From f6ecbfbf857e6a7d1c70a040be42999e93512614 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Thu, 29 Oct 2026 06:22:29 -0400 Subject: [PATCH] fix(ai): capture an $ai_embedding event when an OpenAI embeddings call fails Co-Authored-By: Claude Sonnet 5.5 --- .sampo/changesets/openai-embeddings-error.md | 5 ++ posthog/ai/openai/_embeddings.py | 19 +++++++- posthog/ai/openai/openai.py | 18 ++++++- posthog/ai/openai/openai_async.py | 18 ++++++- posthog/test/ai/openai/test_openai.py | 50 ++++++++++++++++++++ 5 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 .sampo/changesets/openai-embeddings-error.md diff --git a/.sampo/changesets/openai-embeddings-error.md b/.sampo/changesets/openai-embeddings-error.md new file mode 100644 index 000000000..678639cc3 --- /dev/null +++ b/.sampo/changesets/openai-embeddings-error.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Capture an `$ai_embedding` event with `$ai_is_error` and `$ai_error` when an OpenAI `embeddings.create()` call fails, in the sync and async wrappers. Before, a failed embeddings call left no event. diff --git a/posthog/ai/openai/_embeddings.py b/posthog/ai/openai/_embeddings.py index 8cdb04d33..b34c0c358 100644 --- a/posthog/ai/openai/_embeddings.py +++ b/posthog/ai/openai/_embeddings.py @@ -1,6 +1,12 @@ from typing import Any, Dict, Optional -from ..utils import _capture_ai_event, finalize_ai_content, with_privacy_mode +from ..utils import ( + _capture_ai_event, + _format_ai_error, + _get_http_status, + finalize_ai_content, + with_privacy_mode, +) def _capture_embedding_event( @@ -15,6 +21,7 @@ def _capture_embedding_event( properties: Optional[Dict[str, Any]], privacy_mode: bool, groups: Optional[Dict[str, Any]], + error: Optional[BaseException] = None, ) -> None: """Build and capture telemetry shared by sync and async embedding wrappers.""" usage = getattr(response, "usage", None) @@ -28,7 +35,7 @@ def _capture_embedding_event( privacy_mode, finalize_ai_content(request_kwargs.get("input"), posthog_client), ), - "$ai_http_status": 200, + "$ai_http_status": _get_http_status(error) if error is not None else 200, # Omitted when the provider never reported a count: absent means # unknown, 0 is a report of nothing. **({"$ai_input_tokens": input_tokens} if input_tokens is not None else {}), @@ -36,6 +43,14 @@ def _capture_embedding_event( "$ai_trace_id": trace_id, "$ai_base_url": str(base_url), **(properties or {}), + **( + { + "$ai_is_error": True, + "$ai_error": _format_ai_error(posthog_client, privacy_mode, error), + } + if error is not None + else {} + ), } if distinct_id is None: diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index ccdf6e69a..8c1cb6eed 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -495,7 +495,23 @@ def create( ) start_time = time.time() - response = self._original.create(**kwargs) + try: + response = self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise end_time = time.time() _capture_embedding_event( diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 07e0de670..7c28b3152 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -499,7 +499,23 @@ async def create( ) start_time = time.time() - response = await self._original.create(**kwargs) + try: + response = await self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise end_time = time.time() _capture_embedding_event( diff --git a/posthog/test/ai/openai/test_openai.py b/posthog/test/ai/openai/test_openai.py index a0a550191..ef838540a 100644 --- a/posthog/test/ai/openai/test_openai.py +++ b/posthog/test/ai/openai/test_openai.py @@ -460,6 +460,33 @@ def test_embeddings(mock_client, mock_embedding_response): assert isinstance(props["$ai_latency"], float) +def test_embeddings_error(mock_client): + with patch( + "openai.resources.embeddings.Embeddings.create", + side_effect=Exception("Test error"), + ): + client = OpenAI(api_key="test-key", posthog_client=mock_client) + with pytest.raises(Exception, match="Test error"): + client.embeddings.create( + model="text-embedding-3-small", + input="Hello world", + posthog_distinct_id="test-id", + ) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + assert call_args["distinct_id"] == "test-id" + assert call_args["event"] == "$ai_embedding" + assert props["$ai_model"] == "text-embedding-3-small" + assert props["$ai_input"] == "Hello world" + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "Test error" + assert props["$ai_http_status"] == 0 + assert "$ai_input_tokens" not in props + + def test_groups(mock_client, mock_openai_response): with patch( "openai.resources.chat.completions.Completions.create", @@ -1421,6 +1448,29 @@ async def chunk_iterable(): assert isinstance(props["$ai_latency"], float) +@pytest.mark.asyncio +async def test_async_embeddings_error(mock_client): + mock_create = AsyncMock(side_effect=Exception("Test error")) + + with patch("openai.resources.embeddings.AsyncEmbeddings.create", new=mock_create): + client = AsyncOpenAI(api_key="test-key", posthog_client=mock_client) + with pytest.raises(Exception, match="Test error"): + await client.embeddings.create( + model="text-embedding-3-small", + input="Hello world", + posthog_distinct_id="test-id", + ) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + assert call_args["event"] == "$ai_embedding" + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "Test error" + assert props["$ai_http_status"] == 0 + + @pytest.mark.asyncio async def test_async_embeddings_create(mock_client, mock_embedding_response): mock_create = AsyncMock(return_value=mock_embedding_response)