diff --git a/.sampo/changesets/openai-embeddings-error.md b/.sampo/changesets/openai-embeddings-error.md new file mode 100644 index 000000000..678639cc3 --- /dev/null +++ b/.sampo/changesets/openai-embeddings-error.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Capture an `$ai_embedding` event with `$ai_is_error` and `$ai_error` when an OpenAI `embeddings.create()` call fails, in the sync and async wrappers. Before, a failed embeddings call left no event. diff --git a/posthog/ai/openai/_embeddings.py b/posthog/ai/openai/_embeddings.py index 8cdb04d33..b34c0c358 100644 --- a/posthog/ai/openai/_embeddings.py +++ b/posthog/ai/openai/_embeddings.py @@ -1,6 +1,12 @@ from typing import Any, Dict, Optional -from ..utils import _capture_ai_event, finalize_ai_content, with_privacy_mode +from ..utils import ( + _capture_ai_event, + _format_ai_error, + _get_http_status, + finalize_ai_content, + with_privacy_mode, +) def _capture_embedding_event( @@ -15,6 +21,7 @@ def _capture_embedding_event( properties: Optional[Dict[str, Any]], privacy_mode: bool, groups: Optional[Dict[str, Any]], + error: Optional[BaseException] = None, ) -> None: """Build and capture telemetry shared by sync and async embedding wrappers.""" usage = getattr(response, "usage", None) @@ -28,7 +35,7 @@ def _capture_embedding_event( privacy_mode, finalize_ai_content(request_kwargs.get("input"), posthog_client), ), - "$ai_http_status": 200, + "$ai_http_status": _get_http_status(error) if error is not None else 200, # Omitted when the provider never reported a count: absent means # unknown, 0 is a report of nothing. **({"$ai_input_tokens": input_tokens} if input_tokens is not None else {}), @@ -36,6 +43,14 @@ def _capture_embedding_event( "$ai_trace_id": trace_id, "$ai_base_url": str(base_url), **(properties or {}), + **( + { + "$ai_is_error": True, + "$ai_error": _format_ai_error(posthog_client, privacy_mode, error), + } + if error is not None + else {} + ), } if distinct_id is None: diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index d8654f090..1bdcdeb6f 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -498,7 +498,23 @@ def create( ) start_time = time.time() - response = self._original.create(**kwargs) + try: + response = self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise end_time = time.time() _capture_embedding_event( diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 95a898139..1282e1ddf 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -502,7 +502,23 @@ async def create( ) start_time = time.time() - response = await self._original.create(**kwargs) + try: + response = await self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise end_time = time.time() _capture_embedding_event( diff --git a/posthog/test/ai/openai/test_openai.py b/posthog/test/ai/openai/test_openai.py index 6a5f34cf4..dcbfde242 100644 --- a/posthog/test/ai/openai/test_openai.py +++ b/posthog/test/ai/openai/test_openai.py @@ -488,6 +488,33 @@ def mock_create(self, **kwargs): assert stream_options == {"foo": "bar"} +def test_embeddings_error(mock_client): + with patch( + "openai.resources.embeddings.Embeddings.create", + side_effect=Exception("Test error"), + ): + client = OpenAI(api_key="test-key", posthog_client=mock_client) + with pytest.raises(Exception, match="Test error"): + client.embeddings.create( + model="text-embedding-3-small", + input="Hello world", + posthog_distinct_id="test-id", + ) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + assert call_args["distinct_id"] == "test-id" + assert call_args["event"] == "$ai_embedding" + assert props["$ai_model"] == "text-embedding-3-small" + assert props["$ai_input"] == "Hello world" + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "Test error" + assert props["$ai_http_status"] == 0 + assert "$ai_input_tokens" not in props + + def test_groups(mock_client, mock_openai_response): with patch( "openai.resources.chat.completions.Completions.create", @@ -1449,6 +1476,29 @@ async def chunk_iterable(): assert isinstance(props["$ai_latency"], float) +@pytest.mark.asyncio +async def test_async_embeddings_error(mock_client): + mock_create = AsyncMock(side_effect=Exception("Test error")) + + with patch("openai.resources.embeddings.AsyncEmbeddings.create", new=mock_create): + client = AsyncOpenAI(api_key="test-key", posthog_client=mock_client) + with pytest.raises(Exception, match="Test error"): + await client.embeddings.create( + model="text-embedding-3-small", + input="Hello world", + posthog_distinct_id="test-id", + ) + + assert mock_client.capture.call_count == 1 + + call_args = mock_client.capture.call_args[1] + props = call_args["properties"] + assert call_args["event"] == "$ai_embedding" + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "Test error" + assert props["$ai_http_status"] == 0 + + @pytest.mark.asyncio @pytest.mark.parametrize("stream_options", [None, NOT_GIVEN, {"foo": "bar"}]) async def test_async_streaming_stream_options_are_not_mutated(