Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/openai-embeddings-error.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

Capture an `$ai_embedding` event with `$ai_is_error` and `$ai_error` when an OpenAI `embeddings.create()` call fails, in the sync and async wrappers. Before, a failed embeddings call left no event.
19 changes: 17 additions & 2 deletions posthog/ai/openai/_embeddings.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
from typing import Any, Dict, Optional

from ..utils import _capture_ai_event, finalize_ai_content, with_privacy_mode
from ..utils import (
_capture_ai_event,
_format_ai_error,
_get_http_status,
finalize_ai_content,
with_privacy_mode,
)


def _capture_embedding_event(
Expand All @@ -15,6 +21,7 @@ def _capture_embedding_event(
properties: Optional[Dict[str, Any]],
privacy_mode: bool,
groups: Optional[Dict[str, Any]],
error: Optional[BaseException] = None,
) -> None:
"""Build and capture telemetry shared by sync and async embedding wrappers."""
usage = getattr(response, "usage", None)
Expand All @@ -28,14 +35,22 @@ def _capture_embedding_event(
privacy_mode,
finalize_ai_content(request_kwargs.get("input"), posthog_client),
),
"$ai_http_status": 200,
"$ai_http_status": _get_http_status(error) if error is not None else 200,
# Omitted when the provider never reported a count: absent means
# unknown, 0 is a report of nothing.
**({"$ai_input_tokens": input_tokens} if input_tokens is not None else {}),
"$ai_latency": latency,
"$ai_trace_id": trace_id,
"$ai_base_url": str(base_url),
**(properties or {}),
**(
{
"$ai_is_error": True,
"$ai_error": _format_ai_error(posthog_client, privacy_mode, error),
}
if error is not None
else {}
),
}

if distinct_id is None:
Expand Down
18 changes: 17 additions & 1 deletion posthog/ai/openai/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -498,7 +498,23 @@ def create(
)

start_time = time.time()
response = self._original.create(**kwargs)
try:
response = self._original.create(**kwargs)
except Exception as exc:
_capture_embedding_event(
posthog_client=self._client._ph_client,
base_url=self._client.base_url,
response=None,
request_kwargs=kwargs,
latency=time.time() - start_time,
distinct_id=posthog_distinct_id,
trace_id=posthog_trace_id,
properties=posthog_properties,
privacy_mode=posthog_privacy_mode,
groups=posthog_groups,
error=exc,
)
raise
end_time = time.time()

_capture_embedding_event(
Expand Down
18 changes: 17 additions & 1 deletion posthog/ai/openai/openai_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,23 @@ async def create(
)

start_time = time.time()
response = await self._original.create(**kwargs)
try:
response = await self._original.create(**kwargs)
except Exception as exc:
_capture_embedding_event(
posthog_client=self._client._ph_client,
base_url=self._client.base_url,
response=None,
request_kwargs=kwargs,
latency=time.time() - start_time,
distinct_id=posthog_distinct_id,
trace_id=posthog_trace_id,
properties=posthog_properties,
privacy_mode=posthog_privacy_mode,
groups=posthog_groups,
error=exc,
)
raise
end_time = time.time()

_capture_embedding_event(
Expand Down
50 changes: 50 additions & 0 deletions posthog/test/ai/openai/test_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,33 @@ def mock_create(self, **kwargs):
assert stream_options == {"foo": "bar"}


def test_embeddings_error(mock_client):
with patch(
"openai.resources.embeddings.Embeddings.create",
side_effect=Exception("Test error"),
):
client = OpenAI(api_key="test-key", posthog_client=mock_client)
with pytest.raises(Exception, match="Test error"):
client.embeddings.create(
model="text-embedding-3-small",
input="Hello world",
posthog_distinct_id="test-id",
)

assert mock_client.capture.call_count == 1

call_args = mock_client.capture.call_args[1]
props = call_args["properties"]
assert call_args["distinct_id"] == "test-id"
assert call_args["event"] == "$ai_embedding"
assert props["$ai_model"] == "text-embedding-3-small"
assert props["$ai_input"] == "Hello world"
assert props["$ai_is_error"] is True
assert props["$ai_error"] == "Test error"
assert props["$ai_http_status"] == 0
assert "$ai_input_tokens" not in props


def test_groups(mock_client, mock_openai_response):
with patch(
"openai.resources.chat.completions.Completions.create",
Expand Down Expand Up @@ -1449,6 +1476,29 @@ async def chunk_iterable():
assert isinstance(props["$ai_latency"], float)


@pytest.mark.asyncio
async def test_async_embeddings_error(mock_client):
mock_create = AsyncMock(side_effect=Exception("Test error"))

with patch("openai.resources.embeddings.AsyncEmbeddings.create", new=mock_create):
client = AsyncOpenAI(api_key="test-key", posthog_client=mock_client)
with pytest.raises(Exception, match="Test error"):
await client.embeddings.create(
model="text-embedding-3-small",
input="Hello world",
posthog_distinct_id="test-id",
)

assert mock_client.capture.call_count == 1

call_args = mock_client.capture.call_args[1]
props = call_args["properties"]
assert call_args["event"] == "$ai_embedding"
assert props["$ai_is_error"] is True
assert props["$ai_error"] == "Test error"
assert props["$ai_http_status"] == 0


@pytest.mark.asyncio
@pytest.mark.parametrize("stream_options", [None, NOT_GIVEN, {"foo": "bar"}])
async def test_async_streaming_stream_options_are_not_mutated(
Expand Down
Loading