Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/ai-stream-open-error.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

OpenAI (chat completions and responses) and Anthropic wrappers now capture an error `$ai_generation` event when a `stream=True` request fails before the stream opens (for example a 401, 429 or 5xx). Before, only mid-stream errors were captured.
21 changes: 20 additions & 1 deletion posthog/ai/anthropic/anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,26 @@ def _create_streaming(
**kwargs: Any,
):
start_time = time.time()
response = super().create(**kwargs)
try:
response = super().create(**kwargs)
except Exception as exc:
# The request failed before any event arrived: still report it
accumulator = _AnthropicStreamAccumulator()
self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
accumulator.usage_stats,
time.time() - start_time,
accumulator.content_blocks,
accumulator.accumulated_content,
stop_reason=accumulator.stop_reason,
error=exc,
)
raise
return self._track_streaming_response(
response,
posthog_distinct_id,
Expand Down
21 changes: 20 additions & 1 deletion posthog/ai/anthropic/anthropic_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,26 @@ async def _create_streaming(
**kwargs: Any,
):
start_time = time.time()
response = await super().create(**kwargs)
try:
response = await super().create(**kwargs)
except Exception as exc:
# The request failed before any event arrived: still report it
accumulator = _AnthropicStreamAccumulator()
await self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
accumulator.usage_stats,
time.time() - start_time,
accumulator.content_blocks,
accumulator.accumulated_content,
stop_reason=accumulator.stop_reason,
error=exc,
)
raise
return self._track_streaming_response(
response,
posthog_distinct_id,
Expand Down
34 changes: 32 additions & 2 deletions posthog/ai/openai/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,22 @@ def _create_streaming(
):
start_time = time.time()
state = _ResponsesStreamState()
response = self._original.create(**kwargs)
try:
response = self._original.create(**kwargs)
except Exception as exc:
# The request failed before any chunk arrived: still report it
self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
state,
time.time() - start_time,
error=exc,
)
raise

def generator():
stream_error: Optional[BaseException] = None
Expand Down Expand Up @@ -395,7 +410,22 @@ def _create_streaming(
**(stream_options if isinstance(stream_options, dict) else {}),
"include_usage": True,
}
response = self._original.create(**kwargs)
try:
response = self._original.create(**kwargs)
except Exception as exc:
# The request failed before any chunk arrived: still report it
self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
state,
time.time() - start_time,
error=exc,
)
raise

def generator():
stream_error: Optional[BaseException] = None
Expand Down
34 changes: 32 additions & 2 deletions posthog/ai/openai/openai_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,22 @@ async def _create_streaming(
):
start_time = time.time()
state = _ResponsesStreamState()
response = await self._original.create(**kwargs)
try:
response = await self._original.create(**kwargs)
except Exception as exc:
# The request failed before any chunk arrived: still report it
await self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
state,
time.time() - start_time,
error=exc,
)
raise

async def async_generator():
stream_error: Optional[BaseException] = None
Expand Down Expand Up @@ -399,7 +414,22 @@ async def _create_streaming(
**(stream_options if isinstance(stream_options, dict) else {}),
"include_usage": True,
}
response = await self._original.create(**kwargs)
try:
response = await self._original.create(**kwargs)
except Exception as exc:
# The request failed before any chunk arrived: still report it
await self._capture_streaming_event(
posthog_distinct_id,
posthog_trace_id,
posthog_properties,
posthog_privacy_mode,
posthog_groups,
kwargs,
state,
time.time() - start_time,
error=exc,
)
raise

async def async_generator():
stream_error: Optional[BaseException] = None
Expand Down
111 changes: 111 additions & 0 deletions posthog/test/ai/test_stream_midstream_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,114 @@ def test_capture_streaming_event_records_google_genai_error_status(
props = _props(mock_client)
assert props["$ai_http_status"] == code
assert props["$ai_is_error"] is True


class _OpenStatusError(RuntimeError):
status_code = 429


def test_openai_chat_stream_open_error(mock_client):
from posthog.ai.openai import OpenAI

with patch("openai.resources.chat.completions.Completions.create") as create:
create.side_effect = _OpenStatusError("rate limited")
client = OpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
client.chat.completions.create(
model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429


def test_openai_responses_stream_open_error(mock_client):
from posthog.ai.openai import OpenAI

with patch("openai.resources.responses.Responses.create") as create:
create.side_effect = _OpenStatusError("rate limited")
client = OpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
client.responses.create(model="gpt-4", input="hi", stream=True)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429


@pytest.mark.asyncio
async def test_openai_chat_stream_open_error_async(mock_client):
from posthog.ai.openai import AsyncOpenAI

async def create(self, **kwargs):
raise _OpenStatusError("rate limited")

with patch("openai.resources.chat.completions.AsyncCompletions.create", new=create):
client = AsyncOpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
await client.chat.completions.create(
model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429


@pytest.mark.asyncio
async def test_openai_responses_stream_open_error_async(mock_client):
from posthog.ai.openai import AsyncOpenAI

async def create(self, **kwargs):
raise _OpenStatusError("rate limited")

with patch("openai.resources.responses.AsyncResponses.create", new=create):
client = AsyncOpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
await client.responses.create(model="gpt-4", input="hi", stream=True)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429


def test_anthropic_stream_open_error(mock_client):
from posthog.ai.anthropic import Anthropic

with patch("anthropic.resources.messages.Messages.create") as create:
create.side_effect = _OpenStatusError("rate limited")
client = Anthropic(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
client.messages.create(
model="claude-3-opus-20240229",
messages=[{"role": "user", "content": "hi"}],
max_tokens=5,
stream=True,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429


@pytest.mark.asyncio
async def test_anthropic_stream_open_error_async(mock_client):
from posthog.ai.anthropic import AsyncAnthropic

async def create(**kwargs):
raise _OpenStatusError("rate limited")

with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create):
client = AsyncAnthropic(api_key="k", posthog_client=mock_client)
with pytest.raises(_OpenStatusError):
await client.messages.create(
model="claude-3-opus-20240229",
messages=[{"role": "user", "content": "hi"}],
max_tokens=5,
stream=True,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 429