Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/openai-responses-failed-stream-event.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

Record failed OpenAI Responses API stream events as generation errors in the sync and async clients. The original event still reaches the caller, and privacy mode redacts the provider's error message.
10 changes: 10 additions & 0 deletions posthog/ai/openai/_streaming.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,21 @@ class _ResponsesStreamState:
model: Optional[str] = None
stop_reason: Optional[str] = None
service_tier: Optional[str] = None
failure: Optional[RuntimeError] = None

def process_chunk(self, chunk: Any) -> None:
response = getattr(chunk, "response", None)
self._record_response_metadata(response)

if getattr(chunk, "type", None) == "response.failed":
error = getattr(response, "error", None)
message = getattr(error, "message", None)
self.failure = RuntimeError(
message
if isinstance(message, str) and message
else "OpenAI response failed"
)

chunk_usage = extract_openai_usage_from_chunk(chunk, "responses")
if chunk_usage:
merge_usage_stats(self.usage_stats, chunk_usage)
Expand Down
2 changes: 1 addition & 1 deletion posthog/ai/openai/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ def _capture_streaming_event(
model_from_response=state.model,
stop_reason=state.stop_reason,
service_tier=state.service_tier,
error=error,
error=error if error is not None else state.failure,
)
capture_streaming_event(self._client._ph_client, event_data)

Expand Down
2 changes: 1 addition & 1 deletion posthog/ai/openai/openai_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ async def _capture_streaming_event(
model_from_response=state.model,
stop_reason=state.stop_reason,
service_tier=state.service_tier,
error=error,
error=error if error is not None else state.failure,
)
capture_streaming_event(self._client._ph_client, event_data)

Expand Down
82 changes: 82 additions & 0 deletions posthog/test/ai/test_stream_midstream_error.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
"""A stream that raises mid-flight must be captured as an error, not as a 200."""

import json
from unittest.mock import AsyncMock, MagicMock, patch

import httpx
import pytest

from posthog.ai.types import StreamingEventData
Expand Down Expand Up @@ -153,6 +155,86 @@ def stream():
assert props["$ai_error"] == "connection reset"


@pytest.fixture
def failed_response_transport():
event = {
"type": "response.failed",
"sequence_number": 1,
"response": {
"id": "resp_failed",
"status": "failed",
"model": "gpt-4",
"output": [],
"error": {
"code": "server_error",
"message": "provider failed for this input",
},
},
}

def respond(request):
return httpx.Response(
200,
headers={"content-type": "text/event-stream"},
text=f"data: {json.dumps(event)}\n\ndata: [DONE]\n\n",
)

return httpx.MockTransport(respond)


@pytest.mark.parametrize("privacy_mode", [False, True])
def test_openai_responses_failed_event(
mock_client, failed_response_transport, privacy_mode
):
from posthog.ai.openai import OpenAI

with OpenAI(
api_key="test",
posthog_client=mock_client,
http_client=httpx.Client(transport=failed_response_transport),
) as client:
events = list(
client.responses.create(
model="gpt-4",
input="hi",
stream=True,
posthog_privacy_mode=privacy_mode,
)
)

assert [event.type for event in events] == ["response.failed"]
props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 0
assert props["$ai_stop_reason"] == "failed"
if privacy_mode:
assert "provider failed for this input" not in props["$ai_error"]
else:
assert props["$ai_error"] == "provider failed for this input"


@pytest.mark.asyncio
async def test_openai_responses_failed_event_async(
mock_client, failed_response_transport
):
from posthog.ai.openai import AsyncOpenAI

async with AsyncOpenAI(
api_key="test",
posthog_client=mock_client,
http_client=httpx.AsyncClient(transport=failed_response_transport),
) as client:
stream = await client.responses.create(model="gpt-4", input="hi", stream=True)
events = [event async for event in stream]

assert [event.type for event in events] == ["response.failed"]
props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 0
assert props["$ai_error"] == "provider failed for this input"
assert props["$ai_stop_reason"] == "failed"


def test_anthropic_stream_error(mock_client):
from posthog.ai.anthropic import Anthropic

Expand Down