From 114ceaf2e45e793dce4df36b3f142c5a2df58de8 Mon Sep 17 00:00:00 2001 From: Mihir Harshad Khatri Date: Tue, 17 Nov 2026 14:25:21 +0530 Subject: [PATCH] fix(ai): track failed OpenAI Responses stream events Prepared with OpenAI Codex assistance. --- .../openai-responses-failed-stream-event.md | 5 ++ posthog/ai/openai/_streaming.py | 10 +++ posthog/ai/openai/openai.py | 2 +- posthog/ai/openai/openai_async.py | 2 +- .../test/ai/test_stream_midstream_error.py | 82 +++++++++++++++++++ 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 .sampo/changesets/openai-responses-failed-stream-event.md diff --git a/.sampo/changesets/openai-responses-failed-stream-event.md b/.sampo/changesets/openai-responses-failed-stream-event.md new file mode 100644 index 000000000..c647e49e6 --- /dev/null +++ b/.sampo/changesets/openai-responses-failed-stream-event.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Record failed OpenAI Responses API stream events as generation errors in the sync and async clients. The original event still reaches the caller, and privacy mode redacts the provider's error message. diff --git a/posthog/ai/openai/_streaming.py b/posthog/ai/openai/_streaming.py index 49234bee0..c8d2e44eb 100644 --- a/posthog/ai/openai/_streaming.py +++ b/posthog/ai/openai/_streaming.py @@ -23,11 +23,21 @@ class _ResponsesStreamState: model: Optional[str] = None stop_reason: Optional[str] = None service_tier: Optional[str] = None + failure: Optional[RuntimeError] = None def process_chunk(self, chunk: Any) -> None: response = getattr(chunk, "response", None) self._record_response_metadata(response) + if getattr(chunk, "type", None) == "response.failed": + error = getattr(response, "error", None) + message = getattr(error, "message", None) + self.failure = RuntimeError( + message + if isinstance(message, str) and message + else "OpenAI response failed" + ) + chunk_usage = extract_openai_usage_from_chunk(chunk, "responses") if chunk_usage: merge_usage_stats(self.usage_stats, chunk_usage) diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index e00a2d3c0..589318bed 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -244,7 +244,7 @@ def _capture_streaming_event( model_from_response=state.model, stop_reason=state.stop_reason, service_tier=state.service_tier, - error=error, + error=error if error is not None else state.failure, ) capture_streaming_event(self._client._ph_client, event_data) diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 3d105f7a2..990ec9ce9 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -245,7 +245,7 @@ async def _capture_streaming_event( model_from_response=state.model, stop_reason=state.stop_reason, service_tier=state.service_tier, - error=error, + error=error if error is not None else state.failure, ) capture_streaming_event(self._client._ph_client, event_data) diff --git a/posthog/test/ai/test_stream_midstream_error.py b/posthog/test/ai/test_stream_midstream_error.py index 620b5990c..9d444b4cf 100644 --- a/posthog/test/ai/test_stream_midstream_error.py +++ b/posthog/test/ai/test_stream_midstream_error.py @@ -1,7 +1,9 @@ """A stream that raises mid-flight must be captured as an error, not as a 200.""" +import json from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest from posthog.ai.types import StreamingEventData @@ -153,6 +155,86 @@ def stream(): assert props["$ai_error"] == "connection reset" +@pytest.fixture +def failed_response_transport(): + event = { + "type": "response.failed", + "sequence_number": 1, + "response": { + "id": "resp_failed", + "status": "failed", + "model": "gpt-4", + "output": [], + "error": { + "code": "server_error", + "message": "provider failed for this input", + }, + }, + } + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(event)}\n\ndata: [DONE]\n\n", + ) + + return httpx.MockTransport(respond) + + +@pytest.mark.parametrize("privacy_mode", [False, True]) +def test_openai_responses_failed_event( + mock_client, failed_response_transport, privacy_mode +): + from posthog.ai.openai import OpenAI + + with OpenAI( + api_key="test", + posthog_client=mock_client, + http_client=httpx.Client(transport=failed_response_transport), + ) as client: + events = list( + client.responses.create( + model="gpt-4", + input="hi", + stream=True, + posthog_privacy_mode=privacy_mode, + ) + ) + + assert [event.type for event in events] == ["response.failed"] + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + assert props["$ai_stop_reason"] == "failed" + if privacy_mode: + assert "provider failed for this input" not in props["$ai_error"] + else: + assert props["$ai_error"] == "provider failed for this input" + + +@pytest.mark.asyncio +async def test_openai_responses_failed_event_async( + mock_client, failed_response_transport +): + from posthog.ai.openai import AsyncOpenAI + + async with AsyncOpenAI( + api_key="test", + posthog_client=mock_client, + http_client=httpx.AsyncClient(transport=failed_response_transport), + ) as client: + stream = await client.responses.create(model="gpt-4", input="hi", stream=True) + events = [event async for event in stream] + + assert [event.type for event in events] == ["response.failed"] + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + assert props["$ai_error"] == "provider failed for this input" + assert props["$ai_stop_reason"] == "failed" + + def test_anthropic_stream_error(mock_client): from posthog.ai.anthropic import Anthropic