From cc344742214a75a192311b11f7dd67adeec31735 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 13:06:01 -0800 Subject: [PATCH 1/8] fix(ai): send posthog_options on streaming connect errors The error event for a streaming call that fails before the stream starts (OpenAI chat and responses, Anthropic create and messages.stream, Gemini; sync and async) now carries the caller's posthog_options, like the other AI events. Co-Authored-By: Claude Opus 5.5 --- posthog/ai/anthropic/anthropic.py | 2 + posthog/ai/anthropic/anthropic_async.py | 2 + posthog/ai/gemini/gemini.py | 1 + posthog/ai/gemini/gemini_async.py | 1 + posthog/ai/openai/openai.py | 2 + posthog/ai/openai/openai_async.py | 2 + posthog/test/ai/anthropic/test_anthropic.py | 8 +++ .../test/ai/test_stream_midstream_error.py | 59 ++++++++++--------- 8 files changed, 49 insertions(+), 28 deletions(-) diff --git a/posthog/ai/anthropic/anthropic.py b/posthog/ai/anthropic/anthropic.py index f448be786..ee7204325 100644 --- a/posthog/ai/anthropic/anthropic.py +++ b/posthog/ai/anthropic/anthropic.py @@ -169,6 +169,7 @@ def tracked_request(): accumulator.content_blocks, accumulator.accumulated_content, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error. @@ -217,6 +218,7 @@ def _create_streaming( accumulator.content_blocks, accumulator.accumulated_content, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error. diff --git a/posthog/ai/anthropic/anthropic_async.py b/posthog/ai/anthropic/anthropic_async.py index 50d113267..60f267fc4 100644 --- a/posthog/ai/anthropic/anthropic_async.py +++ b/posthog/ai/anthropic/anthropic_async.py @@ -169,6 +169,7 @@ async def tracked_request(): accumulator.content_blocks, accumulator.accumulated_content, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error. @@ -217,6 +218,7 @@ async def _create_streaming( accumulator.content_blocks, accumulator.accumulated_content, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error. diff --git a/posthog/ai/gemini/gemini.py b/posthog/ai/gemini/gemini.py index b06330aba..d0aa8f80c 100644 --- a/posthog/ai/gemini/gemini.py +++ b/posthog/ai/gemini/gemini.py @@ -301,6 +301,7 @@ def _generate_content_streaming( accumulated_content, stop_reason=stop_reason, error=exc, + options=options, ) except Exception: # Telemetry must never replace the provider's own error, and the diff --git a/posthog/ai/gemini/gemini_async.py b/posthog/ai/gemini/gemini_async.py index 7e10a16ff..3184e4056 100644 --- a/posthog/ai/gemini/gemini_async.py +++ b/posthog/ai/gemini/gemini_async.py @@ -284,6 +284,7 @@ async def _generate_content_streaming( accumulated_content, stop_reason=stop_reason, error=exc, + options=options, ) except Exception: # Telemetry must never replace the provider's own error, and the diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index 872da2316..b1b71301a 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -194,6 +194,7 @@ def _create_streaming( state, time.time() - start_time, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error, and the @@ -464,6 +465,7 @@ def _create_streaming( state, time.time() - start_time, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error, and the diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 9b08ccd62..9d59dfcfc 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -195,6 +195,7 @@ async def _create_streaming( state, time.time() - start_time, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error, and the @@ -468,6 +469,7 @@ async def _create_streaming( state, time.time() - start_time, error=exc, + options=posthog_options, ) except Exception: # Telemetry must never replace the provider's own error, and the diff --git a/posthog/test/ai/anthropic/test_anthropic.py b/posthog/test/ai/anthropic/test_anthropic.py index a12133625..1bdb0051d 100644 --- a/posthog/test/ai/anthropic/test_anthropic.py +++ b/posthog/test/ai/anthropic/test_anthropic.py @@ -2316,6 +2316,7 @@ def test_messages_stream_captures_initial_request_error(mock_client): messages=[{"role": "user", "content": "Foo"}], max_tokens=1, posthog_distinct_id="test-user", + posthog_options={"process_person_profile": False}, ) with pytest.raises(RuntimeError, match="401 unauthorized"): with manager: @@ -2325,6 +2326,9 @@ def test_messages_stream_captures_initial_request_error(mock_client): props = mock_client.capture_ai.call_args.kwargs["properties"] assert props["$ai_is_error"] is True assert props["$ai_error"] == "401 unauthorized" + assert mock_client.capture_ai.call_args.kwargs["options"] == { + "process_person_profile": False + } def test_messages_stream_tolerates_provider_manager_internals_changing(mock_client): @@ -2379,6 +2383,7 @@ async def test_async_messages_stream_captures_initial_request_error(mock_client) messages=[{"role": "user", "content": "Foo"}], max_tokens=1, posthog_distinct_id="test-user", + posthog_options={"process_person_profile": False}, ) with pytest.raises(RuntimeError, match="401 unauthorized"): async with manager: @@ -2388,6 +2393,9 @@ async def test_async_messages_stream_captures_initial_request_error(mock_client) props = mock_client.capture_ai.call_args.kwargs["properties"] assert props["$ai_is_error"] is True assert props["$ai_error"] == "401 unauthorized" + assert mock_client.capture_ai.call_args.kwargs["options"] == { + "process_person_profile": False + } def test_async_messages_stream_tolerates_provider_manager_internals_changing( diff --git a/posthog/test/ai/test_stream_midstream_error.py b/posthog/test/ai/test_stream_midstream_error.py index fec3fbbf8..30539472d 100644 --- a/posthog/test/ai/test_stream_midstream_error.py +++ b/posthog/test/ai/test_stream_midstream_error.py @@ -273,6 +273,19 @@ class _ConnectError(RuntimeError): status_code = 401 +# A caller's options must reach the error event too; the distinct id keeps the +# event from being personless, so only the caller can set the option. +_CALLER_OPTIONS = {"process_person_profile": False} +_CALLER = {"posthog_distinct_id": "user-1", "posthog_options": _CALLER_OPTIONS} + + +def _assert_connect_error_event(mock_client): + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + assert mock_client.capture_ai.call_args[1]["options"] == _CALLER_OPTIONS + + def test_openai_chat_stream_connect_error(mock_client): from posthog.ai.openai import OpenAI @@ -284,11 +297,10 @@ def test_openai_chat_stream_connect_error(mock_client): model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True, + **_CALLER, ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) @pytest.mark.asyncio @@ -307,11 +319,10 @@ async def create(**kwargs): model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True, + **_CALLER, ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) def test_openai_responses_stream_connect_error(mock_client): @@ -321,11 +332,9 @@ def test_openai_responses_stream_connect_error(mock_client): create.side_effect = _ConnectError("bad key") client = OpenAI(api_key="k", posthog_client=mock_client) with pytest.raises(_ConnectError): - client.responses.create(model="gpt-4", input="hi", stream=True) + client.responses.create(model="gpt-4", input="hi", stream=True, **_CALLER) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) @pytest.mark.asyncio @@ -338,11 +347,11 @@ async def create(**kwargs): with patch("openai.resources.responses.AsyncResponses.create", side_effect=create): client = AsyncOpenAI(api_key="k", posthog_client=mock_client) with pytest.raises(_ConnectError): - await client.responses.create(model="gpt-4", input="hi", stream=True) + await client.responses.create( + model="gpt-4", input="hi", stream=True, **_CALLER + ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) def test_anthropic_stream_connect_error(mock_client): @@ -357,11 +366,10 @@ def test_anthropic_stream_connect_error(mock_client): messages=[{"role": "user", "content": "hi"}], max_tokens=5, stream=True, + **_CALLER, ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) @pytest.mark.asyncio @@ -379,11 +387,10 @@ async def create(**kwargs): messages=[{"role": "user", "content": "hi"}], max_tokens=5, stream=True, + **_CALLER, ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) def test_gemini_stream_connect_error(mock_client): @@ -399,12 +406,10 @@ def test_gemini_stream_connect_error(mock_client): client = Client(api_key="k", posthog_client=mock_client) with pytest.raises(_ConnectError): client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] + model="gemini-2.0-flash", contents=["hi"], **_CALLER ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) @pytest.mark.asyncio @@ -423,12 +428,10 @@ async def test_gemini_stream_connect_error_async(mock_client): client = AsyncClient(api_key="k", posthog_client=mock_client) with pytest.raises(_ConnectError): await client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] + model="gemini-2.0-flash", contents=["hi"], **_CALLER ) - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 + _assert_connect_error_event(mock_client) @pytest.mark.parametrize( From ca1bb60b6afc1a49bd86fc1b4be14ff16ea19c0d Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 13:39:34 -0800 Subject: [PATCH 2/8] test(ai): keep the connect-error tests on the personless path Co-Authored-By: Claude Opus 5.5 --- posthog/test/ai/test_stream_midstream_error.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/posthog/test/ai/test_stream_midstream_error.py b/posthog/test/ai/test_stream_midstream_error.py index 30539472d..676051144 100644 --- a/posthog/test/ai/test_stream_midstream_error.py +++ b/posthog/test/ai/test_stream_midstream_error.py @@ -273,17 +273,17 @@ class _ConnectError(RuntimeError): status_code = 401 -# A caller's options must reach the error event too; the distinct id keeps the -# event from being personless, so only the caller can set the option. -_CALLER_OPTIONS = {"process_person_profile": False} -_CALLER = {"posthog_distinct_id": "user-1", "posthog_options": _CALLER_OPTIONS} +# A caller's options must reach the error event too. No distinct id, so the +# event is personless and process_person_profile: False joins the caller's. +_CALLER = {"posthog_options": {"cookieless_mode": True}} +_SENT_OPTIONS = {"cookieless_mode": True, "process_person_profile": False} def _assert_connect_error_event(mock_client): props = _props(mock_client) assert props["$ai_is_error"] is True assert props["$ai_http_status"] == 401 - assert mock_client.capture_ai.call_args[1]["options"] == _CALLER_OPTIONS + assert mock_client.capture_ai.call_args[1]["options"] == _SENT_OPTIONS def test_openai_chat_stream_connect_error(mock_client): From 9f6531e09f5b2a1a1f2ad08b37bb4d3e6e91a8c8 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 15:27:55 -0800 Subject: [PATCH 3/8] docs: note that a None $geoip_disable also keeps the SDK marker out Co-Authored-By: Claude Opus 5.5 --- docs/migration-7.x-to-8.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migration-7.x-to-8.0.md b/docs/migration-7.x-to-8.0.md index 9fbb5cb83..f0e0edf2e 100644 --- a/docs/migration-7.x-to-8.0.md +++ b/docs/migration-7.x-to-8.0.md @@ -235,7 +235,7 @@ These changes follow from this order: - **Call properties.** Properties passed to a call now override `super_properties`. - **SDK values.** A `$is_server`, `$geoip_disable`, `$lib`, `$lib_version`, or system property such as `$os` that you set in a call or a context tag now wins over the SDK's value. In 7.x.x the SDK overwrote it. -- **SDK values and `super_properties`.** `super_properties` win over the SDK's value too. So `super_properties={"$geoip_disable": False}` turns GeoIP lookup on for events, even with `disable_geoip=True`. +- **SDK values and `super_properties`.** `super_properties` win over the SDK's value too. So `super_properties={"$geoip_disable": False}` turns GeoIP lookup on for events, even with `disable_geoip=True`. So does `None`, in the event's properties, a context tag or `super_properties`: a `None` property counts as set, so the SDK adds no `True`. - **Feature flag properties.** Feature flag properties from `flags` or `send_feature_flags` still lose to the same key in the call's `properties`, a context tag, or `super_properties`, as in 7.x.x. - **Trace IDs.** A `$trace_id` or `$span_id` in a context tag or `super_properties` now replaces the active OpenTelemetry span's IDs. In 7.x.x the span's IDs beat context tags. - **Trace ID pairs.** Setting either key in the call's `properties`, a context tag, or `super_properties` stops the SDK adding both. In 7.x.x the SDK still added the other key. From 2a92b2ca6bf71aeec4f1f1f254907fcc196f8330 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 15:58:59 -0800 Subject: [PATCH 4/8] docs: correct the NaN example in the invalid-events changeset Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/invalid-events-and-sizes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sampo/changesets/invalid-events-and-sizes.md b/.sampo/changesets/invalid-events-and-sizes.md index e7b310d8e..176007c1e 100644 --- a/.sampo/changesets/invalid-events-and-sizes.md +++ b/.sampo/changesets/invalid-events-and-sizes.md @@ -2,4 +2,4 @@ pypi/posthog: major --- -An invalid event no longer costs the rest of its batch. An event with a missing, empty or non-string name, or with a non-string `distinct_id` after `before_send`, is dropped locally, an event that cannot be serialized (for example with `NaN`) is dropped alone, and a property key JSON cannot represent is dropped from its event. The analytics per-event limit is now capture's 10 MiB request limit instead of 900 KiB. A personal API key (`phx_...`) disables the client like an empty key. The SDK's own log lines no longer include event content or invalid timestamp values. `AsyncPosthog` follows the same rules. +An invalid event no longer costs the rest of its batch. An event with a missing, empty or non-string name, or with a non-string `distinct_id` after `before_send`, is dropped locally, an event that cannot be serialized is dropped alone (a non-finite float such as `NaN` is still sent as `null`), and a property key JSON cannot represent is dropped from its event. The analytics per-event limit is now capture's 10 MiB request limit instead of 900 KiB. A personal API key (`phx_...`) disables the client like an empty key. The SDK's own log lines no longer include event content or invalid timestamp values. `AsyncPosthog` follows the same rules. From acdc8183513323371bd9877a6142f83525413a37 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 16:07:27 -0800 Subject: [PATCH 5/8] fix(benchmarks): serve the merge-base SDK's /batch in paired runs The paired benchmark runs the merge-base SDK through this branch's receiver. It now accepts legacy /batch posts again, answering in the legacy format, beside capture v1 posts, so the baseline is measured instead of failing with 404. The receiver test covers both routes and their reply formats. Co-Authored-By: Claude Opus 5.5 --- benchmarks/server.py | 23 +++++++++++++++-------- benchmarks/test_benchmarks.py | 28 ++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 12 deletions(-) diff --git a/benchmarks/server.py b/benchmarks/server.py index e5071531d..534eec070 100644 --- a/benchmarks/server.py +++ b/benchmarks/server.py @@ -102,7 +102,9 @@ def do_POST(self): definitions(spec["flag_count"], generation) ).encode() self.reply(200, b"{}") - elif path == "/i/v1/analytics/events": + # /batch serves the merge-base SDK in a paired run, which + # still posts legacy batches; this SDK posts capture v1. + elif path in ("/batch", "/i/v1/analytics/events"): if batch_delay: time.sleep(batch_delay) batch = body["batch"] @@ -136,13 +138,18 @@ def do_POST(self): owner.seen[sequence] = 1 owner.state["unique_sequences"] += 1 owner.state["missing_sequences"] -= 1 - # Capture v1 reports a result per event uuid; anything - # else is an unreadable response to the SDK. - results = { - str(event.get("uuid")): {"result": "ok"} - for event in batch - } - self.reply(200, json.dumps({"results": results}).encode()) + if path == "/batch": + self.reply(200, b'{"status":1}') + else: + # Capture v1 reports a result per event uuid; anything + # else is an unreadable response to the SDK. + results = { + str(event.get("uuid")): {"result": "ok"} + for event in batch + } + self.reply( + 200, json.dumps({"results": results}).encode() + ) else: self.unexpected(path) except (ValueError, KeyError, TypeError, AttributeError): diff --git a/benchmarks/test_benchmarks.py b/benchmarks/test_benchmarks.py index f6073086a..dcfd89b46 100644 --- a/benchmarks/test_benchmarks.py +++ b/benchmarks/test_benchmarks.py @@ -364,8 +364,16 @@ def flush(self, timeout_seconds=10, _calls=calls, _submitted=submitted): def test_balanced_duplicate_loss_fails_and_delivery_order_is_not_required(self): spec = next(s for s in workloads(True) if s["id"] == "capture.drain.burst.32") - for duplicate in (False, True): - with self.subTest(duplicate=duplicate), BenchmarkServer(spec) as server: + # The merge-base SDK in a paired run posts to /batch; this SDK posts to capture v1. + for route, duplicate in [ + (route, duplicate) + for route in ("/batch", "/i/v1/analytics/events") + for duplicate in (False, True) + ]: + with ( + self.subTest(route=route, duplicate=duplicate), + BenchmarkServer(spec) as server, + ): workload = Workload(spec, server.url) try: events = workload.events @@ -373,11 +381,23 @@ def test_balanced_duplicate_loss_fails_and_delivery_order_is_not_required(self): # Replace the final 16 events with the preceding 16: total and # per-identity counts stay identical, but sequences do not. events = events[:-16] + events[-32:-16] - control( + reply = control( server.url, - "/i/v1/analytics/events", + route, {"batch": list(reversed(events))}, ) + if route == "/batch": + self.assertEqual(reply, {"status": 1}) + else: + self.assertEqual( + reply, + { + "results": { + str(event.get("uuid")): {"result": "ok"} + for event in events + } + }, + ) workload.captured = spec["operations"] for event in workload.events: workload.captured_by_user[event["distinct_id"]] += 1 From 2f0cd843eedf5366a53d5c31f1657b1e535e4880 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 16:14:14 -0800 Subject: [PATCH 6/8] test: count only the client under test in the flag cache check Pollers left running by other test modules share the patched module-level get, and one of them failed this check on CI. The check now ignores calls made with another client's key; a mutation that makes this client fetch still fails 20 cases. Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_evaluate_flags.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index 72efc507f..15aa0fcff 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -1437,7 +1437,14 @@ def _refresh_from(self, client, source, *flags): mock.patch("posthog.client.get") as get, ): client._load_feature_flags() - get.assert_not_called() + # Pollers left running by other test modules share the patched + # module-level `get`, so only this client's key counts. + own_calls = [ + call + for call in get.call_args_list + if call.args and call.args[0] == client.personal_api_key + ] + self.assertEqual(own_calls, []) @staticmethod def _never_resolvable_flag(key, reason, **kwargs): From 48a111512d4c44f55206e3344dbcdb24191dc0d5 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Mon, 16 Nov 2026 18:41:08 -0800 Subject: [PATCH 7/8] ci: run compliance tests on sdk-test-harness 1.31.1 Pin by digest. The capture v1 suites are unchanged from 1.26.0; the new release only adds feature flag contract tests. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/sdk-compliance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdk-compliance.yml b/.github/workflows/sdk-compliance.yml index 235299d57..c65042df0 100644 --- a/.github/workflows/sdk-compliance.yml +++ b/.github/workflows/sdk-compliance.yml @@ -41,7 +41,7 @@ jobs: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.compression }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} env: - HARNESS_IMAGE: ghcr.io/posthog/sdk-test-harness:1.26.0 + HARNESS_IMAGE: ghcr.io/posthog/sdk-test-harness:1.31.1@sha256:f84d5c70dd72ea4436957ea6dbbc1f6836b6f6cbb472c749711d0f3659d85377 steps: - name: Check out source code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 36b58a4e1a3e61c81266d83f172ca57c6a5272e4 Mon Sep 17 00:00:00 2001 From: Eli Reisman Date: Tue, 17 Nov 2026 10:28:05 -0800 Subject: [PATCH 8/8] test: give the unresolvable-flag warning tests their own secret key The cache test counts only fetches made with its client's key, but about 30 other test clients also use "test", so a poller leaked from one of them would still match and flake the check. A key no other test uses removes that overlap. Co-Authored-By: Claude Opus 5.5 --- posthog/test/test_evaluate_flags.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/test/test_evaluate_flags.py b/posthog/test/test_evaluate_flags.py index 15aa0fcff..89a8c71f0 100644 --- a/posthog/test/test_evaluate_flags.py +++ b/posthog/test/test_evaluate_flags.py @@ -1141,7 +1141,8 @@ class TestUnresolvableFlagWarning(unittest.TestCase): def _client(self, **kwargs): client = Client( FAKE_TEST_API_KEY, - secret_key="test", + # A key no other test uses, so a poller leaked by another module never matches it. + secret_key="test-unresolvable-flag-warning", enable_local_evaluation=False, send=False, **kwargs,