Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/sdk-compliance.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ jobs:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ matrix.compression }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
env:
HARNESS_IMAGE: ghcr.io/posthog/sdk-test-harness:1.26.0
HARNESS_IMAGE: ghcr.io/posthog/sdk-test-harness:1.31.1@sha256:f84d5c70dd72ea4436957ea6dbbc1f6836b6f6cbb472c749711d0f3659d85377
steps:
- name: Check out source code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand Down
2 changes: 1 addition & 1 deletion .sampo/changesets/invalid-events-and-sizes.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
pypi/posthog: major
---

An invalid event no longer costs the rest of its batch. An event with a missing, empty or non-string name, or with a non-string `distinct_id` after `before_send`, is dropped locally, an event that cannot be serialized (for example with `NaN`) is dropped alone, and a property key JSON cannot represent is dropped from its event. The analytics per-event limit is now capture's 10 MiB request limit instead of 900 KiB. A personal API key (`phx_...`) disables the client like an empty key. The SDK's own log lines no longer include event content or invalid timestamp values. `AsyncPosthog` follows the same rules.
An invalid event no longer costs the rest of its batch. An event with a missing, empty or non-string name, or with a non-string `distinct_id` after `before_send`, is dropped locally, an event that cannot be serialized is dropped alone (a non-finite float such as `NaN` is still sent as `null`), and a property key JSON cannot represent is dropped from its event. The analytics per-event limit is now capture's 10 MiB request limit instead of 900 KiB. A personal API key (`phx_...`) disables the client like an empty key. The SDK's own log lines no longer include event content or invalid timestamp values. `AsyncPosthog` follows the same rules.
23 changes: 15 additions & 8 deletions benchmarks/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,9 @@ def do_POST(self):
definitions(spec["flag_count"], generation)
).encode()
self.reply(200, b"{}")
elif path == "/i/v1/analytics/events":
# /batch serves the merge-base SDK in a paired run, which
# still posts legacy batches; this SDK posts capture v1.
elif path in ("/batch", "/i/v1/analytics/events"):
if batch_delay:
time.sleep(batch_delay)
batch = body["batch"]
Expand Down Expand Up @@ -136,13 +138,18 @@ def do_POST(self):
owner.seen[sequence] = 1
owner.state["unique_sequences"] += 1
owner.state["missing_sequences"] -= 1
# Capture v1 reports a result per event uuid; anything
# else is an unreadable response to the SDK.
results = {
str(event.get("uuid")): {"result": "ok"}
for event in batch
}
self.reply(200, json.dumps({"results": results}).encode())
if path == "/batch":
self.reply(200, b'{"status":1}')
else:
# Capture v1 reports a result per event uuid; anything
# else is an unreadable response to the SDK.
results = {
str(event.get("uuid")): {"result": "ok"}
for event in batch
}
self.reply(
200, json.dumps({"results": results}).encode()
)
else:
self.unexpected(path)
except (ValueError, KeyError, TypeError, AttributeError):
Expand Down
28 changes: 24 additions & 4 deletions benchmarks/test_benchmarks.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,20 +364,40 @@ def flush(self, timeout_seconds=10, _calls=calls, _submitted=submitted):

def test_balanced_duplicate_loss_fails_and_delivery_order_is_not_required(self):
spec = next(s for s in workloads(True) if s["id"] == "capture.drain.burst.32")
for duplicate in (False, True):
with self.subTest(duplicate=duplicate), BenchmarkServer(spec) as server:
# The merge-base SDK in a paired run posts to /batch; this SDK posts to capture v1.
for route, duplicate in [
(route, duplicate)
for route in ("/batch", "/i/v1/analytics/events")
for duplicate in (False, True)
]:
with (
self.subTest(route=route, duplicate=duplicate),
BenchmarkServer(spec) as server,
):
workload = Workload(spec, server.url)
try:
events = workload.events
if duplicate:
# Replace the final 16 events with the preceding 16: total and
# per-identity counts stay identical, but sequences do not.
events = events[:-16] + events[-32:-16]
control(
reply = control(
server.url,
"/i/v1/analytics/events",
route,
{"batch": list(reversed(events))},
)
if route == "/batch":
self.assertEqual(reply, {"status": 1})
else:
self.assertEqual(
reply,
{
"results": {
str(event.get("uuid")): {"result": "ok"}
for event in events
}
},
)
workload.captured = spec["operations"]
for event in workload.events:
workload.captured_by_user[event["distinct_id"]] += 1
Expand Down
2 changes: 1 addition & 1 deletion docs/migration-7.x-to-8.0.md
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ These changes follow from this order:

- **Call properties.** Properties passed to a call now override `super_properties`.
- **SDK values.** A `$is_server`, `$geoip_disable`, `$lib`, `$lib_version`, or system property such as `$os` that you set in a call or a context tag now wins over the SDK's value. In 7.x.x the SDK overwrote it.
- **SDK values and `super_properties`.** `super_properties` win over the SDK's value too. So `super_properties={"$geoip_disable": False}` turns GeoIP lookup on for events, even with `disable_geoip=True`.
- **SDK values and `super_properties`.** `super_properties` win over the SDK's value too. So `super_properties={"$geoip_disable": False}` turns GeoIP lookup on for events, even with `disable_geoip=True`. So does `None`, in the event's properties, a context tag or `super_properties`: a `None` property counts as set, so the SDK adds no `True`.
- **Feature flag properties.** Feature flag properties from `flags` or `send_feature_flags` still lose to the same key in the call's `properties`, a context tag, or `super_properties`, as in 7.x.x.
- **Trace IDs.** A `$trace_id` or `$span_id` in a context tag or `super_properties` now replaces the active OpenTelemetry span's IDs. In 7.x.x the span's IDs beat context tags.
- **Trace ID pairs.** Setting either key in the call's `properties`, a context tag, or `super_properties` stops the SDK adding both. In 7.x.x the SDK still added the other key.
Expand Down
2 changes: 2 additions & 0 deletions posthog/ai/anthropic/anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ def tracked_request():
accumulator.content_blocks,
accumulator.accumulated_content,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error.
Expand Down Expand Up @@ -217,6 +218,7 @@ def _create_streaming(
accumulator.content_blocks,
accumulator.accumulated_content,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error.
Expand Down
2 changes: 2 additions & 0 deletions posthog/ai/anthropic/anthropic_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,7 @@ async def tracked_request():
accumulator.content_blocks,
accumulator.accumulated_content,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error.
Expand Down Expand Up @@ -217,6 +218,7 @@ async def _create_streaming(
accumulator.content_blocks,
accumulator.accumulated_content,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error.
Expand Down
1 change: 1 addition & 0 deletions posthog/ai/gemini/gemini.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,7 @@ def _generate_content_streaming(
accumulated_content,
stop_reason=stop_reason,
error=exc,
options=options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down
1 change: 1 addition & 0 deletions posthog/ai/gemini/gemini_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,7 @@ async def _generate_content_streaming(
accumulated_content,
stop_reason=stop_reason,
error=exc,
options=options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down
2 changes: 2 additions & 0 deletions posthog/ai/openai/openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ def _create_streaming(
state,
time.time() - start_time,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down Expand Up @@ -464,6 +465,7 @@ def _create_streaming(
state,
time.time() - start_time,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down
2 changes: 2 additions & 0 deletions posthog/ai/openai/openai_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ async def _create_streaming(
state,
time.time() - start_time,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down Expand Up @@ -468,6 +469,7 @@ async def _create_streaming(
state,
time.time() - start_time,
error=exc,
options=posthog_options,
)
except Exception:
# Telemetry must never replace the provider's own error, and the
Expand Down
8 changes: 8 additions & 0 deletions posthog/test/ai/anthropic/test_anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -2316,6 +2316,7 @@ def test_messages_stream_captures_initial_request_error(mock_client):
messages=[{"role": "user", "content": "Foo"}],
max_tokens=1,
posthog_distinct_id="test-user",
posthog_options={"process_person_profile": False},
)
with pytest.raises(RuntimeError, match="401 unauthorized"):
with manager:
Expand All @@ -2325,6 +2326,9 @@ def test_messages_stream_captures_initial_request_error(mock_client):
props = mock_client.capture_ai.call_args.kwargs["properties"]
assert props["$ai_is_error"] is True
assert props["$ai_error"] == "401 unauthorized"
assert mock_client.capture_ai.call_args.kwargs["options"] == {
"process_person_profile": False
}


def test_messages_stream_tolerates_provider_manager_internals_changing(mock_client):
Expand Down Expand Up @@ -2379,6 +2383,7 @@ async def test_async_messages_stream_captures_initial_request_error(mock_client)
messages=[{"role": "user", "content": "Foo"}],
max_tokens=1,
posthog_distinct_id="test-user",
posthog_options={"process_person_profile": False},
)
with pytest.raises(RuntimeError, match="401 unauthorized"):
async with manager:
Expand All @@ -2388,6 +2393,9 @@ async def test_async_messages_stream_captures_initial_request_error(mock_client)
props = mock_client.capture_ai.call_args.kwargs["properties"]
assert props["$ai_is_error"] is True
assert props["$ai_error"] == "401 unauthorized"
assert mock_client.capture_ai.call_args.kwargs["options"] == {
"process_person_profile": False
}


def test_async_messages_stream_tolerates_provider_manager_internals_changing(
Expand Down
59 changes: 31 additions & 28 deletions posthog/test/ai/test_stream_midstream_error.py
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,19 @@ class _ConnectError(RuntimeError):
status_code = 401


# A caller's options must reach the error event too. No distinct id, so the
# event is personless and process_person_profile: False joins the caller's.
_CALLER = {"posthog_options": {"cookieless_mode": True}}
_SENT_OPTIONS = {"cookieless_mode": True, "process_person_profile": False}


def _assert_connect_error_event(mock_client):
props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
assert mock_client.capture_ai.call_args[1]["options"] == _SENT_OPTIONS


def test_openai_chat_stream_connect_error(mock_client):
from posthog.ai.openai import OpenAI

Expand All @@ -284,11 +297,10 @@ def test_openai_chat_stream_connect_error(mock_client):
model="gpt-4",
messages=[{"role": "user", "content": "hi"}],
stream=True,
**_CALLER,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


@pytest.mark.asyncio
Expand All @@ -307,11 +319,10 @@ async def create(**kwargs):
model="gpt-4",
messages=[{"role": "user", "content": "hi"}],
stream=True,
**_CALLER,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


def test_openai_responses_stream_connect_error(mock_client):
Expand All @@ -321,11 +332,9 @@ def test_openai_responses_stream_connect_error(mock_client):
create.side_effect = _ConnectError("bad key")
client = OpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_ConnectError):
client.responses.create(model="gpt-4", input="hi", stream=True)
client.responses.create(model="gpt-4", input="hi", stream=True, **_CALLER)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


@pytest.mark.asyncio
Expand All @@ -338,11 +347,11 @@ async def create(**kwargs):
with patch("openai.resources.responses.AsyncResponses.create", side_effect=create):
client = AsyncOpenAI(api_key="k", posthog_client=mock_client)
with pytest.raises(_ConnectError):
await client.responses.create(model="gpt-4", input="hi", stream=True)
await client.responses.create(
model="gpt-4", input="hi", stream=True, **_CALLER
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


def test_anthropic_stream_connect_error(mock_client):
Expand All @@ -357,11 +366,10 @@ def test_anthropic_stream_connect_error(mock_client):
messages=[{"role": "user", "content": "hi"}],
max_tokens=5,
stream=True,
**_CALLER,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


@pytest.mark.asyncio
Expand All @@ -379,11 +387,10 @@ async def create(**kwargs):
messages=[{"role": "user", "content": "hi"}],
max_tokens=5,
stream=True,
**_CALLER,
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


def test_gemini_stream_connect_error(mock_client):
Expand All @@ -399,12 +406,10 @@ def test_gemini_stream_connect_error(mock_client):
client = Client(api_key="k", posthog_client=mock_client)
with pytest.raises(_ConnectError):
client.models.generate_content_stream(
model="gemini-2.0-flash", contents=["hi"]
model="gemini-2.0-flash", contents=["hi"], **_CALLER
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


@pytest.mark.asyncio
Expand All @@ -423,12 +428,10 @@ async def test_gemini_stream_connect_error_async(mock_client):
client = AsyncClient(api_key="k", posthog_client=mock_client)
with pytest.raises(_ConnectError):
await client.models.generate_content_stream(
model="gemini-2.0-flash", contents=["hi"]
model="gemini-2.0-flash", contents=["hi"], **_CALLER
)

props = _props(mock_client)
assert props["$ai_is_error"] is True
assert props["$ai_http_status"] == 401
_assert_connect_error_event(mock_client)


@pytest.mark.parametrize(
Expand Down
12 changes: 10 additions & 2 deletions posthog/test/test_evaluate_flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -1141,7 +1141,8 @@ class TestUnresolvableFlagWarning(unittest.TestCase):
def _client(self, **kwargs):
client = Client(
FAKE_TEST_API_KEY,
secret_key="test",
# A key no other test uses, so a poller leaked by another module never matches it.
secret_key="test-unresolvable-flag-warning",
enable_local_evaluation=False,
send=False,
**kwargs,
Expand Down Expand Up @@ -1437,7 +1438,14 @@ def _refresh_from(self, client, source, *flags):
mock.patch("posthog.client.get") as get,
):
client._load_feature_flags()
get.assert_not_called()
# Pollers left running by other test modules share the patched
# module-level `get`, so only this client's key counts.
own_calls = [
call
for call in get.call_args_list
if call.args and call.args[0] == client.personal_api_key
Comment thread
eli-r-ph marked this conversation as resolved.
Resolved
]
self.assertEqual(own_calls, [])

@staticmethod
def _never_resolvable_flag(key, reason, **kwargs):
Expand Down
Loading