Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/openai-agents-response-span-usage.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

`posthog.ai.openai_agents`: report token counts on response spans when the Agents SDK runs with `trace_include_sensitive_data=False`. The SDK then leaves `span_data.response` unset but still records usage on `span_data.usage`, which the processor ignored, so those `$ai_generation` events had no input, output or cache-read tokens.
17 changes: 14 additions & 3 deletions posthog/ai/openai_agents/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ def _parse_iso_timestamp(iso_str: Optional[str]) -> Optional[float]:
return None


def _usage_count(usage: Any, key: str) -> Optional[int]:
"""Read a top-level token count from a usage dict or usage object."""
if isinstance(usage, dict):
return usage.get(key)
return getattr(usage, key, None) if usage else None


def _usage_detail(usage: Any, details_keys: tuple, key: str) -> Optional[int]:
"""Read a token count nested under a usage details field.

Expand Down Expand Up @@ -733,13 +740,17 @@ def _handle_response_span(
response = span_data.response
response_id = response.id if response else None

# Try to extract usage from response
# Try to extract usage from response. With trace_include_sensitive_data
# off the SDK leaves the response unset but still records usage on the
# span, as a dict of the same shape.
usage = getattr(response, "usage", None) if response else None
if usage is None:
usage = getattr(span_data, "usage", None)
total_cost_usd = getattr(usage, "cost", None) if usage else None
# None when the response never reported a count: absent means unknown,
# 0 is a report of nothing.
input_tokens = getattr(usage, "input_tokens", None) if usage else None
output_tokens = getattr(usage, "output_tokens", None) if usage else None
input_tokens = _usage_count(usage, "input_tokens")
output_tokens = _usage_count(usage, "output_tokens")

# Try to extract model from response
model = getattr(response, "model", None) if response else None
Expand Down
30 changes: 30 additions & 0 deletions posthog/test/ai/openai_agents/test_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,36 @@ def test_response_span_reads_cache_tokens(self, processor, mock_client, mock_spa
assert "$ai_reasoning_tokens" not in props
assert props["$ai_cache_reporting_exclusive"] is False

def test_response_span_reads_span_usage_without_response(
self, processor, mock_client, mock_span
):
"""With trace_include_sensitive_data=False the Agents SDK leaves
span_data.response unset but still records the token usage on
span_data.usage."""
mock_span.span_data = ResponseSpanData(
usage={
"requests": 1,
"input_tokens": 100,
"output_tokens": 50,
"total_tokens": 150,
"input_tokens_details": {
"cached_tokens": 60,
"cache_write_tokens": 0,
},
"output_tokens_details": {"reasoning_tokens": 40},
}
)

processor.on_span_start(mock_span)
processor.on_span_end(mock_span)

props = mock_client.capture_ai.call_args[1]["properties"]
assert props["$ai_input_tokens"] == 100
assert props["$ai_output_tokens"] == 50
assert props["$ai_total_tokens"] == 150
assert props["$ai_cache_read_input_tokens"] == 60
assert props["$ai_cache_reporting_exclusive"] is False

def test_cached_compatible_claude_usage_is_declared_inclusive(self):
"""A Claude model behind an OpenAI-compatible endpoint reports inclusive
cached tokens. Without an explicit convention, the cost backend infers
Expand Down