Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/openai-cache-write-tokens.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

`posthog.ai.openai`: report OpenAI cache-write tokens as `$ai_cache_creation_input_tokens`, for streaming and non-streaming calls. OpenAI reports them in `input_tokens_details.cache_write_tokens` (Responses) or `prompt_tokens_details.cache_write_tokens` (Chat Completions) and bills them above the input rate, but the wrapper read only `cached_tokens`, so cache writes were costed as plain input tokens.
17 changes: 16 additions & 1 deletion posthog/ai/openai/openai_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A
)
detail_fields = (
("cache_read_input_tokens", "input_tokens_details", "cached_tokens"),
(
"cache_creation_input_tokens",
"input_tokens_details",
"cache_write_tokens",
),
("reasoning_tokens", "output_tokens_details", "reasoning_tokens"),
)
else:
Expand All @@ -701,6 +706,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A
)
detail_fields = (
("cache_read_input_tokens", "prompt_tokens_details", "cached_tokens"),
(
"cache_creation_input_tokens",
"prompt_tokens_details",
"cache_write_tokens",
),
("reasoning_tokens", "completion_tokens_details", "reasoning_tokens"),
)
counts = {}
Expand All @@ -716,7 +726,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A
def _add_openai_response_details(
result: Dict[str, Any], counts: Dict[str, Any]
) -> None:
for field in ("cache_read_input_tokens", "reasoning_tokens"):
for field in (
"cache_read_input_tokens",
"cache_creation_input_tokens",
"reasoning_tokens",
):
value = counts.get(field)
if value is not None and value > 0:
result[field] = value
Expand All @@ -739,6 +753,7 @@ def _extract_openai_stream_token_usage(raw: Any, provider_type: str) -> TokenUsa
# Unlike non-streaming usage, explicitly reported zero details are retained.
for output, field, detail in (
("cache_read_input_tokens", input_details, "cached_tokens"),
("cache_creation_input_tokens", input_details, "cache_write_tokens"),
("reasoning_tokens", output_details, "reasoning_tokens"),
):
if hasattr(raw, field) and hasattr(getattr(raw, field), detail):
Expand Down
86 changes: 86 additions & 0 deletions posthog/test/ai/openai/test_openai.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from openai.types.responses import (
ParsedResponse,
Response,
ResponseCompletedEvent,
ResponseFunctionToolCall,
ResponseOutputMessage,
ResponseOutputText,
Expand Down Expand Up @@ -638,6 +639,91 @@ def test_cached_tokens(mock_client, mock_openai_response_with_cached_tokens):
assert isinstance(props["$ai_latency"], float)


@pytest.mark.parametrize("stream", [False, True])
def test_responses_cache_write_tokens(
mock_client, mock_openai_response_with_responses_api, stream
):
response = mock_openai_response_with_responses_api.model_copy(
update={
"usage": make_response_usage(
input_tokens=100,
output_tokens=10,
total_tokens=110,
cached_tokens=20,
cache_write_tokens=50,
)
}
)
provider_result = (
[
ResponseCompletedEvent(
type="response.completed", sequence_number=0, response=response
)
]
if stream
else response
)
with patch(
"openai.resources.responses.Responses.create", return_value=provider_result
):
client = OpenAI(api_key="test-key", posthog_client=mock_client)
result = client.responses.create(
model="gpt-4o-mini",
input="Hello",
stream=stream,
posthog_distinct_id="test-id",
)
if stream:
list(result)

props = mock_client.capture.call_args[1]["properties"]
assert props["$ai_input_tokens"] == 100
assert props["$ai_cache_read_input_tokens"] == 20
assert props["$ai_cache_creation_input_tokens"] == 50


@pytest.mark.parametrize("stream", [False, True])
def test_chat_cache_write_tokens(mock_client, mock_openai_response, stream):
usage = CompletionUsage(
prompt_tokens=100,
completion_tokens=10,
total_tokens=110,
prompt_tokens_details={"cached_tokens": 20, "cache_write_tokens": 50},
)
provider_result = (
[
ChatCompletionChunk(
id="chunk",
model="gpt-4",
object="chat.completion.chunk",
created=0,
choices=[],
usage=usage,
)
]
if stream
else mock_openai_response.model_copy(update={"usage": usage})
)
with patch(
"openai.resources.chat.completions.Completions.create",
return_value=provider_result,
):
client = OpenAI(api_key="test-key", posthog_client=mock_client)
result = client.chat.completions.create(
model="gpt-4",
messages=[{"role": "user", "content": "Hello"}],
stream=stream,
posthog_distinct_id="test-id",
)
if stream:
list(result)

props = mock_client.capture.call_args[1]["properties"]
assert props["$ai_input_tokens"] == 100
assert props["$ai_cache_read_input_tokens"] == 20
assert props["$ai_cache_creation_input_tokens"] == 50


def test_null_token_details_do_not_crash(
mock_client, mock_openai_response_with_null_token_details
):
Expand Down
4 changes: 3 additions & 1 deletion posthog/test/ai/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ def make_response_usage(
total_tokens: int,
cached_tokens: int = 0,
reasoning_tokens: int = 0,
cache_write_tokens: int = 0,
):
"""Build an ``openai.types.responses.ResponseUsage`` across SDK versions.

Expand All @@ -33,7 +34,8 @@ def build(model_cls, known):
output_tokens=output_tokens,
total_tokens=total_tokens,
input_tokens_details=build(
InputTokensDetails, {"cached_tokens": cached_tokens}
InputTokensDetails,
{"cached_tokens": cached_tokens, "cache_write_tokens": cache_write_tokens},
),
output_tokens_details=build(
OutputTokensDetails, {"reasoning_tokens": reasoning_tokens}
Expand Down
Loading