diff --git a/.sampo/changesets/openai-cache-write-tokens.md b/.sampo/changesets/openai-cache-write-tokens.md new file mode 100644 index 000000000..e1c32dac0 --- /dev/null +++ b/.sampo/changesets/openai-cache-write-tokens.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +`posthog.ai.openai`: report OpenAI cache-write tokens as `$ai_cache_creation_input_tokens`, for streaming and non-streaming calls. OpenAI reports them in `input_tokens_details.cache_write_tokens` (Responses) or `prompt_tokens_details.cache_write_tokens` (Chat Completions) and bills them above the input rate, but the wrapper read only `cached_tokens`, so cache writes were costed as plain input tokens. diff --git a/posthog/ai/openai/openai_converter.py b/posthog/ai/openai/openai_converter.py index 72f6dc457..ce421b3dc 100644 --- a/posthog/ai/openai/openai_converter.py +++ b/posthog/ai/openai/openai_converter.py @@ -692,6 +692,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A ) detail_fields = ( ("cache_read_input_tokens", "input_tokens_details", "cached_tokens"), + ( + "cache_creation_input_tokens", + "input_tokens_details", + "cache_write_tokens", + ), ("reasoning_tokens", "output_tokens_details", "reasoning_tokens"), ) else: @@ -701,6 +706,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A ) detail_fields = ( ("cache_read_input_tokens", "prompt_tokens_details", "cached_tokens"), + ( + "cache_creation_input_tokens", + "prompt_tokens_details", + "cache_write_tokens", + ), ("reasoning_tokens", "completion_tokens_details", "reasoning_tokens"), ) counts = {} @@ -716,7 +726,11 @@ def _read_openai_response_usage_fields(usage: Any, api_type: str) -> Dict[str, A def _add_openai_response_details( result: Dict[str, Any], counts: Dict[str, Any] ) -> None: - for field in ("cache_read_input_tokens", "reasoning_tokens"): + for field in ( + "cache_read_input_tokens", + "cache_creation_input_tokens", + "reasoning_tokens", + ): value = counts.get(field) if value is not None and value > 0: result[field] = value @@ -739,6 +753,7 @@ def _extract_openai_stream_token_usage(raw: Any, provider_type: str) -> TokenUsa # Unlike non-streaming usage, explicitly reported zero details are retained. for output, field, detail in ( ("cache_read_input_tokens", input_details, "cached_tokens"), + ("cache_creation_input_tokens", input_details, "cache_write_tokens"), ("reasoning_tokens", output_details, "reasoning_tokens"), ): if hasattr(raw, field) and hasattr(getattr(raw, field), detail): diff --git a/posthog/test/ai/openai/test_openai.py b/posthog/test/ai/openai/test_openai.py index edbe3c9de..a27a2ba06 100644 --- a/posthog/test/ai/openai/test_openai.py +++ b/posthog/test/ai/openai/test_openai.py @@ -25,6 +25,7 @@ from openai.types.responses import ( ParsedResponse, Response, + ResponseCompletedEvent, ResponseFunctionToolCall, ResponseOutputMessage, ResponseOutputText, @@ -638,6 +639,91 @@ def test_cached_tokens(mock_client, mock_openai_response_with_cached_tokens): assert isinstance(props["$ai_latency"], float) +@pytest.mark.parametrize("stream", [False, True]) +def test_responses_cache_write_tokens( + mock_client, mock_openai_response_with_responses_api, stream +): + response = mock_openai_response_with_responses_api.model_copy( + update={ + "usage": make_response_usage( + input_tokens=100, + output_tokens=10, + total_tokens=110, + cached_tokens=20, + cache_write_tokens=50, + ) + } + ) + provider_result = ( + [ + ResponseCompletedEvent( + type="response.completed", sequence_number=0, response=response + ) + ] + if stream + else response + ) + with patch( + "openai.resources.responses.Responses.create", return_value=provider_result + ): + client = OpenAI(api_key="test-key", posthog_client=mock_client) + result = client.responses.create( + model="gpt-4o-mini", + input="Hello", + stream=stream, + posthog_distinct_id="test-id", + ) + if stream: + list(result) + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_input_tokens"] == 100 + assert props["$ai_cache_read_input_tokens"] == 20 + assert props["$ai_cache_creation_input_tokens"] == 50 + + +@pytest.mark.parametrize("stream", [False, True]) +def test_chat_cache_write_tokens(mock_client, mock_openai_response, stream): + usage = CompletionUsage( + prompt_tokens=100, + completion_tokens=10, + total_tokens=110, + prompt_tokens_details={"cached_tokens": 20, "cache_write_tokens": 50}, + ) + provider_result = ( + [ + ChatCompletionChunk( + id="chunk", + model="gpt-4", + object="chat.completion.chunk", + created=0, + choices=[], + usage=usage, + ) + ] + if stream + else mock_openai_response.model_copy(update={"usage": usage}) + ) + with patch( + "openai.resources.chat.completions.Completions.create", + return_value=provider_result, + ): + client = OpenAI(api_key="test-key", posthog_client=mock_client) + result = client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "Hello"}], + stream=stream, + posthog_distinct_id="test-id", + ) + if stream: + list(result) + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_input_tokens"] == 100 + assert props["$ai_cache_read_input_tokens"] == 20 + assert props["$ai_cache_creation_input_tokens"] == 50 + + def test_null_token_details_do_not_crash( mock_client, mock_openai_response_with_null_token_details ): diff --git a/posthog/test/ai/utils.py b/posthog/test/ai/utils.py index 9f27c1f38..306736342 100644 --- a/posthog/test/ai/utils.py +++ b/posthog/test/ai/utils.py @@ -7,6 +7,7 @@ def make_response_usage( total_tokens: int, cached_tokens: int = 0, reasoning_tokens: int = 0, + cache_write_tokens: int = 0, ): """Build an ``openai.types.responses.ResponseUsage`` across SDK versions. @@ -33,7 +34,8 @@ def build(model_cls, known): output_tokens=output_tokens, total_tokens=total_tokens, input_tokens_details=build( - InputTokensDetails, {"cached_tokens": cached_tokens} + InputTokensDetails, + {"cached_tokens": cached_tokens, "cache_write_tokens": cache_write_tokens}, ), output_tokens_details=build( OutputTokensDetails, {"reasoning_tokens": reasoning_tokens}