From 1538804b4fe6265d5b7a06f9fbbab8ac68915b3a Mon Sep 17 00:00:00 2001 From: jaeminbird Date: Wed, 4 Nov 2026 20:20:27 -0500 Subject: [PATCH 1/2] fix(ai): stop subtracting cache tokens twice for non-streamed ChatAnthropic calls ChatAnthropic.invoke copies the raw Anthropic usage into response_metadata, and the callback handler read it before the standard usage_metadata. Its input_tokens already exclude cache reads and writes, so the Anthropic cache subtraction ran a second time and a cached call reported 0 input tokens. Read usage_metadata first, as streamed calls already do. The same check also covers Claude models served by other LangChain providers (ChatAnthropicVertex), which report Anthropic-style usage with cache tokens outside input_tokens. --- .../langchain-anthropic-cache-input-tokens.md | 5 ++ posthog/ai/langchain/callbacks.py | 16 ++-- posthog/test/ai/langchain/test_callbacks.py | 87 +++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 .sampo/changesets/langchain-anthropic-cache-input-tokens.md diff --git a/.sampo/changesets/langchain-anthropic-cache-input-tokens.md b/.sampo/changesets/langchain-anthropic-cache-input-tokens.md new file mode 100644 index 000000000..fd34d4c9b --- /dev/null +++ b/.sampo/changesets/langchain-anthropic-cache-input-tokens.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +The LangChain callback handler no longer reports too few input tokens for a non-streamed `ChatAnthropic` call that reads or writes the prompt cache. It took the raw Anthropic usage from `response_metadata`, whose input tokens already exclude the cache, and subtracted the cache tokens again, so a cached call usually reported 0 input tokens. It now reads LangChain's standard `usage_metadata` first, as it already did for streamed calls. diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 757179aac..e94d843e8 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -969,11 +969,12 @@ def _parse_usage_model( # Both cache_read and cache_write tokens should be subtracted since Anthropic's raw API # reports input_tokens as tokens NOT read from or used to create a cache. # For other providers (OpenAI, etc.), input_tokens already excludes cache tokens as expected. - # Match logic consistent with plugin-server: exact match on provider OR substring match on model + # Match logic consistent with plugin-server: exact match on provider, substring + # match on model, or a Claude model under another provider (e.g. Vertex AI) is_anthropic = False if provider and provider.lower() == "anthropic": is_anthropic = True - elif model and "anthropic" in model.lower(): + elif model and ("anthropic" in model.lower() or model.lower().startswith("claude")): is_anthropic = True if is_anthropic and normalized_usage.input_tokens: @@ -1047,12 +1048,15 @@ def _parse_usage( if isinstance(response_metadata, dict) else None ) - ollama_usage = getattr( - message_chunk, "usage_metadata", None - ) # for Ollama + # LangChain's standard usage, which also covers Ollama + usage_metadata = getattr(message_chunk, "usage_metadata", None) + # Prefer the standard usage: its input_tokens include cache reads + # and writes, as the Anthropic subtraction in _parse_usage_model + # expects. ChatAnthropic also copies the raw API usage, whose + # input_tokens already exclude the cache, into response_metadata. chunk_usage = ( - bedrock_anthropic_usage or bedrock_titan_usage or ollama_usage + usage_metadata or bedrock_anthropic_usage or bedrock_titan_usage ) if chunk_usage: llm_usage = _parse_usage_model(chunk_usage, provider, model) diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 4b0057309..b4ae495e8 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -1724,6 +1724,93 @@ def test_anthropic_provider_subtracts_cache_tokens(mock_client): assert generation_args["properties"]["$ai_cache_read_input_tokens"] == 800 +def test_chat_anthropic_invoke_does_not_subtract_cache_tokens_twice(mock_client): + """ChatAnthropic.invoke puts the raw API usage, whose input_tokens already + exclude the cache, in response_metadata next to the standard usage_metadata.""" + from langchain_core.outputs import LLMResult, ChatGeneration + from langchain_core.messages import AIMessage + from uuid import uuid4 + + cb = CallbackHandler(mock_client) + run_id = uuid4() + cb._set_llm_metadata( + serialized={}, + run_id=run_id, + messages=[{"role": "user", "content": "test"}], + metadata={"ls_provider": "anthropic", "ls_model_name": "claude-sonnet-4-5"}, + ) + + # The message ChatAnthropic.invoke returns for a cached prompt + message = AIMessage( + content="Response", + response_metadata={ + "id": "msg_1", + "model": "claude-sonnet-4-5", + "stop_reason": "end_turn", + "usage": { + "input_tokens": 10, + "output_tokens": 7, + "cache_creation_input_tokens": 100, + "cache_read_input_tokens": 1000, + }, + "model_provider": "anthropic", + }, + usage_metadata={ + "input_tokens": 1110, + "output_tokens": 7, + "total_tokens": 1117, + "input_token_details": {"cache_read": 1000, "cache_creation": 100}, + }, + ) + response = LLMResult(generations=[[ChatGeneration(message=message)]]) + + cb._pop_run_and_capture_generation(run_id, None, response) + + props = mock_client.capture.call_args_list[0][1]["properties"] + assert props["$ai_input_tokens"] == 10 + assert props["$ai_output_tokens"] == 7 + assert props["$ai_cache_read_input_tokens"] == 1000 + assert props["$ai_cache_creation_input_tokens"] == 100 + + +def test_claude_on_vertex_subtracts_cache_tokens(mock_client): + """ChatAnthropicVertex reports provider "anthropicvertex" and a bare Claude + model name. Its usage_metadata input_tokens include the cache, like + ChatAnthropic's, and PostHog prices claude-* models Anthropic-style.""" + from langchain_core.outputs import LLMResult, ChatGeneration + from langchain_core.messages import AIMessage + from uuid import uuid4 + + cb = CallbackHandler(mock_client) + run_id = uuid4() + cb._set_llm_metadata( + serialized={}, + run_id=run_id, + messages=[{"role": "user", "content": "test"}], + metadata={ + "ls_provider": "anthropicvertex", + "ls_model_name": "claude-sonnet-4-5@20250929", + }, + ) + message = AIMessage( + content="Response", + usage_metadata={ + "input_tokens": 1110, + "output_tokens": 7, + "total_tokens": 1117, + "input_token_details": {"cache_read": 1000, "cache_creation": 100}, + }, + ) + response = LLMResult(generations=[[ChatGeneration(message=message)]]) + + cb._pop_run_and_capture_generation(run_id, None, response) + + props = mock_client.capture.call_args_list[0][1]["properties"] + assert props["$ai_input_tokens"] == 10 + assert props["$ai_cache_read_input_tokens"] == 1000 + assert props["$ai_cache_creation_input_tokens"] == 100 + + def test_anthropic_provider_subtracts_cache_write_tokens(mock_client): """Test that Anthropic provider correctly subtracts cache write tokens from input tokens.""" from langchain_core.outputs import LLMResult, ChatGeneration From 5ab73c9a67d92ad870b619fd00159d7f73f70cd2 Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Thu, 5 Nov 2026 12:52:53 +0100 Subject: [PATCH 2/2] test: isolate metrics assertion from background HTTP traffic --- posthog/test/test_metrics_autocapture.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/posthog/test/test_metrics_autocapture.py b/posthog/test/test_metrics_autocapture.py index e69310b49..57221289d 100644 --- a/posthog/test/test_metrics_autocapture.py +++ b/posthog/test/test_metrics_autocapture.py @@ -941,7 +941,12 @@ def test_does_not_propagate_context_to_other_services( for request in app_server.received: headers = {key.lower() for key in request["headers"]} assert not headers & {"traceparent", "tracestate", "baggage"} - assert len(_points(reader, "http.client.request.duration")) == 2 + points = [ + point + for point in _points(reader, "http.client.request.duration") + if point.attributes.get("server.port") == app_server.port + ] + assert len(points) == 2 handle.shutdown() requests_instrumentation = sys.modules["opentelemetry.instrumentation.requests"]