diff --git a/.sampo/changesets/gemini-tool-use-prompt-tokens.md b/.sampo/changesets/gemini-tool-use-prompt-tokens.md new file mode 100644 index 000000000..6c5f98d9a --- /dev/null +++ b/.sampo/changesets/gemini-tool-use-prompt-tokens.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Count Gemini URL context tool use prompt tokens as input tokens in captured generations; Google Search retrieval is not billed as input and stays out of the total. diff --git a/posthog/ai/gemini/gemini.py b/posthog/ai/gemini/gemini.py index dad87da9a..485e2adee 100644 --- a/posthog/ai/gemini/gemini.py +++ b/posthog/ai/gemini/gemini.py @@ -25,6 +25,7 @@ extract_gemini_embedding_token_count as extract_gemini_embedding_token_count, extract_gemini_stop_reason_from_chunk, extract_gemini_usage_from_chunk, + has_gemini_url_context, format_gemini_streaming_output as format_gemini_streaming_output, ) @@ -267,10 +268,14 @@ def generator(): nonlocal accumulated_content nonlocal stop_reason stream_error: Optional[BaseException] = None + url_context_seen = False try: for chunk in response: # Extract usage stats from chunk - chunk_usage = extract_gemini_usage_from_chunk(chunk) + url_context_seen = url_context_seen or has_gemini_url_context(chunk) + chunk_usage = extract_gemini_usage_from_chunk( + chunk, url_context_seen=url_context_seen + ) if chunk_usage: # Gemini reports cumulative totals, not incremental values diff --git a/posthog/ai/gemini/gemini_async.py b/posthog/ai/gemini/gemini_async.py index 6c70efd9b..6823743ed 100644 --- a/posthog/ai/gemini/gemini_async.py +++ b/posthog/ai/gemini/gemini_async.py @@ -25,6 +25,7 @@ extract_gemini_embedding_token_count as extract_gemini_embedding_token_count, extract_gemini_stop_reason_from_chunk, extract_gemini_usage_from_chunk, + has_gemini_url_context, format_gemini_streaming_output as format_gemini_streaming_output, ) @@ -254,10 +255,14 @@ async def async_generator(): nonlocal stop_reason stream_error: Optional[BaseException] = None + url_context_seen = False try: async for chunk in response: # Extract usage stats from chunk - chunk_usage = extract_gemini_usage_from_chunk(chunk) + url_context_seen = url_context_seen or has_gemini_url_context(chunk) + chunk_usage = extract_gemini_usage_from_chunk( + chunk, url_context_seen=url_context_seen + ) if chunk_usage: # Gemini reports cumulative totals, not incremental values diff --git a/posthog/ai/gemini/gemini_converter.py b/posthog/ai/gemini/gemini_converter.py index 09ae82d25..81b4f630b 100644 --- a/posthog/ai/gemini/gemini_converter.py +++ b/posthog/ai/gemini/gemini_converter.py @@ -517,19 +517,42 @@ def extract_gemini_web_search_count(response: Any) -> int: return 0 -def _extract_usage_from_metadata(metadata: Any) -> TokenUsage: +def has_gemini_url_context(response: Any) -> bool: + """Whether a response or chunk carries URL context retrieval metadata.""" + candidates = getattr(response, "candidates", None) + if not isinstance(candidates, (list, tuple)): + return False + return any(getattr(c, "url_context_metadata", None) for c in candidates) + + +def _extract_usage_from_metadata( + metadata: Any, bill_tool_use_tokens: bool = False +) -> TokenUsage: """ Common logic to extract usage from Gemini metadata. Used by both streaming and non-streaming paths. Args: metadata: usage_metadata from Gemini response or chunk + bill_tool_use_tokens: whether tool_use_prompt_token_count is billed as + input (URL context). Google Search retrieval is not billed as input. Returns: TokenUsage with standardized usage """ + input_tokens = getattr(metadata, "prompt_token_count", 0) + + # Built-in tools report the retrieved content separately from + # prompt_token_count. URL context content is billed as input; Google Search + # retrieval is not, so only add it when the response used URL context. + # (A response that used both tools cannot be split from these counts.) + if bill_tool_use_tokens: + tool_use_tokens = getattr(metadata, "tool_use_prompt_token_count", 0) + if isinstance(tool_use_tokens, int) and tool_use_tokens > 0: + input_tokens += tool_use_tokens + usage = TokenUsage( - input_tokens=getattr(metadata, "prompt_token_count", 0), + input_tokens=input_tokens, output_tokens=getattr(metadata, "candidates_token_count", 0), ) @@ -573,7 +596,9 @@ def extract_gemini_usage_from_response(response: Any) -> TokenUsage: if not hasattr(response, "usage_metadata") or not response.usage_metadata: return TokenUsage() - usage = _extract_usage_from_metadata(response.usage_metadata) + usage = _extract_usage_from_metadata( + response.usage_metadata, bill_tool_use_tokens=has_gemini_url_context(response) + ) # Add web search count if present web_search_count = extract_gemini_web_search_count(response) @@ -583,12 +608,17 @@ def extract_gemini_usage_from_response(response: Any) -> TokenUsage: return usage -def extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage: +def extract_gemini_usage_from_chunk( + chunk: Any, url_context_seen: bool = False +) -> TokenUsage: """ Extract usage statistics from a Gemini streaming chunk. Args: chunk: Streaming chunk from Gemini API + url_context_seen: whether an earlier chunk of the same stream carried + URL context metadata. Tool indicators can arrive on a different + chunk than the final usage_metadata, so callers carry this flag. Returns: TokenUsage with standardized usage statistics @@ -605,7 +635,10 @@ def extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage: if not hasattr(chunk, "usage_metadata") or not chunk.usage_metadata: return usage - usage_from_metadata = _extract_usage_from_metadata(chunk.usage_metadata) + usage_from_metadata = _extract_usage_from_metadata( + chunk.usage_metadata, + bill_tool_use_tokens=url_context_seen or has_gemini_url_context(chunk), + ) # Merge the usage from metadata with any web search count we found usage.update(usage_from_metadata) diff --git a/posthog/test/ai/gemini/test_gemini.py b/posthog/test/ai/gemini/test_gemini.py index b616dcd89..c5e8409b3 100644 --- a/posthog/test/ai/gemini/test_gemini.py +++ b/posthog/test/ai/gemini/test_gemini.py @@ -1741,3 +1741,40 @@ def test_ai_lane_client_routes_through_capture_ai( mock_client.capture.assert_not_called() assert mock_client.capture_ai.call_count == 1 assert mock_client.capture_ai.call_args[1]["event"] == "$ai_generation" + + +def _url_context_stream_chunks(): + from google.genai import types + + # URL context metadata arrives on an early chunk; usage on the last one. + first = types.GenerateContentResponse( + candidates=[types.Candidate(url_context_metadata=types.UrlContextMetadata())] + ) + last = types.GenerateContentResponse( + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=20, + tool_use_prompt_token_count=300, + candidates_token_count=10, + total_token_count=330, + ) + ) + return first, last + + +def test_streaming_url_context_on_earlier_chunk_bills_tool_use_tokens( + mock_client, mock_google_genai_client +): + first, last = _url_context_stream_chunks() + mock_google_genai_client.models.generate_content_stream.return_value = iter( + [first, last] + ) + + client = Client(api_key="test-key", posthog_client=mock_client) + response = client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["Summarise this page"] + ) + list(response) + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_input_tokens"] == 320 + assert props["$ai_output_tokens"] == 10 diff --git a/posthog/test/ai/gemini/test_gemini_async.py b/posthog/test/ai/gemini/test_gemini_async.py index b4e63c019..9cdc0fdbb 100644 --- a/posthog/test/ai/gemini/test_gemini_async.py +++ b/posthog/test/ai/gemini/test_gemini_async.py @@ -1178,3 +1178,40 @@ async def mock_streaming_response(): call_args = mock_client.capture.call_args[1] assert call_args["event"] == "$ai_generation" assert call_args["properties"]["$ai_provider"] == "gemini" + + +async def test_async_streaming_url_context_on_earlier_chunk_bills_tool_use_tokens( + mock_client, mock_google_genai_client +): + from google.genai import types + + first = types.GenerateContentResponse( + candidates=[types.Candidate(url_context_metadata=types.UrlContextMetadata())] + ) + last = types.GenerateContentResponse( + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=20, + tool_use_prompt_token_count=300, + candidates_token_count=10, + total_token_count=330, + ) + ) + + async def stream(): + yield first + yield last + + mock_google_genai_client.aio.models.generate_content_stream = AsyncMock( + return_value=stream() + ) + + client = AsyncClient(api_key="test-key", posthog_client=mock_client) + response = await client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["Summarise this page"] + ) + async for _ in response: + pass + + props = mock_client.capture.call_args[1]["properties"] + assert props["$ai_input_tokens"] == 320 + assert props["$ai_output_tokens"] == 10 diff --git a/posthog/test/ai/gemini/test_gemini_converter.py b/posthog/test/ai/gemini/test_gemini_converter.py index 7cdd211c7..4227b6d08 100644 --- a/posthog/test/ai/gemini/test_gemini_converter.py +++ b/posthog/test/ai/gemini/test_gemini_converter.py @@ -10,7 +10,9 @@ GEMINI_AVAILABLE = False from posthog.ai.gemini.gemini_converter import ( + extract_gemini_usage_from_chunk, extract_gemini_content_from_chunk, + extract_gemini_usage_from_response, format_gemini_input, format_gemini_response, format_gemini_streaming_output, @@ -311,3 +313,51 @@ def test_streaming_chunk_thought_part_becomes_thinking_block(self): {"type": "thinking", "thinking": "Hmm, "}, {"type": "text", "text": "done"}, ] + + +class TestGeminiUsage: + @staticmethod + def _response(**candidate_kwargs): + return types.GenerateContentResponse( + candidates=[types.Candidate(**candidate_kwargs)], + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=20, + tool_use_prompt_token_count=300, + candidates_token_count=10, + total_token_count=330, + ), + ) + + def test_url_context_tool_use_tokens_count_as_input(self): + # URL context content is billed as input. + resp = self._response(url_context_metadata=types.UrlContextMetadata()) + for extract in ( + extract_gemini_usage_from_response, + extract_gemini_usage_from_chunk, + ): + usage = extract(resp) + assert usage["input_tokens"] == 320 + assert usage["output_tokens"] == 10 + + def test_search_retrieval_tokens_are_not_billed_as_input(self): + # Google Search retrieval is reported in tool_use_prompt_token_count but + # is not charged as input. + resp = self._response( + grounding_metadata=types.GroundingMetadata(web_search_queries=["q"]) + ) + for extract in ( + extract_gemini_usage_from_response, + extract_gemini_usage_from_chunk, + ): + assert extract(resp)["input_tokens"] == 20 + + def test_input_tokens_without_tool_use(self): + resp = types.GenerateContentResponse( + usage_metadata=types.GenerateContentResponseUsageMetadata( + prompt_token_count=20, + candidates_token_count=10, + total_token_count=30, + ) + ) + usage = extract_gemini_usage_from_response(resp) + assert usage["input_tokens"] == 20 diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 3eadb81bd..fbac711ed 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -113,6 +113,7 @@ alias posthog.ai.gemini.gemini.extract_gemini_stop_reason_from_chunk -> posthog. alias posthog.ai.gemini.gemini.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk alias posthog.ai.gemini.gemini.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.gemini.gemini.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output +alias posthog.ai.gemini.gemini.has_gemini_url_context -> posthog.ai.gemini.gemini_converter.has_gemini_url_context alias posthog.ai.gemini.gemini.merge_system_prompt -> posthog.ai.utils.merge_system_prompt alias posthog.ai.gemini.gemini.merge_usage_stats -> posthog.ai.utils.merge_usage_stats alias posthog.ai.gemini.gemini.setup -> posthog.setup @@ -129,6 +130,7 @@ alias posthog.ai.gemini.gemini_async.extract_gemini_stop_reason_from_chunk -> po alias posthog.ai.gemini.gemini_async.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk alias posthog.ai.gemini.gemini_async.finalize_ai_content -> posthog.ai.utils.finalize_ai_content alias posthog.ai.gemini.gemini_async.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output +alias posthog.ai.gemini.gemini_async.has_gemini_url_context -> posthog.ai.gemini.gemini_converter.has_gemini_url_context alias posthog.ai.gemini.gemini_async.merge_system_prompt -> posthog.ai.utils.merge_system_prompt alias posthog.ai.gemini.gemini_async.merge_usage_stats -> posthog.ai.utils.merge_usage_stats alias posthog.ai.gemini.gemini_async.setup -> posthog.setup @@ -1414,13 +1416,14 @@ function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason(response: function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason_from_chunk(chunk: Any) -> Optional[str] function posthog.ai.gemini.gemini_converter.extract_gemini_system_instruction(config: Any) -> Optional[str] function posthog.ai.gemini.gemini_converter.extract_gemini_tools(kwargs: Dict[str, Any]) -> Optional[Any] -function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage +function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk(chunk: Any, url_context_seen: bool = False) -> TokenUsage function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_response(response: Any) -> TokenUsage function posthog.ai.gemini.gemini_converter.extract_gemini_web_search_count(response: Any) -> int function posthog.ai.gemini.gemini_converter.format_gemini_input(contents: Any) -> List[FormattedMessage] function posthog.ai.gemini.gemini_converter.format_gemini_input_with_system(contents: Any, config: Any = None) -> List[FormattedMessage] function posthog.ai.gemini.gemini_converter.format_gemini_response(response: Any) -> List[FormattedMessage] function posthog.ai.gemini.gemini_converter.format_gemini_streaming_output(accumulated_content: Union[str, List[Any]]) -> List[FormattedMessage] +function posthog.ai.gemini.gemini_converter.has_gemini_url_context(response: Any) -> bool function posthog.ai.media.bytes_to_base64(data: bytes) -> str function posthog.ai.media.ensure_serializable(obj: Any) -> Any function posthog.ai.media.normalize_part_keys(d: dict) -> dict