Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/gemini-tool-use-prompt-tokens.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

Count Gemini URL context tool use prompt tokens as input tokens in captured generations; Google Search retrieval is not billed as input and stays out of the total.
7 changes: 6 additions & 1 deletion posthog/ai/gemini/gemini.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
extract_gemini_embedding_token_count as extract_gemini_embedding_token_count,
extract_gemini_stop_reason_from_chunk,
extract_gemini_usage_from_chunk,
has_gemini_url_context,
format_gemini_streaming_output as format_gemini_streaming_output,
)

Expand Down Expand Up @@ -267,10 +268,14 @@ def generator():
nonlocal accumulated_content
nonlocal stop_reason
stream_error: Optional[BaseException] = None
url_context_seen = False
try:
for chunk in response:
# Extract usage stats from chunk
chunk_usage = extract_gemini_usage_from_chunk(chunk)
url_context_seen = url_context_seen or has_gemini_url_context(chunk)
chunk_usage = extract_gemini_usage_from_chunk(
chunk, url_context_seen=url_context_seen
)

if chunk_usage:
# Gemini reports cumulative totals, not incremental values
Expand Down
7 changes: 6 additions & 1 deletion posthog/ai/gemini/gemini_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
extract_gemini_embedding_token_count as extract_gemini_embedding_token_count,
extract_gemini_stop_reason_from_chunk,
extract_gemini_usage_from_chunk,
has_gemini_url_context,
format_gemini_streaming_output as format_gemini_streaming_output,
)

Expand Down Expand Up @@ -254,10 +255,14 @@ async def async_generator():
nonlocal stop_reason

stream_error: Optional[BaseException] = None
url_context_seen = False
try:
async for chunk in response:
# Extract usage stats from chunk
chunk_usage = extract_gemini_usage_from_chunk(chunk)
url_context_seen = url_context_seen or has_gemini_url_context(chunk)
chunk_usage = extract_gemini_usage_from_chunk(
chunk, url_context_seen=url_context_seen
)

if chunk_usage:
# Gemini reports cumulative totals, not incremental values
Expand Down
43 changes: 38 additions & 5 deletions posthog/ai/gemini/gemini_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -517,19 +517,42 @@ def extract_gemini_web_search_count(response: Any) -> int:
return 0


def _extract_usage_from_metadata(metadata: Any) -> TokenUsage:
def has_gemini_url_context(response: Any) -> bool:
"""Whether a response or chunk carries URL context retrieval metadata."""
candidates = getattr(response, "candidates", None)
if not isinstance(candidates, (list, tuple)):
return False
return any(getattr(c, "url_context_metadata", None) for c in candidates)


def _extract_usage_from_metadata(
metadata: Any, bill_tool_use_tokens: bool = False
) -> TokenUsage:
"""
Common logic to extract usage from Gemini metadata.
Used by both streaming and non-streaming paths.

Args:
metadata: usage_metadata from Gemini response or chunk
bill_tool_use_tokens: whether tool_use_prompt_token_count is billed as
input (URL context). Google Search retrieval is not billed as input.

Returns:
TokenUsage with standardized usage
"""
input_tokens = getattr(metadata, "prompt_token_count", 0)

# Built-in tools report the retrieved content separately from
# prompt_token_count. URL context content is billed as input; Google Search
# retrieval is not, so only add it when the response used URL context.
# (A response that used both tools cannot be split from these counts.)
if bill_tool_use_tokens:
tool_use_tokens = getattr(metadata, "tool_use_prompt_token_count", 0)
if isinstance(tool_use_tokens, int) and tool_use_tokens > 0:
input_tokens += tool_use_tokens

usage = TokenUsage(
input_tokens=getattr(metadata, "prompt_token_count", 0),
input_tokens=input_tokens,
output_tokens=getattr(metadata, "candidates_token_count", 0),
)

Expand Down Expand Up @@ -573,7 +596,9 @@ def extract_gemini_usage_from_response(response: Any) -> TokenUsage:
if not hasattr(response, "usage_metadata") or not response.usage_metadata:
return TokenUsage()

usage = _extract_usage_from_metadata(response.usage_metadata)
usage = _extract_usage_from_metadata(
response.usage_metadata, bill_tool_use_tokens=has_gemini_url_context(response)
)

# Add web search count if present
web_search_count = extract_gemini_web_search_count(response)
Expand All @@ -583,12 +608,17 @@ def extract_gemini_usage_from_response(response: Any) -> TokenUsage:
return usage


def extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage:
def extract_gemini_usage_from_chunk(
chunk: Any, url_context_seen: bool = False
) -> TokenUsage:
"""
Extract usage statistics from a Gemini streaming chunk.

Args:
chunk: Streaming chunk from Gemini API
url_context_seen: whether an earlier chunk of the same stream carried
URL context metadata. Tool indicators can arrive on a different
chunk than the final usage_metadata, so callers carry this flag.

Returns:
TokenUsage with standardized usage statistics
Expand All @@ -605,7 +635,10 @@ def extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage:
if not hasattr(chunk, "usage_metadata") or not chunk.usage_metadata:
return usage

usage_from_metadata = _extract_usage_from_metadata(chunk.usage_metadata)
usage_from_metadata = _extract_usage_from_metadata(
chunk.usage_metadata,
bill_tool_use_tokens=url_context_seen or has_gemini_url_context(chunk),
)

# Merge the usage from metadata with any web search count we found
usage.update(usage_from_metadata)
Expand Down
37 changes: 37 additions & 0 deletions posthog/test/ai/gemini/test_gemini.py
Original file line number Diff line number Diff line change
Expand Up @@ -1741,3 +1741,40 @@ def test_ai_lane_client_routes_through_capture_ai(
mock_client.capture.assert_not_called()
assert mock_client.capture_ai.call_count == 1
assert mock_client.capture_ai.call_args[1]["event"] == "$ai_generation"


def _url_context_stream_chunks():
from google.genai import types

# URL context metadata arrives on an early chunk; usage on the last one.
first = types.GenerateContentResponse(
candidates=[types.Candidate(url_context_metadata=types.UrlContextMetadata())]
)
last = types.GenerateContentResponse(
usage_metadata=types.GenerateContentResponseUsageMetadata(
prompt_token_count=20,
tool_use_prompt_token_count=300,
candidates_token_count=10,
total_token_count=330,
)
)
return first, last


def test_streaming_url_context_on_earlier_chunk_bills_tool_use_tokens(
mock_client, mock_google_genai_client
):
first, last = _url_context_stream_chunks()
mock_google_genai_client.models.generate_content_stream.return_value = iter(
[first, last]
)

client = Client(api_key="test-key", posthog_client=mock_client)
response = client.models.generate_content_stream(
model="gemini-2.0-flash", contents=["Summarise this page"]
)
list(response)

props = mock_client.capture.call_args[1]["properties"]
assert props["$ai_input_tokens"] == 320
assert props["$ai_output_tokens"] == 10
37 changes: 37 additions & 0 deletions posthog/test/ai/gemini/test_gemini_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -1178,3 +1178,40 @@ async def mock_streaming_response():
call_args = mock_client.capture.call_args[1]
assert call_args["event"] == "$ai_generation"
assert call_args["properties"]["$ai_provider"] == "gemini"


async def test_async_streaming_url_context_on_earlier_chunk_bills_tool_use_tokens(
mock_client, mock_google_genai_client
):
from google.genai import types

first = types.GenerateContentResponse(
candidates=[types.Candidate(url_context_metadata=types.UrlContextMetadata())]
)
last = types.GenerateContentResponse(
usage_metadata=types.GenerateContentResponseUsageMetadata(
prompt_token_count=20,
tool_use_prompt_token_count=300,
candidates_token_count=10,
total_token_count=330,
)
)

async def stream():
yield first
yield last

mock_google_genai_client.aio.models.generate_content_stream = AsyncMock(
return_value=stream()
)

client = AsyncClient(api_key="test-key", posthog_client=mock_client)
response = await client.models.generate_content_stream(
model="gemini-2.0-flash", contents=["Summarise this page"]
)
async for _ in response:
pass

props = mock_client.capture.call_args[1]["properties"]
assert props["$ai_input_tokens"] == 320
assert props["$ai_output_tokens"] == 10
50 changes: 50 additions & 0 deletions posthog/test/ai/gemini/test_gemini_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
GEMINI_AVAILABLE = False

from posthog.ai.gemini.gemini_converter import (
extract_gemini_usage_from_chunk,
extract_gemini_content_from_chunk,
extract_gemini_usage_from_response,
format_gemini_input,
format_gemini_response,
format_gemini_streaming_output,
Expand Down Expand Up @@ -311,3 +313,51 @@ def test_streaming_chunk_thought_part_becomes_thinking_block(self):
{"type": "thinking", "thinking": "Hmm, "},
{"type": "text", "text": "done"},
]


class TestGeminiUsage:
@staticmethod
def _response(**candidate_kwargs):
return types.GenerateContentResponse(
candidates=[types.Candidate(**candidate_kwargs)],
usage_metadata=types.GenerateContentResponseUsageMetadata(
prompt_token_count=20,
tool_use_prompt_token_count=300,
candidates_token_count=10,
total_token_count=330,
),
)

def test_url_context_tool_use_tokens_count_as_input(self):
# URL context content is billed as input.
resp = self._response(url_context_metadata=types.UrlContextMetadata())
for extract in (
extract_gemini_usage_from_response,
extract_gemini_usage_from_chunk,
):
usage = extract(resp)
assert usage["input_tokens"] == 320
assert usage["output_tokens"] == 10

def test_search_retrieval_tokens_are_not_billed_as_input(self):
# Google Search retrieval is reported in tool_use_prompt_token_count but
# is not charged as input.
resp = self._response(
grounding_metadata=types.GroundingMetadata(web_search_queries=["q"])
)
for extract in (
extract_gemini_usage_from_response,
extract_gemini_usage_from_chunk,
):
assert extract(resp)["input_tokens"] == 20

def test_input_tokens_without_tool_use(self):
resp = types.GenerateContentResponse(
usage_metadata=types.GenerateContentResponseUsageMetadata(
prompt_token_count=20,
candidates_token_count=10,
total_token_count=30,
)
)
usage = extract_gemini_usage_from_response(resp)
assert usage["input_tokens"] == 20
5 changes: 4 additions & 1 deletion references/public_api_snapshot.txt
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ alias posthog.ai.gemini.gemini.extract_gemini_stop_reason_from_chunk -> posthog.
alias posthog.ai.gemini.gemini.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk
alias posthog.ai.gemini.gemini.finalize_ai_content -> posthog.ai.utils.finalize_ai_content
alias posthog.ai.gemini.gemini.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output
alias posthog.ai.gemini.gemini.has_gemini_url_context -> posthog.ai.gemini.gemini_converter.has_gemini_url_context
alias posthog.ai.gemini.gemini.merge_system_prompt -> posthog.ai.utils.merge_system_prompt
alias posthog.ai.gemini.gemini.merge_usage_stats -> posthog.ai.utils.merge_usage_stats
alias posthog.ai.gemini.gemini.setup -> posthog.setup
Expand All @@ -129,6 +130,7 @@ alias posthog.ai.gemini.gemini_async.extract_gemini_stop_reason_from_chunk -> po
alias posthog.ai.gemini.gemini_async.extract_gemini_usage_from_chunk -> posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk
alias posthog.ai.gemini.gemini_async.finalize_ai_content -> posthog.ai.utils.finalize_ai_content
alias posthog.ai.gemini.gemini_async.format_gemini_streaming_output -> posthog.ai.gemini.gemini_converter.format_gemini_streaming_output
alias posthog.ai.gemini.gemini_async.has_gemini_url_context -> posthog.ai.gemini.gemini_converter.has_gemini_url_context
alias posthog.ai.gemini.gemini_async.merge_system_prompt -> posthog.ai.utils.merge_system_prompt
alias posthog.ai.gemini.gemini_async.merge_usage_stats -> posthog.ai.utils.merge_usage_stats
alias posthog.ai.gemini.gemini_async.setup -> posthog.setup
Expand Down Expand Up @@ -1414,13 +1416,14 @@ function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason(response:
function posthog.ai.gemini.gemini_converter.extract_gemini_stop_reason_from_chunk(chunk: Any) -> Optional[str]
function posthog.ai.gemini.gemini_converter.extract_gemini_system_instruction(config: Any) -> Optional[str]
function posthog.ai.gemini.gemini_converter.extract_gemini_tools(kwargs: Dict[str, Any]) -> Optional[Any]
function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk(chunk: Any) -> TokenUsage
function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_chunk(chunk: Any, url_context_seen: bool = False) -> TokenUsage
function posthog.ai.gemini.gemini_converter.extract_gemini_usage_from_response(response: Any) -> TokenUsage
function posthog.ai.gemini.gemini_converter.extract_gemini_web_search_count(response: Any) -> int
function posthog.ai.gemini.gemini_converter.format_gemini_input(contents: Any) -> List[FormattedMessage]
function posthog.ai.gemini.gemini_converter.format_gemini_input_with_system(contents: Any, config: Any = None) -> List[FormattedMessage]
function posthog.ai.gemini.gemini_converter.format_gemini_response(response: Any) -> List[FormattedMessage]
function posthog.ai.gemini.gemini_converter.format_gemini_streaming_output(accumulated_content: Union[str, List[Any]]) -> List[FormattedMessage]
function posthog.ai.gemini.gemini_converter.has_gemini_url_context(response: Any) -> bool
function posthog.ai.media.bytes_to_base64(data: bytes) -> str
function posthog.ai.media.ensure_serializable(obj: Any) -> Any
function posthog.ai.media.normalize_part_keys(d: dict) -> dict
Expand Down
Loading