From f26dc95d43b22b4bb2ad612d93a5fabc544f66b9 Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Thu, 19 Nov 2026 19:05:26 -0800 Subject: [PATCH 1/4] feat(ai): send tool results and errors on tool spans Claude Agent SDK tool spans now wait for the matching tool result and carry the output, error flag, error text and latency. LangChain tool spans set $ai_span_type to tool, mark tools that return an error ToolMessage, and converted ToolMessages keep tool_call_id, name and error status, so tool results can be matched to tool names. Co-Authored-By: Claude Opus 5.5 --- .sampo/changesets/ai-tool-span-results.md | 5 + posthog/ai/claude_agent_sdk/client.py | 26 +- posthog/ai/claude_agent_sdk/processor.py | 287 ++++++++++++------ posthog/ai/langchain/callbacks.py | 34 ++- posthog/ai/utils.py | 31 ++ .../test/ai/claude_agent_sdk/test_client.py | 53 +++- .../ai/claude_agent_sdk/test_processor.py | 136 +++++++++ posthog/test/ai/langchain/test_callbacks.py | 113 +++++++ 8 files changed, 578 insertions(+), 107 deletions(-) create mode 100644 .sampo/changesets/ai-tool-span-results.md diff --git a/.sampo/changesets/ai-tool-span-results.md b/.sampo/changesets/ai-tool-span-results.md new file mode 100644 index 000000000..5c11869f4 --- /dev/null +++ b/.sampo/changesets/ai-tool-span-results.md @@ -0,0 +1,5 @@ +--- +posthog: patch +--- + +Send tool results on AI tool spans. Claude Agent SDK tool spans now wait for the tool's result and carry `$ai_output_state`, `$ai_is_error`, `$ai_error` and `$ai_latency`. LangChain tool spans set `$ai_span_type: "tool"` and mark tools that return an error `ToolMessage`, and converted `ToolMessage`s keep `tool_call_id`, `name` and an error `status`. diff --git a/posthog/ai/claude_agent_sdk/client.py b/posthog/ai/claude_agent_sdk/client.py index f8fb124e6..8a591efb4 100644 --- a/posthog/ai/claude_agent_sdk/client.py +++ b/posthog/ai/claude_agent_sdk/client.py @@ -32,6 +32,7 @@ from posthog.ai.claude_agent_sdk.processor import ( PostHogClaudeAgentProcessor, _GenerationTracker, + _ToolSpanTracker, _from_subagent, _promote_input, _starts_generation, @@ -114,6 +115,7 @@ def __init__( # Shared state across turns self._tracker = _GenerationTracker() + self._tool_spans = _ToolSpanTracker(self._processor) self._generation_index = 0 self._current_generation_span_id: Optional[str] = None self._current_input: Optional[List[Dict[str, Any]]] = None @@ -186,16 +188,21 @@ def _process_message(self, message: Any) -> None: elif isinstance(message, AssistantMessage): self._process_assistant_message(message) - elif isinstance(message, UserMessage) and not _from_subagent(message): - if not self._tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant turn; - # the result fallback reports the final turn only. - self._pending_output = [] - self._next_input = _format_user_input( - message.content, self._processor._client, self._next_input - ) + elif isinstance(message, UserMessage): + if not _from_subagent(message): + if not self._tracker.had_any_stream_events: + # Without stream events, a tool result ends the assistant + # turn; the result fallback reports the final turn only. + self._pending_output = [] + self._next_input = _format_user_input( + message.content, self._processor._client, self._next_input + ) + # Subagent tool spans are tracked too, so match their results. + self._tool_spans.finish(message.content) elif isinstance(message, ResultMessage): + # The turn is over: spans still waiting for a result go out now. + self._tool_spans.flush() if not self._tracker.had_any_stream_events: self._processor._emit_generation_from_result( message, @@ -217,7 +224,7 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None: parent_id = self._tracker.current_span_id or self._current_generation_span_id for block in message.content: if isinstance(block, ToolUseBlock): - self._processor._emit_tool_span( + self._tool_spans.start( block, self._trace_id, parent_id, @@ -237,6 +244,7 @@ async def disconnect(self) -> None: """ # Emit the trace event covering the entire session try: + self._tool_spans.flush() latency = time.time() - self._query_start resolved_id = self._processor._resolve_distinct_id(self._distinct_id) diff --git a/posthog/ai/claude_agent_sdk/processor.py b/posthog/ai/claude_agent_sdk/processor.py index ec9b46c3d..4b64d4d42 100644 --- a/posthog/ai/claude_agent_sdk/processor.py +++ b/posthog/ai/claude_agent_sdk/processor.py @@ -14,6 +14,7 @@ from claude_agent_sdk import ( AssistantMessage, ResultMessage, + ToolResultBlock, ToolUseBlock, UserMessage, ) @@ -34,6 +35,7 @@ from posthog.ai.utils import ( _ai_event_options, _capture_processor_event, + _format_tool_error, _resolve_ai_client, finalize_ai_content, with_privacy_mode as _with_privacy_mode, @@ -62,6 +64,60 @@ class _GenerationData: stop_reason: Optional[str] = None +@dataclass +class _PendingToolSpan: + """A tool $ai_span waiting for the ToolResultBlock of its tool use.""" + + properties: Dict[str, Any] + start_time: float + distinct_id: Optional[str] + trace_id: str + privacy: bool + groups: Dict[str, Any] + options: Optional[Dict[str, Any]] + + +class _ToolSpanTracker: + """Holds each tool span until its result arrives, keyed by tool_use_id. + + The span is sent with the result's output, error flag and latency. Spans + still waiting when the query or session ends are sent without a result. + """ + + def __init__(self, processor: "PostHogClaudeAgentProcessor") -> None: + self._processor = processor + self._pending: Dict[str, _PendingToolSpan] = {} + + def start(self, block: "ToolUseBlock", *args: Any, **kwargs: Any) -> None: + previous = self._pending.pop(block.id, None) + if previous is not None: + self._processor._emit_tool_span(previous) + self._pending[block.id] = self._processor._start_tool_span( + block, *args, **kwargs + ) + + def finish(self, content: Any) -> None: + """Send the spans whose results are in a UserMessage's content.""" + if not isinstance(content, list): + return + for block in content: + if not isinstance(block, ToolResultBlock): + continue + span = self._pending.pop(block.tool_use_id, None) + if span is not None: + self._processor._emit_tool_span(span, block) + + def flush(self) -> None: + """Send every span still waiting for a result, without one.""" + pending = list(self._pending.values()) + self._pending.clear() + for span in pending: + try: + self._processor._emit_tool_span(span) + except Exception as e: + log.debug(f"PostHog tool span error (non-fatal): {e}") + + class _GenerationTracker: """Tracks StreamEvent boundaries to reconstruct per-generation metrics. @@ -286,6 +342,7 @@ async def query( options = replace(options, include_partial_messages=True) tracker = _GenerationTracker() + tool_spans = _ToolSpanTracker(self) query_start = time.time() generation_index = 0 current_generation_span_id: Optional[str] = None @@ -315,90 +372,110 @@ async def query( # Accumulate assistant output per generation pending_output: List[Dict[str, Any]] = [] - async for message in original_query( - prompt=prompt, options=options, transport=transport - ): - # All instrumentation is wrapped in try/except so PostHog errors - # never interrupt the underlying Claude Agent SDK query. - try: - if isinstance(message, StreamEvent): - if _starts_generation(message): - if tracker.current_span_id is not None: - # The previous request never finished (it was - # retried), so its partial output is dropped. + try: + async for message in original_query( + prompt=prompt, options=options, transport=transport + ): + # All instrumentation is wrapped in try/except so PostHog errors + # never interrupt the underlying Claude Agent SDK query. + try: + if isinstance(message, StreamEvent): + if _starts_generation(message): + if tracker.current_span_id is not None: + # The previous request never finished (it was + # retried), so its partial output is dropped. + pending_output = [] + current_input, next_input = _promote_input( + current_input, next_input + ) + tracker.process_stream_event(message) + + # Emit $ai_generation when a turn completes + if tracker.has_completed_generation(): + gen = tracker.pop_generation() + generation_index += 1 + current_generation_span_id = gen.span_id + self._emit_generation( + gen, + trace_id, + generation_index, + current_input, + pending_output or None, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + current_input = None pending_output = [] - current_input, next_input = _promote_input( - current_input, next_input - ) - tracker.process_stream_event(message) - - # Emit $ai_generation when a turn completes - if tracker.has_completed_generation(): - gen = tracker.pop_generation() - generation_index += 1 - current_generation_span_id = gen.span_id - self._emit_generation( - gen, - trace_id, - generation_index, - current_input, - pending_output or None, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, + + elif isinstance(message, AssistantMessage): + tracker.set_model(message.model) + # Use the in-progress generation's span_id as parent for tool spans. + # AssistantMessage arrives before message_stop, so current_generation_span_id + # would be stale (from the previous turn). tracker.current_span_id gives us + # the correct in-progress generation. + parent_id = ( + tracker.current_span_id or current_generation_span_id ) - current_input = None - pending_output = [] - - elif isinstance(message, AssistantMessage): - tracker.set_model(message.model) - # Use the in-progress generation's span_id as parent for tool spans. - # AssistantMessage arrives before message_stop, so current_generation_span_id - # would be stale (from the previous turn). tracker.current_span_id gives us - # the correct in-progress generation. - parent_id = tracker.current_span_id or current_generation_span_id - for block in message.content: - if isinstance(block, ToolUseBlock): - self._emit_tool_span( - block, + for block in message.content: + if isinstance(block, ToolUseBlock): + tool_spans.start( + block, + trace_id, + parent_id, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + if not _from_subagent(message): + # A subagent's own API calls are not this query's generations. + pending_output = _append_assistant_output( + pending_output, message.content + ) + + elif isinstance(message, UserMessage): + if not _from_subagent(message): + # UserMessages carry tool results; queue them as input + # for the *next* generation. + if not tracker.had_any_stream_events: + # Without stream events, a tool result ends the + # assistant turn; the result fallback reports the + # final turn only. + pending_output = [] + next_input = _format_user_input( + message.content, self._client, next_input + ) + # Subagent tool spans are tracked too, so match their results. + tool_spans.finish(message.content) + + elif isinstance(message, ResultMessage): + # Spans still waiting for a result go out before the trace. + tool_spans.flush() + # Fallback: if no StreamEvents were received, emit a single + # generation from ResultMessage aggregate data + if not tracker.had_any_stream_events: + self._emit_generation_from_result( + message, trace_id, - parent_id, + tracker.last_model, + query_start, + initial_input, + pending_output, distinct_id_override, extra_props, privacy, groups, extra_options=extra_options, ) - if not _from_subagent(message): - # A subagent's own API calls are not this query's generations. - pending_output = _append_assistant_output( - pending_output, message.content - ) - elif isinstance(message, UserMessage) and not _from_subagent(message): - # UserMessages carry tool results; queue them as input for the - # *next* generation. - if not tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant - # turn; the result fallback reports the final turn only. - pending_output = [] - next_input = _format_user_input( - message.content, self._client, next_input - ) - - elif isinstance(message, ResultMessage): - # Fallback: if no StreamEvents were received, emit a single - # generation from ResultMessage aggregate data - if not tracker.had_any_stream_events: - self._emit_generation_from_result( + self._emit_trace( message, trace_id, - tracker.last_model, query_start, - initial_input, - pending_output, distinct_id_override, extra_props, privacy, @@ -406,21 +483,14 @@ async def query( extra_options=extra_options, ) - self._emit_trace( - message, - trace_id, - query_start, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, - ) + except Exception as e: + log.debug(f"PostHog instrumentation error (non-fatal): {e}") - except Exception as e: - log.debug(f"PostHog instrumentation error (non-fatal): {e}") - - yield message + yield message + finally: + # Send tool spans whose results never arrived (the query ended + # without a result or the caller stopped iterating). + tool_spans.flush() def _emit_generation( self, @@ -573,7 +643,7 @@ def _emit_generation_from_result( options=extra_options, ) - def _emit_tool_span( + def _start_tool_span( self, block: "ToolUseBlock", trace_id: str, @@ -583,9 +653,8 @@ def _emit_tool_span( privacy: bool, groups: Dict[str, Any], extra_options: Optional[Dict[str, Any]] = None, - ) -> None: - resolved_id = self._resolve_distinct_id(distinct_id) - + ) -> "_PendingToolSpan": + """Build the $ai_span for a tool use; it is sent once its result arrives.""" properties: Dict[str, Any] = { "$ai_trace_id": trace_id, "$ai_span_id": str(uuid.uuid4()), @@ -602,13 +671,45 @@ def _emit_tool_span( _ensure_serializable(block.input), self._client ) + return _PendingToolSpan( + properties=properties, + start_time=time.time(), + distinct_id=self._resolve_distinct_id(distinct_id), + trace_id=trace_id, + privacy=privacy, + groups=groups, + options=extra_options, + ) + + def _emit_tool_span( + self, + span: "_PendingToolSpan", + result: Optional["ToolResultBlock"] = None, + ) -> None: + """Send a tool $ai_span, with its result when the tool returned one.""" + properties = span.properties + if result is not None: + properties["$ai_latency"] = time.time() - span.start_time + is_error = bool(result.is_error) + properties["$ai_is_error"] = is_error + if _with_privacy_mode(self._client, span.privacy, True): + properties["$ai_output_state"] = finalize_ai_content( + format_tool_result_content(result, self._client), self._client + ) + if is_error: + properties["$ai_error"] = _format_tool_error( + self._client, + span.privacy, + format_tool_result_content(result, self._client), + ) + self._capture_event( "$ai_span", properties, - resolved_id or trace_id, - groups, - personless=resolved_id is None, - options=extra_options, + span.distinct_id or span.trace_id, + span.groups, + personless=span.distinct_id is None, + options=span.options, ) def _emit_trace( diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0972cf43a..29cc92641 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -50,6 +50,7 @@ _user_distinct_id, finalize_ai_content, _format_ai_error, + _format_tool_error, get_model_params, _privacy_mode_enabled, _redact_mcp_tool_credentials, @@ -72,6 +73,8 @@ class SpanMetadata: """End time of the run.""" input: Optional[Any] """Input of the run: messages, prompt variables, etc.""" + span_type: Optional[str] = None + """`$ai_span_type` of the run's span, e.g. "tool" for tool runs.""" @property def latency(self) -> float: @@ -303,7 +306,7 @@ def on_tool_start( ) self._set_parent_of_run(run_id, parent_run_id) self._set_trace_or_span_metadata( - serialized, input_str, run_id, parent_run_id, **kwargs + serialized, input_str, run_id, parent_run_id, span_type="tool", **kwargs ) def on_tool_end( @@ -461,12 +464,17 @@ def _set_trace_or_span_metadata( input: Any, run_id: UUID, parent_run_id: Optional[UUID] = None, + span_type: Optional[str] = None, **kwargs, ): default_name = "trace" if parent_run_id is None else "span" run_name = _get_langchain_run_name(serialized, **kwargs) or default_name self._runs[run_id] = SpanMetadata( - name=run_name, input=input, start_time=time.time(), end_time=None + name=run_name, + input=input, + start_time=time.time(), + end_time=None, + span_type=span_type, ) def _set_llm_metadata( @@ -575,6 +583,8 @@ def _capture_trace_or_span( } if parent_run_id is not None: event_properties["$ai_parent_id"] = parent_run_id + if run.span_type is not None and event_name == "$ai_span": + event_properties["$ai_span_type"] = run.span_type if self._properties: event_properties.update(self._properties) @@ -593,6 +603,16 @@ def _capture_trace_or_span( self._privacy_mode, finalize_ai_content(outputs, self._ph_client), ) + if ( + isinstance(outputs, ToolMessage) + and getattr(outputs, "status", None) == "error" + ): + # A tool that reports its failure as a result (e.g. with + # handle_tool_error) ends without an exception. + event_properties["$ai_is_error"] = True + event_properties["$ai_error"] = _format_tool_error( + self._ph_client, self._privacy_mode, outputs.content + ) _capture_ai_event( self._ph_client, @@ -860,7 +880,15 @@ def _convert_message_to_dict(message: BaseMessage) -> dict[str, Any]: elif isinstance(message, SystemMessage): message_dict = {"role": "system", "content": message.content} elif isinstance(message, ToolMessage): - message_dict = {"role": "tool", "content": message.content} + message_dict = { + "role": "tool", + "content": message.content, + "tool_call_id": message.tool_call_id, + } + if message.name: + message_dict["name"] = message.name + if getattr(message, "status", None) == "error": + message_dict["status"] = "error" elif isinstance(message, FunctionMessage): message_dict = {"role": "function", "content": message.content} else: diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 62bd937a6..268a1eca5 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1,5 +1,6 @@ import asyncio import copy +import json import logging import re import time @@ -1081,6 +1082,36 @@ def _format_ai_error( return describe(exc) +_PRIVACY_SAFE_TOOL_ERROR = "ToolError" + + +def _format_tool_error( + ph_client: Any, privacy_mode: Optional[bool], content: Any +) -> str: + """ + The `$ai_error` value for a tool result the tool reported as an error. + + The result content can echo prompts and user data, so privacy mode keeps + only a fixed label. + """ + if _privacy_mode_enabled(ph_client, privacy_mode): + return _PRIVACY_SAFE_TOOL_ERROR + if isinstance(content, str): + return content + if isinstance(content, list): + texts = [ + block["text"] + for block in content + if isinstance(block, dict) and isinstance(block.get("text"), str) + ] + if texts: + return "\n".join(texts) + try: + return json.dumps(content, default=str) + except Exception: + return str(content) + + _MCP_TOOL_CREDENTIAL_FIELDS = ("headers", "authorization") _REDACTED_MCP_CREDENTIAL = "[REDACTED]" diff --git a/posthog/test/ai/claude_agent_sdk/test_client.py b/posthog/test/ai/claude_agent_sdk/test_client.py index f902313e1..97cbcf03c 100644 --- a/posthog/test/ai/claude_agent_sdk/test_client.py +++ b/posthog/test/ai/claude_agent_sdk/test_client.py @@ -178,6 +178,11 @@ async def test_multiturn_stream_tool_parentage_input_promotion_and_final_trace(s assert span["$ai_parent_id"] == one["$ai_span_id"] assert span["$ai_input_state"] == {"query": "stock"} assert span["$ai_span_name"] == "lookup" + assert span["$ai_is_error"] is False + assert span["$ai_output_state"][0]["source"]["data"] == ( + "[base64 image redacted]" + ) + assert span["$ai_latency"] >= 0 await client.query("follow up") second = [ @@ -410,7 +415,7 @@ def cli_tool_turn(): ] -async def run_turn(sdk, monkeypatch, use_client, messages): +async def run_turn(sdk, monkeypatch, use_client, messages, event="$ai_generation"): ph = analytics() if use_client: client = PostHogClaudeSDKClient(posthog_client=ph) @@ -428,7 +433,51 @@ async def query(**kwargs): ) processor = PostHogClaudeAgentProcessor(client=ph) assert [msg async for msg in processor.query(prompt="question")] == messages - return [event["properties"] for event in events(ph, "$ai_generation")] + return [captured["properties"] for captured in events(ph, event)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_cli_order_tool_spans_carry_their_own_results( + sdk, monkeypatch, use_client +): + turn = cli_tool_turn() + turn[6] = UserMessage( + content=[ToolResultBlock(tool_use_id="tool-2", content="denied", is_error=True)] + ) + spans = await run_turn(sdk, monkeypatch, use_client, turn, event="$ai_span") + by_input = {span["$ai_input_state"]["path"]: span for span in spans} + assert len(spans) == 2 + assert by_input["a"]["$ai_output_state"] == "a text" + assert by_input["a"]["$ai_is_error"] is False + assert "$ai_error" not in by_input["a"] + assert by_input["b"]["$ai_output_state"] == "denied" + assert by_input["b"]["$ai_is_error"] is True + assert by_input["b"]["$ai_error"] == "denied" + + +@pytest.mark.asyncio +async def test_disconnect_sends_tool_spans_still_waiting_for_a_result(sdk): + ph = analytics() + messages = [ + AssistantMessage( + model="claude", + content=[ToolUseBlock(id="tool-1", name="Read", input={"path": "a"})], + ), + ] + client = PostHogClaudeSDKClient(posthog_client=ph) + await client.query("question") + respond(sdk, messages) + assert [msg async for msg in client.receive_response()] == messages + assert not events(ph, "$ai_span") + + await client.disconnect() + + [span] = events(ph, "$ai_span") + assert span["properties"]["$ai_span_name"] == "Read" + assert "$ai_is_error" not in span["properties"] + captured = [call.kwargs["event"] for call in ph.capture_ai.call_args_list] + assert captured.index("$ai_span") < captured.index("$ai_trace") @pytest.mark.asyncio diff --git a/posthog/test/ai/claude_agent_sdk/test_processor.py b/posthog/test/ai/claude_agent_sdk/test_processor.py index 6aceb6ce8..447fcf257 100644 --- a/posthog/test/ai/claude_agent_sdk/test_processor.py +++ b/posthog/test/ai/claude_agent_sdk/test_processor.py @@ -18,7 +18,9 @@ ResultMessage, StreamEvent, TextBlock, + ToolResultBlock, ToolUseBlock, + UserMessage, ) from posthog.ai.claude_agent_sdk import ( @@ -477,6 +479,140 @@ async def test_emits_span_for_tool_use(self, processor, mock_client): assert props["$ai_span_name"] == "Read" assert props["$ai_span_type"] == "tool" assert props["$ai_input_state"] == {"file_path": "/tmp/test.py"} + # No ToolResultBlock arrived, so the span is sent without a result. + assert "$ai_output_state" not in props + assert "$ai_is_error" not in props + + @staticmethod + def _tool_turn(result_block, parent_tool_use_id=None): + return [ + _make_message_start(), + _make_assistant_message( + tool_uses=[ + {"id": "tu_1", "name": "Bash", "input": {"command": "ls /nope"}} + ] + ), + _make_message_stop(), + UserMessage(content=[result_block], parent_tool_use_id=parent_tool_use_id), + _make_result_message(), + ] + + @staticmethod + async def _run(proc, messages): + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + async for _ in proc.query(prompt="Run it", options=ClaudeAgentOptions()): + pass + + @staticmethod + def _spans(mock_client): + return [ + c.kwargs["properties"] + for c in mock_client.capture_ai.call_args_list + if c.kwargs.get("event") == "$ai_span" + ] + + @pytest.mark.asyncio + async def test_span_carries_tool_result(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="file.txt", is_error=None) + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_span_name"] == "Bash" + assert props["$ai_input_state"] == {"command": "ls /nope"} + assert props["$ai_output_state"] == "file.txt" + assert props["$ai_is_error"] is False + assert "$ai_error" not in props + assert props["$ai_latency"] >= 0 + + @pytest.mark.asyncio + async def test_span_marks_tool_error(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock( + tool_use_id="tu_1", + content=[{"type": "text", "text": "No such file or directory"}], + is_error=True, + ) + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_output_state"] == [ + {"type": "text", "text": "No such file or directory"} + ] + assert props["$ai_error"] == "No such file or directory" + assert props["$ai_latency"] >= 0 + + @pytest.mark.asyncio + async def test_privacy_mode_redacts_tool_result_and_error(self, mock_client): + proc = PostHogClaudeAgentProcessor( + client=mock_client, distinct_id="user", privacy_mode=True + ) + await self._run( + proc, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="secret", is_error=True) + ), + ) + + [props] = self._spans(mock_client) + assert "$ai_input_state" not in props + assert "$ai_output_state" not in props + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "ToolError" + + @pytest.mark.asyncio + async def test_subagent_tool_result_is_matched(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="boom", is_error=True), + parent_tool_use_id="task_1", + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "boom" + + @pytest.mark.asyncio + async def test_span_is_sent_once_and_before_the_trace(self, processor, mock_client): + await self._run( + processor, + self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="ok")), + ) + + events = [c.kwargs["event"] for c in mock_client.capture_ai.call_args_list] + assert events.count("$ai_span") == 1 + assert events.index("$ai_span") < events.index("$ai_trace") + + @pytest.mark.asyncio + async def test_pending_span_is_sent_when_caller_stops_early( + self, processor, mock_client + ): + messages = self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="ok")) + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + stream = processor.query(prompt="Run it", options=ClaudeAgentOptions()) + async for message in stream: + if isinstance(message, AssistantMessage): + break + await stream.aclose() + + [props] = self._spans(mock_client) + assert props["$ai_span_name"] == "Bash" + assert "$ai_is_error" not in props class TestTraceEmission: diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 10df3378a..b859f1243 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -2247,6 +2247,119 @@ def test_convert_message_to_dict_tool_calls(): ] +def test_convert_message_to_dict_tool_message(): + from langchain_core.messages import ToolMessage + + from posthog.ai.langchain.callbacks import _convert_message_to_dict + + assert _convert_message_to_dict( + ToolMessage(content="Sunny", tool_call_id="call_1", name="get_weather") + ) == { + "role": "tool", + "content": "Sunny", + "tool_call_id": "call_1", + "name": "get_weather", + } + assert _convert_message_to_dict( + ToolMessage(content="City not found", tool_call_id="call_2", status="error") + ) == { + "role": "tool", + "content": "City not found", + "tool_call_id": "call_2", + "status": "error", + } + + +def _invoke_tool_in_chain(mock_client, tool_obj, args, **handler_kwargs): + """Run a tool call as a child of a chain, so the tool run is an $ai_span.""" + callbacks = CallbackHandler(mock_client, **handler_kwargs) + call = {"name": tool_obj.name, "args": args, "id": "call_1", "type": "tool_call"} + chain = RunnableLambda(lambda _, config: tool_obj.invoke(call, config=config)) + try: + chain.invoke({}, config={"callbacks": [callbacks]}) + except Exception: + pass + return [ + c[1]["properties"] + for c in mock_client.capture_ai.call_args_list + if c[1]["event"] == "$ai_span" + ] + + +def _failing_weather_tool(handle_tool_error): + from langchain_core.tools import ToolException + + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + raise ToolException(f"City not found: {city}") + + get_weather.handle_tool_error = handle_tool_error + return get_weather + + +def test_tool_span_type_and_result(mock_client): + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + return f"Sunny in {city}" + + [span] = _invoke_tool_in_chain(mock_client, get_weather, {"city": "Paris"}) + assert span["$ai_span_name"] == "get_weather" + assert span["$ai_span_type"] == "tool" + assert "$ai_is_error" not in span + assert span["$ai_output_state"].content == "Sunny in Paris" + assert span["$ai_output_state"].tool_call_id == "call_1" + + +def test_tool_span_marks_error_tool_message(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, _failing_weather_tool(True), {"city": "Atlantis"} + ) + assert span["$ai_span_type"] == "tool" + assert span["$ai_is_error"] is True + assert span["$ai_error"] == "City not found: Atlantis" + + +def test_tool_span_error_tool_message_in_privacy_mode(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, + _failing_weather_tool(True), + {"city": "Atlantis"}, + privacy_mode=True, + ) + assert span["$ai_is_error"] is True + assert span["$ai_error"] == "ToolError" + assert span["$ai_output_state"] is None + + +def test_tool_span_marks_raised_tool_error(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, _failing_weather_tool(False), {"city": "Atlantis"} + ) + assert span["$ai_span_type"] == "tool" + assert span["$ai_is_error"] is True + assert "City not found: Atlantis" in span["$ai_error"] + + +def test_non_tool_spans_have_no_span_type(mock_client): + prompt = ChatPromptTemplate.from_messages([("user", "Foo")]) + chain = ( + RunnableLambda(lambda x: x) + | prompt + | FakeMessagesListChatModel(responses=[AIMessage(content="Bar")]) + ) + chain.invoke({}, config={"callbacks": [CallbackHandler(mock_client)]}) + + spans = [ + c[1]["properties"] + for c in mock_client.capture_ai.call_args_list + if c[1]["event"] == "$ai_span" + ] + assert spans + assert all("$ai_span_type" not in span for span in spans) + + def test_tool_definition(mock_client): """Test that tools defined in invocation parameters are captured in $ai_tools property""" callbacks = CallbackHandler(mock_client) From d4df1ca9e5034456c8c5943c489f3ead28bf7f75 Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Thu, 19 Nov 2026 19:15:02 -0800 Subject: [PATCH 2/4] fix(ai): redact tool error media, close inner query stream, keep metadata positional args - Redact structured tool error content before serializing $ai_error. - Close processor.query() when the public query() stream closes, so pending tool spans are sent. - Make SpanMetadata.span_type keyword-only so GenerationMetadata positional args keep their order. - Regenerate the public API snapshot. Co-Authored-By: Claude Opus 5.5 --- posthog/ai/claude_agent_sdk/__init__.py | 12 +++++++--- posthog/ai/langchain/callbacks.py | 4 ++-- posthog/ai/utils.py | 2 ++ .../ai/claude_agent_sdk/test_processor.py | 5 ++-- posthog/test/ai/langchain/test_callbacks.py | 7 ++++++ posthog/test/ai/test_utils.py | 23 ++++++++++++++++++- references/public_api_snapshot.txt | 5 ++-- 7 files changed, 48 insertions(+), 10 deletions(-) diff --git a/posthog/ai/claude_agent_sdk/__init__.py b/posthog/ai/claude_agent_sdk/__init__.py index 4e492d313..8d03b1135 100644 --- a/posthog/ai/claude_agent_sdk/__init__.py +++ b/posthog/ai/claude_agent_sdk/__init__.py @@ -151,7 +151,7 @@ async def query( yield message return - async for message in processor.query( + stream = processor.query( prompt=prompt, options=options, transport=transport, @@ -160,5 +160,11 @@ async def query( posthog_privacy_mode=posthog_privacy_mode, posthog_groups=posthog_groups, posthog_options=posthog_options, - ): - yield message + ) + try: + async for message in stream: + yield message + finally: + # Closing this stream early must also close the processor's, so it + # sends the tool spans still waiting for a result. + await stream.aclose() diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 29cc92641..86eaac4b1 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -8,7 +8,7 @@ import json import logging import time -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ( Any, Dict, @@ -73,7 +73,7 @@ class SpanMetadata: """End time of the run.""" input: Optional[Any] """Input of the run: messages, prompt variables, etc.""" - span_type: Optional[str] = None + span_type: Optional[str] = field(default=None, kw_only=True) """`$ai_span_type` of the run's span, e.g. "tool" for tool runs.""" @property diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 268a1eca5..0d8e120eb 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1106,6 +1106,8 @@ def _format_tool_error( ] if texts: return "\n".join(texts) + # Structured content can carry media, so redact it before serializing. + content = finalize_ai_content(content, ph_client) try: return json.dumps(content, default=str) except Exception: diff --git a/posthog/test/ai/claude_agent_sdk/test_processor.py b/posthog/test/ai/claude_agent_sdk/test_processor.py index 447fcf257..1d2d9be2b 100644 --- a/posthog/test/ai/claude_agent_sdk/test_processor.py +++ b/posthog/test/ai/claude_agent_sdk/test_processor.py @@ -596,15 +596,16 @@ async def test_span_is_sent_once_and_before_the_trace(self, processor, mock_clie assert events.index("$ai_span") < events.index("$ai_trace") @pytest.mark.asyncio + @pytest.mark.parametrize("entry_point", ["module_query", "processor_query"]) async def test_pending_span_is_sent_when_caller_stops_early( - self, processor, mock_client + self, mock_client, entry_point ): messages = self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="ok")) with patch( "posthog.ai.claude_agent_sdk.processor.original_query", side_effect=lambda **kw: _fake_query(messages), ): - stream = processor.query(prompt="Run it", options=ClaudeAgentOptions()) + stream = _claude_query(entry_point, mock_client, "user", None) async for message in stream: if isinstance(message, AssistantMessage): break diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index b859f1243..3d92fcaa6 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -126,6 +126,13 @@ def test_metadata_capture(mock_client): callbacks._pop_run_metadata(uuid.uuid4()) # should not raise +def test_generation_metadata_keeps_positional_arguments(): + run = GenerationMetadata("test", 1.0, None, None, "openai", "gpt-4o") + assert run.provider == "openai" + assert run.model == "gpt-4o" + assert run.span_type is None + + def test_run_metadata_capture(mock_client): callbacks = CallbackHandler(mock_client) run_id = uuid.uuid4() diff --git a/posthog/test/ai/test_utils.py b/posthog/test/ai/test_utils.py index 49add2881..56796f4a7 100644 --- a/posthog/test/ai/test_utils.py +++ b/posthog/test/ai/test_utils.py @@ -6,7 +6,11 @@ import pytest from posthog.ai.types import TokenUsage -from posthog.ai.utils import merge_usage_stats, serialize_raw_usage +from posthog.ai.utils import ( + _format_tool_error, + merge_usage_stats, + serialize_raw_usage, +) class PydanticUsage: @@ -164,3 +168,20 @@ def test_merge_usage_stats_invalid_mode_does_not_mutate_target(): with pytest.raises(ValueError, match="Invalid mode: unknown"): merge_usage_stats(target, TokenUsage(input_tokens=4), mode="unknown") assert target == {"input_tokens": 10} + + +def test_format_tool_error_redacts_media_blocks(): + client = SimpleNamespace(privacy_mode=False, enable_full_ai_capture=False) + content = [ + { + "type": "image", + "source_type": "base64", + "data": "iVBORw0KGgo" + "A" * 3000, + "mime_type": "image/png", + } + ] + + error = _format_tool_error(client, False, content) + + assert "iVBORw0KGgo" not in error + assert "redacted" in error diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index c9e0b430e..eac0cfab2 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -621,6 +621,7 @@ attribute posthog.ai.langchain.callbacks.SpanMetadata.end_time: Optional[float] attribute posthog.ai.langchain.callbacks.SpanMetadata.input: Optional[Any] attribute posthog.ai.langchain.callbacks.SpanMetadata.latency: float attribute posthog.ai.langchain.callbacks.SpanMetadata.name: str +attribute posthog.ai.langchain.callbacks.SpanMetadata.span_type: Optional[str] = field(default=None, kw_only=True) attribute posthog.ai.langchain.callbacks.SpanMetadata.start_time: float attribute posthog.ai.langchain.callbacks.log = logging.getLogger('posthog') attribute posthog.ai.langchain.middleware.PostHogMiddleware.state_schema = _PostHogMiddlewareState @@ -1331,9 +1332,9 @@ class posthog.ai.gemini.gemini_async.AsyncModels(api_key: Optional[str] = None, class posthog.ai.gemini.gemini_converter.GeminiMessage class posthog.ai.gemini.gemini_converter.GeminiPart class posthog.ai.langchain.callbacks.CallbackHandler(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[Dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, options: Optional[Dict[str, Any]] = None) -class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None) +class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None, *, span_type: Optional[str] = None) class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None, web_search_count: Optional[int] = None) -class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any]) +class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], *, span_type: Optional[str] = None) class posthog.ai.langchain.middleware.PostHogMiddleware(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[dict[str, Any]] = None, options: Optional[dict[str, Any]] = None) class posthog.ai.openai.openai.OpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) class posthog.ai.openai.openai.WrappedBeta From 8f4da4b4f0808c35a539cc904aeef3b064b78a40 Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Thu, 19 Nov 2026 19:18:57 -0800 Subject: [PATCH 3/4] fix(ai): redact string tool error content too Run every tool error value through finalize_ai_content, not just structured content, so a data URL result is redacted like $ai_output_state. Co-Authored-By: Claude Opus 5.5 --- posthog/ai/utils.py | 5 +++-- posthog/test/ai/test_utils.py | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 0d8e120eb..680affe56 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1096,6 +1096,9 @@ def _format_tool_error( """ if _privacy_mode_enabled(ph_client, privacy_mode): return _PRIVACY_SAFE_TOOL_ERROR + # The content can carry media (data URLs, base64 blocks), so redact it + # the same way as `$ai_output_state` before building the message. + content = finalize_ai_content(content, ph_client) if isinstance(content, str): return content if isinstance(content, list): @@ -1106,8 +1109,6 @@ def _format_tool_error( ] if texts: return "\n".join(texts) - # Structured content can carry media, so redact it before serializing. - content = finalize_ai_content(content, ph_client) try: return json.dumps(content, default=str) except Exception: diff --git a/posthog/test/ai/test_utils.py b/posthog/test/ai/test_utils.py index 56796f4a7..647eb2a31 100644 --- a/posthog/test/ai/test_utils.py +++ b/posthog/test/ai/test_utils.py @@ -185,3 +185,11 @@ def test_format_tool_error_redacts_media_blocks(): assert "iVBORw0KGgo" not in error assert "redacted" in error + + +def test_format_tool_error_redacts_data_url_string(): + client = SimpleNamespace(privacy_mode=False, enable_full_ai_capture=False) + + error = _format_tool_error(client, False, "data:image/png;base64," + "A" * 3000) + + assert "AAAA" not in error From 2846827ae62b0d9f664266a9d23f78afb1612f83 Mon Sep 17 00:00:00 2001 From: Daniel Visca Date: Thu, 19 Nov 2026 19:44:33 -0800 Subject: [PATCH 4/4] refactor(ai): track LangChain tool runs privately instead of a SpanMetadata field Keeps the public SpanMetadata/GenerationMetadata API unchanged; the public API snapshot is back to main's. Co-Authored-By: Claude Opus 5.5 --- posthog/ai/langchain/callbacks.py | 19 +++++++++++-------- posthog/test/ai/langchain/test_callbacks.py | 1 - references/public_api_snapshot.txt | 5 ++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 86eaac4b1..628f22dd5 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -8,13 +8,14 @@ import json import logging import time -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import ( Any, Dict, List, Optional, Sequence, + Set, Union, cast, ) @@ -73,8 +74,6 @@ class SpanMetadata: """End time of the run.""" input: Optional[Any] """Input of the run: messages, prompt variables, etc.""" - span_type: Optional[str] = field(default=None, kw_only=True) - """`$ai_span_type` of the run's span, e.g. "tool" for tool runs.""" @property def latency(self) -> float: @@ -173,6 +172,8 @@ def __init__( self._groups = groups or {} self._options = _ai_event_options(options) self._runs = {} + # Runs started by on_tool_start; their spans get `$ai_span_type: tool`. + self._tool_run_ids: Set[UUID] = set() self._parent_tree = {} def on_chain_start( @@ -306,8 +307,9 @@ def on_tool_start( ) self._set_parent_of_run(run_id, parent_run_id) self._set_trace_or_span_metadata( - serialized, input_str, run_id, parent_run_id, span_type="tool", **kwargs + serialized, input_str, run_id, parent_run_id, **kwargs ) + self._tool_run_ids.add(run_id) def on_tool_end( self, @@ -464,7 +466,6 @@ def _set_trace_or_span_metadata( input: Any, run_id: UUID, parent_run_id: Optional[UUID] = None, - span_type: Optional[str] = None, **kwargs, ): default_name = "trace" if parent_run_id is None else "span" @@ -474,7 +475,6 @@ def _set_trace_or_span_metadata( input=input, start_time=time.time(), end_time=None, - span_type=span_type, ) def _set_llm_metadata( @@ -543,6 +543,7 @@ def _pop_run_and_capture_trace_or_span( self._pop_parent_of_run(run_id) run = self._pop_run_metadata(run_id) if not run: + self._tool_run_ids.discard(run_id) return if isinstance(run, GenerationMetadata): log.warning( @@ -583,8 +584,10 @@ def _capture_trace_or_span( } if parent_run_id is not None: event_properties["$ai_parent_id"] = parent_run_id - if run.span_type is not None and event_name == "$ai_span": - event_properties["$ai_span_type"] = run.span_type + if run_id in self._tool_run_ids: + self._tool_run_ids.discard(run_id) + if event_name == "$ai_span": + event_properties["$ai_span_type"] = "tool" if self._properties: event_properties.update(self._properties) diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 3d92fcaa6..b318c6f96 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -130,7 +130,6 @@ def test_generation_metadata_keeps_positional_arguments(): run = GenerationMetadata("test", 1.0, None, None, "openai", "gpt-4o") assert run.provider == "openai" assert run.model == "gpt-4o" - assert run.span_type is None def test_run_metadata_capture(mock_client): diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index eac0cfab2..c9e0b430e 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -621,7 +621,6 @@ attribute posthog.ai.langchain.callbacks.SpanMetadata.end_time: Optional[float] attribute posthog.ai.langchain.callbacks.SpanMetadata.input: Optional[Any] attribute posthog.ai.langchain.callbacks.SpanMetadata.latency: float attribute posthog.ai.langchain.callbacks.SpanMetadata.name: str -attribute posthog.ai.langchain.callbacks.SpanMetadata.span_type: Optional[str] = field(default=None, kw_only=True) attribute posthog.ai.langchain.callbacks.SpanMetadata.start_time: float attribute posthog.ai.langchain.callbacks.log = logging.getLogger('posthog') attribute posthog.ai.langchain.middleware.PostHogMiddleware.state_schema = _PostHogMiddlewareState @@ -1332,9 +1331,9 @@ class posthog.ai.gemini.gemini_async.AsyncModels(api_key: Optional[str] = None, class posthog.ai.gemini.gemini_converter.GeminiMessage class posthog.ai.gemini.gemini_converter.GeminiPart class posthog.ai.langchain.callbacks.CallbackHandler(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[Dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, options: Optional[Dict[str, Any]] = None) -class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None, *, span_type: Optional[str] = None) +class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None) class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None, web_search_count: Optional[int] = None) -class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], *, span_type: Optional[str] = None) +class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any]) class posthog.ai.langchain.middleware.PostHogMiddleware(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[dict[str, Any]] = None, options: Optional[dict[str, Any]] = None) class posthog.ai.openai.openai.OpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) class posthog.ai.openai.openai.WrappedBeta