diff --git a/.sampo/changesets/ai-tool-span-results.md b/.sampo/changesets/ai-tool-span-results.md new file mode 100644 index 000000000..5c11869f4 --- /dev/null +++ b/.sampo/changesets/ai-tool-span-results.md @@ -0,0 +1,5 @@ +--- +posthog: patch +--- + +Send tool results on AI tool spans. Claude Agent SDK tool spans now wait for the tool's result and carry `$ai_output_state`, `$ai_is_error`, `$ai_error` and `$ai_latency`. LangChain tool spans set `$ai_span_type: "tool"` and mark tools that return an error `ToolMessage`, and converted `ToolMessage`s keep `tool_call_id`, `name` and an error `status`. diff --git a/posthog/ai/claude_agent_sdk/__init__.py b/posthog/ai/claude_agent_sdk/__init__.py index 4e492d313..8d03b1135 100644 --- a/posthog/ai/claude_agent_sdk/__init__.py +++ b/posthog/ai/claude_agent_sdk/__init__.py @@ -151,7 +151,7 @@ async def query( yield message return - async for message in processor.query( + stream = processor.query( prompt=prompt, options=options, transport=transport, @@ -160,5 +160,11 @@ async def query( posthog_privacy_mode=posthog_privacy_mode, posthog_groups=posthog_groups, posthog_options=posthog_options, - ): - yield message + ) + try: + async for message in stream: + yield message + finally: + # Closing this stream early must also close the processor's, so it + # sends the tool spans still waiting for a result. + await stream.aclose() diff --git a/posthog/ai/claude_agent_sdk/client.py b/posthog/ai/claude_agent_sdk/client.py index f8fb124e6..8a591efb4 100644 --- a/posthog/ai/claude_agent_sdk/client.py +++ b/posthog/ai/claude_agent_sdk/client.py @@ -32,6 +32,7 @@ from posthog.ai.claude_agent_sdk.processor import ( PostHogClaudeAgentProcessor, _GenerationTracker, + _ToolSpanTracker, _from_subagent, _promote_input, _starts_generation, @@ -114,6 +115,7 @@ def __init__( # Shared state across turns self._tracker = _GenerationTracker() + self._tool_spans = _ToolSpanTracker(self._processor) self._generation_index = 0 self._current_generation_span_id: Optional[str] = None self._current_input: Optional[List[Dict[str, Any]]] = None @@ -186,16 +188,21 @@ def _process_message(self, message: Any) -> None: elif isinstance(message, AssistantMessage): self._process_assistant_message(message) - elif isinstance(message, UserMessage) and not _from_subagent(message): - if not self._tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant turn; - # the result fallback reports the final turn only. - self._pending_output = [] - self._next_input = _format_user_input( - message.content, self._processor._client, self._next_input - ) + elif isinstance(message, UserMessage): + if not _from_subagent(message): + if not self._tracker.had_any_stream_events: + # Without stream events, a tool result ends the assistant + # turn; the result fallback reports the final turn only. + self._pending_output = [] + self._next_input = _format_user_input( + message.content, self._processor._client, self._next_input + ) + # Subagent tool spans are tracked too, so match their results. + self._tool_spans.finish(message.content) elif isinstance(message, ResultMessage): + # The turn is over: spans still waiting for a result go out now. + self._tool_spans.flush() if not self._tracker.had_any_stream_events: self._processor._emit_generation_from_result( message, @@ -217,7 +224,7 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None: parent_id = self._tracker.current_span_id or self._current_generation_span_id for block in message.content: if isinstance(block, ToolUseBlock): - self._processor._emit_tool_span( + self._tool_spans.start( block, self._trace_id, parent_id, @@ -237,6 +244,7 @@ async def disconnect(self) -> None: """ # Emit the trace event covering the entire session try: + self._tool_spans.flush() latency = time.time() - self._query_start resolved_id = self._processor._resolve_distinct_id(self._distinct_id) diff --git a/posthog/ai/claude_agent_sdk/processor.py b/posthog/ai/claude_agent_sdk/processor.py index ec9b46c3d..4b64d4d42 100644 --- a/posthog/ai/claude_agent_sdk/processor.py +++ b/posthog/ai/claude_agent_sdk/processor.py @@ -14,6 +14,7 @@ from claude_agent_sdk import ( AssistantMessage, ResultMessage, + ToolResultBlock, ToolUseBlock, UserMessage, ) @@ -34,6 +35,7 @@ from posthog.ai.utils import ( _ai_event_options, _capture_processor_event, + _format_tool_error, _resolve_ai_client, finalize_ai_content, with_privacy_mode as _with_privacy_mode, @@ -62,6 +64,60 @@ class _GenerationData: stop_reason: Optional[str] = None +@dataclass +class _PendingToolSpan: + """A tool $ai_span waiting for the ToolResultBlock of its tool use.""" + + properties: Dict[str, Any] + start_time: float + distinct_id: Optional[str] + trace_id: str + privacy: bool + groups: Dict[str, Any] + options: Optional[Dict[str, Any]] + + +class _ToolSpanTracker: + """Holds each tool span until its result arrives, keyed by tool_use_id. + + The span is sent with the result's output, error flag and latency. Spans + still waiting when the query or session ends are sent without a result. + """ + + def __init__(self, processor: "PostHogClaudeAgentProcessor") -> None: + self._processor = processor + self._pending: Dict[str, _PendingToolSpan] = {} + + def start(self, block: "ToolUseBlock", *args: Any, **kwargs: Any) -> None: + previous = self._pending.pop(block.id, None) + if previous is not None: + self._processor._emit_tool_span(previous) + self._pending[block.id] = self._processor._start_tool_span( + block, *args, **kwargs + ) + + def finish(self, content: Any) -> None: + """Send the spans whose results are in a UserMessage's content.""" + if not isinstance(content, list): + return + for block in content: + if not isinstance(block, ToolResultBlock): + continue + span = self._pending.pop(block.tool_use_id, None) + if span is not None: + self._processor._emit_tool_span(span, block) + + def flush(self) -> None: + """Send every span still waiting for a result, without one.""" + pending = list(self._pending.values()) + self._pending.clear() + for span in pending: + try: + self._processor._emit_tool_span(span) + except Exception as e: + log.debug(f"PostHog tool span error (non-fatal): {e}") + + class _GenerationTracker: """Tracks StreamEvent boundaries to reconstruct per-generation metrics. @@ -286,6 +342,7 @@ async def query( options = replace(options, include_partial_messages=True) tracker = _GenerationTracker() + tool_spans = _ToolSpanTracker(self) query_start = time.time() generation_index = 0 current_generation_span_id: Optional[str] = None @@ -315,90 +372,110 @@ async def query( # Accumulate assistant output per generation pending_output: List[Dict[str, Any]] = [] - async for message in original_query( - prompt=prompt, options=options, transport=transport - ): - # All instrumentation is wrapped in try/except so PostHog errors - # never interrupt the underlying Claude Agent SDK query. - try: - if isinstance(message, StreamEvent): - if _starts_generation(message): - if tracker.current_span_id is not None: - # The previous request never finished (it was - # retried), so its partial output is dropped. + try: + async for message in original_query( + prompt=prompt, options=options, transport=transport + ): + # All instrumentation is wrapped in try/except so PostHog errors + # never interrupt the underlying Claude Agent SDK query. + try: + if isinstance(message, StreamEvent): + if _starts_generation(message): + if tracker.current_span_id is not None: + # The previous request never finished (it was + # retried), so its partial output is dropped. + pending_output = [] + current_input, next_input = _promote_input( + current_input, next_input + ) + tracker.process_stream_event(message) + + # Emit $ai_generation when a turn completes + if tracker.has_completed_generation(): + gen = tracker.pop_generation() + generation_index += 1 + current_generation_span_id = gen.span_id + self._emit_generation( + gen, + trace_id, + generation_index, + current_input, + pending_output or None, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + current_input = None pending_output = [] - current_input, next_input = _promote_input( - current_input, next_input - ) - tracker.process_stream_event(message) - - # Emit $ai_generation when a turn completes - if tracker.has_completed_generation(): - gen = tracker.pop_generation() - generation_index += 1 - current_generation_span_id = gen.span_id - self._emit_generation( - gen, - trace_id, - generation_index, - current_input, - pending_output or None, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, + + elif isinstance(message, AssistantMessage): + tracker.set_model(message.model) + # Use the in-progress generation's span_id as parent for tool spans. + # AssistantMessage arrives before message_stop, so current_generation_span_id + # would be stale (from the previous turn). tracker.current_span_id gives us + # the correct in-progress generation. + parent_id = ( + tracker.current_span_id or current_generation_span_id ) - current_input = None - pending_output = [] - - elif isinstance(message, AssistantMessage): - tracker.set_model(message.model) - # Use the in-progress generation's span_id as parent for tool spans. - # AssistantMessage arrives before message_stop, so current_generation_span_id - # would be stale (from the previous turn). tracker.current_span_id gives us - # the correct in-progress generation. - parent_id = tracker.current_span_id or current_generation_span_id - for block in message.content: - if isinstance(block, ToolUseBlock): - self._emit_tool_span( - block, + for block in message.content: + if isinstance(block, ToolUseBlock): + tool_spans.start( + block, + trace_id, + parent_id, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + if not _from_subagent(message): + # A subagent's own API calls are not this query's generations. + pending_output = _append_assistant_output( + pending_output, message.content + ) + + elif isinstance(message, UserMessage): + if not _from_subagent(message): + # UserMessages carry tool results; queue them as input + # for the *next* generation. + if not tracker.had_any_stream_events: + # Without stream events, a tool result ends the + # assistant turn; the result fallback reports the + # final turn only. + pending_output = [] + next_input = _format_user_input( + message.content, self._client, next_input + ) + # Subagent tool spans are tracked too, so match their results. + tool_spans.finish(message.content) + + elif isinstance(message, ResultMessage): + # Spans still waiting for a result go out before the trace. + tool_spans.flush() + # Fallback: if no StreamEvents were received, emit a single + # generation from ResultMessage aggregate data + if not tracker.had_any_stream_events: + self._emit_generation_from_result( + message, trace_id, - parent_id, + tracker.last_model, + query_start, + initial_input, + pending_output, distinct_id_override, extra_props, privacy, groups, extra_options=extra_options, ) - if not _from_subagent(message): - # A subagent's own API calls are not this query's generations. - pending_output = _append_assistant_output( - pending_output, message.content - ) - elif isinstance(message, UserMessage) and not _from_subagent(message): - # UserMessages carry tool results; queue them as input for the - # *next* generation. - if not tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant - # turn; the result fallback reports the final turn only. - pending_output = [] - next_input = _format_user_input( - message.content, self._client, next_input - ) - - elif isinstance(message, ResultMessage): - # Fallback: if no StreamEvents were received, emit a single - # generation from ResultMessage aggregate data - if not tracker.had_any_stream_events: - self._emit_generation_from_result( + self._emit_trace( message, trace_id, - tracker.last_model, query_start, - initial_input, - pending_output, distinct_id_override, extra_props, privacy, @@ -406,21 +483,14 @@ async def query( extra_options=extra_options, ) - self._emit_trace( - message, - trace_id, - query_start, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, - ) + except Exception as e: + log.debug(f"PostHog instrumentation error (non-fatal): {e}") - except Exception as e: - log.debug(f"PostHog instrumentation error (non-fatal): {e}") - - yield message + yield message + finally: + # Send tool spans whose results never arrived (the query ended + # without a result or the caller stopped iterating). + tool_spans.flush() def _emit_generation( self, @@ -573,7 +643,7 @@ def _emit_generation_from_result( options=extra_options, ) - def _emit_tool_span( + def _start_tool_span( self, block: "ToolUseBlock", trace_id: str, @@ -583,9 +653,8 @@ def _emit_tool_span( privacy: bool, groups: Dict[str, Any], extra_options: Optional[Dict[str, Any]] = None, - ) -> None: - resolved_id = self._resolve_distinct_id(distinct_id) - + ) -> "_PendingToolSpan": + """Build the $ai_span for a tool use; it is sent once its result arrives.""" properties: Dict[str, Any] = { "$ai_trace_id": trace_id, "$ai_span_id": str(uuid.uuid4()), @@ -602,13 +671,45 @@ def _emit_tool_span( _ensure_serializable(block.input), self._client ) + return _PendingToolSpan( + properties=properties, + start_time=time.time(), + distinct_id=self._resolve_distinct_id(distinct_id), + trace_id=trace_id, + privacy=privacy, + groups=groups, + options=extra_options, + ) + + def _emit_tool_span( + self, + span: "_PendingToolSpan", + result: Optional["ToolResultBlock"] = None, + ) -> None: + """Send a tool $ai_span, with its result when the tool returned one.""" + properties = span.properties + if result is not None: + properties["$ai_latency"] = time.time() - span.start_time + is_error = bool(result.is_error) + properties["$ai_is_error"] = is_error + if _with_privacy_mode(self._client, span.privacy, True): + properties["$ai_output_state"] = finalize_ai_content( + format_tool_result_content(result, self._client), self._client + ) + if is_error: + properties["$ai_error"] = _format_tool_error( + self._client, + span.privacy, + format_tool_result_content(result, self._client), + ) + self._capture_event( "$ai_span", properties, - resolved_id or trace_id, - groups, - personless=resolved_id is None, - options=extra_options, + span.distinct_id or span.trace_id, + span.groups, + personless=span.distinct_id is None, + options=span.options, ) def _emit_trace( diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0972cf43a..628f22dd5 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -15,6 +15,7 @@ List, Optional, Sequence, + Set, Union, cast, ) @@ -50,6 +51,7 @@ _user_distinct_id, finalize_ai_content, _format_ai_error, + _format_tool_error, get_model_params, _privacy_mode_enabled, _redact_mcp_tool_credentials, @@ -170,6 +172,8 @@ def __init__( self._groups = groups or {} self._options = _ai_event_options(options) self._runs = {} + # Runs started by on_tool_start; their spans get `$ai_span_type: tool`. + self._tool_run_ids: Set[UUID] = set() self._parent_tree = {} def on_chain_start( @@ -305,6 +309,7 @@ def on_tool_start( self._set_trace_or_span_metadata( serialized, input_str, run_id, parent_run_id, **kwargs ) + self._tool_run_ids.add(run_id) def on_tool_end( self, @@ -466,7 +471,10 @@ def _set_trace_or_span_metadata( default_name = "trace" if parent_run_id is None else "span" run_name = _get_langchain_run_name(serialized, **kwargs) or default_name self._runs[run_id] = SpanMetadata( - name=run_name, input=input, start_time=time.time(), end_time=None + name=run_name, + input=input, + start_time=time.time(), + end_time=None, ) def _set_llm_metadata( @@ -535,6 +543,7 @@ def _pop_run_and_capture_trace_or_span( self._pop_parent_of_run(run_id) run = self._pop_run_metadata(run_id) if not run: + self._tool_run_ids.discard(run_id) return if isinstance(run, GenerationMetadata): log.warning( @@ -575,6 +584,10 @@ def _capture_trace_or_span( } if parent_run_id is not None: event_properties["$ai_parent_id"] = parent_run_id + if run_id in self._tool_run_ids: + self._tool_run_ids.discard(run_id) + if event_name == "$ai_span": + event_properties["$ai_span_type"] = "tool" if self._properties: event_properties.update(self._properties) @@ -593,6 +606,16 @@ def _capture_trace_or_span( self._privacy_mode, finalize_ai_content(outputs, self._ph_client), ) + if ( + isinstance(outputs, ToolMessage) + and getattr(outputs, "status", None) == "error" + ): + # A tool that reports its failure as a result (e.g. with + # handle_tool_error) ends without an exception. + event_properties["$ai_is_error"] = True + event_properties["$ai_error"] = _format_tool_error( + self._ph_client, self._privacy_mode, outputs.content + ) _capture_ai_event( self._ph_client, @@ -860,7 +883,15 @@ def _convert_message_to_dict(message: BaseMessage) -> dict[str, Any]: elif isinstance(message, SystemMessage): message_dict = {"role": "system", "content": message.content} elif isinstance(message, ToolMessage): - message_dict = {"role": "tool", "content": message.content} + message_dict = { + "role": "tool", + "content": message.content, + "tool_call_id": message.tool_call_id, + } + if message.name: + message_dict["name"] = message.name + if getattr(message, "status", None) == "error": + message_dict["status"] = "error" elif isinstance(message, FunctionMessage): message_dict = {"role": "function", "content": message.content} else: diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index 62bd937a6..680affe56 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -1,5 +1,6 @@ import asyncio import copy +import json import logging import re import time @@ -1081,6 +1082,39 @@ def _format_ai_error( return describe(exc) +_PRIVACY_SAFE_TOOL_ERROR = "ToolError" + + +def _format_tool_error( + ph_client: Any, privacy_mode: Optional[bool], content: Any +) -> str: + """ + The `$ai_error` value for a tool result the tool reported as an error. + + The result content can echo prompts and user data, so privacy mode keeps + only a fixed label. + """ + if _privacy_mode_enabled(ph_client, privacy_mode): + return _PRIVACY_SAFE_TOOL_ERROR + # The content can carry media (data URLs, base64 blocks), so redact it + # the same way as `$ai_output_state` before building the message. + content = finalize_ai_content(content, ph_client) + if isinstance(content, str): + return content + if isinstance(content, list): + texts = [ + block["text"] + for block in content + if isinstance(block, dict) and isinstance(block.get("text"), str) + ] + if texts: + return "\n".join(texts) + try: + return json.dumps(content, default=str) + except Exception: + return str(content) + + _MCP_TOOL_CREDENTIAL_FIELDS = ("headers", "authorization") _REDACTED_MCP_CREDENTIAL = "[REDACTED]" diff --git a/posthog/test/ai/claude_agent_sdk/test_client.py b/posthog/test/ai/claude_agent_sdk/test_client.py index f902313e1..97cbcf03c 100644 --- a/posthog/test/ai/claude_agent_sdk/test_client.py +++ b/posthog/test/ai/claude_agent_sdk/test_client.py @@ -178,6 +178,11 @@ async def test_multiturn_stream_tool_parentage_input_promotion_and_final_trace(s assert span["$ai_parent_id"] == one["$ai_span_id"] assert span["$ai_input_state"] == {"query": "stock"} assert span["$ai_span_name"] == "lookup" + assert span["$ai_is_error"] is False + assert span["$ai_output_state"][0]["source"]["data"] == ( + "[base64 image redacted]" + ) + assert span["$ai_latency"] >= 0 await client.query("follow up") second = [ @@ -410,7 +415,7 @@ def cli_tool_turn(): ] -async def run_turn(sdk, monkeypatch, use_client, messages): +async def run_turn(sdk, monkeypatch, use_client, messages, event="$ai_generation"): ph = analytics() if use_client: client = PostHogClaudeSDKClient(posthog_client=ph) @@ -428,7 +433,51 @@ async def query(**kwargs): ) processor = PostHogClaudeAgentProcessor(client=ph) assert [msg async for msg in processor.query(prompt="question")] == messages - return [event["properties"] for event in events(ph, "$ai_generation")] + return [captured["properties"] for captured in events(ph, event)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_cli_order_tool_spans_carry_their_own_results( + sdk, monkeypatch, use_client +): + turn = cli_tool_turn() + turn[6] = UserMessage( + content=[ToolResultBlock(tool_use_id="tool-2", content="denied", is_error=True)] + ) + spans = await run_turn(sdk, monkeypatch, use_client, turn, event="$ai_span") + by_input = {span["$ai_input_state"]["path"]: span for span in spans} + assert len(spans) == 2 + assert by_input["a"]["$ai_output_state"] == "a text" + assert by_input["a"]["$ai_is_error"] is False + assert "$ai_error" not in by_input["a"] + assert by_input["b"]["$ai_output_state"] == "denied" + assert by_input["b"]["$ai_is_error"] is True + assert by_input["b"]["$ai_error"] == "denied" + + +@pytest.mark.asyncio +async def test_disconnect_sends_tool_spans_still_waiting_for_a_result(sdk): + ph = analytics() + messages = [ + AssistantMessage( + model="claude", + content=[ToolUseBlock(id="tool-1", name="Read", input={"path": "a"})], + ), + ] + client = PostHogClaudeSDKClient(posthog_client=ph) + await client.query("question") + respond(sdk, messages) + assert [msg async for msg in client.receive_response()] == messages + assert not events(ph, "$ai_span") + + await client.disconnect() + + [span] = events(ph, "$ai_span") + assert span["properties"]["$ai_span_name"] == "Read" + assert "$ai_is_error" not in span["properties"] + captured = [call.kwargs["event"] for call in ph.capture_ai.call_args_list] + assert captured.index("$ai_span") < captured.index("$ai_trace") @pytest.mark.asyncio diff --git a/posthog/test/ai/claude_agent_sdk/test_processor.py b/posthog/test/ai/claude_agent_sdk/test_processor.py index 6aceb6ce8..1d2d9be2b 100644 --- a/posthog/test/ai/claude_agent_sdk/test_processor.py +++ b/posthog/test/ai/claude_agent_sdk/test_processor.py @@ -18,7 +18,9 @@ ResultMessage, StreamEvent, TextBlock, + ToolResultBlock, ToolUseBlock, + UserMessage, ) from posthog.ai.claude_agent_sdk import ( @@ -477,6 +479,141 @@ async def test_emits_span_for_tool_use(self, processor, mock_client): assert props["$ai_span_name"] == "Read" assert props["$ai_span_type"] == "tool" assert props["$ai_input_state"] == {"file_path": "/tmp/test.py"} + # No ToolResultBlock arrived, so the span is sent without a result. + assert "$ai_output_state" not in props + assert "$ai_is_error" not in props + + @staticmethod + def _tool_turn(result_block, parent_tool_use_id=None): + return [ + _make_message_start(), + _make_assistant_message( + tool_uses=[ + {"id": "tu_1", "name": "Bash", "input": {"command": "ls /nope"}} + ] + ), + _make_message_stop(), + UserMessage(content=[result_block], parent_tool_use_id=parent_tool_use_id), + _make_result_message(), + ] + + @staticmethod + async def _run(proc, messages): + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + async for _ in proc.query(prompt="Run it", options=ClaudeAgentOptions()): + pass + + @staticmethod + def _spans(mock_client): + return [ + c.kwargs["properties"] + for c in mock_client.capture_ai.call_args_list + if c.kwargs.get("event") == "$ai_span" + ] + + @pytest.mark.asyncio + async def test_span_carries_tool_result(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="file.txt", is_error=None) + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_span_name"] == "Bash" + assert props["$ai_input_state"] == {"command": "ls /nope"} + assert props["$ai_output_state"] == "file.txt" + assert props["$ai_is_error"] is False + assert "$ai_error" not in props + assert props["$ai_latency"] >= 0 + + @pytest.mark.asyncio + async def test_span_marks_tool_error(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock( + tool_use_id="tu_1", + content=[{"type": "text", "text": "No such file or directory"}], + is_error=True, + ) + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_output_state"] == [ + {"type": "text", "text": "No such file or directory"} + ] + assert props["$ai_error"] == "No such file or directory" + assert props["$ai_latency"] >= 0 + + @pytest.mark.asyncio + async def test_privacy_mode_redacts_tool_result_and_error(self, mock_client): + proc = PostHogClaudeAgentProcessor( + client=mock_client, distinct_id="user", privacy_mode=True + ) + await self._run( + proc, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="secret", is_error=True) + ), + ) + + [props] = self._spans(mock_client) + assert "$ai_input_state" not in props + assert "$ai_output_state" not in props + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "ToolError" + + @pytest.mark.asyncio + async def test_subagent_tool_result_is_matched(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="boom", is_error=True), + parent_tool_use_id="task_1", + ), + ) + + [props] = self._spans(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "boom" + + @pytest.mark.asyncio + async def test_span_is_sent_once_and_before_the_trace(self, processor, mock_client): + await self._run( + processor, + self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="ok")), + ) + + events = [c.kwargs["event"] for c in mock_client.capture_ai.call_args_list] + assert events.count("$ai_span") == 1 + assert events.index("$ai_span") < events.index("$ai_trace") + + @pytest.mark.asyncio + @pytest.mark.parametrize("entry_point", ["module_query", "processor_query"]) + async def test_pending_span_is_sent_when_caller_stops_early( + self, mock_client, entry_point + ): + messages = self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="ok")) + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + stream = _claude_query(entry_point, mock_client, "user", None) + async for message in stream: + if isinstance(message, AssistantMessage): + break + await stream.aclose() + + [props] = self._spans(mock_client) + assert props["$ai_span_name"] == "Bash" + assert "$ai_is_error" not in props class TestTraceEmission: diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 10df3378a..b318c6f96 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -126,6 +126,12 @@ def test_metadata_capture(mock_client): callbacks._pop_run_metadata(uuid.uuid4()) # should not raise +def test_generation_metadata_keeps_positional_arguments(): + run = GenerationMetadata("test", 1.0, None, None, "openai", "gpt-4o") + assert run.provider == "openai" + assert run.model == "gpt-4o" + + def test_run_metadata_capture(mock_client): callbacks = CallbackHandler(mock_client) run_id = uuid.uuid4() @@ -2247,6 +2253,119 @@ def test_convert_message_to_dict_tool_calls(): ] +def test_convert_message_to_dict_tool_message(): + from langchain_core.messages import ToolMessage + + from posthog.ai.langchain.callbacks import _convert_message_to_dict + + assert _convert_message_to_dict( + ToolMessage(content="Sunny", tool_call_id="call_1", name="get_weather") + ) == { + "role": "tool", + "content": "Sunny", + "tool_call_id": "call_1", + "name": "get_weather", + } + assert _convert_message_to_dict( + ToolMessage(content="City not found", tool_call_id="call_2", status="error") + ) == { + "role": "tool", + "content": "City not found", + "tool_call_id": "call_2", + "status": "error", + } + + +def _invoke_tool_in_chain(mock_client, tool_obj, args, **handler_kwargs): + """Run a tool call as a child of a chain, so the tool run is an $ai_span.""" + callbacks = CallbackHandler(mock_client, **handler_kwargs) + call = {"name": tool_obj.name, "args": args, "id": "call_1", "type": "tool_call"} + chain = RunnableLambda(lambda _, config: tool_obj.invoke(call, config=config)) + try: + chain.invoke({}, config={"callbacks": [callbacks]}) + except Exception: + pass + return [ + c[1]["properties"] + for c in mock_client.capture_ai.call_args_list + if c[1]["event"] == "$ai_span" + ] + + +def _failing_weather_tool(handle_tool_error): + from langchain_core.tools import ToolException + + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + raise ToolException(f"City not found: {city}") + + get_weather.handle_tool_error = handle_tool_error + return get_weather + + +def test_tool_span_type_and_result(mock_client): + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + return f"Sunny in {city}" + + [span] = _invoke_tool_in_chain(mock_client, get_weather, {"city": "Paris"}) + assert span["$ai_span_name"] == "get_weather" + assert span["$ai_span_type"] == "tool" + assert "$ai_is_error" not in span + assert span["$ai_output_state"].content == "Sunny in Paris" + assert span["$ai_output_state"].tool_call_id == "call_1" + + +def test_tool_span_marks_error_tool_message(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, _failing_weather_tool(True), {"city": "Atlantis"} + ) + assert span["$ai_span_type"] == "tool" + assert span["$ai_is_error"] is True + assert span["$ai_error"] == "City not found: Atlantis" + + +def test_tool_span_error_tool_message_in_privacy_mode(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, + _failing_weather_tool(True), + {"city": "Atlantis"}, + privacy_mode=True, + ) + assert span["$ai_is_error"] is True + assert span["$ai_error"] == "ToolError" + assert span["$ai_output_state"] is None + + +def test_tool_span_marks_raised_tool_error(mock_client): + [span] = _invoke_tool_in_chain( + mock_client, _failing_weather_tool(False), {"city": "Atlantis"} + ) + assert span["$ai_span_type"] == "tool" + assert span["$ai_is_error"] is True + assert "City not found: Atlantis" in span["$ai_error"] + + +def test_non_tool_spans_have_no_span_type(mock_client): + prompt = ChatPromptTemplate.from_messages([("user", "Foo")]) + chain = ( + RunnableLambda(lambda x: x) + | prompt + | FakeMessagesListChatModel(responses=[AIMessage(content="Bar")]) + ) + chain.invoke({}, config={"callbacks": [CallbackHandler(mock_client)]}) + + spans = [ + c[1]["properties"] + for c in mock_client.capture_ai.call_args_list + if c[1]["event"] == "$ai_span" + ] + assert spans + assert all("$ai_span_type" not in span for span in spans) + + def test_tool_definition(mock_client): """Test that tools defined in invocation parameters are captured in $ai_tools property""" callbacks = CallbackHandler(mock_client) diff --git a/posthog/test/ai/test_utils.py b/posthog/test/ai/test_utils.py index 49add2881..647eb2a31 100644 --- a/posthog/test/ai/test_utils.py +++ b/posthog/test/ai/test_utils.py @@ -6,7 +6,11 @@ import pytest from posthog.ai.types import TokenUsage -from posthog.ai.utils import merge_usage_stats, serialize_raw_usage +from posthog.ai.utils import ( + _format_tool_error, + merge_usage_stats, + serialize_raw_usage, +) class PydanticUsage: @@ -164,3 +168,28 @@ def test_merge_usage_stats_invalid_mode_does_not_mutate_target(): with pytest.raises(ValueError, match="Invalid mode: unknown"): merge_usage_stats(target, TokenUsage(input_tokens=4), mode="unknown") assert target == {"input_tokens": 10} + + +def test_format_tool_error_redacts_media_blocks(): + client = SimpleNamespace(privacy_mode=False, enable_full_ai_capture=False) + content = [ + { + "type": "image", + "source_type": "base64", + "data": "iVBORw0KGgo" + "A" * 3000, + "mime_type": "image/png", + } + ] + + error = _format_tool_error(client, False, content) + + assert "iVBORw0KGgo" not in error + assert "redacted" in error + + +def test_format_tool_error_redacts_data_url_string(): + client = SimpleNamespace(privacy_mode=False, enable_full_ai_capture=False) + + error = _format_tool_error(client, False, "data:image/png;base64," + "A" * 3000) + + assert "AAAA" not in error