diff --git a/.sampo/changesets/ai-tool-span-outcomes.md b/.sampo/changesets/ai-tool-span-outcomes.md new file mode 100644 index 000000000..be837b93f --- /dev/null +++ b/.sampo/changesets/ai-tool-span-outcomes.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +`posthog.ai.claude_agent_sdk` and `posthog.ai.langchain`: tool spans now carry the tool's outcome, so per-tool failure rates can be computed. The Claude Agent SDK integration emits each tool `$ai_span` when its `ToolResultBlock` arrives, with `$ai_output_state`, `$ai_is_error`, `$ai_error` and `$ai_latency` (a tool use whose result never arrives is still emitted when the query or session ends). The LangChain callback handler sets `$ai_span_type: "tool"` on tool spans, marks a tool error handled by `handle_tool_error` (a `ToolMessage` with `status="error"`) with `$ai_is_error` and `$ai_error`, and keeps `tool_call_id`, `name` and an error `status` on tool messages in `$ai_input`. diff --git a/posthog/ai/claude_agent_sdk/client.py b/posthog/ai/claude_agent_sdk/client.py index f8fb124e6..09e199fde 100644 --- a/posthog/ai/claude_agent_sdk/client.py +++ b/posthog/ai/claude_agent_sdk/client.py @@ -32,9 +32,11 @@ from posthog.ai.claude_agent_sdk.processor import ( PostHogClaudeAgentProcessor, _GenerationTracker, + _PendingToolSpan, _from_subagent, _promote_input, _starts_generation, + _tool_result_blocks, ) from posthog.client import Client @@ -119,6 +121,8 @@ def __init__( self._current_input: Optional[List[Dict[str, Any]]] = None self._next_input: Optional[List[Dict[str, Any]]] = None self._pending_output: List[Dict[str, Any]] = [] + # Tool spans waiting for their result, keyed by tool_use_id + self._pending_tools: Dict[str, _PendingToolSpan] = {} self._query_start = time.time() async def connect(self, prompt: Any = None) -> None: @@ -186,16 +190,23 @@ def _process_message(self, message: Any) -> None: elif isinstance(message, AssistantMessage): self._process_assistant_message(message) - elif isinstance(message, UserMessage) and not _from_subagent(message): - if not self._tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant turn; - # the result fallback reports the final turn only. - self._pending_output = [] - self._next_input = _format_user_input( - message.content, self._processor._client, self._next_input - ) + elif isinstance(message, UserMessage): + # Tool results close their tool spans, subagent ones included. + for block in _tool_result_blocks(message): + self._processor._finish_tool_span( + self._pending_tools, block, self._privacy + ) + if not _from_subagent(message): + if not self._tracker.had_any_stream_events: + # Without stream events, a tool result ends the assistant + # turn; the result fallback reports the final turn only. + self._pending_output = [] + self._next_input = _format_user_input( + message.content, self._processor._client, self._next_input + ) elif isinstance(message, ResultMessage): + self._processor._flush_tool_spans(self._pending_tools) if not self._tracker.had_any_stream_events: self._processor._emit_generation_from_result( message, @@ -217,7 +228,8 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None: parent_id = self._tracker.current_span_id or self._current_generation_span_id for block in message.content: if isinstance(block, ToolUseBlock): - self._processor._emit_tool_span( + self._processor._start_tool_span( + self._pending_tools, block, self._trace_id, parent_id, @@ -237,6 +249,8 @@ async def disconnect(self) -> None: """ # Emit the trace event covering the entire session try: + # Tool spans whose result never arrived are emitted without one. + self._processor._flush_tool_spans(self._pending_tools) latency = time.time() - self._query_start resolved_id = self._processor._resolve_distinct_id(self._distinct_id) diff --git a/posthog/ai/claude_agent_sdk/processor.py b/posthog/ai/claude_agent_sdk/processor.py index ec9b46c3d..d607b8367 100644 --- a/posthog/ai/claude_agent_sdk/processor.py +++ b/posthog/ai/claude_agent_sdk/processor.py @@ -62,6 +62,36 @@ class _GenerationData: stop_reason: Optional[str] = None +@dataclass +class _PendingToolSpan: + """A tool span waiting for its ToolResultBlock.""" + + properties: Dict[str, Any] + start_time: float + distinct_id: Any + groups: Dict[str, Any] + extra_options: Optional[Dict[str, Any]] = None + privacy: bool = False + + +_TOOL_SPAN_CONTENT_PROPERTIES = ("$ai_input_state", "$ai_output_state", "$ai_error") + + +def _tool_error_text(content: Any) -> str: + """The `$ai_error` text for an errored tool result's content.""" + if isinstance(content, str): + return content + if isinstance(content, list): + texts = [ + item["text"] + for item in content + if isinstance(item, dict) and isinstance(item.get("text"), str) + ] + if texts: + return "\n".join(texts) + return "" if content is None else str(content) + + class _GenerationTracker: """Tracks StreamEvent boundaries to reconstruct per-generation metrics. @@ -162,7 +192,9 @@ class PostHogClaudeAgentProcessor: Emits: - $ai_generation: one per Anthropic API call (reconstructed from StreamEvents) - - $ai_span: one per tool use (ToolUseBlock in AssistantMessage) + - $ai_span: one per tool use (ToolUseBlock in AssistantMessage), emitted + when its ToolResultBlock arrives so it carries the result, error flag + and latency - $ai_trace: one per query() call (on ResultMessage) """ @@ -315,90 +347,114 @@ async def query( # Accumulate assistant output per generation pending_output: List[Dict[str, Any]] = [] - async for message in original_query( - prompt=prompt, options=options, transport=transport - ): - # All instrumentation is wrapped in try/except so PostHog errors - # never interrupt the underlying Claude Agent SDK query. - try: - if isinstance(message, StreamEvent): - if _starts_generation(message): - if tracker.current_span_id is not None: - # The previous request never finished (it was - # retried), so its partial output is dropped. + # Tool spans waiting for their result, keyed by tool_use_id + pending_tools: Dict[str, _PendingToolSpan] = {} + + try: + async for message in original_query( + prompt=prompt, options=options, transport=transport + ): + # All instrumentation is wrapped in try/except so PostHog errors + # never interrupt the underlying Claude Agent SDK query. + try: + if isinstance(message, StreamEvent): + if _starts_generation(message): + if tracker.current_span_id is not None: + # The previous request never finished (it was + # retried), so its partial output is dropped. + pending_output = [] + current_input, next_input = _promote_input( + current_input, next_input + ) + tracker.process_stream_event(message) + + # Emit $ai_generation when a turn completes + if tracker.has_completed_generation(): + gen = tracker.pop_generation() + generation_index += 1 + current_generation_span_id = gen.span_id + self._emit_generation( + gen, + trace_id, + generation_index, + current_input, + pending_output or None, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + current_input = None pending_output = [] - current_input, next_input = _promote_input( - current_input, next_input - ) - tracker.process_stream_event(message) - - # Emit $ai_generation when a turn completes - if tracker.has_completed_generation(): - gen = tracker.pop_generation() - generation_index += 1 - current_generation_span_id = gen.span_id - self._emit_generation( - gen, - trace_id, - generation_index, - current_input, - pending_output or None, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, + + elif isinstance(message, AssistantMessage): + tracker.set_model(message.model) + # Use the in-progress generation's span_id as parent for tool spans. + # AssistantMessage arrives before message_stop, so current_generation_span_id + # would be stale (from the previous turn). tracker.current_span_id gives us + # the correct in-progress generation. + parent_id = ( + tracker.current_span_id or current_generation_span_id ) - current_input = None - pending_output = [] - - elif isinstance(message, AssistantMessage): - tracker.set_model(message.model) - # Use the in-progress generation's span_id as parent for tool spans. - # AssistantMessage arrives before message_stop, so current_generation_span_id - # would be stale (from the previous turn). tracker.current_span_id gives us - # the correct in-progress generation. - parent_id = tracker.current_span_id or current_generation_span_id - for block in message.content: - if isinstance(block, ToolUseBlock): - self._emit_tool_span( - block, + for block in message.content: + if isinstance(block, ToolUseBlock): + self._start_tool_span( + pending_tools, + block, + trace_id, + parent_id, + distinct_id_override, + extra_props, + privacy, + groups, + extra_options=extra_options, + ) + if not _from_subagent(message): + # A subagent's own API calls are not this query's generations. + pending_output = _append_assistant_output( + pending_output, message.content + ) + + elif isinstance(message, UserMessage): + # Tool results close their tool spans, subagent ones included. + for block in _tool_result_blocks(message): + self._finish_tool_span(pending_tools, block, privacy) + if not _from_subagent(message): + # UserMessages carry tool results; queue them as input + # for the *next* generation. + if not tracker.had_any_stream_events: + # Without stream events, a tool result ends the + # assistant turn; the result fallback reports the + # final turn only. + pending_output = [] + next_input = _format_user_input( + message.content, self._client, next_input + ) + + elif isinstance(message, ResultMessage): + self._flush_tool_spans(pending_tools) + # Fallback: if no StreamEvents were received, emit a single + # generation from ResultMessage aggregate data + if not tracker.had_any_stream_events: + self._emit_generation_from_result( + message, trace_id, - parent_id, + tracker.last_model, + query_start, + initial_input, + pending_output, distinct_id_override, extra_props, privacy, groups, extra_options=extra_options, ) - if not _from_subagent(message): - # A subagent's own API calls are not this query's generations. - pending_output = _append_assistant_output( - pending_output, message.content - ) - elif isinstance(message, UserMessage) and not _from_subagent(message): - # UserMessages carry tool results; queue them as input for the - # *next* generation. - if not tracker.had_any_stream_events: - # Without stream events, a tool result ends the assistant - # turn; the result fallback reports the final turn only. - pending_output = [] - next_input = _format_user_input( - message.content, self._client, next_input - ) - - elif isinstance(message, ResultMessage): - # Fallback: if no StreamEvents were received, emit a single - # generation from ResultMessage aggregate data - if not tracker.had_any_stream_events: - self._emit_generation_from_result( + self._emit_trace( message, trace_id, - tracker.last_model, query_start, - initial_input, - pending_output, distinct_id_override, extra_props, privacy, @@ -406,22 +462,18 @@ async def query( extra_options=extra_options, ) - self._emit_trace( - message, - trace_id, - query_start, - distinct_id_override, - extra_props, - privacy, - groups, - extra_options=extra_options, - ) + except Exception as e: + log.debug(f"PostHog instrumentation error (non-fatal): {e}") + yield message + finally: + # Tool spans still waiting for a result (the query ended + # early, failed or was cancelled) are emitted without one. + try: + self._flush_tool_spans(pending_tools) except Exception as e: log.debug(f"PostHog instrumentation error (non-fatal): {e}") - yield message - def _emit_generation( self, gen: _GenerationData, @@ -573,8 +625,9 @@ def _emit_generation_from_result( options=extra_options, ) - def _emit_tool_span( + def _start_tool_span( self, + pending: Dict[str, _PendingToolSpan], block: "ToolUseBlock", trace_id: str, parent_span_id: Optional[str], @@ -584,8 +637,12 @@ def _emit_tool_span( groups: Dict[str, Any], extra_options: Optional[Dict[str, Any]] = None, ) -> None: - resolved_id = self._resolve_distinct_id(distinct_id) + """Hold a tool span until its ToolResultBlock arrives. + The span is emitted by ``_finish_tool_span`` with the result, error + flag and latency, or by ``_flush_tool_spans`` without them if the + result never arrives. + """ properties: Dict[str, Any] = { "$ai_trace_id": trace_id, "$ai_span_id": str(uuid.uuid4()), @@ -602,13 +659,68 @@ def _emit_tool_span( _ensure_serializable(block.input), self._client ) + span = _PendingToolSpan( + properties=properties, + start_time=time.time(), + distinct_id=distinct_id, + groups=groups, + extra_options=extra_options, + privacy=privacy, + ) + tool_use_id = getattr(block, "id", None) + if tool_use_id is None: + # Nothing can match a result to this span, so emit it now. + self._capture_tool_span(span) + return + # A repeated id would orphan the earlier span; emit it first. + previous = pending.pop(tool_use_id, None) + if previous is not None: + self._capture_tool_span(previous) + pending[tool_use_id] = span + + def _finish_tool_span( + self, + pending: Dict[str, _PendingToolSpan], + block: Any, + privacy: bool, + ) -> None: + """Emit the held tool span for a ToolResultBlock with its outcome.""" + tool_use_id = getattr(block, "tool_use_id", None) + span = pending.pop(tool_use_id, None) if tool_use_id is not None else None + if span is None: + return + properties = span.properties + properties["$ai_latency"] = time.time() - span.start_time + is_error = bool(getattr(block, "is_error", False)) + properties["$ai_is_error"] = is_error + if _with_privacy_mode(self._client, privacy, True): + content = format_tool_result_content(block, self._client) + properties["$ai_output_state"] = finalize_ai_content(content, self._client) + if is_error: + properties["$ai_error"] = _tool_error_text(content) + self._capture_tool_span(span) + + def _flush_tool_spans(self, pending: Dict[str, _PendingToolSpan]) -> None: + """Emit tool spans whose result never arrived, without an outcome.""" + while pending: + _, span = pending.popitem() + self._capture_tool_span(span) + + def _capture_tool_span(self, span: _PendingToolSpan) -> None: + if _with_privacy_mode(self._client, span.privacy, True) is None: + # Privacy may have been enabled while the tool ran; drop content + # buffered before then. + for key in _TOOL_SPAN_CONTENT_PROPERTIES: + span.properties.pop(key, None) + resolved_id = self._resolve_distinct_id(span.distinct_id) + trace_id = span.properties["$ai_trace_id"] self._capture_event( "$ai_span", - properties, + span.properties, resolved_id or trace_id, - groups, + span.groups, personless=resolved_id is None, - options=extra_options, + options=span.extra_options, ) def _emit_trace( @@ -680,6 +792,14 @@ def _from_subagent(message: Any) -> bool: return getattr(message, "parent_tool_use_id", None) is not None +def _tool_result_blocks(message: "UserMessage") -> List[Any]: + """The ToolResultBlocks in a UserMessage.""" + content = message.content + if not isinstance(content, list): + return [] + return [block for block in content if hasattr(block, "tool_use_id")] + + def _starts_generation(message: "StreamEvent") -> bool: """Whether a main-agent API call is starting (subagent calls are skipped).""" return ( diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0972cf43a..01a7bdfeb 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -8,7 +8,7 @@ import json import logging import time -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import ( Any, Dict, @@ -72,6 +72,8 @@ class SpanMetadata: """End time of the run.""" input: Optional[Any] """Input of the run: messages, prompt variables, etc.""" + span_type: Optional[str] = field(default=None, kw_only=True) + """`$ai_span_type` of the run, e.g. "tool" for tool runs.""" @property def latency(self) -> float: @@ -303,7 +305,7 @@ def on_tool_start( ) self._set_parent_of_run(run_id, parent_run_id) self._set_trace_or_span_metadata( - serialized, input_str, run_id, parent_run_id, **kwargs + serialized, input_str, run_id, parent_run_id, span_type="tool", **kwargs ) def on_tool_end( @@ -461,12 +463,17 @@ def _set_trace_or_span_metadata( input: Any, run_id: UUID, parent_run_id: Optional[UUID] = None, + span_type: Optional[str] = None, **kwargs, ): default_name = "trace" if parent_run_id is None else "span" run_name = _get_langchain_run_name(serialized, **kwargs) or default_name self._runs[run_id] = SpanMetadata( - name=run_name, input=input, start_time=time.time(), end_time=None + name=run_name, + input=input, + start_time=time.time(), + end_time=None, + span_type=span_type, ) def _set_llm_metadata( @@ -575,6 +582,8 @@ def _capture_trace_or_span( } if parent_run_id is not None: event_properties["$ai_parent_id"] = parent_run_id + if run.span_type is not None and event_name == "$ai_span": + event_properties["$ai_span_type"] = run.span_type if self._properties: event_properties.update(self._properties) @@ -593,6 +602,15 @@ def _capture_trace_or_span( self._privacy_mode, finalize_ai_content(outputs, self._ph_client), ) + if isinstance(outputs, ToolMessage) and outputs.status == "error": + # A tool error handled by `handle_tool_error` ends the run + # normally, with the error as a ToolMessage. + event_properties["$ai_is_error"] = True + event_properties["$ai_error"] = with_privacy_mode( + self._ph_client, + self._privacy_mode, + _tool_message_text(outputs), + ) _capture_ai_event( self._ph_client, @@ -847,6 +865,19 @@ def _convert_lc_tool_calls_to_oai( return tool_calls +def _tool_message_text(message: ToolMessage) -> str: + """The text of a ToolMessage, for `$ai_error`.""" + content = message.content + if isinstance(content, str): + return content + texts = [ + item if isinstance(item, str) else item.get("text", "") + for item in content + if isinstance(item, str) or isinstance(item, dict) + ] + return "\n".join(text for text in texts if text) + + def _convert_message_to_dict(message: BaseMessage) -> dict[str, Any]: # assistant message if isinstance(message, HumanMessage): @@ -860,7 +891,15 @@ def _convert_message_to_dict(message: BaseMessage) -> dict[str, Any]: elif isinstance(message, SystemMessage): message_dict = {"role": "system", "content": message.content} elif isinstance(message, ToolMessage): - message_dict = {"role": "tool", "content": message.content} + message_dict = { + "role": "tool", + "content": message.content, + "tool_call_id": message.tool_call_id, + } + if message.name: + message_dict["name"] = message.name + if message.status == "error": + message_dict["status"] = message.status elif isinstance(message, FunctionMessage): message_dict = {"role": "function", "content": message.content} else: diff --git a/posthog/test/ai/claude_agent_sdk/test_client.py b/posthog/test/ai/claude_agent_sdk/test_client.py index f902313e1..14c62244f 100644 --- a/posthog/test/ai/claude_agent_sdk/test_client.py +++ b/posthog/test/ai/claude_agent_sdk/test_client.py @@ -599,3 +599,62 @@ async def test_result_fallback_does_not_carry_output_into_the_next_query(sdk): assert second["$ai_output_choices"] == [ {"role": "assistant", "content": [{"type": "text", "text": "second"}]} ] + + +async def tool_spans_for(sdk, monkeypatch, use_client, messages): + ph = analytics() + if use_client: + async with PostHogClaudeSDKClient(posthog_client=ph) as client: + await client.query("question") + respond(sdk, messages) + assert [msg async for msg in client.receive_response()] == messages + else: + + async def query(**kwargs): + for message in messages: + yield message + + monkeypatch.setattr( + "posthog.ai.claude_agent_sdk.processor.original_query", query + ) + processor = PostHogClaudeAgentProcessor(client=ph) + assert [msg async for msg in processor.query(prompt="question")] == messages + return [event["properties"] for event in events(ph, "$ai_span")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_tool_spans_carry_the_result_of_their_own_tool_use( + sdk, monkeypatch, use_client +): + messages = cli_tool_turn() + # The CLI sends results in any order; tool-2's result is an error. + messages[6] = UserMessage( + content=[ + ToolResultBlock(tool_use_id="tool-2", content="no such file", is_error=True) + ] + ) + spans = await tool_spans_for(sdk, monkeypatch, use_client, messages) + by_input = {span["$ai_input_state"]["path"]: span for span in spans} + assert by_input["a"]["$ai_is_error"] is False + assert by_input["a"]["$ai_output_state"] == "a text" + assert "$ai_error" not in by_input["a"] + assert by_input["b"]["$ai_is_error"] is True + assert by_input["b"]["$ai_error"] == "no such file" + assert all(span["$ai_latency"] >= 0 for span in spans) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("use_client", [False, True]) +async def test_tool_span_without_result_is_still_emitted(sdk, monkeypatch, use_client): + messages = [ + stream("message_start", message={"model": "claude", "usage": {}}), + AssistantMessage( + model="claude", + content=[ToolUseBlock(id="tool-1", name="Read", input={"path": "a"})], + ), + stream("message_stop"), + ] + (span,) = await tool_spans_for(sdk, monkeypatch, use_client, messages) + assert span["$ai_span_name"] == "Read" + assert "$ai_is_error" not in span diff --git a/posthog/test/ai/claude_agent_sdk/test_processor.py b/posthog/test/ai/claude_agent_sdk/test_processor.py index 6aceb6ce8..cb0f05421 100644 --- a/posthog/test/ai/claude_agent_sdk/test_processor.py +++ b/posthog/test/ai/claude_agent_sdk/test_processor.py @@ -18,7 +18,9 @@ ResultMessage, StreamEvent, TextBlock, + ToolResultBlock, ToolUseBlock, + UserMessage, ) from posthog.ai.claude_agent_sdk import ( @@ -477,6 +479,161 @@ async def test_emits_span_for_tool_use(self, processor, mock_client): assert props["$ai_span_name"] == "Read" assert props["$ai_span_type"] == "tool" assert props["$ai_input_state"] == {"file_path": "/tmp/test.py"} + # No ToolResultBlock arrived, so the span is flushed without an outcome. + assert "$ai_is_error" not in props + assert "$ai_output_state" not in props + + @staticmethod + async def _run(proc, messages): + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: _fake_query(messages), + ): + async for _ in proc.query(prompt="go", options=ClaudeAgentOptions()): + pass + + @staticmethod + def _events(mock_client): + return [ + (c.kwargs["event"], c.kwargs["properties"]) + for c in mock_client.capture_ai.call_args_list + ] + + @staticmethod + def _tool_turn(result_block, user_message_kwargs=None): + return [ + _make_message_start(), + _make_assistant_message( + tool_uses=[{"id": "tu_1", "name": "Bash", "input": {"command": "ls"}}] + ), + _make_message_stop(), + UserMessage(content=[result_block], **(user_message_kwargs or {})), + _make_message_start(), + _make_assistant_message(text="done"), + _make_message_stop(), + _make_result_message(), + ] + + @pytest.mark.asyncio + async def test_span_carries_successful_tool_result(self, processor, mock_client): + await self._run( + processor, + self._tool_turn(ToolResultBlock(tool_use_id="tu_1", content="a.txt")), + ) + + events = self._events(mock_client) + names = [name for name, _ in events] + # Emitted when the result arrives, before the next generation. + assert names == ["$ai_generation", "$ai_span", "$ai_generation", "$ai_trace"] + span = events[1][1] + assert span["$ai_parent_id"] == events[0][1]["$ai_span_id"] + assert span["$ai_output_state"] == "a.txt" + assert span["$ai_is_error"] is False + assert "$ai_error" not in span + assert span["$ai_latency"] >= 0 + + @pytest.mark.asyncio + @pytest.mark.parametrize("result_arrives", [True, False]) + async def test_privacy_enabled_mid_tool_drops_buffered_content( + self, processor, mock_client, result_arrives + ): + turn = self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="oops", is_error=True) + ) + if not result_arrives: + turn = turn[:3] + [_make_result_message()] + + async def query(messages): + for i, msg in enumerate(messages): + if i == 3: + # Privacy turns on while the tool runs. + mock_client.privacy_mode = True + yield msg + + with patch( + "posthog.ai.claude_agent_sdk.processor.original_query", + side_effect=lambda **kw: query(turn), + ): + async for _ in processor.query(prompt="go", options=ClaudeAgentOptions()): + pass + + spans = [p for name, p in self._events(mock_client) if name == "$ai_span"] + assert len(spans) == 1 + for key in ("$ai_input_state", "$ai_output_state", "$ai_error"): + assert key not in spans[0] + if result_arrives: + assert spans[0]["$ai_is_error"] is True + + @pytest.mark.asyncio + async def test_span_carries_tool_error(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock( + tool_use_id="tu_1", + content=[{"type": "text", "text": "ls: permission denied"}], + is_error=True, + ) + ), + ) + + spans = [p for name, p in self._events(mock_client) if name == "$ai_span"] + assert len(spans) == 1 + assert spans[0]["$ai_is_error"] is True + assert spans[0]["$ai_error"] == "ls: permission denied" + assert spans[0]["$ai_output_state"] == [ + {"type": "text", "text": "ls: permission denied"} + ] + + @pytest.mark.asyncio + async def test_privacy_mode_keeps_error_flag_but_redacts_result(self, mock_client): + proc = PostHogClaudeAgentProcessor( + client=mock_client, distinct_id="user", privacy_mode=True + ) + await self._run( + proc, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="secret", is_error=True) + ), + ) + + spans = [p for name, p in self._events(mock_client) if name == "$ai_span"] + assert spans[0]["$ai_is_error"] is True + assert "$ai_latency" in spans[0] + assert "$ai_error" not in spans[0] + assert "$ai_output_state" not in spans[0] + assert "$ai_input_state" not in spans[0] + + @pytest.mark.asyncio + async def test_subagent_tool_result_closes_its_span(self, processor, mock_client): + await self._run( + processor, + self._tool_turn( + ToolResultBlock(tool_use_id="tu_1", content="boom", is_error=True), + {"parent_tool_use_id": "tu_parent"}, + ), + ) + + spans = [p for name, p in self._events(mock_client) if name == "$ai_span"] + assert len(spans) == 1 + assert spans[0]["$ai_is_error"] is True + + @pytest.mark.asyncio + async def test_pending_span_is_flushed_when_query_ends_without_result( + self, processor, mock_client + ): + await self._run( + processor, + [ + _make_message_start(), + _make_assistant_message(tool_uses=[{"id": "tu_1", "name": "Bash"}]), + _make_message_stop(), + ], + ) + + spans = [p for name, p in self._events(mock_client) if name == "$ai_span"] + assert len(spans) == 1 + assert "$ai_is_error" not in spans[0] class TestTraceEmission: diff --git a/posthog/test/ai/langchain/test_callbacks.py b/posthog/test/ai/langchain/test_callbacks.py index 10df3378a..c24e39fdc 100644 --- a/posthog/test/ai/langchain/test_callbacks.py +++ b/posthog/test/ai/langchain/test_callbacks.py @@ -1582,6 +1582,125 @@ def span(_): assert mock_client.capture_ai.call_args_list[1][1]["properties"]["$ai_is_error"] +def _tool_spans(mock_client): + return [ + call[1]["properties"] + for call in mock_client.capture_ai.call_args_list + if call[1]["event"] == "$ai_span" + and call[1]["properties"].get("$ai_span_type") == "tool" + ] + + +def _invoke_tool_in_chain(mock_client, tool_to_call, args, privacy_mode=False): + # A ToolCall input makes the tool return a ToolMessage. + chain = RunnableLambda( + lambda x, config: tool_to_call.invoke( + {"name": tool_to_call.name, "args": x, "id": "call_1", "type": "tool_call"}, + config, + ) + ) + callbacks = [CallbackHandler(mock_client, privacy_mode=privacy_mode)] + return chain.invoke(args, config={"callbacks": callbacks}) + + +def test_tool_span_has_tool_span_type(mock_client): + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + return f"Sunny in {city}" + + _invoke_tool_in_chain(mock_client, get_weather, {"city": "Paris"}) + + spans = _tool_spans(mock_client) + assert len(spans) == 1 + assert spans[0]["$ai_span_name"] == "get_weather" + assert "$ai_is_error" not in spans[0] + # Only tool runs are typed. + other_events = [ + call[1] + for call in mock_client.capture_ai.call_args_list + if call[1]["properties"].get("$ai_span_type") is None + ] + assert [event["event"] for event in other_events] == ["$ai_trace"] + + +def test_tool_span_marks_handled_tool_error(mock_client): + from langchain_core.tools import ToolException + + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + raise ToolException(f"weather service unavailable for {city}") + + get_weather.handle_tool_error = True + + output = _invoke_tool_in_chain(mock_client, get_weather, {"city": "Paris"}) + assert output.status == "error" + + spans = _tool_spans(mock_client) + assert len(spans) == 1 + assert spans[0]["$ai_is_error"] is True + assert spans[0]["$ai_error"] == "weather service unavailable for Paris" + + +def test_tool_span_handled_tool_error_respects_privacy_mode(mock_client): + from langchain_core.tools import ToolException + + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + raise ToolException("secret details") + + get_weather.handle_tool_error = True + + _invoke_tool_in_chain( + mock_client, get_weather, {"city": "Paris"}, privacy_mode=True + ) + + spans = _tool_spans(mock_client) + assert spans[0]["$ai_is_error"] is True + assert spans[0]["$ai_error"] is None + assert spans[0]["$ai_output_state"] is None + + +def test_tool_span_unhandled_tool_error(mock_client): + @tool + def get_weather(city: str) -> str: + """Get the weather for a city.""" + raise ValueError("boom") + + with pytest.raises(ValueError): + _invoke_tool_in_chain(mock_client, get_weather, {"city": "Paris"}) + + spans = _tool_spans(mock_client) + assert len(spans) == 1 + assert spans[0]["$ai_is_error"] is True + assert spans[0]["$ai_error"] == "ValueError: boom" + + +def test_convert_message_to_dict_tool_message(): + from langchain_core.messages import ToolMessage + + from posthog.ai.langchain.callbacks import _convert_message_to_dict + + assert _convert_message_to_dict( + ToolMessage(content="Sunny", tool_call_id="call_1", name="get_weather") + ) == { + "role": "tool", + "content": "Sunny", + "tool_call_id": "call_1", + "name": "get_weather", + } + assert _convert_message_to_dict( + ToolMessage(content="boom", tool_call_id="call_2", status="error") + ) == { + "role": "tool", + "content": "boom", + "tool_call_id": "call_2", + "status": "error", + } + + def test_openai_reasoning_tokens(mock_client): """Test that OpenAI reasoning tokens are captured correctly.""" prompt = ChatPromptTemplate.from_messages( diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index c9e0b430e..eac0cfab2 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -621,6 +621,7 @@ attribute posthog.ai.langchain.callbacks.SpanMetadata.end_time: Optional[float] attribute posthog.ai.langchain.callbacks.SpanMetadata.input: Optional[Any] attribute posthog.ai.langchain.callbacks.SpanMetadata.latency: float attribute posthog.ai.langchain.callbacks.SpanMetadata.name: str +attribute posthog.ai.langchain.callbacks.SpanMetadata.span_type: Optional[str] = field(default=None, kw_only=True) attribute posthog.ai.langchain.callbacks.SpanMetadata.start_time: float attribute posthog.ai.langchain.callbacks.log = logging.getLogger('posthog') attribute posthog.ai.langchain.middleware.PostHogMiddleware.state_schema = _PostHogMiddlewareState @@ -1331,9 +1332,9 @@ class posthog.ai.gemini.gemini_async.AsyncModels(api_key: Optional[str] = None, class posthog.ai.gemini.gemini_converter.GeminiMessage class posthog.ai.gemini.gemini_converter.GeminiPart class posthog.ai.langchain.callbacks.CallbackHandler(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[Dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[Dict[str, Any]] = None, options: Optional[Dict[str, Any]] = None) -class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None) +class posthog.ai.langchain.callbacks.GenerationMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], provider: Optional[str] = None, model: Optional[str] = None, model_params: Optional[Dict[str, Any]] = None, base_url: Optional[str] = None, tools: Optional[List[Dict[str, Any]]] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_options: Optional[Dict[str, Any]] = None, *, span_type: Optional[str] = None) class posthog.ai.langchain.callbacks.ModelUsage(input_tokens: Optional[int], output_tokens: Optional[int], cache_write_tokens: Optional[int], cache_read_tokens: Optional[int], reasoning_tokens: Optional[int], cache_write_5m_tokens: Optional[int] = None, cache_write_1h_tokens: Optional[int] = None, web_search_count: Optional[int] = None) -class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any]) +class posthog.ai.langchain.callbacks.SpanMetadata(name: str, start_time: float, end_time: Optional[float], input: Optional[Any], *, span_type: Optional[str] = None) class posthog.ai.langchain.middleware.PostHogMiddleware(client: Optional[Client] = None, *, distinct_id: Optional[Union[str, int, UUID]] = None, trace_id: Optional[Union[str, int, float, UUID]] = None, properties: Optional[dict[str, Any]] = None, privacy_mode: bool = False, groups: Optional[dict[str, Any]] = None, options: Optional[dict[str, Any]] = None) class posthog.ai.openai.openai.OpenAI(posthog_client: Optional[PostHogClient] = None, **kwargs) class posthog.ai.openai.openai.WrappedBeta