Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/ai-tool-span-outcomes.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

`posthog.ai.claude_agent_sdk` and `posthog.ai.langchain`: tool spans now carry the tool's outcome, so per-tool failure rates can be computed. The Claude Agent SDK integration emits each tool `$ai_span` when its `ToolResultBlock` arrives, with `$ai_output_state`, `$ai_is_error`, `$ai_error` and `$ai_latency` (a tool use whose result never arrives is still emitted when the query or session ends). The LangChain callback handler sets `$ai_span_type: "tool"` on tool spans, marks a tool error handled by `handle_tool_error` (a `ToolMessage` with `status="error"`) with `$ai_is_error` and `$ai_error`, and keeps `tool_call_id`, `name` and an error `status` on tool messages in `$ai_input`.
32 changes: 23 additions & 9 deletions posthog/ai/claude_agent_sdk/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,11 @@
from posthog.ai.claude_agent_sdk.processor import (
PostHogClaudeAgentProcessor,
_GenerationTracker,
_PendingToolSpan,
_from_subagent,
_promote_input,
_starts_generation,
_tool_result_blocks,
)
from posthog.client import Client

Expand Down Expand Up @@ -119,6 +121,8 @@ def __init__(
self._current_input: Optional[List[Dict[str, Any]]] = None
self._next_input: Optional[List[Dict[str, Any]]] = None
self._pending_output: List[Dict[str, Any]] = []
# Tool spans waiting for their result, keyed by tool_use_id
self._pending_tools: Dict[str, _PendingToolSpan] = {}
self._query_start = time.time()

async def connect(self, prompt: Any = None) -> None:
Expand Down Expand Up @@ -186,16 +190,23 @@ def _process_message(self, message: Any) -> None:
elif isinstance(message, AssistantMessage):
self._process_assistant_message(message)

elif isinstance(message, UserMessage) and not _from_subagent(message):
if not self._tracker.had_any_stream_events:
# Without stream events, a tool result ends the assistant turn;
# the result fallback reports the final turn only.
self._pending_output = []
self._next_input = _format_user_input(
message.content, self._processor._client, self._next_input
)
elif isinstance(message, UserMessage):
# Tool results close their tool spans, subagent ones included.
for block in _tool_result_blocks(message):
self._processor._finish_tool_span(
self._pending_tools, block, self._privacy
)
if not _from_subagent(message):
if not self._tracker.had_any_stream_events:
# Without stream events, a tool result ends the assistant
# turn; the result fallback reports the final turn only.
self._pending_output = []
self._next_input = _format_user_input(
message.content, self._processor._client, self._next_input
)

elif isinstance(message, ResultMessage):
self._processor._flush_tool_spans(self._pending_tools)
if not self._tracker.had_any_stream_events:
self._processor._emit_generation_from_result(
message,
Expand All @@ -217,7 +228,8 @@ def _process_assistant_message(self, message: "AssistantMessage") -> None:
parent_id = self._tracker.current_span_id or self._current_generation_span_id
for block in message.content:
if isinstance(block, ToolUseBlock):
self._processor._emit_tool_span(
self._processor._start_tool_span(
self._pending_tools,
block,
self._trace_id,
parent_id,
Expand All @@ -237,6 +249,8 @@ async def disconnect(self) -> None:
"""
# Emit the trace event covering the entire session
try:
# Tool spans whose result never arrived are emitted without one.
self._processor._flush_tool_spans(self._pending_tools)
latency = time.time() - self._query_start
resolved_id = self._processor._resolve_distinct_id(self._distinct_id)

Expand Down
Loading
Loading