diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md new file mode 100644 index 000000000..718b5a7a0 --- /dev/null +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -0,0 +1,10 @@ +--- +pypi/posthog: minor +--- + +MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: + +- `instrument()` sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception` when a `tools/call` names a tool your server never registered. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. A raw low-level server has no registry the SDK can read, so a failed call there stays a failed tool call. +- FastMCP's stock logging, timing, error-handling, and similar middleware no longer turns off `llm_model` injection on standalone `fastmcp.FastMCP`. +- Manual capture without a `session_id` now sends a fresh `ses_` session id and uses it as the `distinct_id`, instead of no `$session_id` and an `anonymous` person. +- Truncated string properties now fit their limit with the `...` marker included, so `$mcp_tool_name` and other names stop at 256 characters instead of 259. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index 58f177233..464036161 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -191,8 +191,10 @@ On MCP SDK 1.x, standalone FastMCP application middleware can replace or reroute When it overrides a listing or dispatch hook, PostHog does not inject `llm_model`: a fresh replica cannot safely distinguish that field from a replacement tool's own argument. Client metadata capture remains enabled. This also applies to pass-through logging or -authorization middleware with those hooks; argument-based model capture is skipped so an -application-owned value cannot be mistaken for analytics. No additional catalog lookup runs +authorization middleware of your own with those hooks; argument-based model capture is skipped +so an application-owned value cannot be mistaken for analytics. FastMCP's stock middleware, +such as `LoggingMiddleware`, `TimingMiddleware`, and `ErrorHandlingMiddleware`, never reroutes a +tool and does not count; its tool-injection middleware does. No additional catalog lookup runs during a tool call. For a custom dispatcher, `PostHogMCP` enables model capture and conversation correlation @@ -339,6 +341,23 @@ call = posthog.prepare_tool_call( ) ``` +## Calls to tools your server does not have + +A `tools/call` naming a tool your server never registered is not a tool call. +`instrument()` sends `$mcp_unknown_tool` with the requested name as +`$mcp_tool_name`, and no `$mcp_tool_call` or `$exception`. The agent gets the +same error as before. A call naming no tool sends nothing. + +The SDK asks the tool registry only after the call fails, and records a failed +tool call whenever the registry can't rule the tool out: + +- A disabled tool, or a tool called with invalid arguments, still exists. +- A raw low-level server has no registry the SDK can read. +- On standalone `fastmcp.FastMCP`, middleware of your own that overrides a + listing or dispatch hook can serve tools the registry lacks. +- On standalone `fastmcp.FastMCP` 3 and later, a mounted server, proxy, or + other provider hides its disabled tools. + ## Virtual tools on a paginated `tools/list` `instrument()` advertises up to two tools of its own: `get_more_tools` diff --git a/posthog/mcp/_event_types.py b/posthog/mcp/_event_types.py index 7392f90a9..005d08457 100644 --- a/posthog/mcp/_event_types.py +++ b/posthog/mcp/_event_types.py @@ -23,3 +23,4 @@ class MCPAnalyticsEventType: MCP_RESOURCES_READ = "mcp:resources/read" MCP_TOOLS_CALL = "mcp:tools/call" MCP_TOOLS_LIST = "mcp:tools/list" + MCP_UNKNOWN_TOOL = "mcp:unknown_tool" diff --git a/posthog/mcp/_instrument_fastmcp.py b/posthog/mcp/_instrument_fastmcp.py index 78bd1324f..366060141 100644 --- a/posthog/mcp/_instrument_fastmcp.py +++ b/posthog/mcp/_instrument_fastmcp.py @@ -167,6 +167,10 @@ async def wrapped( name, call_arguments, context=context, convert_result=convert_result ) except Exception as error: + if await lifecycle.record_if_unknown_tool( + _name_owned_by_real_tool(server, name) + ): + raise # The minted prompt-back was never delivered to the agent — don't stamp # an orphan conversation_id it can't echo (an agent-supplied id is kept). await lifecycle.record_error(error, (time.monotonic() - start) * 1000) diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index 00bd4ef4d..3d84988a4 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -15,10 +15,11 @@ from __future__ import annotations import functools +import importlib import inspect import time from functools import lru_cache -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple, Type import mcp.types as mcp_types @@ -36,6 +37,7 @@ copy_tools_list_result, extract_tools, is_first_listing_page, + is_tool_result_error, mutate_tool_schema, prepare_request, raw_listing_owns_tool_name, @@ -282,6 +284,10 @@ async def handler(req: Any) -> Any: try: result = await original(req) except Exception as error: + if await lifecycle.record_if_unknown_tool( + await registered_tool_exists(high_level, name) + ): + raise # The @server.call_tool() decorator converts raises into # CallToolResult(isError=True), but a handler wired straight into # request_handlers can raise — capture before re-raising so the failed @@ -333,6 +339,12 @@ async def handler(req: Any) -> Any: else call_result ) + if is_tool_result_error(call_result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(high_level, name), + conversation_id_delivered=delivered, + ): + return result + await lifecycle.record_result( call_result, duration_ms, @@ -573,7 +585,7 @@ async def _name_owned_by_real_tool( @lru_cache(maxsize=1) -def _tool_lookup_not_found_errors() -> Tuple[type, ...]: +def _tool_lookup_not_found_errors() -> Tuple[Type[BaseException], ...]: """The fastmcp exceptions that mean "no live tool by that name" -- an answer. Anything else out of ``get_tool`` is the lookup itself failing. Empty when fastmcp is absent or has moved them, which makes every failure delegate: @@ -793,13 +805,82 @@ def _dispatch_can_differ(server: Any) -> bool: ) +# Stock fastmcp middleware that observes, times, retries, throttles, denies, +# caches, or trims a call to the tool it names, and never provides, renames, or +# reroutes one. The tool-injection family is left out on purpose. +_STOCK_DISPATCH_PRESERVING_MIDDLEWARE = ( + ("logging", "LoggingMiddleware"), + ("logging", "StructuredLoggingMiddleware"), + ("timing", "TimingMiddleware"), + ("timing", "DetailedTimingMiddleware"), + ("error_handling", "ErrorHandlingMiddleware"), + ("error_handling", "RetryMiddleware"), + ("rate_limiting", "RateLimitingMiddleware"), + ("rate_limiting", "SlidingWindowRateLimitingMiddleware"), + ("ping", "PingMiddleware"), + ("authorization", "AuthMiddleware"), + ("caching", "ResponseCachingMiddleware"), + ("response_limiting", "ResponseLimitingMiddleware"), +) + + @functools.lru_cache(maxsize=1) def _builtin_middleware_types() -> tuple: """The middleware a bare ``FastMCP()`` installs on its own, probed rather - than named so a new built-in in a later release is still recognised.""" + than named so a new built-in in a later release is still recognised, plus + the stock middleware above that this fastmcp release ships.""" from fastmcp import FastMCP - return tuple(type(middleware) for middleware in FastMCP("posthog-probe").middleware) + stock = [] + for module, name in _STOCK_DISPATCH_PRESERVING_MIDDLEWARE: + try: + stock.append( + getattr( + importlib.import_module(f"fastmcp.server.middleware.{module}"), name + ) + ) + except (ImportError, AttributeError): + continue + return tuple( + type(middleware) for middleware in FastMCP("posthog-probe").middleware + ) + tuple(stock) + + +async def registered_tool_exists(server: Any, name: str) -> Optional[bool]: + """Whether jlowin's FastMCP registers ``name``, enabled or not, asked after a + failed call to tell an unknown tool from a failed one. A disabled tool still + exists: calling it is a failed call. ``None`` when nothing authoritative can + say: no registry (a raw low-level server, whose listing may hide tools its + dispatcher serves), middleware that can serve tools the registry lacks, or a + mounted server or other provider, which hides its disabled tools.""" + if server is None or _dispatch_can_differ(server): + return None + try: + from fastmcp import FastMCP + + # From 3.x, get_tool hides a disabled tool; the providers still hold it. + found = ( + await server.get_tool(name) is not None + or await super(FastMCP, server).get_tool(name) is not None + ) + except _tool_lookup_not_found_errors(): + found = False + except Exception as err: # noqa: BLE001 - analytics must not break the call + log(f"PostHog MCP: could not look up tool {name!r} - {err}") + return None + if found: + return True + return None if _serves_other_providers(server) else False + + +def _serves_other_providers(server: Any) -> bool: + """Whether a FastMCP server serves tools from anywhere but its own + registry: a mounted server, a proxy, or a custom provider. FastMCP 2.x + keeps mounts in ``_mounted_servers``; 3.x lists them in ``providers``.""" + if getattr(server, "_mounted_servers", None): + return True + local = getattr(server, "_local_provider", None) + return any(provider is not local for provider in getattr(server, "providers", ())) def _reference_chain(schema: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 58d45f778..b7d597789 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -53,6 +53,7 @@ _to_jsonable, copy_tools_list_result, is_first_listing_page, + is_tool_result_error, mutate_tool_schema, params_to_request_dict, prepare_request, @@ -66,6 +67,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) +from ._instrument_lowlevel import registered_tool_exists from ._internal import MCPAnalyticsData from ._model_parameters import ( can_inject_model_parameter, @@ -450,6 +452,10 @@ async def wrapped( name, call_arguments, context=context, convert_result=convert_result ) except Exception as error: + if await lifecycle.record_if_unknown_tool( + _name_owned_by_real_tool_v2(server, name) + ): + raise # The outer MCPServer layer converts the raise after this seam; no # freshly minted handle could have been delivered yet. await lifecycle.record_error(error, (time.monotonic() - start) * 1000) @@ -669,6 +675,10 @@ async def handler(ctx: Any, params: Any) -> Any: try: result = await original(ctx, params) except Exception as error: + if await lifecycle.record_if_unknown_tool( + await registered_tool_exists(standalone, name) + ): + raise # Raw v2 handlers raise through to JSON-RPC errors; no freshly minted # handle could have been delivered yet. await lifecycle.record_error(error, (time.monotonic() - start) * 1000) @@ -685,6 +695,12 @@ async def handler(ctx: Any, params: Any) -> Any: lifecycle.minted_conversation_id, ) + if is_tool_result_error(result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(standalone, name), + conversation_id_delivered=delivered, + ): + return result + await lifecycle.record_result( result, duration_ms, conversation_id_delivered=delivered ) diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index ae8d7b30b..9b1db44c7 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -615,6 +615,32 @@ async def record_feedback(self, *, conversation_id_delivered: bool = False) -> s ) return await handle_feedback(report, self.feedback_options) + async def record_if_unknown_tool( + self, tool_exists: Optional[bool], *, conversation_id_delivered: bool = False + ) -> bool: + """Record the failed call as ``$mcp_unknown_tool`` when ``tool_exists`` + proves the server has no tool by that name, and say whether the call is + settled that way. A call naming no tool is settled with no event. The + adapter then re-raises or returns its response.""" + if not self.name: + return True + if tool_exists is not False: + return False + conversation_id = self._anchored_conversation_id(conversation_id_delivered) + session_id = await self.prepare_session(conversation_id) + await record_unknown_tool( + self.data, + session_id, + conversation_id=conversation_id, + name=self.name, + arguments=self.arguments, + client_name=self.client_name, + client_version=self.client_version, + protocol_version=self.protocol_version, + extra=self.extra, + ) + return True + async def record_error(self, error: Any, duration_ms: float) -> None: # A freshly minted handle cannot anchor or be captured when dispatch # raised: no adapter had an opportunity to deliver it to the agent. @@ -1374,6 +1400,39 @@ async def record_missing_capability( log(f"record_missing_capability failed (event dropped): {err}") +async def record_unknown_tool( + data: MCPAnalyticsData, + session_id: str, + *, + conversation_id: Optional[str], + name: str, + arguments: Optional[Dict[str, Any]], + client_name: Optional[str], + client_version: Optional[str], + protocol_version: Optional[str], + extra: Optional[Dict[str, Any]], +) -> None: + """Record a call naming a tool the server never registered as + ``$mcp_unknown_tool``. It is not a tool call, so it carries no outcome.""" + try: + event: Dict[str, Any] = { + "event_type": MCPAnalyticsEventType.MCP_UNKNOWN_TOOL, + "session_id": session_id, + "conversation_id": conversation_id, + "resource_name": name, + "client_name": client_name, + "client_version": client_version, + "protocol_version": protocol_version, + } + await _apply_event_properties( + data, event, build_tool_call_request(name, arguments), extra + ) + stamp_transport_identity(event, extra) + fire_and_forget(capture_event(data, event), data) + except Exception as err: # noqa: BLE001 - isolate analytics from the tool path + log(f"record_unknown_tool failed (event dropped): {err}") + + async def record_feedback( data: MCPAnalyticsData, session_id: str, diff --git a/posthog/mcp/_posthog_events.py b/posthog/mcp/_posthog_events.py index 51fc5ff75..f7b58f0e7 100644 --- a/posthog/mcp/_posthog_events.py +++ b/posthog/mcp/_posthog_events.py @@ -29,6 +29,7 @@ MCPAnalyticsEventType.MCP_RESOURCES_READ: PostHogMCPAnalyticsEvent.RESOURCE_READ, MCPAnalyticsEventType.MCP_TOOLS_CALL: PostHogMCPAnalyticsEvent.TOOL_CALL, MCPAnalyticsEventType.MCP_TOOLS_LIST: PostHogMCPAnalyticsEvent.TOOLS_LIST, + MCPAnalyticsEventType.MCP_UNKNOWN_TOOL: "$mcp_unknown_tool", } _P = PostHogMCPAnalyticsProperty @@ -112,8 +113,12 @@ def _is_tool_call(event: Event) -> bool: def _add_common_properties(event: Event, properties: Dict[str, Any]) -> None: if event.get("resource_name"): - properties[_P.RESOURCE_NAME] = event["resource_name"] - if _is_tool_call(event): + is_unknown_tool = ( + event.get("event_type") == MCPAnalyticsEventType.MCP_UNKNOWN_TOOL + ) + if not is_unknown_tool: + properties[_P.RESOURCE_NAME] = event["resource_name"] + if is_unknown_tool or _is_tool_call(event): properties[_P.TOOL_NAME] = event["resource_name"] if event.get("tool_description") and _is_tool_call(event): properties[_P.TOOL_DESCRIPTION] = event["tool_description"] diff --git a/posthog/mcp/_sanitization.py b/posthog/mcp/_sanitization.py index b14f4796d..426d2d29f 100644 --- a/posthog/mcp/_sanitization.py +++ b/posthog/mcp/_sanitization.py @@ -14,6 +14,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit +from ._event_types import MCPAnalyticsEventType + # SDK-injected arguments stripped from captured $mcp_parameters (they surface as # dedicated properties: $mcp_intent and $mcp_conversation_id). _INJECTED_ARGUMENT_NAMES = ("context", "conversation_id") @@ -628,8 +630,15 @@ def sanitize_event(event: Dict[str, Any]) -> Dict[str, Any]: if result.get("parameters") is not None: result["parameters"] = sanitize_captured_value(result["parameters"]) + # An unknown tool's name is whatever the agent sent, not one of the + # server's identifiers, so it gets the free-text pass. if result.get("resource_name") is not None: - result["resource_name"] = _sanitize_resource_name(result["resource_name"]) + sanitize_name = ( + sanitize_free_text + if result.get("event_type") == MCPAnalyticsEventType.MCP_UNKNOWN_TOOL + else _sanitize_resource_name + ) + result["resource_name"] = sanitize_name(result["resource_name"]) # The intent comes straight from an agent-narrated `context` string, so it # can contain a secret the LLM read aloud or personal data it narrated about diff --git a/posthog/mcp/_truncation.py b/posthog/mcp/_truncation.py index 674241a6c..382c64639 100644 --- a/posthog/mcp/_truncation.py +++ b/posthog/mcp/_truncation.py @@ -85,9 +85,7 @@ def _visit( return "[Infinity]" if value > 0 else "[-Infinity]" return value if isinstance(value, str): - if len(value) > max_string_length: - return value[:max_string_length] + _TRUNCATION_SUFFIX - return value + return truncate_text(value, max_string_length) if isinstance(value, datetime): return value.isoformat() if callable(value): @@ -161,12 +159,18 @@ def _visit_object( # --- field-level helpers ----------------------------------------------------- +def truncate_text(value: str, max_length: int) -> str: + """``value`` cut to at most ``max_length`` characters, the truncation marker + included, keeping its start.""" + if len(value) <= max_length: + return value + return value[: max_length - len(_TRUNCATION_SUFFIX)] + _TRUNCATION_SUFFIX + + def _truncate_string(value: Optional[str], max_length: int) -> Optional[str]: if not isinstance(value, str): return value - if len(value) <= max_length: - return value - return value[:max_length] + _TRUNCATION_SUFFIX + return truncate_text(value, max_length) def _truncate_stack_frames(frames: Optional[List[Any]]) -> Optional[List[Any]]: @@ -214,8 +218,7 @@ def _truncate_response_content(response: Any) -> Any: new_content.append( { **block, - "text": block["text"][:_MAX_CONTENT_TEXT_LENGTH] - + _TRUNCATION_SUFFIX, + "text": truncate_text(block["text"], _MAX_CONTENT_TEXT_LENGTH), } ) else: diff --git a/posthog/mcp/feedback.py b/posthog/mcp/feedback.py index 5d1bc1d08..5cb732b6c 100644 --- a/posthog/mcp/feedback.py +++ b/posthog/mcp/feedback.py @@ -11,6 +11,7 @@ from ._internal import _maybe_await from ._sanitization import sanitize_free_text, sanitize_free_text_value +from ._truncation import truncate_text from .constants import PostHogMCPAnalyticsProperty from .logger import log from .tools import VIRTUAL_TOOL_HINTS @@ -26,7 +27,6 @@ # Free-text fields are agent-narrated, like `$mcp_intent`; bound them the same way. _MAX_FEEDBACK_TEXT_LENGTH = 2048 _MAX_FEEDBACK_TOOL_NAME_LENGTH = 256 -_TRUNCATION_SUFFIX = "..." _DEFAULT_FEEDBACK_DESCRIPTION = ( "Send feedback about this server to its developers: a capability you needed that no tool " @@ -247,10 +247,6 @@ def build_feedback_intent(report: FeedbackReport) -> str: return "\n\n".join(part for part in (report.summary, report.details) if part) -def _truncate_feedback_text(value: str, max_length: int) -> str: - return value[:max_length] + _TRUNCATION_SUFFIX if len(value) > max_length else value - - def _capture_free_text(value: str) -> str: """Agent-narrated free text can contain a secret the LLM read aloud or personal data it narrated, so it gets exactly the ``$mcp_intent`` pass @@ -258,7 +254,7 @@ def _capture_free_text(value: str) -> str: load-bearing, the URL rewrite would percent-encode the ``@`` the email pattern anchors on), then a length bound. The event pipeline does not process ``event["properties"]``, so this happens here.""" - return _truncate_feedback_text(sanitize_free_text(value), _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(sanitize_free_text(value), _MAX_FEEDBACK_TEXT_LENGTH) def _capture_extra_value(value: Any) -> Any: @@ -268,11 +264,11 @@ def _capture_extra_value(value: Any) -> Any: JSON-stringified and everything is bounded. Unserializable values are dropped.""" sanitized = sanitize_free_text_value(value) if isinstance(sanitized, str): - return _truncate_feedback_text(sanitized, _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(sanitized, _MAX_FEEDBACK_TEXT_LENGTH) if sanitized is None or isinstance(sanitized, (bool, int, float)): return sanitized try: - return _truncate_feedback_text(json.dumps(sanitized), _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(json.dumps(sanitized), _MAX_FEEDBACK_TEXT_LENGTH) except Exception: # noqa: BLE001 - capture must never raise into the tool path return None @@ -305,7 +301,7 @@ def build_feedback_event_properties(report: FeedbackReport) -> JsonRecord: # Nominally an identifier, but the schema can't stop an agent from # writing prose into it — so it gets the same free-text pass as the # other fields. - properties[PostHogMCPAnalyticsProperty.FEEDBACK_TOOL] = _truncate_feedback_text( + properties[PostHogMCPAnalyticsProperty.FEEDBACK_TOOL] = truncate_text( sanitize_free_text(report.tool_name), _MAX_FEEDBACK_TOOL_NAME_LENGTH, ) diff --git a/posthog/mcp/posthog_mcp.py b/posthog/mcp/posthog_mcp.py index 69951eade..2e1c4d3e5 100644 --- a/posthog/mcp/posthog_mcp.py +++ b/posthog/mcp/posthog_mcp.py @@ -70,7 +70,7 @@ resolve_collect_feedback_options, resolve_send_feedback_tool_name, ) -from .session import derive_session_id_from_conversation +from .session import derive_session_id_from_conversation, new_session_id from .tools import build_report_missing_descriptor from .types import ( CollectFeedbackOptions, @@ -712,7 +712,7 @@ def _base_event( ) -> Dict[str, Any]: event: Dict[str, Any] = { "event_type": event_type, - "session_id": session_id, + "session_id": session_id or new_session_id(), "conversation_id": conversation_id, "timestamp": timestamp or datetime.now(timezone.utc), "properties": properties, diff --git a/posthog/test/mcp/test_feedback.py b/posthog/test/mcp/test_feedback.py index b45721c38..5110a0d60 100644 --- a/posthog/test/mcp/test_feedback.py +++ b/posthog/test/mcp/test_feedback.py @@ -237,9 +237,9 @@ def test_properties_bound_free_text_and_tool_name(): {"feedback_type": "issue", "summary": "s" * 5000, "tool_name": "t" * 500} ) props = build_feedback_event_properties(report) - assert len(props["$mcp_feedback_summary"]) == 2048 + 3 + assert len(props["$mcp_feedback_summary"]) == 2048 assert props["$mcp_feedback_summary"].endswith("...") - assert len(props["$mcp_feedback_tool"]) == 256 + 3 + assert len(props["$mcp_feedback_tool"]) == 256 def test_properties_capture_declared_extras_only(): diff --git a/posthog/test/mcp/test_pipeline.py b/posthog/test/mcp/test_pipeline.py index 6a4dda12f..ea7457023 100644 --- a/posthog/test/mcp/test_pipeline.py +++ b/posthog/test/mcp/test_pipeline.py @@ -821,7 +821,7 @@ def test_sanitize_event_passes_through_non_string_intent(): def test_normalize_caps_long_strings(): out = normalize("x" * 40000) assert out.endswith("...") - assert len(out) == 32_768 + 3 + assert len(out) == 32_768 def test_normalize_detects_cycles(): diff --git a/posthog/test/mcp/test_posthog_mcp.py b/posthog/test/mcp/test_posthog_mcp.py index 51bbb679b..fb2e69961 100644 --- a/posthog/test/mcp/test_posthog_mcp.py +++ b/posthog/test/mcp/test_posthog_mcp.py @@ -783,3 +783,17 @@ async def test_capture_tool_call_records_prepared_conversation_and_session(): props = _events(captured, "$mcp_tool_call")[0]["properties"] assert props["$mcp_conversation_id"] == prepared.conversation_id assert props["$session_id"] == prepared.session_id + + +def test_capture_without_a_session_gets_a_fresh_one_per_call(): + client, captured = make_client() + + client.capture_tool_call("t") + client.capture_tool_call("t") + client.flush() + + first, second = captured + session = first["properties"]["$session_id"] + assert session.startswith("ses_") + assert first["distinct_id"] == session + assert second["properties"]["$session_id"] != session diff --git a/posthog/test/mcp/test_truncation.py b/posthog/test/mcp/test_truncation.py index a0752c6c4..a3d47be49 100644 --- a/posthog/test/mcp/test_truncation.py +++ b/posthog/test/mcp/test_truncation.py @@ -20,7 +20,7 @@ def test_normalize_caps_long_strings(): out = normalize("x" * (MAX_STRING_LENGTH + 100)) - assert out.endswith("...") and len(out) == MAX_STRING_LENGTH + 3 + assert out.endswith("...") and len(out) == MAX_STRING_LENGTH def test_normalize_keeps_bool_distinct_from_int(): @@ -102,9 +102,9 @@ def test_truncate_event_caps_metadata_fields(): "llm_model": "m" * 500, } ) - assert len(out["user_intent"]) == 2048 + 3 - assert len(out["resource_name"]) == 256 + 3 - assert len(out["llm_model"]) == 256 + 3 + assert len(out["user_intent"]) == 2048 + assert len(out["resource_name"]) == 256 + assert len(out["llm_model"]) == 256 def test_truncate_event_caps_exception_value_and_frames(): @@ -119,7 +119,7 @@ def test_truncate_event_caps_exception_value_and_frames(): } ) exc = out["error"]["$exception_list"][0] - assert len(exc["value"]) == 2048 + 3 + assert len(exc["value"]) == 2048 assert exc["stacktrace"]["frames"] == frames[:25] + frames[-25:] @@ -127,7 +127,7 @@ def test_truncate_event_caps_response_text_block(): out = truncate_event( {"response": {"content": [{"type": "text", "text": "t" * 40000}]}} ) - assert len(out["response"]["content"][0]["text"]) == 32_768 + 3 + assert len(out["response"]["content"][0]["text"]) == 32_768 # --- size-targeted budget ---------------------------------------------------- diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py new file mode 100644 index 000000000..8d1396d26 --- /dev/null +++ b/posthog/test/mcp/test_unknown_tool.py @@ -0,0 +1,369 @@ +"""``$mcp_unknown_tool``: a call naming a tool the server never registered is +not a tool call. Runs under both MCP SDK majors over an in-process client, so +the response compared against an uninstrumented twin is the one an agent gets.""" + +import importlib +import importlib.util +import json +from contextlib import asynccontextmanager + +import mcp.types as mcp_types +import pytest + +from posthog.mcp import instrument +from posthog.mcp.types import MCPAnalyticsOptions +from posthog.test.mcp._helpers import ( + MCP_MAJOR, + FakeClient, + events_named, + flush_background, +) + +_needs_standalone = pytest.mark.skipif( + importlib.util.find_spec("fastmcp") is None, reason="standalone fastmcp" +) + + +def _fastmcp_can_disable() -> bool: + try: + from fastmcp import FastMCP + except ImportError: + return False + return hasattr(FastMCP, "disable") + + +_needs_server_disable = pytest.mark.skipif( + not _fastmcp_can_disable(), reason="FastMCP.disable and providers, fastmcp 3+" +) + + +def _high_level(): + if MCP_MAJOR >= 2: + from mcp.server.mcpserver import MCPServer as HighLevel + else: + from mcp.server.fastmcp import FastMCP as HighLevel + server = HighLevel("unknown-tool") + + @server.tool() + def search_docs(query: str) -> str: + return f"found {query}" + + return server + + +def _standalone(*middleware): + """``middleware`` names stock fastmcp middleware as ``"."`` + under ``fastmcp.server.middleware``.""" + from fastmcp import FastMCP + + server = FastMCP("unknown-tool") + + @server.tool + def search_docs(query: str) -> str: + return f"found {query}" + + for path in middleware: + module, name = path.rsplit(".", 1) + stock = importlib.import_module(f"fastmcp.server.middleware.{module}") + server.add_middleware(getattr(stock, name)()) + return server + + +def _standalone_with_disabled_tool(): + server = _standalone() + + @server.tool + def retired(query: str) -> str: + return query + + server.disable(names={"retired"}) + return server + + +def _standalone_with_application_middleware(): + from fastmcp.server.middleware.logging import LoggingMiddleware + + class AuditMiddleware(LoggingMiddleware): + async def on_call_tool(self, context, call_next): + return await call_next(context) + + server = _standalone() + server.add_middleware(AuditMiddleware()) + return server + + +def _mounted(): + """A parent serving a sub-server's tools under the ``ns`` namespace, one + of them disabled on the sub-server.""" + from fastmcp import FastMCP + + sub = FastMCP("sub") + + @sub.tool + def retired(query: str) -> str: + return query + + sub.disable(names={"retired"}) + parent = _standalone() + parent.mount(sub, namespace="ns") + return parent + + +def _with_mounted_server(): + """A parent serving a sub-server's tools under the ``ns`` prefix, on every + fastmcp major (2.x spells the argument ``prefix``, 3.x ``namespace``).""" + from fastmcp import FastMCP + + sub = FastMCP("sub") + + @sub.tool + def lookup(query: str) -> str: + return query + + parent = _standalone() + try: + parent.mount(sub, namespace="ns") + except TypeError: + parent.mount(sub, prefix="ns") + return parent + + +def _raw(listed): + """A raw low-level server whose own dispatcher rejects names it does not + know and advertises ``listed`` on tools/list.""" + from mcp.server.lowlevel import Server + + schema = {"type": "object", "properties": {"query": {"type": "string"}}} + listing = mcp_types.ListToolsResult( + tools=[ + mcp_types.Tool( + name=name, + **{"input_schema" if MCP_MAJOR >= 2 else "inputSchema": schema}, + ) + for name in listed + ] + ) + + def reply(name): + if name not in listed: + raise ValueError(f"Unknown tool: {name}") + return [mcp_types.TextContent(type="text", text="ok")] + + if MCP_MAJOR >= 2: + + async def on_call_tool(ctx, params): + return mcp_types.CallToolResult(content=reply(params.name)) + + async def on_list_tools(ctx, params): + return listing + + return Server( + "unknown-tool", on_call_tool=on_call_tool, on_list_tools=on_list_tools + ) + + server = Server("unknown-tool") + + @server.call_tool() + async def call_tool(name, arguments): + return reply(name) + + async def list_tools(req): + return mcp_types.ServerResult(listing) + + server.request_handlers[mcp_types.ListToolsRequest] = list_tools + return server + + +@asynccontextmanager +async def _connected(server): + if type(server).__module__.startswith("fastmcp"): + from fastmcp import Client + + async with Client(server) as client: + yield lambda name, args: client.call_tool(name, args, raise_on_error=False) + elif MCP_MAJOR >= 2: + from mcp import Client + + async with Client(server) as client: + yield client.call_tool + else: + from mcp.shared.memory import create_connected_server_and_client_session + + async with create_connected_server_and_client_session(server) as session: + yield session.call_tool + + +async def _response(server, name, arguments): + async with _connected(server) as call: + try: + result = await call(name, arguments) + except Exception as error: # noqa: BLE001 - the error is the response + return type(error).__name__, str(error) + is_error = result.is_error if hasattr(result, "is_error") else result.isError + return is_error, [getattr(block, "text", None) for block in result.content] + + +@pytest.mark.parametrize( + "build", + [ + pytest.param(_high_level, id="high-level"), + pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), + *( + pytest.param( + lambda path=path: _standalone(path), + id=f"standalone-fastmcp-with-{path.split('.')[0]}-middleware", + marks=_needs_standalone, + ) + for path in ( + "logging.LoggingMiddleware", + "timing.TimingMiddleware", + "error_handling.ErrorHandlingMiddleware", + ) + ), + ], +) +async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build): + expected = await _response(build(), "no_such_tool", {}) + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + assert await _response(server, "no_such_tool", {}) == expected + await flush_background() + + [unknown] = events_named(client, "$mcp_unknown_tool") + assert unknown["properties"]["$mcp_tool_name"] == "no_such_tool" + assert unknown["properties"]["$session_id"] + assert events_named(client, "$mcp_tool_call") == [] + assert events_named(client, "$exception") == [] + + +@pytest.mark.parametrize( + "build, name, arguments", + [ + pytest.param(_high_level, "search_docs", {}, id="invalid-arguments"), + pytest.param( + _standalone_with_disabled_tool, + "retired", + {"query": "q"}, + id="disabled-standalone-tool", + marks=_needs_server_disable, + ), + pytest.param( + _mounted, + "ns_retired", + {"query": "q"}, + id="disabled-tool-on-mounted-server", + marks=_needs_server_disable, + ), + pytest.param( + _mounted, + "ns_no_such_tool", + {}, + id="server-with-mounted-provider", + marks=_needs_server_disable, + ), + pytest.param( + _with_mounted_server, + "ns_no_such_tool", + {}, + id="server-with-mounted-server", + marks=_needs_standalone, + ), + pytest.param( + _standalone_with_application_middleware, + "no_such_tool", + {}, + id="standalone-with-application-middleware", + marks=_needs_standalone, + ), + pytest.param( + lambda: _raw(["search_docs"]), + "no_such_tool", + {}, + id="raw-low-level-unknown-name", + ), + ], +) +async def test_a_call_the_sdk_cannot_rule_out_is_a_failed_tool_call( + build, name, arguments +): + expected = await _response(build(), name, arguments) + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + assert await _response(server, name, arguments) == expected + await flush_background() + + [call] = events_named(client, "$mcp_tool_call") + assert call["properties"]["$mcp_tool_name"] == name + assert call["properties"]["$mcp_is_error"] is True + assert events_named(client, "$mcp_unknown_tool") == [] + + +_SHAPES = [ + pytest.param(_high_level, id="high-level"), + pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), +] + + +@pytest.mark.parametrize( + "build", [*_SHAPES, pytest.param(lambda: _raw(["search_docs"]), id="raw-low-level")] +) +async def test_a_call_naming_no_tool_emits_nothing(build): + expected = await _response(build(), "", {}) + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + assert await _response(server, "", {}) == expected + await flush_background() + + assert [e["event"] for e in client.events if e["event"] != "$mcp_initialize"] == [] + + +@pytest.mark.parametrize("build", _SHAPES) +async def test_an_unknown_tool_name_is_cut_to_256_characters(build): + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + await _response(server, "x" * 5000, {}) + await flush_background() + + [unknown] = events_named(client, "$mcp_unknown_tool") + name = unknown["properties"]["$mcp_tool_name"] + assert len(name) == 256 + assert name.startswith("x" * 200) + + +@pytest.mark.parametrize( + "build, delivers_handle", + [ + pytest.param( + _standalone, True, id="standalone-error-result", marks=_needs_standalone + ), + pytest.param(_high_level, False, id="high-level-raise"), + ], +) +async def test_an_unknown_tool_result_carries_the_conversation_handle_it_delivers( + build, delivers_handle +): + client = FakeClient() + server = build() + instrument(server, client) + + _, texts = await _response(server, "no_such_tool", {}) + await flush_background() + + handles = [ + json.loads(text)["conversation_id"] + for text in texts + if text and text.startswith('{"conversation_id"') + ] + [unknown] = events_named(client, "$mcp_unknown_tool") + assert len(handles) == int(delivers_handle) + assert unknown["properties"].get("$mcp_conversation_id") == next( + iter(handles), None + )