From 8040746b7ed8353873fea7a49955369a6ffe5c24 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 14:10:51 -0300 Subject: [PATCH 1/7] feat(mcp): align posthog.mcp with the sdk-specs MCP spec before v1 What: - get_more_tools and send_feedback annotate readOnlyHint, destructiveHint, idempotentHint and openWorldHint as false, with no title, so the spec's exact annotations JSON matches. Hosts cannot override them. - The injected `context` argument uses the spec's shared description string. - `llm_model` is never added to a schema's `required` list. - A tools/call naming a tool the server never registered sends $mcp_unknown_tool (tool name, identity, session) instead of $mcp_tool_call and $exception. The client gets the same response. A disabled tool, invalid arguments, a raw server with no tools/list handler or a name missing from a paginated first page, and standalone FastMCP with dispatch-overriding middleware stay failed tool calls. PostHogMCP.capture_unknown_tool() covers custom dispatchers. - Every adapter's raise path and isError-result path calls one ToolCallLifecycle.record_if_unknown_tool(). The standalone FastMCP registry lookup and its middleware check moved to _instrumentation.py, so _instrument_v2 no longer imports from _instrument_lowlevel. - Minor sampo changeset. Why: posthog.mcp is about to be declared stable. Customers' insights over $mcp_* events must not shift after that, so every SDK matches the spec now. Tested: - pytest posthog/test/mcp: 678 passed, 10 skipped - annotations test compares the exact JSON; fails 2/4 with a title present - ruff format --check . and ruff check .: clean - mypy --no-site-packages --config-file mypy.ini . | mypy-baseline filter: clean - python -W error -c "import posthog": ok; make public_api_check: up to date - full pytest without provider API keys: 6142 passed, 3 failed in test_metrics_autocapture.py fork tests (macOS objc fork-safety abort, unrelated to this change) Reviewer notes: _standalone_tool_exists catches the cached fastmcp not-found errors, which include DisabledError; get_tool raises only NotFoundError in fastmcp 2.14 and 3.2, so a disabled tool still reads as existing. Client conversation ids are untouched (posthog-python#1193). --- .sampo/changesets/mcp-v1-spec-align.md | 12 ++ posthog/mcp/README.md | 27 +++ posthog/mcp/_event_types.py | 1 + posthog/mcp/_instrument_fastmcp.py | 4 + posthog/mcp/_instrument_lowlevel.py | 79 ++------- posthog/mcp/_instrument_v2.py | 29 +++- posthog/mcp/_instrumentation.py | 176 +++++++++++++++++++- posthog/mcp/_internal.py | 33 +++- posthog/mcp/_posthog_events.py | 9 +- posthog/mcp/_sanitization.py | 11 +- posthog/mcp/constants.py | 1 + posthog/mcp/posthog_mcp.py | 37 +++++ posthog/test/mcp/test_lowlevel.py | 5 +- posthog/test/mcp/test_unknown_tool.py | 217 +++++++++++++++++++++++++ posthog/test/mcp/test_v2_lowlevel.py | 14 +- references/public_api_snapshot.txt | 2 + 16 files changed, 561 insertions(+), 96 deletions(-) create mode 100644 .sampo/changesets/mcp-v1-spec-align.md create mode 100644 posthog/test/mcp/test_unknown_tool.py diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md new file mode 100644 index 000000000..c6460362a --- /dev/null +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -0,0 +1,12 @@ +--- +pypi/posthog: minor +--- + +MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: + +- A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. +- `get_more_tools` and `send_feedback` declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as `false`, and their annotations no longer carry a `title`. +- The injected `context` argument has a shorter description, the same in every PostHog MCP SDK. +- The injected `llm_model` argument is always optional. + +The annotations and descriptions change your published tool definitions. If your server is listed in the ChatGPT or Claude directories, those platforms may re-review the updated tools. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index 58f177233..2b60c71ba 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -339,11 +339,38 @@ call = posthog.prepare_tool_call( ) ``` +## Calls to tools your server does not have + +A `tools/call` that names a tool your server never registered is not a tool +call. `instrument()` sends `$mcp_unknown_tool` with the requested name as +`$mcp_tool_name`, and no `$mcp_tool_call` or `$exception`. The agent gets the +same error as before. The SDK decides only after the call fails, from the tool +registry: + +- High-level servers and standalone `fastmcp.FastMCP` ask their registry. A + disabled tool still exists, so calling it is a failed tool call. +- A raw low-level server's registry is its own `tools/list` handler. A failed + call to a name that handler does not list is unknown. With no `tools/list` + handler, or a name missing from a first page that has more pages after it, the + SDK cannot tell and records a failed tool call. +- On standalone `fastmcp.FastMCP`, middleware that overrides a listing or + dispatch hook can serve tools the registry lacks, so the SDK records a failed + tool call. + +Invalid arguments for a tool that exists are a failed tool call. A custom +dispatcher calls `posthog.capture_unknown_tool(tool_name, session_id=...)` in +place of `capture_tool_call` when its own lookup finds no tool. + ## Virtual tools on a paginated `tools/list` `instrument()` advertises up to two tools of its own: `get_more_tools` (`report_missing=True`) and `send_feedback` (`collect_feedback=True`). +Both declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and +`openWorldHint` as `false`, and no option changes that. A call records one event +in your PostHog project: it writes outside the conversation, deletes nothing, +records again on every repeat, and reaches nothing else. + A client concatenates every `tools/list` page into one list, so each virtual tool is appended to the **first page only** — the page every client reads, including clients that never follow `nextCursor`. "First page" means a request with no diff --git a/posthog/mcp/_event_types.py b/posthog/mcp/_event_types.py index 7392f90a9..005d08457 100644 --- a/posthog/mcp/_event_types.py +++ b/posthog/mcp/_event_types.py @@ -23,3 +23,4 @@ class MCPAnalyticsEventType: MCP_RESOURCES_READ = "mcp:resources/read" MCP_TOOLS_CALL = "mcp:tools/call" MCP_TOOLS_LIST = "mcp:tools/list" + MCP_UNKNOWN_TOOL = "mcp:unknown_tool" diff --git a/posthog/mcp/_instrument_fastmcp.py b/posthog/mcp/_instrument_fastmcp.py index 78bd1324f..366060141 100644 --- a/posthog/mcp/_instrument_fastmcp.py +++ b/posthog/mcp/_instrument_fastmcp.py @@ -167,6 +167,10 @@ async def wrapped( name, call_arguments, context=context, convert_result=convert_result ) except Exception as error: + if await lifecycle.record_if_unknown_tool( + _name_owned_by_real_tool(server, name) + ): + raise # The minted prompt-back was never delivered to the agent — don't stamp # an orphan conversation_id it can't echo (an agent-supplied id is kept). await lifecycle.record_error(error, (time.monotonic() - start) * 1000) diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index 00bd4ef4d..14563b8ca 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -14,11 +14,9 @@ from __future__ import annotations -import functools import inspect import time -from functools import lru_cache -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional, Tuple import mcp.types as mcp_types @@ -30,15 +28,19 @@ from ._conversation_id import build_prompt_back from ._event_types import MCPAnalyticsEventType from ._instrumentation import ( - advertised_tool_names, apply_virtual_tool_injection, collect_listed_tools, copy_tools_list_result, extract_tools, + first_listing_page, is_first_listing_page, + is_tool_result_error, mutate_tool_schema, prepare_request, + _dispatch_can_differ, + _tool_lookup_not_found_errors, raw_listing_owns_tool_name, + registered_tool_exists, record_resource_request, request_to_dict, resource_listing_response, @@ -48,7 +50,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) -from ._internal import MCPAnalyticsData +from ._internal import FirstListingPage, MCPAnalyticsData from ._model_parameters import request_meta_from_context from ._model_parameters import can_inject_model_parameter, is_capture_model_enabled from ._output_instructions import mirror_instructions_into_structured_content @@ -282,6 +284,10 @@ async def handler(req: Any) -> Any: try: result = await original(req) except Exception as error: + if await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, high_level, name) + ): + raise # The @server.call_tool() decorator converts raises into # CallToolResult(isError=True), but a handler wired straight into # request_handlers can raise — capture before re-raising so the failed @@ -294,6 +300,10 @@ async def handler(req: Any) -> Any: # The low-level handler already converted any exception to a # CallToolResult(isError=True); record_tool_call detects that from the result. call_result = getattr(result, "root", result) + if is_tool_result_error(call_result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, high_level, name) + ): + return result # Deliver the handle before capture, over both channels a result has: # mirrored into structuredContent on every response (for tools whose @@ -401,8 +411,8 @@ def _wrap_list_tools( if original is None or getattr(original, _WRAPPED_FLAG, False): return - async def probe_raw_tool_names(_ctx: Any = None) -> Optional[Set[str]]: - """The names the host's own handler advertises on its first page. Calls + async def probe_raw_tool_names(_ctx: Any = None) -> Optional[FirstListingPage]: + """The first page the host's own handler serves. Calls ``original``, not the wrapper, so the probe never recurses into instrumentation or appends a virtual tool. Read by ``_name_owned_by_real_tool`` on raw low-level servers, which have no @@ -442,7 +452,7 @@ async def probe_raw_tool_names(_ctx: Any = None) -> Optional[Set[str]]: context_required=context_required, high_level=high_level, ) - return advertised_tool_names(tools) + return first_listing_page(result) data.raw_tool_names_probe = probe_raw_tool_names @@ -572,26 +582,6 @@ async def _name_owned_by_real_tool( return await raw_listing_owns_tool_name(data, name, server) -@lru_cache(maxsize=1) -def _tool_lookup_not_found_errors() -> Tuple[type, ...]: - """The fastmcp exceptions that mean "no live tool by that name" -- an answer. - Anything else out of ``get_tool`` is the lookup itself failing. Empty when - fastmcp is absent or has moved them, which makes every failure delegate: - the safe direction.""" - try: - from fastmcp import exceptions - except Exception: # noqa: BLE001 - no fastmcp on this path - return () - return tuple( - err - for err in ( - getattr(exceptions, "NotFoundError", None), - getattr(exceptions, "DisabledError", None), - ) - if isinstance(err, type) and issubclass(err, BaseException) - ) - - _INJECTED_KEYS = ("context", "conversation_id", "llm_model") @@ -769,39 +759,6 @@ def _server_dereferences(server: Any) -> bool: ) -_DISPATCH_HOOKS = ("on_message", "on_request", "on_list_tools", "on_call_tool") - - -def _dispatch_can_differ(server: Any) -> bool: - """Whether application middleware can provide, shadow, or reroute a tool, - making the registry an unreliable witness for what actually runs. Any - override of a listing or dispatch hook can; FastMCP's own built-ins are - excluded. Without a trustworthy registry a cold instance keeps ``llm_model`` - (strips fail closed), which is exactly what it did before this path.""" - try: - from fastmcp.server.middleware import Middleware - except ImportError: # FastMCP before middleware existed: nothing can differ - return False - - return any( - type(middleware) not in _builtin_middleware_types() # subclasses are the app's - and any( - getattr(type(middleware), hook) is not getattr(Middleware, hook) - for hook in _DISPATCH_HOOKS - ) - for middleware in getattr(server, "middleware", ()) - ) - - -@functools.lru_cache(maxsize=1) -def _builtin_middleware_types() -> tuple: - """The middleware a bare ``FastMCP()`` installs on its own, probed rather - than named so a new built-in in a later release is still recognised.""" - from fastmcp import FastMCP - - return tuple(type(middleware) for middleware in FastMCP("posthog-probe").middleware) - - def _reference_chain(schema: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: """The schema and each local root ``$ref`` target in turn; ``None`` for a reference that is external, dangling, or cyclic.""" diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 58d45f778..751518c8e 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -34,7 +34,7 @@ from collections.abc import Mapping from dataclasses import replace from datetime import datetime, timezone -from typing import Any, Dict, FrozenSet, Optional, Set, Tuple +from typing import Any, Dict, FrozenSet, Optional, Tuple import mcp.types as mcp_types @@ -46,17 +46,19 @@ from ._conversation_id import build_prompt_back from ._event_types import MCPAnalyticsEventType from ._instrumentation import ( - advertised_tool_names, apply_virtual_tool_injection, collect_listed_tools, _is_sse_request, _to_jsonable, copy_tools_list_result, + first_listing_page, is_first_listing_page, + is_tool_result_error, mutate_tool_schema, params_to_request_dict, prepare_request, raw_listing_owns_tool_name, + registered_tool_exists, record_initialize, record_resource_request, resource_listing_response, @@ -66,7 +68,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) -from ._internal import MCPAnalyticsData +from ._internal import FirstListingPage, MCPAnalyticsData from ._model_parameters import ( can_inject_model_parameter, is_capture_model_enabled, @@ -450,6 +452,10 @@ async def wrapped( name, call_arguments, context=context, convert_result=convert_result ) except Exception as error: + if await lifecycle.record_if_unknown_tool( + _name_owned_by_real_tool_v2(server, name) + ): + raise # The outer MCPServer layer converts the raise after this seam; no # freshly minted handle could have been delivered yet. await lifecycle.record_error(error, (time.monotonic() - start) * 1000) @@ -669,11 +675,19 @@ async def handler(ctx: Any, params: Any) -> Any: try: result = await original(ctx, params) except Exception as error: + if await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, standalone, name, ctx) + ): + raise # Raw v2 handlers raise through to JSON-RPC errors; no freshly minted # handle could have been delivered yet. await lifecycle.record_error(error, (time.monotonic() - start) * 1000) raise duration_ms = (time.monotonic() - start) * 1000 + if is_tool_result_error(result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, standalone, name, ctx) + ): + return result delivered = False if lifecycle.conversation_id: @@ -774,11 +788,10 @@ def _wrap_v2_list_tools( return original = entry.handler - async def probe_raw_tool_names(ctx: Any = None) -> Optional[Set[str]]: - """The host's own first-page tool names — ``original``, not the wrapper, - so the probe never recurses or appends a virtual tool.""" - result = await original(ctx, None) - return advertised_tool_names(list(getattr(result, "tools", []) or [])) + async def probe_raw_tool_names(ctx: Any = None) -> Optional[FirstListingPage]: + """The host's own first page — ``original``, not the wrapper, so the + probe never recurses or appends a virtual tool.""" + return first_listing_page(await original(ctx, None)) data.raw_tool_names_probe = probe_raw_tool_names diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index ae8d7b30b..c8a197bfc 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -11,11 +11,12 @@ import asyncio import concurrent.futures import copy +import functools import os import threading from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, FrozenSet, List, Literal, Optional, Set +from typing import Any, Dict, FrozenSet, List, Literal, Optional, Set, Tuple, Type from ._capture import capture_event from ._context_parameters import ( @@ -41,7 +42,12 @@ resolve_send_feedback_tool_name, ) from ._intent import resolve_tool_call_intent, set_event_intent -from ._internal import MCPAnalyticsData, handle_identify, resolve_event_properties +from ._internal import ( + FirstListingPage, + MCPAnalyticsData, + handle_identify, + resolve_event_properties, +) from ._tool_input import get_tool_input_properties from ._model_parameters import ( add_model_parameter_to_schema, @@ -615,6 +621,27 @@ async def record_feedback(self, *, conversation_id_delivered: bool = False) -> s ) return await handle_feedback(report, self.feedback_options) + async def record_if_unknown_tool(self, tool_exists: Optional[bool]) -> bool: + """Record the failed call as ``$mcp_unknown_tool`` when ``tool_exists`` + proves the server has no tool by that name, and say whether it did. The + adapter then re-raises or returns the host's response unchanged.""" + if tool_exists is not False: + return False + conversation_id = self._anchored_conversation_id(False) + session_id = await self.prepare_session(conversation_id) + await record_unknown_tool( + self.data, + session_id, + conversation_id=conversation_id, + name=self.name, + arguments=self.arguments, + client_name=self.client_name, + client_version=self.client_version, + protocol_version=self.protocol_version, + extra=self.extra, + ) + return True + async def record_error(self, error: Any, duration_ms: float) -> None: # A freshly minted handle cannot anchor or be captured when dispatch # raised: no adapter had an opportunity to deliver it to the agent. @@ -967,6 +994,18 @@ def is_first_listing_page(params: Any) -> bool: return getattr(params, "cursor", None) is None +def first_listing_page(result: Any) -> FirstListingPage: + """The names a host's first ``tools/list`` page advertises, and whether a + cursor leads to more. 1.x wraps the result in a ``ServerResult`` and spells + the cursor ``nextCursor``; 2.x returns it bare as ``next_cursor``.""" + root = getattr(result, "root", result) + cursor = getattr(root, "next_cursor", getattr(root, "nextCursor", None)) + return FirstListingPage( + advertised_tool_names(list(getattr(root, "tools", []) or [])), + cursor is not None, + ) + + def advertised_tool_names(tools: list) -> Set[str]: """The names a listing page advertises. The virtual tools are appended to a *copy* of the host's result, so a page always reads back as the host wrote @@ -1045,26 +1084,116 @@ def resolve_virtual_tool_injection( async def raw_listing_owns_tool_name( - data: MCPAnalyticsData, name: str, ctx: Any = None + data: MCPAnalyticsData, name: str, ctx: Any = None, *, whole_catalogue: bool = False ) -> Optional[bool]: """Whether the host's *own* ``tools/list`` handler advertises ``name`` on its first page, asked at call time. For the raw low-level paths, 1.x and v2, - which have no tool registry to query instead. + which have no tool registry to query instead. With ``whole_catalogue``, a + name missing from a page that has more pages after it is unanswered. Tri-state. ``True``/``False`` are answers; ``None`` means the question could not be asked, and callers must not intercept on it -- guessing "not owned" would swallow a real tool of the host's. Runs once per call to a virtual - tool's name, never for ordinary traffic. + tool's name and per failed call, never for ordinary traffic. """ probe = data.raw_tool_names_probe if probe is None: return None try: - names = await probe(ctx) + page = await probe(ctx) except Exception as err: # noqa: BLE001 - analytics must not break the call warn_ownership_lookup_failed(name, err) return None - return None if names is None else name in names + if page is None: + return None + if name in page.names: + return True + return None if whole_catalogue and page.has_more else False + + +async def registered_tool_exists( + data: MCPAnalyticsData, standalone: Any, name: str, ctx: Any = None +) -> Optional[bool]: + """Whether the server registers ``name`` at all, asked after a failed call + to tell an unknown tool from a failed one. jlowin's standalone FastMCP asks + its registry; a raw low-level server asks its whole ``tools/list`` + catalogue. ``None`` when neither can say.""" + if standalone is not None: + return await _standalone_tool_exists(standalone, name) + return await raw_listing_owns_tool_name(data, name, ctx, whole_catalogue=True) + + +async def _standalone_tool_exists(server: Any, name: str) -> Optional[bool]: + """Whether jlowin's FastMCP registers ``name``, enabled or not. A disabled + tool still exists: calling it is a failed call. Middleware that can serve + tools the registry lacks makes the registry no witness.""" + if _dispatch_can_differ(server): + return None + try: + if await server.get_tool(name) is not None: + return True + from fastmcp import FastMCP + + # From 3.x, get_tool hides a disabled tool; the providers still hold it. + return await super(FastMCP, server).get_tool(name) is not None + except _tool_lookup_not_found_errors(): + return False + except Exception as err: # noqa: BLE001 - analytics must not break the call + log(f"PostHog MCP: could not look up tool {name!r} - {err}") + return None + + +@functools.lru_cache(maxsize=1) +def _tool_lookup_not_found_errors() -> Tuple[Type[BaseException], ...]: + """The fastmcp exceptions that mean "no live tool by that name" -- an answer. + Anything else out of ``get_tool`` is the lookup itself failing. Empty when + fastmcp is absent or has moved them, which makes every failure delegate: + the safe direction.""" + try: + from fastmcp import exceptions + except Exception: # noqa: BLE001 - no fastmcp on this path + return () + return tuple( + err + for err in ( + getattr(exceptions, "NotFoundError", None), + getattr(exceptions, "DisabledError", None), + ) + if isinstance(err, type) and issubclass(err, BaseException) + ) + + +_DISPATCH_HOOKS = ("on_message", "on_request", "on_list_tools", "on_call_tool") + + +def _dispatch_can_differ(server: Any) -> bool: + """Whether application middleware can provide, shadow, or reroute a tool, + making the registry an unreliable witness for what actually runs. Any + override of a listing or dispatch hook can; FastMCP's own built-ins are + excluded. Without a trustworthy registry a cold instance keeps ``llm_model`` + (strips fail closed), which is exactly what it did before this path.""" + try: + from fastmcp.server.middleware import Middleware + except ImportError: # FastMCP before middleware existed: nothing can differ + return False + + return any( + type(middleware) not in _builtin_middleware_types() # subclasses are the app's + and any( + getattr(type(middleware), hook) is not getattr(Middleware, hook) + for hook in _DISPATCH_HOOKS + ) + for middleware in getattr(server, "middleware", ()) + ) + + +@functools.lru_cache(maxsize=1) +def _builtin_middleware_types() -> tuple: + """The middleware a bare ``FastMCP()`` installs on its own, probed rather + than named so a new built-in in a later release is still recognised.""" + from fastmcp import FastMCP + + return tuple(type(middleware) for middleware in FastMCP("posthog-probe").middleware) def warn_ownership_lookup_failed(name: str, err: Exception) -> None: @@ -1374,6 +1503,39 @@ async def record_missing_capability( log(f"record_missing_capability failed (event dropped): {err}") +async def record_unknown_tool( + data: MCPAnalyticsData, + session_id: str, + *, + conversation_id: Optional[str], + name: str, + arguments: Optional[Dict[str, Any]], + client_name: Optional[str], + client_version: Optional[str], + protocol_version: Optional[str], + extra: Optional[Dict[str, Any]], +) -> None: + """Record a call naming a tool the server never registered as + ``$mcp_unknown_tool``. It is not a tool call, so it carries no outcome.""" + try: + event: Dict[str, Any] = { + "event_type": MCPAnalyticsEventType.MCP_UNKNOWN_TOOL, + "session_id": session_id, + "conversation_id": conversation_id, + "resource_name": name, + "client_name": client_name, + "client_version": client_version, + "protocol_version": protocol_version, + } + await _apply_event_properties( + data, event, build_tool_call_request(name, arguments), extra + ) + stamp_transport_identity(event, extra) + fire_and_forget(capture_event(data, event), data) + except Exception as err: # noqa: BLE001 - isolate analytics from the tool path + log(f"record_unknown_tool failed (event dropped): {err}") + + async def record_feedback( data: MCPAnalyticsData, session_id: str, diff --git a/posthog/mcp/_internal.py b/posthog/mcp/_internal.py index 44a1eeb48..58d907ddc 100644 --- a/posthog/mcp/_internal.py +++ b/posthog/mcp/_internal.py @@ -16,13 +16,28 @@ from collections import OrderedDict from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import Any, Awaitable, Callable, Dict, FrozenSet, Optional, Set, Tuple +from typing import ( + Any, + Awaitable, + Callable, + Dict, + FrozenSet, + NamedTuple, + Optional, + Set, + Tuple, +) from .logger import log from ._sink import McpEventSink from .types import MCPAnalyticsOptions, UserIdentity +class FirstListingPage(NamedTuple): + names: Set[str] + has_more: bool + + class IdentityCache: """Bounded LRU of session identities, isolated per server so identities never bleed across server instances.""" @@ -75,14 +90,14 @@ class MCPAnalyticsData: warned_virtual_tool_collisions: Set[Tuple[str, str, str]] = field( default_factory=set ) - # Adapter-supplied probe returning the names the host's own (original, - # un-instrumented) tools/list handler advertises on its first page, or None - # when it can't be determined. Registered by the adapters with no tool - # registry to query — raw low-level servers. Takes the adapter's request - # context, which the 1.x handler shape ignores. - raw_tool_names_probe: Optional[Callable[[Any], Awaitable[Optional[Set[str]]]]] = ( - None - ) + # Adapter-supplied probe returning the first page the host's own (original, + # un-instrumented) tools/list handler serves, or None when it can't be + # determined. Registered by the adapters with no tool registry to query — + # raw low-level servers. Takes the adapter's request context, which the 1.x + # handler shape ignores. + raw_tool_names_probe: Optional[ + Callable[[Any], Awaitable[Optional[FirstListingPage]]] + ] = None last_activity: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) identified_sessions: IdentityCache = field(default_factory=IdentityCache) tool_categories: Dict[str, str] = field(default_factory=dict) diff --git a/posthog/mcp/_posthog_events.py b/posthog/mcp/_posthog_events.py index 51fc5ff75..7b087fd2b 100644 --- a/posthog/mcp/_posthog_events.py +++ b/posthog/mcp/_posthog_events.py @@ -29,6 +29,7 @@ MCPAnalyticsEventType.MCP_RESOURCES_READ: PostHogMCPAnalyticsEvent.RESOURCE_READ, MCPAnalyticsEventType.MCP_TOOLS_CALL: PostHogMCPAnalyticsEvent.TOOL_CALL, MCPAnalyticsEventType.MCP_TOOLS_LIST: PostHogMCPAnalyticsEvent.TOOLS_LIST, + MCPAnalyticsEventType.MCP_UNKNOWN_TOOL: PostHogMCPAnalyticsEvent.UNKNOWN_TOOL, } _P = PostHogMCPAnalyticsProperty @@ -112,8 +113,12 @@ def _is_tool_call(event: Event) -> bool: def _add_common_properties(event: Event, properties: Dict[str, Any]) -> None: if event.get("resource_name"): - properties[_P.RESOURCE_NAME] = event["resource_name"] - if _is_tool_call(event): + is_unknown_tool = ( + event.get("event_type") == MCPAnalyticsEventType.MCP_UNKNOWN_TOOL + ) + if not is_unknown_tool: + properties[_P.RESOURCE_NAME] = event["resource_name"] + if is_unknown_tool or _is_tool_call(event): properties[_P.TOOL_NAME] = event["resource_name"] if event.get("tool_description") and _is_tool_call(event): properties[_P.TOOL_DESCRIPTION] = event["tool_description"] diff --git a/posthog/mcp/_sanitization.py b/posthog/mcp/_sanitization.py index b14f4796d..426d2d29f 100644 --- a/posthog/mcp/_sanitization.py +++ b/posthog/mcp/_sanitization.py @@ -14,6 +14,8 @@ from typing import Any, Dict, List, Optional, Set, Tuple from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit, urlunsplit +from ._event_types import MCPAnalyticsEventType + # SDK-injected arguments stripped from captured $mcp_parameters (they surface as # dedicated properties: $mcp_intent and $mcp_conversation_id). _INJECTED_ARGUMENT_NAMES = ("context", "conversation_id") @@ -628,8 +630,15 @@ def sanitize_event(event: Dict[str, Any]) -> Dict[str, Any]: if result.get("parameters") is not None: result["parameters"] = sanitize_captured_value(result["parameters"]) + # An unknown tool's name is whatever the agent sent, not one of the + # server's identifiers, so it gets the free-text pass. if result.get("resource_name") is not None: - result["resource_name"] = _sanitize_resource_name(result["resource_name"]) + sanitize_name = ( + sanitize_free_text + if result.get("event_type") == MCPAnalyticsEventType.MCP_UNKNOWN_TOOL + else _sanitize_resource_name + ) + result["resource_name"] = sanitize_name(result["resource_name"]) # The intent comes straight from an agent-narrated `context` string, so it # can contain a secret the LLM read aloud or personal data it narrated about diff --git a/posthog/mcp/constants.py b/posthog/mcp/constants.py index 5fc725b35..345536e30 100644 --- a/posthog/mcp/constants.py +++ b/posthog/mcp/constants.py @@ -55,6 +55,7 @@ class PostHogMCPAnalyticsEvent: RESOURCES_LIST = "$mcp_resources_list" TOOL_CALL = "$mcp_tool_call" TOOLS_LIST = "$mcp_tools_list" + UNKNOWN_TOOL = "$mcp_unknown_tool" class PostHogMCPAnalyticsProperty: diff --git a/posthog/mcp/posthog_mcp.py b/posthog/mcp/posthog_mcp.py index 69951eade..8f75fbf74 100644 --- a/posthog/mcp/posthog_mcp.py +++ b/posthog/mcp/posthog_mcp.py @@ -105,6 +105,7 @@ class _ConversationOwnership: class PostHogMCP(Client): """A drop-in posthog ``Client`` with ``capture_tool_call`` / ``capture_initialize`` / ``capture_tools_list`` / ``capture_missing_capability`` / ``capture_feedback`` + / ``capture_unknown_tool`` plus ``prepare_tool_list``, ``prepare_tool_call`` and ``prepare_tool_result`` helpers. ``capture`` adds the configured server build to custom events. Other inherited methods work unchanged.""" @@ -371,6 +372,42 @@ def capture_missing_capability( _apply_model(event, llm_model, llm_model_source) self._emit(event) + def capture_unknown_tool( + self, + tool_name: str, + *, + protocol_version: Optional[str] = None, + distinct_id: Optional[str] = None, + session_id: Optional[str] = None, + conversation_id: Optional[str] = None, + client_user_agent: Optional[str] = None, + vendor_client: Optional[str] = None, + set_properties: Optional[JsonRecord] = None, + groups: Optional[Dict[str, str]] = None, + properties: Optional[JsonRecord] = None, + timestamp: Optional[datetime] = None, + ) -> None: + """Capture a call naming a tool your server does not have. Emits + ``$mcp_unknown_tool`` with the requested name as ``$mcp_tool_name``. + Capture it instead of :meth:`capture_tool_call`, not as well: it is + not a tool call. A tool that exists, disabled or called with invalid + arguments, is a failed tool call.""" + event = self._base_event( + MCPAnalyticsEventType.MCP_UNKNOWN_TOOL, + distinct_id, + session_id, + conversation_id, + set_properties, + groups, + properties, + timestamp, + client_user_agent, + vendor_client, + ) + event["resource_name"] = tool_name + event["protocol_version"] = protocol_version + self._emit(event) + def capture_feedback( self, *, diff --git a/posthog/test/mcp/test_lowlevel.py b/posthog/test/mcp/test_lowlevel.py index 62dbd1ed0..dbc79ff2b 100644 --- a/posthog/test/mcp/test_lowlevel.py +++ b/posthog/test/mcp/test_lowlevel.py @@ -36,7 +36,7 @@ async def list_tools(): @server.call_tool() async def call_tool(name, arguments): - if name == "echo": + if name == "echo" and arguments.get("msg") != "boom": return [mcp_types.TextContent(type="text", text=str(arguments.get("msg")))] raise ValueError("boom") @@ -350,9 +350,8 @@ async def test_tool_call_error_captured_from_is_error_result(): instrument(server, client) handler = server.request_handlers[mcp_types.CallToolRequest] - # "fail" is unlisted -> no validation -> handler raises -> isError result result = await handler( - _call_request("fail", {"context": "trying a tool that errors"}) + _call_request("echo", {"msg": "boom", "context": "trying a tool that errors"}) ) await _flush() diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py new file mode 100644 index 000000000..b0a7e9b1e --- /dev/null +++ b/posthog/test/mcp/test_unknown_tool.py @@ -0,0 +1,217 @@ +"""``$mcp_unknown_tool``: a call naming a tool the server never registered is +not a tool call. Runs under both MCP SDK majors over an in-process client, so +the response compared against an uninstrumented twin is the one an agent gets.""" + +import importlib.util +from contextlib import asynccontextmanager + +import mcp.types as mcp_types +import pytest + +from posthog.mcp import PostHogMCP, instrument +from posthog.mcp.types import MCPAnalyticsOptions +from posthog.test.mcp._helpers import ( + MCP_MAJOR, + FakeClient, + events_named, + flush_background, +) + +_needs_standalone = pytest.mark.skipif( + importlib.util.find_spec("fastmcp") is None, reason="standalone fastmcp" +) + + +def _high_level(): + if MCP_MAJOR >= 2: + from mcp.server.mcpserver import MCPServer as HighLevel + else: + from mcp.server.fastmcp import FastMCP as HighLevel + server = HighLevel("unknown-tool") + + @server.tool() + def search_docs(query: str) -> str: + return f"found {query}" + + return server + + +def _standalone(): + from fastmcp import FastMCP + + server = FastMCP("unknown-tool") + + @server.tool + def search_docs(query: str) -> str: + return f"found {query}" + + @server.tool + def retired(query: str) -> str: + return query + + server.disable(names={"retired"}) + return server + + +def _raw(pages): + """A raw low-level server whose own dispatcher rejects names it does not + know. ``pages`` are the tools/list pages of tool names, chained by cursor; + ``None`` registers no tools/list handler at all.""" + from mcp.server.lowlevel import Server + + registered = {name for page in pages or [] for name in page} + schema = {"type": "object", "properties": {"query": {"type": "string"}}} + + def page_at(cursor): + index = int(cursor) if cursor else 0 + tools = [ + mcp_types.Tool( + name=name, + **{"input_schema" if MCP_MAJOR >= 2 else "inputSchema": schema}, + ) + for name in pages[index] + ] + next_cursor = str(index + 1) if index + 1 < len(pages) else None + if MCP_MAJOR >= 2: + return mcp_types.ListToolsResult(tools=tools, next_cursor=next_cursor) + return mcp_types.ListToolsResult(tools=tools, nextCursor=next_cursor) + + def reply(name): + if name not in registered: + raise ValueError(f"Unknown tool: {name}") + return [mcp_types.TextContent(type="text", text="ok")] + + if MCP_MAJOR >= 2: + + async def on_call_tool(ctx, params): + return mcp_types.CallToolResult(content=reply(params.name)) + + async def on_list_tools(ctx, params): + return page_at(getattr(params, "cursor", None)) + + handlers = {"on_call_tool": on_call_tool} + if pages is not None: + handlers["on_list_tools"] = on_list_tools + return Server("unknown-tool", **handlers) + + server = Server("unknown-tool") + + @server.call_tool() + async def call_tool(name, arguments): + return reply(name) + + if pages is not None: + + async def list_tools(req): + params = getattr(req, "params", None) if req else None + return mcp_types.ServerResult(page_at(getattr(params, "cursor", None))) + + server.request_handlers[mcp_types.ListToolsRequest] = list_tools + return server + + +@asynccontextmanager +async def _connected(server): + if type(server).__module__.startswith("fastmcp"): + from fastmcp import Client + + async with Client(server) as client: + yield lambda name, args: client.call_tool(name, args, raise_on_error=False) + elif MCP_MAJOR >= 2: + from mcp import Client + + async with Client(server) as client: + yield client.call_tool + else: + from mcp.shared.memory import create_connected_server_and_client_session + + async with create_connected_server_and_client_session(server) as session: + yield session.call_tool + + +async def _response(server, name, arguments): + async with _connected(server) as call: + try: + result = await call(name, arguments) + except Exception as error: # noqa: BLE001 - the error is the response + return type(error).__name__, str(error) + is_error = getattr(result, "is_error", getattr(result, "isError", None)) + return is_error, [getattr(block, "text", None) for block in result.content] + + +@pytest.mark.parametrize( + "build", + [ + pytest.param(_high_level, id="high-level"), + pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), + pytest.param(lambda: _raw([["search_docs"]]), id="raw-low-level"), + ], +) +async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build): + expected = await _response(build(), "no_such_tool", {}) + client = FakeClient() + server = build() + instrument(server, client) + + assert await _response(server, "no_such_tool", {}) == expected + await flush_background() + + [unknown] = events_named(client, "$mcp_unknown_tool") + assert unknown["properties"]["$mcp_tool_name"] == "no_such_tool" + assert unknown["properties"]["$session_id"] + assert events_named(client, "$mcp_tool_call") == [] + assert events_named(client, "$exception") == [] + + +@pytest.mark.parametrize( + "build, name, arguments", + [ + pytest.param(_high_level, "search_docs", {}, id="invalid-arguments"), + pytest.param( + _standalone, + "retired", + {"query": "q"}, + id="disabled-standalone-tool", + marks=_needs_standalone, + ), + pytest.param( + lambda: _raw(None), "no_such_tool", {}, id="raw-low-level-without-listing" + ), + pytest.param( + lambda: _raw([["search_docs"], ["export"]]), + "no_such_tool", + {}, + id="raw-low-level-with-more-pages", + ), + ], +) +async def test_a_call_the_sdk_cannot_rule_out_is_a_failed_tool_call( + build, name, arguments +): + expected = await _response(build(), name, arguments) + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + assert await _response(server, name, arguments) == expected + await flush_background() + + [call] = events_named(client, "$mcp_tool_call") + assert call["properties"]["$mcp_tool_name"] == name + assert call["properties"]["$mcp_is_error"] is True + assert events_named(client, "$mcp_unknown_tool") == [] + + +async def test_a_custom_dispatcher_captures_an_unknown_tool(): + posthog = PostHogMCP("phc_test") + captured = [] + posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) + + posthog.capture_unknown_tool("export for jane@example.com", session_id="ses_1") + posthog.flush() + + [unknown] = events_named(captured, "$mcp_unknown_tool") + assert unknown["properties"]["$mcp_tool_name"] == "export for [redacted]" + assert unknown["properties"]["$session_id"] == "ses_1" + assert "$mcp_resource_name" not in unknown["properties"] + assert events_named(captured, "$mcp_tool_call") == [] diff --git a/posthog/test/mcp/test_v2_lowlevel.py b/posthog/test/mcp/test_v2_lowlevel.py index 319cbc3eb..09340dae3 100644 --- a/posthog/test/mcp/test_v2_lowlevel.py +++ b/posthog/test/mcp/test_v2_lowlevel.py @@ -28,14 +28,14 @@ def make_server(*, resource_error: bool = False, listing: str = "resources") -> Server: async def on_call_tool(ctx, params): - if params.name == "boom": + args = params.arguments or {} + if args.get("fail") == "raise": raise ValueError("explode") - if params.name == "soft-fail": + if args.get("fail") == "result": return mcp_types.CallToolResult( content=[mcp_types.TextContent(type="text", text="failed politely")], is_error=True, ) - args = params.arguments or {} return mcp_types.CallToolResult( content=[ mcp_types.TextContent( @@ -410,7 +410,11 @@ async def test_raised_error_is_captured_and_reraised(): instrument(server, client) with pytest.raises(ValueError): - await _call_tool(server, "boom", {"context": "attempting the risky operation"}) + await _call_tool( + server, + "add", + {"fail": "raise", "context": "attempting the risky operation"}, + ) await _flush() calls = _events(client, "$mcp_tool_call") @@ -426,7 +430,7 @@ async def test_is_error_result_is_captured(): instrument(server, client) result = await _call_tool( - server, "soft-fail", {"context": "expecting a polite failure"} + server, "add", {"fail": "result", "context": "expecting a polite failure"} ) await _flush() diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index bfffd5d64..98e77b016 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -1023,6 +1023,7 @@ attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.RESOURCES_LIST = '$mcp_ attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.RESOURCE_READ = '$mcp_resource_read' attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.TOOLS_LIST = '$mcp_tools_list' attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.TOOL_CALL = '$mcp_tool_call' +attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.UNKNOWN_TOOL = '$mcp_unknown_tool' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_NAME = '$mcp_client_name' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_USER_AGENT = '$mcp_client_user_agent' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_VERSION = '$mcp_client_version' @@ -1893,6 +1894,7 @@ method posthog.mcp.posthog_mcp.PostHogMCP.capture_initialize(*, client_name: Opt method posthog.mcp.posthog_mcp.PostHogMCP.capture_missing_capability(*, context: Optional[str] = None, llm_model: Optional[str] = None, llm_model_source: Optional[MCPAnalyticsModelSource] = None, parameters: Any = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.capture_tool_call(tool_name: str, *, intent: Optional[str] = None, intent_source: Optional[str] = None, parameters: Any = None, response: Any = None, duration_ms: Optional[float] = None, is_error: bool = False, error: Any = None, error_type: Optional[str] = None, category: Optional[str] = None, tool_description: Optional[str] = None, llm_model: Optional[str] = None, llm_model_source: Optional[MCPAnalyticsModelSource] = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.capture_tools_list(*, tool_names: Optional[List[str]] = None, parameters: Any = None, response: Any = None, duration_ms: Optional[float] = None, is_error: bool = False, error: Any = None, error_type: Optional[str] = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None +method posthog.mcp.posthog_mcp.PostHogMCP.capture_unknown_tool(tool_name: str, *, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.flush(timeout_seconds: Optional[float] = 10) -> None method posthog.mcp.posthog_mcp.PostHogMCP.prepare_tool_call(name: str, args: Optional[JsonRecord] = None, *, request_meta: Optional[JsonRecord] = None, original_tool: Any = None, session_id: Optional[str] = None) -> PreparedToolCall method posthog.mcp.posthog_mcp.PostHogMCP.prepare_tool_list(tools: List[Any], context: Union[bool, MCPAnalyticsContextOptions] = True, report_missing: bool = False, collect_feedback: bool = False) -> List[Any] From 38a4466acc2864ed61622862a99793544a87e891 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 14:46:33 -0300 Subject: [PATCH 2/7] fix(mcp): close unknown-tool gaps found in review of the spec alignment What: - A disabled tool on a mounted FastMCP 3+ server stays a failed $mcp_tool_call. A server serving other providers (mounts, proxies) can't prove a name absent, so existence answers "can't tell". - FastMCP's stock middleware (logging, timing, error handling, retry, rate limiting, ping, auth, caching, response limiting) no longer hides the registry. Tool-injection middleware and app middleware still do. - A call naming no tool emits nothing, automatic or manual. - Truncated strings fit their limit with the "..." marker included, so $mcp_tool_name stops at 256 characters (the spec's Property reference). - first_listing_page reads nextCursor only when next_cursor is absent, so fastmcp 4 on MCP 2 logs no deprecation warning. - A raw server's unknown-tool error result carries the conversation handle again, and $mcp_unknown_tool carries it once delivered, as origin/main did and as posthog-js does. - Unknown-tool tests skip only the FastMCP.disable cases on fastmcp 2. - Changeset names the listing probe and hidden-tool upgrade differences. Why: posthog.mcp must match the sdk-specs MCP spec before it is declared stable, so customers' insights don't shift afterwards. Tested: - pytest posthog/test/mcp (mcp 1.30 + fastmcp 3.2): 694 passed, 11 skipped - test_unknown_tool.py on mcp 1.30 + fastmcp 2.14.7: 21 passed, 4 skipped; mcp 2.3: 15 passed, 10 skipped; mcp 2.3 + fastmcp 4.1.6: 25 passed - ruff format --check, ruff check, mypy | mypy-baseline filter, make public_api_check: clean Reviewer notes: posthog-js truncates with the marker past the limit too and needs the same fix. test_fastmcp_v2.py:: test_jlowin_middleware_defaults_work_across_fresh_instances fails on fastmcp 2.14.7 on origin/main as well ($mcp_input_keys is ["[redacted]"]); not touched here. --- .sampo/changesets/mcp-v1-spec-align.md | 5 +- posthog/mcp/README.md | 18 +- posthog/mcp/_instrument_lowlevel.py | 10 +- posthog/mcp/_instrument_v2.py | 10 +- posthog/mcp/_instrumentation.py | 83 +++++++-- posthog/mcp/_truncation.py | 19 +- posthog/mcp/feedback.py | 14 +- posthog/mcp/posthog_mcp.py | 4 +- posthog/test/mcp/test_feedback.py | 4 +- posthog/test/mcp/test_pipeline.py | 2 +- posthog/test/mcp/test_truncation.py | 12 +- posthog/test/mcp/test_unknown_tool.py | 233 ++++++++++++++++++++++++- 12 files changed, 352 insertions(+), 62 deletions(-) diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md index c6460362a..5422db59a 100644 --- a/.sampo/changesets/mcp-v1-spec-align.md +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -4,7 +4,10 @@ pypi/posthog: minor MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: -- A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. +- A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. +- A tool your server's `tools/list` handler does not list, such as a hidden or per-user tool, now counts as an unknown tool when a call to it fails. +- On a low-level server whose tool registry the SDK can't read, a failed call runs your `tools/list` handler once to check the name. +- Truncated string properties now fit their limit with the `...` marker included, so `$mcp_tool_name` and other names stop at 256 characters instead of 259. - `get_more_tools` and `send_feedback` declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as `false`, and their annotations no longer carry a `title`. - The injected `context` argument has a shorter description, the same in every PostHog MCP SDK. - The injected `llm_model` argument is always optional. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index 2b60c71ba..be3bfd67d 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -191,8 +191,10 @@ On MCP SDK 1.x, standalone FastMCP application middleware can replace or reroute When it overrides a listing or dispatch hook, PostHog does not inject `llm_model`: a fresh replica cannot safely distinguish that field from a replacement tool's own argument. Client metadata capture remains enabled. This also applies to pass-through logging or -authorization middleware with those hooks; argument-based model capture is skipped so an -application-owned value cannot be mistaken for analytics. No additional catalog lookup runs +authorization middleware of your own with those hooks; argument-based model capture is skipped +so an application-owned value cannot be mistaken for analytics. FastMCP's stock middleware, +such as `LoggingMiddleware`, `TimingMiddleware`, and `ErrorHandlingMiddleware`, never reroutes a +tool and does not count; its tool-injection middleware does. No additional catalog lookup runs during a tool call. For a custom dispatcher, `PostHogMCP` enables model capture and conversation correlation @@ -353,9 +355,15 @@ registry: call to a name that handler does not list is unknown. With no `tools/list` handler, or a name missing from a first page that has more pages after it, the SDK cannot tell and records a failed tool call. -- On standalone `fastmcp.FastMCP`, middleware that overrides a listing or - dispatch hook can serve tools the registry lacks, so the SDK records a failed - tool call. +- On standalone `fastmcp.FastMCP`, middleware of your own that overrides a + listing or dispatch hook can serve tools the registry lacks, so the SDK + records a failed tool call. FastMCP's stock middleware, other than its + tool-injection middleware, does not count. +- On standalone `fastmcp.FastMCP` 3 and later with a mounted server, a proxy, + or another provider, the provider hides its disabled tools, so a name the SDK + can't find is a failed tool call. + +A call naming no tool sends nothing. Invalid arguments for a tool that exists are a failed tool call. A custom dispatcher calls `posthog.capture_unknown_tool(tool_name, session_id=...)` in diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index 14563b8ca..c16fb0007 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -300,10 +300,6 @@ async def handler(req: Any) -> Any: # The low-level handler already converted any exception to a # CallToolResult(isError=True); record_tool_call detects that from the result. call_result = getattr(result, "root", result) - if is_tool_result_error(call_result) and await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, high_level, name) - ): - return result # Deliver the handle before capture, over both channels a result has: # mirrored into structuredContent on every response (for tools whose @@ -343,6 +339,12 @@ async def handler(req: Any) -> Any: else call_result ) + if is_tool_result_error(call_result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, high_level, name), + conversation_id_delivered=delivered, + ): + return result + await lifecycle.record_result( call_result, duration_ms, diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 751518c8e..3f6b8afe4 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -684,10 +684,6 @@ async def handler(ctx: Any, params: Any) -> Any: await lifecycle.record_error(error, (time.monotonic() - start) * 1000) raise duration_ms = (time.monotonic() - start) * 1000 - if is_tool_result_error(result) and await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, standalone, name, ctx) - ): - return result delivered = False if lifecycle.conversation_id: @@ -699,6 +695,12 @@ async def handler(ctx: Any, params: Any) -> Any: lifecycle.minted_conversation_id, ) + if is_tool_result_error(result) and await lifecycle.record_if_unknown_tool( + await registered_tool_exists(data, standalone, name, ctx), + conversation_id_delivered=delivered, + ): + return result + await lifecycle.record_result( result, duration_ms, conversation_id_delivered=delivered ) diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index c8a197bfc..6744f6294 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -12,6 +12,7 @@ import concurrent.futures import copy import functools +import importlib import os import threading from dataclasses import dataclass @@ -621,13 +622,18 @@ async def record_feedback(self, *, conversation_id_delivered: bool = False) -> s ) return await handle_feedback(report, self.feedback_options) - async def record_if_unknown_tool(self, tool_exists: Optional[bool]) -> bool: + async def record_if_unknown_tool( + self, tool_exists: Optional[bool], *, conversation_id_delivered: bool = False + ) -> bool: """Record the failed call as ``$mcp_unknown_tool`` when ``tool_exists`` - proves the server has no tool by that name, and say whether it did. The - adapter then re-raises or returns the host's response unchanged.""" + proves the server has no tool by that name, and say whether the call is + settled that way. A call naming no tool is settled with no event. The + adapter then re-raises or returns its response.""" + if not self.name: + return True if tool_exists is not False: return False - conversation_id = self._anchored_conversation_id(False) + conversation_id = self._anchored_conversation_id(conversation_id_delivered) session_id = await self.prepare_session(conversation_id) await record_unknown_tool( self.data, @@ -999,7 +1005,12 @@ def first_listing_page(result: Any) -> FirstListingPage: cursor leads to more. 1.x wraps the result in a ``ServerResult`` and spells the cursor ``nextCursor``; 2.x returns it bare as ``next_cursor``.""" root = getattr(result, "root", result) - cursor = getattr(root, "next_cursor", getattr(root, "nextCursor", None)) + # fastmcp 4 bridges the 1.x spelling onto 2.x results with a warning. + cursor = ( + root.next_cursor + if hasattr(root, "next_cursor") + else getattr(root, "nextCursor", None) + ) return FirstListingPage( advertised_tool_names(list(getattr(root, "tools", []) or [])), cursor is not None, @@ -1126,21 +1137,33 @@ async def registered_tool_exists( async def _standalone_tool_exists(server: Any, name: str) -> Optional[bool]: """Whether jlowin's FastMCP registers ``name``, enabled or not. A disabled tool still exists: calling it is a failed call. Middleware that can serve - tools the registry lacks makes the registry no witness.""" + tools the registry lacks makes the registry no witness, and so does a + mounted server or other provider, which hides its disabled tools.""" if _dispatch_can_differ(server): return None try: - if await server.get_tool(name) is not None: - return True from fastmcp import FastMCP # From 3.x, get_tool hides a disabled tool; the providers still hold it. - return await super(FastMCP, server).get_tool(name) is not None + found = ( + await server.get_tool(name) is not None + or await super(FastMCP, server).get_tool(name) is not None + ) except _tool_lookup_not_found_errors(): - return False + found = False except Exception as err: # noqa: BLE001 - analytics must not break the call log(f"PostHog MCP: could not look up tool {name!r} - {err}") return None + if found: + return True + return None if _serves_other_providers(server) else False + + +def _serves_other_providers(server: Any) -> bool: + """Whether a FastMCP 3+ server serves tools from anywhere but its own + registry: a mounted server, a proxy, or a custom provider.""" + local = getattr(server, "_local_provider", None) + return any(provider is not local for provider in getattr(server, "providers", ())) @functools.lru_cache(maxsize=1) @@ -1178,7 +1201,8 @@ def _dispatch_can_differ(server: Any) -> bool: return False return any( - type(middleware) not in _builtin_middleware_types() # subclasses are the app's + type(middleware) + not in _dispatch_preserving_middleware_types() # subclasses are the app's and any( getattr(type(middleware), hook) is not getattr(Middleware, hook) for hook in _DISPATCH_HOOKS @@ -1187,13 +1211,44 @@ def _dispatch_can_differ(server: Any) -> bool: ) +# Stock fastmcp middleware that observes, times, retries, throttles, denies, +# caches, or trims a call to the tool it names, and never provides, renames, or +# reroutes one. The tool-injection family is left out on purpose. +_STOCK_DISPATCH_PRESERVING_MIDDLEWARE = ( + ("logging", "LoggingMiddleware"), + ("logging", "StructuredLoggingMiddleware"), + ("timing", "TimingMiddleware"), + ("timing", "DetailedTimingMiddleware"), + ("error_handling", "ErrorHandlingMiddleware"), + ("error_handling", "RetryMiddleware"), + ("rate_limiting", "RateLimitingMiddleware"), + ("rate_limiting", "SlidingWindowRateLimitingMiddleware"), + ("ping", "PingMiddleware"), + ("authorization", "AuthMiddleware"), + ("caching", "ResponseCachingMiddleware"), + ("response_limiting", "ResponseLimitingMiddleware"), +) + + @functools.lru_cache(maxsize=1) -def _builtin_middleware_types() -> tuple: +def _dispatch_preserving_middleware_types() -> FrozenSet[type]: """The middleware a bare ``FastMCP()`` installs on its own, probed rather - than named so a new built-in in a later release is still recognised.""" + than named so a new built-in in a later release is still recognised, plus + the stock middleware above that this fastmcp release ships.""" from fastmcp import FastMCP - return tuple(type(middleware) for middleware in FastMCP("posthog-probe").middleware) + stock = set() + for module, name in _STOCK_DISPATCH_PRESERVING_MIDDLEWARE: + try: + found = getattr( + importlib.import_module(f"fastmcp.server.middleware.{module}"), name + ) + except (ImportError, AttributeError): + continue + stock.add(found) + return frozenset( + {type(middleware) for middleware in FastMCP("posthog-probe").middleware} | stock + ) def warn_ownership_lookup_failed(name: str, err: Exception) -> None: diff --git a/posthog/mcp/_truncation.py b/posthog/mcp/_truncation.py index 674241a6c..382c64639 100644 --- a/posthog/mcp/_truncation.py +++ b/posthog/mcp/_truncation.py @@ -85,9 +85,7 @@ def _visit( return "[Infinity]" if value > 0 else "[-Infinity]" return value if isinstance(value, str): - if len(value) > max_string_length: - return value[:max_string_length] + _TRUNCATION_SUFFIX - return value + return truncate_text(value, max_string_length) if isinstance(value, datetime): return value.isoformat() if callable(value): @@ -161,12 +159,18 @@ def _visit_object( # --- field-level helpers ----------------------------------------------------- +def truncate_text(value: str, max_length: int) -> str: + """``value`` cut to at most ``max_length`` characters, the truncation marker + included, keeping its start.""" + if len(value) <= max_length: + return value + return value[: max_length - len(_TRUNCATION_SUFFIX)] + _TRUNCATION_SUFFIX + + def _truncate_string(value: Optional[str], max_length: int) -> Optional[str]: if not isinstance(value, str): return value - if len(value) <= max_length: - return value - return value[:max_length] + _TRUNCATION_SUFFIX + return truncate_text(value, max_length) def _truncate_stack_frames(frames: Optional[List[Any]]) -> Optional[List[Any]]: @@ -214,8 +218,7 @@ def _truncate_response_content(response: Any) -> Any: new_content.append( { **block, - "text": block["text"][:_MAX_CONTENT_TEXT_LENGTH] - + _TRUNCATION_SUFFIX, + "text": truncate_text(block["text"], _MAX_CONTENT_TEXT_LENGTH), } ) else: diff --git a/posthog/mcp/feedback.py b/posthog/mcp/feedback.py index 5d1bc1d08..5cb732b6c 100644 --- a/posthog/mcp/feedback.py +++ b/posthog/mcp/feedback.py @@ -11,6 +11,7 @@ from ._internal import _maybe_await from ._sanitization import sanitize_free_text, sanitize_free_text_value +from ._truncation import truncate_text from .constants import PostHogMCPAnalyticsProperty from .logger import log from .tools import VIRTUAL_TOOL_HINTS @@ -26,7 +27,6 @@ # Free-text fields are agent-narrated, like `$mcp_intent`; bound them the same way. _MAX_FEEDBACK_TEXT_LENGTH = 2048 _MAX_FEEDBACK_TOOL_NAME_LENGTH = 256 -_TRUNCATION_SUFFIX = "..." _DEFAULT_FEEDBACK_DESCRIPTION = ( "Send feedback about this server to its developers: a capability you needed that no tool " @@ -247,10 +247,6 @@ def build_feedback_intent(report: FeedbackReport) -> str: return "\n\n".join(part for part in (report.summary, report.details) if part) -def _truncate_feedback_text(value: str, max_length: int) -> str: - return value[:max_length] + _TRUNCATION_SUFFIX if len(value) > max_length else value - - def _capture_free_text(value: str) -> str: """Agent-narrated free text can contain a secret the LLM read aloud or personal data it narrated, so it gets exactly the ``$mcp_intent`` pass @@ -258,7 +254,7 @@ def _capture_free_text(value: str) -> str: load-bearing, the URL rewrite would percent-encode the ``@`` the email pattern anchors on), then a length bound. The event pipeline does not process ``event["properties"]``, so this happens here.""" - return _truncate_feedback_text(sanitize_free_text(value), _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(sanitize_free_text(value), _MAX_FEEDBACK_TEXT_LENGTH) def _capture_extra_value(value: Any) -> Any: @@ -268,11 +264,11 @@ def _capture_extra_value(value: Any) -> Any: JSON-stringified and everything is bounded. Unserializable values are dropped.""" sanitized = sanitize_free_text_value(value) if isinstance(sanitized, str): - return _truncate_feedback_text(sanitized, _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(sanitized, _MAX_FEEDBACK_TEXT_LENGTH) if sanitized is None or isinstance(sanitized, (bool, int, float)): return sanitized try: - return _truncate_feedback_text(json.dumps(sanitized), _MAX_FEEDBACK_TEXT_LENGTH) + return truncate_text(json.dumps(sanitized), _MAX_FEEDBACK_TEXT_LENGTH) except Exception: # noqa: BLE001 - capture must never raise into the tool path return None @@ -305,7 +301,7 @@ def build_feedback_event_properties(report: FeedbackReport) -> JsonRecord: # Nominally an identifier, but the schema can't stop an agent from # writing prose into it — so it gets the same free-text pass as the # other fields. - properties[PostHogMCPAnalyticsProperty.FEEDBACK_TOOL] = _truncate_feedback_text( + properties[PostHogMCPAnalyticsProperty.FEEDBACK_TOOL] = truncate_text( sanitize_free_text(report.tool_name), _MAX_FEEDBACK_TOOL_NAME_LENGTH, ) diff --git a/posthog/mcp/posthog_mcp.py b/posthog/mcp/posthog_mcp.py index 8f75fbf74..e3ea4ffa4 100644 --- a/posthog/mcp/posthog_mcp.py +++ b/posthog/mcp/posthog_mcp.py @@ -391,7 +391,9 @@ def capture_unknown_tool( ``$mcp_unknown_tool`` with the requested name as ``$mcp_tool_name``. Capture it instead of :meth:`capture_tool_call`, not as well: it is not a tool call. A tool that exists, disabled or called with invalid - arguments, is a failed tool call.""" + arguments, is a failed tool call. A call naming no tool emits nothing.""" + if not tool_name: + return event = self._base_event( MCPAnalyticsEventType.MCP_UNKNOWN_TOOL, distinct_id, diff --git a/posthog/test/mcp/test_feedback.py b/posthog/test/mcp/test_feedback.py index b45721c38..5110a0d60 100644 --- a/posthog/test/mcp/test_feedback.py +++ b/posthog/test/mcp/test_feedback.py @@ -237,9 +237,9 @@ def test_properties_bound_free_text_and_tool_name(): {"feedback_type": "issue", "summary": "s" * 5000, "tool_name": "t" * 500} ) props = build_feedback_event_properties(report) - assert len(props["$mcp_feedback_summary"]) == 2048 + 3 + assert len(props["$mcp_feedback_summary"]) == 2048 assert props["$mcp_feedback_summary"].endswith("...") - assert len(props["$mcp_feedback_tool"]) == 256 + 3 + assert len(props["$mcp_feedback_tool"]) == 256 def test_properties_capture_declared_extras_only(): diff --git a/posthog/test/mcp/test_pipeline.py b/posthog/test/mcp/test_pipeline.py index 6a4dda12f..ea7457023 100644 --- a/posthog/test/mcp/test_pipeline.py +++ b/posthog/test/mcp/test_pipeline.py @@ -821,7 +821,7 @@ def test_sanitize_event_passes_through_non_string_intent(): def test_normalize_caps_long_strings(): out = normalize("x" * 40000) assert out.endswith("...") - assert len(out) == 32_768 + 3 + assert len(out) == 32_768 def test_normalize_detects_cycles(): diff --git a/posthog/test/mcp/test_truncation.py b/posthog/test/mcp/test_truncation.py index a0752c6c4..a3d47be49 100644 --- a/posthog/test/mcp/test_truncation.py +++ b/posthog/test/mcp/test_truncation.py @@ -20,7 +20,7 @@ def test_normalize_caps_long_strings(): out = normalize("x" * (MAX_STRING_LENGTH + 100)) - assert out.endswith("...") and len(out) == MAX_STRING_LENGTH + 3 + assert out.endswith("...") and len(out) == MAX_STRING_LENGTH def test_normalize_keeps_bool_distinct_from_int(): @@ -102,9 +102,9 @@ def test_truncate_event_caps_metadata_fields(): "llm_model": "m" * 500, } ) - assert len(out["user_intent"]) == 2048 + 3 - assert len(out["resource_name"]) == 256 + 3 - assert len(out["llm_model"]) == 256 + 3 + assert len(out["user_intent"]) == 2048 + assert len(out["resource_name"]) == 256 + assert len(out["llm_model"]) == 256 def test_truncate_event_caps_exception_value_and_frames(): @@ -119,7 +119,7 @@ def test_truncate_event_caps_exception_value_and_frames(): } ) exc = out["error"]["$exception_list"][0] - assert len(exc["value"]) == 2048 + 3 + assert len(exc["value"]) == 2048 assert exc["stacktrace"]["frames"] == frames[:25] + frames[-25:] @@ -127,7 +127,7 @@ def test_truncate_event_caps_response_text_block(): out = truncate_event( {"response": {"content": [{"type": "text", "text": "t" * 40000}]}} ) - assert len(out["response"]["content"][0]["text"]) == 32_768 + 3 + assert len(out["response"]["content"][0]["text"]) == 32_768 # --- size-targeted budget ---------------------------------------------------- diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index b0a7e9b1e..3f851d97a 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -2,7 +2,9 @@ not a tool call. Runs under both MCP SDK majors over an in-process client, so the response compared against an uninstrumented twin is the one an agent gets.""" +import importlib import importlib.util +import json from contextlib import asynccontextmanager import mcp.types as mcp_types @@ -22,6 +24,19 @@ ) +def _fastmcp_can_disable() -> bool: + try: + from fastmcp import FastMCP + except ImportError: + return False + return hasattr(FastMCP, "disable") + + +_needs_server_disable = pytest.mark.skipif( + not _fastmcp_can_disable(), reason="FastMCP.disable and providers, fastmcp 3+" +) + + def _high_level(): if MCP_MAJOR >= 2: from mcp.server.mcpserver import MCPServer as HighLevel @@ -36,7 +51,9 @@ def search_docs(query: str) -> str: return server -def _standalone(): +def _standalone(*middleware): + """``middleware`` names stock fastmcp middleware as ``"."`` + under ``fastmcp.server.middleware``.""" from fastmcp import FastMCP server = FastMCP("unknown-tool") @@ -45,6 +62,16 @@ def _standalone(): def search_docs(query: str) -> str: return f"found {query}" + for path in middleware: + module, name = path.rsplit(".", 1) + stock = importlib.import_module(f"fastmcp.server.middleware.{module}") + server.add_middleware(getattr(stock, name)()) + return server + + +def _standalone_with_disabled_tool(): + server = _standalone() + @server.tool def retired(query: str) -> str: return query @@ -53,10 +80,41 @@ def retired(query: str) -> str: return server -def _raw(pages): +def _standalone_with_application_middleware(): + from fastmcp.server.middleware.logging import LoggingMiddleware + + class AuditMiddleware(LoggingMiddleware): + async def on_call_tool(self, context, call_next): + return await call_next(context) + + server = _standalone() + server.add_middleware(AuditMiddleware()) + return server + + +def _mounted(): + """A parent serving a sub-server's tools under the ``ns`` namespace, one + of them disabled on the sub-server.""" + from fastmcp import FastMCP + + sub = FastMCP("sub") + + @sub.tool + def retired(query: str) -> str: + return query + + sub.disable(names={"retired"}) + parent = _standalone() + parent.mount(sub, namespace="ns") + return parent + + +def _raw(pages, *, reject_with_result=False): """A raw low-level server whose own dispatcher rejects names it does not know. ``pages`` are the tools/list pages of tool names, chained by cursor; - ``None`` registers no tools/list handler at all.""" + ``None`` registers no tools/list handler at all. On MCP 2.x the rejection + raises unless ``reject_with_result``; 1.x's decorator always turns it into + an ``isError`` result.""" from mcp.server.lowlevel import Server registered = {name for page in pages or [] for name in page} @@ -84,7 +142,15 @@ def reply(name): if MCP_MAJOR >= 2: async def on_call_tool(ctx, params): - return mcp_types.CallToolResult(content=reply(params.name)) + try: + return mcp_types.CallToolResult(content=reply(params.name)) + except ValueError as error: + if not reject_with_result: + raise + return mcp_types.CallToolResult( + content=[mcp_types.TextContent(type="text", text=str(error))], + is_error=True, + ) async def on_list_tools(ctx, params): return page_at(getattr(params, "cursor", None)) @@ -135,7 +201,7 @@ async def _response(server, name, arguments): result = await call(name, arguments) except Exception as error: # noqa: BLE001 - the error is the response return type(error).__name__, str(error) - is_error = getattr(result, "is_error", getattr(result, "isError", None)) + is_error = result.is_error if hasattr(result, "is_error") else result.isError return is_error, [getattr(block, "text", None) for block in result.content] @@ -144,6 +210,18 @@ async def _response(server, name, arguments): [ pytest.param(_high_level, id="high-level"), pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), + *( + pytest.param( + lambda path=path: _standalone(path), + id=f"standalone-fastmcp-with-{path.split('.')[0]}-middleware", + marks=_needs_standalone, + ) + for path in ( + "logging.LoggingMiddleware", + "timing.TimingMiddleware", + "error_handling.ErrorHandlingMiddleware", + ) + ), pytest.param(lambda: _raw([["search_docs"]]), id="raw-low-level"), ], ) @@ -151,7 +229,7 @@ async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build expected = await _response(build(), "no_such_tool", {}) client = FakeClient() server = build() - instrument(server, client) + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) assert await _response(server, "no_such_tool", {}) == expected await flush_background() @@ -168,10 +246,31 @@ async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build [ pytest.param(_high_level, "search_docs", {}, id="invalid-arguments"), pytest.param( - _standalone, + _standalone_with_disabled_tool, "retired", {"query": "q"}, id="disabled-standalone-tool", + marks=_needs_server_disable, + ), + pytest.param( + _mounted, + "ns_retired", + {"query": "q"}, + id="disabled-tool-on-mounted-server", + marks=_needs_server_disable, + ), + pytest.param( + _mounted, + "ns_no_such_tool", + {}, + id="server-with-mounted-provider", + marks=_needs_server_disable, + ), + pytest.param( + _standalone_with_application_middleware, + "no_such_tool", + {}, + id="standalone-with-application-middleware", marks=_needs_standalone, ), pytest.param( @@ -215,3 +314,123 @@ async def test_a_custom_dispatcher_captures_an_unknown_tool(): assert unknown["properties"]["$session_id"] == "ses_1" assert "$mcp_resource_name" not in unknown["properties"] assert events_named(captured, "$mcp_tool_call") == [] + + +_SHAPES = [ + pytest.param(_high_level, id="high-level"), + pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), + pytest.param(lambda: _raw([["search_docs"]]), id="raw-low-level"), +] + + +@pytest.mark.parametrize("build", _SHAPES) +async def test_a_call_naming_no_tool_emits_nothing(build): + expected = await _response(build(), "", {}) + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + assert await _response(server, "", {}) == expected + await flush_background() + + assert [e["event"] for e in client.events if e["event"] != "$mcp_initialize"] == [] + + +@pytest.mark.parametrize("build", _SHAPES) +async def test_an_unknown_tool_name_is_cut_to_256_characters(build): + client = FakeClient() + server = build() + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + await _response(server, "x" * 5000, {}) + await flush_background() + + [unknown] = events_named(client, "$mcp_unknown_tool") + name = unknown["properties"]["$mcp_tool_name"] + assert len(name) == 256 + assert name.startswith("x" * 200) + + +@pytest.mark.parametrize( + "tool_name, expected_names", + [ + pytest.param("", [], id="no-tool"), + pytest.param("y" * 5000, [256], id="long-name"), + ], +) +def test_a_custom_dispatcher_follows_the_unknown_tool_name_rules( + tool_name, expected_names +): + posthog = PostHogMCP("phc_test") + captured = [] + posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) + + posthog.capture_unknown_tool(tool_name, session_id="ses_1") + posthog.flush() + + assert [ + len(e["properties"]["$mcp_tool_name"]) + for e in events_named(captured, "$mcp_unknown_tool") + ] == expected_names + + +@pytest.mark.parametrize( + "build, delivers_handle", + [ + pytest.param( + lambda: _raw([["search_docs"]], reject_with_result=True), + True, + id="raw-low-level-error-result", + ), + pytest.param(_high_level, False, id="high-level-raise"), + ], +) +async def test_an_unknown_tool_result_carries_the_conversation_handle_it_delivers( + build, delivers_handle +): + client = FakeClient() + server = build() + instrument(server, client) + + _, texts = await _response(server, "no_such_tool", {}) + await flush_background() + + handles = [ + json.loads(text)["conversation_id"] + for text in texts + if text and text.startswith('{"conversation_id"') + ] + [unknown] = events_named(client, "$mcp_unknown_tool") + assert len(handles) == int(delivers_handle) + assert unknown["properties"].get("$mcp_conversation_id") == next( + iter(handles), None + ) + + +@pytest.mark.skipif(MCP_MAJOR < 2, reason="MCP 2 renamed nextCursor to next_cursor") +async def test_checking_the_host_listing_never_reads_the_deprecated_cursor( + monkeypatch, +): + """fastmcp 4 installs a ``nextCursor`` property on MCP 2 results that warns + on first read; this one stands in for it and counts the reads.""" + reads = [] + + def deprecated_cursor(result): + reads.append(result) + return result.next_cursor + + monkeypatch.setattr( + mcp_types.ListToolsResult, + "nextCursor", + property(deprecated_cursor), + raising=False, + ) + client = FakeClient() + server = _raw([["search_docs"]], reject_with_result=True) + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + await _response(server, "no_such_tool", {}) + await flush_background() + + assert reads == [] + assert len(events_named(client, "$mcp_unknown_tool")) == 1 From 1735d578e5a408925446192417027ec5c8b426ca Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 15:02:39 -0300 Subject: [PATCH 3/7] fix(mcp): bound the listing probe and give manual capture a session What: raw_listing_owns_tool_name waits at most LISTING_PROBE_DEADLINE_SECONDS (1s) for the host's tools/list and treats a timeout as None (can't tell), so a failed call records a failed $mcp_tool_call and the client gets its error promptly. PostHogMCP._base_event mints a fresh ses_ when no session_id is passed, so capture_unknown_tool (and capture_tool_call and the other manual captures) no longer send no $session_id and an anonymous distinct_id. Why: PostHog Review on #1199. A stalled tools/list handler held every failed call forever; the spec's Session identity and Manual capture require a non-empty $session_id, fresh per call when nothing groups it. Tested: pytest posthog/test/mcp (697 passed, 11 skipped); ruff format/check, mypy | mypy-baseline filter, make public_api_check all clean. New tests failed first: stalled listing (timed out), manual capture without session (KeyError). Reviewer: a manual capture passing conversation_id but no session_id still gets a fresh session, not one derived from the handle. --- .sampo/changesets/mcp-v1-spec-align.md | 3 +- posthog/mcp/_instrumentation.py | 12 +++++- posthog/mcp/posthog_mcp.py | 4 +- posthog/test/mcp/test_unknown_tool.py | 56 ++++++++++++++++++++++++-- 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md index 5422db59a..a4ab81ba2 100644 --- a/.sampo/changesets/mcp-v1-spec-align.md +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -6,7 +6,8 @@ MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: - A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. - A tool your server's `tools/list` handler does not list, such as a hidden or per-user tool, now counts as an unknown tool when a call to it fails. -- On a low-level server whose tool registry the SDK can't read, a failed call runs your `tools/list` handler once to check the name. +- On a low-level server whose tool registry the SDK can't read, a failed call runs your `tools/list` handler once to check the name, waiting at most one second. A handler that is slower leaves the call recorded as a failed tool call. +- Manual capture without a `session_id` now sends a fresh `ses_` session id and uses it as the `distinct_id`, instead of no `$session_id` and an `anonymous` person. - Truncated string properties now fit their limit with the `...` marker included, so `$mcp_tool_name` and other names stop at 256 characters instead of 259. - `get_more_tools` and `send_feedback` declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as `false`, and their annotations no longer carry a `title`. - The injected `context` argument has a shorter description, the same in every PostHog MCP SDK. diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index 6744f6294..586cf046a 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -1094,6 +1094,9 @@ def resolve_virtual_tool_injection( return injectable +LISTING_PROBE_DEADLINE_SECONDS = 1.0 + + async def raw_listing_owns_tool_name( data: MCPAnalyticsData, name: str, ctx: Any = None, *, whole_catalogue: bool = False ) -> Optional[bool]: @@ -1105,13 +1108,18 @@ async def raw_listing_owns_tool_name( Tri-state. ``True``/``False`` are answers; ``None`` means the question could not be asked, and callers must not intercept on it -- guessing "not owned" would swallow a real tool of the host's. Runs once per call to a virtual - tool's name and per failed call, never for ordinary traffic. + tool's name and per failed call, never for ordinary traffic. A listing that + takes longer than ``LISTING_PROBE_DEADLINE_SECONDS`` is unanswered, so a + stalled handler cannot hold the client's response. """ probe = data.raw_tool_names_probe if probe is None: return None try: - page = await probe(ctx) + page = await asyncio.wait_for(probe(ctx), LISTING_PROBE_DEADLINE_SECONDS) + except asyncio.TimeoutError: + log(f"PostHog MCP: tools/list did not answer in time to look up {name!r}") + return None except Exception as err: # noqa: BLE001 - analytics must not break the call warn_ownership_lookup_failed(name, err) return None diff --git a/posthog/mcp/posthog_mcp.py b/posthog/mcp/posthog_mcp.py index e3ea4ffa4..5a3e47d06 100644 --- a/posthog/mcp/posthog_mcp.py +++ b/posthog/mcp/posthog_mcp.py @@ -70,7 +70,7 @@ resolve_collect_feedback_options, resolve_send_feedback_tool_name, ) -from .session import derive_session_id_from_conversation +from .session import derive_session_id_from_conversation, new_session_id from .tools import build_report_missing_descriptor from .types import ( CollectFeedbackOptions, @@ -751,7 +751,7 @@ def _base_event( ) -> Dict[str, Any]: event: Dict[str, Any] = { "event_type": event_type, - "session_id": session_id, + "session_id": session_id or new_session_id(), "conversation_id": conversation_id, "timestamp": timestamp or datetime.now(timezone.utc), "properties": properties, diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index 3f851d97a..2c25d90f1 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -2,6 +2,7 @@ not a tool call. Runs under both MCP SDK majors over an in-process client, so the response compared against an uninstrumented twin is the one an agent gets.""" +import asyncio import importlib import importlib.util import json @@ -10,7 +11,7 @@ import mcp.types as mcp_types import pytest -from posthog.mcp import PostHogMCP, instrument +from posthog.mcp import PostHogMCP, _instrumentation as instrumentation, instrument from posthog.mcp.types import MCPAnalyticsOptions from posthog.test.mcp._helpers import ( MCP_MAJOR, @@ -109,14 +110,19 @@ def retired(query: str) -> str: return parent -def _raw(pages, *, reject_with_result=False): +def _raw(pages, *, reject_with_result=False, stall_listing=False): """A raw low-level server whose own dispatcher rejects names it does not know. ``pages`` are the tools/list pages of tool names, chained by cursor; ``None`` registers no tools/list handler at all. On MCP 2.x the rejection raises unless ``reject_with_result``; 1.x's decorator always turns it into - an ``isError`` result.""" + an ``isError`` result. With ``stall_listing``, a client's tools/list never + answers; the server's own cache refresh, which passes no request, does.""" from mcp.server.lowlevel import Server + async def stall_for_a_client(request): + if stall_listing and request is not None: + await asyncio.Event().wait() + registered = {name for page in pages or [] for name in page} schema = {"type": "object", "properties": {"query": {"type": "string"}}} @@ -153,6 +159,7 @@ async def on_call_tool(ctx, params): ) async def on_list_tools(ctx, params): + await stall_for_a_client(params) return page_at(getattr(params, "cursor", None)) handlers = {"on_call_tool": on_call_tool} @@ -169,6 +176,7 @@ async def call_tool(name, arguments): if pages is not None: async def list_tools(req): + await stall_for_a_client(req) params = getattr(req, "params", None) if req else None return mcp_types.ServerResult(page_at(getattr(params, "cursor", None))) @@ -434,3 +442,45 @@ def deprecated_cursor(result): assert reads == [] assert len(events_named(client, "$mcp_unknown_tool")) == 1 + + +async def test_a_stalled_listing_does_not_delay_a_failed_calls_error(monkeypatch): + monkeypatch.setattr(instrumentation, "LISTING_PROBE_DEADLINE_SECONDS", 0.05) + client = FakeClient() + server = _raw([["search_docs"]], reject_with_result=True, stall_listing=True) + instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) + + is_error, _ = await asyncio.wait_for( + _response(server, "no_such_tool", {}), timeout=5 + ) + await flush_background() + + assert is_error is True + [call] = events_named(client, "$mcp_tool_call") + assert call["properties"]["$mcp_is_error"] is True + assert events_named(client, "$mcp_unknown_tool") == [] + + +@pytest.mark.parametrize( + "capture", + [ + pytest.param(lambda posthog: posthog.capture_tool_call("t"), id="tool-call"), + pytest.param( + lambda posthog: posthog.capture_unknown_tool("t"), id="unknown-tool" + ), + ], +) +def test_manual_capture_without_a_session_gets_a_fresh_one_per_call(capture): + posthog = PostHogMCP("phc_test") + captured = [] + posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) + + capture(posthog) + capture(posthog) + posthog.flush() + + first, second = captured + session = first["properties"]["$session_id"] + assert session.startswith("ses_") + assert first["distinct_id"] == session + assert second["properties"]["$session_id"] != session From 3a26e71cc5e3a7f84b19698b8d4d04a44b8a41e8 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 16:39:53 -0300 Subject: [PATCH 4/7] fix(mcp): make the stalled-listing test stall on mcp 2.x What: `_raw(stall_listing=True)` now stalls every tools/list on mcp 2.x; on 1.x it still spares the server's own cache refresh. Why: the v2 adapter probes the host listing as `original(ctx, None)`, and the helper stalled only when a request was present, so the probe was answered at once, the call was recorded as `$mcp_unknown_tool`, and the test found no `$mcp_tool_call` (CI: MCP SDK v2 and v2-bare, Python 3.10). The product code was right: the deadline path falls back to a failed call on mcp 2. Tested: `pytest posthog/test/mcp` on mcp 1.30 + fastmcp 3.2 (697 passed), mcp 2.3 (536), mcp 2.3 + fastmcp 4.1.6 (579), and Python 3.10 with mcp 2.x bare (536) and with fastmcp 4 (579). Raising the deadline to 999s makes the test fail on mcp 2. ruff format/check and mypy baseline clean. --- posthog/test/mcp/test_unknown_tool.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index 2c25d90f1..7eeef2846 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -115,12 +115,13 @@ def _raw(pages, *, reject_with_result=False, stall_listing=False): know. ``pages`` are the tools/list pages of tool names, chained by cursor; ``None`` registers no tools/list handler at all. On MCP 2.x the rejection raises unless ``reject_with_result``; 1.x's decorator always turns it into - an ``isError`` result. With ``stall_listing``, a client's tools/list never - answers; the server's own cache refresh, which passes no request, does.""" + an ``isError`` result. With ``stall_listing``, tools/list never answers on + MCP 2.x; on 1.x only a client's does, the server's own cache refresh, which + passes no request, still does.""" from mcp.server.lowlevel import Server - async def stall_for_a_client(request): - if stall_listing and request is not None: + async def stall_unless_cache_refresh(request): + if stall_listing and (request is not None or MCP_MAJOR >= 2): await asyncio.Event().wait() registered = {name for page in pages or [] for name in page} @@ -159,7 +160,7 @@ async def on_call_tool(ctx, params): ) async def on_list_tools(ctx, params): - await stall_for_a_client(params) + await stall_unless_cache_refresh(params) return page_at(getattr(params, "cursor", None)) handlers = {"on_call_tool": on_call_tool} @@ -176,7 +177,7 @@ async def call_tool(name, arguments): if pages is not None: async def list_tools(req): - await stall_for_a_client(req) + await stall_unless_cache_refresh(req) params = getattr(req, "params", None) if req else None return mcp_types.ServerResult(page_at(getattr(params, "cursor", None))) From 3a3c20c148cfbda127c8f9ecc5e3d76744c11a87 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 17:01:47 -0300 Subject: [PATCH 5/7] fix(mcp): stop treating a tools/list listing as proof a tool is unknown A call is now unknown only on authoritative evidence: jlowin FastMCP's registry when nothing overrides dispatch, or the official high-level server's registry. A raw low-level server's own dispatcher decides which names exist and its listing may hide tools it serves, so there the SDK answers "can't tell" and records a failed $mcp_tool_call (spec, "Tool call event"). This removes the failure-path listing probe, its one-second deadline, the pagination flag and FirstListingPage. The deadline also sat in the check that decides whether get_more_tools and send_feedback are virtual-tool calls, so a tools/list slower than one second sent those calls to the application. That check is back to origin/main's. Tested with: - new tests failed on the previous head: a slow tools/list still answers get_more_tools and send_feedback; a raw low-level server records a failed $mcp_tool_call for an unknown name and for a hidden tool - pytest posthog/test/mcp: .venv (mcp 1.30, fastmcp 3.2) 696 passed; v2 (mcp 2.3) 533 passed; v2f4 (mcp 2.3, fastmcp 4.1) 577 passed; f2 (mcp 1.30, fastmcp 2.14) 688 passed, 1 failed - ruff format --check, ruff check, mypy | mypy-baseline filter, make public_api_check: clean Reviewer notes: the f2 failure, test_jlowin_middleware_defaults_work_across_fresh_instances, also fails on origin/main in that venv. The unknown-tool handle-delivery test now covers the error-result branch through standalone FastMCP, since a raw low-level server no longer records unknown tools. --- .sampo/changesets/mcp-v1-spec-align.md | 3 +- posthog/mcp/README.md | 7 +- posthog/mcp/_instrument_lowlevel.py | 16 +-- posthog/mcp/_instrument_v2.py | 19 ++-- posthog/mcp/_instrumentation.py | 65 ++--------- posthog/mcp/_internal.py | 33 ++---- posthog/test/mcp/test_unknown_tool.py | 151 +++++++++++-------------- 7 files changed, 107 insertions(+), 187 deletions(-) diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md index a4ab81ba2..8581e5ddf 100644 --- a/.sampo/changesets/mcp-v1-spec-align.md +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -5,8 +5,7 @@ pypi/posthog: minor MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: - A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. -- A tool your server's `tools/list` handler does not list, such as a hidden or per-user tool, now counts as an unknown tool when a call to it fails. -- On a low-level server whose tool registry the SDK can't read, a failed call runs your `tools/list` handler once to check the name, waiting at most one second. A handler that is slower leaves the call recorded as a failed tool call. +- A raw low-level server with its own `tools/call` dispatcher keeps recording a call to a name it does not know as a failed tool call, since the SDK has no registry there to check the name against. - Manual capture without a `session_id` now sends a fresh `ses_` session id and uses it as the `distinct_id`, instead of no `$session_id` and an `anonymous` person. - Truncated string properties now fit their limit with the `...` marker included, so `$mcp_tool_name` and other names stop at 256 characters instead of 259. - `get_more_tools` and `send_feedback` declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as `false`, and their annotations no longer carry a `title`. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index be3bfd67d..d775473a1 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -351,10 +351,9 @@ registry: - High-level servers and standalone `fastmcp.FastMCP` ask their registry. A disabled tool still exists, so calling it is a failed tool call. -- A raw low-level server's registry is its own `tools/list` handler. A failed - call to a name that handler does not list is unknown. With no `tools/list` - handler, or a name missing from a first page that has more pages after it, the - SDK cannot tell and records a failed tool call. +- A raw low-level server has no registry the SDK can read, and its own + dispatcher decides which names exist, so a failed call there is always a + failed tool call, including one to a name the dispatcher does not know. - On standalone `fastmcp.FastMCP`, middleware of your own that overrides a listing or dispatch hook can serve tools the registry lacks, so the SDK records a failed tool call. FastMCP's stock middleware, other than its diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index c16fb0007..f40a49e93 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -16,7 +16,7 @@ import inspect import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import mcp.types as mcp_types @@ -28,11 +28,11 @@ from ._conversation_id import build_prompt_back from ._event_types import MCPAnalyticsEventType from ._instrumentation import ( + advertised_tool_names, apply_virtual_tool_injection, collect_listed_tools, copy_tools_list_result, extract_tools, - first_listing_page, is_first_listing_page, is_tool_result_error, mutate_tool_schema, @@ -50,7 +50,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) -from ._internal import FirstListingPage, MCPAnalyticsData +from ._internal import MCPAnalyticsData from ._model_parameters import request_meta_from_context from ._model_parameters import can_inject_model_parameter, is_capture_model_enabled from ._output_instructions import mirror_instructions_into_structured_content @@ -285,7 +285,7 @@ async def handler(req: Any) -> Any: result = await original(req) except Exception as error: if await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, high_level, name) + await registered_tool_exists(high_level, name) ): raise # The @server.call_tool() decorator converts raises into @@ -340,7 +340,7 @@ async def handler(req: Any) -> Any: ) if is_tool_result_error(call_result) and await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, high_level, name), + await registered_tool_exists(high_level, name), conversation_id_delivered=delivered, ): return result @@ -413,8 +413,8 @@ def _wrap_list_tools( if original is None or getattr(original, _WRAPPED_FLAG, False): return - async def probe_raw_tool_names(_ctx: Any = None) -> Optional[FirstListingPage]: - """The first page the host's own handler serves. Calls + async def probe_raw_tool_names(_ctx: Any = None) -> Optional[Set[str]]: + """The names the host's own handler advertises on its first page. Calls ``original``, not the wrapper, so the probe never recurses into instrumentation or appends a virtual tool. Read by ``_name_owned_by_real_tool`` on raw low-level servers, which have no @@ -454,7 +454,7 @@ async def probe_raw_tool_names(_ctx: Any = None) -> Optional[FirstListingPage]: context_required=context_required, high_level=high_level, ) - return first_listing_page(result) + return advertised_tool_names(tools) data.raw_tool_names_probe = probe_raw_tool_names diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 3f6b8afe4..33889011f 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -34,7 +34,7 @@ from collections.abc import Mapping from dataclasses import replace from datetime import datetime, timezone -from typing import Any, Dict, FrozenSet, Optional, Tuple +from typing import Any, Dict, FrozenSet, Optional, Set, Tuple import mcp.types as mcp_types @@ -46,12 +46,12 @@ from ._conversation_id import build_prompt_back from ._event_types import MCPAnalyticsEventType from ._instrumentation import ( + advertised_tool_names, apply_virtual_tool_injection, collect_listed_tools, _is_sse_request, _to_jsonable, copy_tools_list_result, - first_listing_page, is_first_listing_page, is_tool_result_error, mutate_tool_schema, @@ -68,7 +68,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) -from ._internal import FirstListingPage, MCPAnalyticsData +from ._internal import MCPAnalyticsData from ._model_parameters import ( can_inject_model_parameter, is_capture_model_enabled, @@ -676,7 +676,7 @@ async def handler(ctx: Any, params: Any) -> Any: result = await original(ctx, params) except Exception as error: if await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, standalone, name, ctx) + await registered_tool_exists(standalone, name) ): raise # Raw v2 handlers raise through to JSON-RPC errors; no freshly minted @@ -696,7 +696,7 @@ async def handler(ctx: Any, params: Any) -> Any: ) if is_tool_result_error(result) and await lifecycle.record_if_unknown_tool( - await registered_tool_exists(data, standalone, name, ctx), + await registered_tool_exists(standalone, name), conversation_id_delivered=delivered, ): return result @@ -790,10 +790,11 @@ def _wrap_v2_list_tools( return original = entry.handler - async def probe_raw_tool_names(ctx: Any = None) -> Optional[FirstListingPage]: - """The host's own first page — ``original``, not the wrapper, so the - probe never recurses or appends a virtual tool.""" - return first_listing_page(await original(ctx, None)) + async def probe_raw_tool_names(ctx: Any = None) -> Optional[Set[str]]: + """The host's own first-page tool names — ``original``, not the wrapper, + so the probe never recurses or appends a virtual tool.""" + result = await original(ctx, None) + return advertised_tool_names(list(getattr(result, "tools", []) or [])) data.raw_tool_names_probe = probe_raw_tool_names diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index 586cf046a..1598b44f1 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -44,7 +44,6 @@ ) from ._intent import resolve_tool_call_intent, set_event_intent from ._internal import ( - FirstListingPage, MCPAnalyticsData, handle_identify, resolve_event_properties, @@ -1000,23 +999,6 @@ def is_first_listing_page(params: Any) -> bool: return getattr(params, "cursor", None) is None -def first_listing_page(result: Any) -> FirstListingPage: - """The names a host's first ``tools/list`` page advertises, and whether a - cursor leads to more. 1.x wraps the result in a ``ServerResult`` and spells - the cursor ``nextCursor``; 2.x returns it bare as ``next_cursor``.""" - root = getattr(result, "root", result) - # fastmcp 4 bridges the 1.x spelling onto 2.x results with a warning. - cursor = ( - root.next_cursor - if hasattr(root, "next_cursor") - else getattr(root, "nextCursor", None) - ) - return FirstListingPage( - advertised_tool_names(list(getattr(root, "tools", []) or [])), - cursor is not None, - ) - - def advertised_tool_names(tools: list) -> Set[str]: """The names a listing page advertises. The virtual tools are appended to a *copy* of the host's result, so a page always reads back as the host wrote @@ -1094,60 +1076,37 @@ def resolve_virtual_tool_injection( return injectable -LISTING_PROBE_DEADLINE_SECONDS = 1.0 - - async def raw_listing_owns_tool_name( - data: MCPAnalyticsData, name: str, ctx: Any = None, *, whole_catalogue: bool = False + data: MCPAnalyticsData, name: str, ctx: Any = None ) -> Optional[bool]: """Whether the host's *own* ``tools/list`` handler advertises ``name`` on its first page, asked at call time. For the raw low-level paths, 1.x and v2, - which have no tool registry to query instead. With ``whole_catalogue``, a - name missing from a page that has more pages after it is unanswered. + which have no tool registry to query instead. Tri-state. ``True``/``False`` are answers; ``None`` means the question could not be asked, and callers must not intercept on it -- guessing "not owned" would swallow a real tool of the host's. Runs once per call to a virtual - tool's name and per failed call, never for ordinary traffic. A listing that - takes longer than ``LISTING_PROBE_DEADLINE_SECONDS`` is unanswered, so a - stalled handler cannot hold the client's response. + tool's name, never for ordinary traffic. """ probe = data.raw_tool_names_probe if probe is None: return None try: - page = await asyncio.wait_for(probe(ctx), LISTING_PROBE_DEADLINE_SECONDS) - except asyncio.TimeoutError: - log(f"PostHog MCP: tools/list did not answer in time to look up {name!r}") - return None + names = await probe(ctx) except Exception as err: # noqa: BLE001 - analytics must not break the call warn_ownership_lookup_failed(name, err) return None - if page is None: - return None - if name in page.names: - return True - return None if whole_catalogue and page.has_more else False + return None if names is None else name in names -async def registered_tool_exists( - data: MCPAnalyticsData, standalone: Any, name: str, ctx: Any = None -) -> Optional[bool]: - """Whether the server registers ``name`` at all, asked after a failed call - to tell an unknown tool from a failed one. jlowin's standalone FastMCP asks - its registry; a raw low-level server asks its whole ``tools/list`` - catalogue. ``None`` when neither can say.""" - if standalone is not None: - return await _standalone_tool_exists(standalone, name) - return await raw_listing_owns_tool_name(data, name, ctx, whole_catalogue=True) - - -async def _standalone_tool_exists(server: Any, name: str) -> Optional[bool]: - """Whether jlowin's FastMCP registers ``name``, enabled or not. A disabled - tool still exists: calling it is a failed call. Middleware that can serve - tools the registry lacks makes the registry no witness, and so does a +async def registered_tool_exists(server: Any, name: str) -> Optional[bool]: + """Whether jlowin's FastMCP registers ``name``, enabled or not, asked after a + failed call to tell an unknown tool from a failed one. A disabled tool still + exists: calling it is a failed call. ``None`` when nothing authoritative can + say: no registry (a raw low-level server, whose listing may hide tools its + dispatcher serves), middleware that can serve tools the registry lacks, or a mounted server or other provider, which hides its disabled tools.""" - if _dispatch_can_differ(server): + if server is None or _dispatch_can_differ(server): return None try: from fastmcp import FastMCP diff --git a/posthog/mcp/_internal.py b/posthog/mcp/_internal.py index 58d907ddc..44a1eeb48 100644 --- a/posthog/mcp/_internal.py +++ b/posthog/mcp/_internal.py @@ -16,28 +16,13 @@ from collections import OrderedDict from dataclasses import dataclass, field from datetime import datetime, timezone -from typing import ( - Any, - Awaitable, - Callable, - Dict, - FrozenSet, - NamedTuple, - Optional, - Set, - Tuple, -) +from typing import Any, Awaitable, Callable, Dict, FrozenSet, Optional, Set, Tuple from .logger import log from ._sink import McpEventSink from .types import MCPAnalyticsOptions, UserIdentity -class FirstListingPage(NamedTuple): - names: Set[str] - has_more: bool - - class IdentityCache: """Bounded LRU of session identities, isolated per server so identities never bleed across server instances.""" @@ -90,14 +75,14 @@ class MCPAnalyticsData: warned_virtual_tool_collisions: Set[Tuple[str, str, str]] = field( default_factory=set ) - # Adapter-supplied probe returning the first page the host's own (original, - # un-instrumented) tools/list handler serves, or None when it can't be - # determined. Registered by the adapters with no tool registry to query — - # raw low-level servers. Takes the adapter's request context, which the 1.x - # handler shape ignores. - raw_tool_names_probe: Optional[ - Callable[[Any], Awaitable[Optional[FirstListingPage]]] - ] = None + # Adapter-supplied probe returning the names the host's own (original, + # un-instrumented) tools/list handler advertises on its first page, or None + # when it can't be determined. Registered by the adapters with no tool + # registry to query — raw low-level servers. Takes the adapter's request + # context, which the 1.x handler shape ignores. + raw_tool_names_probe: Optional[Callable[[Any], Awaitable[Optional[Set[str]]]]] = ( + None + ) last_activity: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) identified_sessions: IdentityCache = field(default_factory=IdentityCache) tool_categories: Dict[str, str] = field(default_factory=dict) diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index 7eeef2846..ce0860eed 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -11,7 +11,7 @@ import mcp.types as mcp_types import pytest -from posthog.mcp import PostHogMCP, _instrumentation as instrumentation, instrument +from posthog.mcp import PostHogMCP, instrument from posthog.mcp.types import MCPAnalyticsOptions from posthog.test.mcp._helpers import ( MCP_MAJOR, @@ -110,39 +110,29 @@ def retired(query: str) -> str: return parent -def _raw(pages, *, reject_with_result=False, stall_listing=False): +def _raw(listed, *, hidden=(), reject_with_result=False, listing_seconds=0.0): """A raw low-level server whose own dispatcher rejects names it does not - know. ``pages`` are the tools/list pages of tool names, chained by cursor; - ``None`` registers no tools/list handler at all. On MCP 2.x the rejection - raises unless ``reject_with_result``; 1.x's decorator always turns it into - an ``isError`` result. With ``stall_listing``, tools/list never answers on - MCP 2.x; on 1.x only a client's does, the server's own cache refresh, which - passes no request, still does.""" + know. tools/list advertises ``listed`` and takes ``listing_seconds`` to + answer; ``hidden`` tools are dispatched but never listed, and fail. On MCP + 2.x a rejection raises unless ``reject_with_result``; 1.x's decorator + always turns it into an ``isError`` result.""" from mcp.server.lowlevel import Server - async def stall_unless_cache_refresh(request): - if stall_listing and (request is not None or MCP_MAJOR >= 2): - await asyncio.Event().wait() - - registered = {name for page in pages or [] for name in page} schema = {"type": "object", "properties": {"query": {"type": "string"}}} - - def page_at(cursor): - index = int(cursor) if cursor else 0 - tools = [ + listing = mcp_types.ListToolsResult( + tools=[ mcp_types.Tool( name=name, **{"input_schema" if MCP_MAJOR >= 2 else "inputSchema": schema}, ) - for name in pages[index] + for name in listed ] - next_cursor = str(index + 1) if index + 1 < len(pages) else None - if MCP_MAJOR >= 2: - return mcp_types.ListToolsResult(tools=tools, next_cursor=next_cursor) - return mcp_types.ListToolsResult(tools=tools, nextCursor=next_cursor) + ) def reply(name): - if name not in registered: + if name in hidden: + raise ValueError(f"Tool disabled for this user: {name}") + if name not in listed: raise ValueError(f"Unknown tool: {name}") return [mcp_types.TextContent(type="text", text="ok")] @@ -160,13 +150,12 @@ async def on_call_tool(ctx, params): ) async def on_list_tools(ctx, params): - await stall_unless_cache_refresh(params) - return page_at(getattr(params, "cursor", None)) + await asyncio.sleep(listing_seconds) + return listing - handlers = {"on_call_tool": on_call_tool} - if pages is not None: - handlers["on_list_tools"] = on_list_tools - return Server("unknown-tool", **handlers) + return Server( + "unknown-tool", on_call_tool=on_call_tool, on_list_tools=on_list_tools + ) server = Server("unknown-tool") @@ -174,14 +163,11 @@ async def on_list_tools(ctx, params): async def call_tool(name, arguments): return reply(name) - if pages is not None: + async def list_tools(req): + await asyncio.sleep(listing_seconds) + return mcp_types.ServerResult(listing) - async def list_tools(req): - await stall_unless_cache_refresh(req) - params = getattr(req, "params", None) if req else None - return mcp_types.ServerResult(page_at(getattr(params, "cursor", None))) - - server.request_handlers[mcp_types.ListToolsRequest] = list_tools + server.request_handlers[mcp_types.ListToolsRequest] = list_tools return server @@ -231,7 +217,6 @@ async def _response(server, name, arguments): "error_handling.ErrorHandlingMiddleware", ) ), - pytest.param(lambda: _raw([["search_docs"]]), id="raw-low-level"), ], ) async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build): @@ -283,13 +268,16 @@ async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build marks=_needs_standalone, ), pytest.param( - lambda: _raw(None), "no_such_tool", {}, id="raw-low-level-without-listing" - ), - pytest.param( - lambda: _raw([["search_docs"], ["export"]]), + lambda: _raw(["search_docs"]), "no_such_tool", {}, - id="raw-low-level-with-more-pages", + id="raw-low-level-unknown-name", + ), + pytest.param( + lambda: _raw(["search_docs"], hidden=["export"]), + "export", + {"query": "q"}, + id="raw-low-level-hidden-tool", ), ], ) @@ -328,11 +316,12 @@ async def test_a_custom_dispatcher_captures_an_unknown_tool(): _SHAPES = [ pytest.param(_high_level, id="high-level"), pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), - pytest.param(lambda: _raw([["search_docs"]]), id="raw-low-level"), ] -@pytest.mark.parametrize("build", _SHAPES) +@pytest.mark.parametrize( + "build", [*_SHAPES, pytest.param(lambda: _raw(["search_docs"]), id="raw-low-level")] +) async def test_a_call_naming_no_tool_emits_nothing(build): expected = await _response(build(), "", {}) client = FakeClient() @@ -387,9 +376,7 @@ def test_a_custom_dispatcher_follows_the_unknown_tool_name_rules( "build, delivers_handle", [ pytest.param( - lambda: _raw([["search_docs"]], reject_with_result=True), - True, - id="raw-low-level-error-result", + _standalone, True, id="standalone-error-result", marks=_needs_standalone ), pytest.param(_high_level, False, id="high-level-raise"), ], @@ -416,50 +403,40 @@ async def test_an_unknown_tool_result_carries_the_conversation_handle_it_deliver ) -@pytest.mark.skipif(MCP_MAJOR < 2, reason="MCP 2 renamed nextCursor to next_cursor") -async def test_checking_the_host_listing_never_reads_the_deprecated_cursor( - monkeypatch, -): - """fastmcp 4 installs a ``nextCursor`` property on MCP 2 results that warns - on first read; this one stands in for it and counts the reads.""" - reads = [] - - def deprecated_cursor(result): - reads.append(result) - return result.next_cursor - - monkeypatch.setattr( - mcp_types.ListToolsResult, - "nextCursor", - property(deprecated_cursor), - raising=False, - ) - client = FakeClient() - server = _raw([["search_docs"]], reject_with_result=True) - instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) - - await _response(server, "no_such_tool", {}) - await flush_background() - - assert reads == [] - assert len(events_named(client, "$mcp_unknown_tool")) == 1 - - -async def test_a_stalled_listing_does_not_delay_a_failed_calls_error(monkeypatch): - monkeypatch.setattr(instrumentation, "LISTING_PROBE_DEADLINE_SECONDS", 0.05) +@pytest.mark.parametrize( + "name, arguments, event", + [ + pytest.param( + "get_more_tools", + {"context": "bulk export"}, + "$mcp_missing_capability", + id="get-more-tools", + ), + pytest.param( + "send_feedback", + {"feedback_type": "other", "summary": "Slow export."}, + "$mcp_feedback", + id="send-feedback", + ), + ], +) +async def test_a_slow_listing_still_answers_a_virtual_tool_call(name, arguments, event): client = FakeClient() - server = _raw([["search_docs"]], reject_with_result=True, stall_listing=True) - instrument(server, client, MCPAnalyticsOptions(enable_conversation_id=False)) - - is_error, _ = await asyncio.wait_for( - _response(server, "no_such_tool", {}), timeout=5 + server = _raw(["search_docs"], reject_with_result=True, listing_seconds=1.1) + instrument( + server, + client, + MCPAnalyticsOptions( + enable_conversation_id=False, report_missing=True, collect_feedback=True + ), ) + + is_error, _ = await _response(server, name, arguments) await flush_background() - assert is_error is True - [call] = events_named(client, "$mcp_tool_call") - assert call["properties"]["$mcp_is_error"] is True - assert events_named(client, "$mcp_unknown_tool") == [] + assert is_error is False + assert len(events_named(client, event)) == 1 + assert events_named(client, "$mcp_tool_call") == [] @pytest.mark.parametrize( From 5799773188429f928ed1c3ee36f50eba5acf41df Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 18:05:37 -0300 Subject: [PATCH 6/7] fix(mcp): drop the manual unknown-tool API and shrink the spec alignment What: removed the public PostHogMCP.capture_unknown_tool() and the public PostHogMCPAnalyticsEvent.UNKNOWN_TOOL constant; the event name now lives in the private event-name map. The registry helpers stay in _instrument_lowlevel next to _dispatch_can_differ instead of moving to _instrumentation. Dropped the slow-listing and hidden-tool test cases (they guarded a listing probe this branch no longer has and pass on main), the lowlevel test fixture edits, and README text that documented #1204's annotations. The manual-session test moved to test_posthog_mcp.py. Why: stamphog refused #1199 for a public API change and its size. The spec makes manual capture methods optional, so capture_unknown_tool can return in its own PR. Tested with: pytest posthog/test/mcp on .venv, v2, f2, v2f4 (689, 526, 681, 570 passed; f2's one failure is test_jlowin_middleware_defaults_work_across_fresh_instances, failing on main until #1203). ruff format/check, mypy-baseline filter clean. make public_api_check: snapshot up to date, identical to origin/main. --- .sampo/changesets/mcp-v1-spec-align.md | 9 +- posthog/mcp/README.md | 37 +++---- posthog/mcp/_instrument_lowlevel.py | 127 +++++++++++++++++++++++- posthog/mcp/_instrument_v2.py | 2 +- posthog/mcp/_instrumentation.py | 129 +------------------------ posthog/mcp/_posthog_events.py | 2 +- posthog/mcp/constants.py | 1 - posthog/mcp/posthog_mcp.py | 39 -------- posthog/test/mcp/test_lowlevel.py | 5 +- posthog/test/mcp/test_posthog_mcp.py | 14 +++ posthog/test/mcp/test_unknown_tool.py | 129 +------------------------ posthog/test/mcp/test_v2_lowlevel.py | 14 +-- references/public_api_snapshot.txt | 2 - 13 files changed, 166 insertions(+), 344 deletions(-) diff --git a/.sampo/changesets/mcp-v1-spec-align.md b/.sampo/changesets/mcp-v1-spec-align.md index 8581e5ddf..718b5a7a0 100644 --- a/.sampo/changesets/mcp-v1-spec-align.md +++ b/.sampo/changesets/mcp-v1-spec-align.md @@ -4,12 +4,7 @@ pypi/posthog: minor MCP analytics now matches the cross-SDK MCP spec ahead of 1.0: -- A `tools/call` naming a tool your server never registered sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception`. Custom dispatchers can send it with `PostHogMCP.capture_unknown_tool()`. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. -- A raw low-level server with its own `tools/call` dispatcher keeps recording a call to a name it does not know as a failed tool call, since the SDK has no registry there to check the name against. +- `instrument()` sends `$mcp_unknown_tool` instead of a failed `$mcp_tool_call` and `$exception` when a `tools/call` names a tool your server never registered. Insights that count failed tool calls stop counting these. A call naming no tool sends nothing. A raw low-level server has no registry the SDK can read, so a failed call there stays a failed tool call. +- FastMCP's stock logging, timing, error-handling, and similar middleware no longer turns off `llm_model` injection on standalone `fastmcp.FastMCP`. - Manual capture without a `session_id` now sends a fresh `ses_` session id and uses it as the `distinct_id`, instead of no `$session_id` and an `anonymous` person. - Truncated string properties now fit their limit with the `...` marker included, so `$mcp_tool_name` and other names stop at 256 characters instead of 259. -- `get_more_tools` and `send_feedback` declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and `openWorldHint` as `false`, and their annotations no longer carry a `title`. -- The injected `context` argument has a shorter description, the same in every PostHog MCP SDK. -- The injected `llm_model` argument is always optional. - -The annotations and descriptions change your published tool definitions. If your server is listed in the ChatGPT or Claude directories, those platforms may re-review the updated tools. diff --git a/posthog/mcp/README.md b/posthog/mcp/README.md index d775473a1..464036161 100644 --- a/posthog/mcp/README.md +++ b/posthog/mcp/README.md @@ -343,41 +343,26 @@ call = posthog.prepare_tool_call( ## Calls to tools your server does not have -A `tools/call` that names a tool your server never registered is not a tool -call. `instrument()` sends `$mcp_unknown_tool` with the requested name as +A `tools/call` naming a tool your server never registered is not a tool call. +`instrument()` sends `$mcp_unknown_tool` with the requested name as `$mcp_tool_name`, and no `$mcp_tool_call` or `$exception`. The agent gets the -same error as before. The SDK decides only after the call fails, from the tool -registry: - -- High-level servers and standalone `fastmcp.FastMCP` ask their registry. A - disabled tool still exists, so calling it is a failed tool call. -- A raw low-level server has no registry the SDK can read, and its own - dispatcher decides which names exist, so a failed call there is always a - failed tool call, including one to a name the dispatcher does not know. -- On standalone `fastmcp.FastMCP`, middleware of your own that overrides a - listing or dispatch hook can serve tools the registry lacks, so the SDK - records a failed tool call. FastMCP's stock middleware, other than its - tool-injection middleware, does not count. -- On standalone `fastmcp.FastMCP` 3 and later with a mounted server, a proxy, - or another provider, the provider hides its disabled tools, so a name the SDK - can't find is a failed tool call. +same error as before. A call naming no tool sends nothing. -A call naming no tool sends nothing. +The SDK asks the tool registry only after the call fails, and records a failed +tool call whenever the registry can't rule the tool out: -Invalid arguments for a tool that exists are a failed tool call. A custom -dispatcher calls `posthog.capture_unknown_tool(tool_name, session_id=...)` in -place of `capture_tool_call` when its own lookup finds no tool. +- A disabled tool, or a tool called with invalid arguments, still exists. +- A raw low-level server has no registry the SDK can read. +- On standalone `fastmcp.FastMCP`, middleware of your own that overrides a + listing or dispatch hook can serve tools the registry lacks. +- On standalone `fastmcp.FastMCP` 3 and later, a mounted server, proxy, or + other provider hides its disabled tools. ## Virtual tools on a paginated `tools/list` `instrument()` advertises up to two tools of its own: `get_more_tools` (`report_missing=True`) and `send_feedback` (`collect_feedback=True`). -Both declare `readOnlyHint`, `destructiveHint`, `idempotentHint`, and -`openWorldHint` as `false`, and no option changes that. A call records one event -in your PostHog project: it writes outside the conversation, deletes nothing, -records again on every repeat, and reaches nothing else. - A client concatenates every `tools/list` page into one list, so each virtual tool is appended to the **first page only** — the page every client reads, including clients that never follow `nextCursor`. "First page" means a request with no diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index f40a49e93..1635cae58 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -14,9 +14,12 @@ from __future__ import annotations +import functools +import importlib import inspect import time -from typing import Any, Dict, List, Optional, Set, Tuple +from functools import lru_cache +from typing import Any, Dict, List, Optional, Set, Tuple, Type import mcp.types as mcp_types @@ -37,10 +40,7 @@ is_tool_result_error, mutate_tool_schema, prepare_request, - _dispatch_can_differ, - _tool_lookup_not_found_errors, raw_listing_owns_tool_name, - registered_tool_exists, record_resource_request, request_to_dict, resource_listing_response, @@ -584,6 +584,26 @@ async def _name_owned_by_real_tool( return await raw_listing_owns_tool_name(data, name, server) +@lru_cache(maxsize=1) +def _tool_lookup_not_found_errors() -> Tuple[Type[BaseException], ...]: + """The fastmcp exceptions that mean "no live tool by that name" -- an answer. + Anything else out of ``get_tool`` is the lookup itself failing. Empty when + fastmcp is absent or has moved them, which makes every failure delegate: + the safe direction.""" + try: + from fastmcp import exceptions + except Exception: # noqa: BLE001 - no fastmcp on this path + return () + return tuple( + err + for err in ( + getattr(exceptions, "NotFoundError", None), + getattr(exceptions, "DisabledError", None), + ) + if isinstance(err, type) and issubclass(err, BaseException) + ) + + _INJECTED_KEYS = ("context", "conversation_id", "llm_model") @@ -761,6 +781,105 @@ def _server_dereferences(server: Any) -> bool: ) +_DISPATCH_HOOKS = ("on_message", "on_request", "on_list_tools", "on_call_tool") + + +def _dispatch_can_differ(server: Any) -> bool: + """Whether application middleware can provide, shadow, or reroute a tool, + making the registry an unreliable witness for what actually runs. Any + override of a listing or dispatch hook can; FastMCP's own built-ins are + excluded. Without a trustworthy registry a cold instance keeps ``llm_model`` + (strips fail closed), which is exactly what it did before this path.""" + try: + from fastmcp.server.middleware import Middleware + except ImportError: # FastMCP before middleware existed: nothing can differ + return False + + return any( + type(middleware) not in _builtin_middleware_types() # subclasses are the app's + and any( + getattr(type(middleware), hook) is not getattr(Middleware, hook) + for hook in _DISPATCH_HOOKS + ) + for middleware in getattr(server, "middleware", ()) + ) + + +# Stock fastmcp middleware that observes, times, retries, throttles, denies, +# caches, or trims a call to the tool it names, and never provides, renames, or +# reroutes one. The tool-injection family is left out on purpose. +_STOCK_DISPATCH_PRESERVING_MIDDLEWARE = ( + ("logging", "LoggingMiddleware"), + ("logging", "StructuredLoggingMiddleware"), + ("timing", "TimingMiddleware"), + ("timing", "DetailedTimingMiddleware"), + ("error_handling", "ErrorHandlingMiddleware"), + ("error_handling", "RetryMiddleware"), + ("rate_limiting", "RateLimitingMiddleware"), + ("rate_limiting", "SlidingWindowRateLimitingMiddleware"), + ("ping", "PingMiddleware"), + ("authorization", "AuthMiddleware"), + ("caching", "ResponseCachingMiddleware"), + ("response_limiting", "ResponseLimitingMiddleware"), +) + + +@functools.lru_cache(maxsize=1) +def _builtin_middleware_types() -> tuple: + """The middleware a bare ``FastMCP()`` installs on its own, probed rather + than named so a new built-in in a later release is still recognised, plus + the stock middleware above that this fastmcp release ships.""" + from fastmcp import FastMCP + + stock = [] + for module, name in _STOCK_DISPATCH_PRESERVING_MIDDLEWARE: + try: + stock.append( + getattr( + importlib.import_module(f"fastmcp.server.middleware.{module}"), name + ) + ) + except (ImportError, AttributeError): + continue + return tuple( + type(middleware) for middleware in FastMCP("posthog-probe").middleware + ) + tuple(stock) + + +async def registered_tool_exists(server: Any, name: str) -> Optional[bool]: + """Whether jlowin's FastMCP registers ``name``, enabled or not, asked after a + failed call to tell an unknown tool from a failed one. A disabled tool still + exists: calling it is a failed call. ``None`` when nothing authoritative can + say: no registry (a raw low-level server, whose listing may hide tools its + dispatcher serves), middleware that can serve tools the registry lacks, or a + mounted server or other provider, which hides its disabled tools.""" + if server is None or _dispatch_can_differ(server): + return None + try: + from fastmcp import FastMCP + + # From 3.x, get_tool hides a disabled tool; the providers still hold it. + found = ( + await server.get_tool(name) is not None + or await super(FastMCP, server).get_tool(name) is not None + ) + except _tool_lookup_not_found_errors(): + found = False + except Exception as err: # noqa: BLE001 - analytics must not break the call + log(f"PostHog MCP: could not look up tool {name!r} - {err}") + return None + if found: + return True + return None if _serves_other_providers(server) else False + + +def _serves_other_providers(server: Any) -> bool: + """Whether a FastMCP 3+ server serves tools from anywhere but its own + registry: a mounted server, a proxy, or a custom provider.""" + local = getattr(server, "_local_provider", None) + return any(provider is not local for provider in getattr(server, "providers", ())) + + def _reference_chain(schema: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]: """The schema and each local root ``$ref`` target in turn; ``None`` for a reference that is external, dangling, or cyclic.""" diff --git a/posthog/mcp/_instrument_v2.py b/posthog/mcp/_instrument_v2.py index 33889011f..b7d597789 100644 --- a/posthog/mcp/_instrument_v2.py +++ b/posthog/mcp/_instrument_v2.py @@ -58,7 +58,6 @@ params_to_request_dict, prepare_request, raw_listing_owns_tool_name, - registered_tool_exists, record_initialize, record_resource_request, resource_listing_response, @@ -68,6 +67,7 @@ start_tools_list_lifecycle, warn_ownership_lookup_failed, ) +from ._instrument_lowlevel import registered_tool_exists from ._internal import MCPAnalyticsData from ._model_parameters import ( can_inject_model_parameter, diff --git a/posthog/mcp/_instrumentation.py b/posthog/mcp/_instrumentation.py index 1598b44f1..9b1db44c7 100644 --- a/posthog/mcp/_instrumentation.py +++ b/posthog/mcp/_instrumentation.py @@ -11,13 +11,11 @@ import asyncio import concurrent.futures import copy -import functools -import importlib import os import threading from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, FrozenSet, List, Literal, Optional, Set, Tuple, Type +from typing import Any, Dict, FrozenSet, List, Literal, Optional, Set from ._capture import capture_event from ._context_parameters import ( @@ -43,11 +41,7 @@ resolve_send_feedback_tool_name, ) from ._intent import resolve_tool_call_intent, set_event_intent -from ._internal import ( - MCPAnalyticsData, - handle_identify, - resolve_event_properties, -) +from ._internal import MCPAnalyticsData, handle_identify, resolve_event_properties from ._tool_input import get_tool_input_properties from ._model_parameters import ( add_model_parameter_to_schema, @@ -1099,125 +1093,6 @@ async def raw_listing_owns_tool_name( return None if names is None else name in names -async def registered_tool_exists(server: Any, name: str) -> Optional[bool]: - """Whether jlowin's FastMCP registers ``name``, enabled or not, asked after a - failed call to tell an unknown tool from a failed one. A disabled tool still - exists: calling it is a failed call. ``None`` when nothing authoritative can - say: no registry (a raw low-level server, whose listing may hide tools its - dispatcher serves), middleware that can serve tools the registry lacks, or a - mounted server or other provider, which hides its disabled tools.""" - if server is None or _dispatch_can_differ(server): - return None - try: - from fastmcp import FastMCP - - # From 3.x, get_tool hides a disabled tool; the providers still hold it. - found = ( - await server.get_tool(name) is not None - or await super(FastMCP, server).get_tool(name) is not None - ) - except _tool_lookup_not_found_errors(): - found = False - except Exception as err: # noqa: BLE001 - analytics must not break the call - log(f"PostHog MCP: could not look up tool {name!r} - {err}") - return None - if found: - return True - return None if _serves_other_providers(server) else False - - -def _serves_other_providers(server: Any) -> bool: - """Whether a FastMCP 3+ server serves tools from anywhere but its own - registry: a mounted server, a proxy, or a custom provider.""" - local = getattr(server, "_local_provider", None) - return any(provider is not local for provider in getattr(server, "providers", ())) - - -@functools.lru_cache(maxsize=1) -def _tool_lookup_not_found_errors() -> Tuple[Type[BaseException], ...]: - """The fastmcp exceptions that mean "no live tool by that name" -- an answer. - Anything else out of ``get_tool`` is the lookup itself failing. Empty when - fastmcp is absent or has moved them, which makes every failure delegate: - the safe direction.""" - try: - from fastmcp import exceptions - except Exception: # noqa: BLE001 - no fastmcp on this path - return () - return tuple( - err - for err in ( - getattr(exceptions, "NotFoundError", None), - getattr(exceptions, "DisabledError", None), - ) - if isinstance(err, type) and issubclass(err, BaseException) - ) - - -_DISPATCH_HOOKS = ("on_message", "on_request", "on_list_tools", "on_call_tool") - - -def _dispatch_can_differ(server: Any) -> bool: - """Whether application middleware can provide, shadow, or reroute a tool, - making the registry an unreliable witness for what actually runs. Any - override of a listing or dispatch hook can; FastMCP's own built-ins are - excluded. Without a trustworthy registry a cold instance keeps ``llm_model`` - (strips fail closed), which is exactly what it did before this path.""" - try: - from fastmcp.server.middleware import Middleware - except ImportError: # FastMCP before middleware existed: nothing can differ - return False - - return any( - type(middleware) - not in _dispatch_preserving_middleware_types() # subclasses are the app's - and any( - getattr(type(middleware), hook) is not getattr(Middleware, hook) - for hook in _DISPATCH_HOOKS - ) - for middleware in getattr(server, "middleware", ()) - ) - - -# Stock fastmcp middleware that observes, times, retries, throttles, denies, -# caches, or trims a call to the tool it names, and never provides, renames, or -# reroutes one. The tool-injection family is left out on purpose. -_STOCK_DISPATCH_PRESERVING_MIDDLEWARE = ( - ("logging", "LoggingMiddleware"), - ("logging", "StructuredLoggingMiddleware"), - ("timing", "TimingMiddleware"), - ("timing", "DetailedTimingMiddleware"), - ("error_handling", "ErrorHandlingMiddleware"), - ("error_handling", "RetryMiddleware"), - ("rate_limiting", "RateLimitingMiddleware"), - ("rate_limiting", "SlidingWindowRateLimitingMiddleware"), - ("ping", "PingMiddleware"), - ("authorization", "AuthMiddleware"), - ("caching", "ResponseCachingMiddleware"), - ("response_limiting", "ResponseLimitingMiddleware"), -) - - -@functools.lru_cache(maxsize=1) -def _dispatch_preserving_middleware_types() -> FrozenSet[type]: - """The middleware a bare ``FastMCP()`` installs on its own, probed rather - than named so a new built-in in a later release is still recognised, plus - the stock middleware above that this fastmcp release ships.""" - from fastmcp import FastMCP - - stock = set() - for module, name in _STOCK_DISPATCH_PRESERVING_MIDDLEWARE: - try: - found = getattr( - importlib.import_module(f"fastmcp.server.middleware.{module}"), name - ) - except (ImportError, AttributeError): - continue - stock.add(found) - return frozenset( - {type(middleware) for middleware in FastMCP("posthog-probe").middleware} | stock - ) - - def warn_ownership_lookup_failed(name: str, err: Exception) -> None: """A tool-ownership lookup raised instead of answering. Every adapter's probe reports it the same way and then returns ``None``, so the call is delegated diff --git a/posthog/mcp/_posthog_events.py b/posthog/mcp/_posthog_events.py index 7b087fd2b..f7b58f0e7 100644 --- a/posthog/mcp/_posthog_events.py +++ b/posthog/mcp/_posthog_events.py @@ -29,7 +29,7 @@ MCPAnalyticsEventType.MCP_RESOURCES_READ: PostHogMCPAnalyticsEvent.RESOURCE_READ, MCPAnalyticsEventType.MCP_TOOLS_CALL: PostHogMCPAnalyticsEvent.TOOL_CALL, MCPAnalyticsEventType.MCP_TOOLS_LIST: PostHogMCPAnalyticsEvent.TOOLS_LIST, - MCPAnalyticsEventType.MCP_UNKNOWN_TOOL: PostHogMCPAnalyticsEvent.UNKNOWN_TOOL, + MCPAnalyticsEventType.MCP_UNKNOWN_TOOL: "$mcp_unknown_tool", } _P = PostHogMCPAnalyticsProperty diff --git a/posthog/mcp/constants.py b/posthog/mcp/constants.py index 345536e30..5fc725b35 100644 --- a/posthog/mcp/constants.py +++ b/posthog/mcp/constants.py @@ -55,7 +55,6 @@ class PostHogMCPAnalyticsEvent: RESOURCES_LIST = "$mcp_resources_list" TOOL_CALL = "$mcp_tool_call" TOOLS_LIST = "$mcp_tools_list" - UNKNOWN_TOOL = "$mcp_unknown_tool" class PostHogMCPAnalyticsProperty: diff --git a/posthog/mcp/posthog_mcp.py b/posthog/mcp/posthog_mcp.py index 5a3e47d06..2e1c4d3e5 100644 --- a/posthog/mcp/posthog_mcp.py +++ b/posthog/mcp/posthog_mcp.py @@ -105,7 +105,6 @@ class _ConversationOwnership: class PostHogMCP(Client): """A drop-in posthog ``Client`` with ``capture_tool_call`` / ``capture_initialize`` / ``capture_tools_list`` / ``capture_missing_capability`` / ``capture_feedback`` - / ``capture_unknown_tool`` plus ``prepare_tool_list``, ``prepare_tool_call`` and ``prepare_tool_result`` helpers. ``capture`` adds the configured server build to custom events. Other inherited methods work unchanged.""" @@ -372,44 +371,6 @@ def capture_missing_capability( _apply_model(event, llm_model, llm_model_source) self._emit(event) - def capture_unknown_tool( - self, - tool_name: str, - *, - protocol_version: Optional[str] = None, - distinct_id: Optional[str] = None, - session_id: Optional[str] = None, - conversation_id: Optional[str] = None, - client_user_agent: Optional[str] = None, - vendor_client: Optional[str] = None, - set_properties: Optional[JsonRecord] = None, - groups: Optional[Dict[str, str]] = None, - properties: Optional[JsonRecord] = None, - timestamp: Optional[datetime] = None, - ) -> None: - """Capture a call naming a tool your server does not have. Emits - ``$mcp_unknown_tool`` with the requested name as ``$mcp_tool_name``. - Capture it instead of :meth:`capture_tool_call`, not as well: it is - not a tool call. A tool that exists, disabled or called with invalid - arguments, is a failed tool call. A call naming no tool emits nothing.""" - if not tool_name: - return - event = self._base_event( - MCPAnalyticsEventType.MCP_UNKNOWN_TOOL, - distinct_id, - session_id, - conversation_id, - set_properties, - groups, - properties, - timestamp, - client_user_agent, - vendor_client, - ) - event["resource_name"] = tool_name - event["protocol_version"] = protocol_version - self._emit(event) - def capture_feedback( self, *, diff --git a/posthog/test/mcp/test_lowlevel.py b/posthog/test/mcp/test_lowlevel.py index dbc79ff2b..62dbd1ed0 100644 --- a/posthog/test/mcp/test_lowlevel.py +++ b/posthog/test/mcp/test_lowlevel.py @@ -36,7 +36,7 @@ async def list_tools(): @server.call_tool() async def call_tool(name, arguments): - if name == "echo" and arguments.get("msg") != "boom": + if name == "echo": return [mcp_types.TextContent(type="text", text=str(arguments.get("msg")))] raise ValueError("boom") @@ -350,8 +350,9 @@ async def test_tool_call_error_captured_from_is_error_result(): instrument(server, client) handler = server.request_handlers[mcp_types.CallToolRequest] + # "fail" is unlisted -> no validation -> handler raises -> isError result result = await handler( - _call_request("echo", {"msg": "boom", "context": "trying a tool that errors"}) + _call_request("fail", {"context": "trying a tool that errors"}) ) await _flush() diff --git a/posthog/test/mcp/test_posthog_mcp.py b/posthog/test/mcp/test_posthog_mcp.py index 51bbb679b..fb2e69961 100644 --- a/posthog/test/mcp/test_posthog_mcp.py +++ b/posthog/test/mcp/test_posthog_mcp.py @@ -783,3 +783,17 @@ async def test_capture_tool_call_records_prepared_conversation_and_session(): props = _events(captured, "$mcp_tool_call")[0]["properties"] assert props["$mcp_conversation_id"] == prepared.conversation_id assert props["$session_id"] == prepared.session_id + + +def test_capture_without_a_session_gets_a_fresh_one_per_call(): + client, captured = make_client() + + client.capture_tool_call("t") + client.capture_tool_call("t") + client.flush() + + first, second = captured + session = first["properties"]["$session_id"] + assert session.startswith("ses_") + assert first["distinct_id"] == session + assert second["properties"]["$session_id"] != session diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index ce0860eed..142f16e00 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -2,7 +2,6 @@ not a tool call. Runs under both MCP SDK majors over an in-process client, so the response compared against an uninstrumented twin is the one an agent gets.""" -import asyncio import importlib import importlib.util import json @@ -11,7 +10,7 @@ import mcp.types as mcp_types import pytest -from posthog.mcp import PostHogMCP, instrument +from posthog.mcp import instrument from posthog.mcp.types import MCPAnalyticsOptions from posthog.test.mcp._helpers import ( MCP_MAJOR, @@ -110,12 +109,9 @@ def retired(query: str) -> str: return parent -def _raw(listed, *, hidden=(), reject_with_result=False, listing_seconds=0.0): +def _raw(listed): """A raw low-level server whose own dispatcher rejects names it does not - know. tools/list advertises ``listed`` and takes ``listing_seconds`` to - answer; ``hidden`` tools are dispatched but never listed, and fail. On MCP - 2.x a rejection raises unless ``reject_with_result``; 1.x's decorator - always turns it into an ``isError`` result.""" + know and advertises ``listed`` on tools/list.""" from mcp.server.lowlevel import Server schema = {"type": "object", "properties": {"query": {"type": "string"}}} @@ -130,8 +126,6 @@ def _raw(listed, *, hidden=(), reject_with_result=False, listing_seconds=0.0): ) def reply(name): - if name in hidden: - raise ValueError(f"Tool disabled for this user: {name}") if name not in listed: raise ValueError(f"Unknown tool: {name}") return [mcp_types.TextContent(type="text", text="ok")] @@ -139,18 +133,9 @@ def reply(name): if MCP_MAJOR >= 2: async def on_call_tool(ctx, params): - try: - return mcp_types.CallToolResult(content=reply(params.name)) - except ValueError as error: - if not reject_with_result: - raise - return mcp_types.CallToolResult( - content=[mcp_types.TextContent(type="text", text=str(error))], - is_error=True, - ) + return mcp_types.CallToolResult(content=reply(params.name)) async def on_list_tools(ctx, params): - await asyncio.sleep(listing_seconds) return listing return Server( @@ -164,7 +149,6 @@ async def call_tool(name, arguments): return reply(name) async def list_tools(req): - await asyncio.sleep(listing_seconds) return mcp_types.ServerResult(listing) server.request_handlers[mcp_types.ListToolsRequest] = list_tools @@ -273,12 +257,6 @@ async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build {}, id="raw-low-level-unknown-name", ), - pytest.param( - lambda: _raw(["search_docs"], hidden=["export"]), - "export", - {"query": "q"}, - id="raw-low-level-hidden-tool", - ), ], ) async def test_a_call_the_sdk_cannot_rule_out_is_a_failed_tool_call( @@ -298,21 +276,6 @@ async def test_a_call_the_sdk_cannot_rule_out_is_a_failed_tool_call( assert events_named(client, "$mcp_unknown_tool") == [] -async def test_a_custom_dispatcher_captures_an_unknown_tool(): - posthog = PostHogMCP("phc_test") - captured = [] - posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) - - posthog.capture_unknown_tool("export for jane@example.com", session_id="ses_1") - posthog.flush() - - [unknown] = events_named(captured, "$mcp_unknown_tool") - assert unknown["properties"]["$mcp_tool_name"] == "export for [redacted]" - assert unknown["properties"]["$session_id"] == "ses_1" - assert "$mcp_resource_name" not in unknown["properties"] - assert events_named(captured, "$mcp_tool_call") == [] - - _SHAPES = [ pytest.param(_high_level, id="high-level"), pytest.param(_standalone, id="standalone-fastmcp", marks=_needs_standalone), @@ -349,29 +312,6 @@ async def test_an_unknown_tool_name_is_cut_to_256_characters(build): assert name.startswith("x" * 200) -@pytest.mark.parametrize( - "tool_name, expected_names", - [ - pytest.param("", [], id="no-tool"), - pytest.param("y" * 5000, [256], id="long-name"), - ], -) -def test_a_custom_dispatcher_follows_the_unknown_tool_name_rules( - tool_name, expected_names -): - posthog = PostHogMCP("phc_test") - captured = [] - posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) - - posthog.capture_unknown_tool(tool_name, session_id="ses_1") - posthog.flush() - - assert [ - len(e["properties"]["$mcp_tool_name"]) - for e in events_named(captured, "$mcp_unknown_tool") - ] == expected_names - - @pytest.mark.parametrize( "build, delivers_handle", [ @@ -401,64 +341,3 @@ async def test_an_unknown_tool_result_carries_the_conversation_handle_it_deliver assert unknown["properties"].get("$mcp_conversation_id") == next( iter(handles), None ) - - -@pytest.mark.parametrize( - "name, arguments, event", - [ - pytest.param( - "get_more_tools", - {"context": "bulk export"}, - "$mcp_missing_capability", - id="get-more-tools", - ), - pytest.param( - "send_feedback", - {"feedback_type": "other", "summary": "Slow export."}, - "$mcp_feedback", - id="send-feedback", - ), - ], -) -async def test_a_slow_listing_still_answers_a_virtual_tool_call(name, arguments, event): - client = FakeClient() - server = _raw(["search_docs"], reject_with_result=True, listing_seconds=1.1) - instrument( - server, - client, - MCPAnalyticsOptions( - enable_conversation_id=False, report_missing=True, collect_feedback=True - ), - ) - - is_error, _ = await _response(server, name, arguments) - await flush_background() - - assert is_error is False - assert len(events_named(client, event)) == 1 - assert events_named(client, "$mcp_tool_call") == [] - - -@pytest.mark.parametrize( - "capture", - [ - pytest.param(lambda posthog: posthog.capture_tool_call("t"), id="tool-call"), - pytest.param( - lambda posthog: posthog.capture_unknown_tool("t"), id="unknown-tool" - ), - ], -) -def test_manual_capture_without_a_session_gets_a_fresh_one_per_call(capture): - posthog = PostHogMCP("phc_test") - captured = [] - posthog.capture = lambda event, **kw: captured.append({"event": event, **kw}) - - capture(posthog) - capture(posthog) - posthog.flush() - - first, second = captured - session = first["properties"]["$session_id"] - assert session.startswith("ses_") - assert first["distinct_id"] == session - assert second["properties"]["$session_id"] != session diff --git a/posthog/test/mcp/test_v2_lowlevel.py b/posthog/test/mcp/test_v2_lowlevel.py index 09340dae3..319cbc3eb 100644 --- a/posthog/test/mcp/test_v2_lowlevel.py +++ b/posthog/test/mcp/test_v2_lowlevel.py @@ -28,14 +28,14 @@ def make_server(*, resource_error: bool = False, listing: str = "resources") -> Server: async def on_call_tool(ctx, params): - args = params.arguments or {} - if args.get("fail") == "raise": + if params.name == "boom": raise ValueError("explode") - if args.get("fail") == "result": + if params.name == "soft-fail": return mcp_types.CallToolResult( content=[mcp_types.TextContent(type="text", text="failed politely")], is_error=True, ) + args = params.arguments or {} return mcp_types.CallToolResult( content=[ mcp_types.TextContent( @@ -410,11 +410,7 @@ async def test_raised_error_is_captured_and_reraised(): instrument(server, client) with pytest.raises(ValueError): - await _call_tool( - server, - "add", - {"fail": "raise", "context": "attempting the risky operation"}, - ) + await _call_tool(server, "boom", {"context": "attempting the risky operation"}) await _flush() calls = _events(client, "$mcp_tool_call") @@ -430,7 +426,7 @@ async def test_is_error_result_is_captured(): instrument(server, client) result = await _call_tool( - server, "add", {"fail": "result", "context": "expecting a polite failure"} + server, "soft-fail", {"context": "expecting a polite failure"} ) await _flush() diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index 98e77b016..bfffd5d64 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -1023,7 +1023,6 @@ attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.RESOURCES_LIST = '$mcp_ attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.RESOURCE_READ = '$mcp_resource_read' attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.TOOLS_LIST = '$mcp_tools_list' attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.TOOL_CALL = '$mcp_tool_call' -attribute posthog.mcp.constants.PostHogMCPAnalyticsEvent.UNKNOWN_TOOL = '$mcp_unknown_tool' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_NAME = '$mcp_client_name' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_USER_AGENT = '$mcp_client_user_agent' attribute posthog.mcp.constants.PostHogMCPAnalyticsProperty.CLIENT_VERSION = '$mcp_client_version' @@ -1894,7 +1893,6 @@ method posthog.mcp.posthog_mcp.PostHogMCP.capture_initialize(*, client_name: Opt method posthog.mcp.posthog_mcp.PostHogMCP.capture_missing_capability(*, context: Optional[str] = None, llm_model: Optional[str] = None, llm_model_source: Optional[MCPAnalyticsModelSource] = None, parameters: Any = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.capture_tool_call(tool_name: str, *, intent: Optional[str] = None, intent_source: Optional[str] = None, parameters: Any = None, response: Any = None, duration_ms: Optional[float] = None, is_error: bool = False, error: Any = None, error_type: Optional[str] = None, category: Optional[str] = None, tool_description: Optional[str] = None, llm_model: Optional[str] = None, llm_model_source: Optional[MCPAnalyticsModelSource] = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.capture_tools_list(*, tool_names: Optional[List[str]] = None, parameters: Any = None, response: Any = None, duration_ms: Optional[float] = None, is_error: bool = False, error: Any = None, error_type: Optional[str] = None, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None -method posthog.mcp.posthog_mcp.PostHogMCP.capture_unknown_tool(tool_name: str, *, protocol_version: Optional[str] = None, distinct_id: Optional[str] = None, session_id: Optional[str] = None, conversation_id: Optional[str] = None, client_user_agent: Optional[str] = None, vendor_client: Optional[str] = None, set_properties: Optional[JsonRecord] = None, groups: Optional[Dict[str, str]] = None, properties: Optional[JsonRecord] = None, timestamp: Optional[datetime] = None) -> None method posthog.mcp.posthog_mcp.PostHogMCP.flush(timeout_seconds: Optional[float] = 10) -> None method posthog.mcp.posthog_mcp.PostHogMCP.prepare_tool_call(name: str, args: Optional[JsonRecord] = None, *, request_meta: Optional[JsonRecord] = None, original_tool: Any = None, session_id: Optional[str] = None) -> PreparedToolCall method posthog.mcp.posthog_mcp.PostHogMCP.prepare_tool_list(tools: List[Any], context: Union[bool, MCPAnalyticsContextOptions] = True, report_missing: bool = False, collect_feedback: bool = False) -> List[Any] From b4e012cca3333ad1bc19cfe6cc8e2689eedf43e0 Mon Sep 17 00:00:00 2001 From: Lucas Faria Date: Fri, 6 Nov 2026 18:31:18 -0300 Subject: [PATCH 7/7] fix(mcp): treat FastMCP 2.x mounts as unable to rule out a tool Review: _serves_other_providers only checked FastMCP 3.x providers, so on 2.x a mounted server (or one whose mount failed to load) made a missing name look unregistered and emitted $mcp_unknown_tool instead of a failed tool call. It now also checks 2.x's _mounted_servers. Tested: new server-with-mounted-server case fails on fastmcp 2.14.7 before the fix and passes after; pytest posthog/test/mcp on fastmcp 3.2 (690 passed), 2.14.7 and mcp 2.3+fastmcp 4.1 (only failures are the stdio handshake tests, which fail locally without this change, and the 2.x input-keys test that posthog-python#1203 fixes); ruff clean. --- posthog/mcp/_instrument_lowlevel.py | 7 +++++-- posthog/test/mcp/test_unknown_tool.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/posthog/mcp/_instrument_lowlevel.py b/posthog/mcp/_instrument_lowlevel.py index 1635cae58..3d84988a4 100644 --- a/posthog/mcp/_instrument_lowlevel.py +++ b/posthog/mcp/_instrument_lowlevel.py @@ -874,8 +874,11 @@ async def registered_tool_exists(server: Any, name: str) -> Optional[bool]: def _serves_other_providers(server: Any) -> bool: - """Whether a FastMCP 3+ server serves tools from anywhere but its own - registry: a mounted server, a proxy, or a custom provider.""" + """Whether a FastMCP server serves tools from anywhere but its own + registry: a mounted server, a proxy, or a custom provider. FastMCP 2.x + keeps mounts in ``_mounted_servers``; 3.x lists them in ``providers``.""" + if getattr(server, "_mounted_servers", None): + return True local = getattr(server, "_local_provider", None) return any(provider is not local for provider in getattr(server, "providers", ())) diff --git a/posthog/test/mcp/test_unknown_tool.py b/posthog/test/mcp/test_unknown_tool.py index 142f16e00..8d1396d26 100644 --- a/posthog/test/mcp/test_unknown_tool.py +++ b/posthog/test/mcp/test_unknown_tool.py @@ -109,6 +109,25 @@ def retired(query: str) -> str: return parent +def _with_mounted_server(): + """A parent serving a sub-server's tools under the ``ns`` prefix, on every + fastmcp major (2.x spells the argument ``prefix``, 3.x ``namespace``).""" + from fastmcp import FastMCP + + sub = FastMCP("sub") + + @sub.tool + def lookup(query: str) -> str: + return query + + parent = _standalone() + try: + parent.mount(sub, namespace="ns") + except TypeError: + parent.mount(sub, prefix="ns") + return parent + + def _raw(listed): """A raw low-level server whose own dispatcher rejects names it does not know and advertises ``listed`` on tools/list.""" @@ -244,6 +263,13 @@ async def test_a_call_to_a_tool_that_does_not_exist_is_recorded_as_unknown(build id="server-with-mounted-provider", marks=_needs_server_disable, ), + pytest.param( + _with_mounted_server, + "ns_no_such_tool", + {}, + id="server-with-mounted-server", + marks=_needs_standalone, + ), pytest.param( _standalone_with_application_middleware, "no_such_tool",