Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/mcp-reword-injected-descriptions.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

Reworded the descriptions of the injected `conversation_id` and `llm_model` arguments and the `get_more_tools` and `send_feedback` virtual tools so they describe the field instead of instructing the model. This changes your published tool definitions; if your server is listed in the ChatGPT or Claude directories, those platforms may re-review the updated tools.
14 changes: 5 additions & 9 deletions posthog/mcp/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,14 @@
)

DEFAULT_CONVERSATION_ID_DESCRIPTION = (
"Pass the exact conversation_id from the server's previous response, unchanged. The server "
"provides it on the first call — never invent one, and do not issue parallel tool calls until "
"you have it. Keep passing the same conversation_id for the rest of the conversation, including "
"after later user messages or on a different task; do not reset it when the user starts a new "
"request."
"The conversation_id this server returned earlier in this conversation, unchanged, including "
"after later user messages. Omit it if you have none. Recorded for the server developer's "
"analytics."
)

DEFAULT_MODEL_PARAMETER_DESCRIPTION = (
"The exact model identifier you (the assistant) are running as, taken from your "
'system prompt or environment (e.g. "claude-opus-4-8", "gpt-5.2"). Used for '
'analytics only. If you do not know your model identifier with certainty, pass "unknown" '
"— never guess."
'The model identifier you are running as, for example "claude-opus-4-8" or "gpt-5.2". '
'Pass "unknown" if you are not certain. Recorded for the server developer\'s analytics.'
)

POSTHOG_MCP_ANALYTICS_SOURCE = "posthog_mcp_analytics"
Expand Down
11 changes: 4 additions & 7 deletions posthog/mcp/feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,9 @@
_TRUNCATION_SUFFIX = "..."

_DEFAULT_FEEDBACK_DESCRIPTION = (
"Send feedback about this server to its developers. Most important: report a missing "
"capability whenever no available tool fits your task, even if you can work around it "
'(feedback_type "missing_capability"). Also welcome: a tool that failed or confused you, '
"an unhelpful error, or something that worked well. This records the feedback; it does "
"not add or change tools. Do not include user PII or sensitive content."
"Send feedback about this server to its developers: a capability you needed that no tool "
"provides, a tool that failed or misled you, or something that worked well. Records the "
"feedback; it does not add or change tools. Leave out personal or sensitive details."
)

_SEND_FEEDBACK_RESULT_TEXT = (
Expand All @@ -47,8 +45,7 @@
"description": (
"What kind of feedback this is. Use 'missing_capability' when the tool you "
"needed does not exist in the tool list - nothing failed, the capability is "
"absent (this is the most valuable report; send it even if you found a "
"workaround). Use 'issue' when an existing tool behaved badly: it failed, "
"absent. Use 'issue' when an existing tool behaved badly: it failed, "
"returned a confusing error, or its description or schema misled you. Use "
"'praise' when something worked notably well. Use 'other' for anything else."
),
Expand Down
4 changes: 2 additions & 2 deletions posthog/mcp/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ def build_report_missing_descriptor(name: str = GET_MORE_TOOLS_NAME) -> Dict[str
return {
"name": name,
"description": (
"Check for additional tools whenever your task might benefit from specialized "
"capabilities - even if existing tools could work as a fallback."
"Report that the user's request needs a capability this server does not have. "
"Records the report for the server's developers; it does not return or add tools."
),
"inputSchema": {
"type": "object",
Expand Down
44 changes: 43 additions & 1 deletion posthog/test/mcp/test_feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ async def _list_tools_lowlevel(server):
def test_descriptor_defaults():
descriptor = get_feedback_tool_descriptor()
assert descriptor["name"] == SEND_FEEDBACK_TOOL_NAME
assert "missing capability" in descriptor["description"]
assert "a capability you needed that no tool provides" in descriptor["description"]
assert descriptor["inputSchema"]["required"] == ["feedback_type", "summary"]
assert set(descriptor["inputSchema"]["properties"]) == {
"feedback_type",
Expand Down Expand Up @@ -540,6 +540,48 @@ async def test_fastmcp_coexists_with_report_missing():
assert _events(client, "$mcp_tool_call") == []


def _descriptions(node):
if isinstance(node, dict):
for key, value in node.items():
if key == "description" and isinstance(value, str):
yield value
yield from _descriptions(value)
elif isinstance(node, list):
for item in node:
yield from _descriptions(item)


@pytest.mark.parametrize(
"phrase",
[
"parallel tool calls",
"system prompt",
"even if existing tools",
"Most important",
"even if you can work around it",
"send it even if",
],
)
async def test_advertised_descriptions_carry_no_behavioral_instructions(phrase):
server = make_fastmcp()
instrument(
server,
FakeClient(),
MCPAnalyticsOptions(
report_missing=True, collect_feedback=True, capture_model=True
),
)

list_handler = server._mcp_server.request_handlers[mcp_types.ListToolsRequest]
result = await list_handler(mcp_types.ListToolsRequest(method="tools/list"))
tools = result.root.tools
assert {"add", "get_more_tools", "send_feedback"} <= {t.name for t in tools}
descriptions = list(_descriptions([t.model_dump() for t in tools]))

assert descriptions
assert not [d for d in descriptions if phrase.lower() in d.lower()]


# --- on_feedback -----------------------------------------------------------------


Expand Down
2 changes: 1 addition & 1 deletion posthog/test/mcp/test_units.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ def test_add_conversation_id_adds_property():
assert out["properties"]["conversation_id"]["type"] == "string"
# Without this clause, ChatGPT drops the handle on each new user message and every message becomes its own session.
assert (
"rest of the conversation, including after later user messages"
"including after later user messages"
in out["properties"]["conversation_id"]["description"]
)

Expand Down
Loading