Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions .claude/notes/agents.md
Original file line number Diff line number Diff line change
Expand Up @@ -580,11 +580,16 @@ Four cases are worth knowing:
- **Pi's search tool is `find`** (glob-by-pattern), not `glob`; there is no `glob` tool in
its built-in set, so mapping `find` to the canonical `Glob` is what keeps
`command_executed` and `commands_efficiency` comparable.
- **Delegate's host maps most names itself, but not `LoadSkill`.** `delegate-stdio` reports
`ReadFile` as `Read` and `ExecuteSkillApi` as `Skill`, so `DelegateAgent` maps only
`LoadSkill {name}` to `Skill {skill}`; without it `skill_triggered` never sees a Delegate skill
load. The rename is keyed by the host's name: keyed by `Skill`, it would also turn
ExecuteSkillApi's `name`, an API call, into a skill engagement.
- **Delegate's host maps most names itself, but not `LoadSkill` or its Windows shell.**
`delegate-stdio` reports `ReadFile` as `Read`, `ExecuteBashCommand` as `Bash` and
`ExecuteSkillApi` as `Skill`, so `DelegateAgent` maps only `LoadSkill {name}` to
`Skill {skill}` and `ExecutePowershellCommand` to `Bash`. Without the first,
`skill_triggered` never sees a Delegate skill load. Without the second, every Windows shell
call misses `tool_name: Bash` criteria and is matched as JSON, where an escaped quote
(`UiPath.Upgrade.exe\" analyze`) breaks the pattern and the call's plain-English `steps`
can trip an exclusion (coder_eval_uipath nightly 13817754 scored correct migrator and genome
runs 0.8–0.94 this way). The rename is keyed by the host's name: keyed by `Skill`, it would
also turn ExecuteSkillApi's `name`, an API call, into a skill engagement.

Antigravity additionally strips the result payload out of a tool call's arguments: the
harness folds result fields into the same `args` dict at DONE. Beyond a static key list,
Expand Down
5 changes: 3 additions & 2 deletions src/coder_eval/agents/delegate_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,11 +150,12 @@ def _is_config_init_error(message: str) -> bool:
"""``toolStatus`` values the SDK's ``tool_result`` carries for a tool that did not complete."""

# The host reports most tools under their canonical (Claude) names already
# (ReadFile -> Read, ...), but not LoadSkill, which loads a catalog skill by name.
# (ReadFile -> Read, ExecuteBashCommand -> Bash, ...), but not LoadSkill, which loads a
# catalog skill by name, nor ExecutePowershellCommand, the Windows shell tool.
# Keyed by the host's name, not the canonical one: the host also reports
# ExecuteSkillApi as `Skill`, and that call's `name` is an API call, not a skill load.
# Rationale: .claude/notes/agents.md § Tool-name and argument normalization
_TOOL_NAME_MAP: dict[str, str] = {"LoadSkill": "Skill"}
_TOOL_NAME_MAP: dict[str, str] = {"LoadSkill": "Skill", "ExecutePowershellCommand": "Bash"}
_DELEGATE_ARG_RENAME: dict[str, dict[str, str]] = {"LoadSkill": {"name": "skill"}}


Expand Down
17 changes: 15 additions & 2 deletions tests/test_delegate_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,7 +413,7 @@ async def test_happy_path_text_and_tool(self, patch_exec, tmp_path):
events = [
_ev(type="session_start", sessionId="sess-1"),
_ev(type="thinking", content="let me think"),
_tool_call("tool-1", "ExecutePowershellCommand", command="ls"),
_tool_call("tool-1", "Bash", command="ls"),
_tool_result("tool-1", "file.txt"),
_ev(type="thinking", content=""),
_ev(type="message", content="here is my answer", isStepStart=True),
Expand All @@ -428,7 +428,7 @@ async def test_happy_path_text_and_tool(self, patch_exec, tmp_path):
assert not record.crashed
assert len(record.commands) == 1
command = record.commands[0]
assert command.tool_name == "ExecutePowershellCommand"
assert command.tool_name == "Bash"
assert command.parameters == {"command": "ls"}
assert command.result_status == "success"
assert command.result_summary == "file.txt"
Expand Down Expand Up @@ -618,6 +618,19 @@ async def test_load_skill_is_recorded_as_the_canonical_skill_call(self, patch_ex
assert (command.tool_name, command.parameters) == ("Skill", {"skill": "uipath-rpa", "plugin": ""})
assert _engaged_skill_names(command) == {"uipath-rpa"}

async def test_powershell_command_is_recorded_as_the_canonical_bash_call(self, patch_exec, tmp_path):
"""Shell criteria read `Bash` + `command`; on Windows the host sends `ExecutePowershellCommand`."""
command = '& "C:\\Tools\\UiPath.Upgrade.exe" analyze --output-format sarif'
events = [
_tool_call("p1", "ExecutePowershellCommand", command=command, steps=["Run analyze"]),
_tool_result("p1"),
_result(response="done"),
]
agent, _ = await _started_agent(patch_exec, events, tmp_path)
record = await agent.communicate("hi")
[recorded] = record.commands
assert (recorded.tool_name, recorded.parameters) == ("Bash", {"command": command, "steps": ["Run analyze"]})

async def test_skill_api_call_keeps_its_name_arg(self, patch_exec, tmp_path):
"""The host already reports ExecuteSkillApi as `Skill`; its `name` is not a skill load."""
events = [
Expand Down
Loading