diff --git a/.claude/notes/agents.md b/.claude/notes/agents.md index b6d1e147..0b85b359 100644 --- a/.claude/notes/agents.md +++ b/.claude/notes/agents.md @@ -580,11 +580,16 @@ Four cases are worth knowing: - **Pi's search tool is `find`** (glob-by-pattern), not `glob`; there is no `glob` tool in its built-in set, so mapping `find` to the canonical `Glob` is what keeps `command_executed` and `commands_efficiency` comparable. -- **Delegate's host maps most names itself, but not `LoadSkill`.** `delegate-stdio` reports - `ReadFile` as `Read` and `ExecuteSkillApi` as `Skill`, so `DelegateAgent` maps only - `LoadSkill {name}` to `Skill {skill}`; without it `skill_triggered` never sees a Delegate skill - load. The rename is keyed by the host's name: keyed by `Skill`, it would also turn - ExecuteSkillApi's `name`, an API call, into a skill engagement. +- **Delegate's host maps most names itself, but not `LoadSkill` or its Windows shell.** + `delegate-stdio` reports `ReadFile` as `Read`, `ExecuteBashCommand` as `Bash` and + `ExecuteSkillApi` as `Skill`, so `DelegateAgent` maps only `LoadSkill {name}` to + `Skill {skill}` and `ExecutePowershellCommand` to `Bash`. Without the first, + `skill_triggered` never sees a Delegate skill load. Without the second, every Windows shell + call misses `tool_name: Bash` criteria and is matched as JSON, where an escaped quote + (`UiPath.Upgrade.exe\" analyze`) breaks the pattern and the call's plain-English `steps` + can trip an exclusion (coder_eval_uipath nightly 13817754 scored correct migrator and genome + runs 0.8–0.94 this way). The rename is keyed by the host's name: keyed by `Skill`, it would + also turn ExecuteSkillApi's `name`, an API call, into a skill engagement. Antigravity additionally strips the result payload out of a tool call's arguments: the harness folds result fields into the same `args` dict at DONE. Beyond a static key list, diff --git a/src/coder_eval/agents/delegate_agent.py b/src/coder_eval/agents/delegate_agent.py index 1879ae26..31ba91f1 100644 --- a/src/coder_eval/agents/delegate_agent.py +++ b/src/coder_eval/agents/delegate_agent.py @@ -150,11 +150,12 @@ def _is_config_init_error(message: str) -> bool: """``toolStatus`` values the SDK's ``tool_result`` carries for a tool that did not complete.""" # The host reports most tools under their canonical (Claude) names already -# (ReadFile -> Read, ...), but not LoadSkill, which loads a catalog skill by name. +# (ReadFile -> Read, ExecuteBashCommand -> Bash, ...), but not LoadSkill, which loads a +# catalog skill by name, nor ExecutePowershellCommand, the Windows shell tool. # Keyed by the host's name, not the canonical one: the host also reports # ExecuteSkillApi as `Skill`, and that call's `name` is an API call, not a skill load. # Rationale: .claude/notes/agents.md § Tool-name and argument normalization -_TOOL_NAME_MAP: dict[str, str] = {"LoadSkill": "Skill"} +_TOOL_NAME_MAP: dict[str, str] = {"LoadSkill": "Skill", "ExecutePowershellCommand": "Bash"} _DELEGATE_ARG_RENAME: dict[str, dict[str, str]] = {"LoadSkill": {"name": "skill"}} diff --git a/tests/test_delegate_agent.py b/tests/test_delegate_agent.py index 91011ff3..86c5f97b 100644 --- a/tests/test_delegate_agent.py +++ b/tests/test_delegate_agent.py @@ -413,7 +413,7 @@ async def test_happy_path_text_and_tool(self, patch_exec, tmp_path): events = [ _ev(type="session_start", sessionId="sess-1"), _ev(type="thinking", content="let me think"), - _tool_call("tool-1", "ExecutePowershellCommand", command="ls"), + _tool_call("tool-1", "Bash", command="ls"), _tool_result("tool-1", "file.txt"), _ev(type="thinking", content=""), _ev(type="message", content="here is my answer", isStepStart=True), @@ -428,7 +428,7 @@ async def test_happy_path_text_and_tool(self, patch_exec, tmp_path): assert not record.crashed assert len(record.commands) == 1 command = record.commands[0] - assert command.tool_name == "ExecutePowershellCommand" + assert command.tool_name == "Bash" assert command.parameters == {"command": "ls"} assert command.result_status == "success" assert command.result_summary == "file.txt" @@ -618,6 +618,19 @@ async def test_load_skill_is_recorded_as_the_canonical_skill_call(self, patch_ex assert (command.tool_name, command.parameters) == ("Skill", {"skill": "uipath-rpa", "plugin": ""}) assert _engaged_skill_names(command) == {"uipath-rpa"} + async def test_powershell_command_is_recorded_as_the_canonical_bash_call(self, patch_exec, tmp_path): + """Shell criteria read `Bash` + `command`; on Windows the host sends `ExecutePowershellCommand`.""" + command = '& "C:\\Tools\\UiPath.Upgrade.exe" analyze --output-format sarif' + events = [ + _tool_call("p1", "ExecutePowershellCommand", command=command, steps=["Run analyze"]), + _tool_result("p1"), + _result(response="done"), + ] + agent, _ = await _started_agent(patch_exec, events, tmp_path) + record = await agent.communicate("hi") + [recorded] = record.commands + assert (recorded.tool_name, recorded.parameters) == ("Bash", {"command": command, "steps": ["Run analyze"]}) + async def test_skill_api_call_keeps_its_name_arg(self, patch_exec, tmp_path): """The host already reports ExecuteSkillApi as `Skill`; its `name` is not a skill load.""" events = [