FIX: Return a tool error when arguments don't match a FunctionTool's signature

pyrit/prompt_target/common/tool_provider.py

@@ -46,6 +46,18 @@ def __init__(
         self.parameters = parameters
         self.strict = strict
 
+    def validate_arguments(self, *, arguments: dict[str, object]) -> None:
+        """
+        Check model-supplied arguments before the tool is executed.
+
+        The default accepts any arguments. Overrides raise ``ValueError`` to reject a call
+        without executing the tool, so the caller can return the error to the model.
+
+        Args:
+            arguments: Arguments supplied by the model.
+        """
+        return
+
     @abstractmethod
     async def execute_async(self, *, arguments: dict[str, object]) -> object:
         """
@@ -124,6 +136,18 @@ def __init__(
         self._function = function
         self._argument_model: type[BaseModel] = argument_model
 
+    def validate_arguments(self, *, arguments: dict[str, object]) -> None:
+        """
+        Check arguments against the function signature.
+
+        Args:
+            arguments: Arguments supplied by the model.
+
+        Raises:
+            ValidationError: If the arguments do not match the function signature.
+        """
+        self._argument_model.model_validate(arguments)
+
     async def execute_async(self, *, arguments: dict[str, object]) -> object:
         """
         Validate arguments and call the Python function.

pyrit/prompt_target/openai/openai_response_target.py

@@ -1052,6 +1052,16 @@ async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) -
             )
 
         if configured_tool is not None:
+            try:
+                configured_tool.validate_arguments(arguments=args)
+            except ValueError as exc:
+                if self._fail_on_missing_function:
+                    raise
+                logger.warning("Invalid arguments for function '%s': %s", name, exc)
+                return _ToolDispatchResult(
+                    output={"error": "malformed_arguments", "function": name, "raw_arguments": args_json},
+                    invoked=False,
+                )
             return _ToolDispatchResult(output=await configured_tool.execute_async(arguments=args), invoked=True)
 
         if custom_function is not None:

tests/unit/prompt_target/target/test_tool.py

@@ -11,7 +11,7 @@
 from tenacity import wait_none
 
 from pyrit.exceptions import RateLimitException
-from pyrit.models import JsonResponseConfig, Message, MessagePiece
+from pyrit.models import JsonResponseConfig, Message, MessagePiece, ToolExecutionMetadata
 from pyrit.prompt_target import (
     FunctionTool,
     MCPToolProvider,
@@ -59,6 +59,12 @@ async def test_tool_validates_arguments_and_returns_scalar() -> None:
         await add.execute_async(arguments={"x": 2, "unexpected": True})
 
 
+@pytest.mark.parametrize("arguments", [{"x": "two"}, {"y": 3}, {"x": 2, "unexpected": True}])
+def test_function_tool_validate_arguments_rejects_mismatched_arguments(arguments: dict[str, object]) -> None:
+    with pytest.raises(ValidationError):
+        add.validate_arguments(arguments=arguments)
+
+
 async def test_tool_preserves_validated_structured_argument_type() -> None:
     class Location(BaseModel):
         city: str
@@ -397,6 +403,73 @@ async def test_malformed_arguments_reach_model_as_tool_error(*, arguments: str,
         execute.assert_not_called()
 
 
+@pytest.mark.usefixtures("patch_central_database")
+@pytest.mark.parametrize("arguments", ['{"value": "one"}', "{}", '{"value": 1, "unexpected": true}'])
+@pytest.mark.parametrize("strict", [False, True])
+async def test_mismatched_arguments_reach_model_as_tool_error(*, arguments: str, strict: bool) -> None:
+    executed: list[int] = []
+
+    async def record_async(*, value: int) -> int:
+        executed.append(value)
+        return value
+
+    target = OpenAIResponseTarget(
+        model_name="gpt-4",
+        endpoint="https://example.invalid",
+        api_key="not-a-key",
+        tools=[FunctionTool(function=record_async)],
+        fail_on_missing_function=strict,
+    )
+    call = MessagePiece(
+        role="assistant",
+        original_value=json.dumps(
+            {"type": "function_call", "call_id": "call-1", "name": "record_async", "arguments": arguments}
+        ),
+        original_value_data_type="function_call",
+    ).to_message()
+    final = Message.from_prompt(prompt="Please supply valid arguments.", role="assistant")
+    with (
+        patch.object(target, "_handle_openai_request_async", new_callable=AsyncMock, side_effect=[call, final]) as send,
+        patch.object(target._client.responses, "create", new_callable=AsyncMock) as create,
+    ):
+        if strict:
+            with pytest.raises(ValidationError):
+                await target.send_prompt_async(message=Message.from_prompt(prompt="Record", role="user"))
+            send.assert_awaited_once()
+        else:
+            result = await target.send_prompt_async(message=Message.from_prompt(prompt="Record", role="user"))
+            assert [message.api_role for message in result] == ["assistant", "tool", "assistant"]
+            assert send.await_count == 2
+            await send.call_args.kwargs["api_call"]()
+            assert json.loads(create.call_args.kwargs["input"][-1]["output"]) == {
+                "error": "malformed_arguments",
+                "function": "record_async",
+                "raw_arguments": arguments,
+            }
+            execution = ToolExecutionMetadata.from_metadata(metadata=result[1].get_piece().prompt_metadata)
+            assert execution == ToolExecutionMetadata(invoked=False)
+            target.validate_tool_history(result)
+    assert executed == []
+
+
+@pytest.mark.usefixtures("patch_central_database")
+@pytest.mark.parametrize("strict", [False, True])
+async def test_value_error_raised_by_tool_function_propagates(*, strict: bool) -> None:
+    async def lookup_async(*, key: str) -> str:
+        raise ValueError(f"no entry for {key}")
+
+    target = OpenAIResponseTarget(
+        model_name="gpt-4",
+        endpoint="https://example.invalid",
+        api_key="not-a-key",
+        tools=[FunctionTool(function=lookup_async)],
+        fail_on_missing_function=strict,
+    )
+
+    with pytest.raises(ValueError, match="no entry for missing"):
+        await target._execute_call_section_async({"name": "lookup_async", "arguments": '{"key": "missing"}'})
+
+
 async def test_collect_tools_async_preserves_direct_and_provider_order() -> None:
     async def subtract(*, x: int, y: int) -> int:  # pyrit-async-suffix-exempt
         return x - y