From a3edfbefedcbe017a5d1644a939d678a33fe989a Mon Sep 17 00:00:00 2001 From: Roydon Sequeira <136970995+roydonsequeira@users.noreply.github.com> Date: Fri, 30 Oct 2026 11:35:27 +0530 Subject: [PATCH 1/2] FIX: Return a tool error when arguments don't match a FunctionTool's signature OpenAIResponseTarget returns malformed_arguments for tool arguments that are not valid JSON or not a JSON object, but a JSON object that failed the FunctionTool's argument model raised ValidationError out of the send. Tool.validate_arguments now runs before execution. It accepts any arguments by default, and FunctionTool checks them against its argument model, so the dispatcher can return malformed_arguments with invoked=False, or re-raise when fail_on_missing_function=True, without calling the function. Fixes #3668 --- pyrit/prompt_target/common/tool_provider.py | 24 ++++++++ .../openai/openai_response_target.py | 10 ++++ tests/unit/prompt_target/target/test_tool.py | 57 ++++++++++++++++++- 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/pyrit/prompt_target/common/tool_provider.py b/pyrit/prompt_target/common/tool_provider.py index 2b75cef9e0..9200e4a1ad 100644 --- a/pyrit/prompt_target/common/tool_provider.py +++ b/pyrit/prompt_target/common/tool_provider.py @@ -46,6 +46,18 @@ def __init__( self.parameters = parameters self.strict = strict + def validate_arguments(self, *, arguments: dict[str, object]) -> None: + """ + Check model-supplied arguments before the tool is executed. + + The default accepts any arguments. Overrides raise ``ValueError`` to reject a call + without executing the tool, so the caller can return the error to the model. + + Args: + arguments: Arguments supplied by the model. + """ + return + @abstractmethod async def execute_async(self, *, arguments: dict[str, object]) -> object: """ @@ -124,6 +136,18 @@ def __init__( self._function = function self._argument_model: type[BaseModel] = argument_model + def validate_arguments(self, *, arguments: dict[str, object]) -> None: + """ + Check arguments against the function signature. + + Args: + arguments: Arguments supplied by the model. + + Raises: + ValidationError: If the arguments do not match the function signature. + """ + self._argument_model.model_validate(arguments) + async def execute_async(self, *, arguments: dict[str, object]) -> object: """ Validate arguments and call the Python function. diff --git a/pyrit/prompt_target/openai/openai_response_target.py b/pyrit/prompt_target/openai/openai_response_target.py index d0070e4378..e1b8f7696e 100644 --- a/pyrit/prompt_target/openai/openai_response_target.py +++ b/pyrit/prompt_target/openai/openai_response_target.py @@ -1052,6 +1052,16 @@ async def _execute_call_section_async(self, tool_call_section: dict[str, Any]) - ) if configured_tool is not None: + try: + configured_tool.validate_arguments(arguments=args) + except ValueError as exc: + if self._fail_on_missing_function: + raise + logger.warning("Invalid arguments for function '%s': %s", name, exc) + return _ToolDispatchResult( + output={"error": "malformed_arguments", "function": name, "raw_arguments": args_json}, + invoked=False, + ) return _ToolDispatchResult(output=await configured_tool.execute_async(arguments=args), invoked=True) if custom_function is not None: diff --git a/tests/unit/prompt_target/target/test_tool.py b/tests/unit/prompt_target/target/test_tool.py index f38786cd1f..ae93d26f16 100644 --- a/tests/unit/prompt_target/target/test_tool.py +++ b/tests/unit/prompt_target/target/test_tool.py @@ -11,7 +11,7 @@ from tenacity import wait_none from pyrit.exceptions import RateLimitException -from pyrit.models import JsonResponseConfig, Message, MessagePiece +from pyrit.models import JsonResponseConfig, Message, MessagePiece, ToolExecutionMetadata from pyrit.prompt_target import ( FunctionTool, MCPToolProvider, @@ -59,6 +59,12 @@ async def test_tool_validates_arguments_and_returns_scalar() -> None: await add.execute_async(arguments={"x": 2, "unexpected": True}) +@pytest.mark.parametrize("arguments", [{"x": "two"}, {"y": 3}, {"x": 2, "unexpected": True}]) +def test_function_tool_validate_arguments_rejects_mismatched_arguments(arguments: dict[str, object]) -> None: + with pytest.raises(ValidationError): + add.validate_arguments(arguments=arguments) + + async def test_tool_preserves_validated_structured_argument_type() -> None: class Location(BaseModel): city: str @@ -397,6 +403,55 @@ async def test_malformed_arguments_reach_model_as_tool_error(*, arguments: str, execute.assert_not_called() +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize("arguments", ['{"value": "one"}', "{}", '{"value": 1, "unexpected": true}']) +@pytest.mark.parametrize("strict", [False, True]) +async def test_mismatched_arguments_reach_model_as_tool_error(*, arguments: str, strict: bool) -> None: + executed: list[int] = [] + + async def record_async(*, value: int) -> int: + executed.append(value) + return value + + target = OpenAIResponseTarget( + model_name="gpt-4", + endpoint="https://example.invalid", + api_key="not-a-key", + tools=[FunctionTool(function=record_async)], + fail_on_missing_function=strict, + ) + call = MessagePiece( + role="assistant", + original_value=json.dumps( + {"type": "function_call", "call_id": "call-1", "name": "record_async", "arguments": arguments} + ), + original_value_data_type="function_call", + ).to_message() + final = Message.from_prompt(prompt="Please supply valid arguments.", role="assistant") + with ( + patch.object(target, "_handle_openai_request_async", new_callable=AsyncMock, side_effect=[call, final]) as send, + patch.object(target._client.responses, "create", new_callable=AsyncMock) as create, + ): + if strict: + with pytest.raises(ValidationError): + await target.send_prompt_async(message=Message.from_prompt(prompt="Record", role="user")) + send.assert_awaited_once() + else: + result = await target.send_prompt_async(message=Message.from_prompt(prompt="Record", role="user")) + assert [message.api_role for message in result] == ["assistant", "tool", "assistant"] + assert send.await_count == 2 + await send.call_args.kwargs["api_call"]() + assert json.loads(create.call_args.kwargs["input"][-1]["output"]) == { + "error": "malformed_arguments", + "function": "record_async", + "raw_arguments": arguments, + } + execution = ToolExecutionMetadata.from_metadata(metadata=result[1].get_piece().prompt_metadata) + assert execution == ToolExecutionMetadata(invoked=False) + target.validate_tool_history(result) + assert executed == [] + + async def test_collect_tools_async_preserves_direct_and_provider_order() -> None: async def subtract(*, x: int, y: int) -> int: # pyrit-async-suffix-exempt return x - y From 5129d926bffc70122595b213119a5cf6b0c49287 Mon Sep 17 00:00:00 2001 From: Roydon Sequeira <136970995+roydonsequeira@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:23:19 +0530 Subject: [PATCH 2/2] TEST: Check that a ValueError raised by a tool function propagates Only validate_arguments is inside the try that returns malformed_arguments. The new test fails if execute_async is moved into it, so an error raised by the function itself can't be reported to the model as an argument error. --- tests/unit/prompt_target/target/test_tool.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/prompt_target/target/test_tool.py b/tests/unit/prompt_target/target/test_tool.py index ae93d26f16..34173a0524 100644 --- a/tests/unit/prompt_target/target/test_tool.py +++ b/tests/unit/prompt_target/target/test_tool.py @@ -452,6 +452,24 @@ async def record_async(*, value: int) -> int: assert executed == [] +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize("strict", [False, True]) +async def test_value_error_raised_by_tool_function_propagates(*, strict: bool) -> None: + async def lookup_async(*, key: str) -> str: + raise ValueError(f"no entry for {key}") + + target = OpenAIResponseTarget( + model_name="gpt-4", + endpoint="https://example.invalid", + api_key="not-a-key", + tools=[FunctionTool(function=lookup_async)], + fail_on_missing_function=strict, + ) + + with pytest.raises(ValueError, match="no entry for missing"): + await target._execute_call_section_async({"name": "lookup_async", "arguments": '{"key": "missing"}'}) + + async def test_collect_tools_async_preserves_direct_and_provider_order() -> None: async def subtract(*, x: int, y: int) -> int: # pyrit-async-suffix-exempt return x - y