diff --git a/codegen-output/aws-query/pyproject.toml b/codegen-output/aws-query/pyproject.toml index 7fda39478..f13c7e9b4 100644 --- a/codegen-output/aws-query/pyproject.toml +++ b/codegen-output/aws-query/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[xml]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_xml~=0.2.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-query/src/awsquery/client.py b/codegen-output/aws-query/src/awsquery/client.py index bf42c255e..99ca89621 100644 --- a/codegen-output/aws-query/src/awsquery/client.py +++ b/codegen-output/aws-query/src/awsquery/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_xml.plugins import xml_deserialization_plugin from .config import AsyncQueryProtocolConfig, Plugin from .models import ( @@ -145,7 +146,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + xml_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncQueryProtocolConfig: if not self._setup_done: diff --git a/codegen-output/aws-query/src/awsquery/config.py b/codegen-output/aws-query/src/awsquery/config.py index af6bd7422..906bb83a3 100644 --- a/codegen-output/aws-query/src/awsquery/config.py +++ b/codegen-output/aws-query/src/awsquery/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_xml import XMLDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -165,6 +166,7 @@ class _AsyncQueryProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + xml_deserialization_mode: XMLDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -228,6 +230,12 @@ class AsyncQueryProtocolConfig(AsyncAwsConfig): header. """ + xml_deserialization_mode: XMLDeserializationMode | None = None + """ + Controls whether XML response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -236,6 +244,7 @@ class AsyncQueryProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "xml_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("query_protocol") diff --git a/codegen-output/ec2-query/pyproject.toml b/codegen-output/ec2-query/pyproject.toml index ed7ba732d..73eb2f572 100644 --- a/codegen-output/ec2-query/pyproject.toml +++ b/codegen-output/ec2-query/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[xml]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_xml~=0.2.0" ] [project.optional-dependencies] diff --git a/codegen-output/ec2-query/src/ec2query/client.py b/codegen-output/ec2-query/src/ec2query/client.py index 44c1c8135..87bc6d68d 100644 --- a/codegen-output/ec2-query/src/ec2query/client.py +++ b/codegen-output/ec2-query/src/ec2query/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_xml.plugins import xml_deserialization_plugin from .config import AsyncEC2ProtocolConfig, Plugin from .models import ( @@ -121,7 +122,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + xml_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncEC2ProtocolConfig: if not self._setup_done: diff --git a/codegen-output/ec2-query/src/ec2query/config.py b/codegen-output/ec2-query/src/ec2query/config.py index 1c6206aa5..077f3f600 100644 --- a/codegen-output/ec2-query/src/ec2query/config.py +++ b/codegen-output/ec2-query/src/ec2query/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_xml import XMLDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -134,6 +135,7 @@ class _AsyncEC2ProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + xml_deserialization_mode: XMLDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -197,6 +199,12 @@ class AsyncEC2ProtocolConfig(AsyncAwsConfig): header. """ + xml_deserialization_mode: XMLDeserializationMode | None = None + """ + Controls whether XML response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -205,6 +213,7 @@ class AsyncEC2ProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "xml_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("ec2_protocol") diff --git a/codegen-output/rest-xml-with-namespace/pyproject.toml b/codegen-output/rest-xml-with-namespace/pyproject.toml index abe4a6800..d35fb7529 100644 --- a/codegen-output/rest-xml-with-namespace/pyproject.toml +++ b/codegen-output/rest-xml-with-namespace/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[xml]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_xml~=0.2.0" ] [project.optional-dependencies] diff --git a/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/client.py b/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/client.py index c6923d543..a798b3708 100644 --- a/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/client.py +++ b/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_xml.plugins import xml_deserialization_plugin from .config import AsyncRestXmlProtocolNamespaceConfig, Plugin from .models import ( @@ -55,7 +56,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + xml_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncRestXmlProtocolNamespaceConfig: if not self._setup_done: diff --git a/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/config.py b/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/config.py index f3af47919..f2d1726eb 100644 --- a/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/config.py +++ b/codegen-output/rest-xml-with-namespace/src/restxmlwithnamespace/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_xml import XMLDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import SimpleScalarPropertiesInput, SimpleScalarPropertiesOutput @@ -47,6 +48,7 @@ class _AsyncRestXmlProtocolNamespaceConfigOverrides(AwsConfigOverrides, total=Fa ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + xml_deserialization_mode: XMLDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -110,6 +112,12 @@ class AsyncRestXmlProtocolNamespaceConfig(AsyncAwsConfig): header. """ + xml_deserialization_mode: XMLDeserializationMode | None = None + """ + Controls whether XML response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -118,6 +126,7 @@ class AsyncRestXmlProtocolNamespaceConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "xml_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("rest_xml_protocol_namespace") diff --git a/codegen-output/rest-xml/pyproject.toml b/codegen-output/rest-xml/pyproject.toml index e59812bfa..5449c43b3 100644 --- a/codegen-output/rest-xml/pyproject.toml +++ b/codegen-output/rest-xml/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[xml]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_xml~=0.2.0" ] [project.optional-dependencies] diff --git a/codegen-output/rest-xml/src/restxml/client.py b/codegen-output/rest-xml/src/restxml/client.py index 9f29a9777..d7e98a04f 100644 --- a/codegen-output/rest-xml/src/restxml/client.py +++ b/codegen-output/rest-xml/src/restxml/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_xml.plugins import xml_deserialization_plugin from .config import AsyncRestXmlProtocolConfig, Plugin from .models import ( @@ -241,7 +242,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + xml_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncRestXmlProtocolConfig: if not self._setup_done: diff --git a/codegen-output/rest-xml/src/restxml/config.py b/codegen-output/rest-xml/src/restxml/config.py index 3e150ec12..4c78b7a09 100644 --- a/codegen-output/rest-xml/src/restxml/config.py +++ b/codegen-output/rest-xml/src/restxml/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_xml import XMLDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -312,6 +313,7 @@ class _AsyncRestXmlProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + xml_deserialization_mode: XMLDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -375,6 +377,12 @@ class AsyncRestXmlProtocolConfig(AsyncAwsConfig): header. """ + xml_deserialization_mode: XMLDeserializationMode | None = None + """ + Controls whether XML response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -383,6 +391,7 @@ class AsyncRestXmlProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "xml_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("rest_xml_protocol") diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java index cd7bfdd0d..ed4921e63 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SmithyPythonDependency.java @@ -64,6 +64,15 @@ public final class SmithyPythonDependency { Type.DEPENDENCY, false); + /** + * The core smithy-xml python package. + */ + public static final PythonDependency SMITHY_XML = new PythonDependency( + "smithy_xml", + "~=0.2.0", + Type.DEPENDENCY, + false); + /** * The core smithy-cbor python package. */ diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegration.java new file mode 100644 index 000000000..648f3e43d --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegration.java @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import java.util.List; +import java.util.Set; +import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; +import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; +import software.amazon.smithy.aws.traits.protocols.RestXmlTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds configurable XML deserialization to clients that support an XML protocol. + */ +@SmithyInternalApi +public final class XmlDeserializationIntegration implements PythonIntegration { + private static final Set XML_PROTOCOLS = Set.of( + RestXmlTrait.ID, + AwsQueryTrait.ID, + Ec2QueryTrait.ID); + + @Override + public List getClientPlugins(GenerationContext context) { + var protocolGenerator = context.protocolGenerator(); + if (protocolGenerator == null || !isXmlProtocol(protocolGenerator.getProtocol())) { + return List.of(); + } + + var mode = Symbol.builder() + .name("XMLDeserializationMode") + .namespace("smithy_xml", ".") + .addDependency(SmithyPythonDependency.SMITHY_XML) + .build(); + var plugin = SymbolReference.builder() + .symbol(Symbol.builder() + .name("xml_deserialization_plugin") + .namespace("smithy_xml.plugins", ".") + .addDependency(SmithyPythonDependency.SMITHY_XML) + .build()) + .build(); + var property = ConfigProperty.builder() + .name("xml_deserialization_mode") + .type(mode) + .documentation( + "Controls whether XML response payloads are deserialized eagerly or incrementally.") + .nullable(true) + .build(); + + return List.of(RuntimeClientPlugin.builder() + .addConfigProperty(property) + .pythonPlugin(plugin) + .build()); + } + + static boolean isXmlProtocol(ShapeId protocol) { + return XML_PROTOCOLS.contains(protocol); + } +} diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index ccdf63695..e605bc5a4 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -4,6 +4,7 @@ # software.amazon.smithy.python.codegen.integrations.RestJsonIntegration +software.amazon.smithy.python.codegen.integrations.XmlDeserializationIntegration software.amazon.smithy.python.codegen.integrations.Rpcv2CborIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth software.amazon.smithy.python.codegen.integrations.UserAgentIntegration diff --git a/codegen/core/src/test/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegrationTest.java b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegrationTest.java new file mode 100644 index 000000000..23ff23c1d --- /dev/null +++ b/codegen/core/src/test/java/software/amazon/smithy/python/codegen/integrations/XmlDeserializationIntegrationTest.java @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; +import software.amazon.smithy.aws.traits.protocols.AwsQueryTrait; +import software.amazon.smithy.aws.traits.protocols.Ec2QueryTrait; +import software.amazon.smithy.aws.traits.protocols.RestXmlTrait; +import software.amazon.smithy.model.shapes.ShapeId; + +public class XmlDeserializationIntegrationTest { + @Test + public void detectsXmlProtocols() { + assertTrue(XmlDeserializationIntegration.isXmlProtocol(RestXmlTrait.ID)); + assertTrue(XmlDeserializationIntegration.isXmlProtocol(AwsQueryTrait.ID)); + assertTrue(XmlDeserializationIntegration.isXmlProtocol(Ec2QueryTrait.ID)); + assertFalse(XmlDeserializationIntegration.isXmlProtocol(ShapeId.from("example#other"))); + } +} diff --git a/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py b/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py index f3726af15..70e0988a5 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/_private/query/errors.py @@ -86,6 +86,7 @@ def create_aws_query_error( status: int, context: TypedProperties, retry_after: float | None = None, + codec: "XMLCodec | None" = None, ) -> CallError: """Create a modeled or generic CallError from an awsQuery error response.""" code = _parse_aws_query_error_code(body, wrapper_elements) @@ -105,7 +106,7 @@ def create_aws_query_error( ) _assert_xml() - deserializer = XMLCodec().create_deserializer( + deserializer = (codec or XMLCodec()).create_deserializer( body, wrapper_elements=wrapper_elements ) modeled_error = error_shape.deserialize(deserializer) diff --git a/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py b/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py index 0c20ec8b3..ca0b4576d 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py @@ -795,6 +795,7 @@ async def _create_error( status=response.status, context=context, retry_after=parse_retry_after(response), + codec=self.payload_codec, ) def _create_serializer( diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index cff8349b3..a54b12c23 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, cast -from unittest.mock import Mock +from unittest.mock import Mock, patch import pytest from ijson.common import IncompleteJSONError # type: ignore[reportMissingTypeStubs] @@ -30,6 +30,7 @@ from smithy_http import Fields, tuples_to_fields from smithy_http.aio import HTTPRequest, HTTPResponse from smithy_json import JSONSettings +from smithy_xml import XMLDeserializationMode @pytest.mark.parametrize( @@ -511,28 +512,35 @@ async def test_aws_query_resolves_modeled_error_from_query_error_trait() -> None namespace="com.test", service_target="QueryService", version="2020-01-08" ) ) - with pytest.raises(_ModeledQueryError) as exc_info: - await protocol.deserialize_response( - operation=_mock_operation( - _operation_schema("FailingOperation"), - error_schemas=[_INVALID_ACTION_ERROR_SCHEMA], - ), - request=cast(HTTPRequest, Mock()), - response=HTTPResponse( - status=400, - fields=tuples_to_fields([]), - body=( - b"InvalidAction" - b"bad request" + protocol.payload_codec.deserialization_mode = XMLDeserializationMode.STREAMING + with patch.object( + protocol.payload_codec, + "create_deserializer", + wraps=protocol.payload_codec.create_deserializer, + ) as create_deserializer: + with pytest.raises(_ModeledQueryError) as exc_info: + await protocol.deserialize_response( + operation=_mock_operation( + _operation_schema("FailingOperation"), + error_schemas=[_INVALID_ACTION_ERROR_SCHEMA], ), - ), - error_registry=TypeRegistry( - {ShapeID("com.test#InvalidActionError"): _ModeledQueryError} - ), - context=TypedProperties(), - ) + request=cast(HTTPRequest, Mock()), + response=HTTPResponse( + status=400, + fields=tuples_to_fields([]), + body=( + b"InvalidAction" + b"bad request" + ), + ), + error_registry=TypeRegistry( + {ShapeID("com.test#InvalidActionError"): _ModeledQueryError} + ), + context=TypedProperties(), + ) assert exc_info.value.message == "bad request" + create_deserializer.assert_called_once() async def test_aws_query_resolves_modeled_error_from_default_namespace_fallback() -> ( diff --git a/packages/smithy-xml/.changes/next-release/smithy-xml-feature-f40d86af0a1d4f88a11786d45575ea09.json b/packages/smithy-xml/.changes/next-release/smithy-xml-feature-f40d86af0a1d4f88a11786d45575ea09.json new file mode 100644 index 000000000..4a40c9f6e --- /dev/null +++ b/packages/smithy-xml/.changes/next-release/smithy-xml-feature-f40d86af0a1d4f88a11786d45575ea09.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added configurable eager XML deserialization to improve buffered payload performance. Generated XML clients automatically parse `bytes` eagerly while preserving incremental parsing for streaming readers, and can explicitly select eager or streaming behavior." +} diff --git a/packages/smithy-xml/README.md b/packages/smithy-xml/README.md index bcd7da956..c822a01c3 100644 --- a/packages/smithy-xml/README.md +++ b/packages/smithy-xml/README.md @@ -2,3 +2,22 @@ This package provides generic XML serialization and deserialization support for Smithy clients and servers. + +## Deserialization modes + +`XMLCodec` supports eager and streaming deserialization. The default `AUTO` +mode eagerly parses byte payloads and incrementally parses `BytesReader` +payloads: + +```python +from smithy_xml import XMLCodec, XMLDeserializationMode + +codec = XMLCodec( + deserialization_mode=XMLDeserializationMode.STREAMING, +) +``` + +Eager mode is faster for buffered payloads but materializes the complete XML +tree before shape deserialization. Streaming mode preserves incremental parser +consumption for reader inputs. Generated clients using an XML protocol expose +the same choice through `xml_deserialization_mode` on their configuration. diff --git a/packages/smithy-xml/src/smithy_xml/__init__.py b/packages/smithy-xml/src/smithy_xml/__init__.py index 533717dca..591bdf040 100644 --- a/packages/smithy-xml/src/smithy_xml/__init__.py +++ b/packages/smithy-xml/src/smithy_xml/__init__.py @@ -3,7 +3,7 @@ from io import BytesIO from typing import Any -from xml.etree.ElementTree import iterparse +from xml.etree.ElementTree import fromstring, iterparse, parse from smithy_core.codecs import Codec from smithy_core.deserializers import ShapeDeserializer @@ -15,10 +15,13 @@ from ._private.deserializers import XMLShapeDeserializer as _XMLShapeDeserializer from ._private.readers import XMLEventReader as _XMLEventReader from ._private.serializers import XMLShapeSerializer as _XMLShapeSerializer -from .settings import XMLSettings +from ._private.value_deserializer import ( + XMLValueDeserializer as _XMLValueDeserializer, +) +from .settings import XMLDeserializationMode, XMLSettings __version__ = "0.2.0" -__all__ = ("XMLCodec", "XMLSettings") +__all__ = ("XMLCodec", "XMLDeserializationMode", "XMLSettings") class XMLCodec(Codec): @@ -29,6 +32,7 @@ def __init__( use_timestamp_format: bool = True, default_timestamp_format: TimestampFormat = TimestampFormat.DATE_TIME, default_namespace: str | None = None, + deserialization_mode: XMLDeserializationMode = XMLDeserializationMode.AUTO, ) -> None: """Initializes an XMLCodec. @@ -38,11 +42,14 @@ def __init__( `smithy.api#timestampFormat` trait is not enabled or not present. :param default_namespace: Default XML namespace (`xmlns`) applied to the root element during serialization. + :param deserialization_mode: Controls whether XML payloads are parsed eagerly + or incrementally. """ self._settings = XMLSettings( use_timestamp_format=use_timestamp_format, default_timestamp_format=default_timestamp_format, default_namespace=default_namespace, + deserialization_mode=deserialization_mode, ) # Member element renderings, shared by every serializer this codec creates. self._elements: dict[ShapeID, Any] = {} @@ -60,6 +67,21 @@ def create_deserializer( *, wrapper_elements: tuple[str, ...] = (), ) -> ShapeDeserializer: + mode = self._settings.deserialization_mode + if mode is XMLDeserializationMode.EAGER or ( + mode is XMLDeserializationMode.AUTO and isinstance(source, bytes) + ): + root = ( + fromstring(source) # noqa: S314 + if isinstance(source, bytes) + else parse(source).getroot() # noqa: S314 + ) + return _XMLValueDeserializer( + settings=self._settings, + root=root, + wrapper_elements=wrapper_elements, + ) + if isinstance(source, bytes): source = BytesIO(source) reader = _XMLEventReader( @@ -68,3 +90,13 @@ def create_deserializer( return _XMLShapeDeserializer( settings=self._settings, reader=reader, wrapper_elements=wrapper_elements ) + + @property + def deserialization_mode(self) -> XMLDeserializationMode: + """The strategy used to deserialize XML payloads.""" + + return self._settings.deserialization_mode + + @deserialization_mode.setter + def deserialization_mode(self, value: XMLDeserializationMode) -> None: + self._settings.deserialization_mode = value diff --git a/packages/smithy-xml/src/smithy_xml/_private/deserializers.py b/packages/smithy-xml/src/smithy_xml/_private/deserializers.py index 74474baaf..5955b1553 100644 --- a/packages/smithy-xml/src/smithy_xml/_private/deserializers.py +++ b/packages/smithy-xml/src/smithy_xml/_private/deserializers.py @@ -23,26 +23,26 @@ from .traits import member_xml_name -def _local_name(tag: str) -> str: +def local_name(tag: str) -> str: """Strip namespace URI from an element tag: {uri}local -> local.""" if tag.startswith("{"): return tag.split("}", 1)[1] return tag -def _local_attr_name(name: str) -> str: +def local_attr_name(name: str) -> str: """Strip a namespace prefix from a modeled attribute name: prefix:local -> local.""" return name.rpartition(":")[2] -def _validate_element_name(expected: str, elem: Element) -> None: +def validate_element_name(expected: str, elem: Element) -> None: """Raise XMLParseError if the element's local name doesn't match expected.""" - found = _local_name(elem.tag) + found = local_name(elem.tag) if found != expected: raise XMLParseError(f"Expected element '{expected}', got '{found}'") -def _parse_xml_float(text: str) -> float: +def parse_xml_float(text: str) -> float: """Parse an XML float string, handling NaN and Infinity.""" match text: case "NaN": @@ -81,7 +81,7 @@ def __init__( event = next(self._reader) if event.type != "start": raise XMLParseError(f"Expected start element, got '{event.type}'") - _validate_element_name(wrapper, event.elem) + validate_element_name(wrapper, event.elem) self._preconsumed_start = event.elem def is_null(self) -> bool: @@ -107,7 +107,7 @@ def read_integer(self, schema: Schema) -> int: return int(self._read_text()) def read_float(self, schema: Schema) -> float: - return _parse_xml_float(self._read_text()) + return parse_xml_float(self._read_text()) def read_big_decimal(self, schema: Schema) -> Decimal: return Decimal(self._read_text()) @@ -144,13 +144,13 @@ def read_struct( continue # Prefixed names (e.g. ``xsi:name``) are parsed into a # namespace-qualified key, so only the local parts are compared. - expected_attr_name = _local_attr_name(member_xml_name(member_schema)) + expected_attr_name = local_attr_name(member_xml_name(member_schema)) for attr_name, attr_value in start_elem.attrib.items(): - attr_local_name = _local_name(attr_name) + attr_local_name = local_name(attr_name) if attr_local_name == expected_attr_name: consumer( member_schema, - _AttributeDeserializer(attr_value, self._settings), + AttributeDeserializer(attr_value, self._settings), ) break @@ -165,7 +165,7 @@ def read_struct( } while self._reader.peek().type != "end": - tag = _local_name(self._reader.peek().elem.tag) + tag = local_name(self._reader.peek().elem.tag) if tag in flattened_names: flattened_buffers.setdefault(tag, []).extend(self._buffer_element()) @@ -296,7 +296,7 @@ def _read_map_entry( key: str | None = None while self._reader.peek().type != "end": - child_tag = _local_name(self._reader.peek().elem.tag) + child_tag = local_name(self._reader.peek().elem.tag) if child_tag == key_tag: key = self._read_text() elif child_tag == value_tag: @@ -313,7 +313,7 @@ def _read_map_entry( next(self._reader) -class _AttributeDeserializer(SpecificShapeDeserializer): +class AttributeDeserializer(SpecificShapeDeserializer): """Deserializer for a value extracted from an XML attribute string.""" def __init__(self, value: str, settings: XMLSettings) -> None: @@ -348,7 +348,7 @@ def read_big_integer(self, schema: Schema) -> int: return self.read_integer(schema) def read_float(self, schema: Schema) -> float: - return _parse_xml_float(self._value) + return parse_xml_float(self._value) def read_double(self, schema: Schema) -> float: return self.read_float(schema) diff --git a/packages/smithy-xml/src/smithy_xml/_private/value_deserializer.py b/packages/smithy-xml/src/smithy_xml/_private/value_deserializer.py new file mode 100644 index 000000000..d4d566bb4 --- /dev/null +++ b/packages/smithy-xml/src/smithy_xml/_private/value_deserializer.py @@ -0,0 +1,216 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import datetime +from base64 import b64decode +from collections.abc import Callable, Iterable +from decimal import Decimal +from xml.etree.ElementTree import Element + +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.schemas import Schema +from smithy_core.shapes import ShapeID +from smithy_core.traits import ( + TimestampFormatTrait, + XMLAttributeTrait, + XMLFlattenedTrait, +) + +from ..settings import XMLSettings +from .deserializers import ( + AttributeDeserializer, + XMLParseError, + local_attr_name, + local_name, + parse_xml_float, + validate_element_name, +) +from .traits import member_xml_name + + +class XMLValueDeserializer(ShapeDeserializer): + """Deserializer backed by a fully materialized ElementTree value.""" + + def __init__( + self, + settings: XMLSettings, + root: Element, + wrapper_elements: tuple[str, ...] = (), + *, + elements: tuple[Element, ...] | None = None, + xml_names: dict[ShapeID, dict[str, Schema]] | None = None, + ) -> None: + self._settings = settings + self._xml_names = xml_names if xml_names is not None else {} + self._elements = elements + self._from_wrapper = bool(wrapper_elements) + + current = root + for index, wrapper in enumerate(wrapper_elements): + validate_element_name(wrapper, current) + if index < len(wrapper_elements) - 1: + child = next(iter(current), None) + if child is None: + raise XMLParseError( + f"Expected wrapper element '{wrapper_elements[index + 1]}'" + ) + current = child + self._element = current + + def is_null(self) -> bool: + return False + + def read_null(self) -> None: + return None + + def read_boolean(self, schema: Schema) -> bool: + text = self._read_text() + match text: + case "true": + return True + case "false": + return False + case _: + raise XMLParseError(f"Expected 'true' or 'false', got '{text}'") + + def read_blob(self, schema: Schema) -> bytes: + return b64decode(self._read_text()) + + def read_integer(self, schema: Schema) -> int: + return int(self._read_text()) + + def read_float(self, schema: Schema) -> float: + return parse_xml_float(self._read_text()) + + def read_big_decimal(self, schema: Schema) -> Decimal: + return Decimal(self._read_text()) + + def read_string(self, schema: Schema) -> str: + return self._read_text() + + def read_document(self, schema: Schema) -> Document: + raise NotImplementedError("XML does not support document types") + + def read_timestamp(self, schema: Schema) -> datetime.datetime: + format = self._settings.default_timestamp_format + if self._settings.use_timestamp_format: + if format_trait := schema.get_trait(TimestampFormatTrait): + format = format_trait.format + return format.deserialize(self._read_text()) + + def read_struct( + self, + schema: Schema, + consumer: Callable[[Schema, ShapeDeserializer], None], + ) -> None: + element = self._element + xml_names = self._get_xml_names(schema) + + if not self._from_wrapper: + for member_schema in schema.members.values(): + if member_schema.get_trait(XMLAttributeTrait) is None: + continue + expected_name = local_attr_name(member_xml_name(member_schema)) + for attribute_name, value in element.attrib.items(): + if local_name(attribute_name) == expected_name: + consumer( + member_schema, + AttributeDeserializer(value, self._settings), + ) + break + + flattened: dict[str, list[Element]] = {} + for child in element: + name = local_name(child.tag) + member_schema = xml_names.get(name) + if member_schema is None: + continue + if member_schema.get_trait(XMLFlattenedTrait) is not None: + flattened.setdefault(name, []).append(child) + else: + consumer(member_schema, self._for_element(child)) + + for name, elements in flattened.items(): + consumer( + xml_names[name], + self._for_elements(elements), + ) + + def read_list( + self, + schema: Schema, + consumer: Callable[[ShapeDeserializer], None], + ) -> None: + elements: Iterable[Element] + if schema.get_trait(XMLFlattenedTrait) is not None: + elements = self._elements or (self._element,) + else: + elements = self._element + + for element in elements: + consumer(self._for_element(element)) + + def read_map( + self, + schema: Schema, + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + key_tag = member_xml_name(schema.members["key"]) + value_tag = member_xml_name(schema.members["value"]) + entries: Iterable[Element] + if schema.get_trait(XMLFlattenedTrait) is not None: + entries = self._elements or (self._element,) + else: + entries = self._element + + for entry in entries: + self._read_map_entry(entry, key_tag, value_tag, consumer) + + def _read_map_entry( + self, + entry: Element, + key_tag: str, + value_tag: str, + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + key: str | None = None + for child in entry: + name = local_name(child.tag) + if name == key_tag: + key = child.text or "" + elif name == value_tag: + if key is None: + raise XMLParseError( + "Map key element must appear before value element" + ) + consumer(key, self._for_element(child)) + + def _read_text(self) -> str: + return self._element.text or "" + + def _for_element(self, element: Element) -> "XMLValueDeserializer": + return XMLValueDeserializer( + settings=self._settings, + root=element, + xml_names=self._xml_names, + ) + + def _for_elements(self, elements: list[Element]) -> "XMLValueDeserializer": + return XMLValueDeserializer( + settings=self._settings, + root=elements[0], + elements=tuple(elements), + xml_names=self._xml_names, + ) + + def _get_xml_names(self, schema: Schema) -> dict[str, Schema]: + if schema.id in self._xml_names: + return self._xml_names[schema.id] + result = { + member_xml_name(member_schema): member_schema + for member_schema in schema.members.values() + if member_schema.get_trait(XMLAttributeTrait) is None + } + self._xml_names[schema.id] = result + return result diff --git a/packages/smithy-xml/src/smithy_xml/plugins.py b/packages/smithy-xml/src/smithy_xml/plugins.py new file mode 100644 index 000000000..32a041328 --- /dev/null +++ b/packages/smithy-xml/src/smithy_xml/plugins.py @@ -0,0 +1,23 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Protocol + +from . import XMLCodec +from .settings import XMLDeserializationMode + + +class _XMLDeserializationConfig(Protocol): + @property + def protocol(self) -> object | None: ... + + @property + def xml_deserialization_mode(self) -> XMLDeserializationMode | None: ... + + +def xml_deserialization_plugin(config: _XMLDeserializationConfig) -> None: + """Configure XML deserialization on a generated client's protocol codec.""" + + codec = getattr(config.protocol, "payload_codec", None) + if isinstance(codec, XMLCodec) and config.xml_deserialization_mode is not None: + codec.deserialization_mode = config.xml_deserialization_mode diff --git a/packages/smithy-xml/src/smithy_xml/settings.py b/packages/smithy-xml/src/smithy_xml/settings.py index 86549d3e1..7fdeab2c3 100644 --- a/packages/smithy-xml/src/smithy_xml/settings.py +++ b/packages/smithy-xml/src/smithy_xml/settings.py @@ -1,10 +1,24 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass +from enum import StrEnum from smithy_core.types import TimestampFormat +class XMLDeserializationMode(StrEnum): + """Controls how XML payloads are parsed during deserialization.""" + + AUTO = "auto" + """Eagerly parse bytes while preserving incremental reader consumption.""" + + EAGER = "eager" + """Parse and materialize the complete XML tree before deserializing.""" + + STREAMING = "streaming" + """Incrementally deserialize XML using the pull parser.""" + + @dataclass(slots=True) class XMLSettings: """Settings for the XML codec.""" @@ -18,3 +32,6 @@ class XMLSettings: default_namespace: str | None = None """Default XML namespace (`xmlns`) applied to the root element during serialization.""" + + deserialization_mode: XMLDeserializationMode = XMLDeserializationMode.AUTO + """Controls whether XML deserialization is eager, streaming, or automatic.""" diff --git a/packages/smithy-xml/tests/unit/test_deserializers.py b/packages/smithy-xml/tests/unit/test_deserializers.py index f3ee9e703..81da65122 100644 --- a/packages/smithy-xml/tests/unit/test_deserializers.py +++ b/packages/smithy-xml/tests/unit/test_deserializers.py @@ -3,9 +3,12 @@ import math from datetime import datetime from decimal import Decimal +from io import BytesIO from typing import Any +from xml.etree.ElementTree import ParseError import pytest +from smithy_core.deserializers import ShapeDeserializer from smithy_core.prelude import ( BIG_DECIMAL, BLOB, @@ -16,7 +19,9 @@ STRING, TIMESTAMP, ) -from smithy_xml import XMLCodec +from smithy_xml import XMLCodec, XMLDeserializationMode +from smithy_xml._private.deserializers import XMLShapeDeserializer +from smithy_xml._private.value_deserializer import XMLValueDeserializer from . import ( STRING_LIST_SCHEMA, @@ -26,9 +31,14 @@ ) +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) @pytest.mark.parametrize("expected, given", XML_SERDE_CASES) -def test_xml_deserializer(expected: Any, given: bytes) -> None: - codec = XMLCodec() +def test_xml_deserializer( + expected: Any, + given: bytes, + mode: XMLDeserializationMode, +) -> None: + codec = XMLCodec(deserialization_mode=mode) deserializer = codec.create_deserializer(given) match expected: case bool(): @@ -67,6 +77,33 @@ def test_xml_deserializer(expected: Any, given: bytes) -> None: assert actual == expected +@pytest.mark.parametrize( + "mode, source, expected_type", + [ + (XMLDeserializationMode.AUTO, b"", XMLValueDeserializer), + (XMLDeserializationMode.AUTO, BytesIO(b""), XMLShapeDeserializer), + (XMLDeserializationMode.EAGER, BytesIO(b""), XMLValueDeserializer), + (XMLDeserializationMode.STREAMING, b"", XMLShapeDeserializer), + ], +) +def test_deserialization_mode_selects_parser( + mode: XMLDeserializationMode, + source: bytes | BytesIO, + expected_type: type[ShapeDeserializer], +) -> None: + deserializer = XMLCodec(deserialization_mode=mode).create_deserializer(source) + assert isinstance(deserializer, expected_type) + + +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) +def test_invalid_xml_uses_existing_error_type(mode: XMLDeserializationMode) -> None: + with pytest.raises(ParseError): + deserializer = XMLCodec(deserialization_mode=mode).create_deserializer( + b"" + ) + deserializer.read_string(STRING) + + def test_read_document_raises() -> None: """XML does not support document types.""" deserializer = XMLCodec().create_deserializer(b"foo") @@ -97,29 +134,34 @@ def test_deserialize_empty_blob_self_closed() -> None: assert XMLCodec().create_deserializer(b"").read_blob(BLOB) == b"" -def test_wrapper_elements() -> None: +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) +def test_wrapper_elements(mode: XMLDeserializationMode) -> None: """Deserializer can unwrap awsQuery-style response wrappers.""" xml = ( b"" b"hello" b"" ) - deserializer = XMLCodec().create_deserializer( + deserializer = XMLCodec(deserialization_mode=mode).create_deserializer( xml, wrapper_elements=("OpResponse", "OpResult") ) result = SerdeShape.deserialize(deserializer) assert result.string_member == "hello" -def test_wrapper_elements_scalar_read() -> None: +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) +def test_wrapper_elements_scalar_read(mode: XMLDeserializationMode) -> None: xml = b"hello" - deserializer = XMLCodec().create_deserializer( + deserializer = XMLCodec(deserialization_mode=mode).create_deserializer( xml, wrapper_elements=("OpResponse", "OpResult") ) assert deserializer.read_string(STRING) == "hello" -def test_flattened_list_interleaved_with_other_members() -> None: +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) +def test_flattened_list_interleaved_with_other_members( + mode: XMLDeserializationMode, +) -> None: """Flattened list elements can be interleaved with other struct members.""" xml = ( b"" @@ -128,12 +170,15 @@ def test_flattened_list_interleaved_with_other_members() -> None: b"second" b"" ) - result = SerdeShape.deserialize(XMLCodec().create_deserializer(xml)) + result = SerdeShape.deserialize( + XMLCodec(deserialization_mode=mode).create_deserializer(xml) + ) assert result.flattened_list_member == ["first", "second"] assert result.string_member == "middle" -def test_unknown_members_skipped() -> None: +@pytest.mark.parametrize("mode", list(XMLDeserializationMode)) +def test_unknown_members_skipped(mode: XMLDeserializationMode) -> None: xml = ( b"" b"keep" @@ -141,5 +186,7 @@ def test_unknown_members_skipped() -> None: b"5" b"" ) - result = SerdeShape.deserialize(XMLCodec().create_deserializer(xml)) + result = SerdeShape.deserialize( + XMLCodec(deserialization_mode=mode).create_deserializer(xml) + ) assert result == SerdeShape(string_member="keep", integer_member=5) diff --git a/packages/smithy-xml/tests/unit/test_plugins.py b/packages/smithy-xml/tests/unit/test_plugins.py new file mode 100644 index 000000000..4097bf504 --- /dev/null +++ b/packages/smithy-xml/tests/unit/test_plugins.py @@ -0,0 +1,55 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass + +import pytest +from smithy_xml import XMLCodec, XMLDeserializationMode +from smithy_xml.plugins import xml_deserialization_plugin + + +@dataclass +class _Protocol: + payload_codec: object + + +@dataclass +class _Config: + protocol: object | None + xml_deserialization_mode: XMLDeserializationMode | None + + +@pytest.mark.parametrize( + "configured, expected", + [ + (None, XMLDeserializationMode.EAGER), + (XMLDeserializationMode.AUTO, XMLDeserializationMode.AUTO), + (XMLDeserializationMode.EAGER, XMLDeserializationMode.EAGER), + (XMLDeserializationMode.STREAMING, XMLDeserializationMode.STREAMING), + ], +) +def test_configures_xml_codec( + configured: XMLDeserializationMode | None, + expected: XMLDeserializationMode, +) -> None: + codec = XMLCodec(deserialization_mode=XMLDeserializationMode.EAGER) + config = _Config( + protocol=_Protocol(payload_codec=codec), + xml_deserialization_mode=configured, + ) + + xml_deserialization_plugin(config) + + assert codec.deserialization_mode is expected + + +@pytest.mark.parametrize( + "protocol", + [None, object(), _Protocol(payload_codec=object())], +) +def test_ignores_protocols_without_xml_codec(protocol: object | None) -> None: + config = _Config( + protocol=protocol, + xml_deserialization_mode=XMLDeserializationMode.STREAMING, + ) + xml_deserialization_plugin(config)