diff --git a/codegen-output/aws-json-1-0/pyproject.toml b/codegen-output/aws-json-1-0/pyproject.toml index de6d7aa27..b05b8c87e 100644 --- a/codegen-output/aws-json-1-0/pyproject.toml +++ b/codegen-output/aws-json-1-0/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-0/src/awsjson10/client.py b/codegen-output/aws-json-1-0/src/awsjson10/client.py index 5f9cb2388..c6dcee27b 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/client.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJSONRPC10Config, Plugin from .models import ( @@ -94,7 +95,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncJSONRPC10Config: if not self._setup_done: diff --git a/codegen-output/aws-json-1-0/src/awsjson10/config.py b/codegen-output/aws-json-1-0/src/awsjson10/config.py index 1696a93d0..b66993b2f 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/config.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -111,6 +112,7 @@ class _AsyncJSONRPC10ConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -174,6 +176,12 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -182,6 +190,7 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_rpc_10") diff --git a/codegen-output/aws-json-1-1/pyproject.toml b/codegen-output/aws-json-1-1/pyproject.toml index a9b718abf..150cfed84 100644 --- a/codegen-output/aws-json-1-1/pyproject.toml +++ b/codegen-output/aws-json-1-1/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-1/src/awsjson11/client.py b/codegen-output/aws-json-1-1/src/awsjson11/client.py index 897d043d3..cdc16a79d 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/client.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJsonProtocolConfig, Plugin from .models import ( @@ -100,7 +101,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/aws-json-1-1/src/awsjson11/config.py b/codegen-output/aws-json-1-1/src/awsjson11/config.py index bbbeb3e1e..83d1fbe6a 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/config.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -111,6 +112,7 @@ class _AsyncJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -174,6 +176,12 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -182,6 +190,7 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_protocol") diff --git a/codegen-output/rest-json-1/pyproject.toml b/codegen-output/rest-json-1/pyproject.toml index 953379f29..9d63abd61 100644 --- a/codegen-output/rest-json-1/pyproject.toml +++ b/codegen-output/rest-json-1/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[eventstream, json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/rest-json-1/src/restjson/client.py b/codegen-output/rest-json-1/src/restjson/client.py index 5c0a36df8..781c861dd 100644 --- a/codegen-output/rest-json-1/src/restjson/client.py +++ b/codegen-output/rest-json-1/src/restjson/client.py @@ -19,6 +19,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncRestJsonProtocolConfig, Plugin from .models import ( @@ -392,7 +393,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncRestJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/rest-json-1/src/restjson/config.py b/codegen-output/rest-json-1/src/restjson/config.py index 68c6a03da..2894a5685 100644 --- a/codegen-output/rest-json-1/src/restjson/config.py +++ b/codegen-output/rest-json-1/src/restjson/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -556,6 +557,7 @@ class _AsyncRestJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -619,6 +621,12 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -627,6 +635,7 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("rest_json_protocol") diff --git a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java index b5519b0d2..1574c3ab4 100644 --- a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java +++ b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java @@ -56,6 +56,10 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertTrue(config.contains("self.auth_schemes = auth_schemes")); assertTrue(config.contains("default_factory=lambda: AIOHTTPClient()")); assertFalse(config.contains("from smithy_http.aio.crt import AWSCRTHTTPClient")); + assertTrue(config.contains( + "json_deserialization_mode: JSONDeserializationMode | None")); + assertTrue(config.contains( + "\"json_deserialization_mode\": FieldSpec(default=None)")); var client = Files.readString(tempDir.resolve("src/restjson/client.py")); assertInOrder( @@ -73,6 +77,9 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertFalse(client.contains("plugin(self._config)")); assertTrue(client.contains("retry_mode=config.retry_mode")); assertTrue(client.contains("max_attempts=config.max_attempts")); + assertTrue(client.contains( + "from smithy_json.plugins import json_deserialization_plugin")); + assertTrue(client.contains("json_deserialization_plugin,")); var schemas = Files.readString(tempDir.resolve("src/restjson/_private/schemas.py")); assertTrue(schemas.contains("aws.protocols#restJson1")); diff --git a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java index 8d5a84bfa..c64ed5be0 100644 --- a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java +++ b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java @@ -53,6 +53,14 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { var config = Files.readString(tempDir.resolve("src/weather/config.py")); assertTrue(config.contains("self.transport = transport or AIOHTTPClient()")); assertFalse(config.contains("self.transport = transport or AWSCRTHTTPClient()")); + assertTrue(config.contains( + "json_deserialization_mode: JSONDeserializationMode | None")); + assertTrue(config.contains( + "self.json_deserialization_mode = json_deserialization_mode")); + + assertTrue(client.contains( + "from smithy_json.plugins import json_deserialization_plugin")); + assertTrue(client.contains("json_deserialization_plugin,")); // An output that models a "responseMetadata" member gets both the SDK-reserved // response_metadata attribute and the escaped modeled member, so neither shadows diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java new file mode 100644 index 000000000..5efbe1dc9 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import java.util.List; +import java.util.Set; +import software.amazon.smithy.aws.traits.protocols.AwsJson1_0Trait; +import software.amazon.smithy.aws.traits.protocols.AwsJson1_1Trait; +import software.amazon.smithy.aws.traits.protocols.RestJson1Trait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds configurable JSON deserialization to clients that support a JSON protocol. + */ +@SmithyInternalApi +public final class JsonDeserializationIntegration implements PythonIntegration { + private static final Set JSON_PROTOCOLS = Set.of( + RestJson1Trait.ID, + AwsJson1_0Trait.ID, + AwsJson1_1Trait.ID); + + @Override + public List getClientPlugins(GenerationContext context) { + var mode = Symbol.builder() + .name("JSONDeserializationMode") + .namespace("smithy_json", ".") + .addDependency(SmithyPythonDependency.SMITHY_JSON) + .build(); + var plugin = SymbolReference.builder() + .symbol(Symbol.builder() + .name("json_deserialization_plugin") + .namespace("smithy_json.plugins", ".") + .addDependency(SmithyPythonDependency.SMITHY_JSON) + .build()) + .build(); + var property = ConfigProperty.builder() + .name("json_deserialization_mode") + .type(mode) + .documentation( + "Controls whether JSON response payloads are deserialized eagerly or incrementally.") + .nullable(true) + .build(); + + return List.of(RuntimeClientPlugin.builder() + .servicePredicate(JsonDeserializationIntegration::usesJsonProtocol) + .addConfigProperty(property) + .pythonPlugin(plugin) + .build()); + } + + static boolean usesJsonProtocol(Model model, ServiceShape service) { + return ServiceIndex.of(model) + .getProtocols(service) + .keySet() + .stream() + .anyMatch(JSON_PROTOCOLS::contains); + } +} diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index ccdf63695..f85f625be 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -4,6 +4,7 @@ # software.amazon.smithy.python.codegen.integrations.RestJsonIntegration +software.amazon.smithy.python.codegen.integrations.JsonDeserializationIntegration software.amazon.smithy.python.codegen.integrations.Rpcv2CborIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth software.amazon.smithy.python.codegen.integrations.UserAgentIntegration diff --git a/designs/codegen/generated-package.md b/designs/codegen/generated-package.md new file mode 100644 index 000000000..44c90aa6d --- /dev/null +++ b/designs/codegen/generated-package.md @@ -0,0 +1,672 @@ +# Python-Native Generated Package Architecture + +The Python generator emits clients and types packages that use the existing +Smithy Python runtime. Generated packages preserve the public behavior of the +Java generator while loading schemas and model partitions on demand. + +## Goals + +* Generate clients and standalone types packages from the semantic model in + `smithy_python.model`. +* Preserve supported generated package APIs during migration from the Java + generator. +* Avoid constructing user-defined schemas while importing a generated client. +* Use dense member indexes in generated serialization and deserialization. +* Preserve one runtime `Schema` instance for each generated shape. +* Support recursive shape graphs and free-threaded Python. +* Let protocol and platform integrations add generation behavior without + replacing the core generator. +* Measure generation time, import cost, first-use cost, memory, and warm serde + throughout development. + +## Scope + +The first implementation supports standalone types and Rest JSON clients. +Later changes add the remaining protocols after their runtime behavior passes +the existing protocol tests. + +The Java generator remains available during migration. A projection selects +which generator to run, so generated packages can move independently after +their required features reach parity. + +Server generation, custom model validation, and runtime compilation of serde +functions remain future work. Generated dataclasses retain their current +non-slotted layout until compatibility tests and benchmarks support changing +it. + +## Performance Gates + +The benchmark suite records these measurements for equivalent Java-generated +and Python-generated packages: + +* Code generation wall time and peak resident memory. +* Generated source size and compiled bytecode size. +* Package, models, config, and client import time. +* Client construction time. +* First serialization and deserialization time. +* Warm serialization and deserialization time. +* Peak allocations for first use and warm serde. + +Rest JSON provides the first comparison. Rest XML and RPC v2 CBOR join the +suite when the native generator supports them. + +Each measured serde case uses the same model, values, payload, warmup count, +and measured iteration count for both generators. Benchmark output includes +the interpreter version, platform, architecture, and generator commit. + +The implementation must satisfy these behavioral gates: + +* Importing the generated client constructs no user-defined shape schemas. +* Importing a model class constructs no schema until its `schema()` method or + an operation that uses it is accessed. +* Repeated calls to a shape's `schema()` method return the same object. +* Generated serde contains no string-keyed member schema lookup. +* Warm Rest JSON p50 may not regress by more than 5 percent against the Java + generator for the same benchmark case. A larger regression blocks migration + for that case. + +Cold-start measurements establish a baseline before the first client migrates. +The design uses behavioral gates for schema loading because host load and +filesystem state make a single import-time percentage unstable. + +## Generator Architecture + +The generator uses the existing model loader, service selection, CLI, and +plugin environment: + +```text +Smithy JSON AST + | + v +semantic model + | + v +shape selection + | + v +generation context + | + +------> core generators + | + +------> protocol and platform integrations + | + v +generated package +``` + +The proof of concept supplies initial implementations for symbols, writers, +generation context, plugins, models, schemas, clients, and protocol tests. +Those components are ported onto the current semantic model. The repository's +current model loader, shape selection, CLI, and environment handling remain +authoritative. + +### Generation Context + +One generation context owns: + +* The semantic model and selected service. +* Selected shapes in model order. +* Shape and member indexes. +* Symbols and generated names. +* Protocol and platform integrations. +* Dependency and generated-file manifests. +* Schema dependency components and source partitions. + +The context builds indexes once. Generators query those indexes instead of +rescanning every selected shape for each output file. + +### Symbols + +The symbol provider maps semantic shapes and members to Python names, module +paths, runtime dependencies, schema accessors, serializer functions, and +deserializer functions. + +Symbol results are cached by shape ID for the duration of one generation. +Member indexes are assigned once from the selected model order and reused by +schema generation and serde generation. + +Plugins may decorate symbols or supply symbols for integration-owned types. +The core provider remains responsible for collision handling and service +renames. + +### Integrations + +An integration declares the protocols and platform behavior it supports. It +can contribute: + +* Protocol construction. +* Runtime dependencies. +* Generated config fields and defaults. +* Client plugins. +* Traits retained in runtime schemas. +* Code sections in generated files. + +Integrations operate on the semantic model and generation context. Generated +packages depend on the Smithy runtime packages and exclude `smithy-python`. + +### Writers + +Writers collect imports, generated sections, and file contents in memory. The +file manifest writes changed files and removes stale generated files within +the output directory. + +Formatting and linting run after generation when requested. Tests exercise the +unformatted writer output as valid Python so formatting cannot hide syntax +errors. + +## Generated Package Layout + +Generated packages expose stable public modules and place partition details +under `_private`: + +```text +example_service/ + __init__.py + client.py + config.py + models.py + py.typed + _private/ + operations.py + models_0.py + models_1.py + schemas_0.py + schemas_1.py +``` + +Source partitions start with a target size of 128 KiB. The generator estimates +partition size from rendered declarations. The threshold is an implementation +setting, and benchmarks may change it. + +Private module names and partition assignments are unstable. Public model, +client, and config imports remain stable across regenerations. + +## Public Model Facade + +`models.py` exposes generated model classes, enums, unions, and operation +constants. Runtime imports resolve an exported name on first access: + +```python +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ._private.models_0 import GetObjectInput as GetObjectInput + from ._private.models_1 import GetObjectOutput as GetObjectOutput + +_EXPORTS = { + "GetObjectInput": ("._private.models_0", "GetObjectInput"), + "GetObjectOutput": ("._private.models_1", "GetObjectOutput"), +} + + +def __getattr__(name: str) -> Any: + export = _EXPORTS.get(name) + if export is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, symbol_name = export + value = getattr(import_module(module_name, __package__), symbol_name) + globals()[name] = value + return value +``` + +Caching the resolved value in the module makes later access a normal global +lookup. `TYPE_CHECKING` imports expose concrete symbols to static type checkers +without importing model partitions at runtime. + +`__all__` lists the public generated names. `__dir__` combines those names with +the module globals so interactive discovery remains available before +resolution. + +## Model Partitions + +Model partitions contain dataclasses, enums, union variants, and their +generated serde helpers. The generator builds a dependency graph from runtime +model references and groups strongly connected components before assigning +partitions. + +Postponed annotations remove runtime imports needed only for type hints. +Cross-partition imports needed by serde occur through private accessors and +are cached after first use. + +Generated dataclasses keep keyword-only constructors, modeled defaults, +sensitive-member representation, and the existing serialization interfaces. +Modeled errors retain their service error hierarchy. + +## Lazy Schema Partitions + +Schema construction is lazy at the shape level. Each schema partition stores +completed schemas in a dense array indexed by constants generated for that +partition: + +```python +_SCHEMAS: list[Schema | None] = [None] * 24 +_LOCK = RLock() + +_GET_OBJECT_INPUT = 7 +``` + +The warm path performs one array lookup and one `None` check. + +### Dependency Components + +The generator builds a directed graph from each schema to its member targets. +It computes strongly connected components and treats each component as one +initialization unit. + +An acyclic shape forms a component of one. Mutually recursive shapes share a +component. Components form a directed acyclic graph, which gives schema +initialization a stable dependency order. + +A recursive component is never split across source partitions. + +### Component Initialization + +A component builder creates every root schema in the component before it +creates member schemas: + +```python +def _build_component_3() -> None: + node = Schema.collection( + id=ShapeID("example#Node"), + members={"value": None, "next": None}, + ) + + node.members["value"] = Schema.member( + id=node.id.with_member("value"), + target=STRING, + index=0, + ) + node.members["next"] = Schema.member( + id=node.id.with_member("next"), + target=node, + index=1, + ) + + _SCHEMAS[_NODE] = node +``` + +The public accessor uses a double-checked cache lookup: + +```python +def get_node_schema() -> Schema: + schema = _SCHEMAS[_NODE] + if schema is not None: + return schema + + with _LOCK: + schema = _SCHEMAS[_NODE] + if schema is None: + _build_component_3() + schema = _SCHEMAS[_NODE] + assert schema is not None + return schema +``` + +The component lock remains held until every schema in the component is +complete. Another thread sees an empty cache slot, waits for the lock, and +then reads the completed schema. The builder assigns cache slots only after +all roots and members in the component are complete. + +Cross-component dependencies resolve before the current component is +published. The component graph has no cycles, so initialization cannot recurse +back into an unpublished component. + +If construction raises an exception, the cache slots remain empty. A later +call can retry construction. + +### Shape Schema API + +Generated shapes expose a static `schema()` method: + +```python +_NODE_SCHEMA: Schema | None = None +_NODE_MEMBERS: tuple[Schema, ...] | None = None + + +def _node_schema() -> Schema: + global _NODE_SCHEMA, _NODE_MEMBERS + + if (schema := _NODE_SCHEMA) is None: + from .schemas_0 import get_node_schema + + schema = get_node_schema() + _NODE_MEMBERS = schema.members_by_index + _NODE_SCHEMA = schema + return schema + + +def _node_members() -> tuple[Schema, ...]: + members = _NODE_MEMBERS + if members is None: + _node_schema() + members = _NODE_MEMBERS + assert members is not None + return members + + +class Node: + @staticmethod + def schema() -> Schema: + return _node_schema() +``` + +The schema belongs to the modeled shape. It does not vary by Python subclass, +so the method uses `staticmethod`. A class method could create +subclass-specific caches and schema identities. + +The model partition caches the member tuple with the root schema. Generated +serde reads that tuple and does not query the name-keyed member map. + +### Schema Identity and Extensions + +Each shape accessor resolves to one canonical `Schema`. Operations, model +classes, type registries, protocols, and codecs share that object. + +Schema extensions remain lazy and schema-local. HTTP binding metadata, codec +metadata, and future extension values therefore survive across client and +protocol instances after first use. + +Generated code never wraps a schema in a lazy proxy. A proxy would change +schema identity and add resolver work to every attribute access. + +## Generated Serialization + +Structure serialization uses generated member access and dense schema indexes: + +```python +def serialize_members(self, serializer: ShapeSerializer) -> None: + members = _node_members() + serializer.write_string(members[0], self.value) + if self.next is not None: + serializer.write_struct(members[1], self.next) +``` + +`_node_members()` initializes the shape schema on its first call and returns +the cached tuple afterward. + +Generated serialization retains modeled nullability, sparse collection +behavior, timestamp formats, streaming members, and event stream handling. +Protocol-specific routing remains in runtime protocol packages. For example, +the HTTP binding serializer uses the schema's cached HTTP binding extension +while the generated structure writes members in modeled order. + +Collection serializers cache their `member`, `key`, and `value` schemas by +index. Union variants cache their variant member schema. + +## Generated Deserialization + +Structure deserialization uses a slotted callable object as mutable state: + +```python +class _NodeDeserializer: + __slots__ = ("value", "next") + + def __init__(self) -> None: + self.value: str | None = None + self.next: Node | None = None + + def __call__( + self, member: Schema, deserializer: ShapeDeserializer + ) -> None: + match member.expect_member_index(): + case 0: + self.value = deserializer.read_string(member) + case 1: + self.next = Node.deserialize(deserializer) + + def build(self) -> Node: + return Node(value=self.value, next=self.next) +``` + +`deserialize()` allocates one state object, passes it directly as the member +consumer, and calls `build()`: + +```python +@classmethod +def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + state = _NodeDeserializer() + deserializer.read_struct(_node_schema(), state) + return state.build() +``` + +Member indexes select generated cases, and fixed attributes store decoded +values. The primary path allocates one state object for each structure. + +The state initializer applies modeled defaults. Required members use a private +missing sentinel and `build()` reports an absent required value. Mutable +defaults are created for each state object. + +`deserialize_kwargs()` remains available for compatibility. It uses the same +state object and creates a dictionary only when a caller invokes that method. +The primary `deserialize()` path constructs the model directly. + +Union deserializers use one result slot and reject multiple values. List and +map deserializers use slotted callable state objects that retain the target +collection and cached member schemas. + +The first implementation keeps the existing `ShapeDeserializer` interface. +A state-parameter overload may replace callable state objects after every +official codec supports it and benchmarks show a gain. + +## Operations and Client Imports + +Operations resolve on first use from a dense module cache: + +```python +_OPERATIONS: list[APIOperation[Any, Any] | None] = [None] * 18 +_OPERATION_LOCK = Lock() +_GET_OBJECT = 4 + + +def get_get_object_operation() -> APIOperation[Any, Any]: + operation = _OPERATIONS[_GET_OBJECT] + if operation is None: + with _OPERATION_LOCK: + operation = _OPERATIONS[_GET_OBJECT] + if operation is None: + operation = APIOperation( + input=GetObjectInput, + output=GetObjectOutput, + schema=get_get_object_schema(), + input_schema=GetObjectInput.schema(), + output_schema=GetObjectOutput.schema(), + error_registry=_get_get_object_error_registry(), + effective_auth_schemes=_GET_OBJECT_AUTH, + error_schemas=_get_get_object_error_schemas(), + ) + _OPERATIONS[_GET_OBJECT] = operation + return operation +``` + +The client imports model types under `TYPE_CHECKING` for annotations. Each +operation method imports its private operation getter and caches the resolved +operation through the operations module. + +Importing `client.py` loads client infrastructure and config types. It does not +load model or schema partitions. Calling one operation loads that operation's +input, output, errors, and reachable schema components. + +Error registries import modeled error classes when their operation initializes. +Receiving an unknown error does not require loading unrelated modeled errors. + +## Runtime Trait Retention + +The native generator retains traits consumed by runtime packages and generated +SDK metadata. Core generation and integrations register the trait IDs they +need. + +The retained set includes traits used by: + +* Generic schema and validation behavior. +* Serialization and document codecs. +* HTTP binding and protocol framing. +* Authentication, endpoints, checksums, and retries. +* Generated SDK metadata. + +Plugins can add retained trait IDs. Generated docstrings contain documentation +traits, while runtime schemas omit them. Trait definitions and test traits are +excluded from runtime schemas unless an integration registers them. + +Trait filtering reduces generated source size, import parsing, schema +construction, and retained memory. The native and Java generators must expose +the same runtime traits before a client migrates. + +## Compatibility + +The native generator preserves: + +* Public package, model, client, and config module paths. +* Generated model names and member names. +* Keyword-only structure constructors. +* Modeled defaults and nullability. +* Enum and union behavior. +* Modeled error inheritance. +* `serialize()`, `serialize_members()`, `deserialize()`, and + `deserialize_kwargs()`. +* Public operation constants through the models facade. +* Canonical schema identity within one generated package. + +Private partition paths and helper names are implementation details. + +Generated source need not match Java-generated source. Protocol behavior, +public Python behavior, and type-checker behavior define parity. + +## Testing + +### Generator Tests + +Generator unit tests cover: + +* Symbol and member naming. +* Service renames and collisions. +* Partition boundaries. +* Strongly connected component assignment. +* Trait retention. +* Plugin ordering and code sections. +* Stale-file removal. + +Golden tests inspect generated source for lazy imports, dense member access, +and stable public exports. + +### Generated Package Tests + +Each generated fixture must: + +* Compile with the supported Python versions. +* Pass Ruff formatting and lint checks. +* Pass strict Pyright checks. +* Import through every public module. +* Serialize and deserialize recursive shapes. +* Preserve schema identity across model, operation, and registry access. +* Initialize recursive schemas safely from concurrent threads. + +Import tests instrument `Schema` construction and assert the lazy-loading +gates from this design. + +### Protocol Tests + +The existing Smithy protocol tests run against native-generated clients. The +Java generator continues to produce its snapshots during migration so +behavioral differences remain visible. + +Each protocol moves only after its required request, response, error, streaming, +and event stream tests pass. + +## Rollout + +Implementation proceeds in these stages: + +1. Port generation context, symbols, writers, plugins, and package setup onto + the current semantic model. +2. Generate standalone types with partitioned models and lazy schemas. +3. Add dense generated serialization and deserialization. +4. Generate Rest JSON clients and protocol tests. +5. Add AWS platform behavior required by the Rest JSON test services. +6. Add Rest XML, RPC v2 CBOR, AWS JSON, AWS Query, and EC2 Query support. +7. Compare each protocol against the Java generator and migrate projections + independently. + +Each stage includes its benchmark results. Performance regressions remain +local to the stage that introduced them. + +## Tradeoffs + +Lazy construction moves schema work from import time to the first operation +that reaches a shape. Applications that invoke an operation immediately still +pay the construction cost, but applications importing a large client for a +small operation avoid unrelated schema work. + +Partitioning adds private modules and lazy export machinery. It reduces Python +source compiled during common imports and keeps the public module layout +stable. + +Dense indexes depend on generated member order. Regeneration may assign a +different index after a model change. Indexes remain private to one generated +artifact and are never serialized or exposed as a compatibility contract. + +The component lock adds synchronization to a schema cache miss. Cache hits do +not acquire the lock. + +Callable deserialization state allocates one object for each aggregate value. +It avoids the closure and dictionary allocations in the proof of concept while +preserving the public deserializer interface. + +## Alternatives + +### Rebase the Proof of Concept + +The proof of concept predates the current semantic model, selection rules, and +CLI behavior. Rebasing it would replace tested model code and mix generator +work with model regressions. Porting generator components keeps those concerns +separate. + +### Eager Monolithic Schemas + +One schema module is easy to generate, but importing any model constructs the +entire reachable schema graph. Large services pay that cost before selecting +an operation. + +### One File per Shape + +One file per shape maximizes import granularity but creates thousands of files +for large services. Partitioning captures most import savings with lower +filesystem and wheel overhead. + +### Lazy Schema Proxies + +A proxy can defer schema construction without changing generated call sites. +It changes runtime type and identity behavior and adds resolution to schema +attribute access. Concrete schemas with lazy accessors keep the hot path +direct. + +### Class Methods for Schema Access + +A class method receives the concrete Python subclass and can create a separate +cache entry for it. Smithy schema identity belongs to the modeled shape, so a +static method provides the required ownership. + +### Dictionary-Based Deserialization + +Collecting member values in a dictionary preserves keyword construction with +little generated code. It hashes a string for each decoded member and expands +the dictionary during construction. Generated index dispatch and slotted state +avoid that work. + +### Slotted Public Models + +Slotted dataclasses reduce instance memory and can speed attribute access. They +remove `__dict__` and restrict subclass behavior, which changes observable +Python behavior. A later design can adopt slots after compatibility review and +benchmarks. + +## Future Work + +* Add a state-parameter deserializer API if it outperforms callable state across + JSON, XML, CBOR, and HTTP binding paths. +* Precompute validation masks and constraint metadata used during every + structure decode. +* Generate endpoint rule bytecode from normalized endpoint rules. +* Support incremental generation from a prior manifest. +* Evaluate partition thresholds on large AWS service models. +* Evaluate slotted generated models as a new artifact compatibility mode. diff --git a/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json new file mode 100644 index 000000000..de1bae3a8 --- /dev/null +++ b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added configurable eager JSON deserialization to improve buffered payload performance. Generated JSON clients automatically parse `bytes` eagerly while preserving incremental parsing for streaming readers, and can explicitly select eager or streaming behavior." +} diff --git a/packages/smithy-json/README.md b/packages/smithy-json/README.md index 29b5448d6..ecabd8054 100644 --- a/packages/smithy-json/README.md +++ b/packages/smithy-json/README.md @@ -2,3 +2,22 @@ This package provides generic JSON serialization and deserialization support for Smithy clients and servers. + +## Deserialization modes + +`JSONCodec` supports eager and streaming deserialization. The default `AUTO` +mode eagerly parses byte payloads and incrementally parses `BytesReader` +payloads: + +```python +from smithy_json import JSONCodec, JSONDeserializationMode + +codec = JSONCodec( + deserialization_mode=JSONDeserializationMode.STREAMING, +) +``` + +Eager mode is faster for buffered payloads but temporarily materializes the +complete JSON value tree. Streaming mode avoids that additional allocation. +Generated clients using a JSON protocol expose the same choice through +`json_deserialization_mode` on their configuration. diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index 7d20bba87..4867f6577 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -11,10 +11,18 @@ from ._private.deserializers import JSONShapeDeserializer as _JSONShapeDeserializer from ._private.documents import JSONDocument from ._private.serializers import JSONShapeSerializer as _JSONShapeSerializer -from .settings import JSONSettings +from ._private.value_deserializer import ( + JSONValueDeserializer as _JSONValueDeserializer, +) +from .settings import JSONDeserializationMode, JSONSettings __version__ = "0.3.0" -__all__ = ("JSONCodec", "JSONDocument", "JSONSettings") +__all__ = ( + "JSONCodec", + "JSONDeserializationMode", + "JSONDocument", + "JSONSettings", +) class JSONCodec(Codec): @@ -27,6 +35,7 @@ def __init__( default_timestamp_format: TimestampFormat = TimestampFormat.DATE_TIME, default_namespace: str | None = None, document_class: type[JSONDocument] = JSONDocument, + deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO, ) -> None: """Initializes a JSONCodec. @@ -39,6 +48,8 @@ def __init__( :param default_namespace: The default namespace to use when determining a document's discriminator. :param document_class: The document class to deserialize to. + :param deserialization_mode: Controls whether JSON payloads are parsed eagerly, + incrementally, or selected automatically based on the source type. """ self._settings = JSONSettings( use_json_name=use_json_name, @@ -46,6 +57,7 @@ def __init__( default_timestamp_format=default_timestamp_format, default_namespace=default_namespace, document_class=document_class, + deserialization_mode=deserialization_mode, ) @property @@ -56,6 +68,22 @@ def create_serializer(self, sink: BytesWriter) -> "ShapeSerializer": return _JSONShapeSerializer(sink, settings=self._settings) def create_deserializer(self, source: bytes | BytesReader) -> "ShapeDeserializer": + mode = self._settings.deserialization_mode + if mode is JSONDeserializationMode.EAGER or ( + mode is JSONDeserializationMode.AUTO and isinstance(source, bytes) + ): + return _JSONValueDeserializer(source, settings=self._settings) + if isinstance(source, bytes): source = BytesIO(source) return _JSONShapeDeserializer(source, settings=self._settings) + + @property + def deserialization_mode(self) -> JSONDeserializationMode: + """The strategy used to deserialize JSON payloads.""" + + return self._settings.deserialization_mode + + @deserialization_mode.setter + def deserialization_mode(self, value: JSONDeserializationMode) -> None: + self._settings.deserialization_mode = value diff --git a/packages/smithy-json/src/smithy_json/_private/value_deserializer.py b/packages/smithy-json/src/smithy_json/_private/value_deserializer.py new file mode 100644 index 000000000..f5641a117 --- /dev/null +++ b/packages/smithy-json/src/smithy_json/_private/value_deserializer.py @@ -0,0 +1,205 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import datetime +import json +from base64 import b64decode +from collections.abc import Callable, Mapping, Sequence +from decimal import Decimal +from json import JSONDecodeError + +from ijson.common import IncompleteJSONError # type: ignore +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import SmithyError +from smithy_core.interfaces import BytesReader +from smithy_core.schemas import Schema +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import JSONNameTrait, TimestampFormatTrait +from smithy_core.types import TimestampFormat + +from ..settings import JSONSettings + +type JSONValue = ( + Mapping[str, "JSONValue"] + | Sequence["JSONValue"] + | str + | int + | Decimal + | bool + | None +) + + +class JSONValueError(SmithyError): + """Raised when a parsed JSON value does not match the requested Smithy type.""" + + def __init__(self, expected: str, value: JSONValue) -> None: + super().__init__( + f"Error parsing JSON. Expected value of type `{expected}`, " + f"but found: `{type(value).__name__}`: {value}" + ) + + +def _reject_constant(value: str) -> None: + raise ValueError(f"Invalid JSON constant: {value}") + + +class JSONValueDeserializer(ShapeDeserializer): + """Deserializes Smithy shapes from a fully materialized JSON value.""" + + def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: + try: + if isinstance(source, bytes): + self._value: JSONValue = json.loads( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + else: + self._value = json.load( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + except (JSONDecodeError, UnicodeDecodeError, ValueError) as error: + raise IncompleteJSONError(f"parse error: {error}") from error + + self._settings = settings + self._json_names: dict[ShapeID, dict[str, Schema]] = {} + + def is_null(self) -> bool: + return self._value is None + + def read_null(self) -> None: + if self._value is not None: + raise JSONValueError("null", self._value) + + def read_boolean(self, schema: Schema) -> bool: + if not isinstance(self._value, bool): + raise JSONValueError("boolean", self._value) + return self._value + + def read_blob(self, schema: Schema) -> bytes: + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return b64decode(self._value) + + def read_integer(self, schema: Schema) -> int: + if not isinstance(self._value, int) or isinstance(self._value, bool): + raise JSONValueError("number", self._value) + return self._value + + def read_float(self, schema: Schema) -> float: + match self._value: + case Decimal() | "Infinity" | "-Infinity" | "NaN": + return float(self._value) + case int() if not isinstance(self._value, bool): + return self._value + case _: + raise JSONValueError("number", self._value) + + def read_big_decimal(self, schema: Schema) -> Decimal: + match self._value: + case Decimal(): + return self._value + case int() if not isinstance(self._value, bool): + return Decimal(self._value) + case "Infinity" | "-Infinity" | "NaN": + return Decimal(self._value) + case _: + raise JSONValueError("number", self._value) + + def read_string(self, schema: Schema) -> str: + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return self._value + + def read_document(self, schema: Schema) -> Document: + return self._settings.document_class( + value=self._value, + schema=schema, + settings=self._settings, + ) + + def read_timestamp(self, schema: Schema) -> datetime.datetime: + format = self._settings.default_timestamp_format + if self._settings.use_timestamp_format: + if format_trait := schema.get_trait(TimestampFormatTrait): + format = format_trait.format + + match format: + case TimestampFormat.EPOCH_SECONDS: + return format.deserialize(self.read_float(schema=schema)) + case _: + return format.deserialize(self.read_string(schema=schema)) + + def read_struct( + self, + schema: Schema, + consumer: Callable[[Schema, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + struct_schema = schema.member_target or schema + members = self._members_by_wire_name(struct_schema) + try: + for key, member_value in value.items(): + member = members.get(key) + if member is None: + continue + if member_value is None and member.shape_type is not ShapeType.DOCUMENT: + continue + self._value = member_value + consumer(member, self) + finally: + self._value = value + + def read_list( + self, + schema: Schema, + consumer: Callable[[ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Sequence) or isinstance(value, str): + raise JSONValueError("array", value) + + try: + for member_value in value: + self._value = member_value + consumer(self) + finally: + self._value = value + + def read_map( + self, + schema: Schema, + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + try: + for key, member_value in value.items(): + self._value = member_value + consumer(key, self) + finally: + self._value = value + + def _members_by_wire_name(self, schema: Schema) -> Mapping[str, Schema]: + if not self._settings.use_json_name: + return schema.members + + members = self._json_names.get(schema.id) + if members is None: + members = {} + for member_name, member in schema.members.items(): + json_name = member.get_trait(JSONNameTrait) + members[json_name.value if json_name is not None else member_name] = ( + member + ) + self._json_names[schema.id] = members + return members diff --git a/packages/smithy-json/src/smithy_json/plugins.py b/packages/smithy-json/src/smithy_json/plugins.py new file mode 100644 index 000000000..7423bb489 --- /dev/null +++ b/packages/smithy-json/src/smithy_json/plugins.py @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Protocol + +from . import JSONCodec +from .settings import JSONDeserializationMode + + +class _JSONDeserializationConfig(Protocol): + @property + def protocol(self) -> object | None: ... + + @property + def json_deserialization_mode(self) -> JSONDeserializationMode | None: ... + + +def json_deserialization_plugin(config: _JSONDeserializationConfig) -> None: + """Configure JSON deserialization on a generated client's protocol codec.""" + + codec = getattr(config.protocol, "payload_codec", None) + if isinstance(codec, JSONCodec): + codec.deserialization_mode = ( + config.json_deserialization_mode or JSONDeserializationMode.AUTO + ) diff --git a/packages/smithy-json/src/smithy_json/settings.py b/packages/smithy-json/src/smithy_json/settings.py index 6ebde62cf..3b0253705 100644 --- a/packages/smithy-json/src/smithy_json/settings.py +++ b/packages/smithy-json/src/smithy_json/settings.py @@ -1,6 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass +from enum import StrEnum from typing import TYPE_CHECKING from smithy_core.types import TimestampFormat @@ -9,6 +10,19 @@ from ._private.documents import JSONDocument +class JSONDeserializationMode(StrEnum): + """Controls how JSON payloads are parsed during deserialization.""" + + AUTO = "auto" + """Eagerly parse bytes while preserving incremental reader consumption.""" + + EAGER = "eager" + """Consume and materialize the complete JSON value before deserializing.""" + + STREAMING = "streaming" + """Incrementally deserialize JSON using the streaming parser.""" + + @dataclass(slots=True) class JSONSettings: """Settings for the JSON codec.""" @@ -29,3 +43,6 @@ class JSONSettings: default_namespace: str | None = None """The default namespace to use when determining a document's discriminator.""" + + deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO + """Controls whether JSON deserialization is eager, streaming, or automatic.""" diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index 00a6ed2cd..0402f9c6b 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -3,9 +3,11 @@ import math from datetime import datetime from decimal import Decimal +from io import BytesIO from typing import Any import pytest +from ijson.common import IncompleteJSONError # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.prelude import ( @@ -18,7 +20,9 @@ STRING, TIMESTAMP, ) -from smithy_json import JSONCodec, JSONDocument +from smithy_json import JSONCodec, JSONDeserializationMode, JSONDocument +from smithy_json._private.deserializers import JSONShapeDeserializer +from smithy_json._private.value_deserializer import JSONValueDeserializer from . import ( JSON_SERDE_CASES, @@ -28,9 +32,17 @@ ) +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) @pytest.mark.parametrize("expected, given", JSON_SERDE_CASES) -def test_json_deserializer(expected: Any, given: bytes) -> None: - codec = JSONCodec() +def test_json_deserializer( + expected: Any, + given: bytes, + mode: JSONDeserializationMode, +) -> None: + codec = JSONCodec(deserialization_mode=mode) deserializer = codec.create_deserializer(given) match expected: case None: @@ -108,3 +120,46 @@ def test_uses_custom_document() -> None: codec = JSONCodec(document_class=CustomDocument) actual = codec.create_deserializer(b'{"foo": "bar"}').read_document(DOCUMENT) assert isinstance(actual, CustomDocument) + + +@pytest.mark.parametrize( + "mode, source, expected_type", + [ + (JSONDeserializationMode.AUTO, b"{}", JSONValueDeserializer), + (JSONDeserializationMode.AUTO, BytesIO(b"{}"), JSONShapeDeserializer), + (JSONDeserializationMode.EAGER, BytesIO(b"{}"), JSONValueDeserializer), + (JSONDeserializationMode.STREAMING, b"{}", JSONShapeDeserializer), + ], +) +def test_deserialization_mode_selects_parser( + mode: JSONDeserializationMode, + source: bytes | BytesIO, + expected_type: type[ShapeDeserializer], +) -> None: + deserializer = JSONCodec(deserialization_mode=mode).create_deserializer(source) + assert isinstance(deserializer, expected_type) + + +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) +def test_ignores_unknown_structure_members(mode: JSONDeserializationMode) -> None: + actual = JSONCodec(deserialization_mode=mode).deserialize( + b'{"unknown":{"nested":[1,2,3]},"stringMember":"value"}', + SerdeShape, + ) + + assert actual.string_member == "value" + + +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) +def test_invalid_json_uses_existing_error_type(mode: JSONDeserializationMode) -> None: + with pytest.raises(IncompleteJSONError): + deserializer = JSONCodec(deserialization_mode=mode).create_deserializer( + b'{"incomplete":' + ) + deserializer.read_document(DOCUMENT) diff --git a/packages/smithy-json/tests/unit/test_plugins.py b/packages/smithy-json/tests/unit/test_plugins.py new file mode 100644 index 000000000..b16618c0d --- /dev/null +++ b/packages/smithy-json/tests/unit/test_plugins.py @@ -0,0 +1,52 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass + +import pytest +from smithy_json import JSONCodec, JSONDeserializationMode +from smithy_json.plugins import json_deserialization_plugin + + +@dataclass +class _Protocol: + payload_codec: object + + +@dataclass +class _Config: + protocol: object + json_deserialization_mode: JSONDeserializationMode | None + + +@pytest.mark.parametrize( + "configured, expected", + [ + (None, JSONDeserializationMode.AUTO), + (JSONDeserializationMode.AUTO, JSONDeserializationMode.AUTO), + (JSONDeserializationMode.EAGER, JSONDeserializationMode.EAGER), + (JSONDeserializationMode.STREAMING, JSONDeserializationMode.STREAMING), + ], +) +def test_configures_json_codec( + configured: JSONDeserializationMode | None, + expected: JSONDeserializationMode, +) -> None: + codec = JSONCodec(deserialization_mode=JSONDeserializationMode.EAGER) + config = _Config( + protocol=_Protocol(payload_codec=codec), + json_deserialization_mode=configured, + ) + + json_deserialization_plugin(config) + + assert codec.deserialization_mode is expected + + +@pytest.mark.parametrize("protocol", [object(), _Protocol(payload_codec=object())]) +def test_ignores_protocols_without_json_codec(protocol: object) -> None: + config = _Config( + protocol=protocol, + json_deserialization_mode=JSONDeserializationMode.STREAMING, + ) + json_deserialization_plugin(config)