From 82612498d6d93ea5d31074e4f7bbfb880044781c Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:29:51 -0400 Subject: [PATCH 1/4] feat(smithy-json): add configurable eager deserialization Add automatic, eager, and streaming JSON deserialization modes and expose the setting through generated JSON client configurations. --- codegen-output/aws-json-1-0/pyproject.toml | 3 +- .../aws-json-1-0/src/awsjson10/client.py | 7 +- .../aws-json-1-0/src/awsjson10/config.py | 9 + codegen-output/aws-json-1-1/pyproject.toml | 3 +- .../aws-json-1-1/src/awsjson11/client.py | 7 +- .../aws-json-1-1/src/awsjson11/config.py | 9 + codegen-output/rest-json-1/pyproject.toml | 3 +- .../rest-json-1/src/restjson/client.py | 7 +- .../rest-json-1/src/restjson/config.py | 9 + .../python/codegen/test/AwsCodegenTest.java | 7 + .../codegen/test/PythonCodegenTest.java | 8 + .../JsonDeserializationIntegration.java | 69 ++ ...hon.codegen.integrations.PythonIntegration | 1 + designs/codegen/generated-package.md | 672 ++++++++++++++++++ ...ture-4b7f9ca09557405eb9382c56490c6d7a.json | 4 + packages/smithy-json/README.md | 19 + .../smithy-json/src/smithy_json/__init__.py | 32 +- .../_private/value_deserializer.py | 205 ++++++ .../smithy-json/src/smithy_json/plugins.py | 25 + .../smithy-json/src/smithy_json/settings.py | 17 + .../tests/unit/test_deserializers.py | 61 +- .../smithy-json/tests/unit/test_plugins.py | 52 ++ 22 files changed, 1218 insertions(+), 11 deletions(-) create mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java create mode 100644 designs/codegen/generated-package.md create mode 100644 packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json create mode 100644 packages/smithy-json/src/smithy_json/_private/value_deserializer.py create mode 100644 packages/smithy-json/src/smithy_json/plugins.py create mode 100644 packages/smithy-json/tests/unit/test_plugins.py diff --git a/codegen-output/aws-json-1-0/pyproject.toml b/codegen-output/aws-json-1-0/pyproject.toml index de6d7aa27..b05b8c87e 100644 --- a/codegen-output/aws-json-1-0/pyproject.toml +++ b/codegen-output/aws-json-1-0/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-0/src/awsjson10/client.py b/codegen-output/aws-json-1-0/src/awsjson10/client.py index 5f9cb2388..c6dcee27b 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/client.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJSONRPC10Config, Plugin from .models import ( @@ -94,7 +95,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncJSONRPC10Config: if not self._setup_done: diff --git a/codegen-output/aws-json-1-0/src/awsjson10/config.py b/codegen-output/aws-json-1-0/src/awsjson10/config.py index 1696a93d0..b66993b2f 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/config.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -111,6 +112,7 @@ class _AsyncJSONRPC10ConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -174,6 +176,12 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -182,6 +190,7 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_rpc_10") diff --git a/codegen-output/aws-json-1-1/pyproject.toml b/codegen-output/aws-json-1-1/pyproject.toml index a9b718abf..150cfed84 100644 --- a/codegen-output/aws-json-1-1/pyproject.toml +++ b/codegen-output/aws-json-1-1/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-1/src/awsjson11/client.py b/codegen-output/aws-json-1-1/src/awsjson11/client.py index 897d043d3..cdc16a79d 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/client.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/client.py @@ -14,6 +14,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJsonProtocolConfig, Plugin from .models import ( @@ -100,7 +101,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/aws-json-1-1/src/awsjson11/config.py b/codegen-output/aws-json-1-1/src/awsjson11/config.py index bbbeb3e1e..83d1fbe6a 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/config.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -111,6 +112,7 @@ class _AsyncJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -174,6 +176,12 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -182,6 +190,7 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_protocol") diff --git a/codegen-output/rest-json-1/pyproject.toml b/codegen-output/rest-json-1/pyproject.toml index 953379f29..9d63abd61 100644 --- a/codegen-output/rest-json-1/pyproject.toml +++ b/codegen-output/rest-json-1/pyproject.toml @@ -26,7 +26,8 @@ classifiers = [ dependencies = [ "smithy_aws_core[eventstream, json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0" + "smithy_http[aiohttp]~=0.6.0", + "smithy_json~=0.3.0" ] [project.optional-dependencies] diff --git a/codegen-output/rest-json-1/src/restjson/client.py b/codegen-output/rest-json-1/src/restjson/client.py index 5c0a36df8..781c861dd 100644 --- a/codegen-output/rest-json-1/src/restjson/client.py +++ b/codegen-output/rest-json-1/src/restjson/client.py @@ -19,6 +19,7 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin +from smithy_json.plugins import json_deserialization_plugin from .config import AsyncRestJsonProtocolConfig, Plugin from .models import ( @@ -392,7 +393,11 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] + self._client_plugins: list[Plugin] = [ + aws_user_agent_plugin, + json_deserialization_plugin, + user_agent_plugin, + ] async def _ensure_setup(self) -> AsyncRestJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/rest-json-1/src/restjson/config.py b/codegen-output/rest-json-1/src/restjson/config.py index 68c6a03da..2894a5685 100644 --- a/codegen-output/rest-json-1/src/restjson/config.py +++ b/codegen-output/rest-json-1/src/restjson/config.py @@ -25,6 +25,7 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient +from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -556,6 +557,7 @@ class _AsyncRestJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None + json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -619,6 +621,12 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): header. """ + json_deserialization_mode: JSONDeserializationMode | None = None + """ + Controls whether JSON response payloads are deserialized eagerly or + incrementally. + """ + _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -627,6 +635,7 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), + "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("rest_json_protocol") diff --git a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java index b5519b0d2..1574c3ab4 100644 --- a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java +++ b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java @@ -56,6 +56,10 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertTrue(config.contains("self.auth_schemes = auth_schemes")); assertTrue(config.contains("default_factory=lambda: AIOHTTPClient()")); assertFalse(config.contains("from smithy_http.aio.crt import AWSCRTHTTPClient")); + assertTrue(config.contains( + "json_deserialization_mode: JSONDeserializationMode | None")); + assertTrue(config.contains( + "\"json_deserialization_mode\": FieldSpec(default=None)")); var client = Files.readString(tempDir.resolve("src/restjson/client.py")); assertInOrder( @@ -73,6 +77,9 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertFalse(client.contains("plugin(self._config)")); assertTrue(client.contains("retry_mode=config.retry_mode")); assertTrue(client.contains("max_attempts=config.max_attempts")); + assertTrue(client.contains( + "from smithy_json.plugins import json_deserialization_plugin")); + assertTrue(client.contains("json_deserialization_plugin,")); var schemas = Files.readString(tempDir.resolve("src/restjson/_private/schemas.py")); assertTrue(schemas.contains("aws.protocols#restJson1")); diff --git a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java index 8d5a84bfa..c64ed5be0 100644 --- a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java +++ b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java @@ -53,6 +53,14 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { var config = Files.readString(tempDir.resolve("src/weather/config.py")); assertTrue(config.contains("self.transport = transport or AIOHTTPClient()")); assertFalse(config.contains("self.transport = transport or AWSCRTHTTPClient()")); + assertTrue(config.contains( + "json_deserialization_mode: JSONDeserializationMode | None")); + assertTrue(config.contains( + "self.json_deserialization_mode = json_deserialization_mode")); + + assertTrue(client.contains( + "from smithy_json.plugins import json_deserialization_plugin")); + assertTrue(client.contains("json_deserialization_plugin,")); // An output that models a "responseMetadata" member gets both the SDK-reserved // response_metadata attribute and the escaped modeled member, so neither shadows diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java new file mode 100644 index 000000000..5efbe1dc9 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java @@ -0,0 +1,69 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import java.util.List; +import java.util.Set; +import software.amazon.smithy.aws.traits.protocols.AwsJson1_0Trait; +import software.amazon.smithy.aws.traits.protocols.AwsJson1_1Trait; +import software.amazon.smithy.aws.traits.protocols.RestJson1Trait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.codegen.core.SymbolReference; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.ServiceIndex; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Adds configurable JSON deserialization to clients that support a JSON protocol. + */ +@SmithyInternalApi +public final class JsonDeserializationIntegration implements PythonIntegration { + private static final Set JSON_PROTOCOLS = Set.of( + RestJson1Trait.ID, + AwsJson1_0Trait.ID, + AwsJson1_1Trait.ID); + + @Override + public List getClientPlugins(GenerationContext context) { + var mode = Symbol.builder() + .name("JSONDeserializationMode") + .namespace("smithy_json", ".") + .addDependency(SmithyPythonDependency.SMITHY_JSON) + .build(); + var plugin = SymbolReference.builder() + .symbol(Symbol.builder() + .name("json_deserialization_plugin") + .namespace("smithy_json.plugins", ".") + .addDependency(SmithyPythonDependency.SMITHY_JSON) + .build()) + .build(); + var property = ConfigProperty.builder() + .name("json_deserialization_mode") + .type(mode) + .documentation( + "Controls whether JSON response payloads are deserialized eagerly or incrementally.") + .nullable(true) + .build(); + + return List.of(RuntimeClientPlugin.builder() + .servicePredicate(JsonDeserializationIntegration::usesJsonProtocol) + .addConfigProperty(property) + .pythonPlugin(plugin) + .build()); + } + + static boolean usesJsonProtocol(Model model, ServiceShape service) { + return ServiceIndex.of(model) + .getProtocols(service) + .keySet() + .stream() + .anyMatch(JSON_PROTOCOLS::contains); + } +} diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index ccdf63695..f85f625be 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -4,6 +4,7 @@ # software.amazon.smithy.python.codegen.integrations.RestJsonIntegration +software.amazon.smithy.python.codegen.integrations.JsonDeserializationIntegration software.amazon.smithy.python.codegen.integrations.Rpcv2CborIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth software.amazon.smithy.python.codegen.integrations.UserAgentIntegration diff --git a/designs/codegen/generated-package.md b/designs/codegen/generated-package.md new file mode 100644 index 000000000..44c90aa6d --- /dev/null +++ b/designs/codegen/generated-package.md @@ -0,0 +1,672 @@ +# Python-Native Generated Package Architecture + +The Python generator emits clients and types packages that use the existing +Smithy Python runtime. Generated packages preserve the public behavior of the +Java generator while loading schemas and model partitions on demand. + +## Goals + +* Generate clients and standalone types packages from the semantic model in + `smithy_python.model`. +* Preserve supported generated package APIs during migration from the Java + generator. +* Avoid constructing user-defined schemas while importing a generated client. +* Use dense member indexes in generated serialization and deserialization. +* Preserve one runtime `Schema` instance for each generated shape. +* Support recursive shape graphs and free-threaded Python. +* Let protocol and platform integrations add generation behavior without + replacing the core generator. +* Measure generation time, import cost, first-use cost, memory, and warm serde + throughout development. + +## Scope + +The first implementation supports standalone types and Rest JSON clients. +Later changes add the remaining protocols after their runtime behavior passes +the existing protocol tests. + +The Java generator remains available during migration. A projection selects +which generator to run, so generated packages can move independently after +their required features reach parity. + +Server generation, custom model validation, and runtime compilation of serde +functions remain future work. Generated dataclasses retain their current +non-slotted layout until compatibility tests and benchmarks support changing +it. + +## Performance Gates + +The benchmark suite records these measurements for equivalent Java-generated +and Python-generated packages: + +* Code generation wall time and peak resident memory. +* Generated source size and compiled bytecode size. +* Package, models, config, and client import time. +* Client construction time. +* First serialization and deserialization time. +* Warm serialization and deserialization time. +* Peak allocations for first use and warm serde. + +Rest JSON provides the first comparison. Rest XML and RPC v2 CBOR join the +suite when the native generator supports them. + +Each measured serde case uses the same model, values, payload, warmup count, +and measured iteration count for both generators. Benchmark output includes +the interpreter version, platform, architecture, and generator commit. + +The implementation must satisfy these behavioral gates: + +* Importing the generated client constructs no user-defined shape schemas. +* Importing a model class constructs no schema until its `schema()` method or + an operation that uses it is accessed. +* Repeated calls to a shape's `schema()` method return the same object. +* Generated serde contains no string-keyed member schema lookup. +* Warm Rest JSON p50 may not regress by more than 5 percent against the Java + generator for the same benchmark case. A larger regression blocks migration + for that case. + +Cold-start measurements establish a baseline before the first client migrates. +The design uses behavioral gates for schema loading because host load and +filesystem state make a single import-time percentage unstable. + +## Generator Architecture + +The generator uses the existing model loader, service selection, CLI, and +plugin environment: + +```text +Smithy JSON AST + | + v +semantic model + | + v +shape selection + | + v +generation context + | + +------> core generators + | + +------> protocol and platform integrations + | + v +generated package +``` + +The proof of concept supplies initial implementations for symbols, writers, +generation context, plugins, models, schemas, clients, and protocol tests. +Those components are ported onto the current semantic model. The repository's +current model loader, shape selection, CLI, and environment handling remain +authoritative. + +### Generation Context + +One generation context owns: + +* The semantic model and selected service. +* Selected shapes in model order. +* Shape and member indexes. +* Symbols and generated names. +* Protocol and platform integrations. +* Dependency and generated-file manifests. +* Schema dependency components and source partitions. + +The context builds indexes once. Generators query those indexes instead of +rescanning every selected shape for each output file. + +### Symbols + +The symbol provider maps semantic shapes and members to Python names, module +paths, runtime dependencies, schema accessors, serializer functions, and +deserializer functions. + +Symbol results are cached by shape ID for the duration of one generation. +Member indexes are assigned once from the selected model order and reused by +schema generation and serde generation. + +Plugins may decorate symbols or supply symbols for integration-owned types. +The core provider remains responsible for collision handling and service +renames. + +### Integrations + +An integration declares the protocols and platform behavior it supports. It +can contribute: + +* Protocol construction. +* Runtime dependencies. +* Generated config fields and defaults. +* Client plugins. +* Traits retained in runtime schemas. +* Code sections in generated files. + +Integrations operate on the semantic model and generation context. Generated +packages depend on the Smithy runtime packages and exclude `smithy-python`. + +### Writers + +Writers collect imports, generated sections, and file contents in memory. The +file manifest writes changed files and removes stale generated files within +the output directory. + +Formatting and linting run after generation when requested. Tests exercise the +unformatted writer output as valid Python so formatting cannot hide syntax +errors. + +## Generated Package Layout + +Generated packages expose stable public modules and place partition details +under `_private`: + +```text +example_service/ + __init__.py + client.py + config.py + models.py + py.typed + _private/ + operations.py + models_0.py + models_1.py + schemas_0.py + schemas_1.py +``` + +Source partitions start with a target size of 128 KiB. The generator estimates +partition size from rendered declarations. The threshold is an implementation +setting, and benchmarks may change it. + +Private module names and partition assignments are unstable. Public model, +client, and config imports remain stable across regenerations. + +## Public Model Facade + +`models.py` exposes generated model classes, enums, unions, and operation +constants. Runtime imports resolve an exported name on first access: + +```python +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ._private.models_0 import GetObjectInput as GetObjectInput + from ._private.models_1 import GetObjectOutput as GetObjectOutput + +_EXPORTS = { + "GetObjectInput": ("._private.models_0", "GetObjectInput"), + "GetObjectOutput": ("._private.models_1", "GetObjectOutput"), +} + + +def __getattr__(name: str) -> Any: + export = _EXPORTS.get(name) + if export is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_name, symbol_name = export + value = getattr(import_module(module_name, __package__), symbol_name) + globals()[name] = value + return value +``` + +Caching the resolved value in the module makes later access a normal global +lookup. `TYPE_CHECKING` imports expose concrete symbols to static type checkers +without importing model partitions at runtime. + +`__all__` lists the public generated names. `__dir__` combines those names with +the module globals so interactive discovery remains available before +resolution. + +## Model Partitions + +Model partitions contain dataclasses, enums, union variants, and their +generated serde helpers. The generator builds a dependency graph from runtime +model references and groups strongly connected components before assigning +partitions. + +Postponed annotations remove runtime imports needed only for type hints. +Cross-partition imports needed by serde occur through private accessors and +are cached after first use. + +Generated dataclasses keep keyword-only constructors, modeled defaults, +sensitive-member representation, and the existing serialization interfaces. +Modeled errors retain their service error hierarchy. + +## Lazy Schema Partitions + +Schema construction is lazy at the shape level. Each schema partition stores +completed schemas in a dense array indexed by constants generated for that +partition: + +```python +_SCHEMAS: list[Schema | None] = [None] * 24 +_LOCK = RLock() + +_GET_OBJECT_INPUT = 7 +``` + +The warm path performs one array lookup and one `None` check. + +### Dependency Components + +The generator builds a directed graph from each schema to its member targets. +It computes strongly connected components and treats each component as one +initialization unit. + +An acyclic shape forms a component of one. Mutually recursive shapes share a +component. Components form a directed acyclic graph, which gives schema +initialization a stable dependency order. + +A recursive component is never split across source partitions. + +### Component Initialization + +A component builder creates every root schema in the component before it +creates member schemas: + +```python +def _build_component_3() -> None: + node = Schema.collection( + id=ShapeID("example#Node"), + members={"value": None, "next": None}, + ) + + node.members["value"] = Schema.member( + id=node.id.with_member("value"), + target=STRING, + index=0, + ) + node.members["next"] = Schema.member( + id=node.id.with_member("next"), + target=node, + index=1, + ) + + _SCHEMAS[_NODE] = node +``` + +The public accessor uses a double-checked cache lookup: + +```python +def get_node_schema() -> Schema: + schema = _SCHEMAS[_NODE] + if schema is not None: + return schema + + with _LOCK: + schema = _SCHEMAS[_NODE] + if schema is None: + _build_component_3() + schema = _SCHEMAS[_NODE] + assert schema is not None + return schema +``` + +The component lock remains held until every schema in the component is +complete. Another thread sees an empty cache slot, waits for the lock, and +then reads the completed schema. The builder assigns cache slots only after +all roots and members in the component are complete. + +Cross-component dependencies resolve before the current component is +published. The component graph has no cycles, so initialization cannot recurse +back into an unpublished component. + +If construction raises an exception, the cache slots remain empty. A later +call can retry construction. + +### Shape Schema API + +Generated shapes expose a static `schema()` method: + +```python +_NODE_SCHEMA: Schema | None = None +_NODE_MEMBERS: tuple[Schema, ...] | None = None + + +def _node_schema() -> Schema: + global _NODE_SCHEMA, _NODE_MEMBERS + + if (schema := _NODE_SCHEMA) is None: + from .schemas_0 import get_node_schema + + schema = get_node_schema() + _NODE_MEMBERS = schema.members_by_index + _NODE_SCHEMA = schema + return schema + + +def _node_members() -> tuple[Schema, ...]: + members = _NODE_MEMBERS + if members is None: + _node_schema() + members = _NODE_MEMBERS + assert members is not None + return members + + +class Node: + @staticmethod + def schema() -> Schema: + return _node_schema() +``` + +The schema belongs to the modeled shape. It does not vary by Python subclass, +so the method uses `staticmethod`. A class method could create +subclass-specific caches and schema identities. + +The model partition caches the member tuple with the root schema. Generated +serde reads that tuple and does not query the name-keyed member map. + +### Schema Identity and Extensions + +Each shape accessor resolves to one canonical `Schema`. Operations, model +classes, type registries, protocols, and codecs share that object. + +Schema extensions remain lazy and schema-local. HTTP binding metadata, codec +metadata, and future extension values therefore survive across client and +protocol instances after first use. + +Generated code never wraps a schema in a lazy proxy. A proxy would change +schema identity and add resolver work to every attribute access. + +## Generated Serialization + +Structure serialization uses generated member access and dense schema indexes: + +```python +def serialize_members(self, serializer: ShapeSerializer) -> None: + members = _node_members() + serializer.write_string(members[0], self.value) + if self.next is not None: + serializer.write_struct(members[1], self.next) +``` + +`_node_members()` initializes the shape schema on its first call and returns +the cached tuple afterward. + +Generated serialization retains modeled nullability, sparse collection +behavior, timestamp formats, streaming members, and event stream handling. +Protocol-specific routing remains in runtime protocol packages. For example, +the HTTP binding serializer uses the schema's cached HTTP binding extension +while the generated structure writes members in modeled order. + +Collection serializers cache their `member`, `key`, and `value` schemas by +index. Union variants cache their variant member schema. + +## Generated Deserialization + +Structure deserialization uses a slotted callable object as mutable state: + +```python +class _NodeDeserializer: + __slots__ = ("value", "next") + + def __init__(self) -> None: + self.value: str | None = None + self.next: Node | None = None + + def __call__( + self, member: Schema, deserializer: ShapeDeserializer + ) -> None: + match member.expect_member_index(): + case 0: + self.value = deserializer.read_string(member) + case 1: + self.next = Node.deserialize(deserializer) + + def build(self) -> Node: + return Node(value=self.value, next=self.next) +``` + +`deserialize()` allocates one state object, passes it directly as the member +consumer, and calls `build()`: + +```python +@classmethod +def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + state = _NodeDeserializer() + deserializer.read_struct(_node_schema(), state) + return state.build() +``` + +Member indexes select generated cases, and fixed attributes store decoded +values. The primary path allocates one state object for each structure. + +The state initializer applies modeled defaults. Required members use a private +missing sentinel and `build()` reports an absent required value. Mutable +defaults are created for each state object. + +`deserialize_kwargs()` remains available for compatibility. It uses the same +state object and creates a dictionary only when a caller invokes that method. +The primary `deserialize()` path constructs the model directly. + +Union deserializers use one result slot and reject multiple values. List and +map deserializers use slotted callable state objects that retain the target +collection and cached member schemas. + +The first implementation keeps the existing `ShapeDeserializer` interface. +A state-parameter overload may replace callable state objects after every +official codec supports it and benchmarks show a gain. + +## Operations and Client Imports + +Operations resolve on first use from a dense module cache: + +```python +_OPERATIONS: list[APIOperation[Any, Any] | None] = [None] * 18 +_OPERATION_LOCK = Lock() +_GET_OBJECT = 4 + + +def get_get_object_operation() -> APIOperation[Any, Any]: + operation = _OPERATIONS[_GET_OBJECT] + if operation is None: + with _OPERATION_LOCK: + operation = _OPERATIONS[_GET_OBJECT] + if operation is None: + operation = APIOperation( + input=GetObjectInput, + output=GetObjectOutput, + schema=get_get_object_schema(), + input_schema=GetObjectInput.schema(), + output_schema=GetObjectOutput.schema(), + error_registry=_get_get_object_error_registry(), + effective_auth_schemes=_GET_OBJECT_AUTH, + error_schemas=_get_get_object_error_schemas(), + ) + _OPERATIONS[_GET_OBJECT] = operation + return operation +``` + +The client imports model types under `TYPE_CHECKING` for annotations. Each +operation method imports its private operation getter and caches the resolved +operation through the operations module. + +Importing `client.py` loads client infrastructure and config types. It does not +load model or schema partitions. Calling one operation loads that operation's +input, output, errors, and reachable schema components. + +Error registries import modeled error classes when their operation initializes. +Receiving an unknown error does not require loading unrelated modeled errors. + +## Runtime Trait Retention + +The native generator retains traits consumed by runtime packages and generated +SDK metadata. Core generation and integrations register the trait IDs they +need. + +The retained set includes traits used by: + +* Generic schema and validation behavior. +* Serialization and document codecs. +* HTTP binding and protocol framing. +* Authentication, endpoints, checksums, and retries. +* Generated SDK metadata. + +Plugins can add retained trait IDs. Generated docstrings contain documentation +traits, while runtime schemas omit them. Trait definitions and test traits are +excluded from runtime schemas unless an integration registers them. + +Trait filtering reduces generated source size, import parsing, schema +construction, and retained memory. The native and Java generators must expose +the same runtime traits before a client migrates. + +## Compatibility + +The native generator preserves: + +* Public package, model, client, and config module paths. +* Generated model names and member names. +* Keyword-only structure constructors. +* Modeled defaults and nullability. +* Enum and union behavior. +* Modeled error inheritance. +* `serialize()`, `serialize_members()`, `deserialize()`, and + `deserialize_kwargs()`. +* Public operation constants through the models facade. +* Canonical schema identity within one generated package. + +Private partition paths and helper names are implementation details. + +Generated source need not match Java-generated source. Protocol behavior, +public Python behavior, and type-checker behavior define parity. + +## Testing + +### Generator Tests + +Generator unit tests cover: + +* Symbol and member naming. +* Service renames and collisions. +* Partition boundaries. +* Strongly connected component assignment. +* Trait retention. +* Plugin ordering and code sections. +* Stale-file removal. + +Golden tests inspect generated source for lazy imports, dense member access, +and stable public exports. + +### Generated Package Tests + +Each generated fixture must: + +* Compile with the supported Python versions. +* Pass Ruff formatting and lint checks. +* Pass strict Pyright checks. +* Import through every public module. +* Serialize and deserialize recursive shapes. +* Preserve schema identity across model, operation, and registry access. +* Initialize recursive schemas safely from concurrent threads. + +Import tests instrument `Schema` construction and assert the lazy-loading +gates from this design. + +### Protocol Tests + +The existing Smithy protocol tests run against native-generated clients. The +Java generator continues to produce its snapshots during migration so +behavioral differences remain visible. + +Each protocol moves only after its required request, response, error, streaming, +and event stream tests pass. + +## Rollout + +Implementation proceeds in these stages: + +1. Port generation context, symbols, writers, plugins, and package setup onto + the current semantic model. +2. Generate standalone types with partitioned models and lazy schemas. +3. Add dense generated serialization and deserialization. +4. Generate Rest JSON clients and protocol tests. +5. Add AWS platform behavior required by the Rest JSON test services. +6. Add Rest XML, RPC v2 CBOR, AWS JSON, AWS Query, and EC2 Query support. +7. Compare each protocol against the Java generator and migrate projections + independently. + +Each stage includes its benchmark results. Performance regressions remain +local to the stage that introduced them. + +## Tradeoffs + +Lazy construction moves schema work from import time to the first operation +that reaches a shape. Applications that invoke an operation immediately still +pay the construction cost, but applications importing a large client for a +small operation avoid unrelated schema work. + +Partitioning adds private modules and lazy export machinery. It reduces Python +source compiled during common imports and keeps the public module layout +stable. + +Dense indexes depend on generated member order. Regeneration may assign a +different index after a model change. Indexes remain private to one generated +artifact and are never serialized or exposed as a compatibility contract. + +The component lock adds synchronization to a schema cache miss. Cache hits do +not acquire the lock. + +Callable deserialization state allocates one object for each aggregate value. +It avoids the closure and dictionary allocations in the proof of concept while +preserving the public deserializer interface. + +## Alternatives + +### Rebase the Proof of Concept + +The proof of concept predates the current semantic model, selection rules, and +CLI behavior. Rebasing it would replace tested model code and mix generator +work with model regressions. Porting generator components keeps those concerns +separate. + +### Eager Monolithic Schemas + +One schema module is easy to generate, but importing any model constructs the +entire reachable schema graph. Large services pay that cost before selecting +an operation. + +### One File per Shape + +One file per shape maximizes import granularity but creates thousands of files +for large services. Partitioning captures most import savings with lower +filesystem and wheel overhead. + +### Lazy Schema Proxies + +A proxy can defer schema construction without changing generated call sites. +It changes runtime type and identity behavior and adds resolution to schema +attribute access. Concrete schemas with lazy accessors keep the hot path +direct. + +### Class Methods for Schema Access + +A class method receives the concrete Python subclass and can create a separate +cache entry for it. Smithy schema identity belongs to the modeled shape, so a +static method provides the required ownership. + +### Dictionary-Based Deserialization + +Collecting member values in a dictionary preserves keyword construction with +little generated code. It hashes a string for each decoded member and expands +the dictionary during construction. Generated index dispatch and slotted state +avoid that work. + +### Slotted Public Models + +Slotted dataclasses reduce instance memory and can speed attribute access. They +remove `__dict__` and restrict subclass behavior, which changes observable +Python behavior. A later design can adopt slots after compatibility review and +benchmarks. + +## Future Work + +* Add a state-parameter deserializer API if it outperforms callable state across + JSON, XML, CBOR, and HTTP binding paths. +* Precompute validation masks and constraint metadata used during every + structure decode. +* Generate endpoint rule bytecode from normalized endpoint rules. +* Support incremental generation from a prior manifest. +* Evaluate partition thresholds on large AWS service models. +* Evaluate slotted generated models as a new artifact compatibility mode. diff --git a/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json new file mode 100644 index 000000000..de1bae3a8 --- /dev/null +++ b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added configurable eager JSON deserialization to improve buffered payload performance. Generated JSON clients automatically parse `bytes` eagerly while preserving incremental parsing for streaming readers, and can explicitly select eager or streaming behavior." +} diff --git a/packages/smithy-json/README.md b/packages/smithy-json/README.md index 29b5448d6..ecabd8054 100644 --- a/packages/smithy-json/README.md +++ b/packages/smithy-json/README.md @@ -2,3 +2,22 @@ This package provides generic JSON serialization and deserialization support for Smithy clients and servers. + +## Deserialization modes + +`JSONCodec` supports eager and streaming deserialization. The default `AUTO` +mode eagerly parses byte payloads and incrementally parses `BytesReader` +payloads: + +```python +from smithy_json import JSONCodec, JSONDeserializationMode + +codec = JSONCodec( + deserialization_mode=JSONDeserializationMode.STREAMING, +) +``` + +Eager mode is faster for buffered payloads but temporarily materializes the +complete JSON value tree. Streaming mode avoids that additional allocation. +Generated clients using a JSON protocol expose the same choice through +`json_deserialization_mode` on their configuration. diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index 7d20bba87..4867f6577 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -11,10 +11,18 @@ from ._private.deserializers import JSONShapeDeserializer as _JSONShapeDeserializer from ._private.documents import JSONDocument from ._private.serializers import JSONShapeSerializer as _JSONShapeSerializer -from .settings import JSONSettings +from ._private.value_deserializer import ( + JSONValueDeserializer as _JSONValueDeserializer, +) +from .settings import JSONDeserializationMode, JSONSettings __version__ = "0.3.0" -__all__ = ("JSONCodec", "JSONDocument", "JSONSettings") +__all__ = ( + "JSONCodec", + "JSONDeserializationMode", + "JSONDocument", + "JSONSettings", +) class JSONCodec(Codec): @@ -27,6 +35,7 @@ def __init__( default_timestamp_format: TimestampFormat = TimestampFormat.DATE_TIME, default_namespace: str | None = None, document_class: type[JSONDocument] = JSONDocument, + deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO, ) -> None: """Initializes a JSONCodec. @@ -39,6 +48,8 @@ def __init__( :param default_namespace: The default namespace to use when determining a document's discriminator. :param document_class: The document class to deserialize to. + :param deserialization_mode: Controls whether JSON payloads are parsed eagerly, + incrementally, or selected automatically based on the source type. """ self._settings = JSONSettings( use_json_name=use_json_name, @@ -46,6 +57,7 @@ def __init__( default_timestamp_format=default_timestamp_format, default_namespace=default_namespace, document_class=document_class, + deserialization_mode=deserialization_mode, ) @property @@ -56,6 +68,22 @@ def create_serializer(self, sink: BytesWriter) -> "ShapeSerializer": return _JSONShapeSerializer(sink, settings=self._settings) def create_deserializer(self, source: bytes | BytesReader) -> "ShapeDeserializer": + mode = self._settings.deserialization_mode + if mode is JSONDeserializationMode.EAGER or ( + mode is JSONDeserializationMode.AUTO and isinstance(source, bytes) + ): + return _JSONValueDeserializer(source, settings=self._settings) + if isinstance(source, bytes): source = BytesIO(source) return _JSONShapeDeserializer(source, settings=self._settings) + + @property + def deserialization_mode(self) -> JSONDeserializationMode: + """The strategy used to deserialize JSON payloads.""" + + return self._settings.deserialization_mode + + @deserialization_mode.setter + def deserialization_mode(self, value: JSONDeserializationMode) -> None: + self._settings.deserialization_mode = value diff --git a/packages/smithy-json/src/smithy_json/_private/value_deserializer.py b/packages/smithy-json/src/smithy_json/_private/value_deserializer.py new file mode 100644 index 000000000..f5641a117 --- /dev/null +++ b/packages/smithy-json/src/smithy_json/_private/value_deserializer.py @@ -0,0 +1,205 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import datetime +import json +from base64 import b64decode +from collections.abc import Callable, Mapping, Sequence +from decimal import Decimal +from json import JSONDecodeError + +from ijson.common import IncompleteJSONError # type: ignore +from smithy_core.deserializers import ShapeDeserializer +from smithy_core.documents import Document +from smithy_core.exceptions import SmithyError +from smithy_core.interfaces import BytesReader +from smithy_core.schemas import Schema +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import JSONNameTrait, TimestampFormatTrait +from smithy_core.types import TimestampFormat + +from ..settings import JSONSettings + +type JSONValue = ( + Mapping[str, "JSONValue"] + | Sequence["JSONValue"] + | str + | int + | Decimal + | bool + | None +) + + +class JSONValueError(SmithyError): + """Raised when a parsed JSON value does not match the requested Smithy type.""" + + def __init__(self, expected: str, value: JSONValue) -> None: + super().__init__( + f"Error parsing JSON. Expected value of type `{expected}`, " + f"but found: `{type(value).__name__}`: {value}" + ) + + +def _reject_constant(value: str) -> None: + raise ValueError(f"Invalid JSON constant: {value}") + + +class JSONValueDeserializer(ShapeDeserializer): + """Deserializes Smithy shapes from a fully materialized JSON value.""" + + def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: + try: + if isinstance(source, bytes): + self._value: JSONValue = json.loads( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + else: + self._value = json.load( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + except (JSONDecodeError, UnicodeDecodeError, ValueError) as error: + raise IncompleteJSONError(f"parse error: {error}") from error + + self._settings = settings + self._json_names: dict[ShapeID, dict[str, Schema]] = {} + + def is_null(self) -> bool: + return self._value is None + + def read_null(self) -> None: + if self._value is not None: + raise JSONValueError("null", self._value) + + def read_boolean(self, schema: Schema) -> bool: + if not isinstance(self._value, bool): + raise JSONValueError("boolean", self._value) + return self._value + + def read_blob(self, schema: Schema) -> bytes: + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return b64decode(self._value) + + def read_integer(self, schema: Schema) -> int: + if not isinstance(self._value, int) or isinstance(self._value, bool): + raise JSONValueError("number", self._value) + return self._value + + def read_float(self, schema: Schema) -> float: + match self._value: + case Decimal() | "Infinity" | "-Infinity" | "NaN": + return float(self._value) + case int() if not isinstance(self._value, bool): + return self._value + case _: + raise JSONValueError("number", self._value) + + def read_big_decimal(self, schema: Schema) -> Decimal: + match self._value: + case Decimal(): + return self._value + case int() if not isinstance(self._value, bool): + return Decimal(self._value) + case "Infinity" | "-Infinity" | "NaN": + return Decimal(self._value) + case _: + raise JSONValueError("number", self._value) + + def read_string(self, schema: Schema) -> str: + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return self._value + + def read_document(self, schema: Schema) -> Document: + return self._settings.document_class( + value=self._value, + schema=schema, + settings=self._settings, + ) + + def read_timestamp(self, schema: Schema) -> datetime.datetime: + format = self._settings.default_timestamp_format + if self._settings.use_timestamp_format: + if format_trait := schema.get_trait(TimestampFormatTrait): + format = format_trait.format + + match format: + case TimestampFormat.EPOCH_SECONDS: + return format.deserialize(self.read_float(schema=schema)) + case _: + return format.deserialize(self.read_string(schema=schema)) + + def read_struct( + self, + schema: Schema, + consumer: Callable[[Schema, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + struct_schema = schema.member_target or schema + members = self._members_by_wire_name(struct_schema) + try: + for key, member_value in value.items(): + member = members.get(key) + if member is None: + continue + if member_value is None and member.shape_type is not ShapeType.DOCUMENT: + continue + self._value = member_value + consumer(member, self) + finally: + self._value = value + + def read_list( + self, + schema: Schema, + consumer: Callable[[ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Sequence) or isinstance(value, str): + raise JSONValueError("array", value) + + try: + for member_value in value: + self._value = member_value + consumer(self) + finally: + self._value = value + + def read_map( + self, + schema: Schema, + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + try: + for key, member_value in value.items(): + self._value = member_value + consumer(key, self) + finally: + self._value = value + + def _members_by_wire_name(self, schema: Schema) -> Mapping[str, Schema]: + if not self._settings.use_json_name: + return schema.members + + members = self._json_names.get(schema.id) + if members is None: + members = {} + for member_name, member in schema.members.items(): + json_name = member.get_trait(JSONNameTrait) + members[json_name.value if json_name is not None else member_name] = ( + member + ) + self._json_names[schema.id] = members + return members diff --git a/packages/smithy-json/src/smithy_json/plugins.py b/packages/smithy-json/src/smithy_json/plugins.py new file mode 100644 index 000000000..7423bb489 --- /dev/null +++ b/packages/smithy-json/src/smithy_json/plugins.py @@ -0,0 +1,25 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Protocol + +from . import JSONCodec +from .settings import JSONDeserializationMode + + +class _JSONDeserializationConfig(Protocol): + @property + def protocol(self) -> object | None: ... + + @property + def json_deserialization_mode(self) -> JSONDeserializationMode | None: ... + + +def json_deserialization_plugin(config: _JSONDeserializationConfig) -> None: + """Configure JSON deserialization on a generated client's protocol codec.""" + + codec = getattr(config.protocol, "payload_codec", None) + if isinstance(codec, JSONCodec): + codec.deserialization_mode = ( + config.json_deserialization_mode or JSONDeserializationMode.AUTO + ) diff --git a/packages/smithy-json/src/smithy_json/settings.py b/packages/smithy-json/src/smithy_json/settings.py index 6ebde62cf..3b0253705 100644 --- a/packages/smithy-json/src/smithy_json/settings.py +++ b/packages/smithy-json/src/smithy_json/settings.py @@ -1,6 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass +from enum import StrEnum from typing import TYPE_CHECKING from smithy_core.types import TimestampFormat @@ -9,6 +10,19 @@ from ._private.documents import JSONDocument +class JSONDeserializationMode(StrEnum): + """Controls how JSON payloads are parsed during deserialization.""" + + AUTO = "auto" + """Eagerly parse bytes while preserving incremental reader consumption.""" + + EAGER = "eager" + """Consume and materialize the complete JSON value before deserializing.""" + + STREAMING = "streaming" + """Incrementally deserialize JSON using the streaming parser.""" + + @dataclass(slots=True) class JSONSettings: """Settings for the JSON codec.""" @@ -29,3 +43,6 @@ class JSONSettings: default_namespace: str | None = None """The default namespace to use when determining a document's discriminator.""" + + deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO + """Controls whether JSON deserialization is eager, streaming, or automatic.""" diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index 00a6ed2cd..0402f9c6b 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -3,9 +3,11 @@ import math from datetime import datetime from decimal import Decimal +from io import BytesIO from typing import Any import pytest +from ijson.common import IncompleteJSONError # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.prelude import ( @@ -18,7 +20,9 @@ STRING, TIMESTAMP, ) -from smithy_json import JSONCodec, JSONDocument +from smithy_json import JSONCodec, JSONDeserializationMode, JSONDocument +from smithy_json._private.deserializers import JSONShapeDeserializer +from smithy_json._private.value_deserializer import JSONValueDeserializer from . import ( JSON_SERDE_CASES, @@ -28,9 +32,17 @@ ) +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) @pytest.mark.parametrize("expected, given", JSON_SERDE_CASES) -def test_json_deserializer(expected: Any, given: bytes) -> None: - codec = JSONCodec() +def test_json_deserializer( + expected: Any, + given: bytes, + mode: JSONDeserializationMode, +) -> None: + codec = JSONCodec(deserialization_mode=mode) deserializer = codec.create_deserializer(given) match expected: case None: @@ -108,3 +120,46 @@ def test_uses_custom_document() -> None: codec = JSONCodec(document_class=CustomDocument) actual = codec.create_deserializer(b'{"foo": "bar"}').read_document(DOCUMENT) assert isinstance(actual, CustomDocument) + + +@pytest.mark.parametrize( + "mode, source, expected_type", + [ + (JSONDeserializationMode.AUTO, b"{}", JSONValueDeserializer), + (JSONDeserializationMode.AUTO, BytesIO(b"{}"), JSONShapeDeserializer), + (JSONDeserializationMode.EAGER, BytesIO(b"{}"), JSONValueDeserializer), + (JSONDeserializationMode.STREAMING, b"{}", JSONShapeDeserializer), + ], +) +def test_deserialization_mode_selects_parser( + mode: JSONDeserializationMode, + source: bytes | BytesIO, + expected_type: type[ShapeDeserializer], +) -> None: + deserializer = JSONCodec(deserialization_mode=mode).create_deserializer(source) + assert isinstance(deserializer, expected_type) + + +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) +def test_ignores_unknown_structure_members(mode: JSONDeserializationMode) -> None: + actual = JSONCodec(deserialization_mode=mode).deserialize( + b'{"unknown":{"nested":[1,2,3]},"stringMember":"value"}', + SerdeShape, + ) + + assert actual.string_member == "value" + + +@pytest.mark.parametrize( + "mode", + [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], +) +def test_invalid_json_uses_existing_error_type(mode: JSONDeserializationMode) -> None: + with pytest.raises(IncompleteJSONError): + deserializer = JSONCodec(deserialization_mode=mode).create_deserializer( + b'{"incomplete":' + ) + deserializer.read_document(DOCUMENT) diff --git a/packages/smithy-json/tests/unit/test_plugins.py b/packages/smithy-json/tests/unit/test_plugins.py new file mode 100644 index 000000000..b16618c0d --- /dev/null +++ b/packages/smithy-json/tests/unit/test_plugins.py @@ -0,0 +1,52 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from dataclasses import dataclass + +import pytest +from smithy_json import JSONCodec, JSONDeserializationMode +from smithy_json.plugins import json_deserialization_plugin + + +@dataclass +class _Protocol: + payload_codec: object + + +@dataclass +class _Config: + protocol: object + json_deserialization_mode: JSONDeserializationMode | None + + +@pytest.mark.parametrize( + "configured, expected", + [ + (None, JSONDeserializationMode.AUTO), + (JSONDeserializationMode.AUTO, JSONDeserializationMode.AUTO), + (JSONDeserializationMode.EAGER, JSONDeserializationMode.EAGER), + (JSONDeserializationMode.STREAMING, JSONDeserializationMode.STREAMING), + ], +) +def test_configures_json_codec( + configured: JSONDeserializationMode | None, + expected: JSONDeserializationMode, +) -> None: + codec = JSONCodec(deserialization_mode=JSONDeserializationMode.EAGER) + config = _Config( + protocol=_Protocol(payload_codec=codec), + json_deserialization_mode=configured, + ) + + json_deserialization_plugin(config) + + assert codec.deserialization_mode is expected + + +@pytest.mark.parametrize("protocol", [object(), _Protocol(payload_codec=object())]) +def test_ignores_protocols_without_json_codec(protocol: object) -> None: + config = _Config( + protocol=protocol, + json_deserialization_mode=JSONDeserializationMode.STREAMING, + ) + json_deserialization_plugin(config) From d510b7ef7b814fe6b0ccf436aa99b30065d8e32d Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:06:05 -0400 Subject: [PATCH 2/4] remove modes - always eager --- codegen-output/aws-json-1-0/pyproject.toml | 3 +- .../aws-json-1-0/src/awsjson10/client.py | 7 +- .../aws-json-1-0/src/awsjson10/config.py | 9 - codegen-output/aws-json-1-1/pyproject.toml | 3 +- .../aws-json-1-1/src/awsjson11/client.py | 7 +- .../aws-json-1-1/src/awsjson11/config.py | 9 - codegen-output/rest-json-1/pyproject.toml | 3 +- .../rest-json-1/src/restjson/client.py | 7 +- .../rest-json-1/src/restjson/config.py | 9 - .../python/codegen/test/AwsCodegenTest.java | 7 - .../codegen/test/PythonCodegenTest.java | 8 - .../JsonDeserializationIntegration.java | 69 -- ...hon.codegen.integrations.PythonIntegration | 1 - designs/codegen/generated-package.md | 672 ------------------ ...ture-4b7f9ca09557405eb9382c56490c6d7a.json | 2 +- packages/smithy-json/README.md | 21 +- .../smithy-json/src/smithy_json/__init__.py | 36 +- .../smithy-json/src/smithy_json/plugins.py | 25 - .../smithy-json/src/smithy_json/settings.py | 17 - .../tests/unit/test_deserializers.py | 51 +- .../smithy-json/tests/unit/test_plugins.py | 52 -- 21 files changed, 27 insertions(+), 991 deletions(-) delete mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java delete mode 100644 designs/codegen/generated-package.md delete mode 100644 packages/smithy-json/src/smithy_json/plugins.py delete mode 100644 packages/smithy-json/tests/unit/test_plugins.py diff --git a/codegen-output/aws-json-1-0/pyproject.toml b/codegen-output/aws-json-1-0/pyproject.toml index b05b8c87e..de6d7aa27 100644 --- a/codegen-output/aws-json-1-0/pyproject.toml +++ b/codegen-output/aws-json-1-0/pyproject.toml @@ -26,8 +26,7 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0", - "smithy_json~=0.3.0" + "smithy_http[aiohttp]~=0.6.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-0/src/awsjson10/client.py b/codegen-output/aws-json-1-0/src/awsjson10/client.py index c6dcee27b..5f9cb2388 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/client.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/client.py @@ -14,7 +14,6 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin -from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJSONRPC10Config, Plugin from .models import ( @@ -95,11 +94,7 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [ - aws_user_agent_plugin, - json_deserialization_plugin, - user_agent_plugin, - ] + self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] async def _ensure_setup(self) -> AsyncJSONRPC10Config: if not self._setup_done: diff --git a/codegen-output/aws-json-1-0/src/awsjson10/config.py b/codegen-output/aws-json-1-0/src/awsjson10/config.py index b66993b2f..1696a93d0 100644 --- a/codegen-output/aws-json-1-0/src/awsjson10/config.py +++ b/codegen-output/aws-json-1-0/src/awsjson10/config.py @@ -25,7 +25,6 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient -from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -112,7 +111,6 @@ class _AsyncJSONRPC10ConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None - json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -176,12 +174,6 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): header. """ - json_deserialization_mode: JSONDeserializationMode | None = None - """ - Controls whether JSON response payloads are deserialized eagerly or - incrementally. - """ - _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -190,7 +182,6 @@ class AsyncJSONRPC10Config(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), - "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_rpc_10") diff --git a/codegen-output/aws-json-1-1/pyproject.toml b/codegen-output/aws-json-1-1/pyproject.toml index 150cfed84..a9b718abf 100644 --- a/codegen-output/aws-json-1-1/pyproject.toml +++ b/codegen-output/aws-json-1-1/pyproject.toml @@ -26,8 +26,7 @@ classifiers = [ dependencies = [ "smithy_aws_core[json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0", - "smithy_json~=0.3.0" + "smithy_http[aiohttp]~=0.6.0" ] [project.optional-dependencies] diff --git a/codegen-output/aws-json-1-1/src/awsjson11/client.py b/codegen-output/aws-json-1-1/src/awsjson11/client.py index cdc16a79d..897d043d3 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/client.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/client.py @@ -14,7 +14,6 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin -from smithy_json.plugins import json_deserialization_plugin from .config import AsyncJsonProtocolConfig, Plugin from .models import ( @@ -101,11 +100,7 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [ - aws_user_agent_plugin, - json_deserialization_plugin, - user_agent_plugin, - ] + self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] async def _ensure_setup(self) -> AsyncJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/aws-json-1-1/src/awsjson11/config.py b/codegen-output/aws-json-1-1/src/awsjson11/config.py index 83d1fbe6a..bbbeb3e1e 100644 --- a/codegen-output/aws-json-1-1/src/awsjson11/config.py +++ b/codegen-output/aws-json-1-1/src/awsjson11/config.py @@ -25,7 +25,6 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient -from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -112,7 +111,6 @@ class _AsyncJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None - json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -176,12 +174,6 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): header. """ - json_deserialization_mode: JSONDeserializationMode | None = None - """ - Controls whether JSON response payloads are deserialized eagerly or - incrementally. - """ - _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -190,7 +182,6 @@ class AsyncJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), - "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("json_protocol") diff --git a/codegen-output/rest-json-1/pyproject.toml b/codegen-output/rest-json-1/pyproject.toml index 9d63abd61..953379f29 100644 --- a/codegen-output/rest-json-1/pyproject.toml +++ b/codegen-output/rest-json-1/pyproject.toml @@ -26,8 +26,7 @@ classifiers = [ dependencies = [ "smithy_aws_core[eventstream, json]~=0.12.0", "smithy_core~=0.9.0", - "smithy_http[aiohttp]~=0.6.0", - "smithy_json~=0.3.0" + "smithy_http[aiohttp]~=0.6.0" ] [project.optional-dependencies] diff --git a/codegen-output/rest-json-1/src/restjson/client.py b/codegen-output/rest-json-1/src/restjson/client.py index 781c861dd..5c0a36df8 100644 --- a/codegen-output/rest-json-1/src/restjson/client.py +++ b/codegen-output/rest-json-1/src/restjson/client.py @@ -19,7 +19,6 @@ from smithy_core.serializers import SerializeableShape from smithy_http.aio.interfaces import HTTPClient from smithy_http.plugins import user_agent_plugin -from smithy_json.plugins import json_deserialization_plugin from .config import AsyncRestJsonProtocolConfig, Plugin from .models import ( @@ -393,11 +392,7 @@ def __init__( super().__init__() self._config = config self._plugins = plugins - self._client_plugins: list[Plugin] = [ - aws_user_agent_plugin, - json_deserialization_plugin, - user_agent_plugin, - ] + self._client_plugins: list[Plugin] = [aws_user_agent_plugin, user_agent_plugin] async def _ensure_setup(self) -> AsyncRestJsonProtocolConfig: if not self._setup_done: diff --git a/codegen-output/rest-json-1/src/restjson/config.py b/codegen-output/rest-json-1/src/restjson/config.py index 2894a5685..68c6a03da 100644 --- a/codegen-output/rest-json-1/src/restjson/config.py +++ b/codegen-output/rest-json-1/src/restjson/config.py @@ -25,7 +25,6 @@ from smithy_core.interfaces.auth import AuthSchemeResolver from smithy_core.shapes import ShapeID from smithy_http.aio.aiohttp import AIOHTTPClient -from smithy_json import JSONDeserializationMode from .auth import HTTPAuthSchemeResolver from .models import ( @@ -557,7 +556,6 @@ class _AsyncRestJsonProtocolConfigOverrides(AwsConfigOverrides, total=False): ) auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None auth_scheme_resolver: AuthSchemeResolver | None - json_deserialization_mode: JSONDeserializationMode | None @dataclass(kw_only=True, repr=False, init=False) @@ -621,12 +619,6 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): header. """ - json_deserialization_mode: JSONDeserializationMode | None = None - """ - Controls whether JSON response payloads are deserialized eagerly or - incrementally. - """ - _FIELDS: ClassVar[dict[str, FieldSpec]] = { "aws_credentials_identity_resolver": FieldSpec(default=None), "region": FieldSpec(default=None), @@ -635,7 +627,6 @@ class AsyncRestJsonProtocolConfig(AsyncAwsConfig): "aws_session_token": FieldSpec(default=None), "user_agent_extra": FieldSpec(default=None), "sdk_ua_app_id": FieldSpec(default=None), - "json_deserialization_mode": FieldSpec(default=None), **AsyncAwsConfig._FIELDS, "endpoint_uri": FieldSpec( default=None, resolver=EndpointUriResolver("rest_json_protocol") diff --git a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java index 1574c3ab4..b5519b0d2 100644 --- a/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java +++ b/codegen/aws/core/src/it/java/software/amazon/smithy/python/codegen/test/AwsCodegenTest.java @@ -56,10 +56,6 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertTrue(config.contains("self.auth_schemes = auth_schemes")); assertTrue(config.contains("default_factory=lambda: AIOHTTPClient()")); assertFalse(config.contains("from smithy_http.aio.crt import AWSCRTHTTPClient")); - assertTrue(config.contains( - "json_deserialization_mode: JSONDeserializationMode | None")); - assertTrue(config.contains( - "\"json_deserialization_mode\": FieldSpec(default=None)")); var client = Files.readString(tempDir.resolve("src/restjson/client.py")); assertInOrder( @@ -77,9 +73,6 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { assertFalse(client.contains("plugin(self._config)")); assertTrue(client.contains("retry_mode=config.retry_mode")); assertTrue(client.contains("max_attempts=config.max_attempts")); - assertTrue(client.contains( - "from smithy_json.plugins import json_deserialization_plugin")); - assertTrue(client.contains("json_deserialization_plugin,")); var schemas = Files.readString(tempDir.resolve("src/restjson/_private/schemas.py")); assertTrue(schemas.contains("aws.protocols#restJson1")); diff --git a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java index c64ed5be0..8d5a84bfa 100644 --- a/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java +++ b/codegen/core/src/it/java/software/amazon/smithy/python/codegen/test/PythonCodegenTest.java @@ -53,14 +53,6 @@ public void testCodegen(@TempDir Path tempDir) throws IOException { var config = Files.readString(tempDir.resolve("src/weather/config.py")); assertTrue(config.contains("self.transport = transport or AIOHTTPClient()")); assertFalse(config.contains("self.transport = transport or AWSCRTHTTPClient()")); - assertTrue(config.contains( - "json_deserialization_mode: JSONDeserializationMode | None")); - assertTrue(config.contains( - "self.json_deserialization_mode = json_deserialization_mode")); - - assertTrue(client.contains( - "from smithy_json.plugins import json_deserialization_plugin")); - assertTrue(client.contains("json_deserialization_plugin,")); // An output that models a "responseMetadata" member gets both the SDK-reserved // response_metadata attribute and the escaped modeled member, so neither shadows diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java deleted file mode 100644 index 5efbe1dc9..000000000 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/JsonDeserializationIntegration.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0 - */ -package software.amazon.smithy.python.codegen.integrations; - -import java.util.List; -import java.util.Set; -import software.amazon.smithy.aws.traits.protocols.AwsJson1_0Trait; -import software.amazon.smithy.aws.traits.protocols.AwsJson1_1Trait; -import software.amazon.smithy.aws.traits.protocols.RestJson1Trait; -import software.amazon.smithy.codegen.core.Symbol; -import software.amazon.smithy.codegen.core.SymbolReference; -import software.amazon.smithy.model.Model; -import software.amazon.smithy.model.knowledge.ServiceIndex; -import software.amazon.smithy.model.shapes.ServiceShape; -import software.amazon.smithy.model.shapes.ShapeId; -import software.amazon.smithy.python.codegen.ConfigProperty; -import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.SmithyPythonDependency; -import software.amazon.smithy.utils.SmithyInternalApi; - -/** - * Adds configurable JSON deserialization to clients that support a JSON protocol. - */ -@SmithyInternalApi -public final class JsonDeserializationIntegration implements PythonIntegration { - private static final Set JSON_PROTOCOLS = Set.of( - RestJson1Trait.ID, - AwsJson1_0Trait.ID, - AwsJson1_1Trait.ID); - - @Override - public List getClientPlugins(GenerationContext context) { - var mode = Symbol.builder() - .name("JSONDeserializationMode") - .namespace("smithy_json", ".") - .addDependency(SmithyPythonDependency.SMITHY_JSON) - .build(); - var plugin = SymbolReference.builder() - .symbol(Symbol.builder() - .name("json_deserialization_plugin") - .namespace("smithy_json.plugins", ".") - .addDependency(SmithyPythonDependency.SMITHY_JSON) - .build()) - .build(); - var property = ConfigProperty.builder() - .name("json_deserialization_mode") - .type(mode) - .documentation( - "Controls whether JSON response payloads are deserialized eagerly or incrementally.") - .nullable(true) - .build(); - - return List.of(RuntimeClientPlugin.builder() - .servicePredicate(JsonDeserializationIntegration::usesJsonProtocol) - .addConfigProperty(property) - .pythonPlugin(plugin) - .build()); - } - - static boolean usesJsonProtocol(Model model, ServiceShape service) { - return ServiceIndex.of(model) - .getProtocols(service) - .keySet() - .stream() - .anyMatch(JSON_PROTOCOLS::contains); - } -} diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index f85f625be..ccdf63695 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -4,7 +4,6 @@ # software.amazon.smithy.python.codegen.integrations.RestJsonIntegration -software.amazon.smithy.python.codegen.integrations.JsonDeserializationIntegration software.amazon.smithy.python.codegen.integrations.Rpcv2CborIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth software.amazon.smithy.python.codegen.integrations.UserAgentIntegration diff --git a/designs/codegen/generated-package.md b/designs/codegen/generated-package.md deleted file mode 100644 index 44c90aa6d..000000000 --- a/designs/codegen/generated-package.md +++ /dev/null @@ -1,672 +0,0 @@ -# Python-Native Generated Package Architecture - -The Python generator emits clients and types packages that use the existing -Smithy Python runtime. Generated packages preserve the public behavior of the -Java generator while loading schemas and model partitions on demand. - -## Goals - -* Generate clients and standalone types packages from the semantic model in - `smithy_python.model`. -* Preserve supported generated package APIs during migration from the Java - generator. -* Avoid constructing user-defined schemas while importing a generated client. -* Use dense member indexes in generated serialization and deserialization. -* Preserve one runtime `Schema` instance for each generated shape. -* Support recursive shape graphs and free-threaded Python. -* Let protocol and platform integrations add generation behavior without - replacing the core generator. -* Measure generation time, import cost, first-use cost, memory, and warm serde - throughout development. - -## Scope - -The first implementation supports standalone types and Rest JSON clients. -Later changes add the remaining protocols after their runtime behavior passes -the existing protocol tests. - -The Java generator remains available during migration. A projection selects -which generator to run, so generated packages can move independently after -their required features reach parity. - -Server generation, custom model validation, and runtime compilation of serde -functions remain future work. Generated dataclasses retain their current -non-slotted layout until compatibility tests and benchmarks support changing -it. - -## Performance Gates - -The benchmark suite records these measurements for equivalent Java-generated -and Python-generated packages: - -* Code generation wall time and peak resident memory. -* Generated source size and compiled bytecode size. -* Package, models, config, and client import time. -* Client construction time. -* First serialization and deserialization time. -* Warm serialization and deserialization time. -* Peak allocations for first use and warm serde. - -Rest JSON provides the first comparison. Rest XML and RPC v2 CBOR join the -suite when the native generator supports them. - -Each measured serde case uses the same model, values, payload, warmup count, -and measured iteration count for both generators. Benchmark output includes -the interpreter version, platform, architecture, and generator commit. - -The implementation must satisfy these behavioral gates: - -* Importing the generated client constructs no user-defined shape schemas. -* Importing a model class constructs no schema until its `schema()` method or - an operation that uses it is accessed. -* Repeated calls to a shape's `schema()` method return the same object. -* Generated serde contains no string-keyed member schema lookup. -* Warm Rest JSON p50 may not regress by more than 5 percent against the Java - generator for the same benchmark case. A larger regression blocks migration - for that case. - -Cold-start measurements establish a baseline before the first client migrates. -The design uses behavioral gates for schema loading because host load and -filesystem state make a single import-time percentage unstable. - -## Generator Architecture - -The generator uses the existing model loader, service selection, CLI, and -plugin environment: - -```text -Smithy JSON AST - | - v -semantic model - | - v -shape selection - | - v -generation context - | - +------> core generators - | - +------> protocol and platform integrations - | - v -generated package -``` - -The proof of concept supplies initial implementations for symbols, writers, -generation context, plugins, models, schemas, clients, and protocol tests. -Those components are ported onto the current semantic model. The repository's -current model loader, shape selection, CLI, and environment handling remain -authoritative. - -### Generation Context - -One generation context owns: - -* The semantic model and selected service. -* Selected shapes in model order. -* Shape and member indexes. -* Symbols and generated names. -* Protocol and platform integrations. -* Dependency and generated-file manifests. -* Schema dependency components and source partitions. - -The context builds indexes once. Generators query those indexes instead of -rescanning every selected shape for each output file. - -### Symbols - -The symbol provider maps semantic shapes and members to Python names, module -paths, runtime dependencies, schema accessors, serializer functions, and -deserializer functions. - -Symbol results are cached by shape ID for the duration of one generation. -Member indexes are assigned once from the selected model order and reused by -schema generation and serde generation. - -Plugins may decorate symbols or supply symbols for integration-owned types. -The core provider remains responsible for collision handling and service -renames. - -### Integrations - -An integration declares the protocols and platform behavior it supports. It -can contribute: - -* Protocol construction. -* Runtime dependencies. -* Generated config fields and defaults. -* Client plugins. -* Traits retained in runtime schemas. -* Code sections in generated files. - -Integrations operate on the semantic model and generation context. Generated -packages depend on the Smithy runtime packages and exclude `smithy-python`. - -### Writers - -Writers collect imports, generated sections, and file contents in memory. The -file manifest writes changed files and removes stale generated files within -the output directory. - -Formatting and linting run after generation when requested. Tests exercise the -unformatted writer output as valid Python so formatting cannot hide syntax -errors. - -## Generated Package Layout - -Generated packages expose stable public modules and place partition details -under `_private`: - -```text -example_service/ - __init__.py - client.py - config.py - models.py - py.typed - _private/ - operations.py - models_0.py - models_1.py - schemas_0.py - schemas_1.py -``` - -Source partitions start with a target size of 128 KiB. The generator estimates -partition size from rendered declarations. The threshold is an implementation -setting, and benchmarks may change it. - -Private module names and partition assignments are unstable. Public model, -client, and config imports remain stable across regenerations. - -## Public Model Facade - -`models.py` exposes generated model classes, enums, unions, and operation -constants. Runtime imports resolve an exported name on first access: - -```python -from importlib import import_module -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from ._private.models_0 import GetObjectInput as GetObjectInput - from ._private.models_1 import GetObjectOutput as GetObjectOutput - -_EXPORTS = { - "GetObjectInput": ("._private.models_0", "GetObjectInput"), - "GetObjectOutput": ("._private.models_1", "GetObjectOutput"), -} - - -def __getattr__(name: str) -> Any: - export = _EXPORTS.get(name) - if export is None: - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - module_name, symbol_name = export - value = getattr(import_module(module_name, __package__), symbol_name) - globals()[name] = value - return value -``` - -Caching the resolved value in the module makes later access a normal global -lookup. `TYPE_CHECKING` imports expose concrete symbols to static type checkers -without importing model partitions at runtime. - -`__all__` lists the public generated names. `__dir__` combines those names with -the module globals so interactive discovery remains available before -resolution. - -## Model Partitions - -Model partitions contain dataclasses, enums, union variants, and their -generated serde helpers. The generator builds a dependency graph from runtime -model references and groups strongly connected components before assigning -partitions. - -Postponed annotations remove runtime imports needed only for type hints. -Cross-partition imports needed by serde occur through private accessors and -are cached after first use. - -Generated dataclasses keep keyword-only constructors, modeled defaults, -sensitive-member representation, and the existing serialization interfaces. -Modeled errors retain their service error hierarchy. - -## Lazy Schema Partitions - -Schema construction is lazy at the shape level. Each schema partition stores -completed schemas in a dense array indexed by constants generated for that -partition: - -```python -_SCHEMAS: list[Schema | None] = [None] * 24 -_LOCK = RLock() - -_GET_OBJECT_INPUT = 7 -``` - -The warm path performs one array lookup and one `None` check. - -### Dependency Components - -The generator builds a directed graph from each schema to its member targets. -It computes strongly connected components and treats each component as one -initialization unit. - -An acyclic shape forms a component of one. Mutually recursive shapes share a -component. Components form a directed acyclic graph, which gives schema -initialization a stable dependency order. - -A recursive component is never split across source partitions. - -### Component Initialization - -A component builder creates every root schema in the component before it -creates member schemas: - -```python -def _build_component_3() -> None: - node = Schema.collection( - id=ShapeID("example#Node"), - members={"value": None, "next": None}, - ) - - node.members["value"] = Schema.member( - id=node.id.with_member("value"), - target=STRING, - index=0, - ) - node.members["next"] = Schema.member( - id=node.id.with_member("next"), - target=node, - index=1, - ) - - _SCHEMAS[_NODE] = node -``` - -The public accessor uses a double-checked cache lookup: - -```python -def get_node_schema() -> Schema: - schema = _SCHEMAS[_NODE] - if schema is not None: - return schema - - with _LOCK: - schema = _SCHEMAS[_NODE] - if schema is None: - _build_component_3() - schema = _SCHEMAS[_NODE] - assert schema is not None - return schema -``` - -The component lock remains held until every schema in the component is -complete. Another thread sees an empty cache slot, waits for the lock, and -then reads the completed schema. The builder assigns cache slots only after -all roots and members in the component are complete. - -Cross-component dependencies resolve before the current component is -published. The component graph has no cycles, so initialization cannot recurse -back into an unpublished component. - -If construction raises an exception, the cache slots remain empty. A later -call can retry construction. - -### Shape Schema API - -Generated shapes expose a static `schema()` method: - -```python -_NODE_SCHEMA: Schema | None = None -_NODE_MEMBERS: tuple[Schema, ...] | None = None - - -def _node_schema() -> Schema: - global _NODE_SCHEMA, _NODE_MEMBERS - - if (schema := _NODE_SCHEMA) is None: - from .schemas_0 import get_node_schema - - schema = get_node_schema() - _NODE_MEMBERS = schema.members_by_index - _NODE_SCHEMA = schema - return schema - - -def _node_members() -> tuple[Schema, ...]: - members = _NODE_MEMBERS - if members is None: - _node_schema() - members = _NODE_MEMBERS - assert members is not None - return members - - -class Node: - @staticmethod - def schema() -> Schema: - return _node_schema() -``` - -The schema belongs to the modeled shape. It does not vary by Python subclass, -so the method uses `staticmethod`. A class method could create -subclass-specific caches and schema identities. - -The model partition caches the member tuple with the root schema. Generated -serde reads that tuple and does not query the name-keyed member map. - -### Schema Identity and Extensions - -Each shape accessor resolves to one canonical `Schema`. Operations, model -classes, type registries, protocols, and codecs share that object. - -Schema extensions remain lazy and schema-local. HTTP binding metadata, codec -metadata, and future extension values therefore survive across client and -protocol instances after first use. - -Generated code never wraps a schema in a lazy proxy. A proxy would change -schema identity and add resolver work to every attribute access. - -## Generated Serialization - -Structure serialization uses generated member access and dense schema indexes: - -```python -def serialize_members(self, serializer: ShapeSerializer) -> None: - members = _node_members() - serializer.write_string(members[0], self.value) - if self.next is not None: - serializer.write_struct(members[1], self.next) -``` - -`_node_members()` initializes the shape schema on its first call and returns -the cached tuple afterward. - -Generated serialization retains modeled nullability, sparse collection -behavior, timestamp formats, streaming members, and event stream handling. -Protocol-specific routing remains in runtime protocol packages. For example, -the HTTP binding serializer uses the schema's cached HTTP binding extension -while the generated structure writes members in modeled order. - -Collection serializers cache their `member`, `key`, and `value` schemas by -index. Union variants cache their variant member schema. - -## Generated Deserialization - -Structure deserialization uses a slotted callable object as mutable state: - -```python -class _NodeDeserializer: - __slots__ = ("value", "next") - - def __init__(self) -> None: - self.value: str | None = None - self.next: Node | None = None - - def __call__( - self, member: Schema, deserializer: ShapeDeserializer - ) -> None: - match member.expect_member_index(): - case 0: - self.value = deserializer.read_string(member) - case 1: - self.next = Node.deserialize(deserializer) - - def build(self) -> Node: - return Node(value=self.value, next=self.next) -``` - -`deserialize()` allocates one state object, passes it directly as the member -consumer, and calls `build()`: - -```python -@classmethod -def deserialize(cls, deserializer: ShapeDeserializer) -> Self: - state = _NodeDeserializer() - deserializer.read_struct(_node_schema(), state) - return state.build() -``` - -Member indexes select generated cases, and fixed attributes store decoded -values. The primary path allocates one state object for each structure. - -The state initializer applies modeled defaults. Required members use a private -missing sentinel and `build()` reports an absent required value. Mutable -defaults are created for each state object. - -`deserialize_kwargs()` remains available for compatibility. It uses the same -state object and creates a dictionary only when a caller invokes that method. -The primary `deserialize()` path constructs the model directly. - -Union deserializers use one result slot and reject multiple values. List and -map deserializers use slotted callable state objects that retain the target -collection and cached member schemas. - -The first implementation keeps the existing `ShapeDeserializer` interface. -A state-parameter overload may replace callable state objects after every -official codec supports it and benchmarks show a gain. - -## Operations and Client Imports - -Operations resolve on first use from a dense module cache: - -```python -_OPERATIONS: list[APIOperation[Any, Any] | None] = [None] * 18 -_OPERATION_LOCK = Lock() -_GET_OBJECT = 4 - - -def get_get_object_operation() -> APIOperation[Any, Any]: - operation = _OPERATIONS[_GET_OBJECT] - if operation is None: - with _OPERATION_LOCK: - operation = _OPERATIONS[_GET_OBJECT] - if operation is None: - operation = APIOperation( - input=GetObjectInput, - output=GetObjectOutput, - schema=get_get_object_schema(), - input_schema=GetObjectInput.schema(), - output_schema=GetObjectOutput.schema(), - error_registry=_get_get_object_error_registry(), - effective_auth_schemes=_GET_OBJECT_AUTH, - error_schemas=_get_get_object_error_schemas(), - ) - _OPERATIONS[_GET_OBJECT] = operation - return operation -``` - -The client imports model types under `TYPE_CHECKING` for annotations. Each -operation method imports its private operation getter and caches the resolved -operation through the operations module. - -Importing `client.py` loads client infrastructure and config types. It does not -load model or schema partitions. Calling one operation loads that operation's -input, output, errors, and reachable schema components. - -Error registries import modeled error classes when their operation initializes. -Receiving an unknown error does not require loading unrelated modeled errors. - -## Runtime Trait Retention - -The native generator retains traits consumed by runtime packages and generated -SDK metadata. Core generation and integrations register the trait IDs they -need. - -The retained set includes traits used by: - -* Generic schema and validation behavior. -* Serialization and document codecs. -* HTTP binding and protocol framing. -* Authentication, endpoints, checksums, and retries. -* Generated SDK metadata. - -Plugins can add retained trait IDs. Generated docstrings contain documentation -traits, while runtime schemas omit them. Trait definitions and test traits are -excluded from runtime schemas unless an integration registers them. - -Trait filtering reduces generated source size, import parsing, schema -construction, and retained memory. The native and Java generators must expose -the same runtime traits before a client migrates. - -## Compatibility - -The native generator preserves: - -* Public package, model, client, and config module paths. -* Generated model names and member names. -* Keyword-only structure constructors. -* Modeled defaults and nullability. -* Enum and union behavior. -* Modeled error inheritance. -* `serialize()`, `serialize_members()`, `deserialize()`, and - `deserialize_kwargs()`. -* Public operation constants through the models facade. -* Canonical schema identity within one generated package. - -Private partition paths and helper names are implementation details. - -Generated source need not match Java-generated source. Protocol behavior, -public Python behavior, and type-checker behavior define parity. - -## Testing - -### Generator Tests - -Generator unit tests cover: - -* Symbol and member naming. -* Service renames and collisions. -* Partition boundaries. -* Strongly connected component assignment. -* Trait retention. -* Plugin ordering and code sections. -* Stale-file removal. - -Golden tests inspect generated source for lazy imports, dense member access, -and stable public exports. - -### Generated Package Tests - -Each generated fixture must: - -* Compile with the supported Python versions. -* Pass Ruff formatting and lint checks. -* Pass strict Pyright checks. -* Import through every public module. -* Serialize and deserialize recursive shapes. -* Preserve schema identity across model, operation, and registry access. -* Initialize recursive schemas safely from concurrent threads. - -Import tests instrument `Schema` construction and assert the lazy-loading -gates from this design. - -### Protocol Tests - -The existing Smithy protocol tests run against native-generated clients. The -Java generator continues to produce its snapshots during migration so -behavioral differences remain visible. - -Each protocol moves only after its required request, response, error, streaming, -and event stream tests pass. - -## Rollout - -Implementation proceeds in these stages: - -1. Port generation context, symbols, writers, plugins, and package setup onto - the current semantic model. -2. Generate standalone types with partitioned models and lazy schemas. -3. Add dense generated serialization and deserialization. -4. Generate Rest JSON clients and protocol tests. -5. Add AWS platform behavior required by the Rest JSON test services. -6. Add Rest XML, RPC v2 CBOR, AWS JSON, AWS Query, and EC2 Query support. -7. Compare each protocol against the Java generator and migrate projections - independently. - -Each stage includes its benchmark results. Performance regressions remain -local to the stage that introduced them. - -## Tradeoffs - -Lazy construction moves schema work from import time to the first operation -that reaches a shape. Applications that invoke an operation immediately still -pay the construction cost, but applications importing a large client for a -small operation avoid unrelated schema work. - -Partitioning adds private modules and lazy export machinery. It reduces Python -source compiled during common imports and keeps the public module layout -stable. - -Dense indexes depend on generated member order. Regeneration may assign a -different index after a model change. Indexes remain private to one generated -artifact and are never serialized or exposed as a compatibility contract. - -The component lock adds synchronization to a schema cache miss. Cache hits do -not acquire the lock. - -Callable deserialization state allocates one object for each aggregate value. -It avoids the closure and dictionary allocations in the proof of concept while -preserving the public deserializer interface. - -## Alternatives - -### Rebase the Proof of Concept - -The proof of concept predates the current semantic model, selection rules, and -CLI behavior. Rebasing it would replace tested model code and mix generator -work with model regressions. Porting generator components keeps those concerns -separate. - -### Eager Monolithic Schemas - -One schema module is easy to generate, but importing any model constructs the -entire reachable schema graph. Large services pay that cost before selecting -an operation. - -### One File per Shape - -One file per shape maximizes import granularity but creates thousands of files -for large services. Partitioning captures most import savings with lower -filesystem and wheel overhead. - -### Lazy Schema Proxies - -A proxy can defer schema construction without changing generated call sites. -It changes runtime type and identity behavior and adds resolution to schema -attribute access. Concrete schemas with lazy accessors keep the hot path -direct. - -### Class Methods for Schema Access - -A class method receives the concrete Python subclass and can create a separate -cache entry for it. Smithy schema identity belongs to the modeled shape, so a -static method provides the required ownership. - -### Dictionary-Based Deserialization - -Collecting member values in a dictionary preserves keyword construction with -little generated code. It hashes a string for each decoded member and expands -the dictionary during construction. Generated index dispatch and slotted state -avoid that work. - -### Slotted Public Models - -Slotted dataclasses reduce instance memory and can speed attribute access. They -remove `__dict__` and restrict subclass behavior, which changes observable -Python behavior. A later design can adopt slots after compatibility review and -benchmarks. - -## Future Work - -* Add a state-parameter deserializer API if it outperforms callable state across - JSON, XML, CBOR, and HTTP binding paths. -* Precompute validation masks and constraint metadata used during every - structure decode. -* Generate endpoint rule bytecode from normalized endpoint rules. -* Support incremental generation from a prior manifest. -* Evaluate partition thresholds on large AWS service models. -* Evaluate slotted generated models as a new artifact compatibility mode. diff --git a/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json index de1bae3a8..ea94910b4 100644 --- a/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json +++ b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json @@ -1,4 +1,4 @@ { "type": "feature", - "description": "Added configurable eager JSON deserialization to improve buffered payload performance. Generated JSON clients automatically parse `bytes` eagerly while preserving incremental parsing for streaming readers, and can explicitly select eager or streaming behavior." + "description": "JSON payloads are now eagerly parsed before deserialization, improving performance for both `bytes` and `BytesReader` inputs." } diff --git a/packages/smithy-json/README.md b/packages/smithy-json/README.md index ecabd8054..efa8b10ca 100644 --- a/packages/smithy-json/README.md +++ b/packages/smithy-json/README.md @@ -3,21 +3,8 @@ This package provides generic JSON serialization and deserialization support for Smithy clients and servers. -## Deserialization modes +## Deserialization -`JSONCodec` supports eager and streaming deserialization. The default `AUTO` -mode eagerly parses byte payloads and incrementally parses `BytesReader` -payloads: - -```python -from smithy_json import JSONCodec, JSONDeserializationMode - -codec = JSONCodec( - deserialization_mode=JSONDeserializationMode.STREAMING, -) -``` - -Eager mode is faster for buffered payloads but temporarily materializes the -complete JSON value tree. Streaming mode avoids that additional allocation. -Generated clients using a JSON protocol expose the same choice through -`json_deserialization_mode` on their configuration. +`JSONCodec` eagerly parses `bytes` and `BytesReader` payloads before +deserializing Smithy shapes. This improves deserialization performance while +temporarily materializing the complete JSON value tree. diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index 4867f6577..b27b138ff 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -1,28 +1,20 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from io import BytesIO - from smithy_core.codecs import Codec from smithy_core.deserializers import ShapeDeserializer from smithy_core.interfaces import BytesReader, BytesWriter from smithy_core.serializers import ShapeSerializer from smithy_core.types import TimestampFormat -from ._private.deserializers import JSONShapeDeserializer as _JSONShapeDeserializer from ._private.documents import JSONDocument from ._private.serializers import JSONShapeSerializer as _JSONShapeSerializer from ._private.value_deserializer import ( JSONValueDeserializer as _JSONValueDeserializer, ) -from .settings import JSONDeserializationMode, JSONSettings +from .settings import JSONSettings __version__ = "0.3.0" -__all__ = ( - "JSONCodec", - "JSONDeserializationMode", - "JSONDocument", - "JSONSettings", -) +__all__ = ("JSONCodec", "JSONDocument", "JSONSettings") class JSONCodec(Codec): @@ -35,7 +27,6 @@ def __init__( default_timestamp_format: TimestampFormat = TimestampFormat.DATE_TIME, default_namespace: str | None = None, document_class: type[JSONDocument] = JSONDocument, - deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO, ) -> None: """Initializes a JSONCodec. @@ -48,8 +39,6 @@ def __init__( :param default_namespace: The default namespace to use when determining a document's discriminator. :param document_class: The document class to deserialize to. - :param deserialization_mode: Controls whether JSON payloads are parsed eagerly, - incrementally, or selected automatically based on the source type. """ self._settings = JSONSettings( use_json_name=use_json_name, @@ -57,7 +46,6 @@ def __init__( default_timestamp_format=default_timestamp_format, default_namespace=default_namespace, document_class=document_class, - deserialization_mode=deserialization_mode, ) @property @@ -68,22 +56,4 @@ def create_serializer(self, sink: BytesWriter) -> "ShapeSerializer": return _JSONShapeSerializer(sink, settings=self._settings) def create_deserializer(self, source: bytes | BytesReader) -> "ShapeDeserializer": - mode = self._settings.deserialization_mode - if mode is JSONDeserializationMode.EAGER or ( - mode is JSONDeserializationMode.AUTO and isinstance(source, bytes) - ): - return _JSONValueDeserializer(source, settings=self._settings) - - if isinstance(source, bytes): - source = BytesIO(source) - return _JSONShapeDeserializer(source, settings=self._settings) - - @property - def deserialization_mode(self) -> JSONDeserializationMode: - """The strategy used to deserialize JSON payloads.""" - - return self._settings.deserialization_mode - - @deserialization_mode.setter - def deserialization_mode(self, value: JSONDeserializationMode) -> None: - self._settings.deserialization_mode = value + return _JSONValueDeserializer(source, settings=self._settings) diff --git a/packages/smithy-json/src/smithy_json/plugins.py b/packages/smithy-json/src/smithy_json/plugins.py deleted file mode 100644 index 7423bb489..000000000 --- a/packages/smithy-json/src/smithy_json/plugins.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -from typing import Protocol - -from . import JSONCodec -from .settings import JSONDeserializationMode - - -class _JSONDeserializationConfig(Protocol): - @property - def protocol(self) -> object | None: ... - - @property - def json_deserialization_mode(self) -> JSONDeserializationMode | None: ... - - -def json_deserialization_plugin(config: _JSONDeserializationConfig) -> None: - """Configure JSON deserialization on a generated client's protocol codec.""" - - codec = getattr(config.protocol, "payload_codec", None) - if isinstance(codec, JSONCodec): - codec.deserialization_mode = ( - config.json_deserialization_mode or JSONDeserializationMode.AUTO - ) diff --git a/packages/smithy-json/src/smithy_json/settings.py b/packages/smithy-json/src/smithy_json/settings.py index 3b0253705..6ebde62cf 100644 --- a/packages/smithy-json/src/smithy_json/settings.py +++ b/packages/smithy-json/src/smithy_json/settings.py @@ -1,7 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass -from enum import StrEnum from typing import TYPE_CHECKING from smithy_core.types import TimestampFormat @@ -10,19 +9,6 @@ from ._private.documents import JSONDocument -class JSONDeserializationMode(StrEnum): - """Controls how JSON payloads are parsed during deserialization.""" - - AUTO = "auto" - """Eagerly parse bytes while preserving incremental reader consumption.""" - - EAGER = "eager" - """Consume and materialize the complete JSON value before deserializing.""" - - STREAMING = "streaming" - """Incrementally deserialize JSON using the streaming parser.""" - - @dataclass(slots=True) class JSONSettings: """Settings for the JSON codec.""" @@ -43,6 +29,3 @@ class JSONSettings: default_namespace: str | None = None """The default namespace to use when determining a document's discriminator.""" - - deserialization_mode: JSONDeserializationMode = JSONDeserializationMode.AUTO - """Controls whether JSON deserialization is eager, streaming, or automatic.""" diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index 0402f9c6b..3d3697525 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -20,8 +20,7 @@ STRING, TIMESTAMP, ) -from smithy_json import JSONCodec, JSONDeserializationMode, JSONDocument -from smithy_json._private.deserializers import JSONShapeDeserializer +from smithy_json import JSONCodec, JSONDocument from smithy_json._private.value_deserializer import JSONValueDeserializer from . import ( @@ -32,17 +31,9 @@ ) -@pytest.mark.parametrize( - "mode", - [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], -) @pytest.mark.parametrize("expected, given", JSON_SERDE_CASES) -def test_json_deserializer( - expected: Any, - given: bytes, - mode: JSONDeserializationMode, -) -> None: - codec = JSONCodec(deserialization_mode=mode) +def test_json_deserializer(expected: Any, given: bytes) -> None: + codec = JSONCodec() deserializer = codec.create_deserializer(given) match expected: case None: @@ -123,29 +114,19 @@ def test_uses_custom_document() -> None: @pytest.mark.parametrize( - "mode, source, expected_type", + "source", [ - (JSONDeserializationMode.AUTO, b"{}", JSONValueDeserializer), - (JSONDeserializationMode.AUTO, BytesIO(b"{}"), JSONShapeDeserializer), - (JSONDeserializationMode.EAGER, BytesIO(b"{}"), JSONValueDeserializer), - (JSONDeserializationMode.STREAMING, b"{}", JSONShapeDeserializer), + b"{}", + BytesIO(b"{}"), ], ) -def test_deserialization_mode_selects_parser( - mode: JSONDeserializationMode, - source: bytes | BytesIO, - expected_type: type[ShapeDeserializer], -) -> None: - deserializer = JSONCodec(deserialization_mode=mode).create_deserializer(source) - assert isinstance(deserializer, expected_type) +def test_deserializer_is_eager(source: bytes | BytesIO) -> None: + deserializer = JSONCodec().create_deserializer(source) + assert isinstance(deserializer, JSONValueDeserializer) -@pytest.mark.parametrize( - "mode", - [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], -) -def test_ignores_unknown_structure_members(mode: JSONDeserializationMode) -> None: - actual = JSONCodec(deserialization_mode=mode).deserialize( +def test_ignores_unknown_structure_members() -> None: + actual = JSONCodec().deserialize( b'{"unknown":{"nested":[1,2,3]},"stringMember":"value"}', SerdeShape, ) @@ -153,13 +134,7 @@ def test_ignores_unknown_structure_members(mode: JSONDeserializationMode) -> Non assert actual.string_member == "value" -@pytest.mark.parametrize( - "mode", - [JSONDeserializationMode.STREAMING, JSONDeserializationMode.EAGER], -) -def test_invalid_json_uses_existing_error_type(mode: JSONDeserializationMode) -> None: +def test_invalid_json_uses_existing_error_type() -> None: with pytest.raises(IncompleteJSONError): - deserializer = JSONCodec(deserialization_mode=mode).create_deserializer( - b'{"incomplete":' - ) + deserializer = JSONCodec().create_deserializer(b'{"incomplete":') deserializer.read_document(DOCUMENT) diff --git a/packages/smithy-json/tests/unit/test_plugins.py b/packages/smithy-json/tests/unit/test_plugins.py deleted file mode 100644 index b16618c0d..000000000 --- a/packages/smithy-json/tests/unit/test_plugins.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -from dataclasses import dataclass - -import pytest -from smithy_json import JSONCodec, JSONDeserializationMode -from smithy_json.plugins import json_deserialization_plugin - - -@dataclass -class _Protocol: - payload_codec: object - - -@dataclass -class _Config: - protocol: object - json_deserialization_mode: JSONDeserializationMode | None - - -@pytest.mark.parametrize( - "configured, expected", - [ - (None, JSONDeserializationMode.AUTO), - (JSONDeserializationMode.AUTO, JSONDeserializationMode.AUTO), - (JSONDeserializationMode.EAGER, JSONDeserializationMode.EAGER), - (JSONDeserializationMode.STREAMING, JSONDeserializationMode.STREAMING), - ], -) -def test_configures_json_codec( - configured: JSONDeserializationMode | None, - expected: JSONDeserializationMode, -) -> None: - codec = JSONCodec(deserialization_mode=JSONDeserializationMode.EAGER) - config = _Config( - protocol=_Protocol(payload_codec=codec), - json_deserialization_mode=configured, - ) - - json_deserialization_plugin(config) - - assert codec.deserialization_mode is expected - - -@pytest.mark.parametrize("protocol", [object(), _Protocol(payload_codec=object())]) -def test_ignores_protocols_without_json_codec(protocol: object) -> None: - config = _Config( - protocol=protocol, - json_deserialization_mode=JSONDeserializationMode.STREAMING, - ) - json_deserialization_plugin(config) From 9ba0f396c36f6f1cd677f1dcd8c704043563a88f Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Wed, 14 Oct 2026 11:03:12 -0400 Subject: [PATCH 3/4] rename new parser --- .../smithy-json/src/smithy_json/__init__.py | 6 +- .../src/smithy_json/_private/deserializers.py | 321 ++++++++---------- .../_private/value_deserializer.py | 205 ----------- .../tests/unit/test_deserializers.py | 4 +- 4 files changed, 137 insertions(+), 399 deletions(-) delete mode 100644 packages/smithy-json/src/smithy_json/_private/value_deserializer.py diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index b27b138ff..fd78af5a9 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -6,11 +6,9 @@ from smithy_core.serializers import ShapeSerializer from smithy_core.types import TimestampFormat +from ._private.deserializers import JSONShapeDeserializer as _JSONShapeDeserializer from ._private.documents import JSONDocument from ._private.serializers import JSONShapeSerializer as _JSONShapeSerializer -from ._private.value_deserializer import ( - JSONValueDeserializer as _JSONValueDeserializer, -) from .settings import JSONSettings __version__ = "0.3.0" @@ -56,4 +54,4 @@ def create_serializer(self, sink: BytesWriter) -> "ShapeSerializer": return _JSONShapeSerializer(sink, settings=self._settings) def create_deserializer(self, source: bytes | BytesReader) -> "ShapeDeserializer": - return _JSONValueDeserializer(source, settings=self._settings) + return _JSONShapeDeserializer(source, settings=self._settings) diff --git a/packages/smithy-json/src/smithy_json/_private/deserializers.py b/packages/smithy-json/src/smithy_json/_private/deserializers.py index 58d888edb..02ea83687 100644 --- a/packages/smithy-json/src/smithy_json/_private/deserializers.py +++ b/packages/smithy-json/src/smithy_json/_private/deserializers.py @@ -1,14 +1,14 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import datetime +import json from base64 import b64decode -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from decimal import Decimal -from typing import Literal, NamedTuple, Protocol, cast +from json import JSONDecodeError -import ijson # type: ignore -from ijson.common import ObjectBuilder # type: ignore +from ijson.common import IncompleteJSONError # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.exceptions import SmithyError @@ -20,153 +20,106 @@ from ..settings import JSONSettings -# TODO: put these type hints in a pyi somewhere. There here because ijson isn't -# typed. -type JSONParseEventType = Literal[ - "null", - "string", - "number", - "boolean", - "start_array", - "end_array", - "start_map", - "map_key", - "end_map", -] +type JSONValue = ( + Mapping[str, "JSONValue"] + | Sequence["JSONValue"] + | str + | int + | Decimal + | bool + | None +) -type JSONParseEventValue = str | int | float | Decimal | bool | None -type JSON = Mapping[str, "JSON"] | Sequence["JSON"] | JSONParseEventValue +class JSONValueError(SmithyError): + """Raised when a parsed JSON value does not match the requested Smithy type.""" - -class JSONParseEvent(NamedTuple): - path: str - type: JSONParseEventType - value: JSONParseEventValue - - -class JSONTokenError(SmithyError): - def __init__(self, expected: str, event: JSONParseEvent) -> None: + def __init__(self, expected: str, value: JSONValue) -> None: super().__init__( - f"Error parsing JSON. Expected token of type `{expected}` at path " - f"`{event.path}`, but found: `{event.type}`: {event.value}" + f"Error parsing JSON. Expected value of type `{expected}`, " + f"but found: `{type(value).__name__}`: {value}" ) -class TypedObjectBuilder(Protocol): - value: JSON - - def event(self, event: JSONParseEventType, value: JSONParseEventValue): ... - - -class BufferedParser: - """A wrapper around the ijson parser that allows peeking.""" - - def __init__( - self, stream: Iterator[tuple[str, JSONParseEventType, JSONParseEventValue]] - ) -> None: - self._stream = stream - self._pending: JSONParseEvent | None = None - - def __iter__(self): - return self - - def __next__(self) -> JSONParseEvent: - if self._pending is not None: - result = self._pending - self._pending = None - return result - return self._next() - - def _next(self) -> JSONParseEvent: - return JSONParseEvent(*next(self._stream)) - - def peek(self) -> JSONParseEvent: - if self._pending is None: - self._pending = self._next() - return self._pending +def _reject_constant(value: str) -> None: + raise ValueError(f"Invalid JSON constant: {value}") class JSONShapeDeserializer(ShapeDeserializer): - def __init__(self, source: BytesReader, settings: JSONSettings) -> None: - self._stream = BufferedParser(ijson.parse(source)) - self._settings = settings + """Deserializes Smithy shapes from a fully materialized JSON value.""" + + def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: + try: + if isinstance(source, bytes): + self._value: JSONValue = json.loads( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + else: + self._value = json.load( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + except (JSONDecodeError, UnicodeDecodeError, ValueError) as error: + raise IncompleteJSONError(f"parse error: {error}") from error - # A mapping of json name to member name for each shape. Since the deserializer - # is shared and we don't know which shapes will be deserialized, this is - # populated on an as-needed basis. - self._json_names: dict[ShapeID, dict[str, str]] = {} + self._settings = settings + self._json_names: dict[ShapeID, dict[str, Schema]] = {} def is_null(self) -> bool: - return self._stream.peek().type == "null" + return self._value is None def read_null(self) -> None: - event = next(self._stream) - if event.value is not None: - raise JSONTokenError("null", event) - return None + if self._value is not None: + raise JSONValueError("null", self._value) def read_boolean(self, schema: Schema) -> bool: - event = next(self._stream) - if not isinstance(event.value, bool): - raise JSONTokenError("boolean", event) - return event.value + if not isinstance(self._value, bool): + raise JSONValueError("boolean", self._value) + return self._value def read_blob(self, schema: Schema) -> bytes: - event = next(self._stream) - if event.type != "string" or not isinstance(event.value, str): - raise JSONTokenError("string", event) - return b64decode(event.value) + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return b64decode(self._value) def read_integer(self, schema: Schema) -> int: - event = next(self._stream) - if not isinstance(event.value, int): - raise JSONTokenError("number", event) - return event.value + if not isinstance(self._value, int) or isinstance(self._value, bool): + raise JSONValueError("number", self._value) + return self._value def read_float(self, schema: Schema) -> float: - event = next(self._stream) - match event.value: + match self._value: case Decimal() | "Infinity" | "-Infinity" | "NaN": - return float(event.value) - case int() | float(): - return event.value + return float(self._value) + case int() if not isinstance(self._value, bool): + return self._value case _: - raise JSONTokenError("number", event) + raise JSONValueError("number", self._value) def read_big_decimal(self, schema: Schema) -> Decimal: - event = next(self._stream) - match event.value: + match self._value: case Decimal(): - return event.value - case int() | float() | "Infinity" | "-Infinity" | "NaN": - return Decimal(event.value) + return self._value + case int() if not isinstance(self._value, bool): + return Decimal(self._value) + case "Infinity" | "-Infinity" | "NaN": + return Decimal(self._value) case _: - raise JSONTokenError("number", event) + raise JSONValueError("number", self._value) def read_string(self, schema: Schema) -> str: - event = next(self._stream) - if event.type not in ("string", "map_key") or not isinstance(event.value, str): - raise JSONTokenError("string | map_key", event) - return event.value + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return self._value def read_document(self, schema: Schema) -> Document: - start = next(self._stream) - if start.type not in ("start_map", "start_array"): - return self._settings.document_class( - value=start.value, schema=schema, settings=self._settings - ) - - end_type = "end_map" if start.type == "start_map" else "end_array" - builder = cast(TypedObjectBuilder, ObjectBuilder()) - builder.event(start.type, start.value) - while ( - event := next(self._stream) - ).path != start.path or event.type != end_type: - builder.event(event.type, event.value) - return self._settings.document_class( - value=builder.value, schema=schema, settings=self._settings + value=self._value, + schema=schema, + settings=self._settings, ) def read_timestamp(self, schema: Schema) -> datetime.datetime: @@ -182,79 +135,71 @@ def read_timestamp(self, schema: Schema) -> datetime.datetime: return format.deserialize(self.read_string(schema=schema)) def read_struct( - self, schema: Schema, consumer: Callable[[Schema, "ShapeDeserializer"], None] - ): - event = next(self._stream) - if event.type != "start_map": - raise JSONTokenError("start_map", event) - - while self._stream.peek().type != "end_map": - key = self.read_string(schema=schema) - member = self._resolve_member(schema=schema, key=key) - if not member: - self._skip() - continue - if self.is_null() and member.shape_type is not ShapeType.DOCUMENT: - self.read_null() - continue - consumer(member, self) - - next(self._stream) - - def _resolve_member(self, schema: Schema, key: str) -> Schema | None: - if self._settings.use_json_name: - if schema.id not in self._json_names: - self._cache_json_names(schema=schema) - if key in self._json_names[schema.id]: - return schema.members.get(self._json_names[schema.id][key]) - return None - - return schema.members.get(key) - - def _cache_json_names(self, schema: Schema): - result: dict[str, str] = {} - for member_name, member_schema in schema.members.items(): - name: str = member_name - if json_name := member_schema.get_trait(JSONNameTrait): - name = json_name.value - result[name] = member_name - self._json_names[schema.id] = result + self, + schema: Schema, + consumer: Callable[[Schema, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + struct_schema = schema.member_target or schema + members = self._members_by_wire_name(struct_schema) + try: + for key, member_value in value.items(): + member = members.get(key) + if member is None: + continue + if member_value is None and member.shape_type is not ShapeType.DOCUMENT: + continue + self._value = member_value + consumer(member, self) + finally: + self._value = value def read_list( - self, schema: Schema, consumer: Callable[["ShapeDeserializer"], None] - ): - event = next(self._stream) - if event.type != "start_array": - raise JSONTokenError("start_array", event) - - while self._stream.peek().type != "end_array": - consumer(self) + self, + schema: Schema, + consumer: Callable[[ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Sequence) or isinstance(value, str): + raise JSONValueError("array", value) - next(self._stream) + try: + for member_value in value: + self._value = member_value + consumer(self) + finally: + self._value = value def read_map( self, schema: Schema, - consumer: Callable[[str, "ShapeDeserializer"], None], - ): - event = next(self._stream) - if event.type != "start_map": - raise JSONTokenError("start_map", event) - - key_schema = schema.members["key"] - while self._stream.peek().type != "end_map": - consumer(self.read_string(schema=key_schema), self) - - next(self._stream) - - def _skip(self) -> None: - start = next(self._stream) - if start.type not in ("start_map", "start_array"): - return - - end_type = "end_map" if start.type == "start_map" else "end_array" - - while ( - event := next(self._stream) - ).path != start.path or event.type != end_type: - continue + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + try: + for key, member_value in value.items(): + self._value = member_value + consumer(key, self) + finally: + self._value = value + + def _members_by_wire_name(self, schema: Schema) -> Mapping[str, Schema]: + if not self._settings.use_json_name: + return schema.members + + members = self._json_names.get(schema.id) + if members is None: + members = {} + for member_name, member in schema.members.items(): + json_name = member.get_trait(JSONNameTrait) + members[json_name.value if json_name is not None else member_name] = ( + member + ) + self._json_names[schema.id] = members + return members diff --git a/packages/smithy-json/src/smithy_json/_private/value_deserializer.py b/packages/smithy-json/src/smithy_json/_private/value_deserializer.py deleted file mode 100644 index f5641a117..000000000 --- a/packages/smithy-json/src/smithy_json/_private/value_deserializer.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -import datetime -import json -from base64 import b64decode -from collections.abc import Callable, Mapping, Sequence -from decimal import Decimal -from json import JSONDecodeError - -from ijson.common import IncompleteJSONError # type: ignore -from smithy_core.deserializers import ShapeDeserializer -from smithy_core.documents import Document -from smithy_core.exceptions import SmithyError -from smithy_core.interfaces import BytesReader -from smithy_core.schemas import Schema -from smithy_core.shapes import ShapeID, ShapeType -from smithy_core.traits import JSONNameTrait, TimestampFormatTrait -from smithy_core.types import TimestampFormat - -from ..settings import JSONSettings - -type JSONValue = ( - Mapping[str, "JSONValue"] - | Sequence["JSONValue"] - | str - | int - | Decimal - | bool - | None -) - - -class JSONValueError(SmithyError): - """Raised when a parsed JSON value does not match the requested Smithy type.""" - - def __init__(self, expected: str, value: JSONValue) -> None: - super().__init__( - f"Error parsing JSON. Expected value of type `{expected}`, " - f"but found: `{type(value).__name__}`: {value}" - ) - - -def _reject_constant(value: str) -> None: - raise ValueError(f"Invalid JSON constant: {value}") - - -class JSONValueDeserializer(ShapeDeserializer): - """Deserializes Smithy shapes from a fully materialized JSON value.""" - - def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: - try: - if isinstance(source, bytes): - self._value: JSONValue = json.loads( - source, - parse_float=Decimal, - parse_constant=_reject_constant, - ) - else: - self._value = json.load( - source, - parse_float=Decimal, - parse_constant=_reject_constant, - ) - except (JSONDecodeError, UnicodeDecodeError, ValueError) as error: - raise IncompleteJSONError(f"parse error: {error}") from error - - self._settings = settings - self._json_names: dict[ShapeID, dict[str, Schema]] = {} - - def is_null(self) -> bool: - return self._value is None - - def read_null(self) -> None: - if self._value is not None: - raise JSONValueError("null", self._value) - - def read_boolean(self, schema: Schema) -> bool: - if not isinstance(self._value, bool): - raise JSONValueError("boolean", self._value) - return self._value - - def read_blob(self, schema: Schema) -> bytes: - if not isinstance(self._value, str): - raise JSONValueError("string", self._value) - return b64decode(self._value) - - def read_integer(self, schema: Schema) -> int: - if not isinstance(self._value, int) or isinstance(self._value, bool): - raise JSONValueError("number", self._value) - return self._value - - def read_float(self, schema: Schema) -> float: - match self._value: - case Decimal() | "Infinity" | "-Infinity" | "NaN": - return float(self._value) - case int() if not isinstance(self._value, bool): - return self._value - case _: - raise JSONValueError("number", self._value) - - def read_big_decimal(self, schema: Schema) -> Decimal: - match self._value: - case Decimal(): - return self._value - case int() if not isinstance(self._value, bool): - return Decimal(self._value) - case "Infinity" | "-Infinity" | "NaN": - return Decimal(self._value) - case _: - raise JSONValueError("number", self._value) - - def read_string(self, schema: Schema) -> str: - if not isinstance(self._value, str): - raise JSONValueError("string", self._value) - return self._value - - def read_document(self, schema: Schema) -> Document: - return self._settings.document_class( - value=self._value, - schema=schema, - settings=self._settings, - ) - - def read_timestamp(self, schema: Schema) -> datetime.datetime: - format = self._settings.default_timestamp_format - if self._settings.use_timestamp_format: - if format_trait := schema.get_trait(TimestampFormatTrait): - format = format_trait.format - - match format: - case TimestampFormat.EPOCH_SECONDS: - return format.deserialize(self.read_float(schema=schema)) - case _: - return format.deserialize(self.read_string(schema=schema)) - - def read_struct( - self, - schema: Schema, - consumer: Callable[[Schema, ShapeDeserializer], None], - ) -> None: - value = self._value - if not isinstance(value, Mapping): - raise JSONValueError("object", value) - - struct_schema = schema.member_target or schema - members = self._members_by_wire_name(struct_schema) - try: - for key, member_value in value.items(): - member = members.get(key) - if member is None: - continue - if member_value is None and member.shape_type is not ShapeType.DOCUMENT: - continue - self._value = member_value - consumer(member, self) - finally: - self._value = value - - def read_list( - self, - schema: Schema, - consumer: Callable[[ShapeDeserializer], None], - ) -> None: - value = self._value - if not isinstance(value, Sequence) or isinstance(value, str): - raise JSONValueError("array", value) - - try: - for member_value in value: - self._value = member_value - consumer(self) - finally: - self._value = value - - def read_map( - self, - schema: Schema, - consumer: Callable[[str, ShapeDeserializer], None], - ) -> None: - value = self._value - if not isinstance(value, Mapping): - raise JSONValueError("object", value) - - try: - for key, member_value in value.items(): - self._value = member_value - consumer(key, self) - finally: - self._value = value - - def _members_by_wire_name(self, schema: Schema) -> Mapping[str, Schema]: - if not self._settings.use_json_name: - return schema.members - - members = self._json_names.get(schema.id) - if members is None: - members = {} - for member_name, member in schema.members.items(): - json_name = member.get_trait(JSONNameTrait) - members[json_name.value if json_name is not None else member_name] = ( - member - ) - self._json_names[schema.id] = members - return members diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index 3d3697525..d1ffa8cdb 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -21,7 +21,7 @@ TIMESTAMP, ) from smithy_json import JSONCodec, JSONDocument -from smithy_json._private.value_deserializer import JSONValueDeserializer +from smithy_json._private.deserializers import JSONShapeDeserializer from . import ( JSON_SERDE_CASES, @@ -122,7 +122,7 @@ def test_uses_custom_document() -> None: ) def test_deserializer_is_eager(source: bytes | BytesIO) -> None: deserializer = JSONCodec().create_deserializer(source) - assert isinstance(deserializer, JSONValueDeserializer) + assert isinstance(deserializer, JSONShapeDeserializer) def test_ignores_unknown_structure_members() -> None: From dabdbc2d4d09bbba0cb04dc212f3190c1000aca2 Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Wed, 14 Oct 2026 13:57:55 -0400 Subject: [PATCH 4/4] remove error remapping/ijson dependency --- .../tests/unit/aio/test_protocols.py | 7 ++--- packages/smithy-json/pyproject.toml | 1 - .../src/smithy_json/_private/deserializers.py | 29 ++++++++----------- .../src/smithy_json/_private/documents.py | 3 +- .../tests/unit/test_deserializers.py | 6 ++-- uv.lock | 25 +--------------- 6 files changed, 20 insertions(+), 51 deletions(-) diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index 9718f560c..2744d9cf8 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass +from json import JSONDecodeError from typing import Any, cast from unittest.mock import Mock import pytest -from ijson.common import IncompleteJSONError # type: ignore[reportMissingTypeStubs] from smithy_aws_core.aio.protocols import ( AWSErrorIdentifier, AwsJson11ClientProtocol, @@ -467,10 +467,9 @@ async def test_aws_json11_raises_parse_error_for_invalid_error_body() -> None: body=b'{"__type":', ) - # The message depends on which ijson backend is in use. with pytest.raises( - IncompleteJSONError, - match=r"premature EOF|parse error|Incomplete JSON content", + JSONDecodeError, + match="Expecting value", ): await protocol.deserialize_response( operation=operation, diff --git a/packages/smithy-json/pyproject.toml b/packages/smithy-json/pyproject.toml index 89e5f705d..f897d3027 100644 --- a/packages/smithy-json/pyproject.toml +++ b/packages/smithy-json/pyproject.toml @@ -28,7 +28,6 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "ijson>=3.3.0", "smithy-core", ] diff --git a/packages/smithy-json/src/smithy_json/_private/deserializers.py b/packages/smithy-json/src/smithy_json/_private/deserializers.py index 02ea83687..bbd24f5f6 100644 --- a/packages/smithy-json/src/smithy_json/_private/deserializers.py +++ b/packages/smithy-json/src/smithy_json/_private/deserializers.py @@ -6,9 +6,7 @@ from base64 import b64decode from collections.abc import Callable, Mapping, Sequence from decimal import Decimal -from json import JSONDecodeError -from ijson.common import IncompleteJSONError # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.exceptions import SmithyError @@ -49,21 +47,18 @@ class JSONShapeDeserializer(ShapeDeserializer): """Deserializes Smithy shapes from a fully materialized JSON value.""" def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: - try: - if isinstance(source, bytes): - self._value: JSONValue = json.loads( - source, - parse_float=Decimal, - parse_constant=_reject_constant, - ) - else: - self._value = json.load( - source, - parse_float=Decimal, - parse_constant=_reject_constant, - ) - except (JSONDecodeError, UnicodeDecodeError, ValueError) as error: - raise IncompleteJSONError(f"parse error: {error}") from error + if isinstance(source, bytes): + self._value: JSONValue = json.loads( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + else: + self._value = json.load( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) self._settings = settings self._json_names: dict[ShapeID, dict[str, Schema]] = {} diff --git a/packages/smithy-json/src/smithy_json/_private/documents.py b/packages/smithy-json/src/smithy_json/_private/documents.py index 7c0138515..9bdede3bd 100644 --- a/packages/smithy-json/src/smithy_json/_private/documents.py +++ b/packages/smithy-json/src/smithy_json/_private/documents.py @@ -42,8 +42,7 @@ def as_blob(self) -> bytes: return b64decode(expect_type(str, self._value)) def as_float(self) -> float: - # Floats are returned as Decimals by default in ijson, which is the correct - # behavior, since technically JSON numbers have arbitrary precision. + # JSON numbers are parsed as Decimals to preserve their arbitrary precision. return float(expect_type(Decimal, self._value)) def as_timestamp(self) -> datetime: diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index d1ffa8cdb..7e35b3ddb 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -4,10 +4,10 @@ from datetime import datetime from decimal import Decimal from io import BytesIO +from json import JSONDecodeError from typing import Any import pytest -from ijson.common import IncompleteJSONError # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.prelude import ( @@ -134,7 +134,7 @@ def test_ignores_unknown_structure_members() -> None: assert actual.string_member == "value" -def test_invalid_json_uses_existing_error_type() -> None: - with pytest.raises(IncompleteJSONError): +def test_invalid_json_raises_json_decode_error() -> None: + with pytest.raises(JSONDecodeError): deserializer = JSONCodec().create_deserializer(b'{"incomplete":') deserializer.read_document(DOCUMENT) diff --git a/uv.lock b/uv.lock index 3ef01194f..f07a64d38 100644 --- a/uv.lock +++ b/uv.lock @@ -350,25 +350,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] -[[package]] -name = "ijson" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/83/28e9e93a3a61913e334e3a2e78ea9924bb9f9b1ac45898977f9d9dd6133f/ijson-3.3.0.tar.gz", hash = "sha256:7f172e6ba1bee0d4c8f8ebd639577bfe429dee0f3f96775a067b8bae4492d8a0", size = 60079, upload-time = "2024-06-06T08:37:13.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/4d/3992f7383e26a950e02dc704bc6c5786a080d5c25fe0fc5543ef477c1883/ijson-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:988e959f2f3d59ebd9c2962ae71b97c0df58323910d0b368cc190ad07429d1bb", size = 84550, upload-time = "2024-06-06T08:35:16.756Z" }, - { url = "https://files.pythonhosted.org/packages/1b/cc/3d4372e0d0b02a821b982f1fdf10385512dae9b9443c1597719dd37769a9/ijson-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b2f73f0d0fce5300f23a1383d19b44d103bb113b57a69c36fd95b7c03099b181", size = 57572, upload-time = "2024-06-06T08:35:18.077Z" }, - { url = "https://files.pythonhosted.org/packages/02/de/970d48b1ff9da5d9513c86fdd2acef5cb3415541c8069e0d92a151b84adb/ijson-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0ee57a28c6bf523d7cb0513096e4eb4dac16cd935695049de7608ec110c2b751", size = 56902, upload-time = "2024-06-06T08:35:20.065Z" }, - { url = "https://files.pythonhosted.org/packages/5e/a0/4537722c8b3b05e82c23dfe09a3a64dd1e44a013a5ca58b1e77dfe48b2f1/ijson-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0155a8f079c688c2ccaea05de1ad69877995c547ba3d3612c1c336edc12a3a5", size = 127400, upload-time = "2024-06-06T08:35:21.81Z" }, - { url = "https://files.pythonhosted.org/packages/b2/96/54956062a99cf49f7a7064b573dcd756da0563ce57910dc34e27a473d9b9/ijson-3.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ab00721304af1ae1afa4313ecfa1bf16b07f55ef91e4a5b93aeaa3e2bd7917c", size = 118786, upload-time = "2024-06-06T08:35:23.496Z" }, - { url = "https://files.pythonhosted.org/packages/07/74/795319531c5b5504508f595e631d592957f24bed7ff51a15bc4c61e7b24c/ijson-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40ee3821ee90be0f0e95dcf9862d786a7439bd1113e370736bfdf197e9765bfb", size = 126288, upload-time = "2024-06-06T08:35:25.473Z" }, - { url = "https://files.pythonhosted.org/packages/69/6a/e0cec06fbd98851d5d233b59058c1dc2ea767c9bb6feca41aa9164fff769/ijson-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3b6987a0bc3e6d0f721b42c7a0198ef897ae50579547b0345f7f02486898f5", size = 129569, upload-time = "2024-06-06T08:35:26.871Z" }, - { url = "https://files.pythonhosted.org/packages/2a/4f/82c0d896d8dcb175f99ced7d87705057bcd13523998b48a629b90139a0dc/ijson-3.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:63afea5f2d50d931feb20dcc50954e23cef4127606cc0ecf7a27128ed9f9a9e6", size = 121508, upload-time = "2024-06-06T08:35:28.236Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b6/8973474eba4a917885e289d9e138267d3d1f052c2d93b8c968755661a42d/ijson-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5c3e285e0735fd8c5a26d177eca8b52512cdd8687ca86ec77a0c66e9c510182", size = 127896, upload-time = "2024-06-06T08:35:29.61Z" }, - { url = "https://files.pythonhosted.org/packages/94/25/00e66af887adbbe70002e0479c3c2340bdfa17a168e25d4ab5a27b53582d/ijson-3.3.0-cp312-cp312-win32.whl", hash = "sha256:907f3a8674e489abdcb0206723e5560a5cb1fa42470dcc637942d7b10f28b695", size = 49272, upload-time = "2024-06-06T08:35:31.137Z" }, - { url = "https://files.pythonhosted.org/packages/25/a2/e187beee237808b2c417109ae0f4f7ee7c81ecbe9706305d6ac2a509cc45/ijson-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f890d04ad33262d0c77ead53c85f13abfb82f2c8f078dfbf24b78f59534dfdd", size = 51272, upload-time = "2024-06-06T08:35:32.38Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" @@ -781,15 +762,11 @@ provides-extras = ["aiohttp", "awscrt", "cbor"] name = "smithy-json" source = { editable = "packages/smithy-json" } dependencies = [ - { name = "ijson" }, { name = "smithy-core" }, ] [package.metadata] -requires-dist = [ - { name = "ijson", specifier = ">=3.3.0" }, - { name = "smithy-core", editable = "packages/smithy-core" }, -] +requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "smithy-python"