diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index 9718f560c..2744d9cf8 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -2,11 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 from dataclasses import dataclass +from json import JSONDecodeError from typing import Any, cast from unittest.mock import Mock import pytest -from ijson.common import IncompleteJSONError # type: ignore[reportMissingTypeStubs] from smithy_aws_core.aio.protocols import ( AWSErrorIdentifier, AwsJson11ClientProtocol, @@ -467,10 +467,9 @@ async def test_aws_json11_raises_parse_error_for_invalid_error_body() -> None: body=b'{"__type":', ) - # The message depends on which ijson backend is in use. with pytest.raises( - IncompleteJSONError, - match=r"premature EOF|parse error|Incomplete JSON content", + JSONDecodeError, + match="Expecting value", ): await protocol.deserialize_response( operation=operation, diff --git a/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json new file mode 100644 index 000000000..ea94910b4 --- /dev/null +++ b/packages/smithy-json/.changes/next-release/smithy-json-feature-4b7f9ca09557405eb9382c56490c6d7a.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "JSON payloads are now eagerly parsed before deserialization, improving performance for both `bytes` and `BytesReader` inputs." +} diff --git a/packages/smithy-json/README.md b/packages/smithy-json/README.md index 29b5448d6..efa8b10ca 100644 --- a/packages/smithy-json/README.md +++ b/packages/smithy-json/README.md @@ -2,3 +2,9 @@ This package provides generic JSON serialization and deserialization support for Smithy clients and servers. + +## Deserialization + +`JSONCodec` eagerly parses `bytes` and `BytesReader` payloads before +deserializing Smithy shapes. This improves deserialization performance while +temporarily materializing the complete JSON value tree. diff --git a/packages/smithy-json/pyproject.toml b/packages/smithy-json/pyproject.toml index 89e5f705d..f897d3027 100644 --- a/packages/smithy-json/pyproject.toml +++ b/packages/smithy-json/pyproject.toml @@ -28,7 +28,6 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - "ijson>=3.3.0", "smithy-core", ] diff --git a/packages/smithy-json/src/smithy_json/__init__.py b/packages/smithy-json/src/smithy_json/__init__.py index 7d20bba87..fd78af5a9 100644 --- a/packages/smithy-json/src/smithy_json/__init__.py +++ b/packages/smithy-json/src/smithy_json/__init__.py @@ -1,7 +1,5 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from io import BytesIO - from smithy_core.codecs import Codec from smithy_core.deserializers import ShapeDeserializer from smithy_core.interfaces import BytesReader, BytesWriter @@ -56,6 +54,4 @@ def create_serializer(self, sink: BytesWriter) -> "ShapeSerializer": return _JSONShapeSerializer(sink, settings=self._settings) def create_deserializer(self, source: bytes | BytesReader) -> "ShapeDeserializer": - if isinstance(source, bytes): - source = BytesIO(source) return _JSONShapeDeserializer(source, settings=self._settings) diff --git a/packages/smithy-json/src/smithy_json/_private/deserializers.py b/packages/smithy-json/src/smithy_json/_private/deserializers.py index 58d888edb..bbd24f5f6 100644 --- a/packages/smithy-json/src/smithy_json/_private/deserializers.py +++ b/packages/smithy-json/src/smithy_json/_private/deserializers.py @@ -1,14 +1,12 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 import datetime +import json from base64 import b64decode -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from decimal import Decimal -from typing import Literal, NamedTuple, Protocol, cast -import ijson # type: ignore -from ijson.common import ObjectBuilder # type: ignore from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import Document from smithy_core.exceptions import SmithyError @@ -20,153 +18,103 @@ from ..settings import JSONSettings -# TODO: put these type hints in a pyi somewhere. There here because ijson isn't -# typed. -type JSONParseEventType = Literal[ - "null", - "string", - "number", - "boolean", - "start_array", - "end_array", - "start_map", - "map_key", - "end_map", -] +type JSONValue = ( + Mapping[str, "JSONValue"] + | Sequence["JSONValue"] + | str + | int + | Decimal + | bool + | None +) -type JSONParseEventValue = str | int | float | Decimal | bool | None -type JSON = Mapping[str, "JSON"] | Sequence["JSON"] | JSONParseEventValue +class JSONValueError(SmithyError): + """Raised when a parsed JSON value does not match the requested Smithy type.""" - -class JSONParseEvent(NamedTuple): - path: str - type: JSONParseEventType - value: JSONParseEventValue - - -class JSONTokenError(SmithyError): - def __init__(self, expected: str, event: JSONParseEvent) -> None: + def __init__(self, expected: str, value: JSONValue) -> None: super().__init__( - f"Error parsing JSON. Expected token of type `{expected}` at path " - f"`{event.path}`, but found: `{event.type}`: {event.value}" + f"Error parsing JSON. Expected value of type `{expected}`, " + f"but found: `{type(value).__name__}`: {value}" ) -class TypedObjectBuilder(Protocol): - value: JSON - - def event(self, event: JSONParseEventType, value: JSONParseEventValue): ... - - -class BufferedParser: - """A wrapper around the ijson parser that allows peeking.""" - - def __init__( - self, stream: Iterator[tuple[str, JSONParseEventType, JSONParseEventValue]] - ) -> None: - self._stream = stream - self._pending: JSONParseEvent | None = None - - def __iter__(self): - return self - - def __next__(self) -> JSONParseEvent: - if self._pending is not None: - result = self._pending - self._pending = None - return result - return self._next() - - def _next(self) -> JSONParseEvent: - return JSONParseEvent(*next(self._stream)) - - def peek(self) -> JSONParseEvent: - if self._pending is None: - self._pending = self._next() - return self._pending +def _reject_constant(value: str) -> None: + raise ValueError(f"Invalid JSON constant: {value}") class JSONShapeDeserializer(ShapeDeserializer): - def __init__(self, source: BytesReader, settings: JSONSettings) -> None: - self._stream = BufferedParser(ijson.parse(source)) - self._settings = settings + """Deserializes Smithy shapes from a fully materialized JSON value.""" + + def __init__(self, source: bytes | BytesReader, settings: JSONSettings) -> None: + if isinstance(source, bytes): + self._value: JSONValue = json.loads( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) + else: + self._value = json.load( + source, + parse_float=Decimal, + parse_constant=_reject_constant, + ) - # A mapping of json name to member name for each shape. Since the deserializer - # is shared and we don't know which shapes will be deserialized, this is - # populated on an as-needed basis. - self._json_names: dict[ShapeID, dict[str, str]] = {} + self._settings = settings + self._json_names: dict[ShapeID, dict[str, Schema]] = {} def is_null(self) -> bool: - return self._stream.peek().type == "null" + return self._value is None def read_null(self) -> None: - event = next(self._stream) - if event.value is not None: - raise JSONTokenError("null", event) - return None + if self._value is not None: + raise JSONValueError("null", self._value) def read_boolean(self, schema: Schema) -> bool: - event = next(self._stream) - if not isinstance(event.value, bool): - raise JSONTokenError("boolean", event) - return event.value + if not isinstance(self._value, bool): + raise JSONValueError("boolean", self._value) + return self._value def read_blob(self, schema: Schema) -> bytes: - event = next(self._stream) - if event.type != "string" or not isinstance(event.value, str): - raise JSONTokenError("string", event) - return b64decode(event.value) + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return b64decode(self._value) def read_integer(self, schema: Schema) -> int: - event = next(self._stream) - if not isinstance(event.value, int): - raise JSONTokenError("number", event) - return event.value + if not isinstance(self._value, int) or isinstance(self._value, bool): + raise JSONValueError("number", self._value) + return self._value def read_float(self, schema: Schema) -> float: - event = next(self._stream) - match event.value: + match self._value: case Decimal() | "Infinity" | "-Infinity" | "NaN": - return float(event.value) - case int() | float(): - return event.value + return float(self._value) + case int() if not isinstance(self._value, bool): + return self._value case _: - raise JSONTokenError("number", event) + raise JSONValueError("number", self._value) def read_big_decimal(self, schema: Schema) -> Decimal: - event = next(self._stream) - match event.value: + match self._value: case Decimal(): - return event.value - case int() | float() | "Infinity" | "-Infinity" | "NaN": - return Decimal(event.value) + return self._value + case int() if not isinstance(self._value, bool): + return Decimal(self._value) + case "Infinity" | "-Infinity" | "NaN": + return Decimal(self._value) case _: - raise JSONTokenError("number", event) + raise JSONValueError("number", self._value) def read_string(self, schema: Schema) -> str: - event = next(self._stream) - if event.type not in ("string", "map_key") or not isinstance(event.value, str): - raise JSONTokenError("string | map_key", event) - return event.value + if not isinstance(self._value, str): + raise JSONValueError("string", self._value) + return self._value def read_document(self, schema: Schema) -> Document: - start = next(self._stream) - if start.type not in ("start_map", "start_array"): - return self._settings.document_class( - value=start.value, schema=schema, settings=self._settings - ) - - end_type = "end_map" if start.type == "start_map" else "end_array" - builder = cast(TypedObjectBuilder, ObjectBuilder()) - builder.event(start.type, start.value) - while ( - event := next(self._stream) - ).path != start.path or event.type != end_type: - builder.event(event.type, event.value) - return self._settings.document_class( - value=builder.value, schema=schema, settings=self._settings + value=self._value, + schema=schema, + settings=self._settings, ) def read_timestamp(self, schema: Schema) -> datetime.datetime: @@ -182,79 +130,71 @@ def read_timestamp(self, schema: Schema) -> datetime.datetime: return format.deserialize(self.read_string(schema=schema)) def read_struct( - self, schema: Schema, consumer: Callable[[Schema, "ShapeDeserializer"], None] - ): - event = next(self._stream) - if event.type != "start_map": - raise JSONTokenError("start_map", event) - - while self._stream.peek().type != "end_map": - key = self.read_string(schema=schema) - member = self._resolve_member(schema=schema, key=key) - if not member: - self._skip() - continue - if self.is_null() and member.shape_type is not ShapeType.DOCUMENT: - self.read_null() - continue - consumer(member, self) - - next(self._stream) - - def _resolve_member(self, schema: Schema, key: str) -> Schema | None: - if self._settings.use_json_name: - if schema.id not in self._json_names: - self._cache_json_names(schema=schema) - if key in self._json_names[schema.id]: - return schema.members.get(self._json_names[schema.id][key]) - return None - - return schema.members.get(key) - - def _cache_json_names(self, schema: Schema): - result: dict[str, str] = {} - for member_name, member_schema in schema.members.items(): - name: str = member_name - if json_name := member_schema.get_trait(JSONNameTrait): - name = json_name.value - result[name] = member_name - self._json_names[schema.id] = result + self, + schema: Schema, + consumer: Callable[[Schema, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + struct_schema = schema.member_target or schema + members = self._members_by_wire_name(struct_schema) + try: + for key, member_value in value.items(): + member = members.get(key) + if member is None: + continue + if member_value is None and member.shape_type is not ShapeType.DOCUMENT: + continue + self._value = member_value + consumer(member, self) + finally: + self._value = value def read_list( - self, schema: Schema, consumer: Callable[["ShapeDeserializer"], None] - ): - event = next(self._stream) - if event.type != "start_array": - raise JSONTokenError("start_array", event) - - while self._stream.peek().type != "end_array": - consumer(self) + self, + schema: Schema, + consumer: Callable[[ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Sequence) or isinstance(value, str): + raise JSONValueError("array", value) - next(self._stream) + try: + for member_value in value: + self._value = member_value + consumer(self) + finally: + self._value = value def read_map( self, schema: Schema, - consumer: Callable[[str, "ShapeDeserializer"], None], - ): - event = next(self._stream) - if event.type != "start_map": - raise JSONTokenError("start_map", event) - - key_schema = schema.members["key"] - while self._stream.peek().type != "end_map": - consumer(self.read_string(schema=key_schema), self) - - next(self._stream) - - def _skip(self) -> None: - start = next(self._stream) - if start.type not in ("start_map", "start_array"): - return - - end_type = "end_map" if start.type == "start_map" else "end_array" - - while ( - event := next(self._stream) - ).path != start.path or event.type != end_type: - continue + consumer: Callable[[str, ShapeDeserializer], None], + ) -> None: + value = self._value + if not isinstance(value, Mapping): + raise JSONValueError("object", value) + + try: + for key, member_value in value.items(): + self._value = member_value + consumer(key, self) + finally: + self._value = value + + def _members_by_wire_name(self, schema: Schema) -> Mapping[str, Schema]: + if not self._settings.use_json_name: + return schema.members + + members = self._json_names.get(schema.id) + if members is None: + members = {} + for member_name, member in schema.members.items(): + json_name = member.get_trait(JSONNameTrait) + members[json_name.value if json_name is not None else member_name] = ( + member + ) + self._json_names[schema.id] = members + return members diff --git a/packages/smithy-json/src/smithy_json/_private/documents.py b/packages/smithy-json/src/smithy_json/_private/documents.py index 7c0138515..9bdede3bd 100644 --- a/packages/smithy-json/src/smithy_json/_private/documents.py +++ b/packages/smithy-json/src/smithy_json/_private/documents.py @@ -42,8 +42,7 @@ def as_blob(self) -> bytes: return b64decode(expect_type(str, self._value)) def as_float(self) -> float: - # Floats are returned as Decimals by default in ijson, which is the correct - # behavior, since technically JSON numbers have arbitrary precision. + # JSON numbers are parsed as Decimals to preserve their arbitrary precision. return float(expect_type(Decimal, self._value)) def as_timestamp(self) -> datetime: diff --git a/packages/smithy-json/tests/unit/test_deserializers.py b/packages/smithy-json/tests/unit/test_deserializers.py index 00a6ed2cd..7e35b3ddb 100644 --- a/packages/smithy-json/tests/unit/test_deserializers.py +++ b/packages/smithy-json/tests/unit/test_deserializers.py @@ -3,6 +3,8 @@ import math from datetime import datetime from decimal import Decimal +from io import BytesIO +from json import JSONDecodeError from typing import Any import pytest @@ -19,6 +21,7 @@ TIMESTAMP, ) from smithy_json import JSONCodec, JSONDocument +from smithy_json._private.deserializers import JSONShapeDeserializer from . import ( JSON_SERDE_CASES, @@ -108,3 +111,30 @@ def test_uses_custom_document() -> None: codec = JSONCodec(document_class=CustomDocument) actual = codec.create_deserializer(b'{"foo": "bar"}').read_document(DOCUMENT) assert isinstance(actual, CustomDocument) + + +@pytest.mark.parametrize( + "source", + [ + b"{}", + BytesIO(b"{}"), + ], +) +def test_deserializer_is_eager(source: bytes | BytesIO) -> None: + deserializer = JSONCodec().create_deserializer(source) + assert isinstance(deserializer, JSONShapeDeserializer) + + +def test_ignores_unknown_structure_members() -> None: + actual = JSONCodec().deserialize( + b'{"unknown":{"nested":[1,2,3]},"stringMember":"value"}', + SerdeShape, + ) + + assert actual.string_member == "value" + + +def test_invalid_json_raises_json_decode_error() -> None: + with pytest.raises(JSONDecodeError): + deserializer = JSONCodec().create_deserializer(b'{"incomplete":') + deserializer.read_document(DOCUMENT) diff --git a/uv.lock b/uv.lock index 3ef01194f..f07a64d38 100644 --- a/uv.lock +++ b/uv.lock @@ -350,25 +350,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] -[[package]] -name = "ijson" -version = "3.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/83/28e9e93a3a61913e334e3a2e78ea9924bb9f9b1ac45898977f9d9dd6133f/ijson-3.3.0.tar.gz", hash = "sha256:7f172e6ba1bee0d4c8f8ebd639577bfe429dee0f3f96775a067b8bae4492d8a0", size = 60079, upload-time = "2024-06-06T08:37:13.026Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/4d/3992f7383e26a950e02dc704bc6c5786a080d5c25fe0fc5543ef477c1883/ijson-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:988e959f2f3d59ebd9c2962ae71b97c0df58323910d0b368cc190ad07429d1bb", size = 84550, upload-time = "2024-06-06T08:35:16.756Z" }, - { url = "https://files.pythonhosted.org/packages/1b/cc/3d4372e0d0b02a821b982f1fdf10385512dae9b9443c1597719dd37769a9/ijson-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b2f73f0d0fce5300f23a1383d19b44d103bb113b57a69c36fd95b7c03099b181", size = 57572, upload-time = "2024-06-06T08:35:18.077Z" }, - { url = "https://files.pythonhosted.org/packages/02/de/970d48b1ff9da5d9513c86fdd2acef5cb3415541c8069e0d92a151b84adb/ijson-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0ee57a28c6bf523d7cb0513096e4eb4dac16cd935695049de7608ec110c2b751", size = 56902, upload-time = "2024-06-06T08:35:20.065Z" }, - { url = "https://files.pythonhosted.org/packages/5e/a0/4537722c8b3b05e82c23dfe09a3a64dd1e44a013a5ca58b1e77dfe48b2f1/ijson-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0155a8f079c688c2ccaea05de1ad69877995c547ba3d3612c1c336edc12a3a5", size = 127400, upload-time = "2024-06-06T08:35:21.81Z" }, - { url = "https://files.pythonhosted.org/packages/b2/96/54956062a99cf49f7a7064b573dcd756da0563ce57910dc34e27a473d9b9/ijson-3.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ab00721304af1ae1afa4313ecfa1bf16b07f55ef91e4a5b93aeaa3e2bd7917c", size = 118786, upload-time = "2024-06-06T08:35:23.496Z" }, - { url = "https://files.pythonhosted.org/packages/07/74/795319531c5b5504508f595e631d592957f24bed7ff51a15bc4c61e7b24c/ijson-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40ee3821ee90be0f0e95dcf9862d786a7439bd1113e370736bfdf197e9765bfb", size = 126288, upload-time = "2024-06-06T08:35:25.473Z" }, - { url = "https://files.pythonhosted.org/packages/69/6a/e0cec06fbd98851d5d233b59058c1dc2ea767c9bb6feca41aa9164fff769/ijson-3.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:da3b6987a0bc3e6d0f721b42c7a0198ef897ae50579547b0345f7f02486898f5", size = 129569, upload-time = "2024-06-06T08:35:26.871Z" }, - { url = "https://files.pythonhosted.org/packages/2a/4f/82c0d896d8dcb175f99ced7d87705057bcd13523998b48a629b90139a0dc/ijson-3.3.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:63afea5f2d50d931feb20dcc50954e23cef4127606cc0ecf7a27128ed9f9a9e6", size = 121508, upload-time = "2024-06-06T08:35:28.236Z" }, - { url = "https://files.pythonhosted.org/packages/2b/b6/8973474eba4a917885e289d9e138267d3d1f052c2d93b8c968755661a42d/ijson-3.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b5c3e285e0735fd8c5a26d177eca8b52512cdd8687ca86ec77a0c66e9c510182", size = 127896, upload-time = "2024-06-06T08:35:29.61Z" }, - { url = "https://files.pythonhosted.org/packages/94/25/00e66af887adbbe70002e0479c3c2340bdfa17a168e25d4ab5a27b53582d/ijson-3.3.0-cp312-cp312-win32.whl", hash = "sha256:907f3a8674e489abdcb0206723e5560a5cb1fa42470dcc637942d7b10f28b695", size = 49272, upload-time = "2024-06-06T08:35:31.137Z" }, - { url = "https://files.pythonhosted.org/packages/25/a2/e187beee237808b2c417109ae0f4f7ee7c81ecbe9706305d6ac2a509cc45/ijson-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8f890d04ad33262d0c77ead53c85f13abfb82f2c8f078dfbf24b78f59534dfdd", size = 51272, upload-time = "2024-06-06T08:35:32.38Z" }, -] - [[package]] name = "iniconfig" version = "2.1.0" @@ -781,15 +762,11 @@ provides-extras = ["aiohttp", "awscrt", "cbor"] name = "smithy-json" source = { editable = "packages/smithy-json" } dependencies = [ - { name = "ijson" }, { name = "smithy-core" }, ] [package.metadata] -requires-dist = [ - { name = "ijson", specifier = ">=3.3.0" }, - { name = "smithy-core", editable = "packages/smithy-core" }, -] +requires-dist = [{ name = "smithy-core", editable = "packages/smithy-core" }] [[package]] name = "smithy-python"