diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ca429ec..2ee86abad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,11 @@ jobs: run: | make build-py + - name: Check native Python codegen output + if: ${{ matrix.python-version == '3.12' }} + run: | + make generate-codegen-output-py + - name: Build (and test) java packages run: | make build-java diff --git a/.licenserc.yaml b/.licenserc.yaml index e000755b0..012b76f65 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -15,6 +15,7 @@ header: - 'LICENSE' - 'NOTICE' - 'codegen-output/**' + - 'codegen-output-py/**' - '**/smithy_core/rfc3986.py' language: diff --git a/Makefile b/Makefile index 88aee62b3..5b44d9a89 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL:=help -.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests api-snapshot ci lint-py check-py test-py benchmark build-py clean +.PHONY: help install format-java check-java build-java test-protocols typecheck-protocols generate-protocol-tests generate-codegen-output-py api-snapshot ci lint-py check-py test-py benchmark build-py clean help: ## Show this help. @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) @@ -63,6 +63,19 @@ generate-protocol-tests: ## Generates protocol-test clients, copies them to ./co @echo "codegen-output is up to date." +generate-codegen-output-py: ## Generates Python output and asserts no snapshot diff. + rm -rf codegen-output-py + uv run python scripts/codegen-output-py/generate.py + @if ! git diff --quiet --exit-code -- codegen-output-py || [ -n "$$(git ls-files --others --exclude-standard -- codegen-output-py)" ]; then \ + echo "ERROR: generated codegen-output-py differs from the committed snapshot."; \ + echo "Review the diff and commit it if the change is intended:"; \ + git --no-pager status --short -- codegen-output-py; \ + git --no-pager diff -- codegen-output-py; \ + exit 1; \ + fi + @echo "codegen-output-py is up to date." + + api-snapshot: ## Records the public API surface of each package and asserts no git diff. uv run python scripts/api-snapshot/api_snapshot.py @@ -84,6 +97,7 @@ ci: ## Runs the full CI sequence locally (mirrors the CI workflow steps). make test-py make build-py make build-java + make generate-codegen-output-py make generate-protocol-tests test-protocols make typecheck-protocols diff --git a/codegen-output-py/types/.smithy-python-manifest.json b/codegen-output-py/types/.smithy-python-manifest.json new file mode 100644 index 000000000..9c7015360 --- /dev/null +++ b/codegen-output-py/types/.smithy-python-manifest.json @@ -0,0 +1,6 @@ +{ + "files": { + "src/example_types/_private/schemas.py": "d0f35e8979c4aa9d49a1d9c0c8efeb860420bc0ceab3ba3edcc9488c053e9110" + }, + "version": 1 +} diff --git a/codegen-output-py/types/src/example_types/_private/schemas.py b/codegen-output-py/types/src/example_types/_private/schemas.py new file mode 100644 index 000000000..1192d7ce8 --- /dev/null +++ b/codegen-output-py/types/src/example_types/_private/schemas.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from smithy_core._schema_compact import StaticSchema + +n0 = "example" +_s0 = "GetExampleInput" +_s1 = "smithy.api#input" +_s2 = "id" +_s3 = "TagList" +_s4 = "member" +_s5 = "Attributes" +_s6 = "key" +_s7 = "value" +_s8 = "Value" +_s9 = "text" +_s10 = "number" +_s11 = "empty" +_s12 = "Status" +_s13 = "ACTIVE" +_s14 = "smithy.api#enumValue" +_s15 = "active" +_s16 = "INACTIVE" +_s17 = "inactive" +_s18 = "Code" +_s19 = "OK" +_s20 = "NOT_FOUND" +_s21 = "Example" +_s22 = "name" +_s23 = "count" +_s24 = "tags" +_s25 = "attributes" +_s26 = "choice" +_s27 = "status" +_s28 = "code" +_s29 = "GetExampleOutput" +_s30 = "smithy.api#output" +_s31 = "ExampleError" +_s32 = "smithy.api#error" +_s33 = "client" +_s34 = "message" +_s35 = "retryAfterSeconds" + +GET_EXAMPLE_INPUT: StaticSchema = ( + 18, + n0, + _s0, + {_s1: None}, + (_s2,), + ((3, 1),), +) + +TAG_LIST: StaticSchema = ( + 16, + n0, + _s3, + 0, + (_s4,), + ((3, 0),), +) + +ATTRIBUTES: StaticSchema = ( + 17, + n0, + _s5, + 0, + ( + _s6, + _s7, + ), + ( + (3, 0), + (13, 0), + ), +) + +EXAMPLE_VALUE: StaticSchema = ( + 19, + n0, + _s8, + 0, + ( + _s9, + _s10, + _s11, + ), + ( + (3, 0), + (7, 0), + (-1, 0), + ), +) + +STATUS: StaticSchema = ( + 14, + n0, + _s12, + 0, + ( + _s13, + _s16, + ), + ( + (-1, {_s14: _s15}), + (-1, {_s14: _s17}), + ), +) + +CODE: StaticSchema = ( + 15, + n0, + _s18, + 0, + ( + _s19, + _s20, + ), + ( + (-1, {_s14: 200}), + (-1, {_s14: 404}), + ), +) + +RENAMED_EXAMPLE: StaticSchema = ( + 18, + n0, + _s21, + 0, + ( + _s22, + _s23, + _s24, + _s25, + _s26, + _s27, + _s28, + ), + ( + (3, 1), + (-5, 0), + (TAG_LIST, 0), + (ATTRIBUTES, 0), + (EXAMPLE_VALUE, 0), + (STATUS, 0), + (CODE, 0), + ), +) + +GET_EXAMPLE_OUTPUT: StaticSchema = ( + 18, + n0, + _s29, + {_s30: None}, + ( + n0, + _s7, + ), + ( + (RENAMED_EXAMPLE, 0), + (EXAMPLE_VALUE, 0), + ), +) + +EXAMPLE_ERROR: StaticSchema = ( + 18, + n0, + _s31, + {_s32: _s33}, + ( + _s34, + _s35, + ), + ( + (3, 0), + (7, 0), + ), +) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index 3c3f6f2ad..2d9943eba 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -33,14 +33,14 @@ generate as follows: * `--service SHAPE_ID` selects a specific service shape. The ID MUST be absolute and MUST NOT identify a member. The shape MUST exist in the model and MUST be a service that is not marked `@mixin`. -* When `--service` is omitted and the model contains exactly one non-mixin - service shape, that service is used. -* When `--service` is omitted and the model contains more than one non-mixin - service shape, the command fails with an invocation error that lists the - candidates. +* For `client`, omitting `--service` selects the sole non-mixin service. Multiple + services produce an invocation error that lists the candidates. A model + without a service also fails. +* For `types`, omitting `--service` does not infer one. Selection starts from + eligible declarations across the model even when the model contains one or + more services. -The `client` artifact requires a resolved service. The `types` artifact does -not. The CLI MUST NOT synthesize a placeholder service to satisfy generation. +The CLI MUST NOT synthesize a placeholder service to satisfy generation. ### Generated Shapes @@ -50,16 +50,15 @@ resources, errors, and members. This matches the surface produced by the other Smithy code generators. Data shapes in the model that are not connected to the service are not generated, and the CLI reports how many were left out. -When no service is resolved, the `types` artifact generates every data shape in -the model. Smithy guarantees case-insensitively unique shape names only within a -service closure, so in this mode the command fails when two shapes have -case-insensitively equal names, identifying the conflicting shape IDs. +When no service is resolved, the `types` artifact starts with eligible +declaration shapes across the model. Its selector and input/output settings +filter those roots before member-target dependencies are added. Smithy +guarantees case-insensitively unique shape names only within a service closure, +so the command fails when two final declarations have case-insensitively equal +names, identifying the conflicting shape IDs. Trait definitions, prelude shapes, and shapes marked `@mixin` are never -generated. Builds that need a different set of shapes, such as types that are -not bound to any operation, apply smithy-build transforms in the projection. -An option to generate every shape in the model regardless of the service MAY be -added when there is a need for it. +generated. Projection transforms apply before the artifact's selector. The command MUST return zero after successful generation and non-zero when arguments, settings, the model, or generation are invalid. Diagnostics are diff --git a/designs/codegen/index.md b/designs/codegen/index.md index 1cc06e48e..5455a37b9 100644 --- a/designs/codegen/index.md +++ b/designs/codegen/index.md @@ -62,5 +62,6 @@ behavior of generated packages. * [Code Generator CLI](cli.md) * [Service and Data-Shape Selection](selection.md) -* [Native Python Symbols](symbols.md) -* [Native Python Writer](writer.md) +* [Python Symbols](symbols.md) +* [Python Writer](writer.md) +* [Standalone Types Generation](types.md) diff --git a/designs/codegen/selection.md b/designs/codegen/selection.md index ffb35f5b5..8bf34871f 100644 --- a/designs/codegen/selection.md +++ b/designs/codegen/selection.md @@ -1,11 +1,11 @@ # Service and Data-Shape Selection Selection follows [model loading](model.md) and precedes generation. The CLI's -[service selection rules](cli.md#service-selection) apply to both artifacts. +[service selection rules](cli.md#service-selection) differ by artifact. `--service` accepts only an absolute, existing, non-member, non-mixin service ID. -Without it, the sole non-mixin service is used; ambiguity lists the candidates -in model order. Clients require a service, while standalone types do not. -These selection failures are invocation errors (exit 2). +Clients infer a sole service when the option is absent. Standalone types use a +service only when the option is explicit. Selection failures are invocation +errors (exit 2). ## API and traversal @@ -15,6 +15,22 @@ require_service=False)` returns a frozen `Selection` containing the resolved `require_service=True`. The API has no I/O and reuses the loaded shape objects. `resolve_service` is also available for consumers needing only resolution. +Standalone types use: + +```python +select_type_shapes( + model, + *, + service_id=None, + selector=Selector.identity(), + generate_inputs_and_outputs=False, +) +``` + +This function does not infer a service. An explicit service establishes the +initial service closure. Without one, every eligible declaration shape in the +model is an initial candidate. + A service closure uses an iterative worklist and a visited-ID set. Its edges are: * Service operations, resources, and errors. @@ -35,17 +51,36 @@ Filtering eligible shapes in original model order preserves that order rather than worklist or sorted order. The excluded count includes only eligible data shapes outside the closure; the CLI reports it to stderr only when nonzero. -## Types without a service +## Standalone types + +Standalone types selection proceeds in four stages: + +1. Select declaration candidates from an explicit service closure or the + complete model. +2. Intersect the candidates with the selector result. +3. Exclude input and output structures as roots unless + `generate_inputs_and_outputs=True`. +4. Add every member-target dependency reachable from the remaining + declarations. + +Declaration candidates are structures, unions, enums, and integer enums. +Prelude shapes, mixins, trait definitions, and shapes marked `@private` are +ineligible. Dependency closure may add simple and collection shapes needed by +schemas and serde. Final shapes retain model order. + +Final declaration names are grouped case-insensitively. A conflicting group +produces a `ModelError` listing every conflicting ID (exit 1). Dependency-only +shapes do not participate in this check. + +## Selectors -All eligible data shapes are selected. Names are grouped case-insensitively; -any conflicting groups produce a `ModelError` listing every conflicting ID -(exit 1). Ineligible shapes do not participate. This check is intentionally not -run when a service resolves: Smithy validation is responsible for service name -semantics, and selection is not a general validator or symbol provider. +`Selector.parse(source)` builds an immutable syntax tree for the Smithy selector +language. Evaluation uses the loaded semantic model and returns shape IDs. +Parsing occurs once while resolving `TypesSettings`. Selection intersects the +result with eligible declaration candidates and does not reorder either set. -## Current boundary +Selector conformance fixtures run the same expressions and models through the +Java and Python implementations. The selected shape ID sets must match. -Successful selection still reaches the generation-not-implemented diagnostic -(exit 1). No files or output directories are created, and stdout remains clean. -The generator has no new dependencies; selection neither interprets selectors -nor introduces runtime schemas, naming, or writers. +Selection has no output side effects. It returns model objects for generation +and does not construct symbols, schemas, or writers. diff --git a/designs/codegen/symbols.md b/designs/codegen/symbols.md index e81acd92b..33daae01d 100644 --- a/designs/codegen/symbols.md +++ b/designs/codegen/symbols.md @@ -1,4 +1,4 @@ -# Native Python symbols +# Python symbols `SymbolProvider` answers three questions for the Python generator: @@ -123,13 +123,18 @@ identically named types from different modules also remain distinct. ## Type references -`TypeReference(name, module=None, arguments=(), nullable=False)` is an immutable, -hashable description of a type, not a Python annotation string. `name` and -`module` identify the type; `arguments` holds its element, key or value types. -`nullable=True` means the value can also be `None`. `module` is the public module -users import the type from. If the writer later places a definition in a different -physical module, that placement is the writer's concern and does not change the -reference. Resolved references are cached for the provider's lifetime. +`TypeReference(name, module=None, arguments=(), nullable=False, +dependencies=())` is an immutable, hashable description of a type, not a Python +annotation string. `name` and `module` identify the type. `arguments` holds its +element, key, or value types. `nullable=True` means the value can also be +`None`. `dependencies` contains the package requirements needed to use the +reference. + +`module` is the public module users import the type from. Dependency records +supply package names and versions. Module names only control imports. If the +writer later places a definition in a different physical module, that placement +is the writer's concern and does not change the reference. Resolved references +are cached for the provider's lifetime. For example, `list[str]` is represented as: @@ -137,6 +142,12 @@ For example, `list[str]` is represented as: TypeReference("list", "builtins", (TypeReference("str", "builtins"),)) ``` +References to standard-library and generated modules have no dependencies. A +runtime reference such as `smithy_core.documents.Document` carries the +compatible `smithy-core` dependency from the generator's runtime package +catalog. Collection references retain the dependencies of their nested +arguments. + | Smithy type | Python value type | | ----------------------------------------------- | -------------------------------- | | boolean | bool | @@ -182,9 +193,9 @@ Collection expansion does not use Python recursion. A cycle made entirely of collections, such as two lists containing each other, raises `ModelError` with the IDs in the cycle. A failed lookup does not affect later lookups. -Streaming types are deferred until their Python interfaces are defined. The -provider rejects streaming blobs and event-stream unions rather than treating -them as ordinary values. +Streaming blobs resolve to `smithy_core.aio.interfaces.StreamingBlob`, including +when nested in collections. Event-stream unions retain schema names for +protocol integration and do not produce model declarations. ## Inputs and errors diff --git a/designs/codegen/types.md b/designs/codegen/types.md new file mode 100644 index 000000000..9f7135dc3 --- /dev/null +++ b/designs/codegen/types.md @@ -0,0 +1,507 @@ +# Standalone Types Generation + +## Abstract + +This document describes how the Python generator produces standalone types +packages. It defines package settings, shape selection, generation context, +schema and model emission, dependency tracking, and output publication. +Generated packages expose lazy models backed by compact, lazily hydrated +schemas. + +## Goals + +* The `smithy-python generate types` command produces an importable standalone + types package. +* Generation reuses the loaded model, selection, symbol provider, and writers. +* Generated packages expose compatible public modules, declarations, + constructors, schemas, and serde behavior. +* Schemas use compact encoding and hydrate on first use. +* Model classes are constructed and cached on first access. +* Model analysis, symbol resolution, and writer work are computed once and + reused. +* Output is deterministic on every supported generator host. +* Every generated file is validated before the output directory is modified. +* The generation context and output APIs are reusable by other artifacts. + +Generated packages use `smithy-core` schema and model runtime APIs. + +## Compatibility + +| Area | Required behavior | +| --- | --- | +| Package | Preserve the configured package name, version, metadata, and runtime dependencies. | +| Public modules | Preserve import paths such as `.models`. | +| Models | Preserve declaration names, field names, constructor signatures, defaults, error inheritance, enum values, and union variants. | +| Schemas | Preserve shape IDs, shape types, traits, member order, member indexes, and targets. | +| Serde | Preserve generated and runtime serialization and deserialization behavior. | +| Typing | Generated packages must pass strict Pyright checks and expose the same public types. | +| Runtime support | Generated packages depend on handwritten runtime packages, not `smithy-python`. | + +Model classes are built on demand through `ModelRegistry`, and their static +surface is declared in `models.pyi`. An explicit import such as +`from example.models import Widget` returns a class with the expected public +name, module, fields, constructor, and serde behavior. The class is cached after +first access, so later imports and attribute reads return the same object. + +The lazy facade preserves explicit imports, star imports, `dir(models)`, class +identity, dataclass inspection, and pickling by module and qualified name. +Concurrent first access returns one class object. Packages generated without a +service still provide `ServiceError` and use it as the base of modeled errors. + +`models.pyi` defines the complete static typing surface. Runtime classes expose +generated field names in `__annotations__` and enter `models.__dict__` when +first accessed. Class-construction errors surface from that first access. + +Files under `_private` are not public API. The generator may change their layout +if generated public modules and runtime behavior stay compatible. + +## Pipeline + +Generation uses one explicit pipeline: + +```text +process inputs + | + v +artifact settings + | + v +model loading and shape selection + | + v +generation plan and symbol provider + | + v +artifact generator + | + v +registered writers and text files + | + v +staged validation and output commit +``` + +The CLI selects the artifact generator. Artifact generators share the context +and output APIs but control their own generation phases. + +## Settings + +`ArtifactSettings` contains settings shared by generated artifacts: + +```python +@dataclass(frozen=True, slots=True) +class ArtifactSettings: + package: str + package_version: str + package_description: str | None + + +@dataclass(frozen=True, slots=True) +class TypesSettings(ArtifactSettings): + selector: Selector = field(default_factory=Selector.identity) + generate_inputs_and_outputs: bool = False +``` + +The resolved service remains on `Selection`. It is not duplicated in settings. +The CLI subcommand identifies the artifact, so settings do not duplicate that +value. Artifact-specific settings extend the shared frozen dataclass. + +The `types` command accepts: + +* `--package` +* `--package-version` +* `--package-description` +* `--selector` +* `--generate-inputs-and-outputs` + +`package` and `package-version` are required. The package identifies the +generated import package and follows the validation rules in `SymbolProvider`. +The description defaults to `" types package"`. + +The selector setting is parsed once into an immutable selector syntax tree. +The parser and evaluator implement the Smithy selector language and produce the +same selected shape IDs as Java's `Selector.parse()` and `Selector.select()`. + +Types generation never infers a service. The options apply in this order: + +1. An explicit `--service` establishes the service closure. Without it, every + eligible declaration shape is a candidate. +2. `--selector` narrows those declaration shapes. The default identity selector + keeps every candidate. +3. Input and output structures are removed as roots unless + `--generate-inputs-and-outputs` is set. +4. Member-target dependencies of the remaining declarations are added + transitively. + +Filtering the final closure in model order keeps output deterministic. Name +collision checks run against the final declaration set. + +Invocation through the Smithy `run` plugin and environment-variable precedence +follow the [Code Generator CLI](cli.md). Projection transforms remain +supported, but do not replace settings compatibility. + +Unknown settings and invalid combinations fail before output is created. + +## Generation Context + +`GenerationContext` contains the state shared by one artifact generation: + +```python +@dataclass(frozen=True, slots=True) +class GenerationContext[SettingsT: ArtifactSettings]: + model: Model + selection: Selection + settings: SettingsT + symbols: SymbolProvider + plan: GenerationPlan + dependencies: DependencySet + files: FileManifest +``` + +The context is constructed once after model loading and selection. Generators +receive the context directly. They do not reload the model, construct their own +symbol providers, or inspect files written by another generator. + +`TypesGenerator` receives `GenerationContext[TypesSettings]`. Other artifacts +can define their own settings subclasses without casts or a shared union of +artifact settings. + +`GenerationPlan` composes focused plans for each emitter. + +`SchemaPlan` contains shapes in schema emission order, filtered trait values, +schema constant names, and interned schema strings. +`ModelPlan` contains declaration names, rename mappings, and union variant and +unknown-class names. + +Planning uses iterative graph algorithms so model depth does not consume the +Python stack. The symbol provider caches resolved `TypeReference` values. + +The plan contains semantic data, not rendered source. Emitters can partition +files later without recomputing shape relationships. + +## Dependencies + +Generated imports and package requirements use the same dependency record: + +```python +@dataclass(frozen=True, slots=True) +class Dependency: + package: str + specifier: str + group: Literal["runtime", "build", "test", "docs"] + extras: tuple[str, ...] = () +``` + +These records live in `smithy_python.dependencies` so symbols, writers, and +package generation share one representation. + +`TypeReference` gains a tuple of dependencies. `PythonWriter` records +dependencies from every reference it writes. Emitters call +`DependencySet.add()` for requirements that have no matching Python reference, +such as the generated package's build backend. + +`DependencySet` keys entries by package and group. Repeated entries must use the +same version specifier. The set merges and sorts extras, then fails on a +specifier conflict. Package setup reads this set after all emitters finish. +This matches the dependency flow used by the Java generator without inferring +package versions from import names. Build dependencies render under +`[build-system].requires`. Runtime dependencies render under +`[project].dependencies`. + +The generator owns a catalog of compatible Smithy runtime packages. The +`smithy-core` requirement selected by that catalog must support the compact +schema encoding emitted by the generator. + +## Artifact Generators + +An artifact generator has one method: + +```python +class ArtifactGenerator[SettingsT: ArtifactSettings](Protocol): + def generate(self, context: GenerationContext[SettingsT]) -> None: ... +``` + +The CLI maps `types` to `TypesGenerator`. Unknown artifacts fail during argument +parsing. `TypesGenerator` implements +`ArtifactGenerator[TypesSettings]`. + +`TypesGenerator` runs these phases: + +1. Emit schemas. +2. Emit model declarations and serde adapters. +3. Emit package initialization and typing metadata. +4. Emit package metadata and documentation. + +Each phase records output in `FileManifest`. No phase writes directly to disk. +After generation, the CLI asks the manifest to stage, validate, and publish the +artifact. + +## Output Management + +`FileManifest` owns every file produced by one generation: + +```python +class FileManifest: + def python_module(self, path: str, module: str) -> PythonWriter: ... + def text_file(self, path: str, content: str) -> None: ... + def commit(self, output: Path) -> None: ... +``` + +Paths are relative POSIX paths. Absolute paths, empty components, `.` and `..` +are rejected. Two generators cannot claim the same normalized path. + +`python_module` returns the same writer when the same path and module are +requested again. A second request with a different module fails. This supports +separate emitters contributing to one module without creating separate import +planners. + +Commit uses a temporary staging directory adjacent to the output directory: + +1. Validate every registered path and create the staging directory. +2. Render, validate, hash, and stage one file at a time. +3. Publish current files, reconcile stale owned files, and write the artifact + manifest last. + +The rendered string is released after its staged file is written. Peak rendered +source memory is bounded by the largest output file instead of the complete +artifact. Validation compiles Python modules and checks text-file invariants. A +syntax, import-planning, or staging error removes the staging directory and +leaves the existing output untouched. + +Publication uses `os.replace` to atomically update each file. Commit requires +exclusive access to the configured output directory. + +The output directory contains a generated artifact manifest with normalized +paths and content hashes. Current generated paths are authoritative. Commit +replaces a current path when the previous manifest owns it, even if its contents +changed. If a current path already exists and the previous manifest does not own +it, commit fails with an output collision. + +On a later run, the generator may delete a stale file only when the previous +manifest owns it and its content still matches the recorded hash. Modified stale +files are retained and are no longer owned by the manifest. Files absent from +the manifest are retained unless they collide with a current generated path. + +Callers use a dedicated, generator-managed output directory and keep custom +files at non-conflicting paths. Initial generation starts with no files at +generated destinations. + +The Smithy `run` plugin output restriction still applies. Symlinks or resolved +ancestors that escape the output root are rejected. + +## Schema Emission + +The generator emits the compact schema format consumed by +`smithy_core._schema_compact` in `._private.schemas`. + +Each generated schema is module-level immutable data: + +```python +WIDGET: StaticSchema = ( + ShapeType.STRUCTURE.value, + "example", + "Widget", + 0, + ("name",), + ((ShapeType.STRING.value, 1),), +) +``` + +The actual output uses integer shape types, trait bit vectors, interned strings, +and the tuple layout accepted by `smithy_core._schema_compact`. The example is +expanded for readability. + +### Compact Encoding Contract + +The generator and runtime share a versioned compact encoding contract without +adding a runtime dependency from `smithy-python` to `smithy-core`. + +One generator module owns: + +* The encoding version. +* Shape tuple slots and arity. +* Shape type integers and prelude sentinels. +* The ordered trait bit table. +* Trait filtering and value encoding rules. + +`smithy_core._schema_compact` exposes the matching private encoding version. +Generated tuples do not include a version field. A change to tuple layout, +shape type integers, prelude sentinels, or trait bit positions increments the +version and requires a coordinated `smithy-core` dependency update. + +Trait planning applies the Java generator's `SchemaTraitFilterIndex` rules to +each emitted data shape. Generated schemas contain the same trait IDs and values +for the same model. Input and output traits are synthesized under the same +conditions. A schema uses the integer trait bit vector only when every present +trait is represented by a valueless bit. Otherwise it emits the complete trait +mapping. + +The emitter preserves these properties: + +* Importing the schema module does not construct `Schema` objects. +* `hydrate` constructs each connected schema graph on first use and caches it. +* Prelude targets use integer sentinels instead of duplicate schema data. +* Repeated namespaces, names, trait IDs, and string trait values use module + constants. +* Valueless two-state traits use a bit vector. +* Members use model order and receive dense indexes from zero. +* References to schemas already emitted use direct tuple references. +* Forward and recursive references use thunks. + +Schema ordering is computed once from the selected model. Non-recursive shapes +use topological order. Strongly connected components follow in stable model +order. The emitter must produce the same member order and schema semantics as +the Java generator even when its private constant names differ. + +## Model Emission + +Standalone types use `ModelRegistry` to construct model classes on demand: + +* `models.py` contains a `ModelRegistry` and module `__getattr__`. +* `models.pyi` declares the complete typed surface. +* The generated registry receives its schema constant index, rename map, and + complete shape-name set. +* `models.py` computes `__all__` from the public model index. +* Resolving one model does not scan or hydrate every schema. +* A reentrant lock protects first construction and the class cache. +* Runtime classes report `.models` as their module and use a stable + qualified name. +* `__dir__` returns module globals and generated names without constructing + classes. + +The generated stub declares structures, modeled errors, enums, integer enums, +union variants, unknown union variants, and serde methods. Static tooling does +not import or hydrate runtime classes to discover this surface. + +`ModelRegistry` attaches the same constructor defaults, field metadata, error +inheritance, enum values, union behavior, and serde methods that concrete +generated declarations provide. + +The module computes `__all__` instead of emitting a literal list of unresolved +model names: + +```python +_MODEL_NAMES = tuple(_MODEL_INDEX) +__all__ = ("ServiceError", *_MODEL_NAMES) + + +def __dir__() -> list[str]: + return sorted({*globals(), *__all__}) +``` + +This keeps Ruff from reporting `F822` for lazy exports. Star imports read +`__all__` and resolve each model through module `__getattr__`. Computing +`__dir__` from the index does not construct model classes. + +Model member access uses the schema's dense member indexes where positional +dispatch is required. Name maps remain dictionaries where lookup is by a wire +or Python name. The generator does not replace sparse name lookups with arrays +that require an additional name-to-index lookup. + +## Package Files + +The `types` artifact contains: + +* `pyproject.toml` +* `README.md` +* `src//__init__.py` +* `src//models.py` +* `src//models.pyi` +* `src//_private/__init__.py` +* `src//_private/schemas.py` +* `src//py.typed` + +Package setup renders the dependency set collected during generation. + +## Large Models + +Schema and stub partitioning must not change public imports. The output API and +generation plan allow private content to move behind stable facade modules. + +smithy-java partitions schema classes after a model reaches 500 schema fields +and then targets about 50 KB of generated schema content per class. smithy-php +uses a centralized lazy schema registry. Python can combine those approaches: + +* Keep `._private.schemas` as the stable facade. +* Place compact schema tuples in private partition modules. +* Keep every recursive strongly connected component in one partition. +* Route schema constant access through generated indexes. +* Partition large type stubs behind the public `models.pyi` facade. + +Python partitioning thresholds must be based on Python measurements. Relevant inputs include writer render time, peak generator memory, generated import time, and type +checker time. + +Partitioning bounds the syntax tree analyzed by `PythonWriter` and prevents a +single EC2-sized source file from dominating codegen memory. Measurements +establish the partitioning thresholds. Facade modules preserve generated +imports. + +## Performance + +| Optimization | Generator behavior | +| --- | --- | +| Compact schema tuples | Emit the versioned tuple encoding consumed by `smithy-core`. | +| Lazy schema hydration | Hydrate and cache a connected schema graph on first use. | +| Lazy model construction | Use an indexed registry to construct standalone types. | +| Cached symbol resolution | Reuse one symbol provider and cached type references. | +| Dense serde dispatch | Preserve member order and dense member indexes. | + +Generation loads and selects the model once, computes graph data once, interns +schema strings once, and renders each writer once. Staging bounds rendered +source memory to one file. + +Measurements record generator time and peak memory, generated bytes and file +counts, package import cost, first schema and model access, warm serde +throughput, and strict Pyright time. The test set includes a small fixture, the +standalone types integration-test model, and a large AWS model such as EC2. + +## Validation + +Tests cover the compatibility table, selected shape IDs, compact encoding +version and mappings, dependency sets, serde results, deterministic output, +concurrent model construction, star imports, pickle round trips, and generation +without a service. Generated packages must compile on every supported Python +version and pass Ruff and strict Pyright. + +## Alternatives + +### Port the Java Directed Codegen Framework + +The Java framework supplies a lifecycle for shape and integration hooks. Porting +it would make the types artifact depend on extension points that have no Python +consumer. The explicit artifact generator keeps the design smaller. Shared +phases can become hooks after another artifact demonstrates the required +lifecycle. + +### Write Files During Generation + +Direct writes use less temporary memory, but a late syntax or import failure +leaves a partial artifact. Per-file staging bounds memory while preserving a +validation boundary before publication. + +### Import Schema Encoding from smithy-core + +Importing tuple slots and trait bits from `smithy-core` would remove duplicated +constants. It would also add a generator-host runtime dependency and couple the +generator to the runtime selected for its output. A local versioned contract +keeps that boundary explicit. + +### Build Runtime Schemas During Import + +Emitting constructed `Schema` objects would reduce hydration code, but shifts +the cost to every package import. Compact static data preserves cold-start +behavior and shares the runtime cache. + +### Preserve Eager Standalone Models + +Eager standalone declarations would place every class in `models.__dict__` +immediately after import. They would also require a second model emitter and +construct every class during package import. The generator uses one lazy +representation and exposes supported behavior through the public facade and +typed stub. + +### Require Byte-Identical Java Output + +Byte-identical output would couple the Python implementation to Java writer +formatting and internal helper names. Behavioral parity protects users while +allowing Python source organization and import planning to evolve. diff --git a/designs/codegen/writer.md b/designs/codegen/writer.md index e4bc75bd3..1c581d1d1 100644 --- a/designs/codegen/writer.md +++ b/designs/codegen/writer.md @@ -1,9 +1,10 @@ -# Native Python writer +# Python writer `PythonWriter` turns lines of Python text and `TypeReference` values into the source of one module. Pass references from `SymbolProvider.type_reference()` -or construct them directly. The writer chooses annotation spellings and imports. -The caller decides which declarations, fields and defaults to write. +or construct them directly. The writer chooses annotation spellings and +imports, and can record package dependencies from references. The caller +decides which declarations, fields, and defaults to write. ```python from smithy_python.symbols import TypeReference @@ -33,8 +34,9 @@ class Node: ## Writing a module -`PythonWriter(module)` takes the destination module name. `render()` finds the -names bound by the written lines and follows Python's scoping rules. +`PythonWriter(module, dependencies=None)` takes the destination module name and +an optional shared `DependencySet`. `render()` finds the names bound by the +written lines and follows Python's scoping rules. Module-level names, such as classes, functions, assignments, imports and `global` assignments, apply to every reference. A local name, such as a field, parameter or local variable, applies only to references that can see it. A @@ -115,6 +117,22 @@ alias. Referenced packages don't have to be installed in the generator's environ For example, rendering a reference to `smithy_core.documents.Document` does not import `smithy_core`. +## Dependencies + +When a dependency set is provided, `line()` records dependencies from each +`TypeReference` and its nested arguments. Recording happens when the reference +is written so package metadata can be generated before source rendering. +Repeated dependency records are deduplicated by `DependencySet`. + +Imports and package requirements remain separate. A module name controls the +generated import, while a dependency record controls the generated package +metadata. Annotation-only references still contribute dependencies because +type checking a generated package must resolve them. + +Writers created by `FileManifest.python_module()` share the generation +context's dependency set. A standalone writer without a dependency set renders +without recording dependencies. + ## Names that collide An import uses a module-derived alias when its name matches a module-level @@ -140,8 +158,8 @@ Equivalent references share one import, using the lexicographically smallest supplied module/name pair so the choice does not depend on reference order. A same-module reference whose name is a local name visible where it is -referenced raises `CodegenError`. This first version does not add self-imports -or aliases for local declarations. +referenced raises `CodegenError`. The writer does not add self-imports or aliases +for local declarations. The writer returns source in memory. File output, documentation conversion and actual declaration generation remain separate follow-ups. diff --git a/packages/smithy-core/src/smithy_core/_schema_compact.py b/packages/smithy-core/src/smithy_core/_schema_compact.py index 841ee3da7..bcd037692 100644 --- a/packages/smithy-core/src/smithy_core/_schema_compact.py +++ b/packages/smithy-core/src/smithy_core/_schema_compact.py @@ -28,7 +28,7 @@ import threading from collections.abc import Callable -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Final, cast from . import prelude from .schemas import Schema @@ -38,6 +38,8 @@ if TYPE_CHECKING: from .documents import DocumentValue +_ENCODING_VERSION: Final = 2 + type StaticSchema = tuple[Any, ...] # A member target: a prelude int, a sibling tuple, or a lambda returning one. A built # Schema is also accepted so an operation can be hand-initialized with real schemas. @@ -66,10 +68,28 @@ ShapeType.DOCUMENT.value: prelude.DOCUMENT, } -# The UNIT prelude schema, referenced by enum members. Given its own sentinel so it -# does not collide with a simple-type substitute. +# Prelude schemas whose identity or traits differ from the ordinary simple-type +# substitutes use dedicated sentinels. UNIT = -1 -_PRELUDE[UNIT] = prelude.UNIT +PRIMITIVE_BOOLEAN = -2 +PRIMITIVE_BYTE = -3 +PRIMITIVE_SHORT = -4 +PRIMITIVE_INTEGER = -5 +PRIMITIVE_LONG = -6 +PRIMITIVE_FLOAT = -7 +PRIMITIVE_DOUBLE = -8 +_PRELUDE.update( + { + UNIT: prelude.UNIT, + PRIMITIVE_BOOLEAN: prelude.PRIMITIVE_BOOLEAN, + PRIMITIVE_BYTE: prelude.PRIMITIVE_BYTE, + PRIMITIVE_SHORT: prelude.PRIMITIVE_SHORT, + PRIMITIVE_INTEGER: prelude.PRIMITIVE_INTEGER, + PRIMITIVE_LONG: prelude.PRIMITIVE_LONG, + PRIMITIVE_FLOAT: prelude.PRIMITIVE_FLOAT, + PRIMITIVE_DOUBLE: prelude.PRIMITIVE_DOUBLE, + } +) # Bit index -> trait id for valueless two-state traits; OR of bits encodes any # combination. diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-schema-generation.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-schema-generation.json new file mode 100644 index 000000000..ed39c9fc6 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-schema-generation.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added native generation of compact, lazily hydrated schemas with Java-compatible trait filtering, string interning, and recursive references." +} diff --git a/packages/smithy-python/CHANGELOG.md b/packages/smithy-python/CHANGELOG.md index 85d9c7b32..2ca327f21 100644 --- a/packages/smithy-python/CHANGELOG.md +++ b/packages/smithy-python/CHANGELOG.md @@ -4,5 +4,5 @@ ### Features * Added the experimental smithy-python package and CLI scaffold. -* Added service and data-shape selection for the native Python generator, including `--service` for `client` and `types` commands, service closure filtering, and standalone types name conflict diagnostics. +* Added service and data-shape selection for the Python generator, including `--service` for `client` and `types` commands, service closure filtering, and standalone types name conflict diagnostics. * Added a Smithy JSON AST model loader with an embedded prelude, mixin flattening, and apply support. diff --git a/packages/smithy-python/src/smithy_python/_selector.py b/packages/smithy-python/src/smithy_python/_selector.py new file mode 100644 index 000000000..e08b4c850 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/_selector.py @@ -0,0 +1,1293 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Smithy selector parsing and evaluation.""" + +from __future__ import annotations + +import re +from collections import deque +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +from types import MappingProxyType +from typing import Final, NoReturn, Self, cast + +from .exceptions import SelectorSyntaxError +from .model import ( + EnumShape, + IntEnumShape, + MemberShape, + Model, + Node, + OperationShape, + ResourceShape, + ServiceShape, + Shape, + ShapeId, + ShapeType, +) + +type _Receiver = Callable[[Shape], bool] +type _ValueData = ( + None + | bool + | int + | float + | str + | Shape + | ShapeId + | tuple[Node, ...] + | tuple[_Value, ...] + | Mapping[str, Node] +) + +_IDENTIFIER: Final = re.compile(r"(?:[A-Za-z]|_+[A-Za-z0-9])[A-Za-z0-9_]*") +_NUMBER: Final = re.compile(r"-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?") +_NUMBERS: Final = frozenset( + { + ShapeType.BYTE, + ShapeType.SHORT, + ShapeType.INTEGER, + ShapeType.LONG, + ShapeType.FLOAT, + ShapeType.DOUBLE, + ShapeType.BIG_DECIMAL, + ShapeType.BIG_INTEGER, + } +) +_SIMPLE: Final = frozenset( + { + ShapeType.BLOB, + ShapeType.BOOLEAN, + ShapeType.STRING, + ShapeType.TIMESTAMP, + ShapeType.BYTE, + ShapeType.SHORT, + ShapeType.INTEGER, + ShapeType.LONG, + ShapeType.FLOAT, + ShapeType.DOCUMENT, + ShapeType.DOUBLE, + ShapeType.BIG_DECIMAL, + ShapeType.BIG_INTEGER, + ShapeType.ENUM, + ShapeType.INT_ENUM, + } +) +_AGGREGATE: Final = frozenset( + {ShapeType.LIST, ShapeType.MAP, ShapeType.STRUCTURE, ShapeType.UNION} +) +_SERVICE: Final = frozenset( + {ShapeType.SERVICE, ShapeType.RESOURCE, ShapeType.OPERATION} +) +_TOPDOWN_RELATIONSHIPS: Final = frozenset( + { + "operation", + "resource", + "collectionOperation", + "create", + "put", + "read", + "update", + "delete", + "list", + } +) +_ENUM_TRAIT: Final = ShapeId("smithy.api", "enum") +_TRAIT_TRAIT: Final = ShapeId("smithy.api", "trait") +_AUTH_DEFINITION_TRAIT: Final = ShapeId("smithy.api", "authDefinition") +_PRELUDE_AUTH_TRAITS: Final = frozenset( + { + ShapeId("smithy.api", "httpApiKeyAuth"), + ShapeId("smithy.api", "httpBasicAuth"), + ShapeId("smithy.api", "httpBearerAuth"), + ShapeId("smithy.api", "httpDigestAuth"), + } +) +_PRELUDE_TRAIT_TYPES: Final[Mapping[ShapeId, ShapeType]] = { + ShapeId("smithy.api", name): shape_type + for shape_type, names in ( + ( + ShapeType.STRING, + ( + "documentation", + "httpHeader", + "httpPrefixHeaders", + "httpQuery", + "jsonName", + "mediaType", + "pattern", + "resourceIdentifier", + "since", + "title", + "xmlName", + ), + ), + ( + ShapeType.LIST, + ( + "auth", + "createsResources", + "deletesResources", + "enum", + "examples", + "putsResources", + "readsResources", + "references", + "suppress", + "tags", + "updatesResources", + ), + ), + ( + ShapeType.MAP, + ("externalDocumentation", "traitValidators", "unstableFeatures"), + ), + (ShapeType.DOCUMENT, ("default", "enumValue")), + (ShapeType.ENUM, ("error", "timestampFormat")), + (ShapeType.INTEGER, ("httpError",)), + ) + for name in names +} + + +@dataclass(frozen=True, slots=True) +class Selector: + """An immutable parsed Smithy selector.""" + + source: str + _expression: _Expression + _roots: tuple[_Expression, ...] = () + _variable_names: tuple[str, ...] = () + + @classmethod + def identity(cls) -> Self: + """Return a selector that retains every starting shape.""" + return cls("*", _IDENTITY) + + @classmethod + def parse(cls, source: str) -> Self: + """Parse a Smithy selector expression.""" + return cast(Self, _Parser(source).parse(cls)) + + def select(self, model: Model) -> frozenset[ShapeId]: + """Evaluate this selector and return the matching shape IDs.""" + graph = _Graph(model) + roots: list[frozenset[Shape]] = [] + for expression in self._roots: + context = _Context(graph, roots, self._variable_names) + roots.append( + context.collect(expression, graph.shapes, reset_variables=True) + ) + + context = _Context(graph, roots, self._variable_names) + return frozenset( + shape.id + for shape in context.collect( + self._expression, graph.shapes, reset_variables=True + ) + ) + + @property + def is_identity(self) -> bool: + """Whether this selector retains every starting shape.""" + return self._expression is _IDENTITY and not self._roots + + def __str__(self) -> str: + return self.source + + +class _Expression: + __slots__ = () + + def push( + self, context: _Context, shape: Shape, receiver: _Receiver + ) -> bool: # pragma: no cover - abstract dispatch + raise NotImplementedError + + +class _Identity(_Expression): + __slots__ = () + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + return receiver(shape) + + +_IDENTITY: Final = _Identity() + + +@dataclass(frozen=True, slots=True) +class _SequenceExpression(_Expression): + expressions: tuple[_Expression, ...] + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + def apply(index: int, current: Shape) -> bool: + if index == len(self.expressions): + return receiver(current) + return self.expressions[index].push( + context, current, lambda result: apply(index + 1, result) + ) + + return apply(0, shape) + + +@dataclass(frozen=True, slots=True) +class _TypeExpression(_Expression): + types: frozenset[ShapeType] + aliases: frozenset[ShapeType] = frozenset() + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + if shape.type in self.types or shape.type in self.aliases: + return receiver(shape) + return False + + +_TYPE_EXPRESSIONS: Final[Mapping[str, _TypeExpression]] = { + **{ + shape_type.value: _TypeExpression(frozenset({shape_type})) + for shape_type in ShapeType + }, + "number": _TypeExpression(_NUMBERS), + "simpleType": _TypeExpression(_SIMPLE), + "collection": _TypeExpression(frozenset({ShapeType.LIST})), + "set": _TypeExpression(frozenset({ShapeType.LIST})), + "aggregateType": _TypeExpression(_AGGREGATE), + "serviceType": _TypeExpression(_SERVICE), + "dataType": _TypeExpression(_SIMPLE | _AGGREGATE), + "string": _TypeExpression( + frozenset({ShapeType.STRING}), frozenset({ShapeType.ENUM}) + ), + "integer": _TypeExpression( + frozenset({ShapeType.INTEGER}), frozenset({ShapeType.INT_ENUM}) + ), +} + + +@dataclass(frozen=True, slots=True) +class _AnyExpression(_Expression): + expressions: tuple[_Expression, ...] + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + for expression in self.expressions: + if expression.push(context, shape, receiver): + return True + return False + + +@dataclass(frozen=True, slots=True) +class _TestExpression(_Expression): + expressions: tuple[_Expression, ...] + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + if any(context.receives(expression, shape) for expression in self.expressions): + return receiver(shape) + return False + + +@dataclass(frozen=True, slots=True) +class _NotExpression(_Expression): + expression: _Expression + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + if not context.receives(self.expression, shape): + return receiver(shape) + return False + + +@dataclass(frozen=True, slots=True) +class _InExpression(_Expression): + expression: _Expression + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + matched = False + + def find(result: Shape) -> bool: + nonlocal matched + matched = result is shape + return matched + + self.expression.push(context, shape, find) + return receiver(shape) if matched else False + + +@dataclass(frozen=True, slots=True) +class _VariableStoreExpression(_Expression): + slot: int + expression: _Expression + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + context.variables[self.slot] = context.collect(self.expression, (shape,)) + return receiver(shape) + + +@dataclass(frozen=True, slots=True) +class _VariableGetExpression(_Expression): + slot: int + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + return any(receiver(result) for result in context.variable(self.slot)) + + +@dataclass(frozen=True, slots=True) +class _RootExpression(_Expression): + index: int + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + return any(receiver(result) for result in context.roots[self.index]) + + +@dataclass(frozen=True, slots=True) +class _NeighborExpression(_Expression): + forward: bool + labels: frozenset[str] = frozenset() + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + for edge in context.graph.neighbors( + shape, forward=self.forward, include_traits="trait" in self.labels + ): + if (not self.labels or edge.label in self.labels) and receiver(edge.target): + return True + return False + + +class _RecursiveNeighborExpression(_Expression): + __slots__ = () + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + visited = {shape} + pending = deque([shape]) + while pending: + current = pending.popleft() + for edge in context.graph.neighbors( + current, forward=True, include_traits=False + ): + result = edge.target + if result in visited: + continue + visited.add(result) + if receiver(result): + return True + pending.append(result) + return False + + +_RECURSIVE_NEIGHBOR: Final = _RecursiveNeighborExpression() + + +@dataclass(frozen=True, slots=True) +class _RecursiveExpression(_Expression): + expression: _Expression + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + visited: set[Shape] = set() + pending = deque([shape]) + while pending: + current = pending.popleft() + + def enqueue(result: Shape) -> bool: + if result in visited: + return False + visited.add(result) + if receiver(result): + return True + pending.append(result) + return False + + if self.expression.push(context, current, enqueue): + return True + return False + + +@dataclass(frozen=True, slots=True) +class _TopDownExpression(_Expression): + qualifier: _Expression + disqualifier: _Expression | None + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + if shape.type not in _SERVICE: + return False + visited: set[tuple[Shape, bool]] = set() + pending = [(shape, False)] + while pending: + current, qualified = pending.pop() + state = (current, qualified) + if state in visited: + continue + visited.add(state) + if not qualified and context.receives(self.qualifier, current): + qualified = True + if ( + qualified + and self.disqualifier is not None + and context.receives(self.disqualifier, current) + ): + qualified = False + if qualified and receiver(current): + return True + children = [ + edge.target + for edge in context.graph.neighbors( + current, forward=True, include_traits=False + ) + if edge.label in _TOPDOWN_RELATIONSHIPS + ] + pending.extend((child, qualified) for child in reversed(children)) + return False + + +@dataclass(frozen=True, slots=True) +class _AttributeExpression(_Expression): + path: tuple[str, ...] + comparator: str | None + expected: tuple[str, ...] = () + insensitive: bool = False + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + actual = _Value("shape", shape, context).path(self.path) + matches = ( + actual.present + if self.comparator is None + else any( + _compare( + self.comparator, + actual, + _Value.literal(value), + self.insensitive, + ) + for value in self.expected + ) + ) + return receiver(shape) if matches else False + + +@dataclass(frozen=True, slots=True) +class _ScopedOperand: + path: tuple[str, ...] | None = None + literal: str | None = None + + def resolve(self, scope: _Value) -> _Value: + if self.path is not None: + return scope.path(self.path) + assert self.literal is not None # noqa: S101 - parser invariant + return _Value.literal(self.literal) + + +@dataclass(frozen=True, slots=True) +class _ScopedAssertion: + lhs: _ScopedOperand + comparator: str + rhs: tuple[_ScopedOperand, ...] + insensitive: bool + + +@dataclass(frozen=True, slots=True) +class _ScopedAttributeExpression(_Expression): + path: tuple[str, ...] + assertions: tuple[_ScopedAssertion, ...] + + def push(self, context: _Context, shape: Shape, receiver: _Receiver) -> bool: + value = _Value("shape", shape, context).path(self.path) + if not value.present: + return False + for scope in value.flatten(): + if all( + any( + _compare( + assertion.comparator, + assertion.lhs.resolve(scope), + rhs.resolve(scope), + assertion.insensitive, + ) + for rhs in assertion.rhs + ) + for assertion in self.assertions + ): + return receiver(shape) + return False + + +@dataclass(frozen=True, slots=True) +class _Edge: + target: Shape + label: str + + +class _Graph: + __slots__ = ("_forward", "_reverse", "_trait_forward", "_trait_reverse", "shapes") + + def __init__(self, model: Model) -> None: + shapes: list[Shape] = [] + for shape in model.iter_shapes(include_prelude=True): + shapes.append(shape) + shapes.extend(shape.members.values()) + virtual_traits = _virtual_trait_shapes(model, shapes) + self.shapes = (*shapes, *virtual_traits.values()) + self._forward: dict[Shape, tuple[_Edge, ...]] = {} + reverse: dict[Shape, list[_Edge]] = {} + trait_forward: dict[Shape, list[_Edge]] = {} + trait_reverse: dict[Shape, list[_Edge]] = {} + + for shape in self.shapes: + edges = ( + () + if shape.id in virtual_traits + else tuple(self._shape_edges(model, shape)) + ) + self._forward[shape] = edges + for edge in edges: + reverse.setdefault(edge.target, []).append(_Edge(shape, edge.label)) + for trait_id in _trait_ids(shape): + target = model.find_shape(trait_id) or virtual_traits.get(trait_id) + if target is None: + continue + trait_forward.setdefault(shape, []).append(_Edge(target, "trait")) + trait_reverse.setdefault(target, []).append(_Edge(shape, "trait")) + + self._reverse = {shape: tuple(reverse.get(shape, ())) for shape in self.shapes} + self._trait_forward = { + shape: tuple(trait_forward.get(shape, ())) for shape in self.shapes + } + self._trait_reverse = { + shape: tuple(trait_reverse.get(shape, ())) for shape in self.shapes + } + + def neighbors( + self, shape: Shape, *, forward: bool, include_traits: bool + ) -> tuple[_Edge, ...]: + direct = self._forward[shape] if forward else self._reverse[shape] + if not include_traits: + return direct + traits = self._trait_forward[shape] if forward else self._trait_reverse[shape] + return (*direct, *traits) + + @staticmethod + def _shape_edges(model: Model, shape: Shape) -> Iterable[_Edge]: + for mixin in shape.mixins: + target = model.find_shape(mixin) + if target is not None and not isinstance(shape, MemberShape): + yield _Edge(target, "mixin") + if isinstance(shape, MemberShape): + container = model.get_shape(shape.container) + if not isinstance(container, EnumShape | IntEnumShape): + yield _Edge(model.get_shape(shape.target), "") + return + if isinstance(shape, ServiceShape): + for target in shape.operations: + yield _Edge(model.get_shape(target), "operation") + for target in shape.resources: + yield _Edge(model.get_shape(target), "resource") + for target in shape.errors: + yield _Edge(model.get_shape(target), "error") + return + if isinstance(shape, ResourceShape): + for target in shape.identifiers.values(): + yield _Edge(model.get_shape(target), "identifier") + for target in shape.properties.values(): + yield _Edge(model.get_shape(target), "property") + for target in shape.resources: + yield _Edge(model.get_shape(target), "resource") + for label in ("list", "create", "put", "read", "update", "delete"): + target_id = getattr(shape, label) + if target_id is not None: + yield _Edge(model.get_shape(target_id), label) + for target in shape.operations: + yield _Edge(model.get_shape(target), "operation") + for target in shape.collection_operations: + yield _Edge(model.get_shape(target), "collectionOperation") + return + if isinstance(shape, OperationShape): + if shape.input != ShapeId("smithy.api", "Unit"): + yield _Edge(model.get_shape(shape.input), "input") + if shape.output != ShapeId("smithy.api", "Unit"): + yield _Edge(model.get_shape(shape.output), "output") + for target in shape.errors: + yield _Edge(model.get_shape(target), "error") + return + for member in shape.members.values(): + yield _Edge(member, "member") + + +class _Context: + __slots__ = ("_variable_indices", "graph", "roots", "variables") + + def __init__( + self, + graph: _Graph, + roots: Sequence[frozenset[Shape]], + variable_names: tuple[str, ...], + ) -> None: + self.graph = graph + self.roots = roots + self.variables: list[frozenset[Shape] | None] = [None] * len(variable_names) + self._variable_indices = { + name: index for index, name in enumerate(variable_names) + } + + def clear_variables(self) -> None: + self.variables[:] = [None] * len(self.variables) + + def variable(self, slot: int) -> frozenset[Shape]: + return self.variables[slot] or frozenset() + + def variable_by_name(self, name: str) -> frozenset[Shape]: + slot = self._variable_indices.get(name) + if slot is None: + return frozenset() + return self.variable(slot) + + def collect( + self, + expression: _Expression, + shapes: Iterable[Shape], + *, + reset_variables: bool = False, + ) -> frozenset[Shape]: + """Collect every result produced by an expression.""" + matches: set[Shape] = set() + + def capture(result: Shape) -> bool: + matches.add(result) + return False + + for shape in shapes: + if reset_variables: + self.clear_variables() + expression.push(self, shape, capture) + return frozenset(matches) + + def receives(self, expression: _Expression, shape: Shape) -> bool: + received = False + + def stop(_: Shape) -> bool: + nonlocal received + received = True + return True + + expression.push(self, shape, stop) + return received + + +@dataclass(frozen=True, slots=True) +class _Value: + kind: str + value: _ValueData = None + context: _Context | None = None + + @classmethod + def literal(cls, value: bool | int | float | str) -> _Value: + return cls("literal", value) + + @property + def present(self) -> bool: + if self.kind == "empty": + return False + if self.kind == "projection": + return bool(self.value) + return True + + def path(self, path: Sequence[str]) -> _Value: + current = self + for segment in path: + if not current.present: + break + current = current.get_property(segment) + return current + + def get_property(self, key: str) -> _Value: + if self.kind == "literal": + return ( + _Value.literal(len(self.string)) if key == "(length)" else _EMPTY_VALUE + ) + if self.kind == "node": + if isinstance(self.value, Mapping): + mapping = self.value + if key == "(keys)": + return _Value("projection", tuple(map(_Value.literal, mapping))) + if key == "(values)": + return _Value( + "projection", + tuple(_Value("node", value) for value in mapping.values()), + ) + if key == "(length)": + return _Value.literal(len(mapping)) + return _Value("node", mapping[key]) if key in mapping else _EMPTY_VALUE + if isinstance(self.value, tuple): + values = cast(tuple[Node, ...], self.value) + if key == "(values)": + return _Value( + "projection", + tuple(_Value("node", value) for value in values), + ) + if key == "(length)": + return _Value.literal(len(values)) + return _EMPTY_VALUE + if isinstance(self.value, str) and key == "(length)": + return _Value.literal(len(self.value)) + return _EMPTY_VALUE + if self.kind == "projection": + values = self.flatten() + if key == "(first)": + return values[0] if values else _EMPTY_VALUE + projected = tuple( + result + for value in cast(tuple[_Value, ...], self.value) + if (result := value.get_property(key)).present + ) + return _Value("projection", projected) + if self.kind == "id": + shape_id = self.value + assert isinstance(shape_id, ShapeId) # noqa: S101 - internal invariant + if key == "name": + return _Value.literal(shape_id.name) + if key == "namespace": + return _Value.literal(shape_id.namespace) + if key == "member": + return ( + _Value.literal(shape_id.member) + if shape_id.member is not None + else _EMPTY_VALUE + ) + if key == "(length)": + return _Value.literal(len(str(shape_id))) + return _EMPTY_VALUE + if self.kind == "service": + service = self.value + assert isinstance(service, ServiceShape) # noqa: S101 + if key == "version": + return ( + _Value.literal(service.version) + if service.version is not None + else _EMPTY_VALUE + ) + if key == "id": + return _Value("id", service.id) + return _EMPTY_VALUE + if self.kind == "traits": + shape = self.value + assert isinstance(shape, Shape) # noqa: S101 + traits = _traits(shape) + if key == "(keys)": + return _Value( + "projection", tuple(_Value("id", trait) for trait in traits) + ) + if key == "(values)": + return _Value( + "projection", + tuple(_Value("node", value) for value in traits.values()), + ) + if key == "(length)": + return _Value.literal(len(traits)) + trait_id = _absolute_trait(key) + return ( + _Value("node", traits[trait_id]) if trait_id in traits else _EMPTY_VALUE + ) + if self.kind == "shape": + shape = self.value + assert isinstance(shape, Shape) # noqa: S101 + if key == "trait": + return _Value("traits", shape, self.context) + if key == "id": + return _Value("id", shape.id, self.context) + if key == "service" and isinstance(shape, ServiceShape): + return _Value("service", shape, self.context) + if key == "var": + return _Value("variables", context=self.context) + return _EMPTY_VALUE + if self.kind == "variables": + assert self.context is not None # noqa: S101 + return _Value( + "projection", + tuple( + _Value("shape", shape, self.context) + for shape in self.context.variable_by_name(key) + ), + ) + return _EMPTY_VALUE + + @property + def string(self) -> str: + if self.kind == "empty" or self.kind == "projection": + return "" + if self.kind in {"id", "shape", "service"}: + return str(self.value.id if isinstance(self.value, Shape) else self.value) + if self.kind == "node": + if isinstance(self.value, bool): + return str(self.value).lower() + if isinstance(self.value, str | int | float): + return str(self.value) + return "" + return str(self.value) + + def flatten(self) -> tuple[_Value, ...]: + if self.kind != "projection": + return (self,) + result: list[_Value] = [] + for value in cast(tuple[_Value, ...], self.value): + result.extend(value.flatten()) + return tuple(result) + + +_EMPTY_VALUE: Final = _Value("empty") + + +def _absolute_trait(value: str) -> ShapeId: + return ShapeId.from_string( + value, default_namespace=None if "#" in value else "smithy.api" + ) + + +def _trait_ids(shape: Shape) -> tuple[ShapeId, ...]: + traits = list(shape.traits) + if isinstance(shape, EnumShape) and _ENUM_TRAIT not in shape.traits: + traits.append(_ENUM_TRAIT) + return tuple(traits) + + +def _virtual_trait_shapes( + model: Model, shapes: Sequence[Shape] +) -> dict[ShapeId, Shape]: + missing = { + trait_id + for shape in shapes + for trait_id in _trait_ids(shape) + if model.find_shape(trait_id) is None + } + if not missing: + return {} + if model.find_shape(_TRAIT_TRAIT) is None: + missing.add(_TRAIT_TRAIT) + if ( + missing & _PRELUDE_AUTH_TRAITS + and model.find_shape(_AUTH_DEFINITION_TRAIT) is None + ): + missing.add(_AUTH_DEFINITION_TRAIT) + empty: Mapping[str, Node] = MappingProxyType({}) + result: dict[ShapeId, Shape] = {} + for shape_id in missing: + traits: dict[ShapeId, Node] = {_TRAIT_TRAIT: empty} + if shape_id in _PRELUDE_AUTH_TRAITS: + traits[_AUTH_DEFINITION_TRAIT] = empty + result[shape_id] = Shape( + id=shape_id, + type=_PRELUDE_TRAIT_TYPES.get(shape_id, ShapeType.STRUCTURE), + traits=MappingProxyType(traits), + ) + return result + + +def _traits(shape: Shape) -> Mapping[ShapeId, Node]: + if not isinstance(shape, EnumShape) or _ENUM_TRAIT in shape.traits: + return shape.traits + enum_values: list[Mapping[str, Node]] = [] + for member in shape.members.values(): + value = member.get_trait("enumValue") + entry: dict[str, Node] = {"name": member.name} + if value is not None: + entry["value"] = value + for trait_id, trait_value in member.traits.items(): + if trait_id != ShapeId("smithy.api", "enumValue"): + entry[trait_id.name] = trait_value + enum_values.append(entry) + return {**shape.traits, _ENUM_TRAIT: tuple(enum_values)} + + +def _compare(comparator: str, lhs: _Value, rhs: _Value, insensitive: bool) -> bool: + if comparator in {"{<}", "{<<}", "{=}", "{!=}"}: + if lhs.kind != "projection" or rhs.kind != "projection": + return comparator == "{!=}" + left = lhs.flatten() + right = rhs.flatten() + subset = _is_subset(left, right, insensitive) + reverse = _is_subset(right, left, insensitive) + return { + "{<}": subset, + "{<<}": subset and not reverse, + "{=}": subset and reverse, + "{!=}": not (subset and reverse), + }[comparator] + if comparator == "?=": + return (lhs.present and rhs.string == "true") or ( + not lhs.present and rhs.string == "false" + ) + for left in lhs.flatten(): + for right in rhs.flatten(): + if not left.present or not right.present: + continue + a = left.string.casefold() if insensitive else left.string + b = right.string.casefold() if insensitive else right.string + if comparator == "=" and a == b: + return True + if comparator == "!=" and a != b: + return True + if comparator == "^=" and a.startswith(b): + return True + if comparator == "$=" and a.endswith(b): + return True + if comparator == "*=" and b in a: + return True + if comparator in {">", ">=", "<", "<="}: + try: + left_number = Decimal(a) + right_number = Decimal(b) + except InvalidOperation: + continue + if not left_number.is_finite() or not right_number.is_finite(): + continue + if comparator == ">" and left_number > right_number: + return True + if comparator == ">=" and left_number >= right_number: + return True + if comparator == "<" and left_number < right_number: + return True + if comparator == "<=" and left_number <= right_number: + return True + return False + + +def _is_subset(lhs: Sequence[_Value], rhs: Sequence[_Value], insensitive: bool) -> bool: + return all( + any(_compare("=", left, right, insensitive) for right in rhs) for left in lhs + ) + + +class _Parser: + __slots__ = ("_depth", "_position", "_roots", "_source", "_variables") + + def __init__(self, source: str) -> None: + self._source = source + self._position = 0 + self._depth = 0 + self._roots: list[_Expression] = [] + self._variables: dict[str, int] = {} + + def parse(self, selector_type: type[Selector]) -> Selector: + expression = self._parse_sequence() + self._ws() + if not self._eof: + self._error(f"Unexpected selector character: {self._peek()!r}") + names = tuple(self._variables) + return selector_type(self._source, expression, tuple(self._roots), names) + + def _parse_sequence(self) -> _Expression: + self._depth += 1 + if self._depth > 100: + self._error("Selector exceeds the maximum nesting depth of 100") + try: + expressions = [self._parse_expression()] + self._ws() + while not self._eof and self._peek() not in ",])": + expressions.append(self._parse_expression()) + self._ws() + compact = tuple( + expression for expression in expressions if expression is not _IDENTITY + ) + if not compact: + return _IDENTITY + if len(compact) == 1: + return compact[0] + return _SequenceExpression(compact) + finally: + self._depth -= 1 + + def _parse_expression(self) -> _Expression: + self._ws() + if self._eof: + self._error("Unexpected selector EOF") + char = self._peek() + if char == "*": + self._position += 1 + return _IDENTITY + if char == ":": + self._position += 1 + return self._parse_function() + if char == "[": + self._position += 1 + if self._take("@"): + return self._parse_scoped_attribute() + return self._parse_attribute() + if char == ">": + self._position += 1 + return _NeighborExpression(True) + if char == "<": + self._position += 1 + if self._take("-"): + self._expect("[") + labels = self._parse_relationships() + self._expect("-") + return _NeighborExpression(False, frozenset(labels)) + return _NeighborExpression(False) + if char == "~": + self._position += 1 + self._expect(">") + return _RECURSIVE_NEIGHBOR + if char == "-": + self._position += 1 + self._expect("[") + labels = self._parse_relationships() + self._expect("-") + self._expect(">") + return _NeighborExpression(True, frozenset(labels)) + if char == "$": + self._position += 1 + return self._parse_variable() + token = self._identifier() + type_expression = _TYPE_EXPRESSIONS.get(token) + if type_expression is None: + self._error(f"Unknown shape type: {token}") + return type_expression + + def _parse_function(self) -> _Expression: + name = self._identifier() + arguments = self._parse_arguments() + if name == "not": + self._arity(name, arguments, 1) + return _NotExpression(arguments[0]) + if name == "test": + return _TestExpression(arguments) + if name in {"is", "each"}: + return _AnyExpression(arguments) + if name == "in": + self._arity(name, arguments, 1) + return _InExpression(arguments[0]) + if name == "root": + self._arity(name, arguments, 1) + result = _RootExpression(len(self._roots)) + self._roots.append(arguments[0]) + return result + if name == "recursive": + self._arity(name, arguments, 1) + return _RecursiveExpression(arguments[0]) + if name == "topdown": + if len(arguments) not in {1, 2}: + self._error( + f"The :topdown function accepts 1 or 2 arguments, found {len(arguments)}" + ) + return _TopDownExpression( + arguments[0], arguments[1] if len(arguments) == 2 else None + ) + return _AnyExpression(()) + + def _parse_arguments(self) -> tuple[_Expression, ...]: + self._ws() + self._expect("(") + arguments: list[_Expression] = [] + while True: + arguments.append(self._parse_sequence()) + self._ws() + if self._take(")"): + return tuple(arguments) + self._expect(",") + + def _parse_variable(self) -> _Expression: + self._ws() + if self._take("{"): + self._ws() + name = self._identifier() + self._ws() + self._expect("}") + return _VariableGetExpression(self._variable_slot(name)) + name = self._identifier() + self._ws() + self._expect("(") + expression = self._parse_sequence() + self._ws() + self._expect(")") + return _VariableStoreExpression(self._variable_slot(name), expression) + + def _parse_relationships(self) -> tuple[str, ...]: + labels: list[str] = [] + while True: + self._ws() + labels.append(self._identifier()) + self._ws() + if self._take("]"): + return tuple(labels) + self._expect(",") + + def _parse_attribute(self) -> _Expression: + path = self._parse_path(allow_empty=False) + self._ws() + if self._take("]"): + return _AttributeExpression(path, None) + comparator = self._parse_comparator() + values = [self._attribute_value()] + self._ws() + while self._take(","): + values.append(self._attribute_value()) + self._ws() + insensitive = self._take("i") + self._ws() + self._expect("]") + return _AttributeExpression(path, comparator, tuple(values), insensitive) + + def _parse_scoped_attribute(self) -> _Expression: + path = self._parse_path(allow_empty=True) + self._ws() + self._expect(":") + assertions: list[_ScopedAssertion] = [] + while True: + lhs = self._scoped_operand() + comparator = self._parse_comparator() + rhs = [self._scoped_operand()] + self._ws() + while self._take(","): + rhs.append(self._scoped_operand()) + self._ws() + insensitive = self._take("i") + self._ws() + assertions.append( + _ScopedAssertion(lhs, comparator, tuple(rhs), insensitive) + ) + if not self._take("&"): + break + self._expect("&") + self._expect("]") + return _ScopedAttributeExpression(path, tuple(assertions)) + + def _scoped_operand(self) -> _ScopedOperand: + self._ws() + if self._take("@"): + self._expect("{") + self._ws() + path = () if self._peek() == "}" else self._parse_path(allow_empty=False) + self._expect("}") + self._ws() + return _ScopedOperand(path=path) + return _ScopedOperand(literal=self._attribute_value()) + + def _parse_path( + self, *, allow_empty: bool, separator: str = "|" + ) -> tuple[str, ...]: + self._ws() + if allow_empty and self._peek() == ":": + return () + path = [self._path_segment()] + self._ws() + while self._take(separator): + path.append(self._path_segment()) + self._ws() + return tuple(path) + + def _path_segment(self) -> str: + self._ws() + if self._take("("): + name = self._identifier() + self._expect(")") + return f"({name})" + if self._peek() in {"'", '"'}: + return self._attribute_value() + start = self._position + while ( + not self._eof + and not self._peek().isspace() + and self._peek() not in "|,]}:=!^$*?><{" + ): + self._position += 1 + if start == self._position: + self._error("Expected an attribute path segment") + return self._source[start : self._position] + + def _parse_comparator(self) -> str: + self._ws() + for token in ( + "{!=}", + "{<<}", + "{<}", + "{=}", + "!=", + "^=", + "$=", + "*=", + "?=", + ">=", + "<=", + ): + if self._take(token): + self._ws() + return token + for token in ("=", ">", "<"): + if self._take(token): + self._ws() + return token + self._error("Expected an attribute comparator") + + def _attribute_value(self) -> str: + self._ws() + if self._peek() in {"'", '"'}: + quote = self._peek() + self._position += 1 + start = self._position + while not self._eof and self._peek() != quote: + self._position += 1 + if self._eof: + self._error(f"Expected {quote!r} to close attribute value") + value = self._source[start : self._position] + self._position += 1 + self._ws() + return value + match = _NUMBER.match(self._source, self._position) + if match is not None: + self._position = match.end() + self._ws() + return match.group() + start = self._position + while ( + not self._eof + and not self._peek().isspace() + and self._peek() not in ",]|}" + and not self._source.startswith("&&", self._position) + ): + self._position += 1 + if start == self._position: + self._error("Expected an attribute value") + value = self._source[start : self._position] + self._ws() + return value + + def _identifier(self) -> str: + self._ws() + match = _IDENTIFIER.match(self._source, self._position) + if match is None: + self._error("Expected an identifier") + self._position = match.end() + return match.group() + + def _variable_slot(self, name: str) -> int: + slot = self._variables.get(name) + if slot is None: + slot = len(self._variables) + self._variables[name] = slot + return slot + + def _arity( + self, name: str, arguments: tuple[_Expression, ...], expected: int + ) -> None: + if len(arguments) != expected: + self._error(f"The :{name} function requires {expected} argument") + + def _ws(self) -> None: + while not self._eof and self._peek().isspace(): + self._position += 1 + + def _take(self, token: str) -> bool: + if self._source.startswith(token, self._position): + self._position += len(token) + return True + return False + + def _expect(self, token: str) -> None: + if not self._take(token): + self._error(f"Expected {token!r}") + + def _peek(self) -> str: + return self._source[self._position] if not self._eof else "" + + @property + def _eof(self) -> bool: + return self._position >= len(self._source) + + def _error(self, message: str) -> NoReturn: + line = self._source.count("\n", 0, self._position) + 1 + line_start = self._source.rfind("\n", 0, self._position) + 1 + column = self._position - line_start + 1 + raise SelectorSyntaxError(f"{message} at line {line}, column {column}") diff --git a/packages/smithy-python/src/smithy_python/dependencies.py b/packages/smithy-python/src/smithy_python/dependencies.py new file mode 100644 index 000000000..db98207b7 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/dependencies.py @@ -0,0 +1,116 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Dependency records shared by symbols, writers, and package generation.""" + +from __future__ import annotations + +from collections.abc import Collection, Iterable, Iterator +from dataclasses import dataclass +from typing import Literal + +from . import exceptions as _exceptions + +__all__ = [ + "HATCHLING", + "SMITHY_CORE", + "Dependency", + "DependencyGroup", + "DependencySet", +] + +type DependencyGroup = Literal["runtime", "build", "test", "docs"] + +_GROUP_ORDER: dict[DependencyGroup, int] = { + "runtime": 0, + "build": 1, + "test": 2, + "docs": 3, +} + + +@dataclass(frozen=True, slots=True) +class Dependency: + """A generated package requirement and the group that owns it.""" + + package: str + specifier: str + group: DependencyGroup + extras: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not self.package or self.package.strip() != self.package: + raise ValueError("Dependency package must be a non-empty trimmed string") + if self.group not in _GROUP_ORDER: + raise ValueError(f"Unknown dependency group: {self.group!r}") + if any(not extra or extra.strip() != extra for extra in self.extras): + raise ValueError("Dependency extras must be non-empty trimmed strings") + object.__setattr__(self, "extras", tuple(sorted(set(self.extras)))) + + +class DependencySet(Collection[Dependency]): + """Deterministic dependencies keyed by package and group.""" + + def __init__(self, dependencies: Iterable[Dependency] = ()) -> None: + self._dependencies: dict[tuple[str, DependencyGroup], Dependency] = {} + for dependency in dependencies: + self.add(dependency) + + def add(self, dependency: Dependency) -> None: + """Add a dependency, merging extras or rejecting version conflicts.""" + key = (dependency.package, dependency.group) + previous = self._dependencies.get(key) + if previous is None: + self._dependencies[key] = dependency + return + if previous.specifier != dependency.specifier: + raise _exceptions.CodegenError( + f"Conflicting {dependency.group} dependency specifiers for " + f"{dependency.package!r}: {previous.specifier!r} and " + f"{dependency.specifier!r}" + ) + extras = tuple(sorted(set(previous.extras) | set(dependency.extras))) + if extras != previous.extras: + self._dependencies[key] = Dependency( + package=previous.package, + specifier=previous.specifier, + group=previous.group, + extras=extras, + ) + + def for_group(self, group: DependencyGroup) -> tuple[Dependency, ...]: + """Return one dependency group in deterministic package order.""" + return tuple(dependency for dependency in self if dependency.group == group) + + def __contains__(self, value: object) -> bool: + if not isinstance(value, Dependency): + return False + return self._dependencies.get((value.package, value.group)) == value + + def __iter__(self) -> Iterator[Dependency]: + return iter( + sorted( + self._dependencies.values(), + key=lambda dependency: ( + _GROUP_ORDER[dependency.group], + dependency.package, + ), + ) + ) + + def __len__(self) -> int: + return len(self._dependencies) + + +SMITHY_CORE = Dependency( + package="smithy-core", + specifier="~=0.10.0", + group="runtime", +) +"""Compatible smithy-core runtime for generated artifacts.""" + +HATCHLING = Dependency( + package="hatchling", + specifier="", + group="build", +) +"""Build backend used by generated artifacts.""" diff --git a/packages/smithy-python/src/smithy_python/exceptions.py b/packages/smithy-python/src/smithy_python/exceptions.py index 49e088cd8..89208d756 100644 --- a/packages/smithy-python/src/smithy_python/exceptions.py +++ b/packages/smithy-python/src/smithy_python/exceptions.py @@ -34,3 +34,7 @@ class InvalidShapeIdError(ModelError, ValueError): class ShapeNotFoundError(ModelError, LookupError): """Raised when a shape ID does not resolve to a shape in the model.""" + + +class SelectorSyntaxError(CodegenError, ValueError): + """Raised when a Smithy selector expression is invalid.""" diff --git a/packages/smithy-python/src/smithy_python/files.py b/packages/smithy-python/src/smithy_python/files.py new file mode 100644 index 000000000..736f71821 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/files.py @@ -0,0 +1,301 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Staged output and ownership tracking for generated artifacts.""" + +from __future__ import annotations + +import hashlib +import itertools +import json +import os +import tempfile +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, cast + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import writer as _writer + +__all__ = ["FileManifest"] + +_MANIFEST_PATH: Final = ".smithy-python-manifest.json" +_MANIFEST_VERSION: Final = 1 + + +class FileManifest: + """Collect, validate, publish, and track files for one generated artifact.""" + + def __init__(self, dependencies: _dependencies.DependencySet | None = None) -> None: + self._dependencies = ( + dependencies if dependencies is not None else _dependencies.DependencySet() + ) + self._files: dict[str, _GeneratedFile] = {} + + @property + def dependencies(self) -> _dependencies.DependencySet: + """Dependencies recorded by writers owned by this manifest.""" + return self._dependencies + + def python_module(self, path: str, module: str) -> _writer.PythonWriter: + """Return the shared writer for a generated Python module.""" + path = _normalize_path(path) + self._claim_public_path(path) + existing = self._files.get(path) + if isinstance(existing, _PythonFile): + if existing.module != module: + raise _exceptions.CodegenError( + f"Generated path {path!r} already uses module " + f"{existing.module!r}, not {module!r}" + ) + return existing.writer + if existing is not None: + raise _exceptions.CodegenError( + f"Generated path {path!r} is already a text file" + ) + writer = _writer.PythonWriter(module, dependencies=self._dependencies) + self._files[path] = _PythonFile(module, writer) + return writer + + def text_file(self, path: str, content: str) -> None: + """Register one generated text file.""" + path = _normalize_path(path) + self._claim_public_path(path) + if path in self._files: + raise _exceptions.CodegenError( + f"Generated path {path!r} is already claimed" + ) + self._files[path] = _TextFile(content) + + def commit(self, output: Path) -> None: + """Stage, validate, and publish all registered files.""" + output = output.absolute() + self._validate_path_tree() + if output.is_symlink() and not output.exists(): + raise _exceptions.CodegenError( + f"Output path is a dangling symlink: {output}" + ) + if output.exists() and not output.is_dir(): + raise _exceptions.CodegenError(f"Output path is not a directory: {output}") + + previous = self._load_previous_manifest(output) + for path in self._files: + self._validate_destination(output, path, owned=path in previous) + for path in previous: + self._validate_safe_path(output, path) + + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory( + prefix=f".{output.name or 'output'}.smithy-python-", + dir=output.parent, + ) as directory: + staging = Path(directory) + hashes = self._stage_files(staging) + manifest_content = self._render_manifest(hashes) + manifest_target = staging / _MANIFEST_PATH + manifest_target.write_text(manifest_content, encoding="utf-8") + self._publish(output, staging, hashes, previous) + + @staticmethod + def _claim_public_path(path: str) -> None: + if path == _MANIFEST_PATH: + raise _exceptions.CodegenError( + f"Generated path {_MANIFEST_PATH!r} is reserved for output ownership" + ) + + def _validate_path_tree(self) -> None: + paths = sorted(PurePosixPath(path).parts for path in self._files) + for previous, current in itertools.pairwise(paths): + if len(previous) < len(current) and current[: len(previous)] == previous: + raise _exceptions.CodegenError( + f"Generated path {'/'.join(previous)!r} conflicts with " + f"{'/'.join(current)!r}" + ) + + @staticmethod + def _validate_safe_path(output: Path, path: str) -> None: + root = output.resolve(strict=False) + current = output + for part in PurePosixPath(path).parts: + current /= part + if current.is_symlink(): + if not current.exists(): + raise _exceptions.CodegenError( + f"Generated path {path!r} contains a dangling symlink" + ) + resolved = current.resolve(strict=False) + if not resolved.is_relative_to(root): + raise _exceptions.CodegenError( + f"Generated path {path!r} escapes the output directory" + ) + + @classmethod + def _validate_destination(cls, output: Path, path: str, *, owned: bool) -> None: + cls._validate_safe_path(output, path) + parts = PurePosixPath(path).parts + current = output + for index, part in enumerate(parts): + current /= part + if index < len(parts) - 1 and current.is_symlink(): + raise _exceptions.CodegenError( + f"Generated path {path!r} contains a symlinked ancestor" + ) + if current.exists(): + is_target = index == len(parts) - 1 + if (is_target and current.is_dir()) or ( + not is_target and not current.is_dir() + ): + raise _exceptions.CodegenError( + f"Generated path {path!r} conflicts with {current}" + ) + if is_target and not owned: + raise _exceptions.CodegenError( + f"Generated path {path!r} already exists but is not owned " + "by the output manifest" + ) + + @staticmethod + def _load_previous_manifest(output: Path) -> dict[str, str]: + manifest = output / _MANIFEST_PATH + if manifest.is_symlink(): + raise _exceptions.CodegenError( + f"Output ownership manifest cannot be a symlink: {manifest}" + ) + if not manifest.exists(): + return {} + try: + value: object = json.loads(manifest.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise _exceptions.CodegenError( + f"Cannot read output ownership manifest: {error}" + ) from error + if not isinstance(value, dict): + raise _exceptions.CodegenError("Invalid output ownership manifest") + manifest_data = cast(dict[object, object], value) + files = manifest_data.get("files") + if manifest_data.get("version") != _MANIFEST_VERSION or not isinstance( + files, dict + ): + raise _exceptions.CodegenError("Invalid output ownership manifest") + file_data = cast(dict[object, object], files) + result: dict[str, str] = {} + for raw_path, digest in file_data.items(): + if not isinstance(raw_path, str) or not isinstance(digest, str): + raise _exceptions.CodegenError("Invalid output ownership manifest") + path = _normalize_path(raw_path) + if ( + path != raw_path + or path == _MANIFEST_PATH + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise _exceptions.CodegenError("Invalid output ownership manifest") + result[path] = digest + return result + + def _stage_files(self, staging: Path) -> dict[str, str]: + hashes: dict[str, str] = {} + for path in sorted(self._files): + generated = self._files[path] + if isinstance(generated, _PythonFile): + content = generated.writer.render() + try: + compile(content, path, "exec", dont_inherit=True) + except (SyntaxError, ValueError) as error: + raise _exceptions.CodegenError( + f"Generated Python module {path!r} is invalid: {error}" + ) from error + else: + content = generated.content + if "\0" in content: + raise _exceptions.CodegenError( + f"Generated text file {path!r} contains a null byte" + ) + encoded = content.encode() + target = staging.joinpath(*PurePosixPath(path).parts) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(encoded) + hashes[path] = hashlib.sha256(encoded).hexdigest() + return hashes + + @staticmethod + def _render_manifest(hashes: dict[str, str]) -> str: + return ( + json.dumps( + {"version": _MANIFEST_VERSION, "files": hashes}, + indent=2, + sort_keys=True, + ) + + "\n" + ) + + @staticmethod + def _publish( + output: Path, + staging: Path, + hashes: dict[str, str], + previous: dict[str, str], + ) -> None: + output.mkdir(parents=True, exist_ok=True) + for path in hashes: + source = staging.joinpath(*PurePosixPath(path).parts) + target = output.joinpath(*PurePosixPath(path).parts) + target.parent.mkdir(parents=True, exist_ok=True) + os.replace(source, target) + + for path in sorted(previous.keys() - hashes.keys()): + parts = PurePosixPath(path).parts + target = output.joinpath(*parts) + if not target.is_file() or any( + output.joinpath(*parts[:index]).is_symlink() + for index in range(1, len(parts) + 1) + ): + continue + try: + unchanged = _digest_file(target) == previous[path] + except OSError: + unchanged = False + if unchanged: + target.unlink() + + os.replace(staging / _MANIFEST_PATH, output / _MANIFEST_PATH) + + +@dataclass(slots=True) +class _PythonFile: + module: str + writer: _writer.PythonWriter + + +@dataclass(frozen=True, slots=True) +class _TextFile: + content: str + + +type _GeneratedFile = _PythonFile | _TextFile + + +def _normalize_path(path: str) -> str: + parts = path.split("/") + if ( + not path + or path.startswith("/") + or "\\" in path + or any( + part in ("", ".", "..") + or ( + len(part) >= 2 + and part[0].isascii() + and part[0].isalpha() + and part[1] == ":" + ) + for part in parts + ) + ): + raise _exceptions.CodegenError(f"Invalid generated path: {path!r}") + return str(PurePosixPath(path)) + + +def _digest_file(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() diff --git a/packages/smithy-python/src/smithy_python/generation.py b/packages/smithy-python/src/smithy_python/generation.py new file mode 100644 index 000000000..3bec23e70 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/generation.py @@ -0,0 +1,138 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Shared settings and state for artifact generators.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Protocol + +from . import dependencies as _dependencies +from . import files as _files +from . import model as _model +from . import schemas as _schemas +from . import selection as _selection +from . import symbols as _symbols + +__all__ = [ + "ArtifactGenerator", + "ArtifactSettings", + "GenerationContext", + "GenerationPlan", + "ModelPlan", + "TypesSettings", +] + + +@dataclass(frozen=True, slots=True) +class ArtifactSettings: + """Settings shared by generated artifacts.""" + + package: str + package_version: str + package_description: str | None + + +@dataclass(frozen=True, slots=True) +class TypesSettings(ArtifactSettings): + """Settings for a standalone types package.""" + + selector: _selection.Selector = field(default_factory=_selection.Selector.identity) + generate_inputs_and_outputs: bool = False + + +@dataclass(frozen=True, slots=True) +class ModelPlan: + """Derived semantic data used to emit lazy model declarations.""" + + declaration_names: Mapping[_model.ShapeId, str] + renames: Mapping[str, str] + union_variant_names: Mapping[_model.ShapeId, str] + union_unknown_names: Mapping[_model.ShapeId, str] + + @classmethod + def from_shapes( + cls, + shapes: tuple[_model.Shape, ...], + symbols: _symbols.SymbolProvider, + schema_names: Mapping[_model.ShapeId, str], + ) -> ModelPlan: + """Plan declaration and union names in stable shape order.""" + declarations: dict[_model.ShapeId, str] = {} + variants: dict[_model.ShapeId, str] = {} + unknowns: dict[_model.ShapeId, str] = {} + for shape in shapes: + if shape.type not in _model.DECLARATION_TYPES or shape.has_trait( + "streaming" + ): + continue + declarations[shape.id] = symbols.declaration_name(shape.id) + if shape.type is _model.ShapeType.UNION: + for member in shape.members.values(): + variants[member.id] = symbols.union_variant_name(member.id) + unknowns[shape.id] = symbols.union_unknown_name(shape.id) + + return cls( + declaration_names=MappingProxyType(declarations), + renames=MappingProxyType( + { + declaration: schema_names[shape_id] + for shape_id, declaration in declarations.items() + if declaration != shape_id.name + } + ), + union_variant_names=MappingProxyType(variants), + union_unknown_names=MappingProxyType(unknowns), + ) + + +@dataclass(frozen=True, slots=True) +class GenerationPlan: + """Derived semantic data shared by artifact emitters.""" + + schemas: _schemas.SchemaPlan + models: ModelPlan + + @classmethod + def from_selection( + cls, + model: _model.Model, + selection: _selection.Selection, + symbols: _symbols.SymbolProvider, + ) -> GenerationPlan: + """Build focused plans in stable selection order.""" + selected = selection.shapes + schemas = _schemas.SchemaPlan.from_shapes(model, selected, symbols) + return cls( + schemas=schemas, + models=ModelPlan.from_shapes(selected, symbols, schemas.names), + ) + + +@dataclass(frozen=True, slots=True) +class GenerationContext[SettingsT: ArtifactSettings]: + """State shared by one artifact generation.""" + + model: _model.Model + selection: _selection.Selection + settings: SettingsT + symbols: _symbols.SymbolProvider + plan: GenerationPlan + dependencies: _dependencies.DependencySet + files: _files.FileManifest + + def __post_init__(self) -> None: + if self.files.dependencies is not self.dependencies: + raise ValueError( + "GenerationContext dependencies must be shared with FileManifest" + ) + + +class ArtifactGenerator[SettingsT: ArtifactSettings](Protocol): + """Generate one artifact into the context's file manifest.""" + + def generate(self, context: GenerationContext[SettingsT]) -> None: + """Register every generated file and dependency.""" + ... diff --git a/packages/smithy-python/src/smithy_python/model/__init__.py b/packages/smithy-python/src/smithy_python/model/__init__.py index 58f26309d..9aea01767 100644 --- a/packages/smithy-python/src/smithy_python/model/__init__.py +++ b/packages/smithy-python/src/smithy_python/model/__init__.py @@ -12,6 +12,7 @@ from ._model import Model from ._shape_id import ShapeId from ._shapes import ( + DECLARATION_TYPES, EnumShape, IntEnumShape, JsonValue, @@ -31,6 +32,7 @@ ) __all__ = [ + "DECLARATION_TYPES", "EnumShape", "IntEnumShape", "InvalidShapeIdError", diff --git a/packages/smithy-python/src/smithy_python/model/_shapes.py b/packages/smithy-python/src/smithy_python/model/_shapes.py index 5417823a5..e7357daab 100644 --- a/packages/smithy-python/src/smithy_python/model/_shapes.py +++ b/packages/smithy-python/src/smithy_python/model/_shapes.py @@ -75,6 +75,14 @@ class ShapeType(StrEnum): ShapeType.DOCUMENT, } ) +DECLARATION_TYPES: Final = frozenset( + { + ShapeType.STRUCTURE, + ShapeType.UNION, + ShapeType.ENUM, + ShapeType.INT_ENUM, + } +) MIXIN_TRAIT: Final = ShapeId(PRELUDE_NAMESPACE, "mixin") UNIT: Final = ShapeId(PRELUDE_NAMESPACE, "Unit") diff --git a/packages/smithy-python/src/smithy_python/schemas.py b/packages/smithy-python/src/smithy_python/schemas.py new file mode 100644 index 000000000..c47084e45 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/schemas.py @@ -0,0 +1,513 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Planning and emission for compact, lazily hydrated schemas.""" + +from __future__ import annotations + +from collections.abc import Collection, Mapping +from dataclasses import dataclass +from math import isfinite +from types import MappingProxyType +from typing import TYPE_CHECKING, Final + +from . import dependencies as _dependencies +from . import exceptions as _exceptions +from . import model as _model +from . import symbols as _symbols + +if TYPE_CHECKING: + from .generation import GenerationContext, TypesSettings + from .writer import PythonWriter + +__all__ = ["SchemaGenerator", "SchemaPlan"] + + +@dataclass(frozen=True, slots=True) +class SchemaPlan: + """Derived semantic data used to emit compact schemas.""" + + shapes: tuple[_model.Shape, ...] + names: Mapping[_model.ShapeId, str] + strings: Mapping[str, str] + traits: Mapping[_model.ShapeId, _TraitPlan] + + @classmethod + def from_shapes( + cls, + model: _model.Model, + shapes: tuple[_model.Shape, ...], + symbols: _symbols.SymbolProvider, + ) -> SchemaPlan: + """Plan schema order, filtered traits, names, and interned strings.""" + emitted = _schema_order(shapes) + included = _included_traits(model) + inputs = { + operation.input for operation in model.iter_shapes(_model.OperationShape) + } + outputs = { + operation.output for operation in model.iter_shapes(_model.OperationShape) + } + strings = _StringStore() + traits: dict[_model.ShapeId, _TraitPlan] = {} + for shape in emitted: + additional: list[_model.ShapeId] = [] + if shape.type is _model.ShapeType.STRUCTURE: + if shape.id in inputs: + additional.append(_INPUT_TRAIT) + if shape.id in outputs: + additional.append(_OUTPUT_TRAIT) + shape_traits = _plan_traits(shape, included, additional) + traits[shape.id] = shape_traits + strings.intern(shape.id.namespace, preferred="n") + strings.intern(shape.id.name) + _intern_traits(strings, shape_traits) + for member in shape.members.values(): + member_traits = _plan_traits(member, included) + traits[member.id] = member_traits + strings.intern(member.name) + _intern_traits(strings, member_traits) + + return cls( + shapes=emitted, + names=MappingProxyType( + {shape.id: symbols.schema_name(shape.id) for shape in emitted} + ), + strings=MappingProxyType(strings.values), + traits=MappingProxyType(traits), + ) + + +class SchemaGenerator: + """Emit the compact schema module for a standalone types package.""" + + def generate(self, context: GenerationContext[TypesSettings]) -> None: + """Register ``._private.schemas`` with the file manifest.""" + package = context.settings.package + path = f"src/{package.replace('.', '/')}/_private/schemas.py" + writer = context.files.python_module(path, f"{package}._private.schemas") + plan = context.plan.schemas + for value, name in plan.strings.items(): + writer.line(name, " = ", repr(value)) + if plan.strings: + writer.line() + + written: set[_model.ShapeId] = set() + for shape in plan.shapes: + _write_shape(writer, plan, shape, written) + written.add(shape.id) + + +# This contract is intentionally local to the generator. It must match +# smithy_core._schema_compact without making smithy-python import smithy-core. +_ENCODING_VERSION: Final = 2 +_UNIT_INT: Final = -1 +_PRIMITIVE_BOOLEAN_INT: Final = -2 +_PRIMITIVE_BYTE_INT: Final = -3 +_PRIMITIVE_SHORT_INT: Final = -4 +_PRIMITIVE_INTEGER_INT: Final = -5 +_PRIMITIVE_LONG_INT: Final = -6 +_PRIMITIVE_FLOAT_INT: Final = -7 +_PRIMITIVE_DOUBLE_INT: Final = -8 + +_TYPE_INTS: Final[Mapping[_model.ShapeType, int]] = { + _model.ShapeType.BLOB: 1, + _model.ShapeType.BOOLEAN: 2, + _model.ShapeType.STRING: 3, + _model.ShapeType.TIMESTAMP: 4, + _model.ShapeType.BYTE: 5, + _model.ShapeType.SHORT: 6, + _model.ShapeType.INTEGER: 7, + _model.ShapeType.LONG: 8, + _model.ShapeType.FLOAT: 9, + _model.ShapeType.DOUBLE: 10, + _model.ShapeType.BIG_INTEGER: 11, + _model.ShapeType.BIG_DECIMAL: 12, + _model.ShapeType.DOCUMENT: 13, + _model.ShapeType.ENUM: 14, + _model.ShapeType.INT_ENUM: 15, + _model.ShapeType.LIST: 16, + _model.ShapeType.MAP: 17, + _model.ShapeType.STRUCTURE: 18, + _model.ShapeType.UNION: 19, + _model.ShapeType.SERVICE: 21, + _model.ShapeType.OPERATION: 23, +} + + +def _shape_id(value: str) -> _model.ShapeId: + return _model.ShapeId.from_string(value) + + +_PRELUDE_INTS: Final[Mapping[_model.ShapeId, int]] = { + _shape_id("smithy.api#Blob"): 1, + _shape_id("smithy.api#Boolean"): 2, + _shape_id("smithy.api#String"): 3, + _shape_id("smithy.api#Timestamp"): 4, + _shape_id("smithy.api#Byte"): 5, + _shape_id("smithy.api#Short"): 6, + _shape_id("smithy.api#Integer"): 7, + _shape_id("smithy.api#Long"): 8, + _shape_id("smithy.api#Float"): 9, + _shape_id("smithy.api#Double"): 10, + _shape_id("smithy.api#BigInteger"): 11, + _shape_id("smithy.api#BigDecimal"): 12, + _shape_id("smithy.api#Document"): 13, + _shape_id("smithy.api#PrimitiveBoolean"): _PRIMITIVE_BOOLEAN_INT, + _shape_id("smithy.api#PrimitiveByte"): _PRIMITIVE_BYTE_INT, + _shape_id("smithy.api#PrimitiveShort"): _PRIMITIVE_SHORT_INT, + _shape_id("smithy.api#PrimitiveInteger"): _PRIMITIVE_INTEGER_INT, + _shape_id("smithy.api#PrimitiveLong"): _PRIMITIVE_LONG_INT, + _shape_id("smithy.api#PrimitiveFloat"): _PRIMITIVE_FLOAT_INT, + _shape_id("smithy.api#PrimitiveDouble"): _PRIMITIVE_DOUBLE_INT, + _shape_id("smithy.api#Unit"): _UNIT_INT, +} + +# Bit position must match smithy_core._schema_compact._TRAIT_BITS. +_TRAIT_BITS: Final = tuple( + map( + _shape_id, + ( + "smithy.api#required", + "smithy.api#idempotencyToken", + "smithy.api#sensitive", + "smithy.api#httpLabel", + "smithy.api#httpPayload", + "smithy.api#idempotent", + "smithy.api#sparse", + "smithy.api#httpResponseCode", + "smithy.api#hostLabel", + "smithy.api#requiresLength", + "smithy.api#httpQueryParams", + "smithy.api#notProperty", + ), + ) +) +_TRAIT_BIT_INDEX: Final = { + trait_id: index for index, trait_id in enumerate(_TRAIT_BITS) +} + +# This is the client-side set from Java's SchemaTraitFilterIndex. Constraint +# traits are intentionally omitted because standalone types do not enable +# server-side validation. +_INCLUDED_TRAITS: Final = frozenset( + map( + _shape_id, + ( + "smithy.api#sparse", + "smithy.api#sensitive", + "smithy.api#idempotencyToken", + "smithy.api#default", + "smithy.api#enumValue", + "smithy.api#required", + "smithy.api#input", + "smithy.api#output", + "smithy.api#timestampFormat", + "smithy.synthetic#originalShapeId", + "aws.api#service", + "smithy.rules#endpointBdd", + "smithy.api#longPoll", + "smithy.api#requestCompression", + "smithy.api#jsonName", + "smithy.api#mediaType", + "smithy.api#xmlAttribute", + "smithy.api#xmlFlattened", + "smithy.api#xmlName", + "smithy.api#xmlNamespace", + "smithy.api#streaming", + "smithy.api#endpoint", + "smithy.api#error", + "smithy.api#requiresLength", + "smithy.api#eventHeader", + "smithy.api#eventPayload", + "smithy.api#httpError", + "smithy.api#http", + "smithy.api#httpHeader", + "smithy.api#httpQuery", + "smithy.api#httpLabel", + "smithy.api#httpPayload", + "smithy.api#httpPrefixHeaders", + "smithy.api#httpQueryParams", + "smithy.api#httpResponseCode", + "smithy.api#hostLabel", + ), + ) +) +_DEFAULT_TRAIT: Final = _shape_id("smithy.api#default") +_INPUT_TRAIT: Final = _shape_id("smithy.api#input") +_OUTPUT_TRAIT: Final = _shape_id("smithy.api#output") +_DEFINITION_TRAITS: Final = ( + _shape_id("smithy.api#protocolDefinition"), + _shape_id("smithy.api#authDefinition"), +) + +_STATIC_SCHEMA: Final = _symbols.TypeReference( + "StaticSchema", + "smithy_core._schema_compact", + dependencies=(_dependencies.SMITHY_CORE,), +) +_MAPPING_PROXY: Final = _symbols.TypeReference("MappingProxyType", "types") +_TYPED_MAPPING_PROXY: Final = _symbols.TypeReference( + "MappingProxyType", + "types", + arguments=( + _symbols.TypeReference("str", "builtins"), + _symbols.TypeReference("object", "builtins"), + ), +) + +type _SourcePart = str | _symbols.TypeReference +type _TraitPlan = int | Mapping[_model.ShapeId, _model.Node] + + +def _write_shape( + writer: PythonWriter, + plan: SchemaPlan, + shape: _model.Shape, + written: set[_model.ShapeId], +) -> None: + writer.line(plan.names[shape.id], ": ", _STATIC_SCHEMA, " = (") + with writer.indent(): + writer.line(f"{_TYPE_INTS[shape.type]},") + writer.line(plan.strings[shape.id.namespace], ",") + writer.line(plan.strings[shape.id.name], ",") + writer.line(*_trait_parts(plan, plan.traits[shape.id]), ",") + if shape.members: + writer.line( + "(", + ", ".join( + plan.strings[member.name] for member in shape.members.values() + ), + ",),", + ) + writer.line("(") + with writer.indent(): + for member in shape.members.values(): + writer.line( + "(", + _schema_ref(plan, member.target, written), + ", ", + *_trait_parts(plan, plan.traits[member.id]), + "),", + ) + writer.line("),") + writer.line(")") + writer.line() + + +def _included_traits(model: _model.Model) -> frozenset[_model.ShapeId]: + included = set(_INCLUDED_TRAITS) + for shape in model.iter_shapes(): + for definition_trait in _DEFINITION_TRAITS: + if not shape.has_trait(definition_trait): + continue + included.add(shape.id) + definition = shape.get_trait(definition_trait) + if not isinstance(definition, Mapping): + continue + referenced = definition.get("traits") + if not isinstance(referenced, tuple): + continue + for value in referenced: + if isinstance(value, str): + included.add(_shape_id(value)) + return frozenset(included) + + +def _plan_traits( + shape: _model.Shape, + included: frozenset[_model.ShapeId], + additional: Collection[_model.ShapeId] = (), +) -> _TraitPlan: + bits = 0 + values: dict[_model.ShapeId, _model.Node] = {} + valueless_bits: list[_model.ShapeId] = [] + for trait_id, value in shape.traits.items(): + if trait_id not in included: + continue + valueless = ( + trait_id != _DEFAULT_TRAIT and isinstance(value, Mapping) and not value + ) + bit = _TRAIT_BIT_INDEX.get(trait_id) + if valueless and bit is not None: + bits |= 1 << bit + valueless_bits.append(trait_id) + else: + values[trait_id] = None if valueless else value + for trait_id in additional: + bit = _TRAIT_BIT_INDEX.get(trait_id) + if bit is not None: + bits |= 1 << bit + valueless_bits.append(trait_id) + else: + values[trait_id] = None + if not values: + return bits + values.update(dict.fromkeys(valueless_bits)) + return MappingProxyType(values) + + +def _trait_parts(plan: SchemaPlan, traits: _TraitPlan) -> tuple[_SourcePart, ...]: + if isinstance(traits, int): + return (str(traits),) + parts: list[_SourcePart] = ["{"] + for index, (trait_id, value) in enumerate(traits.items()): + if index: + parts.append(", ") + parts.extend((plan.strings[str(trait_id)], ": ")) + parts.extend( + _node_parts( + value, + strings=plan.strings, + intern_string=True, + ) + ) + parts.append("}") + return tuple(parts) + + +def _node_parts( + value: _model.Node, + *, + strings: Mapping[str, str], + intern_string: bool = False, +) -> tuple[_SourcePart, ...]: + if value is None: + return ("None",) + if isinstance(value, bool): + return ("True" if value else "False",) + if isinstance(value, str): + if intern_string: + return (strings[value],) + return (repr(value),) + if isinstance(value, float) and not isfinite(value): + raise _exceptions.CodegenError("Cannot emit a non-finite numeric trait value") + if isinstance(value, int | float): + return (repr(value),) + if isinstance(value, tuple): + parts: list[_SourcePart] = ["("] + for index, item in enumerate(value): + if index: + parts.append(", ") + parts.extend(_node_parts(item, strings=strings)) + if len(value) == 1: + parts.append(",") + parts.append(")") + return tuple(parts) + + proxy = _TYPED_MAPPING_PROXY if not value else _MAPPING_PROXY + parts = [proxy, "({"] + for index, (key, item) in enumerate(value.items()): + if index: + parts.append(", ") + parts.extend((repr(key), ": ")) + parts.extend(_node_parts(item, strings=strings)) + parts.append("})") + return tuple(parts) + + +def _schema_ref( + plan: SchemaPlan, target: _model.ShapeId, written: set[_model.ShapeId] +) -> str: + if (prelude := _PRELUDE_INTS.get(target)) is not None: + return str(prelude) + try: + name = plan.names[target] + except KeyError as error: + raise _exceptions.CodegenError( + f"Schema target {target} was not selected" + ) from error + return name if target in written else f"lambda: {name}" + + +def _schema_order( + shapes: tuple[_model.Shape, ...], +) -> tuple[_model.Shape, ...]: + by_id = {shape.id: shape for shape in shapes} + graph = { + shape.id: tuple( + member.target for member in shape.members.values() if member.target in by_id + ) + for shape in shapes + } + reverse: dict[_model.ShapeId, list[_model.ShapeId]] = { + shape.id: [] for shape in shapes + } + for source, targets in graph.items(): + for target in targets: + reverse[target].append(source) + + finished: list[_model.ShapeId] = [] + visited: set[_model.ShapeId] = set() + for root in graph: + if root in visited: + continue + visited.add(root) + pending: list[tuple[_model.ShapeId, int]] = [(root, 0)] + while pending: + current, index = pending[-1] + targets = graph[current] + if index == len(targets): + finished.append(current) + pending.pop() + continue + target = targets[index] + pending[-1] = (current, index + 1) + if target not in visited: + visited.add(target) + pending.append((target, 0)) + + recursive: set[_model.ShapeId] = set() + visited.clear() + for root in reversed(finished): + if root in visited: + continue + component: list[_model.ShapeId] = [] + component_pending = [root] + visited.add(root) + while component_pending: + current = component_pending.pop() + component.append(current) + for target in reverse[current]: + if target not in visited: + visited.add(target) + component_pending.append(target) + if len(component) > 1 or root in graph[root]: + recursive.update(component) + + return ( + *(by_id[shape_id] for shape_id in finished if shape_id not in recursive), + *(shape for shape in shapes if shape.id in recursive), + ) + + +def _intern_traits(strings: _StringStore, traits: _TraitPlan) -> None: + if isinstance(traits, int): + return + for trait_id, value in traits.items(): + strings.intern(str(trait_id)) + if isinstance(value, str): + strings.intern(value) + + +class _StringStore: + __slots__ = ("_allocated", "_next", "values") + + def __init__(self) -> None: + self.values: dict[str, str] = {} + self._allocated: set[str] = set() + self._next: dict[str, int] = {} + + def intern(self, value: str, *, preferred: str | None = None) -> str: + existing = self.values.get(value) + if existing is not None: + return existing + prefix = preferred or "_s" + suffix = self._next.get(prefix, 0) + candidate = f"{prefix}{suffix}" + while candidate in self._allocated: + suffix += 1 + candidate = f"{prefix}{suffix}" + self._next[prefix] = suffix + 1 + self._allocated.add(candidate) + self.values[value] = candidate + return candidate diff --git a/packages/smithy-python/src/smithy_python/selection.py b/packages/smithy-python/src/smithy_python/selection.py index c6b0ec414..62779f1b0 100644 --- a/packages/smithy-python/src/smithy_python/selection.py +++ b/packages/smithy-python/src/smithy_python/selection.py @@ -7,8 +7,10 @@ from collections.abc import Iterator from dataclasses import dataclass +from ._selector import Selector from .exceptions import InvalidInvocationError, InvalidShapeIdError, ModelError from .model import ( + DECLARATION_TYPES, MemberShape, Model, OperationShape, @@ -19,7 +21,13 @@ ShapeType, ) -__all__ = ["Selection", "resolve_service", "select_shapes"] +__all__ = [ + "Selection", + "Selector", + "resolve_service", + "select_shapes", + "select_type_shapes", +] @dataclass(frozen=True, slots=True) @@ -32,6 +40,44 @@ class Selection: """Eligible data shapes omitted by the service closure (zero without a service).""" +def resolve_service( + model: Model, *, service_id: str | None = None, require_service: bool = False +) -> ServiceShape | None: + """Resolve an absolute service ID or the sole non-mixin service. + + Invalid or ambiguous selections raise :class:`InvalidInvocationError`. + """ + if service_id is not None: + try: + shape_id = ShapeId.from_string(service_id) + except InvalidShapeIdError as error: + raise InvalidInvocationError(f"--service: {error}") from error + if shape_id.is_member: + raise InvalidInvocationError( + f"--service {service_id!r} is a member ID, not a service" + ) + shape = model.find_shape(shape_id) + if shape is None: + raise InvalidInvocationError(f"--service {service_id!r} is not defined") + if not isinstance(shape, ServiceShape): + raise InvalidInvocationError(f"--service {service_id!r} is not a service") + if shape.is_mixin: + raise InvalidInvocationError(f"--service {service_id!r} is a mixin") + return shape + + services = model.services() + if len(services) > 1: + candidates = ", ".join(str(service.id) for service in services) + raise InvalidInvocationError( + f"Multiple services found; select one with --service: {candidates}" + ) + if services: + return services[0] + if require_service: + raise InvalidInvocationError("Client generation requires a service") + return None + + def select_shapes( model: Model, *, service_id: str | None = None, require_service: bool = False ) -> Selection: @@ -61,19 +107,116 @@ def select_shapes( _check_names(eligible) return Selection(service=None, shapes=eligible, excluded_count=0) + visited = _closure(model, service.id) + selected = tuple(shape for shape in eligible if shape.id in visited) + return Selection( + service=service, shapes=selected, excluded_count=len(eligible) - len(selected) + ) + + +def select_type_shapes( + model: Model, + *, + service_id: str | None = None, + selector: Selector | None = None, + generate_inputs_and_outputs: bool = False, +) -> Selection: + """Select declarations and their schema dependencies for standalone types. + + Unlike client selection, this function never infers a service. A selector + narrows declaration roots before their transitive member targets are added. + """ + service = ( + resolve_service(model, service_id=service_id) + if service_id is not None + else None + ) + declarations = tuple( + shape + for shape in model.iter_shapes() + if shape.type in DECLARATION_TYPES + and not shape.is_mixin + and not shape.has_trait("trait") + and not shape.has_trait("private") + ) + if service is None: + candidates = declarations + excluded_count = 0 + else: + closure = _closure(model, service.id) + candidates = tuple(shape for shape in declarations if shape.id in closure) + excluded_count = len(declarations) - len(candidates) + + resolved_selector = selector or Selector.identity() + selected_ids = ( + frozenset(shape.id for shape in candidates) + if resolved_selector.is_identity + else resolved_selector.select(model) + ) + operation_inputs_and_outputs = { + shape_id + for operation in model.iter_shapes(OperationShape) + for shape_id in (operation.input, operation.output) + } + roots = tuple( + shape + for shape in candidates + if shape.id in selected_ids + and ( + generate_inputs_and_outputs + or ( + shape.id not in operation_inputs_and_outputs + and not shape.has_trait("input") + and not shape.has_trait("output") + ) + ) + ) + + closure = {shape.id for shape in roots} + pending = list(closure) + while pending: + shape = model.get_shape(pending.pop()) + for member in shape.members.values(): + target = model.get_shape(member.target) + if ( + model.is_prelude(target) + or target.id in closure + or target.is_mixin + or target.has_trait("trait") + or target.type + in { + ShapeType.SERVICE, + ShapeType.OPERATION, + ShapeType.RESOURCE, + ShapeType.MEMBER, + } + ): + continue + closure.add(target.id) + pending.append(target.id) + + shapes = tuple(shape for shape in model.iter_shapes() if shape.id in closure) + _check_names( + tuple( + shape + for shape in shapes + if shape.type in DECLARATION_TYPES and not shape.has_trait("streaming") + ), + service=service, + ) + return Selection(service=service, shapes=shapes, excluded_count=excluded_count) + + +def _closure(model: Model, root: ShapeId) -> set[ShapeId]: visited: set[ShapeId] = set() - pending = [service.id] + pending = [root] while pending: shape_id = pending.pop() if shape_id in visited: continue visited.add(shape_id) pending.extend(_references(model.get_shape(shape_id))) - - selected = tuple(shape for shape in eligible if shape.id in visited) - return Selection( - service=service, shapes=selected, excluded_count=len(eligible) - len(selected) - ) + return visited def _references(shape: Shape) -> Iterator[ShapeId]: @@ -109,51 +252,16 @@ def _references(shape: Shape) -> Iterator[ShapeId]: yield from shape.resources -def _check_names(shapes: tuple[Shape, ...]) -> None: +def _check_names( + shapes: tuple[Shape, ...], *, service: ServiceShape | None = None +) -> None: by_name: dict[str, list[ShapeId]] = {} for shape in shapes: - by_name.setdefault(shape.id.name.casefold(), []).append(shape.id) + name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name + by_name.setdefault(name.casefold(), []).append(shape.id) conflicts = [ids for ids in by_name.values() if len(ids) > 1] if conflicts: details = "; ".join( ", ".join(str(shape_id) for shape_id in ids) for ids in conflicts ) raise ModelError(f"Case-insensitive data shape name conflicts: {details}") - - -def resolve_service( - model: Model, *, service_id: str | None = None, require_service: bool = False -) -> ServiceShape | None: - """Resolve an absolute service ID or the sole non-mixin service. - - Invalid or ambiguous selections raise :class:`InvalidInvocationError`. - """ - if service_id is not None: - try: - shape_id = ShapeId.from_string(service_id) - except InvalidShapeIdError as error: - raise InvalidInvocationError(f"--service: {error}") from error - if shape_id.is_member: - raise InvalidInvocationError( - f"--service {service_id!r} is a member ID, not a service" - ) - shape = model.find_shape(shape_id) - if shape is None: - raise InvalidInvocationError(f"--service {service_id!r} is not defined") - if not isinstance(shape, ServiceShape): - raise InvalidInvocationError(f"--service {service_id!r} is not a service") - if shape.is_mixin: - raise InvalidInvocationError(f"--service {service_id!r} is a mixin") - return shape - - services = model.services() - if len(services) > 1: - candidates = ", ".join(str(service.id) for service in services) - raise InvalidInvocationError( - f"Multiple services found; select one with --service: {candidates}" - ) - if services: - return services[0] - if require_service: - raise InvalidInvocationError("Client generation requires a service") - return None diff --git a/packages/smithy-python/src/smithy_python/symbols.py b/packages/smithy-python/src/smithy_python/symbols.py index f2657baa2..1fe1d417e 100644 --- a/packages/smithy-python/src/smithy_python/symbols.py +++ b/packages/smithy-python/src/smithy_python/symbols.py @@ -9,7 +9,9 @@ from collections.abc import Callable, Collection from dataclasses import dataclass, replace +from . import dependencies as _dependencies from .model import ( + DECLARATION_TYPES, ListShape, MapShape, MemberShape, @@ -23,17 +25,6 @@ __all__ = ["SymbolProvider", "TypeReference"] -# Union of Python 3.12-3.15 hard keywords, independent of the host interpreter. -# Soft keywords (match, case, type, lazy, _) remain ordinary identifiers here. -# Update this when adding support for another Python version. -_KEYWORDS = frozenset( - "False None True and as assert async await break class continue def del " - "elif else except finally for from global if import in is lambda nonlocal " - "not or pass raise return try while with yield".split() -) -_ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) -_NAMED = _ENUMS | {ShapeType.STRUCTURE, ShapeType.UNION} - @dataclass(frozen=True, slots=True) class TypeReference: @@ -41,61 +32,15 @@ class TypeReference: ``module=None`` denotes the None literal. ``nullable`` permits None in addition to this reference, and is used only for sparse collection entries. - No imports or annotation rendering are performed. + Dependencies are recorded when a writer consumes the reference. No imports + or annotation rendering are performed here. """ name: str module: str | None = None arguments: tuple[TypeReference, ...] = () nullable: bool = False - - -# Types whose reference never depends on the shape: Smithy's simple types, with -# enums widened to str/int so unknown values remain representable. -_FIXED_TYPES = { - ShapeType.BOOLEAN: TypeReference("bool", "builtins"), - ShapeType.STRING: TypeReference("str", "builtins"), - ShapeType.ENUM: TypeReference("str", "builtins"), - ShapeType.INT_ENUM: TypeReference("int", "builtins"), - **dict.fromkeys( - ( - ShapeType.BYTE, - ShapeType.SHORT, - ShapeType.INTEGER, - ShapeType.LONG, - ShapeType.BIG_INTEGER, - ), - TypeReference("int", "builtins"), - ), - ShapeType.FLOAT: TypeReference("float", "builtins"), - ShapeType.DOUBLE: TypeReference("float", "builtins"), - ShapeType.BLOB: TypeReference("bytes", "builtins"), - ShapeType.BIG_DECIMAL: TypeReference("Decimal", "decimal"), - ShapeType.TIMESTAMP: TypeReference("datetime", "datetime"), - ShapeType.DOCUMENT: TypeReference("Document", "smithy_core.documents"), -} - - -def _name( - value: str, - *, - pascal: bool = False, - constant: bool = False, - reserved: Collection[str] = (), -) -> str: - if pascal: - result = "".join(word[:1].upper() + word[1:] for word in value.split("_")) - else: - value = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1_\2", value) - value = re.sub(r"([a-z0-9])([A-Z])", r"\1_\2", value) - result = "_".join(word.lower() for word in value.split("_") if word) - if constant: - result = result.upper() - if result[:1].isdigit(): - result = "_" + result - if result in _KEYWORDS or result in reserved: - result += "_" - return result + dependencies: tuple[_dependencies.Dependency, ...] = () class SymbolProvider: @@ -119,10 +64,15 @@ def __init__( self._package = package self._reserved_member_names: dict[ShapeId, frozenset[str]] = {} self._type_references: dict[ShapeId, TypeReference] = {} + self._declaration_names: dict[ShapeId, str] = {} + self._schema_names = _plan_schema_names(selection) + self._union_variant_names: dict[ShapeId, str] = {} + self._union_unknown_names: dict[ShapeId, str] = {} self._selected = frozenset(shape.id for shape in selection.shapes) + self._shape_names: frozenset[str] = frozenset() declarations: dict[str, ShapeId] = {} for shape in selection.shapes: - if shape.type not in _NAMED or shape.has_trait("streaming"): + if shape.type not in DECLARATION_TYPES or shape.has_trait("streaming"): continue self._check_collision( declarations, self.declaration_name(shape.id), shape.id @@ -134,38 +84,25 @@ def __init__( members: dict[str, ShapeId] = {} for member in shape.members.values(): self._check_collision(members, self.member_name(member.id), member.id) - - @staticmethod - def _check_collision( - scope: dict[str, ShapeId], name: str, shape_id: ShapeId - ) -> None: - if name in scope: - raise ModelError( - f"Python name collision {name!r}: {scope[name]}, {shape_id}" + self._shape_names = frozenset(declarations) + for shape in selection.shapes: + if shape.type is not ShapeType.UNION or shape.has_trait("streaming"): + continue + for member in shape.members.values(): + self._check_collision( + declarations, self.union_variant_name(member.id), member.id + ) + self._check_collision( + declarations, self.union_unknown_name(shape.id), shape.id ) - scope[name] = shape_id - - def _shape(self, shape_id: ShapeId | str) -> Shape: - shape = self._model.get_shape(shape_id) - if shape.id.root not in self._selected and not self._model.is_prelude(shape): - raise ModelError(f"Shape {shape.id} is not selected") - if ( - shape.is_mixin - or shape.has_trait("trait") - or shape.type - in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) - ): - raise ModelError(f"No data symbol for {shape.id} ({shape.type})") - if shape.type in (ShapeType.BLOB, ShapeType.UNION) and shape.has_trait( - "streaming" - ): - raise ModelError(f"Unsupported streaming {shape.type}: {shape.id}") - return shape def declaration_name(self, shape_id: ShapeId | str) -> str: """Return a generated type's PascalCase name; aliases have no declaration.""" shape = self._shape(shape_id) - if shape.type not in _NAMED or self._model.is_prelude(shape): + cached = self._declaration_names.get(shape.id) + if cached is not None: + return cached + if shape.type not in DECLARATION_TYPES or self._model.is_prelude(shape): raise ModelError(f"No generated declaration for {shape.id}") service = self._selection.service name = service.rename.get(shape.id, shape.id.name) if service else shape.id.name @@ -174,6 +111,59 @@ def declaration_name(self, shape_id: ShapeId | str) -> str: raise ModelError( f"Invalid Python declaration name {result!r} for {shape.id}" ) + self._declaration_names[shape.id] = result + return result + + def schema_name(self, shape_id: ShapeId | str) -> str: + """Return the module-level compact schema constant name.""" + shape = self._selected_shape(shape_id) + if ( + shape.id.is_member + or shape.is_mixin + or shape.has_trait("trait") + or shape.type + in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) + ): + raise ModelError(f"No data schema for {shape.id} ({shape.type})") + cached = self._schema_names.get(shape.id) + if cached is not None: + return cached + if self._model.is_prelude(shape): + return _name(shape.id.name, constant=True) + raise ModelError(f"No schema name planned for {shape.id}") + + def union_variant_name(self, shape_id: ShapeId | str) -> str: + """Return the generated class name for a union member.""" + member = self._shape(shape_id) + if not isinstance(member, MemberShape): + raise ModelError(f"Union variant naming requires a member ID: {member.id}") + cached = self._union_variant_names.get(member.id) + if cached is not None: + return cached + container = self._shape(member.container) + if container.type is not ShapeType.UNION: + raise ModelError(f"{member.id} is not a union member") + container_name = self.declaration_name(container.id) + capitalized = member.name[:1].upper() + member.name[1:] + result = container_name + capitalized + if result in self._shape_names: + result = container_name + "_" + capitalized + self._union_variant_names[member.id] = result + return result + + def union_unknown_name(self, shape_id: ShapeId | str) -> str: + """Return the generated unknown-variant class name for a union.""" + shape = self._shape(shape_id) + cached = self._union_unknown_names.get(shape.id) + if cached is not None: + return cached + if shape.type is not ShapeType.UNION: + raise ModelError(f"{shape.id} is not a union") + container_name = self.declaration_name(shape.id) + result = container_name + "Unknown" + if result in self._shape_names: + result = container_name + "_Unknown" + self._union_unknown_names[shape.id] = result return result def type_reference(self, shape_id: ShapeId | str) -> TypeReference: @@ -199,7 +189,7 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: else (shape.key.target, shape.value.target) ) if expanded: - arguments = tuple(resolved[target] for target in targets) + arguments = tuple(self.type_reference(target) for target in targets) if shape.has_trait("sparse"): arguments = ( *arguments[:-1], @@ -226,6 +216,8 @@ def type_reference(self, shape_id: ShapeId | str) -> TypeReference: resolved[current] = TypeReference( self.declaration_name(current), f"{self._package}.models" ) + elif shape.type is ShapeType.BLOB and shape.has_trait("streaming"): + resolved[current] = _STREAMING_BLOB elif shape.type in _FIXED_TYPES: resolved[current] = _FIXED_TYPES[shape.type] else: @@ -240,10 +232,127 @@ def member_name(self, shape_id: ShapeId | str) -> str: if not isinstance(member, MemberShape): raise ModelError(f"Member naming requires a member ID: {member.id}") container = self._shape(member.container) - if container.type not in _NAMED or self._model.is_prelude(container): + if container.type not in DECLARATION_TYPES or self._model.is_prelude(container): raise ModelError(f"No generated member name for {member.id}") return _name( member.name, constant=container.type in _ENUMS, reserved=self._reserved_member_names.get(container.id, ()), ) + + @staticmethod + def _check_collision( + scope: dict[str, ShapeId], name: str, shape_id: ShapeId + ) -> None: + if name in scope: + raise ModelError( + f"Python name collision {name!r}: {scope[name]}, {shape_id}" + ) + scope[name] = shape_id + + def _selected_shape(self, shape_id: ShapeId | str) -> Shape: + shape = self._model.get_shape(shape_id) + if shape.id.root not in self._selected and not self._model.is_prelude(shape): + raise ModelError(f"Shape {shape.id} is not selected") + return shape + + def _shape(self, shape_id: ShapeId | str) -> Shape: + shape = self._selected_shape(shape_id) + if ( + shape.is_mixin + or shape.has_trait("trait") + or shape.type + in (ShapeType.SERVICE, ShapeType.OPERATION, ShapeType.RESOURCE) + ): + raise ModelError(f"No data symbol for {shape.id} ({shape.type})") + if shape.type is ShapeType.UNION and shape.has_trait("streaming"): + raise ModelError(f"Unsupported streaming {shape.type}: {shape.id}") + return shape + + +# Union of Python 3.12-3.15 hard keywords, independent of the host interpreter. +# Soft keywords (match, case, type, lazy, _) remain ordinary identifiers here. +# Update this when adding support for another Python version. +_KEYWORDS = frozenset( + "False None True and as assert async await break class continue def del " + "elif else except finally for from global if import in is lambda nonlocal " + "not or pass raise return try while with yield".split() +) +_ENUMS = frozenset({ShapeType.ENUM, ShapeType.INT_ENUM}) +_ACRONYM_BOUNDARY = re.compile(r"([A-Z]+)([A-Z][a-z])") +_WORD_BOUNDARY = re.compile(r"([a-z0-9])([A-Z])") + +_STREAMING_BLOB = TypeReference( + "StreamingBlob", + "smithy_core.aio.interfaces", + dependencies=(_dependencies.SMITHY_CORE,), +) + +# Types whose reference never depends on the shape: Smithy's simple types, with +# enums widened to str/int so unknown values remain representable. +_FIXED_TYPES = { + ShapeType.BOOLEAN: TypeReference("bool", "builtins"), + ShapeType.STRING: TypeReference("str", "builtins"), + ShapeType.ENUM: TypeReference("str", "builtins"), + ShapeType.INT_ENUM: TypeReference("int", "builtins"), + **dict.fromkeys( + ( + ShapeType.BYTE, + ShapeType.SHORT, + ShapeType.INTEGER, + ShapeType.LONG, + ShapeType.BIG_INTEGER, + ), + TypeReference("int", "builtins"), + ), + ShapeType.FLOAT: TypeReference("float", "builtins"), + ShapeType.DOUBLE: TypeReference("float", "builtins"), + ShapeType.BLOB: TypeReference("bytes", "builtins"), + ShapeType.BIG_DECIMAL: TypeReference("Decimal", "decimal"), + ShapeType.TIMESTAMP: TypeReference("datetime", "datetime"), + ShapeType.DOCUMENT: TypeReference( + "Document", "smithy_core.documents", dependencies=(_dependencies.SMITHY_CORE,) + ), +} + + +def _name( + value: str, + *, + pascal: bool = False, + constant: bool = False, + reserved: Collection[str] = (), +) -> str: + if pascal: + result = "".join(word[:1].upper() + word[1:] for word in value.split("_")) + else: + value = _ACRONYM_BOUNDARY.sub(r"\1_\2", value) + value = _WORD_BOUNDARY.sub(r"\1_\2", value) + result = "_".join(word.lower() for word in value.split("_") if word) + if constant: + result = result.upper() + if result[:1].isdigit(): + result = "_" + result + if result in _KEYWORDS or result in reserved: + result += "_" + return result + + +def _plan_schema_names(selection: Selection) -> dict[ShapeId, str]: + names: dict[ShapeId, str] = {} + used: set[str] = set() + for shape in selection.shapes: + selected = ( + selection.service.rename.get(shape.id, shape.id.name) + if selection.service + else shape.id.name + ) + base = _name(selected, constant=True) + name = base + suffix = 2 + while name in used: + name = f"{base}_{suffix}" + suffix += 1 + used.add(name) + names[shape.id] = name + return names diff --git a/packages/smithy-python/src/smithy_python/writer.py b/packages/smithy-python/src/smithy_python/writer.py index b9b10208b..d91ffdf2e 100644 --- a/packages/smithy-python/src/smithy_python/writer.py +++ b/packages/smithy-python/src/smithy_python/writer.py @@ -16,6 +16,7 @@ from typing import Literal from unicodedata import normalize +from . import dependencies as _dependencies from .exceptions import CodegenError from .symbols import TypeReference @@ -53,6 +54,218 @@ _COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) +class PythonWriter: + """Write trusted Python lines, retaining type references until rendering.""" + + def __init__( + self, module: str, *, dependencies: _dependencies.DependencySet | None = None + ) -> None: + _validate_module(module, relative=False) + self._module = _binding(module) + self._dependencies = dependencies + self._depth = 0 + self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] + + def line(self, *parts: str | TypeReference) -> None: + """Append one line, concatenating its parts without separators.""" + if self._dependencies is not None: + pending = [part for part in parts if isinstance(part, TypeReference)] + while pending: + reference = pending.pop() + for dependency in reference.dependencies: + self._dependencies.add(dependency) + pending.extend(reference.arguments) + self._lines.append((self._depth, parts)) + + @contextmanager + def indent(self) -> Generator[None]: + """Indent by four spaces for the duration of the block.""" + self._depth += 1 + try: + yield + finally: + self._depth -= 1 + + def render(self) -> str: + """Return complete source without changing the recorded lines.""" + names, imports = self._plan_imports(*self._names()) + header = "from __future__ import annotations\n" + if imports: + header += "\n" + "\n".join(item.statement() for item in imports) + "\n" + body = self._body(lambda reference: self._annotation(reference, names)) + return header + ("\n\n" + "\n".join(body) + "\n" if body else "") + + def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: + body: list[str] = [] + for depth, parts in self._lines: + text = "".join( + part if isinstance(part, str) else reference(part) for part in parts + ) + body.append(" " * depth + text if text else "") + return body + + def _references(self) -> list[TypeReference]: + return [ + part + for _, parts in self._lines + for part in parts + if isinstance(part, TypeReference) + ] + + def _names(self) -> tuple[frozenset[str], list[frozenset[str]]]: + """Return module-level names and the local names each reference sees.""" + prefix = _placeholder_prefix("".join(self._body(lambda _: ""))) + numbers = itertools.count() + source = "\n".join(self._body(lambda _: f"{prefix}{next(numbers)}_")) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", SyntaxWarning) + try: + tree = ast.parse(source, feature_version=(3, 12)) + except SyntaxError as error: + raise CodegenError( + f"Written lines are not valid Python 3.12 (line {error.lineno}): " + f"{error.msg}" + ) from error + for warning in caught: + # Invalid escape sequences become errors in future Python versions. + if issubclass(warning.category, SyntaxWarning): + raise CodegenError( + f"Written lines raise SyntaxWarning (line {warning.lineno}): " + f"{warning.message}" + ) + return _analyze(tree, prefix, len(self._references())) + + def _plan_imports( + self, module_names: frozenset[str], shadows: list[frozenset[str]] + ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: + identities: set[tuple[str | None, str]] = set() + # The local names visible where each identity is referenced. References + # in one scope share a set, so keep the distinct sets, not their union. + local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} + for reference, shadow in zip(self._references(), shadows, strict=True): + pending = [reference] + while pending: + ref = pending.pop() + if ref.module is None and (ref.name != "None" or ref.arguments): + raise CodegenError( + f"Only the None literal may omit its module: {ref.name!r}" + ) + if ref.module is not None: + _validate_identifier(ref.name, "reference name") + _validate_module(ref.module, relative=True) + identities.add((ref.module, ref.name)) + key = _identity(ref.module, ref.name) + local_names.setdefault(key, set()).add(shadow) + pending.extend(ref.arguments) + # Equivalent Python identifiers share one import. Pick a supplied spelling + # deterministically, without rewriting the caller's source names. + representatives: dict[tuple[str | None, str], tuple[str | None, str]] = {} + for module, name in sorted( + identities, key=lambda item: (item[0] or "", item[1]) + ): + representatives.setdefault(_identity(module, name), (module, name)) + current_names = { + name for module, name in representatives if module == self._module + } + for name in sorted(current_names): + if _is_local(name, local_names[(self._module, name)]): + raise CodegenError( + f"Same-module reference {self._module}.{name} conflicts with local name {name!r}" + ) + reserved = module_names | current_names + external = sorted( + (module, name) + for (module_binding, _), (module, name) in representatives.items() + if module is not None and module_binding not in ("builtins", self._module) + ) + counts = Counter(_binding(name) for _, name in external) + names: dict[tuple[str | None, str], str] = {} + # Each import with the local names of the places that use it. + imports: list[tuple[_Import, set[frozenset[str]]]] = [] + for module, name in external: + binding = _binding(name) + scoped = local_names[_identity(module, name)] + alias = ( + f"_{module.replace('.', '_')}_{name}" + if binding in reserved + or _is_local(binding, scoped) + or binding in _BUILTINS + or binding == "annotations" + or counts[binding] > 1 + else name + ) + names[(module, name)] = alias + imports.append((_Import(module, name, alias), scoped)) + qualified: set[frozenset[str]] = set() + for key, (module, name) in representatives.items(): + module_binding, name_binding = key + if module is None: + names[(module, name)] = "None" + elif module_binding == "builtins": + scoped = local_names[key] + if name_binding in reserved or _is_local(name_binding, scoped): + names[(module, name)] = f"_builtins.{name}" + qualified |= scoped + else: + names[(module, name)] = name + elif module_binding == self._module: + names[(module, name)] = name + if qualified: + imports.append((_Import("builtins", None, "_builtins"), qualified)) + + # Check the complete plan, including unaliased imports. No binding gets + # priority merely because its reference was encountered first. + bindings = dict.fromkeys(reserved, "a module-level name") + imports.sort( + key=lambda item: (item[0].module, item[0].name or "", item[0].alias) + ) + for item, scoped in imports: + owner = f"{item.module}.{item.name}" if item.name else item.module + binding = _binding(item.alias) + if binding in bindings: + conflict = bindings[binding] + elif binding in _BUILTINS: + conflict = "a builtin" + elif _is_local(binding, scoped): + conflict = "a local name" + else: + bindings[binding] = owner + continue + raise CodegenError( + f"Import binding {item.alias!r} for {owner} conflicts with {conflict}" + ) + for module, name in identities: + names[(module, name)] = names[representatives[_identity(module, name)]] + return names, [item for item, _ in imports] + + @staticmethod + def _annotation( + reference: TypeReference, names: dict[tuple[str | None, str], str] + ) -> str: + # Emit tokens rather than recursing through potentially deep collections. + pending: list[str | TypeReference] = [reference] + result: list[str] = [] + while pending: + part = pending.pop() + if isinstance(part, str): + result.append(part) + continue + if part.module is None: + result.append("None") + continue + result.append(names[(part.module, part.name)]) + if part.nullable: + pending.append(" | None") + if part.arguments: + pending.append("]") + for index in range(len(part.arguments) - 1, -1, -1): + pending.append(part.arguments[index]) + if index: + pending.append(", ") + pending.append("[") + return "".join(result) + + @dataclass(frozen=True, slots=True) class _Import: """One planned import. ``name=None`` imports the module itself.""" @@ -342,205 +555,3 @@ def type_scope( visible[scope] = frozenset(scope.visible_names()) shadows.append(visible[scope]) return frozenset(module.names), shadows - - -class PythonWriter: - """Write trusted Python lines, retaining type references until rendering.""" - - def __init__(self, module: str) -> None: - _validate_module(module, relative=False) - self._module = _binding(module) - self._depth = 0 - self._lines: list[tuple[int, tuple[str | TypeReference, ...]]] = [] - - def line(self, *parts: str | TypeReference) -> None: - """Append one line, concatenating its parts without separators.""" - self._lines.append((self._depth, parts)) - - @contextmanager - def indent(self) -> Generator[None]: - """Indent by four spaces for the duration of the block.""" - self._depth += 1 - try: - yield - finally: - self._depth -= 1 - - def _body(self, reference: Callable[[TypeReference], str]) -> list[str]: - body: list[str] = [] - for depth, parts in self._lines: - text = "".join( - part if isinstance(part, str) else reference(part) for part in parts - ) - body.append(" " * depth + text if text else "") - return body - - def _references(self) -> list[TypeReference]: - return [ - part - for _, parts in self._lines - for part in parts - if isinstance(part, TypeReference) - ] - - def _names(self) -> tuple[frozenset[str], list[frozenset[str]]]: - """Return module-level names and the local names each reference sees.""" - prefix = _placeholder_prefix("".join(self._body(lambda _: ""))) - numbers = itertools.count() - source = "\n".join(self._body(lambda _: f"{prefix}{next(numbers)}_")) - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always", SyntaxWarning) - try: - tree = ast.parse(source, feature_version=(3, 12)) - except SyntaxError as error: - raise CodegenError( - f"Written lines are not valid Python 3.12 (line {error.lineno}): " - f"{error.msg}" - ) from error - for warning in caught: - # Invalid escape sequences become errors in future Python versions. - if issubclass(warning.category, SyntaxWarning): - raise CodegenError( - f"Written lines raise SyntaxWarning (line {warning.lineno}): " - f"{warning.message}" - ) - return _analyze(tree, prefix, len(self._references())) - - def _plan_imports( - self, module_names: frozenset[str], shadows: list[frozenset[str]] - ) -> tuple[dict[tuple[str | None, str], str], list[_Import]]: - identities: set[tuple[str | None, str]] = set() - # The local names visible where each identity is referenced. References - # in one scope share a set, so keep the distinct sets, not their union. - local_names: dict[tuple[str | None, str], set[frozenset[str]]] = {} - for reference, shadow in zip(self._references(), shadows, strict=True): - pending = [reference] - while pending: - ref = pending.pop() - if ref.module is None and (ref.name != "None" or ref.arguments): - raise CodegenError( - f"Only the None literal may omit its module: {ref.name!r}" - ) - if ref.module is not None: - _validate_identifier(ref.name, "reference name") - _validate_module(ref.module, relative=True) - identities.add((ref.module, ref.name)) - key = _identity(ref.module, ref.name) - local_names.setdefault(key, set()).add(shadow) - pending.extend(ref.arguments) - # Equivalent Python identifiers share one import. Pick a supplied spelling - # deterministically, without rewriting the caller's source names. - representatives: dict[tuple[str | None, str], tuple[str | None, str]] = {} - for module, name in sorted( - identities, key=lambda item: (item[0] or "", item[1]) - ): - representatives.setdefault(_identity(module, name), (module, name)) - current_names = { - name for module, name in representatives if module == self._module - } - for name in sorted(current_names): - if _is_local(name, local_names[(self._module, name)]): - raise CodegenError( - f"Same-module reference {self._module}.{name} conflicts with local name {name!r}" - ) - reserved = module_names | current_names - external = sorted( - (module, name) - for (module_binding, _), (module, name) in representatives.items() - if module is not None and module_binding not in ("builtins", self._module) - ) - counts = Counter(_binding(name) for _, name in external) - names: dict[tuple[str | None, str], str] = {} - # Each import with the local names of the places that use it. - imports: list[tuple[_Import, set[frozenset[str]]]] = [] - for module, name in external: - binding = _binding(name) - scoped = local_names[_identity(module, name)] - alias = ( - f"_{module.replace('.', '_')}_{name}" - if binding in reserved - or _is_local(binding, scoped) - or binding in _BUILTINS - or binding == "annotations" - or counts[binding] > 1 - else name - ) - names[(module, name)] = alias - imports.append((_Import(module, name, alias), scoped)) - qualified: set[frozenset[str]] = set() - for key, (module, name) in representatives.items(): - module_binding, name_binding = key - if module is None: - names[(module, name)] = "None" - elif module_binding == "builtins": - scoped = local_names[key] - if name_binding in reserved or _is_local(name_binding, scoped): - names[(module, name)] = f"_builtins.{name}" - qualified |= scoped - else: - names[(module, name)] = name - elif module_binding == self._module: - names[(module, name)] = name - if qualified: - imports.append((_Import("builtins", None, "_builtins"), qualified)) - - # Check the complete plan, including unaliased imports. No binding gets - # priority merely because its reference was encountered first. - bindings = dict.fromkeys(reserved, "a module-level name") - imports.sort( - key=lambda item: (item[0].module, item[0].name or "", item[0].alias) - ) - for item, scoped in imports: - owner = f"{item.module}.{item.name}" if item.name else item.module - binding = _binding(item.alias) - if binding in bindings: - conflict = bindings[binding] - elif binding in _BUILTINS: - conflict = "a builtin" - elif _is_local(binding, scoped): - conflict = "a local name" - else: - bindings[binding] = owner - continue - raise CodegenError( - f"Import binding {item.alias!r} for {owner} conflicts with {conflict}" - ) - for module, name in identities: - names[(module, name)] = names[representatives[_identity(module, name)]] - return names, [item for item, _ in imports] - - @staticmethod - def _annotation( - reference: TypeReference, names: dict[tuple[str | None, str], str] - ) -> str: - # Emit tokens rather than recursing through potentially deep collections. - pending: list[str | TypeReference] = [reference] - result: list[str] = [] - while pending: - part = pending.pop() - if isinstance(part, str): - result.append(part) - continue - if part.module is None: - result.append("None") - continue - result.append(names[(part.module, part.name)]) - if part.nullable: - pending.append(" | None") - if part.arguments: - pending.append("]") - for index in range(len(part.arguments) - 1, -1, -1): - pending.append(part.arguments[index]) - if index: - pending.append(", ") - pending.append("[") - return "".join(result) - - def render(self) -> str: - """Return complete source without changing the recorded lines.""" - names, imports = self._plan_imports(*self._names()) - header = "from __future__ import annotations\n" - if imports: - header += "\n" + "\n".join(item.statement() for item in imports) + "\n" - body = self._body(lambda reference: self._annotation(reference, names)) - return header + ("\n\n" + "\n".join(body) + "\n" if body else "") diff --git a/packages/smithy-python/tests/unit/test_data_selection.py b/packages/smithy-python/tests/unit/test_data_selection.py index 894cc7188..1439e9c8d 100644 --- a/packages/smithy-python/tests/unit/test_data_selection.py +++ b/packages/smithy-python/tests/unit/test_data_selection.py @@ -6,7 +6,7 @@ import pytest from smithy_python.exceptions import InvalidInvocationError, ModelError from smithy_python.model import Model, load_model -from smithy_python.selection import select_shapes +from smithy_python.selection import Selector, select_shapes, select_type_shapes def _model(shapes: dict[str, object]) -> Model: @@ -310,3 +310,134 @@ def test_direct_api_errors_and_empty_selection() -> None: select_shapes( _model({"a#Name": {"type": "string"}, "b#name": {"type": "string"}}) ) + + +def test_type_selection_does_not_infer_service_and_adds_dependencies() -> None: + model = _model( + { + "a#First": {"type": "service", "errors": [{"target": "a#Hidden"}]}, + "a#Second": {"type": "service", "errors": [{"target": "a#Public"}]}, + "a#Shared": {"type": "string"}, + "a#Public": { + "type": "structure", + "members": {"value": {"target": "a#Shared"}}, + }, + "a#Hidden": {"type": "structure"}, + "a#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + }, + "a#Private": { + "type": "structure", + "traits": {"smithy.api#private": {}}, + }, + } + ) + + result = select_type_shapes( + model, selector=Selector.parse("structure [id|name=Public]") + ) + assert result.service is None + assert [str(shape.id) for shape in result.shapes] == ["a#Shared", "a#Public"] + + result = select_type_shapes(model, service_id="a#Second") + assert result.service is model.get_shape("a#Second") + assert result.shapes == ( + model.get_shape("a#Shared"), + model.get_shape("a#Public"), + ) + assert result.excluded_count == 2 + + +def test_type_selection_controls_input_output_roots() -> None: + model = _model( + { + "a#Shared": {"type": "string"}, + "a#Input": { + "type": "structure", + "traits": {"smithy.api#input": {}}, + "members": {"value": {"target": "a#Shared"}}, + }, + "a#Output": { + "type": "structure", + "traits": {"smithy.api#output": {}}, + }, + "a#ImplicitInput": { + "type": "structure", + "members": {"value": {"target": "a#Shared"}}, + }, + "a#ImplicitOutput": {"type": "structure"}, + "a#Operation": { + "type": "operation", + "input": {"target": "a#ImplicitInput"}, + "output": {"target": "a#ImplicitOutput"}, + }, + "a#Data": {"type": "structure"}, + } + ) + + assert [str(shape.id) for shape in select_type_shapes(model).shapes] == ["a#Data"] + assert [ + str(shape.id) + for shape in select_type_shapes(model, generate_inputs_and_outputs=True).shapes + ] == [ + "a#Shared", + "a#Input", + "a#Output", + "a#ImplicitInput", + "a#ImplicitOutput", + "a#Data", + ] + + +def test_type_selection_checks_only_selected_declaration_names() -> None: + model = _model( + { + "a#Name": {"type": "structure"}, + "b#name": {"type": "structure"}, + } + ) + result = select_type_shapes(model, selector=Selector.parse("[id|namespace=a]")) + assert result.shapes == (model.get_shape("a#Name"),) + + with pytest.raises(ModelError, match="a#Name, b#name"): + select_type_shapes(model) + + +def test_type_selection_excludes_streaming_unions_from_name_checks() -> None: + model = _model( + { + "a#Events": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + }, + "b#Events": {"type": "structure"}, + } + ) + + result = select_type_shapes(model) + + assert result.shapes == ( + model.get_shape("a#Events"), + model.get_shape("b#Events"), + ) + + +def test_type_selection_uses_service_renames_for_name_conflicts() -> None: + model = _model( + { + "a#Service": { + "type": "service", + "errors": [{"target": "a#Name"}, {"target": "b#name"}], + "rename": {"b#name": "OtherName"}, + }, + "a#Name": {"type": "structure"}, + "b#name": {"type": "structure"}, + } + ) + + result = select_type_shapes(model, service_id="a#Service") + assert result.shapes == ( + model.get_shape("a#Name"), + model.get_shape("b#name"), + ) diff --git a/packages/smithy-python/tests/unit/test_dependencies.py b/packages/smithy-python/tests/unit/test_dependencies.py new file mode 100644 index 000000000..f295d1533 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_dependencies.py @@ -0,0 +1,70 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from smithy_python.dependencies import ( + HATCHLING, + SMITHY_CORE, + Dependency, + DependencySet, +) +from smithy_python.exceptions import CodegenError + + +def test_dependency_normalizes_extras() -> None: + dependency = Dependency( + package="example", + specifier="~=1.0", + group="runtime", + extras=("xml", "auth", "xml"), + ) + assert dependency.extras == ("auth", "xml") + + +@pytest.mark.parametrize( + "dependency", + [ + lambda: Dependency("", "", "runtime"), + lambda: Dependency(" example", "", "runtime"), + lambda: Dependency("example", "", "invalid"), # type: ignore[arg-type] + lambda: Dependency("example", "", "runtime", ("",)), + ], +) +def test_dependency_rejects_invalid_values(dependency: object) -> None: + with pytest.raises(ValueError): + dependency() # type: ignore[operator] + + +def test_dependency_set_merges_extras_and_orders_groups() -> None: + dependencies = DependencySet( + ( + Dependency("docs", "", "docs"), + Dependency("testing", "", "test"), + Dependency("builder", "", "build"), + Dependency("z-runtime", "~=1.0", "runtime", ("xml",)), + ) + ) + dependencies.add(Dependency("z-runtime", "~=1.0", "runtime", ("auth", "xml"))) + dependencies.add(Dependency("a-runtime", "~=2.0", "runtime")) + + assert tuple(dependencies) == ( + Dependency("a-runtime", "~=2.0", "runtime"), + Dependency("z-runtime", "~=1.0", "runtime", ("auth", "xml")), + Dependency("builder", "", "build"), + Dependency("testing", "", "test"), + Dependency("docs", "", "docs"), + ) + assert dependencies.for_group("runtime") == tuple(dependencies)[:2] + + +def test_dependency_set_rejects_specifier_conflicts() -> None: + dependencies = DependencySet((Dependency("example", "~=1.0", "runtime"),)) + with pytest.raises(CodegenError, match="Conflicting runtime dependency"): + dependencies.add(Dependency("example", "~=2.0", "runtime")) + + assert tuple(dependencies) == (Dependency("example", "~=1.0", "runtime"),) + + +def test_runtime_dependency_catalog() -> None: + assert SMITHY_CORE == Dependency("smithy-core", "~=0.10.0", "runtime") + assert HATCHLING == Dependency("hatchling", "", "build") diff --git a/packages/smithy-python/tests/unit/test_files.py b/packages/smithy-python/tests/unit/test_files.py new file mode 100644 index 000000000..098d0e54f --- /dev/null +++ b/packages/smithy-python/tests/unit/test_files.py @@ -0,0 +1,255 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.exceptions import CodegenError +from smithy_python.files import FileManifest +from smithy_python.symbols import TypeReference + +_OWNERSHIP_MANIFEST = ".smithy-python-manifest.json" + + +def test_file_manifest_reuses_python_writer_and_records_dependencies( + tmp_path: Path, +) -> None: + dependencies = DependencySet() + files = FileManifest(dependencies) + writer = files.python_module("example/models.py", "example.models") + assert writer is files.python_module("example/models.py", "example.models") + writer.line( + "value: ", + TypeReference( + "list", + "builtins", + ( + TypeReference( + "Document", + "smithy_core.documents", + dependencies=(SMITHY_CORE,), + ), + ), + ), + ) + files.text_file("example/py.typed", "") + + output = tmp_path / "output" + files.commit(output) + + assert ( + output.joinpath("example/models.py").read_text(encoding="utf-8") + == "from __future__ import annotations\n\n" + "from smithy_core.documents import Document\n\n\n" + "value: list[Document]\n" + ) + assert output.joinpath("example/py.typed").read_text(encoding="utf-8") == "" + assert tuple(dependencies) == (SMITHY_CORE,) + ownership = json.loads( + output.joinpath(_OWNERSHIP_MANIFEST).read_text(encoding="utf-8") + ) + assert ownership["version"] == 1 + assert tuple(ownership["files"]) == ("example/models.py", "example/py.typed") + assert all(len(digest) == 64 for digest in ownership["files"].values()) + + +@pytest.mark.parametrize( + "path", + ( + "", + "/models.py", + "example//models.py", + "./models.py", + "../models.py", + "a\\b", + "D:/outside/models.py", + "D:outside/models.py", + "nested/D:/outside/models.py", + ), +) +def test_file_manifest_rejects_invalid_paths(path: str) -> None: + files = FileManifest() + with pytest.raises(CodegenError, match="Invalid generated path"): + files.text_file(path, "") + + +def test_file_manifest_rejects_conflicting_claims(tmp_path: Path) -> None: + files = FileManifest() + files.python_module("models.py", "example.models") + with pytest.raises(CodegenError, match="already uses module"): + files.python_module("models.py", "other.models") + with pytest.raises(CodegenError, match="already claimed"): + files.text_file("models.py", "") + with pytest.raises(CodegenError, match="reserved"): + files.text_file(_OWNERSHIP_MANIFEST, "") + + files.text_file("package", "") + files.text_file("package/models.py", "") + with pytest.raises(CodegenError, match="conflicts"): + files.commit(tmp_path / "output") + + +def test_validation_failure_leaves_existing_output_untouched(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + existing = output / "existing.py" + existing.write_text("value = 'original'\n", encoding="utf-8") + + files = FileManifest() + files.python_module("invalid.py", "invalid").line("if") + with pytest.raises(CodegenError, match="not valid Python"): + files.commit(output) + + assert existing.read_text(encoding="utf-8") == "value = 'original'\n" + assert not output.joinpath("invalid.py").exists() + assert not output.joinpath(_OWNERSHIP_MANIFEST).exists() + + +def test_unowned_current_file_is_not_overwritten(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + existing = output / "current.txt" + existing.write_text("user\n", encoding="utf-8") + + files = FileManifest() + files.text_file("current.txt", "generated\n") + with pytest.raises(CodegenError, match="not owned"): + files.commit(output) + + assert existing.read_text(encoding="utf-8") == "user\n" + assert not output.joinpath(_OWNERSHIP_MANIFEST).exists() + + +def test_owned_current_file_is_replaced_even_when_modified(tmp_path: Path) -> None: + output = tmp_path / "output" + first = FileManifest() + first.text_file("current.txt", "first\n") + first.commit(output) + output.joinpath("current.txt").write_text("modified\n", encoding="utf-8") + + second = FileManifest() + second.text_file("current.txt", "second\n") + second.commit(output) + + assert output.joinpath("current.txt").read_text(encoding="utf-8") == "second\n" + + +def test_stale_owned_files_are_deleted_only_when_unmodified(tmp_path: Path) -> None: + output = tmp_path / "output" + first = FileManifest() + first.text_file("delete.txt", "generated\n") + first.text_file("retain.txt", "generated\n") + first.commit(output) + output.joinpath("retain.txt").write_text("modified\n", encoding="utf-8") + output.joinpath("unowned.txt").write_text("unowned\n", encoding="utf-8") + + second = FileManifest() + second.text_file("current.txt", "current\n") + second.commit(output) + + assert not output.joinpath("delete.txt").exists() + assert output.joinpath("retain.txt").read_text(encoding="utf-8") == "modified\n" + assert output.joinpath("unowned.txt").read_text(encoding="utf-8") == "unowned\n" + ownership = json.loads( + output.joinpath(_OWNERSHIP_MANIFEST).read_text(encoding="utf-8") + ) + assert tuple(ownership["files"]) == ("current.txt",) + + +def test_stale_owned_path_changed_to_directory_is_retained(tmp_path: Path) -> None: + output = tmp_path / "output" + first = FileManifest() + first.text_file("stale", "generated\n") + first.commit(output) + output.joinpath("stale").unlink() + output.joinpath("stale").mkdir() + output.joinpath("stale/user.txt").write_text("user\n", encoding="utf-8") + + FileManifest().commit(output) + + assert output.joinpath("stale/user.txt").read_text(encoding="utf-8") == "user\n" + + +def test_stale_owned_path_with_symlinked_ancestor_is_retained(tmp_path: Path) -> None: + output = tmp_path / "output" + first = FileManifest() + first.text_file("old/file.txt", "generated\n") + first.commit(output) + output.joinpath("old/file.txt").unlink() + output.joinpath("old").rmdir() + output.joinpath("current").mkdir() + try: + output.joinpath("old").symlink_to("current", target_is_directory=True) + except OSError as error: + pytest.skip(f"Cannot create symlinks: {error}") + + second = FileManifest() + second.text_file("current/file.txt", "generated\n") + second.commit(output) + + assert output.joinpath("current/file.txt").read_text() == "generated\n" + + +def test_without_previous_manifest_existing_files_are_not_deleted( + tmp_path: Path, +) -> None: + output = tmp_path / "output" + output.mkdir() + existing = output / "existing.txt" + existing.write_text("existing\n", encoding="utf-8") + + FileManifest().commit(output) + + assert existing.read_text(encoding="utf-8") == "existing\n" + + +def test_symlink_that_escapes_output_is_rejected(tmp_path: Path) -> None: + output = tmp_path / "output" + outside = tmp_path / "outside" + output.mkdir() + outside.mkdir() + try: + output.joinpath("escape").symlink_to(outside, target_is_directory=True) + except OSError as error: + pytest.skip(f"Cannot create symlinks: {error}") + + files = FileManifest() + files.text_file("escape/generated.txt", "content\n") + with pytest.raises(CodegenError, match="escapes"): + files.commit(output) + + assert not outside.joinpath("generated.txt").exists() + + +def test_symlinked_destination_ancestor_is_rejected(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + output.joinpath("real").mkdir() + try: + output.joinpath("alias").symlink_to("real", target_is_directory=True) + except OSError as error: + pytest.skip(f"Cannot create symlinks: {error}") + + files = FileManifest() + files.text_file("alias/generated.txt", "content\n") + with pytest.raises(CodegenError, match="symlinked ancestor"): + files.commit(output) + + assert not output.joinpath("real/generated.txt").exists() + + +def test_invalid_previous_manifest_prevents_publication(tmp_path: Path) -> None: + output = tmp_path / "output" + output.mkdir() + output.joinpath(_OWNERSHIP_MANIFEST).write_text("{}", encoding="utf-8") + files = FileManifest() + files.text_file("generated.txt", "content\n") + + with pytest.raises(CodegenError, match="Invalid output ownership manifest"): + files.commit(output) + + assert not output.joinpath("generated.txt").exists() diff --git a/packages/smithy-python/tests/unit/test_generation.py b/packages/smithy-python/tests/unit/test_generation.py new file mode 100644 index 000000000..2b001d0ab --- /dev/null +++ b/packages/smithy-python/tests/unit/test_generation.py @@ -0,0 +1,136 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest +from smithy_python.dependencies import DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import ( + ArtifactSettings, + GenerationContext, + GenerationPlan, + TypesSettings, +) +from smithy_python.model import load_model +from smithy_python.selection import Selector, select_shapes, select_type_shapes +from smithy_python.symbols import SymbolProvider + + +def test_artifact_settings_are_immutable_values() -> None: + settings = ArtifactSettings( + package="example.types", + package_version="1.2.3", + package_description=None, + ) + assert settings.package == "example.types" + assert settings.package_version == "1.2.3" + + +def test_types_settings_default_to_identity_selection() -> None: + settings = TypesSettings( + package="example.types", + package_version="1.2.3", + package_description="Example types", + ) + assert settings.selector == Selector.identity() + assert not settings.generate_inputs_and_outputs + + +def test_generation_context_shares_manifest_dependencies() -> None: + model = load_model('{"smithy": "2.0"}') + selection = select_shapes(model) + settings = TypesSettings("example", "1.0.0", None) + symbols = SymbolProvider(model, selection, package=settings.package) + dependencies = DependencySet() + context = GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + assert context.files.dependencies is context.dependencies + + with pytest.raises(ValueError, match="must be shared"): + GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=context.plan, + dependencies=dependencies, + files=FileManifest(), + ) + + +def test_generation_plan_orders_schemas_and_plans_names() -> None: + model = load_model( + { + "smithy": "2.0", + "shapes": { + "example#Leaf": {"type": "string"}, + "example#CycleB": { + "type": "structure", + "members": {"a": {"target": "example#CycleA"}}, + }, + "example#CycleA": { + "type": "structure", + "members": {"b": {"target": "example#CycleB"}}, + }, + "example#Root": { + "type": "structure", + "members": {"leaf": {"target": "example#Leaf"}}, + }, + "example#Choice": { + "type": "union", + "members": {"foo": {"target": "example#Leaf"}}, + }, + "example#ChoiceFoo": {"type": "structure"}, + "example#ChoiceUnknown": {"type": "structure"}, + }, + } + ) + selection = select_type_shapes(model) + symbols = SymbolProvider(model, selection, package="example") + plan = GenerationPlan.from_selection(model, selection, symbols) + + assert tuple(map(str, (shape.id for shape in plan.schemas.shapes))) == ( + "example#Leaf", + "example#Root", + "example#Choice", + "example#ChoiceFoo", + "example#ChoiceUnknown", + "example#CycleB", + "example#CycleA", + ) + assert plan.schemas.names[model.get_shape("example#Choice").id] == "CHOICE" + assert ( + plan.models.union_variant_names[model.get_shape("example#Choice$foo").id] + == "Choice_Foo" + ) + assert ( + plan.models.union_unknown_names[model.get_shape("example#Choice").id] + == "Choice_Unknown" + ) + assert plan.schemas.strings["example"] == "n0" + + +def test_generation_plan_keeps_streaming_schemas_out_of_model_declarations() -> None: + model = load_model( + { + "smithy": "2.0", + "shapes": { + "example#Events": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + } + }, + } + ) + selection = select_type_shapes(model) + symbols = SymbolProvider(model, selection, package="example") + plan = GenerationPlan.from_selection(model, selection, symbols) + shape_id = model.get_shape("example#Events").id + assert plan.schemas.names[shape_id] == "EVENTS" + assert shape_id not in plan.models.declaration_names diff --git a/packages/smithy-python/tests/unit/test_schemas.py b/packages/smithy-python/tests/unit/test_schemas.py new file mode 100644 index 000000000..ca5bc72cb --- /dev/null +++ b/packages/smithy-python/tests/unit/test_schemas.py @@ -0,0 +1,365 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# pyright: reportPrivateUsage=false + +from __future__ import annotations + +import runpy +from collections.abc import Mapping +from pathlib import Path +from types import MappingProxyType +from typing import Any, cast + +import pytest +from smithy_core import _schema_compact, prelude +from smithy_core.shapes import ShapeType as RuntimeShapeType +from smithy_core.traits import DefaultTrait, RequiredTrait, XMLNamespaceTrait +from smithy_python.dependencies import SMITHY_CORE, DependencySet +from smithy_python.exceptions import CodegenError +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import ShapeId, load_model +from smithy_python.schemas import ( + _ENCODING_VERSION, + _PRELUDE_INTS, + _TRAIT_BITS, + _TYPE_INTS, + _UNIT_INT, + SchemaGenerator, + _StringStore, +) +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + + +def _context(model_data: Mapping[str, object]) -> GenerationContext[TypesSettings]: + model = load_model(model_data) + selection = select_type_shapes(model, generate_inputs_and_outputs=True) + settings = TypesSettings("example.types", "1.0.0", None) + symbols = SymbolProvider(model, selection, package=settings.package) + dependencies = DependencySet() + return GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + + +def test_compact_encoding_contract_matches_runtime() -> None: + assert _ENCODING_VERSION == _schema_compact._ENCODING_VERSION + assert tuple(map(str, _TRAIT_BITS)) == _schema_compact._TRAIT_BITS + assert _UNIT_INT == _schema_compact.UNIT + assert {*_PRELUDE_INTS.values(), _UNIT_INT} == set(_schema_compact._PRELUDE) + for shape_type, integer in _TYPE_INTS.items(): + assert RuntimeShapeType[shape_type.name].value == integer + + +def test_schema_plan_filters_traits_and_synthesizes_input_output() -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#protocol": { + "type": "structure", + "traits": { + "smithy.api#trait": {}, + "smithy.api#protocolDefinition": { + "traits": ["example#wireTrait"] + }, + }, + }, + "example#wireTrait": { + "type": "structure", + "traits": {"smithy.api#trait": {}}, + }, + "example#Data": { + "type": "structure", + "traits": { + "example#protocol": {}, + "example#wireTrait": {"enabled": True}, + "example#ignored": "no", + }, + "members": { + "emptyDocument": { + "target": "smithy.api#Document", + "traits": {"smithy.api#default": {}}, + }, + "nullDocument": { + "target": "smithy.api#Document", + "traits": {"smithy.api#default": None}, + }, + "required": { + "target": "smithy.api#String", + "traits": {"smithy.api#required": {}}, + }, + }, + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Data"}, + "output": {"target": "example#Data"}, + }, + }, + } + ) + plan = context.plan.schemas + data = ShapeId.from_string("example#Data") + traits = plan.traits[data] + assert not isinstance(traits, int) + assert tuple(map(str, traits)) == ( + "example#protocol", + "example#wireTrait", + "smithy.api#input", + "smithy.api#output", + ) + assert traits[ShapeId.from_string("example#protocol")] is None + assert traits[ShapeId.from_string("example#wireTrait")] == MappingProxyType( + {"enabled": True} + ) + assert traits[ShapeId.from_string("smithy.api#input")] is None + assert traits[ShapeId.from_string("smithy.api#output")] is None + + empty_traits = plan.traits[ShapeId.from_string("example#Data$emptyDocument")] + null_traits = plan.traits[ShapeId.from_string("example#Data$nullDocument")] + required = plan.traits[ShapeId.from_string("example#Data$required")] + assert not isinstance(empty_traits, int) + assert not isinstance(null_traits, int) + assert empty_traits[ShapeId.from_string("smithy.api#default")] == MappingProxyType( + {} + ) + assert null_traits[ShapeId.from_string("smithy.api#default")] is None + assert required == 1 + assert "example#ignored" not in plan.strings + + +@pytest.mark.parametrize("value", [float("inf"), float("-inf"), float("nan")]) +def test_schema_generator_rejects_non_finite_trait_values(value: float) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Value": { + "type": "structure", + "members": { + "value": { + "target": "smithy.api#String", + "traits": {"smithy.api#default": value}, + } + }, + } + }, + } + ) + + with pytest.raises(CodegenError, match="non-finite numeric trait value"): + SchemaGenerator().generate(context) + + +def test_string_store_uses_short_monotonic_aliases() -> None: + strings = _StringStore() + + assert strings.intern("example", preferred="n") == "n0" + aliases = tuple(strings.intern(f"value{index}") for index in range(2_000)) + + assert aliases[0] == "_s0" + assert aliases[-1] == "_s1999" + assert len(set(aliases)) == len(aliases) + assert max(map(len, aliases)) == 6 + + +def test_schema_generator_emits_lazy_hydratable_schemas(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Name": { + "type": "string", + "traits": {"smithy.api#sensitive": {}}, + }, + "example#Choice": { + "type": "enum", + "members": { + "ONE": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "one"}, + } + }, + }, + "example#Node": { + "type": "structure", + "members": { + "name": { + "target": "example#Name", + "traits": {"smithy.api#required": {}}, + }, + "next": {"target": "example#Node"}, + "choice": {"target": "example#Choice"}, + }, + }, + }, + } + ) + SchemaGenerator().generate(context) + assert SMITHY_CORE in context.dependencies + context.files.commit(tmp_path) + + path = tmp_path / "src/example/types/_private/schemas.py" + source = path.read_text() + assert "from smithy_core._schema_compact import StaticSchema" in source + assert "lambda: NODE" in source + assert "MappingProxyType" not in source + + values = runpy.run_path(str(path)) + static_name = cast(tuple[Any, ...], values["NAME"]) + static_node = cast(tuple[Any, ...], values["NODE"]) + static_choice = cast(tuple[Any, ...], values["CHOICE"]) + assert static_name == (RuntimeShapeType.STRING.value, "example", "Name", 1 << 2) + assert static_node[5][0][1] == 1 + assert callable(static_node[5][1][0]) + assert static_choice[5][0][0] == _UNIT_INT + + node = _schema_compact.hydrate(static_node) + assert str(node.id) == "example#Node" + assert tuple(node.members) == ("name", "next", "choice") + assert node.members_by_index[0].expect_member_index() == 0 + assert node.members_by_index[0].get_trait(RequiredTrait) is not None + assert node.members_by_index[1].expect_member_target() is node + assert str(node.members_by_index[2].expect_member_target().id) == "example#Choice" + + +def test_schema_generator_preserves_primitive_prelude_targets(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Data": { + "type": "structure", + "members": { + "boolean": {"target": "smithy.api#PrimitiveBoolean"}, + "byte": {"target": "smithy.api#PrimitiveByte"}, + "short": {"target": "smithy.api#PrimitiveShort"}, + "integer": {"target": "smithy.api#PrimitiveInteger"}, + "long": {"target": "smithy.api#PrimitiveLong"}, + "float": {"target": "smithy.api#PrimitiveFloat"}, + "double": {"target": "smithy.api#PrimitiveDouble"}, + }, + } + }, + } + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + values = runpy.run_path(str(tmp_path / "src/example/types/_private/schemas.py")) + data = _schema_compact.hydrate(cast(tuple[Any, ...], values["DATA"])) + expected = { + "boolean": (prelude.PRIMITIVE_BOOLEAN, False), + "byte": (prelude.PRIMITIVE_BYTE, 0), + "short": (prelude.PRIMITIVE_SHORT, 0), + "integer": (prelude.PRIMITIVE_INTEGER, 0), + "long": (prelude.PRIMITIVE_LONG, 0), + "float": (prelude.PRIMITIVE_FLOAT, 0), + "double": (prelude.PRIMITIVE_DOUBLE, 0), + } + for name, (schema, default) in expected.items(): + target = data.members[name].expect_member_target() + assert target is schema + trait = target.get_trait(DefaultTrait) + assert trait is not None + assert trait.value == default + + +def test_schema_generator_emits_immutable_trait_documents(tmp_path: Path) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "example#Data": { + "type": "structure", + "traits": { + "smithy.api#xmlNamespace": { + "uri": "https://example.com", + "values": [{"name": "one"}], + } + }, + "members": { + "document": { + "target": "smithy.api#Document", + "traits": {"smithy.api#default": {}}, + } + }, + } + }, + } + ) + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + path = tmp_path / "src/example/types/_private/schemas.py" + source = path.read_text() + assert "from types import MappingProxyType" in source + assert "MappingProxyType[str, object]({})" in source + + values = runpy.run_path(str(path)) + data = _schema_compact.hydrate(cast(tuple[Any, ...], values["DATA"])) + namespace = data.get_trait(XMLNamespaceTrait) + assert namespace is not None + document = namespace.document_value + assert isinstance(document, MappingProxyType) + assert isinstance(cast(Mapping[str, object], document)["values"], tuple) + + +def test_schema_output_is_deterministic(tmp_path: Path) -> None: + model = { + "smithy": "2.0", + "shapes": { + "example#B": {"type": "string"}, + "example#A": { + "type": "structure", + "members": {"b": {"target": "example#B"}}, + }, + }, + } + outputs: list[str] = [] + for index in range(2): + context = _context(model) + SchemaGenerator().generate(context) + output = tmp_path / str(index) + context.files.commit(output) + outputs.append((output / "src/example/types/_private/schemas.py").read_text()) + assert outputs[0] == outputs[1] + + +def test_schema_names_are_deconflicted_for_private_dependencies( + tmp_path: Path, +) -> None: + context = _context( + { + "smithy": "2.0", + "shapes": { + "a#Name": {"type": "string"}, + "b#Name": {"type": "integer"}, + "a#First": { + "type": "structure", + "members": {"value": {"target": "a#Name"}}, + }, + "b#Second": { + "type": "structure", + "members": {"value": {"target": "b#Name"}}, + }, + }, + } + ) + names = context.plan.schemas.names + assert names[ShapeId.from_string("a#Name")] == "NAME" + assert names[ShapeId.from_string("b#Name")] == "NAME_2" + + SchemaGenerator().generate(context) + context.files.commit(tmp_path) + values = runpy.run_path(str(tmp_path / "src/example/types/_private/schemas.py")) + first = _schema_compact.hydrate(cast(tuple[Any, ...], values["FIRST"])) + second = _schema_compact.hydrate(cast(tuple[Any, ...], values["SECOND"])) + assert first.members["value"].shape_type is RuntimeShapeType.STRING + assert second.members["value"].shape_type is RuntimeShapeType.INTEGER diff --git a/packages/smithy-python/tests/unit/test_selectors.py b/packages/smithy-python/tests/unit/test_selectors.py new file mode 100644 index 000000000..858f68e66 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_selectors.py @@ -0,0 +1,326 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pytest +from smithy_python.exceptions import SelectorSyntaxError +from smithy_python.model import Model, load_model +from smithy_python.selection import Selector + + +def _model(shapes: dict[str, object]) -> Model: + return load_model({"smithy": "2.0", "shapes": shapes}) + + +def _select(model: Model, source: str) -> set[str]: + return set(map(str, Selector.parse(source).select(model))) + + +def test_shape_types_categories_and_enum_aliases() -> None: + model = _model( + { + "example#Text": {"type": "string"}, + "example#Count": {"type": "integer"}, + "example#Choice": { + "type": "enum", + "members": { + "A": { + "target": "smithy.api#Unit", + "traits": {"smithy.api#enumValue": "a"}, + } + }, + }, + "example#Numbers": { + "type": "list", + "member": {"target": "example#Count"}, + }, + } + ) + + assert _select(model, "number [id|namespace=example]") == {"example#Count"} + assert _select(model, "string [id|namespace=example]") == { + "example#Text", + "example#Choice", + } + assert _select(model, "aggregateType [id|namespace=example]") == {"example#Numbers"} + assert _select(model, "member [id|namespace=example]") == { + "example#Choice$A", + "example#Numbers$member", + } + + +def test_attributes_projections_scopes_and_comparators() -> None: + model = _model( + { + "example#Match": { + "type": "string", + "traits": { + "example#config": { + "enabled": True, + "min": 3, + "max": 3, + "mode": "foo", + "tags": ["internal", "external"], + } + }, + }, + "example#NoMatch": { + "type": "string", + "traits": { + "example#config": { + "enabled": False, + "min": 1, + "max": 3, + "mode": "foo", + "tags": [], + } + }, + }, + } + ) + + assert _select(model, "[trait|example#config|min >= 3]") == {"example#Match"} + assert _select(model, "[trait|example#config|tags|(values) = external]") == { + "example#Match" + } + assert _select(model, "[@trait|example#config: @{min} = @{max}]") == { + "example#Match" + } + assert _select( + model, + "[@trait|example#config|tags|(values): @{} = external]", + ) == {"example#Match"} + assert _select( + model, + "[@trait|example#config: @{mode}=foo&&@{enabled}=true]", + ) == {"example#Match"} + assert _select(model, "[id|name ^= match i]") == {"example#Match"} + + +def test_null_trait_value_is_present() -> None: + model = _model( + { + "example#Container": { + "type": "structure", + "members": { + "value": { + "target": "smithy.api#Document", + "traits": {"smithy.api#default": None}, + } + }, + } + } + ) + + assert _select(model, "member [trait|default]") == {"example#Container$value"} + + +def test_neighbors_recursive_functions_and_variables() -> None: + model = _model( + { + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + }, + "example#Operation": { + "type": "operation", + "input": {"target": "example#Input"}, + "output": {"target": "example#Output"}, + }, + "example#Input": { + "type": "structure", + "members": {"value": {"target": "example#InputNumber"}}, + }, + "example#Output": { + "type": "structure", + "members": {"value": {"target": "example#OutputNumber"}}, + }, + "example#InputNumber": {"type": "integer"}, + "example#OutputNumber": {"type": "long"}, + } + ) + + assert _select(model, "service ~> number") == { + "example#InputNumber", + "example#OutputNumber", + } + assert _select(model, "number :test(< member < structure)") == { + "example#InputNumber", + "example#OutputNumber", + } + assert _select( + model, + "service $outputs(~> operation -[output]-> ~> number) " + "~> operation -[input]-> ~> number :not(:in(${outputs}))", + ) == {"example#InputNumber"} + assert _select( + model, + "number :in(:root(service ~> operation -[output]-> ~> number))", + ) == {"example#OutputNumber"} + + +def test_traits_recursive_selector_and_topdown() -> None: + model = _model( + { + "example#Marker": { + "type": "structure", + "traits": {"smithy.api#trait": {}}, + }, + "example#Base": { + "type": "structure", + "traits": {"smithy.api#mixin": {}}, + }, + "example#Derived": { + "type": "structure", + "mixins": [{"target": "example#Base"}], + "traits": {"smithy.api#sensitive": {}}, + }, + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "traits": {"example#Marker": {}}, + }, + "example#Operation": { + "type": "operation", + "traits": {"example#Marker": {}}, + }, + } + ) + + assert _select(model, "[id=example#Derived] :recursive(-[mixin]->)") == { + "example#Base" + } + assert _select(model, "-[trait]-> [trait|trait] [id|namespace=example]") == { + "example#Marker" + } + assert _select( + model, + "[id=smithy.api#sensitive] <-[trait]- structure", + ) == {"example#Derived"} + assert _select(model, ":topdown([trait|example#Marker])") == { + "example#Service", + "example#Operation", + } + + +def test_virtual_prelude_traits_preserve_shape_types() -> None: + model = _model( + { + "example#Data": { + "type": "structure", + "traits": {"smithy.api#xmlName": "renamed"}, + } + } + ) + + assert _select( + model, + "structure :test(-[trait]-> string [id=smithy.api#xmlName])", + ) == {"example#Data"} + + +@pytest.mark.parametrize( + ("meta_trait", "applied_trait"), + ( + ("smithy.api#trait", "example#Missing"), + ("smithy.api#authDefinition", "smithy.api#httpApiKeyAuth"), + ), +) +def test_virtual_traits_do_not_shadow_existing_meta_traits( + meta_trait: str, applied_trait: str +) -> None: + model = _model( + { + meta_trait: { + "type": "structure", + "members": {"value": {"target": "smithy.api#String"}}, + }, + "example#Data": { + "type": "structure", + "traits": {applied_trait: {}}, + }, + } + ) + + assert _select(model, f"[id={meta_trait}] > member") == {f"{meta_trait}$value"} + + +def test_topdown_includes_resource_operations() -> None: + model = _model( + { + "example#Marker": { + "type": "structure", + "traits": {"smithy.api#trait": {}}, + }, + "example#Service": { + "type": "service", + "version": "1", + "resources": [{"target": "example#Resource"}], + "traits": {"example#Marker": {}}, + }, + "example#Resource": { + "type": "resource", + "create": {"target": "example#Create"}, + "collectionOperations": [{"target": "example#Collection"}], + }, + "example#Create": {"type": "operation"}, + "example#Collection": {"type": "operation"}, + } + ) + + assert _select(model, ":topdown([trait|example#Marker])") == { + "example#Service", + "example#Resource", + "example#Create", + "example#Collection", + } + + +def test_topdown_tracks_qualification_per_path() -> None: + model = _model( + { + "example#Marker": { + "type": "structure", + "traits": {"smithy.api#trait": {}}, + }, + "example#Service": { + "type": "service", + "version": "1", + "operations": [{"target": "example#Operation"}], + "resources": [{"target": "example#Resource"}], + }, + "example#Resource": { + "type": "resource", + "create": {"target": "example#Operation"}, + "traits": {"example#Marker": {}}, + }, + "example#Operation": {"type": "operation"}, + } + ) + + assert _select(model, ":topdown([trait|example#Marker])") == { + "example#Resource", + "example#Operation", + } + + +@pytest.mark.parametrize("value", ["NaN", "Infinity", "-Infinity"]) +def test_ordered_comparisons_ignore_non_finite_values(value: str) -> None: + model = _model( + { + "example#Value": { + "type": "string", + "traits": {"smithy.api#default": value}, + } + } + ) + + assert _select(model, "[trait|default > 0]") == set() + + +def test_invalid_selector_reports_location() -> None: + with pytest.raises(SelectorSyntaxError, match=r"line 1, column"): + Selector.parse("structure [trait|") diff --git a/packages/smithy-python/tests/unit/test_symbols.py b/packages/smithy-python/tests/unit/test_symbols.py index 6ce95f771..4e0c77866 100644 --- a/packages/smithy-python/tests/unit/test_symbols.py +++ b/packages/smithy-python/tests/unit/test_symbols.py @@ -5,7 +5,8 @@ import keyword import pytest -from smithy_python.model import MemberShape, Model, load_model +from smithy_python.dependencies import SMITHY_CORE +from smithy_python.model import MemberShape, Model, ShapeId, load_model from smithy_python.selection import select_shapes from smithy_python.symbols import SymbolProvider @@ -89,8 +90,10 @@ def test_type_references(kind: str, name: str, module: str) -> None: model = model_with({"example#HTTPServer": {"type": kind}}) symbols = SymbolProvider(model, select_shapes(model), package="example.client") ref = symbols.type_reference("example#HTTPServer") - assert ref == TypeReference(name, module) - assert hash(ref) == hash(TypeReference(name, module)) + dependencies = (SMITHY_CORE,) if kind == "document" else () + expected = TypeReference(name, module, dependencies=dependencies) + assert ref == expected + assert hash(ref) == hash(expected) with pytest.raises(FrozenInstanceError): setattr(ref, "name", "Changed") if module != "example.client.models" and kind not in ("enum", "intEnum"): @@ -99,6 +102,23 @@ def test_type_references(kind: str, name: str, module: str) -> None: assert symbols.type_reference("smithy.api#Unit") == TypeReference("None") +def test_streaming_union_has_a_schema_name_without_a_model_symbol() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Events": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + } + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="example") + assert symbols.schema_name("example#Events") == "EVENTS" + with pytest.raises(ModelError, match="Unsupported streaming union"): + symbols.declaration_name("example#Events") + + @pytest.mark.parametrize("kind,value_type", [("enum", "str"), ("intEnum", "int")]) @pytest.mark.parametrize("sparse", [False, True]) def test_enum_value_references(kind: str, value_type: str, sparse: bool) -> None: @@ -186,6 +206,68 @@ def test_service_rename_only_changes_declaration() -> None: assert str(selection.shapes[0].id) == "example#Original" +def test_union_names_disambiguate_generated_declaration_names() -> None: + model = model_with( + { + "example#Choice": { + "type": "union", + "members": {"foo": {"target": "smithy.api#String"}}, + }, + "example#Choice_Foo": {"type": "structure"}, + "example#Choice_Unknown": {"type": "structure"}, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + + assert symbols.declaration_name("example#Choice_Foo") == "ChoiceFoo" + assert symbols.union_variant_name("example#Choice$foo") == "Choice_Foo" + assert symbols.declaration_name("example#Choice_Unknown") == "ChoiceUnknown" + assert symbols.union_unknown_name("example#Choice") == "Choice_Unknown" + + +def test_union_names_disambiguate_service_renames() -> None: + from smithy_python.model import ServiceShape + from smithy_python.selection import Selection + + model = model_with( + { + "example#Service": { + "type": "service", + "rename": {"example#Other": "ChoiceFoo"}, + }, + "example#Choice": { + "type": "union", + "members": {"foo": {"target": "smithy.api#String"}}, + }, + "example#Other": {"type": "structure"}, + } + ) + selection = Selection( + model.expect_shape("example#Service", ServiceShape), + (model.get_shape("example#Choice"), model.get_shape("example#Other")), + 0, + ) + symbols = SymbolProvider(model, selection, package="pkg") + + assert symbols.declaration_name("example#Other") == "ChoiceFoo" + assert symbols.union_variant_name("example#Choice$foo") == "Choice_Foo" + + +def test_synthesized_union_name_collisions_are_rejected() -> None: + from smithy_python.model import ModelError + + model = model_with( + { + "example#Choice": { + "type": "union", + "members": {"unknown": {"target": "smithy.api#String"}}, + } + } + ) + with pytest.raises(ModelError, match="ChoiceUnknown"): + SymbolProvider(model, select_shapes(model), package="pkg") + + @pytest.mark.parametrize("sparse", [False, True]) def test_collections_and_recursive_named_types(sparse: bool) -> None: from smithy_python.symbols import TypeReference @@ -253,6 +335,49 @@ def test_collections_and_recursive_named_types(sparse: bool) -> None: assert symbols.type_reference(target) == TypeReference("str", "builtins") +def test_type_reference_override_applies_within_collections() -> None: + from smithy_python.symbols import TypeReference + + model = model_with( + { + "example#Times": { + "type": "list", + "member": {"target": "smithy.api#Timestamp"}, + }, + "example#NestedTimes": { + "type": "list", + "member": {"target": "example#Times"}, + }, + "example#TimeMap": { + "type": "map", + "key": {"target": "smithy.api#String"}, + "value": {"target": "smithy.api#Timestamp"}, + }, + } + ) + + class CustomSymbolProvider(SymbolProvider): + def type_reference(self, shape_id: ShapeId | str) -> TypeReference: + if str(shape_id) == "smithy.api#Timestamp": + return TypeReference("CustomTimestamp", "example.timestamps") + return super().type_reference(shape_id) + + symbols = CustomSymbolProvider(model, select_shapes(model), package="pkg") + timestamp = TypeReference("CustomTimestamp", "example.timestamps") + times = TypeReference("list", "builtins", (timestamp,)) + + assert symbols.type_reference("smithy.api#Timestamp") == timestamp + assert symbols.type_reference("example#Times") == times + assert symbols.type_reference("example#NestedTimes") == TypeReference( + "list", "builtins", (times,) + ) + assert symbols.type_reference("example#TimeMap") == TypeReference( + "dict", + "builtins", + (TypeReference("str", "builtins"), timestamp), + ) + + @pytest.mark.parametrize("mutual", [False, True]) def test_collection_only_cycles(mutual: bool) -> None: from smithy_python.model import ModelError @@ -348,7 +473,7 @@ def test_only_emitted_declarations_collide() -> None: "Document", "pkg.models" ) assert symbols.type_reference("smithy.api#Document") == TypeReference( - "Document", "smithy_core.documents" + "Document", "smithy_core.documents", dependencies=(SMITHY_CORE,) ) assert symbols.member_name("example#Document$list") == "list" assert symbols.member_name("example#Document$dict") == "dict" @@ -400,13 +525,39 @@ def test_rename_normalization(name: str, expected: str) -> None: ) -@pytest.mark.parametrize("kind", ["blob", "union"]) -def test_streaming_rejected_when_resolved(kind: str) -> None: +def test_streaming_blob_type_reference() -> None: + from smithy_python.symbols import TypeReference + + model = model_with( + { + "example#Stream": { + "type": "blob", + "traits": {"smithy.api#streaming": {}}, + }, + "example#Streams": {"type": "list", "member": {"target": "example#Stream"}}, + } + ) + symbols = SymbolProvider(model, select_shapes(model), package="pkg") + stream = TypeReference( + "StreamingBlob", + "smithy_core.aio.interfaces", + dependencies=(SMITHY_CORE,), + ) + assert symbols.type_reference("example#Stream") == stream + assert symbols.type_reference("example#Streams") == TypeReference( + "list", "builtins", (stream,) + ) + + +def test_streaming_union_rejected_when_resolved() -> None: from smithy_python.model import ModelError model = model_with( { - "example#Stream": {"type": kind, "traits": {"smithy.api#streaming": {}}}, + "example#Stream": { + "type": "union", + "traits": {"smithy.api#streaming": {}}, + }, "example#Streams": {"type": "list", "member": {"target": "example#Stream"}}, } ) diff --git a/packages/smithy-python/tests/unit/test_writer.py b/packages/smithy-python/tests/unit/test_writer.py index e789321aa..78852e0cc 100644 --- a/packages/smithy-python/tests/unit/test_writer.py +++ b/packages/smithy-python/tests/unit/test_writer.py @@ -13,11 +13,46 @@ from typing import get_type_hints import pytest +from smithy_python.dependencies import Dependency, DependencySet from smithy_python.exceptions import CodegenError from smithy_python.symbols import TypeReference from smithy_python.writer import PythonWriter +def test_writer_records_direct_and_nested_dependencies() -> None: + dependencies = DependencySet() + runtime = Dependency("runtime", "~=1.0", "runtime") + nested = Dependency("nested", "~=2.0", "runtime") + writer = PythonWriter("models", dependencies=dependencies) + writer.line( + "value: ", + TypeReference( + "list", + "builtins", + ( + TypeReference( + "Value", + "external", + dependencies=(nested,), + ), + ), + dependencies=(runtime,), + ), + ) + + assert tuple(dependencies) == (nested, runtime) + + +def test_standalone_writer_does_not_require_dependency_collection() -> None: + dependency = Dependency("runtime", "~=1.0", "runtime") + writer = PythonWriter("models") + writer.line( + "value: ", + TypeReference("Value", "external", dependencies=(dependency,)), + ) + assert "from external import Value" in writer.render() + + def load_written_module( writer: PythonWriter, directory: Path, monkeypatch: pytest.MonkeyPatch ) -> ModuleType: diff --git a/scripts/api-snapshot/api.json b/scripts/api-snapshot/api.json index 804aace53..d82b8bfe0 100644 --- a/scripts/api-snapshot/api.json +++ b/scripts/api-snapshot/api.json @@ -1487,6 +1487,13 @@ "main": "function", "select_shapes": "rex-function" }, + "smithy_python.dependencies": { + "Dependency": "class", + "DependencyGroup": "rex-type_alias", + "DependencySet": "class", + "HATCHLING": "Dependency", + "SMITHY_CORE": "Dependency" + }, "smithy_python.environment": { "PluginEnvironment": "class" }, @@ -1495,9 +1502,21 @@ "InvalidInvocationError": "exception", "InvalidShapeIdError": "exception", "ModelError": "exception", + "SelectorSyntaxError": "exception", "ShapeNotFoundError": "exception", "SmithyPythonError": "exception" }, + "smithy_python.files": { + "FileManifest": "class" + }, + "smithy_python.generation": { + "ArtifactGenerator": "protocol", + "ArtifactSettings": "class", + "GenerationContext": "class", + "GenerationPlan": "class", + "ModelPlan": "class", + "TypesSettings": "class" + }, "smithy_python.model": { "EnumShape": "rex-class", "IntEnumShape": "rex-class", @@ -1522,6 +1541,10 @@ "load_model": "rex-function", "to_json": "rex-function" }, + "smithy_python.schemas": { + "SchemaGenerator": "class", + "SchemaPlan": "class" + }, "smithy_python.selection": { "InvalidInvocationError": "rex-exception", "InvalidShapeIdError": "rex-exception", @@ -1531,12 +1554,14 @@ "OperationShape": "rex-class", "ResourceShape": "rex-class", "Selection": "class", + "Selector": "rex-class", "ServiceShape": "rex-class", "Shape": "rex-class", "ShapeId": "rex-class", "ShapeType": "rex-class", "resolve_service": "function", - "select_shapes": "function" + "select_shapes": "function", + "select_type_shapes": "function" }, "smithy_python.symbols": { "ListShape": "rex-class", diff --git a/scripts/codegen-output-py/generate.py b/scripts/codegen-output-py/generate.py new file mode 100644 index 000000000..54ea35f82 --- /dev/null +++ b/scripts/codegen-output-py/generate.py @@ -0,0 +1,79 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Regenerate the committed Python codegen snapshot.""" + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +from smithy_python.dependencies import DependencySet +from smithy_python.files import FileManifest +from smithy_python.generation import GenerationContext, GenerationPlan, TypesSettings +from smithy_python.model import load_model +from smithy_python.schemas import SchemaGenerator +from smithy_python.selection import select_type_shapes +from smithy_python.symbols import SymbolProvider + +_ROOT = Path(__file__).resolve().parents[2] +_MODEL = Path(__file__).with_name("model.json") +_OUTPUT = _ROOT / "codegen-output-py" / "types" +_PACKAGE = "example_types" +_SERVICE = "example#ExampleService" + + +def main() -> None: + model = load_model(_MODEL.read_bytes()) + selection = select_type_shapes( + model, + service_id=_SERVICE, + generate_inputs_and_outputs=True, + ) + settings = TypesSettings( + package=_PACKAGE, + package_version="1.0.0", + package_description="Example Python codegen output", + ) + symbols = SymbolProvider(model, selection, package=settings.package) + dependencies = DependencySet() + context = GenerationContext( + model=model, + selection=selection, + settings=settings, + symbols=symbols, + plan=GenerationPlan.from_selection(model, selection, symbols), + dependencies=dependencies, + files=FileManifest(dependencies), + ) + SchemaGenerator().generate(context) + context.files.commit(_OUTPUT) + _format_output() + + +def _format_output() -> None: + config = _ROOT / "pyproject.toml" + for command in ("check", "format"): + args = [sys.executable, "-m", "ruff", command] + if command == "check": + args.append("--fix") + subprocess.run( + [*args, str(_OUTPUT), "--config", str(config)], + check=True, + ) + + manifest = _OUTPUT / ".smithy-python-manifest.json" + files = { + path.relative_to(_OUTPUT).as_posix(): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in sorted(_OUTPUT.rglob("*")) + if path.is_file() and path != manifest + } + manifest.write_text( + json.dumps({"version": 1, "files": files}, indent=2, sort_keys=True) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/codegen-output-py/model.json b/scripts/codegen-output-py/model.json new file mode 100644 index 000000000..6d4110a28 --- /dev/null +++ b/scripts/codegen-output-py/model.json @@ -0,0 +1,171 @@ +{ + "smithy": "2.0", + "shapes": { + "example#ExampleService": { + "type": "service", + "version": "2026-10-23", + "operations": [ + { + "target": "example#GetExample" + } + ], + "rename": { + "example#Example": "RenamedExample", + "example#Value": "ExampleValue" + } + }, + "example#GetExample": { + "type": "operation", + "input": { + "target": "example#GetExampleInput" + }, + "output": { + "target": "example#GetExampleOutput" + }, + "errors": [ + { + "target": "example#ExampleError" + } + ] + }, + "example#GetExampleInput": { + "type": "structure", + "traits": { + "smithy.api#documentation": "Input for retrieving an example." + }, + "members": { + "id": { + "target": "smithy.api#String", + "traits": { + "smithy.api#required": {} + } + } + } + }, + "example#GetExampleOutput": { + "type": "structure", + "members": { + "example": { + "target": "example#Example" + }, + "value": { + "target": "example#Value" + } + } + }, + "example#ExampleError": { + "type": "structure", + "traits": { + "smithy.api#documentation": "An example modeled error.", + "smithy.api#error": "client" + }, + "members": { + "message": { + "target": "smithy.api#String" + }, + "retryAfterSeconds": { + "target": "smithy.api#Integer" + } + } + }, + "example#Example": { + "type": "structure", + "traits": { + "smithy.api#documentation": "A representative generated structure." + }, + "members": { + "name": { + "target": "smithy.api#String", + "traits": { + "smithy.api#documentation": "The example name.", + "smithy.api#required": {} + } + }, + "count": { + "target": "smithy.api#PrimitiveInteger" + }, + "tags": { + "target": "example#TagList" + }, + "attributes": { + "target": "example#Attributes" + }, + "choice": { + "target": "example#Value" + }, + "status": { + "target": "example#Status" + }, + "code": { + "target": "example#Code" + } + } + }, + "example#TagList": { + "type": "list", + "member": { + "target": "smithy.api#String" + } + }, + "example#Attributes": { + "type": "map", + "key": { + "target": "smithy.api#String" + }, + "value": { + "target": "smithy.api#Document" + } + }, + "example#Value": { + "type": "union", + "traits": { + "smithy.api#documentation": "A representative generated union." + }, + "members": { + "text": { + "target": "smithy.api#String" + }, + "number": { + "target": "smithy.api#Integer" + }, + "empty": { + "target": "smithy.api#Unit" + } + } + }, + "example#Status": { + "type": "enum", + "members": { + "ACTIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "active" + } + }, + "INACTIVE": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": "inactive" + } + } + } + }, + "example#Code": { + "type": "intEnum", + "members": { + "OK": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": 200 + } + }, + "NOT_FOUND": { + "target": "smithy.api#Unit", + "traits": { + "smithy.api#enumValue": 404 + } + } + } + } + } +}