diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2afa63d..6183cde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,9 @@ jobs: # what a commit's CI proves. Find the newest tag with: # gh release list --repo mas-bandwidth/serialize --limit 1 # Bump family-wide, deliberately, in its own commit per repo. SERIALIZE_TAG - # below is the ONE place this repository names a version of the reference. + # below is the ONE place this repository names a RELEASE of the reference; the + # corpus commit beside it names the vendoring and is a different pin for a + # different job, explained where it is declared. # (The spec-sync job above deliberately stays on upstream main: drift DETECTION # is that job's whole purpose.) interop: @@ -90,6 +92,14 @@ jobs: runs-on: ubuntu-latest env: SERIALIZE_TAG: v1.16.0 + # TWO PINS, TWO JOBS FOR THEM. SERIALIZE_TAG above is the wire + # compatibility pin and is a released tag. SERIALIZE_CORPUS_COMMIT is the + # commit STANDARD.md and conformance/ in this repository are vendored + # from, and the reference conformance runner is built from it, because a + # runner older than the corpus cannot drive the operations the corpus + # carries. The spec-sync job proves this repository's vendored copies + # match upstream main, so this commit is upstream main at the vendoring. + SERIALIZE_CORPUS_COMMIT: 7e0515e952f3373d001ec1899adf9dffb823e1c5 steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -100,15 +110,20 @@ jobs: - name: Check out the C++ reference at the pinned release run: git clone --quiet --depth 1 --branch "$SERIALIZE_TAG" https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize + - name: Check out the C++ reference at the corpus commit + run: | + git clone --quiet https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize-corpus + git -C /tmp/cpp-serialize-corpus checkout --quiet "$SERIALIZE_CORPUS_COMMIT" + # Asserts stay ON (no -DNDEBUG): they are the C++ half of "API misuse # panics", and the degenerate ranges the message carries must pass with the # library's own checks enabled rather than around them. No -ffp-contract=off # either: the compressed float arithmetic is pinned in the library's source, # and building with default flags proves that rather than masking it. - - name: Build the C++ halves against the pinned library + - name: Build the C++ halves run: | c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_interop interop/interop.cpp - c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_conformance /tmp/cpp-serialize/conformance.cpp + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize-corpus -o /tmp/cpp_conformance /tmp/cpp-serialize-corpus/conformance.cpp - name: Both halves write byte identical wire data run: | diff --git a/README.md b/README.md index 3061fdc..014a8f8 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ specification in [mas-bandwidth/serialize](https://github.com/mas-bandwidth/serialize), which CI checks for drift — is the authority on every byte. -Version 1.1.1 (`SerializeUtil.VERSION`). +Version 1.1.2 (`SerializeUtil.VERSION`). ## Getting it @@ -26,7 +26,7 @@ package is not yet published: there is no artifact on Maven Central under any coordinate, and there is no Maven or Gradle build here to produce one. Publishing it is a separate round. -Fifteen files under `src/serialize/`, zero dependencies, Java 17 language +Sixteen files under `src/serialize/`, zero dependencies, Java 17 language level. Take it either way: **Copy the package in** — drop `src/serialize/` into your own source tree and @@ -85,6 +85,9 @@ primitive-specialized holder cells (`IntRef`, `LongRef`, `BoolRef`, pairs of long halves, mirroring the family's emulated pair types. - **Utilities**: `SerializeUtil` — `bitsRequired` / `64` / `128`, zig-zag conversion, the compressed-float step count. +- **Composition**: `serializeObject`, which runs a nested `Serializer` + inline and contributes no bytes of its own — no framing, no length + prefix, no alignment inserted around it. - **The bitpacker underneath**: `BitWriter` and `BitReader`, the family wire in branchless 64-bit window loads. Every stream and both bitpackers expose `reset(...)` for allocation-free reuse. @@ -146,11 +149,19 @@ gates. The suite runs every vector in [`conformance/`](conformance), the family's shared corpus, vendored from mas-bandwidth/serialize and -checked for drift by CI: an accepted vector must decode to the stated -value and consume the stated bits, a refused vector must be refused, and -nothing regenerates its own expectations. It also pins the family's -golden vectors byte for byte — the golden wire message covering every -operation class, the discriminating compressed-float vectors (bit +checked for drift by CI. The directory is discovered at run time rather +than named in the source, and a vector whose operation the runner cannot +drive fails rather than being skipped. An accepted vector must decode to +the stated value and consume the stated bits; a vector marked +`writer = canonical` is re-emitted through the write stream and compared +byte for byte, flush included; a vector carrying `measure_at_least` is +held to that floor on the measure stream. A refused vector must be +refused, must leave the caller's scalar destination unwritten, and must +leave the stream terminal, which is checked by behavior: every later step +refuses too, and a further read fails, consumes no bits and writes +nothing. Nothing regenerates its own expectations. The suite also pins +the family's golden vectors byte for byte — the golden wire message +covering every operation class, the discriminating compressed-float vectors (bit patterns, not tolerances), the string and wide-string pins, every relative-integer tier, and the fixed point shapes at every group count — plus a sabotage sweep proving every consumed bit of the golden stream is diff --git a/STANDARD.md b/STANDARD.md index 467275c..88cf1bd 100644 --- a/STANDARD.md +++ b/STANDARD.md @@ -12,6 +12,9 @@ structure your code. **The format version is 1.1.** It names the wire, not a library release. Two endpoints interoperate when they run releases carrying the same format version, and a release states which format version it implements. +[COMPATIBILITY.md](COMPATIBILITY.md) lists the release of every implementation +in the family that carries it, pinned to the commit of this document and of the +corpus that those releases vendor. The rulings that moved the format off 1.0, which was this document as first written: @@ -25,6 +28,9 @@ written: * **2026-09-04.** `int_relative` carries the non-negative int32 domain, and every tier's reconstruction is refused outside it. +Every ruling in this list is part of format version 1.1, the 2026-09-04 +refusal ruling included, and none of them moves the version again. + ## Architecture serialize is a **bit packer**. Values are written as variable numbers of bits @@ -90,6 +96,11 @@ Writes the low `bits` bits of `value`, where `bits` is in `[1,64]`. * For `bits > 32` the value is split: the **low 32 bits are written first as a 32-bit group**, then the remaining `bits - 32` high bits as a second group. +The bound `value < 2^bits` holds at every width in `[1,64]` and not only at 32 +or fewer. A `bits` call of width zero is caller error rather than a zero-bit +field, and the ranged operations below branch around this primitive when the +range is degenerate. + Fixed-width helpers are aliases for exactly this, and carry no range information of their own: @@ -143,6 +154,17 @@ entirely by the range: bits_required( min, max ) = ( min == max ) ? 0 : 32 - count_leading_zeros( max - min ) +The arithmetic is unsigned and wrapping: `max - min` and `value - min` are +computed in the unsigned domain of the operation's width, `uint32` for `int`, +`uint64` for `int64` and `uint128` for `int128`, and the leading zero count is +taken at that same width, so a range as wide as `[INT32_MIN, INT32_MAX]` is +exact rather than overflowing. + +The wider widths carry the same rule under their own names. +`bits_required64( min, max )` is zero when `min == max` and otherwise +`64 - count_leading_zeros64( max - min )` over the unsigned 64-bit difference, +and `bits_required128` is that rule again at 128 bits. + `value - min` is written in that many bits. Note the consequences: * a range of `[0,7]` costs 3 bits; @@ -162,7 +184,10 @@ but quantizes across them, and a zero `delta` has no quantization to define, so it requires `min < max` and a checked build asserts that. Readers must check that the decoded value lies within `[min,max]` and fail -otherwise. +otherwise. The check is stated in offset form for `int`, `int64` and `fixed` +alike: the reader compares the decoded offset against `max - min` in the +unsigned domain before it forms the value, which cannot overflow on hostile +input and is equivalent to the value form. The range must be identical on both sides. The format carries no self-description: a stream is only interpretable by a reader that performs the @@ -181,6 +206,9 @@ bits are written first, followed by the remaining `bits - 32` high bits. `serialize_bits( value, 64 )` and always costs a full 64 bits. The names are similar and the encodings are not. +Readers must check that the decoded value lies within `[min,max]` and fail +otherwise, in the offset form stated under `serialize_int`. + ### int128 (ranged) serialize_int128( stream, value, min, max ) @@ -226,12 +254,24 @@ value scaled by `2^fraction_bits`. `min` and `max` are bounds in **whole real units**, and all four parameters are compile-time constants of the call site — they are part of the format, exactly like a ranged integer's bounds. +The legal storage widths are 8, 16, 32, 64 and 128 bits, always signed, and +`integer_bits` is at least 1 because the sign bit counts toward it. +"Compile-time constants" describes the C++ call site rather than the wire, so a +language whose generics cannot take integer parameters passes the four +parameters at runtime and produces the same bytes. `value` is a raw backed +fixed point value of the storage type rather than a floating point number, so +the operation performs no conversion of its own. + The encoding is an offset encoding over the **raw** (scaled) bounds: raw_min = min << fraction_bits raw_max = max << fraction_bits bits = bit length of ( raw_max - raw_min ) — bits_required, at whatever width the range needs +The raw range lives in the unsigned domain of the storage width, exactly as +`int128`'s bounds do, so a Q64.64 field over the full `int64` unit range has an +exact raw range of `2^128 - 2^64` rather than an overflowing one. + `raw_value - raw_min` is written in `bits` bits, split into 32-bit groups from least significant upward exactly as `serialize_bits` splits wide values: `bits <= 32` is a single group, otherwise full 32-bit groups from the bottom @@ -268,7 +308,9 @@ for `raw >= 0`, `-( ( -raw + half ) >> drop )` for `raw < 0`, with negative raw rounds ties toward +infinity and diverges by exactly one raw step, on exact ties of negative raws only. A rounding rule is not wire shape — no protocol identifier can see the divergence — so a conformance vector must -pin a negative tie value that distinguishes the two rules. +pin a negative tie value that distinguishes the two rules. The rule binds every +conversion between floating point and fixed point that a runtime chooses to +expose, and never a byte on the wire. ### int_relative @@ -291,6 +333,11 @@ each answering "does it fit in this tier?": The tier names are this document's, and the conformance vectors use them. The five `bounded-*` tiers are named for their payload width. +**The writer takes the first tier the difference fits, and that encoding is the +canonical one.** The reader accepts any tier whose payload decodes within the +rules, so a stream carrying a difference in a wider tier than a writer would +have chosen is a valid stream rather than a refused one. + A difference of 1 — the common case for sequence numbers — costs a single bit. **The final tier transmits `current`, not the difference.** Every tier above it @@ -314,7 +361,9 @@ operation, not of the caller's storage type: a 64-bit or unsigned `previous` of `2^31` is caller error everywhere, exactly as a negative one is. `previous` is the caller's own state and never arrives off the wire, so a `previous` outside the domain is caller error, asserted in checked builds, and this document -defines no wire meaning for it. +defines no wire meaning for it. The operation's API type is the signed 32-bit +integer, and the domain binds whatever wider or unsigned storage a caller keeps +its sequence in. **Every tier's reconstruction must be checked.** The reader must reconstruct `current` in a width that cannot wrap, then compare the result against the @@ -376,6 +425,10 @@ The writer clamps `(value - min) / delta` to `[0,1]`, multiplies by integer to `max_integer_value`**, and writes the result in `bits` bits. The reader divides by `max_integer_value`, multiplies by `delta`, and adds `min`. +`delta`, the quotient `delta / res` and the clamp are computed in `float32` +like the rest of this operation. A `res` of zero or below is caller error, +asserted in checked builds, and this document defines no wire meaning for it. + **This arithmetic is `float32`, and the two roundings are part of the format.** The product `normalized * max_integer_value` rounds to `float32` BEFORE `0.5` is added, and that sum rounds to `float32` before the floor. Two roundings, not @@ -395,6 +448,13 @@ one. Specifically, an implementation must not: product suffices — the spec forbids fusing across it. Rust does not fuse unless `mul_add` is called explicitly. +The rule is a property of the arithmetic rather than a list of languages: no +fused multiply-add across the expressions named here, in any language, and +every runtime documents the barrier it uses to hold that. The witness value +`8388608.0` and the two witness ranges named below are owed as corpus vectors, +and stand among the remaining silences under Provenance until the corpus carries +them. + **The integer clamp is normative — added 2026-08-23 (schema#109; ruling: Glenn, live).** Once `max_integer_value >= 2^23` the `float32` ulp at the top of the range reaches 1, so the rounded sum can exceed `max_integer_value` @@ -502,6 +562,9 @@ The terminator is not transmitted; the reader appends it. Because `buffer_size` is an operand rather than a transmitted value, the same string serialized against different buffer sizes produces different bytes. +`buffer_size` is a wire operand that bounds the length field and describes no +destination, the destination is the language's own string type, and `length` +counts bytes. **`string` payloads are well-formed UTF-8 by contract** *(adopted 2026-08-15 from the schema enactment, writer-trusted per the doctrine above)*. The wire @@ -521,7 +584,9 @@ in this document: * **Invalid UTF-8 fails the read.** The payload the contract above promises is the payload the reader insists on: a stream carrying malformed UTF-8 was not produced by a conforming writer, and the reader refuses it rather than - handing it to the application. + handing it to the application. Invalid means not well formed under Unicode + Table 3-7, so overlong forms, encoded surrogates, code points above + `10FFFF` and truncated sequences are all refused. * **An interior NUL fails the read** — a zero byte anywhere among the `length` transmitted bytes. A conforming writer derives `length` from `strlen`, so no zero byte can reach the wire from conformance; a stream carrying one gives @@ -544,7 +609,9 @@ carry.)* serialize_wstring( stream, string, buffer_size ) A null-terminated wide string. `buffer_size` counts **wide characters, not -bytes**. +bytes**. `wstring` is a required operation in every implementation, its +destination is the language's own string type, and a runtime whose strings are +not UTF-16 recombines surrogate pairs into code points on read. 1. The length, as `serialize_int( length, 0, buffer_size - 1 )`. 2. Each character as a **32-bit group**, in order. @@ -637,6 +704,10 @@ exist for convenience and to avoid a branch, not to encode anything differently. This document therefore specifies each operation once, under its `serialize_` name. +The API surface is not constrained. A single stream abstraction with a read, a +write and a measure conformer carries every operation in this document, and +these variants are one implementation's convenience rather than an obligation. + ## The Measure Stream Until 2026-08-15 this document disclaimed the measure stream in its third @@ -695,7 +766,8 @@ specified here because all nine implementations ship it and had already begun to disagree, not because it is expected to survive into entropy-coded encodings. -**Testable**: for every message in the conformance corpus, +**The measure stream is a required operation**, and every implementation ships +it. **Testable**: for every sequence vector in the shared corpus, `measure >= bits written`, at every starting bit position; and the worked example discriminates — a conservative measure reports 23 bits for `{ bits(8); align; bits(8) }` where an exact-from-zero measure reports 16. @@ -710,7 +782,15 @@ surface. **Reading past the end must fail.** An operation that would consume more bits than remain in the stream fails the read. It must not produce a partial value, zero-fill the missing bits, or wrap. The failure is terminal under the rule -below. +below. A stream's length is a count of bytes, so the test an operation performs +is `bit_index + n > 8 * bytes`, and `align` can never run past the end because +it never crosses a byte boundary. + +**A trap, a crash or an abort on malformed input is non-conforming.** Refusal +is the only conforming answer to a stream this document says must be refused, +and a runtime whose integer arithmetic traps on overflow unconditionally uses +wrapping operations on the read path, so that no input can turn a refusal into +a trap. **A refused primitive read must leave its destination unwritten.** The rule is per primitive read: when a read of a scalar fails, the caller's value must be @@ -740,6 +820,16 @@ enforces that rather than the caller's discipline. Two shapes satisfy it: stream, and a reader that throws, satisfy the rule as written, because neither hands the caller a stream to continue on. +Both shapes conform, and the speed rule does not forbid a language's own idiom +for failure, so a runtime that reports failure by throwing conforms where +throwing is that idiom, with the poisoned position remaining the recommended +latch. + +Every read consults the failure state before it does anything else, zero-bit +reads included, so a degenerate ranged read, an `align` on an already aligned +stream, a `bytes` call of zero count and `object` all refuse on a stream that +has already failed. + A failure persists until the stream is **re-initialized**, which is the operation that points a stream at a new buffer, or until the stream is discarded. An implementation with no re-initialization discards. @@ -761,6 +851,12 @@ allocation contract its caller is under — a caller holding the wrong contract is reading out of bounds, and that is a property of the implementation's documentation, not of the wire. +The eight-byte slack contract is the fast path, and every runtime states that +its callers are under it. A copying entry point that takes an exact-length +input and copies it into a slack backed buffer is admitted as a convenience, as +the C implementation's padded wrapper is, because it prices the copy once at +the boundary rather than per operation in the hot path. + **Trailing bits: writers must write zero; readers must not look; tools may judge.** *(Adopted 2026-08-15; the ruling verbatim: "Yes, I am OK with writers must write zero, readers must ignore non-zero. And it's good for a @@ -812,7 +908,9 @@ because it is the nearest working example is how inventions travel disguised as specification; every port-to-port inheritance in the audit was carrying one. If the standard lacks the information needed to implement correctly and fast, **the standard is too loose: tighten it here, upstream — never improvise in a -port.** +port.** Every sentence that sends a reader to the C++ implementation for a rule +is owed a rule in this document and a vector in the corpus, and what those +sentences still cover is listed as the remaining silences under Provenance. **The check model.** The caller is responsible for well-formed writes. Where the language has checked builds, write-side contract validation uses them and @@ -846,7 +944,9 @@ than the C++ implementation for any reason other than a documented language necessity is defective, and the deviation and its necessity must be documented where the divergence lives. Performance parity is part of conformance in spirit: C and C++ at total parity; systems languages within a few percent, -with every residual attributed to a named language mechanism. +with every residual attributed to a named language mechanism. Swift is a +systems language for that parity class, and ARC traffic and bounds checks are +nameable residuals once they are measured and attributed. ## Compatibility Notes @@ -871,7 +971,10 @@ verifying every claim against its golden test vector. disagree, the implementation is a bug.** Where this document is silent, the behavior of the C++ implementation (`mas-bandwidth/serialize`) breaks the tie, and it holds that standing only until this document is amended to state the -rule itself. A port never copies an implementation over the text of this +rule itself. That tie-break is a marker over a silence rather than a source: +what it still covers is listed as the remaining silences below, each owed a +rule here and a vector in the corpus, and this sentence goes when that list is +empty. A port never copies an implementation over the text of this document. `serialize.h` is one implementation among nine: C, C++, C#, Dart, Elixir, Go, Java, JavaScript and Rust. It was the first, which is a fact about history and not about standing. @@ -895,13 +998,60 @@ explicit refusal test. An implementation conforms when it reproduces every vector byte for byte and refuses everything this document says must be refused. **The shared corpus is the conformance instrument.** It is the `conformance/` -directory of this repository, one file per operation, holding the accepted and -refused vectors this document's rules require. Every implementation vendors and -syncs that directory the way it vendors this document, and its test suite must -run every vector in it. No checker reimplements the codec and then checks that -reimplementation against itself. A suite that regenerates its own expectations -proves only that a port agrees with itself, which is how one wrong reading of -this document travels to nine implementations under green results. +directory of this repository, one file per covered operation, holding the +accepted and refused vectors this document's rules require. The covered +operations are: + +| file | operation | what it pins | +|---|---|---| +| `bits.txt` | `bits`, and the `uint8`/`uint16`/`uint32`/`uint64` aliases | every width from 1 to 64, and the low 32-bit group before the high one above 32 | +| `bool.txt` | `bool` | one bit, and a reader that must not look at the rest of the byte | +| `uint128.txt` | `uint128` | 128 raw bits, the low 64-bit half first | +| `align.txt` | `align` | zero bits to the boundary, nothing when already aligned, and non-zero padding refused | +| `int.txt` | `int` | the width rule, the degenerate range at zero bits, and a decoded value outside `[min,max]` refused | +| `int64.txt` | `int64` | the same, at the 32/33-bit group boundary and across a span the signed domain cannot hold | +| `int128.txt` | `int128` | the ranged 128 bit codec, including ranges wider than `2^127` | +| `fixed.txt` | `fixed` | the offset over the raw bounds at six Q formats, and the degenerate range at zero bits on every storage width | +| `int_relative.txt` | `int_relative` | the domain, the refusal of a reconstruction outside it in every tier, and each bounded tier's own payload range | +| `float.txt` | `float` | NaN payloads, a signaling NaN, negative zero and denormals, compared as bit patterns | +| `double.txt` | `double` | the same at 64 bits, and the low half before the high one | +| `compressed_float.txt` | `compressed_float` | both clamp witnesses, the values clamp as a field width, and a non-zero `min` decode pinned bit exactly | +| `bytes.txt` | `bytes` | the align that comes first, and a zero count that still performs it | +| `string.txt` | `string` | valid UTF-8 accepted, and invalid UTF-8, an interior NUL and an out-of-range length refused | +| `wstring.txt` | `wstring` | no alignment anywhere, a surrogate pair accepted, and an unpaired surrogate, a group above `0xFFFF` and a zero group refused | +| `object.txt` | `object` | that it adds no bytes of its own, each vector twinned with the same operations unnested | +| `sequence.txt` | sequences of operations | alignment after an odd width, a zero-bit field between wide ones, terminal failure, and the measure floor | +| `message.txt` | the Worked Example's message | three operations across an alignment boundary, byte for byte | + +Adding a file here is how a rule becomes a family obligation, and the table +above moves with the directory. + +Every implementation vendors and syncs that directory the way it vendors this +document, and its test suite must run every vector in it. No checker +reimplements the codec and then checks that reimplementation against itself. A +suite that regenerates its own expectations proves only that a port agrees with +itself, which is how one wrong reading of this document travels to nine +implementations under green results. + +**A runner discovers the directory. It does not name the files.** A runner +holding a list of filenames in its source runs the vectors someone remembered to +add to that list, so a newly vendored file goes untested in every implementation +whose list was not edited, silently and under green results. Enumerate the +directory instead: at run time, or at build time from a glob that re-runs when +the directory changes. An empty directory fails the run, because an empty corpus +is a broken checkout and not a pass, and so does a vector whose operation the +runner has no code for, because an operation nobody implemented must be red +rather than skipped. + +**The remaining silences.** These are the rules a reader can get today only by +reading an implementation, each named by the section that owes it and by what a +vector would pin. The tie-break sentence above stands until this list is empty. + +| section | what a vector would pin | why the corpus does not carry it | +|---|---|---| +| `fixed` | a negative tie value, which separates half away from zero from an arithmetic shift | the rounding happens where a value is quantized into a Q format or narrowed out of one, and no `serialize_fixed` call rounds: the wire round trip is exact, this document says as much, and no record whose input is a byte stream can reach the rule. It needs a record shape this format does not have | +| `compressed_float` | the between-quanta writer inputs, and the fusion witness as a WRITE | a vector's input is a stream, so it can pin what a writer left on the wire but not the value a writer was handed. Both clamp declarations' widths, boundaries and refusals are pinned, and `8388608.0` is pinned as a decode in each of them; the writer inputs need a record shape this format does not have | +| Worked Example | the whole 112-byte golden message | this document prints 15 of its bytes and `message.txt` carries exactly those. The rest exist only inside an implementation, and a vector copied off the implementation it judges is the failure this section opens by naming | **The vector format.** A vector file is text. `#` begins a comment, blank lines separate records, and each record is `key` and value, one per line: @@ -912,13 +1062,84 @@ separate records, and each record is `key` and value, one per line: | `name` | a stable identifier for the vector | | `param` | one parameter as `name = value`, repeated once per parameter | | `bytes` | the stream, as hexadecimal byte pairs, empty for a zero-bit read | -| `expect` | the word `refused`, or `value = ` and the decoded value | +| `expect` | the word `refused`, or `value = ` and the decoded value, or `bits = 0x` and the decoded bit pattern | | `consumed` | bits a conforming reader consumes, accepted reads only | +| `writer` | the word `canonical`, on a vector that also pins the bytes a writer emits | +| `measure_at_least` | the smallest number of bits a conforming measure may report, on a sequence | `consumed` is stated only for accepted reads. **After a refusal the stream position is not part of the contract**, so no vector states it and no implementation is judged on it. +**A vector binds the reader only, unless it carries `writer = canonical`.** An +accepted vector states what a reader must decode from those bytes and states +nothing about what a writer emits for that value, which is what lets a vector +pin a valid non-canonical encoding such as an `int_relative` absolute tier +carrying a difference a writer would have sent in a narrower tier. A vector +carrying `writer = canonical` additionally pins the bytes a conforming writer +produces for that value. + +**Values are typed by the operation's table.** A `param` takes the type its own +section gives that parameter, so `min` and `max` under `fixed` are whole real +units, `buffer_size` under `string` is a byte count, and `res` under +`compressed_float` is a `float32`. + +**Lexical rules.** `#` begins a comment at the start of a line and nowhere +else, numbers are written as signed decimal or as `0x` hexadecimal, and a +parser must accept values up to 128 bits wide. `expect bits = 0x...` is the +spelling for a value compared as a bit pattern rather than as a number, which +is how `float` and `double` vectors are stated. + +**A sequence vector states more than one operation.** Its `operation` is +`sequence`, its steps are `param step = ` lines in order, and `expect value` +lists one entry per step separated by ` | `, with `-` for a step that produces +no value of its own. Sequences carry what a single-operation record cannot: +alignment cost, which depends on the bit index the previous operation left +behind; a zero-bit field, which must leave that index exactly where it found +it; a nested `object`, spelled `object ` to wrap the next `n` steps; and +terminal failure, where a refusal must poison the stream for every later step +even where the bytes those steps would read are still present. + +**A measure is checked against a floor, never against a number.** A sequence +may state `measure_at_least`, and a conforming measure must report at least it. +The key is a floor rather than an equality because a measure is a bound and not +the packet size: the value stated is the true worst case over every starting +bit position, so the expected implementation's 7 bits per alignment-performing +operation passes, a tighter correct bound passes, and the exact-from-zero +accounting this document calls non-conforming falls below it. + +**A harness presents every stream with the slack the contract requires.** The +bytes of a vector are the stream, and a harness running it against an +implementation under the eight-byte slack contract allocates that slack behind +them, including for the empty stream of a zero-bit read. + +**What a runner checks.** These are obligations. A runner that checks less than +this is not running the instrument. + +* For an **accepted** vector: the decoded value equals the value the record + states, compared as a number, or as a bit pattern where the record states + `expect bits = 0x`; and the bits consumed equal `consumed`. +* For a vector carrying `writer = canonical`: additionally, the bytes a writer + emits for that value, byte for byte. +* For a **refused** vector: the read is refused, the caller's scalar destination + holds exactly what it held before the call, and the stream is left terminal. + Terminality is checked by behavior rather than by an accessor, so the check + ports to every implementation: issue a further read on the same stream and + require that it also fails, consumes no bits, and writes nothing to its own + destination. + +**A gate nobody has seen go red is a claim, not a measurement.** The corpus is +an instrument, and an instrument is trusted because it has been shown to +respond. Two controls establish that, and both belong in the test chain rather +than in a README: a corpus file whose operation has no runner, which every gate +must reject rather than skip; and a reader with one line of a refusal rule +deleted, run against the real corpus, which must go red on exactly the vectors +that pin that rule. A control that stops finding anything to break is a failure +of the control and not a licence to remove it. This repository carries all +three as `ctest` targets, the sabotage generated at configure time from a +scratch copy of the header so that a needle which no longer matches is a hard +error. + **Conformance vectors must discriminate.** A value taken from the middle of a range, or one that lands where every plausible reading agrees, proves nothing — the `compressed_float` divergence above survived years of green test suites diff --git a/USAGE.md b/USAGE.md index f1f8b64..ce0b8d8 100644 --- a/USAGE.md +++ b/USAGE.md @@ -395,6 +395,41 @@ the wire: for the same bounds, signed and unsigned storage produce identical bytes. Reads refuse raw values smuggled into the bit headroom; an invalid declaration is caller misuse, asserted under `-ea`. +## Nesting: composition, not an encoding + +`serializeObject(serializer)` runs a nested object's own serialize +function inline. It contributes **no bytes of its own** — no framing, no +length prefix and no alignment is inserted around it — so the nested +operations land at exactly the position the call sits at, and nesting a +run of fields changes nothing about the bytes. + +`Serializer` is a one-method interface, so a message type implements it +directly and a small nested field can be a lambda: + +```java +final class Transform implements Serializer +{ + final FloatRef x = new FloatRef(); + final FloatRef y = new FloatRef(); + + @Override public boolean serialize( BitStream stream ) + { + return stream.serializeFloat( x ) && stream.serializeFloat( y ); + } +} + +// the same three fields, nested and flat, produce the same bytes +stream.serializeInt( health, 0, 100 ) && +stream.serializeObject( transform ); + +stream.serializeInt( health, 0, 100 ) && +stream.serializeFloat( transform.x ) && stream.serializeFloat( transform.y ); +``` + +A refusal inside the nesting propagates out of it, and a read stream +consults its failure state first, so a nested object on a stream that has +already failed refuses without invoking the object at all. + ## Measuring `MeasureStream` prices a message without a buffer. For everything except diff --git a/conformance/align.txt b/conformance/align.txt new file mode 100644 index 0000000..706904d --- /dev/null +++ b/conformance/align.txt @@ -0,0 +1,116 @@ +# serialize conformance vectors: serialize_align +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "align": pads with ZERO BITS until the bit index is a multiple +# of 8, and if the stream is already aligned NOTHING is written. "Readers must +# verify that the padding bits are zero and fail the read if they are not." +# +# An align at bit index 0 is a no-op, so every interesting property of this +# operation lives at a non-zero bit index. `preceding_bits` places the stream +# there: the reader first performs serialize_bits over that width and discards +# the value, then performs the align under test. `consumed` counts both. +# +# `value` is the padding the align consumed, which STANDARD.md requires to be +# zero on every accepted read — so every accepted vector here states 0, and +# every refusal is a stream whose padding is not zero. +# +# On a `writer = canonical` vector the writer leg emits the `preceding_bits` +# bits the reader just decoded and then performs the align, and the result must +# be the vector's bytes: that is what pins "pads with zero bits" and "if the +# stream is already aligned, nothing is written" on the write side. +# +# The align never reads past the end: the padding always lies inside the byte +# the preceding bits are in, and a vector's stream is a whole number of bytes. +# So the past-end rule has no vector in this file, and that is the rule's +# absence rather than a gap. + +# Already aligned: nothing is consumed. An implementation that pads a whole +# byte when the index is already a multiple of 8 runs off the end of this +# one-byte stream and refuses it. + +operation align +name align-no-op-when-already-aligned +param preceding_bits = 8 +bytes AB +expect value = 0 +consumed 8 +writer canonical + +# The strongest form of the same rule: at bit index 0 the stream is aligned, so +# the align consumes nothing — proved from an EMPTY stream, where any read at +# all is an error. + +operation align +name align-no-op-at-bit-index-zero +param preceding_bits = 0 +bytes +expect value = 0 +consumed 0 +writer canonical + +operation align +name align-pads-one-bit +param preceding_bits = 7 +bytes 7F +expect value = 0 +consumed 8 +writer canonical + +operation align +name align-pads-four-bits +param preceding_bits = 4 +bytes 0A +expect value = 0 +consumed 8 +writer canonical + +operation align +name align-pads-seven-bits +param preceding_bits = 1 +bytes 01 +expect value = 0 +consumed 8 +writer canonical + +# The padding must be zero, and readers must fail the read otherwise. Each +# refusal below shares its `preceding_bits` with an accepted vector above, so a +# reader that refuses on the bit index rather than on the padding contents +# fails the accepted twin. + +operation align +name align-refuse-one-non-zero-pad-bit +param preceding_bits = 7 +bytes FF +expect refused + +operation align +name align-refuse-four-non-zero-pad-bits +param preceding_bits = 4 +bytes AA +expect refused + +operation align +name align-refuse-seven-non-zero-pad-bits +param preceding_bits = 1 +bytes FF +expect refused + +# Only the highest padding bit is set. A reader that checks the first padding +# bit, or that tests the padding one bit at a time and stops early, accepts +# this stream; the document requires all of the padding to be zero. + +operation align +name align-refuse-only-the-highest-pad-bit-set +param preceding_bits = 1 +bytes 81 +expect refused + +# Only the lowest padding bit is set — the mirror of the vector above. + +operation align +name align-refuse-only-the-lowest-pad-bit-set +param preceding_bits = 1 +bytes 03 +expect refused diff --git a/conformance/bits.txt b/conformance/bits.txt new file mode 100644 index 0000000..8824828 --- /dev/null +++ b/conformance/bits.txt @@ -0,0 +1,180 @@ +# serialize conformance vectors: serialize_bits +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "bits": the low `bits` bits of the value, where `bits` is in +# [1,64]. For bits <= 32 a single group; for bits > 32 the LOW 32 BITS FIRST as +# a 32-bit group, then the remaining bits - 32 high bits. The fixed width +# helpers serialize_uint8 / uint16 / uint32 / uint64 are aliases for exactly +# this and carry no range information of their own, so they are covered here +# rather than in files of their own; the vectors that stand for them say so in +# their names. +# +# `bits` has no range rule: every bit pattern of the stated width is a legal +# value, so the only refusal STANDARD.md states for this operation is reading +# past the end of the stream ("Reader Obligations"). Every refusal vector below +# is that rule, and each has an accepted twin over the same width so a reader +# that refuses on the width rather than on the remaining length fails the twin. + +operation bits +name bits-1-zero +param bits = 1 +bytes 00 +expect value = 0 +consumed 1 +writer canonical + +operation bits +name bits-1-one +param bits = 1 +bytes 01 +expect value = 1 +consumed 1 +writer canonical + +# The low bit is the value; STANDARD.md, "Trailing bits": readers must not +# reject a stream for the contents of the unused bits of the final byte. This +# vector is reader-only because no conforming writer emits those bits set. + +operation bits +name bits-1-reader-ignores-trailing-bits +param bits = 1 +bytes FF +expect value = 1 +consumed 1 + +operation bits +name bits-8-alias-uint8 +param bits = 8 +bytes 7F +expect value = 127 +consumed 8 +writer canonical + +operation bits +name bits-8-max +param bits = 8 +bytes FF +expect value = 255 +consumed 8 +writer canonical + +operation bits +name bits-16-alias-uint16 +param bits = 16 +bytes 34 12 +expect value = 4660 +consumed 16 +writer canonical + +operation bits +name bits-24 +param bits = 24 +bytes EF CD AB +expect value = 11259375 +consumed 24 +writer canonical + +operation bits +name bits-32-alias-uint32 +param bits = 32 +bytes EF BE AD DE +expect value = 3735928559 +consumed 32 +writer canonical + +operation bits +name bits-32-max +param bits = 32 +bytes FF FF FF FF +expect value = 4294967295 +consumed 32 +writer canonical + +# The 32/33 boundary, where the single group becomes low-32-then-high. A writer +# that emits the high group first puts this value's single set bit in bit 0 +# instead of bit 32 and produces 01 00 00 00 00. + +operation bits +name bits-33-low-group-precedes-high-group +param bits = 33 +bytes 00 00 00 00 01 +expect value = 4294967296 +consumed 33 +writer canonical + +operation bits +name bits-33-max +param bits = 33 +bytes FF FF FF FF 01 +expect value = 8589934591 +consumed 33 +writer canonical + +operation bits +name bits-64-alias-uint64 +param bits = 64 +bytes F0 DE BC 9A 78 56 34 12 +expect value = 1311768467463790320 +consumed 64 +writer canonical + +operation bits +name bits-64-high-half-only +param bits = 64 +bytes 00 00 00 00 FF FF FF FF +expect value = 18446744069414584320 +consumed 64 +writer canonical + +operation bits +name bits-64-low-half-only +param bits = 64 +bytes FF FF FF FF 00 00 00 00 +expect value = 4294967295 +consumed 64 +writer canonical + +operation bits +name bits-64-max +param bits = 64 +bytes FF FF FF FF FF FF FF FF +expect value = 18446744073709551615 +consumed 64 +writer canonical + +# Reading past the end. Each refusal states a width the stream cannot supply; +# the accepted twin above it supplies exactly enough. + +operation bits +name bits-refuse-past-end-empty-stream +param bits = 1 +bytes +expect refused + +operation bits +name bits-refuse-past-end-16-of-8 +param bits = 16 +bytes FF +expect refused + +operation bits +name bits-accept-16-of-16 +param bits = 16 +bytes FF FF +expect value = 65535 +consumed 16 +writer canonical + +operation bits +name bits-refuse-past-end-33-of-32 +param bits = 33 +bytes FF FF FF FF +expect refused + +operation bits +name bits-refuse-past-end-64-of-56 +param bits = 64 +bytes FF FF FF FF FF FF FF +expect refused diff --git a/conformance/bool.txt b/conformance/bool.txt new file mode 100644 index 0000000..96fea43 --- /dev/null +++ b/conformance/bool.txt @@ -0,0 +1,47 @@ +# serialize conformance vectors: serialize_bool +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "bool": one bit, 1 for true and 0 for false. There is no range +# rule — both bit values are legal — so the only refusal STANDARD.md states for +# this operation is reading past the end of the stream. + +operation bool +name bool-false +bytes 00 +expect value = false +consumed 1 +writer canonical + +operation bool +name bool-true +bytes 01 +expect value = true +consumed 1 +writer canonical + +# STANDARD.md, "Trailing bits": readers must not reject a stream for the +# contents of the unused bits of the final byte, and no read operation examines +# them. A reader that tests the whole byte rather than the low bit fails this. +# Reader-only: no conforming writer emits those bits set. + +operation bool +name bool-true-reader-ignores-trailing-bits +bytes FF +expect value = true +consumed 1 + +# Only the low bit is the value. A reader that tests the byte against zero +# would call this true. + +operation bool +name bool-false-with-higher-bits-set +bytes FE +expect value = false +consumed 1 + +operation bool +name bool-refuse-past-end-empty-stream +bytes +expect refused diff --git a/conformance/bytes.txt b/conformance/bytes.txt new file mode 100644 index 0000000..5d8d9ba --- /dev/null +++ b/conformance/bytes.txt @@ -0,0 +1,124 @@ +# serialize conformance vectors: serialize_bytes +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "bytes": ALIGNS FIRST, then writes `count` raw bytes. "The +# alignment is part of the format, not an optimization — a reader that does not +# align will desynchronize." `count` is not written; both sides agree on it. +# +# `count` may be zero, and "the alignment is performed regardless": a +# zero-length serialize_bytes pads to the byte boundary and writes nothing +# else. Skipping the alignment when count is zero desynchronizes every field +# that follows, and a round-trip self-test cannot catch it because both halves +# skip the same align — so the corpus must, and does, below. +# +# `preceding_bits` places the stream at a non-aligned bit index before the +# operation, exactly as in align.txt; without it the operation's own align is +# always a no-op and the format's load-bearing clause goes untested. `value` is +# the decoded payload as hexadecimal byte pairs, empty for count = 0. +# +# Two refusal rules reach this operation: the alignment padding must be zero +# ("align"), and reading past the end must fail ("Reader Obligations"). Note +# that STANDARD.md leaves the buffer's contents UNSPECIFIED after a refused +# read into a caller-owned buffer, so no vector here states anything about it. + +operation bytes +name bytes-zero-count-aligned-start +param preceding_bits = 0 +param count = 0 +bytes +expect value = +consumed 0 +writer canonical + +# The load-bearing clause: count = 0 still performs the align. Three bits of +# value, then five bits of zero padding. An implementation that skips the align +# when count is zero consumes 3 bits here instead of 8. + +operation bytes +name bytes-zero-count-still-performs-the-align +param preceding_bits = 3 +param count = 0 +bytes 05 +expect value = +consumed 8 +writer canonical + +operation bytes +name bytes-zero-count-refuse-non-zero-align-padding +param preceding_bits = 3 +param count = 0 +bytes FD +expect refused + +operation bytes +name bytes-single-byte-aligned-start +param preceding_bits = 0 +param count = 1 +bytes FF +expect value = FF +consumed 8 +writer canonical + +operation bytes +name bytes-all-zero +param preceding_bits = 0 +param count = 3 +bytes 00 00 00 +expect value = 00 00 00 +consumed 24 +writer canonical + +# The golden message's byte block. + +operation bytes +name bytes-seven-bytes-aligned-start +param preceding_bits = 0 +param count = 7 +bytes DE AD BE EF CA FE 01 +expect value = DE AD BE EF CA FE 01 +consumed 56 +writer canonical + +operation bytes +name bytes-after-a-partial-byte +param preceding_bits = 4 +param count = 2 +bytes 0A AB CD +expect value = AB CD +consumed 24 +writer canonical + +operation bytes +name bytes-refuse-non-zero-align-padding +param preceding_bits = 4 +param count = 2 +bytes FA AB CD +expect refused + +operation bytes +name bytes-refuse-past-end +param preceding_bits = 0 +param count = 4 +bytes DE AD BE +expect refused + +# The align lands exactly on the end of the stream and there is nothing left +# for the payload. A reader that checks the length before aligning accepts. + +operation bytes +name bytes-refuse-past-end-after-the-align +param preceding_bits = 1 +param count = 1 +bytes 01 +expect refused + +operation bytes +name bytes-accept-one-byte-after-the-align +param preceding_bits = 1 +param count = 1 +bytes 01 5A +expect value = 5A +consumed 16 +writer canonical diff --git a/conformance/compressed_float.txt b/conformance/compressed_float.txt new file mode 100644 index 0000000..f6b845e --- /dev/null +++ b/conformance/compressed_float.txt @@ -0,0 +1,347 @@ +# serialize conformance vectors: serialize_compressed_float +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "compressed_float": let delta = max - min and +# values = delta / res, CLAMPED TO [1, 4294967040] (the largest float below +# 2^32). Then max_integer_value = ceil( values ) and +# bits = bits_required( 0, max_integer_value ). The high end of that clamp is +# what keeps bits_required's 32-bit form well defined, and it is visible on the +# wire as the field's width — the `bits-clamped` vectors below pin it. +# +# The reader divides the integer by max_integer_value, multiplies by delta, and +# adds min. "The reader's arithmetic is pinned the same way" as the writer's: +# it is float32 with EVERY STEP ROUNDING — the quotient rounds, the product +# rounds BEFORE min is added, and the sum rounds. An implementation must not +# widen any step to double, and must not contract the multiply and the add into +# a fused multiply-add: fused, the decode rounds once instead of twice, and +# whenever min is non-zero the decoded value can land one ulp away from the +# conformant result. "Conformance vectors over a non-zero-min range must pin +# decoded values BIT-EXACTLY — a tolerance comparison cannot see a one-ulp +# divergence." Every expectation here is therefore `expect bits`. +# +# The corpus is a reader instrument and these are reader vectors. The writer's +# own discriminating band — the between-quanta inputs and the fused-writer +# witness 8388608.0 — is a property of the value chosen for a write, which no +# record whose input is a byte stream can state; that gap is reported with this +# corpus rather than papered over. What the writer side leaves on the wire IS +# pinned here: the two declarations the document names for the integer clamp, +# [0, 8388609] and [0, 16777215] at resolution 1, appear below as the widths +# and accept boundaries a reader must agree with. +# +# The refusal rules are "Readers must reject an integer greater than +# max_integer_value" and reading past the end. Non-finite inputs and non-finite +# declarations are non-conforming on the WRITE side and "the read path is +# untouched: the poison is in the declaration or the input value, never on the +# wire, so there is nothing for a reader to refuse" — so no vector here states +# one, and that is the document's ruling rather than a gap. + +# [0,10] at resolution 0.01: values = 1000 exactly, max_integer_value = 1000, +# ten bits. + +operation compressed_float +name compressed-float-zero-integer-over-0-10 +param min = 0 +param max = 10 +param res = 0.01 +bytes 00 00 +expect bits = 0x00000000 +consumed 10 + +# The golden message's field: integer 500, which normalizes to exactly 0.5 and +# lands exactly on a quantum, where float32, double and a fused multiply-add +# all produce the same answer. It is the on-quantum ANCHOR and it can never +# discriminate; it is here to keep the width and the decode honest, and the +# vectors that can fail are below it. + +operation compressed_float +name compressed-float-on-quantum-anchor-over-0-10 +param min = 0 +param max = 10 +param res = 0.01 +bytes F4 01 +expect bits = 0x40A00000 +consumed 10 + +# Integer 1 over [0,10]: float32( 1 / 1000 ) * 10 + 0, two roundings. + +operation compressed_float +name compressed-float-integer-one-over-0-10 +param min = 0 +param max = 10 +param res = 0.01 +bytes 01 00 +expect bits = 0x3C23D70B +consumed 10 + +operation compressed_float +name compressed-float-accepts-max-integer-value-over-0-10 +param min = 0 +param max = 10 +param res = 0.01 +bytes E8 03 +expect bits = 0x41200000 +consumed 10 + +operation compressed_float +name compressed-float-refuse-integer-one-past-max-integer-value +param min = 0 +param max = 10 +param res = 0.01 +bytes E9 03 +expect refused + +operation compressed_float +name compressed-float-refuse-integer-all-ten-bits +param min = 0 +param max = 10 +param res = 0.01 +bytes FF 03 +expect refused + +# A NON-ZERO min, where the reader's second rounding becomes visible: the +# product rounds BEFORE min is added, and a fused decode lands one ulp away. +# [-100,100] at resolution 0.01: values = 20000, fifteen bits. + +operation compressed_float +name compressed-float-nonzero-min-integer-zero +param min = -100 +param max = 100 +param res = 0.01 +bytes 00 00 +expect bits = 0xC2C80000 +consumed 15 + +# Integer 10000 normalizes to exactly 0.5; the product is exactly 100.0 and the +# sum is exactly +0.0. The on-quantum anchor for this declaration. + +operation compressed_float +name compressed-float-nonzero-min-midpoint +param min = -100 +param max = 100 +param res = 0.01 +bytes 10 27 +expect bits = 0x00000000 +consumed 15 + +# Integer 384 over [-100,100]: the format's answer is 0xC2C051EC and a fused +# reader decode is 0xC2C051EB. One ulp, invisible to any tolerance comparison, +# and it changes the bytes a re-encode produces — which is how a fusing +# platform breaks round-tripping with a conforming one. + +operation compressed_float +name compressed-float-nonzero-min-fused-decode-differs-by-one-ulp +param min = -100 +param max = 100 +param res = 0.01 +bytes 80 01 +expect bits = 0xC2C051EC +consumed 15 + +# Integer 5743 over [-100,100], the family's brute-forced off-quantum value. + +operation compressed_float +name compressed-float-nonzero-min-off-quantum +param min = -100 +param max = 100 +param res = 0.01 +bytes 6F 16 +expect bits = 0xC22A47AE +consumed 15 + +operation compressed_float +name compressed-float-nonzero-min-accepts-max-integer-value +param min = -100 +param max = 100 +param res = 0.01 +bytes 20 4E +expect bits = 0x42C80000 +consumed 15 + +operation compressed_float +name compressed-float-nonzero-min-refuse-integer-one-past-max-integer-value +param min = -100 +param max = 100 +param res = 0.01 +bytes 21 4E +expect refused + +operation compressed_float +name compressed-float-nonzero-min-refuse-integer-all-fifteen-bits +param min = -100 +param max = 100 +param res = 0.01 +bytes FF 7F +expect refused + +# The integer clamp's band, the first of the two declarations STANDARD.md names +# (schema#109, adopted 2026-08-23): [0, 8388609] at resolution 1. Here +# max_integer_value is 8388609 and the field is 24 bits, so codes 8388610 +# through 16777215 are expressible and every one of them is the "reader +# rejects" class the ruling closes. The accept boundary is the largest legal +# code and the refusal is exactly one past it. + +operation compressed_float +name compressed-float-clamp-band-accepts-max-integer-value +param min = 0 +param max = 8388609 +param res = 1 +bytes 01 00 80 +expect bits = 0x4B000001 +consumed 24 + +# 2^23 itself, the value STANDARD.md names as the fusion witness: "measured +# exhaustively over [0, 16777215] at resolution 1, a fused writer moves +# 4,194,304 inputs (every even float in the top binade), the first being 2^23 +# itself." That witness is a writer INPUT and no record whose own input is a +# byte stream can state it, so what is pinned here is the reader's side of the +# same band, at the same value, in both declarations the ruling names. + +operation compressed_float +name compressed-float-clamp-band-fusion-witness-value +param min = 0 +param max = 8388609 +param res = 1 +bytes 00 00 80 +expect bits = 0x4B000000 +consumed 24 + +operation compressed_float +name compressed-float-clamp-band-refuse-integer-one-past-max-integer-value +param min = 0 +param max = 8388609 +param res = 1 +bytes 02 00 80 +expect refused + +operation compressed_float +name compressed-float-clamp-band-refuse-integer-all-twenty-four-bits +param min = 0 +param max = 8388609 +param res = 1 +bytes FF FF FF +expect refused + +operation compressed_float +name compressed-float-clamp-band-integer-zero +param min = 0 +param max = 8388609 +param res = 1 +bytes 00 00 00 +expect bits = 0x00000000 +consumed 24 + +# The second declaration STANDARD.md names, [0, 16777215] at resolution 1 — +# the wire-divergence class, where implementations historically disagreed about +# the bytes because the rounded sum could exceed the field. Here +# max_integer_value is 16777215, which is exactly the largest 24-bit code, so +# EVERY code is legal and this declaration has no refusal vector: the rule's +# absence rather than a gap. What it pins is the width, which is 24 and not 25. + +operation compressed_float +name compressed-float-wire-divergence-band-accepts-max-integer-value +param min = 0 +param max = 16777215 +param res = 1 +bytes FF FF FF +expect bits = 0x4B7FFFFF +consumed 24 + +operation compressed_float +name compressed-float-wire-divergence-band-fusion-witness-value +param min = 0 +param max = 16777215 +param res = 1 +bytes 00 00 80 +expect bits = 0x4B000000 +consumed 24 + +operation compressed_float +name compressed-float-wire-divergence-band-integer-zero +param min = 0 +param max = 16777215 +param res = 1 +bytes 00 00 00 +expect bits = 0x00000000 +consumed 24 + +# The HIGH end of the values clamp, which is visible on the wire as the field's +# width. delta is 2^33 and res is 1, so values is 8589934592 — above +# 4294967040, so it clamps there and max_integer_value is 4294967040, giving a +# 32-bit field. Without the clamp the width would be 34 bits and the reader +# would consume two bits too many, so `consumed 32` is what pins the rule. + +operation compressed_float +name compressed-float-values-clamped-at-the-high-end +param min = 0 +param max = 8589934592 +param res = 1 +bytes 00 FF FF FF +expect bits = 0x50000000 +consumed 32 + +operation compressed_float +name compressed-float-values-clamped-integer-zero +param min = 0 +param max = 8589934592 +param res = 1 +bytes 00 00 00 00 +expect bits = 0x00000000 +consumed 32 + +operation compressed_float +name compressed-float-values-clamped-refuse-integer-one-past-max-integer-value +param min = 0 +param max = 8589934592 +param res = 1 +bytes 01 FF FF FF +expect refused + +operation compressed_float +name compressed-float-values-clamped-refuse-integer-all-thirty-two-bits +param min = 0 +param max = 8589934592 +param res = 1 +bytes FF FF FF FF +expect refused + +# The LOW end of the values clamp: delta / res is below 1, so values clamps to +# 1, max_integer_value is 1 and the field is a single bit. Every code is legal +# at that width, so this declaration has no refusal vector. + +operation compressed_float +name compressed-float-values-clamped-at-the-low-end-zero +param min = 0 +param max = 1 +param res = 10 +bytes 00 +expect bits = 0x00000000 +consumed 1 + +operation compressed_float +name compressed-float-values-clamped-at-the-low-end-one +param min = 0 +param max = 1 +param res = 10 +bytes 01 +expect bits = 0x3F800000 +consumed 1 + +# Reading past the end. + +operation compressed_float +name compressed-float-refuse-past-end-8-of-10 +param min = 0 +param max = 10 +param res = 0.01 +bytes FF +expect refused + +operation compressed_float +name compressed-float-refuse-past-end-empty-stream +param min = 0 +param max = 10 +param res = 0.01 +bytes +expect refused diff --git a/conformance/double.txt b/conformance/double.txt new file mode 100644 index 0000000..aa7fbd8 --- /dev/null +++ b/conformance/double.txt @@ -0,0 +1,148 @@ +# serialize conformance vectors: serialize_double +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "double": the 64 bits of the IEEE-754 double-precision +# representation, written as one 64-bit group — which is serialize_bits over 64 +# bits, so the low 32-bit group precedes the high one and the bytes on an +# aligned stream are the value in little-endian order. +# +# "Bit transparency — both directions" binds this operation exactly as it binds +# float, and so does the comparison rule: these vectors compare BIT PATTERNS, +# not values. Every expectation below is `expect bits`. +# +# There is no range rule and no refusal of any pattern, so the only refusal +# STANDARD.md states for this operation is reading past the end. + +operation double +name double-positive-zero +bytes 00 00 00 00 00 00 00 00 +expect bits = 0x0000000000000000 +consumed 64 +writer canonical + +operation double +name double-negative-zero +bytes 00 00 00 00 00 00 00 80 +expect bits = 0x8000000000000000 +consumed 64 +writer canonical + +operation double +name double-one +bytes 00 00 00 00 00 00 F0 3F +expect bits = 0x3FF0000000000000 +consumed 64 +writer canonical + +# The golden message's double field, 1/3. + +operation double +name double-one-third +bytes 55 55 55 55 55 55 D5 3F +expect bits = 0x3FD5555555555555 +consumed 64 +writer canonical + +# The whole value sits in one half or the other, so an implementation that +# writes the high 32-bit group first swaps these two vectors' bytes. + +operation double +name double-low-half-only +bytes EF CD AB 89 00 00 00 00 +expect bits = 0x0000000089ABCDEF +consumed 64 +writer canonical + +operation double +name double-high-half-only +bytes 00 00 00 00 EF CD AB 89 +expect bits = 0x89ABCDEF00000000 +consumed 64 +writer canonical + +operation double +name double-positive-infinity +bytes 00 00 00 00 00 00 F0 7F +expect bits = 0x7FF0000000000000 +consumed 64 +writer canonical + +operation double +name double-negative-infinity +bytes 00 00 00 00 00 00 F0 FF +expect bits = 0xFFF0000000000000 +consumed 64 +writer canonical + +operation double +name double-quiet-nan +bytes 00 00 00 00 00 00 F8 7F +expect bits = 0x7FF8000000000000 +consumed 64 +writer canonical + +# A SIGNALING NaN: the quiet bit is clear, and a reader that quiets NaNs +# rewrites it to 0x7FF8000000000001. + +operation double +name double-signaling-nan +bytes 01 00 00 00 00 00 F0 7F +expect bits = 0x7FF0000000000001 +consumed 64 +writer canonical + +operation double +name double-nan-with-payload +bytes 00 00 00 00 00 AD FE 7F +expect bits = 0x7FFEAD0000000000 +consumed 64 +writer canonical + +operation double +name double-negative-nan-all-payload-bits +bytes FF FF FF FF FF FF FF FF +expect bits = 0xFFFFFFFFFFFFFFFF +consumed 64 +writer canonical + +operation double +name double-smallest-positive-denormal +bytes 01 00 00 00 00 00 00 00 +expect bits = 0x0000000000000001 +consumed 64 +writer canonical + +operation double +name double-largest-denormal +bytes FF FF FF FF FF FF 0F 00 +expect bits = 0x000FFFFFFFFFFFFF +consumed 64 +writer canonical + +operation double +name double-smallest-positive-normal +bytes 00 00 00 00 00 00 10 00 +expect bits = 0x0010000000000000 +consumed 64 +writer canonical + +operation double +name double-largest-finite +bytes FF FF FF FF FF FF EF 7F +expect bits = 0x7FEFFFFFFFFFFFFF +consumed 64 +writer canonical + +# Reading past the end. + +operation double +name double-refuse-past-end-56-of-64 +bytes FF FF FF FF FF FF FF +expect refused + +operation double +name double-refuse-past-end-empty-stream +bytes +expect refused diff --git a/conformance/fixed.txt b/conformance/fixed.txt new file mode 100644 index 0000000..2a0f9bd --- /dev/null +++ b/conformance/fixed.txt @@ -0,0 +1,421 @@ +# serialize conformance vectors: serialize_fixed +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "fixed (Q format, ranged)": a value in an integer storage type +# of exactly integer_bits + fraction_bits bits, the stored integer being the +# real value scaled by 2^fraction_bits. The encoding is an offset encoding over +# the RAW (scaled) bounds: +# +# raw_min = min << fraction_bits +# raw_max = max << fraction_bits +# bits = bit length of ( raw_max - raw_min ) +# +# and raw_value - raw_min is written in that many bits, split into 32-bit +# groups from least significant upward exactly as serialize_bits splits wide +# values. With fraction_bits = 0 the operation IS a ranged integer, and for +# storage of 64 bits or fewer the bytes are identical to serialize_int64 over +# the raw bounds. +# +# `value` is the RAW stored integer, not the real value: the raw integer is +# what the wire carries and what the round trip is exact in, and stating it +# avoids every question of how a real value is spelled in each of the nine +# implementations' number syntax. raw = real * 2^fraction_bits. +# +# All four parameters are compile-time constants of the call site, so a runner +# supports a fixed set of declarations and a vector naming a declaration the +# runner does not carry must FAIL rather than pass — the same contract the +# corpus states for an operation with no runner. The declarations used here are +# Q8.8, Q32.0, Q16.16, Q48.16, Q112.16 and Q64.64. +# +# The refusal rules are "Readers must check that the decoded offset is at most +# raw_max - raw_min and fail otherwise — reject, never clamp", and reading past +# the end. + +# The document's own worked example: Q8.8 over +/- 100 whole units, raw_min +# -25600, raw range 51200, 16 bits. The golden message's bytes are C0 60, the +# offset 0x60C0 = 24768, and the raw value -832, which is -3.25 in Q8.8. + +operation fixed +name fixed-q8-8-worked-example +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes C0 60 +expect value = -832 +consumed 16 +writer canonical + +operation fixed +name fixed-q8-8-accepts-its-raw-minimum +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes 00 00 +expect value = -25600 +consumed 16 +writer canonical + +operation fixed +name fixed-q8-8-accepts-offset-equal-to-the-raw-span +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes 00 C8 +expect value = 25600 +consumed 16 +writer canonical + +operation fixed +name fixed-q8-8-refuse-offset-one-past-the-raw-span +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes 01 C8 +expect refused + +operation fixed +name fixed-q8-8-refuse-offset-all-ones +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes FF FF +expect refused + +# fraction_bits = 0: the operation is a ranged integer. + +operation fixed +name fixed-zero-fraction-bits-is-a-ranged-int +param integer_bits = 32 +param fraction_bits = 0 +param min = 0 +param max = 7 +bytes 05 +expect value = 5 +consumed 3 +writer canonical + +operation fixed +name fixed-zero-fraction-bits-accepts-its-maximum +param integer_bits = 32 +param fraction_bits = 0 +param min = 0 +param max = 5 +bytes 05 +expect value = 5 +consumed 3 +writer canonical + +operation fixed +name fixed-zero-fraction-bits-refuse-offset-one-past-the-span +param integer_bits = 32 +param fraction_bits = 0 +param min = 0 +param max = 5 +bytes 06 +expect refused + +# A degenerate range is legal and costs ZERO BITS — on every storage width. +# STANDARD.md is explicit that the storage width must not change this: a +# Q112.16 field over a degenerate range costs zero bits, not fraction_bits of +# zeros. The wide rows below are the widths where the implementations +# historically behaved four different ways. + +operation fixed +name fixed-degenerate-range-q16-16 +param integer_bits = 16 +param fraction_bits = 16 +param min = 7 +param max = 7 +bytes +expect value = 458752 +consumed 0 +writer canonical + +operation fixed +name fixed-degenerate-range-q112-16-negative +param integer_bits = 112 +param fraction_bits = 16 +param min = -9 +param max = -9 +bytes +expect value = -589824 +consumed 0 +writer canonical + +operation fixed +name fixed-degenerate-range-q64-64-zero +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 0 +bytes +expect value = 0 +consumed 0 +writer canonical + +# The strongest form: a wide raw value recovered from an EMPTY stream, so any +# read at all is an error and a pass proves zero bits rather than +# fraction_bits of zeros. + +operation fixed +name fixed-degenerate-range-q64-64-nonzero +param integer_bits = 64 +param fraction_bits = 64 +param min = 3 +param max = 3 +bytes +expect value = 55340232221128654848 +consumed 0 +writer canonical + +# A 32-bit raw span: the last width written as a single group. + +operation fixed +name fixed-q16-16-raw-span-32-bits +param integer_bits = 16 +param fraction_bits = 16 +param min = -32768 +param max = 32767 +bytes EF BE AD DE +expect value = 1588444911 +consumed 32 +writer canonical + +operation fixed +name fixed-q16-16-accepts-offset-equal-to-the-raw-span +param integer_bits = 16 +param fraction_bits = 16 +param min = -32768 +param max = 32767 +bytes 00 00 FF FF +expect value = 2147418112 +consumed 32 +writer canonical + +operation fixed +name fixed-q16-16-refuse-offset-one-past-the-raw-span +param integer_bits = 16 +param fraction_bits = 16 +param min = -32768 +param max = 32767 +bytes 01 00 FF FF +expect refused + +operation fixed +name fixed-q16-16-refuse-offset-all-ones +param integer_bits = 16 +param fraction_bits = 16 +param min = -32768 +param max = 32767 +bytes FF FF FF FF +expect refused + +# A 34-bit raw span in Q48.16: two groups, 32 bits then a 2-bit remainder. + +operation fixed +name fixed-q48-16-raw-span-34-bits-two-groups +param integer_bits = 48 +param fraction_bits = 16 +param min = 0 +param max = 131072 +bytes 00 00 00 00 01 +expect value = 4294967296 +consumed 34 +writer canonical + +operation fixed +name fixed-q48-16-accepts-offset-equal-to-the-raw-span +param integer_bits = 48 +param fraction_bits = 16 +param min = 0 +param max = 131072 +bytes 00 00 00 00 02 +expect value = 8589934592 +consumed 34 +writer canonical + +operation fixed +name fixed-q48-16-refuse-offset-one-past-the-raw-span +param integer_bits = 48 +param fraction_bits = 16 +param min = 0 +param max = 131072 +bytes 01 00 00 00 02 +expect refused + +# A 75-bit raw span in Q112.16 — the golden message's wide field shape: two +# full 32-bit groups from the bottom, then an 11-bit remainder on top. Each of +# the three vectors below puts a single set bit in one group, so a decoder that +# assembles the groups in the wrong order, or puts the remainder anywhere but +# the most significant position, decodes the wrong value for one of them. + +operation fixed +name fixed-q112-16-raw-span-75-bits-group-one +param integer_bits = 112 +param fraction_bits = 16 +param min = 0 +param max = 288230376151711744 +bytes 00 00 00 00 01 00 00 00 00 00 +expect value = 4294967296 +consumed 75 +writer canonical + +operation fixed +name fixed-q112-16-raw-span-75-bits-group-two +param integer_bits = 112 +param fraction_bits = 16 +param min = 0 +param max = 288230376151711744 +bytes 00 00 00 00 00 00 00 00 01 00 +expect value = 18446744073709551616 +consumed 75 +writer canonical + +operation fixed +name fixed-q112-16-accepts-offset-equal-to-the-raw-span +param integer_bits = 112 +param fraction_bits = 16 +param min = 0 +param max = 288230376151711744 +bytes 00 00 00 00 00 00 00 00 00 04 +expect value = 18889465931478580854784 +consumed 75 +writer canonical + +operation fixed +name fixed-q112-16-refuse-offset-one-past-the-raw-span +param integer_bits = 112 +param fraction_bits = 16 +param min = 0 +param max = 288230376151711744 +bytes 01 00 00 00 00 00 00 00 00 04 +expect refused + +operation fixed +name fixed-q112-16-refuse-offset-all-ones +param integer_bits = 112 +param fraction_bits = 16 +param min = 0 +param max = 288230376151711744 +bytes FF FF FF FF FF FF FF FF FF 07 +expect refused + +# The same width with a negative raw minimum, so the offset must be added to a +# negative raw base at 128-bit precision. + +operation fixed +name fixed-q112-16-negative-raw-minimum-decodes-zero +param integer_bits = 112 +param fraction_bits = 16 +param min = -144115188075855872 +param max = 144115188075855872 +bytes 00 00 00 00 00 00 00 00 00 02 +expect value = 0 +consumed 75 +writer canonical + +operation fixed +name fixed-q112-16-negative-raw-minimum-accepts-its-minimum +param integer_bits = 112 +param fraction_bits = 16 +param min = -144115188075855872 +param max = 144115188075855872 +bytes 00 00 00 00 00 00 00 00 00 00 +expect value = -9444732965739290427392 +consumed 75 +writer canonical + +# A 127-bit raw span in Q64.64 — four groups, the golden message's widest field. +# Over a zero minimum the decoded raw value is the offset itself. The span is +# raw_max = ( 2^63 - 1 ) << 64 = 2^127 - 2^64, whose bit length is 127 and not +# 128: an implementation that rounds the width up to the storage width reads one +# bit too many and every vector here goes red. + +operation fixed +name fixed-q64-64-raw-span-127-bits-group-one +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 +expect value = 4294967296 +consumed 127 +writer canonical + +operation fixed +name fixed-q64-64-raw-span-127-bits-group-two +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 +expect value = 18446744073709551616 +consumed 127 +writer canonical + +operation fixed +name fixed-q64-64-raw-span-127-bits-group-three +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 +expect value = 79228162514264337593543950336 +consumed 127 +writer canonical + +operation fixed +name fixed-q64-64-accepts-offset-equal-to-the-raw-span +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 00 FF FF FF FF FF FF FF 7F +expect value = 170141183460469231713240559642174554112 +consumed 127 +writer canonical + +operation fixed +name fixed-q64-64-refuse-offset-one-past-the-raw-span +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes 01 00 00 00 00 00 00 00 FF FF FF FF FF FF FF 7F +expect refused + +operation fixed +name fixed-q64-64-refuse-offset-all-ones +param integer_bits = 64 +param fraction_bits = 64 +param min = 0 +param max = 9223372036854775807 +bytes FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF 7F +expect refused + +# Reading past the end. + +operation fixed +name fixed-refuse-past-end +param integer_bits = 8 +param fraction_bits = 8 +param min = -100 +param max = 100 +bytes C0 +expect refused + +operation fixed +name fixed-refuse-past-end-empty-stream +param integer_bits = 16 +param fraction_bits = 16 +param min = -32768 +param max = 32767 +bytes +expect refused diff --git a/conformance/float.txt b/conformance/float.txt new file mode 100644 index 0000000..a21de24 --- /dev/null +++ b/conformance/float.txt @@ -0,0 +1,162 @@ +# serialize conformance vectors: serialize_float +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "float": the 32 bits of the IEEE-754 single-precision +# representation, written as a 32-bit group. No conversion, no compression. +# +# "Bit transparency — both directions": every bit pattern is legal on the wire +# — NaNs with any payload, signaling NaNs, infinities, negative zero, denormals +# — and the reader reproduces the transmitted pattern exactly. It must not +# canonicalize, quiet, flush, or refuse any of them. There is no reader +# latitude: a reader that rejects NaN, or canonicalizes payloads, does not +# conform. +# +# The document also states how these vectors must be compared: "Conformance +# vectors for these operations must compare BIT PATTERNS, NOT VALUES: NaN +# compares unequal to itself, -0.0 == 0.0, and a tolerance comparison cannot +# see a quieted signaling bit, so a value-space comparison here proves +# nothing." Every expectation below is therefore `expect bits`, and a runner +# that compares these as floats fails the NaN rows by construction and passes +# the negative-zero row while a canonicalizing reader is broken. +# +# There is no range rule and no refusal of any pattern, so the only refusal +# STANDARD.md states for this operation is reading past the end. + +operation float +name float-positive-zero +bytes 00 00 00 00 +expect bits = 0x00000000 +consumed 32 +writer canonical + +# -0.0 compares EQUAL to 0.0 in value space, so only a bit comparison can see a +# reader that canonicalizes the sign of zero. + +operation float +name float-negative-zero +bytes 00 00 00 80 +expect bits = 0x80000000 +consumed 32 +writer canonical + +operation float +name float-one +bytes 00 00 80 3F +expect bits = 0x3F800000 +consumed 32 +writer canonical + +operation float +name float-negative-one +bytes 00 00 80 BF +expect bits = 0xBF800000 +consumed 32 +writer canonical + +# The golden message's float field. + +operation float +name float-pi +bytes DB 0F 49 40 +expect bits = 0x40490FDB +consumed 32 +writer canonical + +operation float +name float-positive-infinity +bytes 00 00 80 7F +expect bits = 0x7F800000 +consumed 32 +writer canonical + +operation float +name float-negative-infinity +bytes 00 00 80 FF +expect bits = 0xFF800000 +consumed 32 +writer canonical + +operation float +name float-quiet-nan +bytes 00 00 C0 7F +expect bits = 0x7FC00000 +consumed 32 +writer canonical + +# A SIGNALING NaN: the quiet bit is clear. A reader that quiets NaNs — which +# many language runtimes do on an ordinary load and store — turns this into +# 0x7FC00001, and no tolerance comparison can see the difference. + +operation float +name float-signaling-nan +bytes 01 00 80 7F +expect bits = 0x7F800001 +consumed 32 +writer canonical + +# A NaN carrying a payload. A reader that canonicalizes NaNs loses it. + +operation float +name float-nan-with-payload +bytes 00 AD DE 7F +expect bits = 0x7FDEAD00 +consumed 32 +writer canonical + +operation float +name float-negative-nan-all-payload-bits +bytes FF FF FF FF +expect bits = 0xFFFFFFFF +consumed 32 +writer canonical + +# Denormals, which a reader that flushes subnormals to zero destroys. + +operation float +name float-smallest-positive-denormal +bytes 01 00 00 00 +expect bits = 0x00000001 +consumed 32 +writer canonical + +operation float +name float-largest-denormal +bytes FF FF 7F 00 +expect bits = 0x007FFFFF +consumed 32 +writer canonical + +operation float +name float-negative-denormal +bytes 01 00 00 80 +expect bits = 0x80000001 +consumed 32 +writer canonical + +operation float +name float-smallest-positive-normal +bytes 00 00 80 00 +expect bits = 0x00800000 +consumed 32 +writer canonical + +operation float +name float-largest-finite +bytes FF FF 7F 7F +expect bits = 0x7F7FFFFF +consumed 32 +writer canonical + +# Reading past the end. + +operation float +name float-refuse-past-end-24-of-32 +bytes 00 00 00 +expect refused + +operation float +name float-refuse-past-end-empty-stream +bytes +expect refused diff --git a/conformance/int.txt b/conformance/int.txt new file mode 100644 index 0000000..f932ac5 --- /dev/null +++ b/conformance/int.txt @@ -0,0 +1,263 @@ +# serialize conformance vectors: serialize_int +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "int (ranged)": the width is +# +# bits_required( min, max ) = ( min == max ) ? 0 : 32 - count_leading_zeros( max - min ) +# +# and `value - min` is written in that many bits. The document names the +# consequences and this file pins each of them: [0,7] costs 3 bits, [0,8] costs +# 4, and a degenerate range where min == max costs ZERO BITS with the value +# taken from the range alone. +# +# Two refusal rules reach this operation. "Readers must check that the decoded +# value lies within [min,max] and fail otherwise" — in offset form, an offset +# greater than max - min, which is expressible whenever the span is not one +# less than a power of two. And reading past the end must fail. Every refusal +# below carries an accepted twin one step inside the same edge, sharing the +# declaration, so a reader that refuses on the declaration rather than on the +# decoded value fails the twin. + +operation int +name int-degenerate-range-zero-bits +param min = 42 +param max = 42 +bytes +expect value = 42 +consumed 0 +writer canonical + +operation int +name int-degenerate-range-negative +param min = -1000 +param max = -1000 +bytes +expect value = -1000 +consumed 0 +writer canonical + +operation int +name int-degenerate-range-at-int32-min +param min = -2147483648 +param max = -2147483648 +bytes +expect value = -2147483648 +consumed 0 +writer canonical + +# The document's own pair: [0,7] is 3 bits, [0,8] is 4. An implementation that +# computes the width from the count of values rather than the span gets one of +# these wrong, and the `consumed` line is what catches it. + +operation int +name int-range-0-7-costs-three-bits +param min = 0 +param max = 7 +bytes 05 +expect value = 5 +consumed 3 +writer canonical + +operation int +name int-range-0-7-accepts-its-maximum +param min = 0 +param max = 7 +bytes 07 +expect value = 7 +consumed 3 +writer canonical + +operation int +name int-range-0-8-costs-four-bits +param min = 0 +param max = 8 +bytes 08 +expect value = 8 +consumed 4 +writer canonical + +# [0,8] is four bits, so offsets 9 through 15 are expressible and every one of +# them must be refused — reject, never clamp. + +operation int +name int-range-0-8-refuse-offset-one-past-the-span +param min = 0 +param max = 8 +bytes 09 +expect refused + +operation int +name int-range-0-8-refuse-offset-fifteen +param min = 0 +param max = 8 +bytes 0F +expect refused + +operation int +name int-range-0-1-single-bit-zero +param min = 0 +param max = 1 +bytes 00 +expect value = 0 +consumed 1 +writer canonical + +operation int +name int-range-0-1-single-bit-one +param min = 0 +param max = 1 +bytes 01 +expect value = 1 +consumed 1 +writer canonical + +operation int +name int-range-0-2-accepts-its-maximum +param min = 0 +param max = 2 +bytes 02 +expect value = 2 +consumed 2 +writer canonical + +operation int +name int-range-0-2-refuse-offset-three +param min = 0 +param max = 2 +bytes 03 +expect refused + +# A negative min: the offset is value - min, and the reader adds min back. The +# golden message's narrow int field. + +operation int +name int-negative-min-golden-field +param min = -100 +param max = 100 +bytes 3F +expect value = -37 +consumed 8 +writer canonical + +operation int +name int-negative-min-accepts-its-minimum +param min = -100 +param max = 100 +bytes 00 +expect value = -100 +consumed 8 +writer canonical + +operation int +name int-negative-min-accepts-its-maximum +param min = -100 +param max = 100 +bytes C8 +expect value = 100 +consumed 8 +writer canonical + +operation int +name int-negative-min-refuse-offset-one-past-the-span +param min = -100 +param max = 100 +bytes C9 +expect refused + +operation int +name int-negative-min-refuse-offset-all-ones +param min = -100 +param max = 100 +bytes FF +expect refused + +# The full int32 range is 32 bits, and every 32-bit offset is legal over it, so +# this declaration has no refusal vector — that is the rule's absence, not a +# gap. The span is 2^32 - 1, which a reader computing max - min in a signed +# 32-bit type overflows; the maximum vector is where that shows. + +operation int +name int-full-int32-range-golden-field +param min = -2147483648 +param max = 2147483647 +bytes EB 32 A4 78 +expect value = -123456789 +consumed 32 +writer canonical + +operation int +name int-full-int32-range-minimum +param min = -2147483648 +param max = 2147483647 +bytes 00 00 00 00 +expect value = -2147483648 +consumed 32 +writer canonical + +operation int +name int-full-int32-range-maximum +param min = -2147483648 +param max = 2147483647 +bytes FF FF FF FF +expect value = 2147483647 +consumed 32 +writer canonical + +# A 31-bit span whose maximum offset is one below the width's maximum: the +# widest declaration in this file that can express a refusal. + +operation int +name int-span-31-bits-accepts-its-maximum +param min = 0 +param max = 2147483646 +bytes FE FF FF 7F +expect value = 2147483646 +consumed 31 +writer canonical + +operation int +name int-span-31-bits-refuse-offset-one-past-the-span +param min = 0 +param max = 2147483646 +bytes FF FF FF 7F +expect refused + +# Reading past the end. [0,1000] is 10 bits and one byte cannot supply them. + +operation int +name int-refuse-past-end +param min = 0 +param max = 1000 +bytes FF +expect refused + +operation int +name int-accept-ten-bits-of-ten +param min = 0 +param max = 1000 +bytes E8 03 +expect value = 1000 +consumed 10 +writer canonical + +operation int +name int-refuse-past-end-empty-stream +param min = 0 +param max = 1 +bytes +expect refused + +# A degenerate range consumes nothing, so it succeeds on an empty stream where +# every other declaration fails: the same empty stream, two verdicts, decided +# by the declaration alone. + +operation int +name int-degenerate-range-accepts-the-empty-stream +param min = 7 +param max = 7 +bytes +expect value = 7 +consumed 0 +writer canonical diff --git a/conformance/int128.txt b/conformance/int128.txt index 16c92c5..3b647fd 100644 --- a/conformance/int128.txt +++ b/conformance/int128.txt @@ -3,6 +3,26 @@ # The format is specified in STANDARD.md, "The vector format". Records are # separated by blank lines. # +# STANDARD.md, "int128 (ranged)": bits_required128( min, max ) bits, with min +# and max converted to the UNSIGNED 128-bit domain first "so a range wider than +# 2^127 is exact rather than overflowing". The offset value - min is computed +# in that same unsigned domain and written in 32-bit groups from least +# significant upward — bits <= 32 is a single group, otherwise full 32-bit +# groups from the bottom with the final group carrying the remainder, up to +# four groups. +# +# Where the range fits 64 bits or fewer the bytes are identical to +# serialize_int64 over the same bounds, so the narrow vectors below repeat +# int64.txt's declarations deliberately: a field widened from 64 to 128 bits +# must not change the wire. +# +# Do not confuse this with serialize_uint128, which is not ranged and always +# costs a full 128 bits; that operation has its own file. +# +# The refusal rules are "Readers must check that the decoded offset is at most +# max - min in the unsigned domain and fail otherwise — reject, never clamp", +# and reading past the end. + # A degenerate range is legal and costs zero bits: STANDARD.md, "int (ranged)" # for the rule and "int128 (ranged)" for the width. The stream is empty, the # reader consumes nothing, and the value comes from min. A reader that requires @@ -15,3 +35,180 @@ param max = 1267650600228229401496703205383 bytes expect value = 1267650600228229401496703205383 consumed 0 +writer canonical + +operation int128 +name int128-degenerate-range-negative +param min = -1267650600228229401496703205376 +param max = -1267650600228229401496703205376 +bytes +expect value = -1267650600228229401496703205376 +consumed 0 +writer canonical + +# Narrow ranges: byte-identical to serialize_int64 over the same bounds. + +operation int128 +name int128-narrow-range-matches-int64 +param min = 0 +param max = 7 +bytes 05 +expect value = 5 +consumed 3 +writer canonical + +operation int128 +name int128-64-bit-span-matches-int64 +param min = -9223372036854775808 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 80 +expect value = 0 +consumed 64 +writer canonical + +# The library's additive pin, decoded here by the document alone: bounds of +# +/- 2^70 span 2^71 and need 72 bits, so this load-bears the THREE-group +# structure — 32 bits, 32 bits, then an 8-bit remainder on top. A decoder that +# assembles the groups in the wrong order, or puts the remainder anywhere but +# the most significant position, decodes the wrong value. + +operation int128 +name int128-72-bit-span-three-groups +param min = -1180591620717411303424 +param max = 1180591620717411303424 +bytes 11 32 54 76 98 BA DC FE 3F +expect value = -0x0123456789ABCDEF +consumed 72 +writer canonical + +operation int128 +name int128-72-bit-span-accepts-its-minimum +param min = -1180591620717411303424 +param max = 1180591620717411303424 +bytes 00 00 00 00 00 00 00 00 00 +expect value = -1180591620717411303424 +consumed 72 +writer canonical + +operation int128 +name int128-72-bit-span-accepts-offset-equal-to-the-span +param min = -1180591620717411303424 +param max = 1180591620717411303424 +bytes 00 00 00 00 00 00 00 00 80 +expect value = 1180591620717411303424 +consumed 72 +writer canonical + +operation int128 +name int128-72-bit-span-refuse-offset-one-past-the-span +param min = -1180591620717411303424 +param max = 1180591620717411303424 +bytes 01 00 00 00 00 00 00 00 80 +expect refused + +operation int128 +name int128-72-bit-span-refuse-offset-all-ones +param min = -1180591620717411303424 +param max = 1180591620717411303424 +bytes FF FF FF FF FF FF FF FF FF +expect refused + +# The full 128-bit range: the span is 2^128 - 1, which is exactly what "min and +# max are converted to the unsigned 128-bit domain first" buys. Four groups, +# and every 128-bit offset is legal, so this declaration has no refusal vector. + +operation int128 +name int128-full-range-zero +param min = -170141183460469231731687303715884105728 +param max = 170141183460469231731687303715884105727 +bytes 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 80 +expect value = 0 +consumed 128 +writer canonical + +operation int128 +name int128-full-range-minimum +param min = -170141183460469231731687303715884105728 +param max = 170141183460469231731687303715884105727 +bytes 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +expect value = -170141183460469231731687303715884105728 +consumed 128 +writer canonical + +operation int128 +name int128-full-range-maximum +param min = -170141183460469231731687303715884105728 +param max = 170141183460469231731687303715884105727 +bytes FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF +expect value = 170141183460469231731687303715884105727 +consumed 128 +writer canonical + +# The widest declaration that can express a refusal: the span is 2^128 - 2, one +# below the width's maximum. An implementation that computes the span in the +# SIGNED 128-bit domain overflows here and cannot decide either vector. + +operation int128 +name int128-128-bit-span-accepts-its-maximum +param min = -170141183460469231731687303715884105728 +param max = 170141183460469231731687303715884105726 +bytes FE FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF +expect value = 170141183460469231731687303715884105726 +consumed 128 +writer canonical + +operation int128 +name int128-128-bit-span-refuse-offset-one-past-the-span +param min = -170141183460469231731687303715884105728 +param max = 170141183460469231731687303715884105726 +bytes FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF +expect refused + +# The group boundaries, one vector per group, over a zero minimum so the decoded +# value is the offset itself. The bounds are SIGNED, so the widest declaration +# with a zero minimum is [ 0, 2^127 - 1 ], a 127-bit span across four groups — +# three full ones and a 31-bit remainder — and each vector puts a single set bit +# in one group to prove that group's position in the stream. + +operation int128 +name int128-group-boundary-bit-32 +param min = 0 +param max = 170141183460469231731687303715884105727 +bytes 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 +expect value = 4294967296 +consumed 127 +writer canonical + +operation int128 +name int128-group-boundary-bit-64 +param min = 0 +param max = 170141183460469231731687303715884105727 +bytes 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 +expect value = 18446744073709551616 +consumed 127 +writer canonical + +operation int128 +name int128-group-boundary-bit-96 +param min = 0 +param max = 170141183460469231731687303715884105727 +bytes 00 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 +expect value = 79228162514264337593543950336 +consumed 127 +writer canonical + +# Reading past the end. A 101-bit field cannot be read from eight bytes. + +operation int128 +name int128-refuse-past-end +param min = 0 +param max = 1267650600228229401496703205376 +bytes FF FF FF FF FF FF FF FF +expect refused + +operation int128 +name int128-refuse-past-end-empty-stream +param min = 0 +param max = 1 +bytes +expect refused diff --git a/conformance/int64.txt b/conformance/int64.txt new file mode 100644 index 0000000..2271454 --- /dev/null +++ b/conformance/int64.txt @@ -0,0 +1,223 @@ +# serialize conformance vectors: serialize_int64 +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "int64 (ranged)": the 64-bit counterpart of the ranged integer +# and the only ranged 64-bit operation. bits_required64( min, max ) bits are +# used; "if that is 32 or fewer, the value is written as a single group of that +# many bits; otherwise the low 32 bits are written first, followed by the +# remaining bits - 32 high bits." +# +# Do not confuse this with serialize_uint64, which is not ranged and always +# costs a full 64 bits; that operation is covered in bits.txt, whose aliases it +# is one of. The names are similar and the encodings are not. +# +# The refusal rules are the ranged integer's: a decoded value outside [min,max] +# — in offset form an offset greater than max - min — and reading past the end. +# The span is computed in the unsigned 64-bit domain, so the widest +# declarations here are where an implementation that subtracts in a signed type +# diverges. + +operation int64 +name int64-degenerate-range-zero-bits +param min = 1234567890123456789 +param max = 1234567890123456789 +bytes +expect value = 1234567890123456789 +consumed 0 +writer canonical + +operation int64 +name int64-degenerate-range-at-int64-min +param min = -9223372036854775808 +param max = -9223372036854775808 +bytes +expect value = -9223372036854775808 +consumed 0 +writer canonical + +# A narrow range produces the same bytes as serialize_int over the same bounds. + +operation int64 +name int64-narrow-range-matches-int +param min = 0 +param max = 7 +bytes 05 +expect value = 5 +consumed 3 +writer canonical + +operation int64 +name int64-single-bit-range +param min = -1 +param max = 0 +bytes 01 +expect value = 0 +consumed 1 +writer canonical + +# Exactly 32 bits: the last width written as a single group. + +operation int64 +name int64-span-exactly-32-bits-single-group +param min = 0 +param max = 4294967295 +bytes EF BE AD DE +expect value = 3735928559 +consumed 32 +writer canonical + +operation int64 +name int64-span-exactly-32-bits-accepts-its-maximum +param min = 0 +param max = 4294967295 +bytes FF FF FF FF +expect value = 4294967295 +consumed 32 +writer canonical + +# One wider: 33 bits, the first width that splits into a low 32-bit group and a +# high remainder. The set bit lands in the high group, so an implementation +# that writes the high group first puts it in bit 0 instead of bit 32. + +operation int64 +name int64-span-33-bits-low-group-precedes-high-group +param min = 0 +param max = 4294967296 +bytes 00 00 00 00 01 +expect value = 4294967296 +consumed 33 +writer canonical + +# The vector above is also this declaration's accept boundary: the span is +# 2^32, so offset 2^32 is the largest legal offset and the next one is refused. + +operation int64 +name int64-span-33-bits-refuse-offset-one-past-the-span +param min = 0 +param max = 4294967296 +bytes 01 00 00 00 01 +expect refused + +operation int64 +name int64-span-33-bits-accepts-its-full-width +param min = 0 +param max = 8589934591 +bytes FF FF FF FF 01 +expect value = 8589934591 +consumed 33 +writer canonical + +# A 42-bit span with negative bounds: 32 bits in the low group and 10 in the +# high one, and the offset must be added to a negative raw minimum. + +operation int64 +name int64-span-42-bits-negative-bounds +param min = -1099511627776 +param max = 1099511627776 +bytes 00 00 00 00 00 01 +expect value = 0 +consumed 42 +writer canonical + +operation int64 +name int64-span-42-bits-accepts-its-minimum +param min = -1099511627776 +param max = 1099511627776 +bytes 00 00 00 00 00 00 +expect value = -1099511627776 +consumed 42 +writer canonical + +operation int64 +name int64-span-42-bits-accepts-its-maximum +param min = -1099511627776 +param max = 1099511627776 +bytes 00 00 00 00 00 02 +expect value = 1099511627776 +consumed 42 +writer canonical + +operation int64 +name int64-span-42-bits-refuse-offset-one-past-the-span +param min = -1099511627776 +param max = 1099511627776 +bytes 01 00 00 00 00 02 +expect refused + +# The full int64 range: the span is 2^64 - 1, so the width is 64 and every +# 64-bit offset is legal. No refusal vector exists for this declaration, which +# is the rule's absence rather than a gap. + +operation int64 +name int64-full-range-zero +param min = -9223372036854775808 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 80 +expect value = 0 +consumed 64 +writer canonical + +operation int64 +name int64-full-range-minimum +param min = -9223372036854775808 +param max = 9223372036854775807 +bytes 00 00 00 00 00 00 00 00 +expect value = -9223372036854775808 +consumed 64 +writer canonical + +operation int64 +name int64-full-range-maximum +param min = -9223372036854775808 +param max = 9223372036854775807 +bytes FF FF FF FF FF FF FF FF +expect value = 9223372036854775807 +consumed 64 +writer canonical + +# The widest declaration that can express a refusal: the span is 2^64 - 2, one +# below the width's maximum. An implementation computing max - min in a signed +# 64-bit type overflows here and cannot decide either vector correctly. + +operation int64 +name int64-span-64-bits-accepts-its-maximum +param min = -9223372036854775808 +param max = 9223372036854775806 +bytes FE FF FF FF FF FF FF FF +expect value = 9223372036854775806 +consumed 64 +writer canonical + +operation int64 +name int64-span-64-bits-refuse-offset-one-past-the-span +param min = -9223372036854775808 +param max = 9223372036854775806 +bytes FF FF FF FF FF FF FF FF +expect refused + +# Reading past the end. + +operation int64 +name int64-refuse-past-end-41-of-32 +param min = 0 +param max = 1099511627776 +bytes FF FF FF FF +expect refused + +operation int64 +name int64-accept-41-bits-of-41 +param min = 0 +param max = 1099511627776 +bytes 00 00 00 00 00 01 +expect value = 1099511627776 +consumed 41 +writer canonical + +operation int64 +name int64-refuse-past-end-empty-stream +param min = 0 +param max = 1 +bytes +expect refused diff --git a/conformance/int_relative.txt b/conformance/int_relative.txt index 8b70cac..f353bd0 100644 --- a/conformance/int_relative.txt +++ b/conformance/int_relative.txt @@ -4,11 +4,62 @@ # separated by blank lines. `consumed` appears on accepted reads only, because # after a refusal the stream position is not part of the contract. # -# Every refusal here is the domain rule: STANDARD.md, "int_relative", where the -# domain is 0 to 2^31 - 1 inclusive and reconstruction is checked in every -# tier. Each refusal has an accept twin one step inside the domain, sharing its -# bytes, so a reader that refuses on the bytes rather than on the reconstructed -# value fails the twin. +# STANDARD.md, "int_relative": a ladder of one-bit flags, each answering "does +# it fit in this tier?", with the tier names this file uses taken from the +# document's own table: +# +# one-bit 1 — exactly 1 +# bounded-3 0 1 serialize_int( d, 2, 6 ) 2 – 6 +# bounded-5 0 0 1 serialize_int( d, 7, 23 ) 7 – 23 +# bounded-9 0 0 0 1 serialize_int( d, 24, 280 ) 24 – 280 +# bounded-13 0 0 0 0 1 serialize_int( d, 281, 4377 ) 281 – 4377 +# bounded-17 0 0 0 0 0 1 serialize_int( d, 4378, 69914 ) 4378 – 69914 +# absolute 0 0 0 0 0 0 current as 32 raw bits anything +# +# Three refusal rules reach this operation, and this file carries all three in +# every tier they can occur in. +# +# 1. The DOMAIN and the ordering. "Every tier's reconstruction must be +# checked": the reader reconstructs current in a width that cannot wrap and +# must refuse unless current lies in 0 .. 2^31 - 1 and is strictly greater +# than previous. That binds in the one-bit tier, in each of the five +# bounded tiers, and in the absolute tier. Each refusal below has an accept +# twin one step inside the domain, SHARING ITS BYTES, so a reader that +# refuses on the bytes rather than on the reconstructed value fails the twin. +# +# 2. The BOUNDED PAYLOAD's own range. Each bounded tier's payload is a ranged +# integer whose span is not one less than a power of two, so an offset past +# the span is expressible in the tier's width and "readers must check that +# the decoded value lies within [min,max] and fail otherwise" applies inside +# the ladder exactly as it does anywhere else. Every bounded tier carries +# that refusal below, at one past the span and at the width's maximum, each +# beside the accepted vector that pins the span's top. +# +# 3. Reading PAST THE END, in the flags and in each tier's payload. +# +# A writer emits the FIRST tier a difference fits, so the absolute-tier vectors +# whose difference is small are reader-only: no conforming writer produces +# them, and marking them `writer = canonical` would demand wire the document +# forbids. The tier-first-fit vectors carry the mark instead. + +# --------------------------------------------------------------------------- +# the one-bit tier + +# An empty stream cannot supply even the ladder's first flag. + +operation int_relative +name ladder-refuse-past-end-empty-stream +param previous = 100 +bytes +expect refused + +operation int_relative +name one-bit-accept +param previous = 100 +bytes 01 +expect value = 101 +consumed 1 +writer canonical operation int_relative name one-bit-refuse-past-domain @@ -22,6 +73,39 @@ param previous = 2147483646 bytes 01 expect value = 2147483647 consumed 1 +writer canonical + +# --------------------------------------------------------------------------- +# the bounded-3 tier: payload serialize_int( d, 2, 6 ), span 4 in 3 bits, so +# offsets 5 through 7 are expressible and must be refused + +operation int_relative +name bounded-3-accept-low +param previous = 100 +bytes 02 +expect value = 102 +consumed 5 +writer canonical + +operation int_relative +name bounded-3-accept-high +param previous = 100 +bytes 12 +expect value = 106 +consumed 5 +writer canonical + +operation int_relative +name bounded-3-refuse-payload-one-past-the-span +param previous = 100 +bytes 16 +expect refused + +operation int_relative +name bounded-3-refuse-payload-all-three-bits +param previous = 100 +bytes 1E +expect refused operation int_relative name bounded-3-refuse-past-domain @@ -35,6 +119,39 @@ param previous = 2147483645 bytes 02 expect value = 2147483647 consumed 5 +writer canonical + +# --------------------------------------------------------------------------- +# the bounded-5 tier: payload serialize_int( d, 7, 23 ), span 16 in 5 bits, so +# offsets 17 through 31 are expressible and must be refused + +operation int_relative +name bounded-5-accept-low +param previous = 100 +bytes 04 +expect value = 107 +consumed 8 +writer canonical + +operation int_relative +name bounded-5-accept-high +param previous = 100 +bytes 84 +expect value = 123 +consumed 8 +writer canonical + +operation int_relative +name bounded-5-refuse-payload-one-past-the-span +param previous = 100 +bytes 8C +expect refused + +operation int_relative +name bounded-5-refuse-payload-all-five-bits +param previous = 100 +bytes FC +expect refused operation int_relative name bounded-5-refuse-past-domain @@ -48,6 +165,39 @@ param previous = 2147483640 bytes 04 expect value = 2147483647 consumed 8 +writer canonical + +# --------------------------------------------------------------------------- +# the bounded-9 tier: payload serialize_int( d, 24, 280 ), span 256 in 9 bits, +# so offsets 257 through 511 are expressible and must be refused + +operation int_relative +name bounded-9-accept-low +param previous = 100 +bytes 08 00 +expect value = 124 +consumed 13 +writer canonical + +operation int_relative +name bounded-9-accept-high +param previous = 100 +bytes 08 10 +expect value = 380 +consumed 13 +writer canonical + +operation int_relative +name bounded-9-refuse-payload-one-past-the-span +param previous = 100 +bytes 18 10 +expect refused + +operation int_relative +name bounded-9-refuse-payload-all-nine-bits +param previous = 100 +bytes F8 1F +expect refused operation int_relative name bounded-9-refuse-past-domain @@ -61,6 +211,45 @@ param previous = 2147483623 bytes 08 00 expect value = 2147483647 consumed 13 +writer canonical + +operation int_relative +name bounded-9-refuse-past-end-payload-truncated +param previous = 100 +bytes 08 +expect refused + +# --------------------------------------------------------------------------- +# the bounded-13 tier: payload serialize_int( d, 281, 4377 ), span 4096 in 13 +# bits, so offsets 4097 through 8191 are expressible and must be refused + +operation int_relative +name bounded-13-accept-low +param previous = 100 +bytes 10 00 00 +expect value = 381 +consumed 18 +writer canonical + +operation int_relative +name bounded-13-accept-high +param previous = 100 +bytes 10 00 02 +expect value = 4477 +consumed 18 +writer canonical + +operation int_relative +name bounded-13-refuse-payload-one-past-the-span +param previous = 100 +bytes 30 00 02 +expect refused + +operation int_relative +name bounded-13-refuse-payload-all-thirteen-bits +param previous = 100 +bytes F0 FF 03 +expect refused operation int_relative name bounded-13-refuse-past-domain @@ -74,6 +263,48 @@ param previous = 2147483366 bytes 10 00 00 expect value = 2147483647 consumed 18 +writer canonical + +operation int_relative +name bounded-13-refuse-past-end-payload-truncated +param previous = 100 +bytes 10 00 +expect refused + +# --------------------------------------------------------------------------- +# the bounded-17 tier: payload serialize_int( d, 4378, 69914 ), span 65536 in +# 17 bits, so offsets 65537 through 131071 are expressible and must be refused. +# This is the tier a decoder is most likely to be missing outright, in which +# case its flag is read as the start of the absolute form and everything after +# it shifts. + +operation int_relative +name bounded-17-accept-low +param previous = 100 +bytes 20 00 00 +expect value = 4478 +consumed 23 +writer canonical + +operation int_relative +name bounded-17-accept-high +param previous = 100 +bytes 20 00 40 +expect value = 70014 +consumed 23 +writer canonical + +operation int_relative +name bounded-17-refuse-payload-one-past-the-span +param previous = 100 +bytes 60 00 40 +expect refused + +operation int_relative +name bounded-17-refuse-payload-all-seventeen-bits +param previous = 100 +bytes E0 FF 7F +expect refused operation int_relative name bounded-17-refuse-past-domain @@ -87,6 +318,39 @@ param previous = 2147479269 bytes 20 00 00 expect value = 2147483647 consumed 23 +writer canonical + +operation int_relative +name bounded-17-refuse-past-end-payload-truncated +param previous = 100 +bytes 20 00 +expect refused + +# --------------------------------------------------------------------------- +# the absolute tier. "The final tier transmits current, not the difference." +# Its 32 raw bits are UNSIGNED, so a value with the top bit set is outside the +# domain and must be refused; a reader that takes the group into a signed +# sequence type first has already left the domain, and the two readings +# disagree about the same byte sequence. +# +# A writer reaches this tier only when the difference exceeds 69914, so the +# small-difference vectors here are reader-only. + +operation int_relative +name absolute-accept-domain-maximum +param previous = 100 +bytes C0 FF FF FF 1F +expect value = 2147483647 +consumed 38 +writer canonical + +operation int_relative +name absolute-accept-first-difference-past-the-last-tier +param previous = 100 +bytes C0 5F 44 00 00 +expect value = 70015 +consumed 38 +writer canonical operation int_relative name absolute-refuse-not-increasing @@ -101,6 +365,18 @@ bytes C0 FF FF FF 1F expect value = 2147483647 consumed 38 +operation int_relative +name absolute-refuse-current-equal-to-previous +param previous = 100 +bytes 00 19 00 00 00 +expect refused + +operation int_relative +name absolute-refuse-current-below-previous +param previous = 100 +bytes 80 0C 00 00 00 +expect refused + operation int_relative name absolute-refuse-top-bit-set param previous = 100 @@ -114,8 +390,8 @@ bytes C0 FF FF FF 3F expect refused operation int_relative -name absolute-accept-domain-maximum +name absolute-refuse-past-end-payload-truncated param previous = 100 -bytes C0 FF FF FF 1F -expect value = 2147483647 -consumed 38 +bytes 00 00 00 00 +expect refused + diff --git a/conformance/message.txt b/conformance/message.txt new file mode 100644 index 0000000..d5f8596 --- /dev/null +++ b/conformance/message.txt @@ -0,0 +1,48 @@ +# serialize conformance vectors: the message STANDARD.md works through +# +# The format is specified in STANDARD.md, "The vector format". Records use the +# sequence spelling documented at the head of conformance/sequence.txt. +# +# STANDARD.md's "Worked Example" decodes a run of the library's golden message +# field by field, and states the bytes it decodes: the 13-byte wide-string run +# +# 0xE3 0x21 0x00 0x00 0xC0 0x21 0x00 0x00 0x00 0x22 0x00 0x00 0x00 +# +# then "The message then aligns and continues with four fixed point fields. The +# first is serialize_fixed( value, 8, 8, -100, +100 ) — Q8.8, so raw_min is +# -100 << 8 = -25600, the raw range is 51200, and the field costs 16 bits. The +# next two bytes of the golden vector are 0xC0 0x60, which is the offset +# 0x60C0 = 24768; adding raw_min gives a raw value of -832, which is -3.25 in +# Q8.8 — exactly the value the golden message stores." +# +# That is a real multi-operation message whose every byte the document itself +# supplies, and this file is it: a wstring, the align that follows it, and the +# fixed point field the align makes room for. It is the cheapest way to confirm +# an independent implementation is correct, and it is the one place in this +# corpus where three different operations meet across an alignment boundary +# with a byte-for-byte expectation the page prints in full. +# +# It is a FRAGMENT of the 112-byte golden and not the whole of it, deliberately. +# The remaining fields' bytes appear nowhere in STANDARD.md; they exist only as +# an array inside serialize.h, and lifting them here would put an +# implementation's output into the corpus, which is the one thing the corpus +# must never contain — "no checker reimplements the codec and then checks that +# reimplementation against itself", and a vector copied off the implementation +# it is meant to judge is the same failure wearing a different coat. Shipping +# the full golden needs the document to print the rest of its bytes; that is +# filed as an underdetermination rather than worked around here. +# +# The wide string ends at bit 99, three bits into byte 12, so the align pads +# the remaining five bits of that byte — which is why the document's 13-byte +# run ends in 0x00 and the fixed point field begins on byte 13. + +operation sequence +name message-worked-example-wstring-align-fixed +param step = wstring 8 +param step = align +param step = fixed 8 8 -100 100 +bytes E3 21 00 00 C0 21 00 00 00 22 00 00 00 C0 60 +expect value = 043C 0438 0440 | - | -832 +consumed 120 +measure_at_least 122 +writer canonical diff --git a/conformance/object.txt b/conformance/object.txt new file mode 100644 index 0000000..348c4aa --- /dev/null +++ b/conformance/object.txt @@ -0,0 +1,114 @@ +# serialize conformance vectors: serialize_object +# +# The format is specified in STANDARD.md, "The vector format". Records use the +# sequence spelling documented at the head of conformance/sequence.txt, with one +# addition: a step spelled `object ` wraps the NEXT n steps in a nested +# object, and states `-` in the expect list because it produces no value of its +# own. +# +# STANDARD.md, "object": serialize_object "invokes the object's own serialize +# function inline. It contributes NO BYTES OF ITS OWN — it is composition, not +# an encoding. Whatever the nested object writes appears at exactly this +# position in the stream, with no framing, length prefix, or alignment inserted +# around it." +# +# That rule is a statement about two streams being the same, so every vector +# here has a twin in sequence.txt or below it carrying the same operations +# unnested: same bytes, same consumption, same values. An implementation that +# inserts a length, a tag or an alignment around a nested object produces +# different bytes for the nested vector than for its flat twin, and one of the +# pair goes red. +# +# The operation has no refusal rule of its own — it consumes nothing and decides +# nothing — so the refusals below are the nested operations' own, and what they +# pin is that a refusal inside a nested object propagates out of it rather than +# being swallowed by the composition. + +# The flat twin, for comparison: four bits, a bool, then a byte. + +operation sequence +name object-flat-twin +param step = bits 4 +param step = bool +param step = bits 8 +bytes 5A 0D +expect value = 10 | true | 106 +consumed 13 +writer canonical + +# The same three operations, all three inside one nested object. Identical bytes +# and identical consumption: the object added nothing. + +operation sequence +name object-adds-no-bytes-of-its-own +param step = object 3 +param step = bits 4 +param step = bool +param step = bits 8 +bytes 5A 0D +expect value = - | 10 | true | 106 +consumed 13 +writer canonical + +# The nested object in the middle of a stream, with operations before and after +# it, so a framing byte would show up between them rather than at an end. + +operation sequence +name object-nested-in-the-middle-of-a-stream +param step = bits 4 +param step = object 1 +param step = bool +param step = bits 8 +bytes 5A 0D +expect value = 10 | - | true | 106 +consumed 13 +writer canonical + +# No alignment is inserted around a nested object. The object here holds a +# single 3-bit field and is followed by a byte-aligned operation; if the +# composition aligned on entry or exit, the bytes would move. + +operation sequence +name object-inserts-no-alignment +param step = object 1 +param step = bits 3 +param step = bits 8 +bytes 05 00 +expect value = - | 5 | 0 +consumed 11 +writer canonical + +# An object containing an operation that itself aligns: the align belongs to the +# nested operation and lands where the nested operation puts it, not at the +# object boundary. + +operation sequence +name object-containing-an-aligning-operation +param step = bits 3 +param step = object 1 +param step = bytes 1 +bytes 05 AB +expect value = 5 | - | AB +consumed 16 +measure_at_least 18 +writer canonical + +# A refusal inside a nested object propagates out of it. The nested align finds +# non-zero padding, and the read must fail — the composition must not swallow +# the refusal, and the operation after the object must refuse too. + +operation sequence +name object-refusal-propagates-out-of-the-nesting +param step = bits 3 +param step = object 1 +param step = align +param step = bits 8 +bytes FD AB +expect refused + +operation sequence +name object-refusal-past-the-end-propagates +param step = object 1 +param step = bits 16 +bytes FF +expect refused diff --git a/conformance/sequence.txt b/conformance/sequence.txt new file mode 100644 index 0000000..33d7b0d --- /dev/null +++ b/conformance/sequence.txt @@ -0,0 +1,199 @@ +# serialize conformance vectors: sequences of operations +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# Every other file in this corpus tests one operation in isolation, from bit +# index zero. That misses everything the format says about operations meeting +# each other: alignment cost depends on the bit index the previous operation +# left behind, a zero-bit field must leave the index exactly where it found it, +# and a refusal must poison the stream for everything after it. Those are +# rules STANDARD.md states and a single-operation record cannot express, so +# they live here. +# +# A sequence record's steps are given as repeated `param step` lines, one +# operation each, in order. The step spellings are: +# +# bits bool +# align float +# bytes int +# string wstring +# fixed +# +# `expect value` lists one entry per step in order, separated by ` | `, with +# `-` for a step that produces no value of its own. `consumed` is the total +# for the whole sequence. +# +# `expect refused` means the sequence must be refused, and it carries the +# TERMINAL FAILURE rule with it: STANDARD.md says "nothing after a failing +# operation has a defined position, so nothing after it is interpretable, and +# it must be the stream that enforces that rather than the caller's +# discipline", so a runner must check not only that the failing step refuses +# but that EVERY LATER STEP REFUSES TOO — even where the stream still holds +# bytes those steps could have read. The refusing sequences below are built so +# that a reader without the latch passes the failing step's successor. +# +# `measure_at_least` states the smallest number of bits a conforming measure +# may report for the sequence, and a measure must report at least it. It is a +# FLOOR and not an equality, because STANDARD.md makes a measure a bound and +# not the packet size: "a measure must report a size sufficient to serialize +# the message at any starting bit position. It need not be exact, and cannot +# be." Each floor below is the true worst case over every starting bit +# position — exact width for every operation, plus the most an alignment can +# cost where the sequence contains one — so an implementation charging the +# document's expected 7 bits per alignment-performing operation passes, and so +# does a tighter correct bound, while the non-conforming exact-from-zero +# accounting the document names fails. + +# The document's own measure example, verbatim: "{ bits(8); align; bits(8) } is +# 16 bits — 2 bytes — written from an aligned start, where the align is a +# no-op; written from bit offset 1, the align pads 7 bits and the message spans +# 23 bits — 3 bytes of room needed. The exact-from-zero answer of 2 bytes is +# not sufficient at every starting position, which is the one thing a measure +# is for. The conservative answer — 8 + 7 + 8 = 23 bits — is sufficient +# everywhere." A measure reporting 16 fails this record and a measure +# reporting 23 passes it. + +operation sequence +name sequence-measure-worked-example +param step = bits 8 +param step = align +param step = bits 8 +bytes AB CD +expect value = 171 | - | 205 +consumed 16 +measure_at_least 23 +writer canonical + +# The align actually pads here, because the preceding operation left the index +# at bit 3. Read consumption and measure floor differ, which is the point. + +operation sequence +name sequence-align-pads-after-an-odd-width +param step = bits 3 +param step = align +param step = bits 8 +bytes 05 AB +expect value = 5 | - | 171 +consumed 16 +measure_at_least 18 +writer canonical + +# The same stream with the padding set. The align refuses, and the bits(8) +# after it must refuse as well although byte 1 is present and readable — that +# is the terminal-failure rule, and a reader whose failure does not latch +# decodes 171 from the third step. + +operation sequence +name sequence-refusal-is-terminal-after-bad-alignment +param step = bits 3 +param step = align +param step = bits 8 +bytes FD AB +expect refused + +# A second align costs nothing, because the first one left the stream aligned. + +operation sequence +name sequence-second-align-is-a-no-op +param step = bits 1 +param step = align +param step = align +param step = bits 8 +bytes 01 AB +expect value = 1 | - | - | 171 +consumed 16 +measure_at_least 16 +writer canonical + +# serialize_bytes aligns first, so it realigns a stream left mid-byte — and it +# is that align, not the payload, that the measure floor charges for. + +operation sequence +name sequence-bytes-realigns-a-mid-byte-stream +param step = bits 5 +param step = bytes 2 +bytes 15 AB CD +expect value = 21 | AB CD +consumed 24 +measure_at_least 28 +writer canonical + +# A degenerate range costs zero bits, so the second wide field must begin at +# bit 32 exactly. An implementation that charges any bits at all for the +# degenerate int shifts every bit after it and decodes the wrong second value. + +operation sequence +name sequence-zero-bit-field-between-wide-ones +param step = bits 32 +param step = int 5 5 +param step = bits 32 +bytes EF BE AD DE 78 56 34 12 +expect value = 3735928559 | 5 | 305419896 +consumed 64 +measure_at_least 64 +writer canonical + +# The narrow string aligns and the wide string does not, in one stream: the +# string ends byte-aligned at bit 32, and the wstring's three-bit length field +# then puts its single character at bit 35. An implementation that mirrors the +# narrow path in the wide one moves every byte from index 4 onward. + +operation sequence +name sequence-string-aligns-and-wstring-does-not +param step = string 16 +param step = wstring 8 +bytes 03 61 62 63 09 02 00 00 00 +expect value = 61 62 63 | 0041 +consumed 67 +measure_at_least 70 +writer canonical + +# float is bit-transparent at any bit index, and nothing aligns around it: the +# bool before it leaves the stream at bit 1 and the 32-bit group runs from +# there to bit 33. + +operation sequence +name sequence-float-unaligned-between-bools +param step = bool +param step = float +param step = bool +bytes 01 00 00 7F 02 +expect value = true | 0x3F800000 | true +consumed 34 +measure_at_least 34 +writer canonical + +# Terminal failure after a RANGE refusal rather than an alignment one. The +# ranged int over [0,8] is four bits and the stream carries offset 9, which +# must be refused; the bits(8) after it must refuse too, although twelve +# readable bits remain. + +operation sequence +name sequence-refusal-is-terminal-after-a-range-refusal +param step = int 0 8 +param step = bits 8 +bytes 09 AB +expect refused + +# Terminal failure after a PAST-END refusal, with a zero-bit operation as the +# successor: a degenerate range consumes nothing and would otherwise always +# succeed, so this is the vector a reader that treats "consumes no bits" as +# "cannot fail" gets wrong. + +operation sequence +name sequence-refusal-is-terminal-before-a-zero-bit-field +param step = bits 16 +param step = int 7 7 +bytes FF +expect refused + +# Terminal failure inside a composite: the string's payload runs past the end, +# and the wstring after it must refuse rather than resynchronize. + +operation sequence +name sequence-refusal-is-terminal-after-a-string-refusal +param step = string 16 +param step = wstring 8 +bytes 06 67 6F 6C +expect refused diff --git a/conformance/string.txt b/conformance/string.txt new file mode 100644 index 0000000..50e6461 --- /dev/null +++ b/conformance/string.txt @@ -0,0 +1,340 @@ +# serialize conformance vectors: serialize_string +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "string": a null-terminated narrow string, written as +# +# 1. the length, as serialize_int( length, 0, buffer_size - 1 ) — so the bit +# cost depends on buffer_size, which both sides must agree on; +# 2. the characters, as serialize_bytes — WHICH ALIGNS. +# +# The terminator is not transmitted; the reader appends it. Because +# buffer_size is an operand rather than a transmitted value, the same string +# serialized against different buffer sizes produces different bytes, and the +# vectors below pin three buffer sizes for that reason. +# +# `value` is the transmitted payload as hexadecimal byte pairs, empty for the +# empty string. Spelling it as bytes rather than as text keeps this file pure +# ASCII and keeps the expectation free of any question about how each of the +# nine implementations quotes a string. +# +# Four refusal rules reach this operation: +# +# * the length field's own range rule, when buffer_size - 1 is not one less +# than a power of two, so a length past the buffer is expressible; +# * the alignment padding must be zero, inherited from serialize_bytes; +# * INVALID UTF-8 fails the read — "the payload the contract above promises +# is the payload the reader insists on"; +# * an INTERIOR NUL fails the read — a zero byte anywhere among the length +# transmitted bytes, which "gives the payload two lengths" and is its own +# rule because NUL is well-formed UTF-8; +# +# and reading past the end. Every UTF-8 refusal below is paired with the +# nearest well-formed accept, so a reader that refuses a whole byte class +# rather than the malformed sequence fails the twin. +# +# STANDARD.md leaves the buffer's contents UNSPECIFIED after a refused read +# into a caller-owned buffer, so no vector here states anything about it. + +# buffer_size 16: the length field is bits_required( 0, 15 ) = 4 bits, then +# four bits of alignment padding, so the payload starts at byte 1. + +operation string +name string-golden +param buffer_size = 16 +bytes 06 67 6F 6C 64 65 6E +expect value = 67 6F 6C 64 65 6E +consumed 56 +writer canonical + +# The empty string reaches serialize_bytes with count zero, which "pads to the +# byte boundary and writes nothing else". The align is performed regardless: an +# implementation that skips it when the payload is empty consumes 4 bits here +# instead of 8 and desynchronizes everything after it. + +operation string +name string-empty-still-performs-the-align +param buffer_size = 16 +bytes 00 +expect value = +consumed 8 +writer canonical + +operation string +name string-single-byte +param buffer_size = 16 +bytes 01 41 +expect value = 41 +consumed 16 +writer canonical + +operation string +name string-fills-the-buffer +param buffer_size = 16 +bytes 0F 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F +expect value = 41 42 43 44 45 46 47 48 49 4A 4B 4C 4D 4E 4F +consumed 128 +writer canonical + +# buffer_size 8: a three-bit length field, so the same payload produces +# different bytes than it does above. The alignment absorbs five bits here +# instead of four, and the payload still starts at byte 1. + +operation string +name string-buffer-size-8-three-bit-length +param buffer_size = 8 +bytes 04 74 65 73 74 +expect value = 74 65 73 74 +consumed 40 +writer canonical + +# buffer_size 1: the length field is serialize_int( length, 0, 0 ), a +# DEGENERATE RANGE, so it costs zero bits and the only representable string is +# the empty one. The align at bit index 0 is a no-op, the payload is empty, and +# the whole operation consumes nothing at all — from an empty stream, where any +# read is an error. + +operation string +name string-buffer-size-1-zero-bit-length-field +param buffer_size = 1 +bytes +expect value = +consumed 0 +writer canonical + +# The length field's range rule: buffer_size 10 gives bits_required( 0, 9 ) = 4 +# bits, so lengths 10 through 15 are expressible and must be refused. + +operation string +name string-accepts-the-longest-length +param buffer_size = 10 +bytes 09 31 32 33 34 35 36 37 38 39 +expect value = 31 32 33 34 35 36 37 38 39 +consumed 80 +writer canonical + +operation string +name string-refuse-length-one-past-the-buffer +param buffer_size = 10 +bytes 0A +expect refused + +operation string +name string-refuse-length-all-four-bits +param buffer_size = 10 +bytes 0F +expect refused + +# The alignment padding must be zero. The low four bits are a length of 1, the +# high four are the padding, and only the padding differs between this vector +# and string-single-byte above. + +operation string +name string-refuse-non-zero-align-padding +param buffer_size = 16 +bytes F1 41 +expect refused + +# Reading past the end, in the length field and in the payload. + +operation string +name string-refuse-past-end-payload-truncated +param buffer_size = 16 +bytes 06 67 6F 6C +expect refused + +operation string +name string-refuse-past-end-empty-stream +param buffer_size = 16 +bytes +expect refused + +# --------------------------------------------------------------------------- +# the interior NUL rule. A conforming writer derives length from strlen, so no +# zero byte can reach the wire from conformance; a stream carrying one gives +# the payload two lengths — the wire length and the strlen every consumer +# downstream computes — and everything between them rides invisibly past +# whichever side uses the other. The rule is "a zero byte ANYWHERE among the +# length transmitted bytes", so the first, a middle and the last byte each get +# a vector: a reader that only scans strictly-interior positions accepts two of +# these three. + +operation string +name string-refuse-nul-as-the-only-byte +param buffer_size = 16 +bytes 01 00 +expect refused + +operation string +name string-refuse-nul-as-the-first-byte +param buffer_size = 16 +bytes 03 00 61 62 +expect refused + +operation string +name string-refuse-nul-in-the-middle +param buffer_size = 16 +bytes 03 61 00 62 +expect refused + +operation string +name string-refuse-nul-as-the-last-byte +param buffer_size = 16 +bytes 03 61 62 00 +expect refused + +operation string +name string-accept-the-same-three-bytes-without-a-nul +param buffer_size = 16 +bytes 03 61 62 63 +expect value = 61 62 63 +consumed 32 +writer canonical + +# --------------------------------------------------------------------------- +# the UTF-8 well-formedness rule. Each refusal is paired with the nearest +# well-formed sequence, usually the code point one step away from the malformed +# one, so a reader that refuses a byte class wholesale fails the accept. + +operation string +name string-accept-shortest-two-byte-sequence +param buffer_size = 16 +bytes 02 C2 80 +expect value = C2 80 +consumed 24 +writer canonical + +operation string +name string-refuse-overlong-two-byte-nul +param buffer_size = 16 +bytes 02 C0 80 +expect refused + +operation string +name string-refuse-overlong-two-byte-slash +param buffer_size = 16 +bytes 02 C1 AF +expect refused + +operation string +name string-accept-shortest-three-byte-sequence +param buffer_size = 16 +bytes 03 E0 A0 80 +expect value = E0 A0 80 +consumed 32 +writer canonical + +operation string +name string-refuse-overlong-three-byte-slash +param buffer_size = 16 +bytes 03 E0 80 AF +expect refused + +operation string +name string-accept-just-below-the-surrogate-block +param buffer_size = 16 +bytes 03 ED 9F BF +expect value = ED 9F BF +consumed 32 +writer canonical + +operation string +name string-refuse-surrogate-encoded-in-utf8 +param buffer_size = 16 +bytes 03 ED A0 80 +expect refused + +operation string +name string-refuse-low-surrogate-encoded-in-utf8 +param buffer_size = 16 +bytes 03 ED BF BF +expect refused + +operation string +name string-accept-just-above-the-surrogate-block +param buffer_size = 16 +bytes 03 EE 80 80 +expect value = EE 80 80 +consumed 32 +writer canonical + +operation string +name string-accept-astral-code-point +param buffer_size = 16 +bytes 04 F0 9F 92 A9 +expect value = F0 9F 92 A9 +consumed 40 +writer canonical + +operation string +name string-accept-the-largest-code-point +param buffer_size = 16 +bytes 04 F4 8F BF BF +expect value = F4 8F BF BF +consumed 40 +writer canonical + +operation string +name string-refuse-code-point-above-10FFFF +param buffer_size = 16 +bytes 04 F4 90 80 80 +expect refused + +operation string +name string-refuse-five-byte-sequence +param buffer_size = 16 +bytes 05 F8 88 80 80 80 +expect refused + +operation string +name string-refuse-lone-continuation-byte +param buffer_size = 16 +bytes 01 80 +expect refused + +operation string +name string-refuse-continuation-byte-after-ascii +param buffer_size = 16 +bytes 02 41 BF +expect refused + +operation string +name string-refuse-truncated-two-byte-sequence +param buffer_size = 16 +bytes 01 C3 +expect refused + +operation string +name string-accept-the-same-two-byte-sequence-completed +param buffer_size = 16 +bytes 02 C3 A9 +expect value = C3 A9 +consumed 24 +writer canonical + +operation string +name string-refuse-truncated-three-byte-sequence +param buffer_size = 16 +bytes 02 E2 82 +expect refused + +operation string +name string-accept-the-same-three-byte-sequence-completed +param buffer_size = 16 +bytes 03 E2 82 AC +expect value = E2 82 AC +consumed 32 +writer canonical + +operation string +name string-refuse-byte-fe +param buffer_size = 16 +bytes 01 FE +expect refused + +operation string +name string-refuse-byte-ff +param buffer_size = 16 +bytes 01 FF +expect refused diff --git a/conformance/uint128.txt b/conformance/uint128.txt new file mode 100644 index 0000000..a534573 --- /dev/null +++ b/conformance/uint128.txt @@ -0,0 +1,65 @@ +# serialize conformance vectors: serialize_uint128 +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "uint128": a 128-bit unsigned integer, ALWAYS 128 bits on the +# wire, the LOW 64-BIT HALF FIRST and then the high half, each half written +# exactly as serialize_bits( half, 64 ). When the stream is byte aligned the +# result is the 16 bytes of the value in little-endian order. +# +# Do not confuse this with serialize_int128, which is ranged. This operation +# carries no bounds and has no range rule, so the only refusal STANDARD.md +# states for it is reading past the end of the stream. +# +# The half order is the thing an implementation gets wrong, so the two vectors +# that put the whole value in one half are the discriminating pair: an +# implementation that writes the high half first swaps their bytes. + +operation uint128 +name uint128-zero +bytes 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +expect value = 0 +consumed 128 +writer canonical + +operation uint128 +name uint128-low-half-only +bytes 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +expect value = 1 +consumed 128 +writer canonical + +operation uint128 +name uint128-high-half-only +bytes 00 00 00 00 00 00 00 00 01 00 00 00 00 00 00 00 +expect value = 18446744073709551616 +consumed 128 +writer canonical + +# The library's additive pin, decoded here by the document alone: +# 0x0123456789ABCDEFFEDCBA9876543210, low half 0xFEDCBA9876543210 first. + +operation uint128 +name uint128-distinct-halves +bytes 10 32 54 76 98 BA DC FE EF CD AB 89 67 45 23 01 +expect value = 0x0123456789ABCDEFFEDCBA9876543210 +consumed 128 +writer canonical + +operation uint128 +name uint128-max +bytes FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF FF +expect value = 340282366920938463463374607431768211455 +consumed 128 +writer canonical + +operation uint128 +name uint128-refuse-past-end-15-bytes +bytes 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 +expect refused + +operation uint128 +name uint128-refuse-past-end-empty-stream +bytes +expect refused diff --git a/conformance/wstring.txt b/conformance/wstring.txt new file mode 100644 index 0000000..1324edd --- /dev/null +++ b/conformance/wstring.txt @@ -0,0 +1,271 @@ +# serialize conformance vectors: serialize_wstring +# +# The format is specified in STANDARD.md, "The vector format". Records are +# separated by blank lines. +# +# STANDARD.md, "wstring": a null-terminated wide string where buffer_size +# counts WIDE CHARACTERS, NOT BYTES, written as +# +# 1. the length, as serialize_int( length, 0, buffer_size - 1 ); +# 2. each character as a 32-BIT GROUP, in order. +# +# "NO ALIGNMENT IS PERFORMED ANYWHERE IN THIS OPERATION — this is the one place +# where the wide-string path deliberately differs from its narrow counterpart, +# which aligns via serialize_bytes. An implementation that mirrors the narrow +# string path here will produce the wrong bytes." Every accepted vector below +# has a length field narrower than a byte, so the first character always begins +# mid-byte and a stray align moves every byte after it. +# +# Each 32-bit group carries one UTF-16 CODE UNIT, not one code point. `value` +# is therefore stated as the transmitted code units, four hexadecimal digits +# each, and that is the wire-level truth on every platform: a 4-byte wchar_t +# implementation converts at the boundary — splitting astral code points into +# surrogate pairs on write and recombining on read — so its runner compares +# after re-splitting, and both platforms owe the same bytes. +# +# Four refusal rules reach this operation: +# +# * the length field's own range rule, when buffer_size - 1 is not one less +# than a power of two; +# * a GROUP ABOVE 0xFFFF is not a UTF-16 code unit and "the reader must +# refuse it on every platform, whatever the local wchar_t can hold" — +# refusal does not depend on the platform, so the same byte sequence is +# refused everywhere; +# * an UNPAIRED SURROGATE fails the read: a high surrogate (0xD800–0xDBFF) +# not immediately followed by a low surrogate (0xDC00–0xDFFF), a low +# surrogate not immediately preceded by a high surrogate, or a high +# surrogate as the final transmitted group. Well-formed surrogate PAIRS +# remain valid — they are how astral text travels; +# * an INTERIOR NUL fails the read — a zero group among the length +# transmitted groups, by the same two-lengths logic as string; +# +# and reading past the end. + +# The document's own worked example, quoted in "Worked Example": a wchar_t[8] +# buffer holding three characters, producing this 13-byte run. buffer_size 8 +# makes the length field bits_required( 0, 7 ) = 3 bits, and the first +# character begins immediately at bit 3 — the low 5 bits of 0xE3 are the low 5 +# bits of 0x043C. Total 3 + 3x32 = 99 bits. + +operation wstring +name wstring-worked-example +param buffer_size = 8 +bytes E3 21 00 00 C0 21 00 00 00 22 00 00 00 +expect value = 043C 0438 0440 +consumed 99 +writer canonical + +# No alignment, stated as bluntly as the corpus can state it: a one-bit length +# field, so the character occupies bits 1 through 32. An implementation that +# aligns before the characters starts them at bit 8 and produces entirely +# different bytes. + +operation wstring +name wstring-no-alignment-before-the-characters +param buffer_size = 2 +bytes 83 00 00 00 00 +expect value = 0041 +consumed 33 +writer canonical + +operation wstring +name wstring-empty +param buffer_size = 8 +bytes 00 +expect value = +consumed 3 +writer canonical + +# buffer_size 1: the length field is serialize_int( length, 0, 0 ), a +# degenerate range costing zero bits, and there is no alignment to perform, so +# the whole operation consumes nothing — from an empty stream. + +operation wstring +name wstring-buffer-size-1-zero-bit-length-field +param buffer_size = 1 +bytes +expect value = +consumed 0 +writer canonical + +operation wstring +name wstring-single-basic-plane-character +param buffer_size = 8 +bytes 09 02 00 00 00 +expect value = 0041 +consumed 35 +writer canonical + +# --------------------------------------------------------------------------- +# the group-above-0xFFFF rule, pinned at exactly the boundary: 0xFFFF is a +# legal code unit and 0x10000 is not, and the two streams differ by one bit. + +operation wstring +name wstring-accept-group-ffff +param buffer_size = 8 +bytes F9 FF 07 00 00 +expect value = FFFF +consumed 35 +writer canonical + +operation wstring +name wstring-refuse-group-one-above-ffff +param buffer_size = 8 +bytes 01 00 08 00 00 +expect refused + +operation wstring +name wstring-refuse-group-all-thirty-two-bits +param buffer_size = 8 +bytes F9 FF FF FF 07 +expect refused + +# --------------------------------------------------------------------------- +# the unpaired-surrogate rule, at both ends of both surrogate blocks, each +# beside the nearest code unit that is not a surrogate. + +operation wstring +name wstring-accept-just-below-the-surrogate-block +param buffer_size = 8 +bytes F9 BF 06 00 00 +expect value = D7FF +consumed 35 +writer canonical + +operation wstring +name wstring-refuse-first-high-surrogate-alone +param buffer_size = 8 +bytes 01 C0 06 00 00 +expect refused + +operation wstring +name wstring-refuse-last-high-surrogate-alone +param buffer_size = 8 +bytes F9 DF 06 00 00 +expect refused + +operation wstring +name wstring-refuse-first-low-surrogate-alone +param buffer_size = 8 +bytes 01 E0 06 00 00 +expect refused + +operation wstring +name wstring-refuse-last-low-surrogate-alone +param buffer_size = 8 +bytes F9 FF 06 00 00 +expect refused + +operation wstring +name wstring-accept-just-above-the-surrogate-block +param buffer_size = 8 +bytes 01 00 07 00 00 +expect value = E000 +consumed 35 +writer canonical + +# A well-formed surrogate pair is how astral text travels, and it must be +# accepted. This is U+1F4A9 as its pair, D83D DCA9. + +operation wstring +name wstring-accept-surrogate-pair +param buffer_size = 8 +bytes EA C1 06 00 48 E5 06 00 00 +expect value = D83D DCA9 +consumed 67 +writer canonical + +operation wstring +name wstring-refuse-high-surrogate-followed-by-a-non-surrogate +param buffer_size = 8 +bytes EA C1 06 00 08 02 00 00 00 +expect refused + +operation wstring +name wstring-refuse-low-surrogate-not-preceded-by-a-high-one +param buffer_size = 8 +bytes 4A E5 06 00 E8 C1 06 00 00 +expect refused + +operation wstring +name wstring-refuse-high-surrogate-as-the-final-group +param buffer_size = 8 +bytes 0A 02 00 00 E8 C1 06 00 00 +expect refused + +# --------------------------------------------------------------------------- +# the interior NUL rule, in each position a zero group can occupy. + +operation wstring +name wstring-refuse-nul-as-the-only-group +param buffer_size = 8 +bytes 01 00 00 00 00 +expect refused + +operation wstring +name wstring-refuse-nul-as-the-first-of-two-groups +param buffer_size = 8 +bytes 02 00 00 00 08 02 00 00 00 +expect refused + +operation wstring +name wstring-refuse-nul-as-the-final-group +param buffer_size = 8 +bytes 0A 02 00 00 00 00 00 00 00 +expect refused + +# The accept twin of the vector above: the same stream with a non-zero second +# group, so the only difference between the two verdicts is the group's value. + +operation wstring +name wstring-accept-two-basic-plane-groups +param buffer_size = 8 +bytes 0A 02 00 00 10 02 00 00 00 +expect value = 0041 0042 +consumed 67 +writer canonical + +# --------------------------------------------------------------------------- +# the length field's range rule: buffer_size 5 gives bits_required( 0, 4 ) = 3 +# bits, so lengths 5, 6 and 7 are expressible and must be refused. + +operation wstring +name wstring-accept-length-inside-a-five-character-buffer +param buffer_size = 5 +bytes 09 02 00 00 00 +expect value = 0041 +consumed 35 +writer canonical + +operation wstring +name wstring-refuse-length-one-past-the-buffer +param buffer_size = 5 +bytes 05 +expect refused + +operation wstring +name wstring-refuse-length-all-three-bits +param buffer_size = 5 +bytes 07 +expect refused + +# --------------------------------------------------------------------------- +# reading past the end, in the length field and in the groups. + +operation wstring +name wstring-refuse-past-end-second-group-truncated +param buffer_size = 8 +bytes EA C1 06 00 +expect refused + +operation wstring +name wstring-refuse-past-end-first-group-truncated +param buffer_size = 8 +bytes 09 02 +expect refused + +operation wstring +name wstring-refuse-past-end-empty-stream +param buffer_size = 8 +bytes +expect refused diff --git a/src/serialize/BitStream.java b/src/serialize/BitStream.java index febaa2b..23f6bce 100644 --- a/src/serialize/BitStream.java +++ b/src/serialize/BitStream.java @@ -105,6 +105,16 @@ public interface BitStream */ boolean serializeFixed128( Ref value, int integerBits, int fractionBits, long minUnits, long maxUnits ); + /** + * Runs a nested object's own serialize function inline. It contributes no + * bytes of its own: no framing, no length prefix and no alignment is + * inserted around it, so the nested object's operations appear at exactly + * this position in the stream. A refusal inside the nesting propagates out + * of it, and a read stream consults its failure state first, so a nested + * object on a failed stream refuses without invoking the object. + */ + boolean serializeObject( Serializer object ); + /** The number of pad bits an align would cost right now. Always 7 on a measure stream. */ int getAlignBits(); diff --git a/src/serialize/MeasureStream.java b/src/serialize/MeasureStream.java index 9f455ff..35a684b 100644 --- a/src/serialize/MeasureStream.java +++ b/src/serialize/MeasureStream.java @@ -46,10 +46,15 @@ private static boolean checkBits32( int bits ) return true; } - private static boolean checkBits64( int bits ) + // the same width bound the write stream carries, on the caller's 64-bit + // value: a measure runs the writer's contracts so a non-conforming call is + // diagnosed on every writing stream. SerializeUtil.valueFitsInBits is the + // one place the rule lives. + private static boolean checkBits64( long value, int bits ) { assert bits > 0; assert bits <= 64; + assert SerializeUtil.valueFitsInBits( value, bits ); return true; } @@ -80,7 +85,7 @@ public boolean serializeBits( IntRef value, int bits ) @Override public boolean serializeBits64( LongRef value, int bits ) { - assert checkBits64( bits ); + assert checkBits64( value.value, bits ); bitsWritten += bits; return true; } @@ -279,6 +284,12 @@ public boolean serializeFixed128( Ref value, int integerBits, int f return true; } + @Override + public boolean serializeObject( Serializer object ) + { + return object.serialize( this ); + } + /** Always the worst case of 7: the measure does not know the final bit position. */ @Override public int getAlignBits() diff --git a/src/serialize/ReadStream.java b/src/serialize/ReadStream.java index d454c21..e77c169 100644 --- a/src/serialize/ReadStream.java +++ b/src/serialize/ReadStream.java @@ -710,6 +710,13 @@ else if ( bits <= 96 ) return true; } + @Override + public boolean serializeObject( Serializer object ) + { + if ( failed ) return false; // the latch: a failed stream refuses everything after + return object.serialize( this ); + } + @Override public int getAlignBits() { diff --git a/src/serialize/SerializeUtil.java b/src/serialize/SerializeUtil.java index a2e2f85..64b9bb9 100644 --- a/src/serialize/SerializeUtil.java +++ b/src/serialize/SerializeUtil.java @@ -11,7 +11,26 @@ public final class SerializeUtil private SerializeUtil() {} /** The library version, matching the release tag. */ - public static final String VERSION = "1.1.1"; + public static final String VERSION = "1.1.2"; + + /** + * Does a value fit in a field of this width? STANDARD.md bounds a + * {@code bits} field by {@code value < 2^bits} at every width in [1,64], + * not only at 32 or fewer. + * + * This is the write side bound for {@code serializeBits64}, and it runs on + * the caller's 64-bit value before any narrowing to a 32-bit group. A + * check placed after the narrowing is handed a value the narrowing already + * made legal: 2^32 + 5 arrives as 5, which fits four bits. + * + * @param value the caller's value, read as unsigned. + * @param bits the field width, in [1,64]. + * @return true if the value fits in that many bits. + */ + public static boolean valueFitsInBits( long value, int bits ) + { + return bits >= 64 || Long.compareUnsigned( value, ( 1L << bits ) - 1 ) <= 0; + } /** * The number of bits required to serialize an integer in [min,max]. diff --git a/src/serialize/Serializer.java b/src/serialize/Serializer.java new file mode 100644 index 0000000..91a72d5 --- /dev/null +++ b/src/serialize/Serializer.java @@ -0,0 +1,24 @@ +package serialize; + +/** + * An object that serializes itself against any stream: the operand of the + * family's {@code object} operation. + * + * One method, so a message type implements it directly and a nested field + * can also be a lambda or a method reference. The same body runs against a + * {@link WriteStream}, a {@link ReadStream} and a {@link MeasureStream}, + * which is what makes composition free of a second description of the + * message. + * + * @see BitStream#serializeObject(Serializer) + */ +@FunctionalInterface +public interface Serializer +{ + /** + * Serializes this object against the stream. + * @param stream the stream to run against. + * @return false to refuse, which propagates out of the nesting. + */ + boolean serialize( BitStream stream ); +} diff --git a/src/serialize/WriteStream.java b/src/serialize/WriteStream.java index b362a07..2aee64c 100644 --- a/src/serialize/WriteStream.java +++ b/src/serialize/WriteStream.java @@ -51,10 +51,16 @@ private static boolean checkBits32( int bits ) return true; } - private static boolean checkBits64( int bits ) + // the width bound runs on the caller's 64-bit value, before the narrowing + // to a 32-bit group in serializeBits64: a check placed after the narrowing + // is handed a value the narrowing already made legal. + // SerializeUtil.valueFitsInBits is the one place the rule lives, shared + // with MeasureStream. + private static boolean checkBits64( long value, int bits ) { assert bits > 0; assert bits <= 64; + assert SerializeUtil.valueFitsInBits( value, bits ); return true; } @@ -85,7 +91,7 @@ public boolean serializeBits( IntRef value, int bits ) @Override public boolean serializeBits64( LongRef value, int bits ) { - assert checkBits64( bits ); + assert checkBits64( value.value, bits ); if ( bits <= 32 ) { writer.writeBits( (int) value.value, bits ); @@ -444,6 +450,12 @@ public boolean serializeFixed128( Ref value, int integerBits, int f return true; } + @Override + public boolean serializeObject( Serializer object ) + { + return object.serialize( this ); + } + @Override public int getAlignBits() { diff --git a/test/serialize/tests/ConformanceTests.java b/test/serialize/tests/ConformanceTests.java index 0178be3..2406275 100644 --- a/test/serialize/tests/ConformanceTests.java +++ b/test/serialize/tests/ConformanceTests.java @@ -1,42 +1,76 @@ package serialize.tests; +import serialize.BitStream; +import serialize.BoolRef; +import serialize.DoubleRef; +import serialize.FloatRef; import serialize.Int128Value; import serialize.IntRef; +import serialize.LongRef; +import serialize.MeasureStream; import serialize.ReadStream; import serialize.Ref; +import serialize.UInt128Value; +import serialize.WriteStream; import java.io.IOException; import java.io.UncheckedIOException; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import static serialize.tests.Harness.check; -import static serialize.tests.Harness.checkEqual; import static serialize.tests.Harness.test; /** - * The shared conformance corpus, run through this library's reader. + * The shared conformance corpus, run through this library's reader, writer and + * measure. * * {@code conformance/} is a verbatim vendored copy of the corpus in - * mas-bandwidth/serialize, one file per operation, and CI fails if the two - * diverge. Every vector in it runs here: an accepted vector must decode to - * the stated value and consume the stated number of bits, and a refused - * vector must be refused with the destination left unwritten. Nothing in - * this file regenerates an expectation — the corpus is the authority, and a - * suite that computes its own answers proves only that the port agrees with - * itself. + * mas-bandwidth/serialize, one file per covered operation, and CI fails if the + * two diverge. Every vector in it runs here, and nothing in this file + * regenerates an expectation: the corpus is the authority, and a suite that + * computes its own answers proves only that the port agrees with itself. * - * The corpus directory is read relative to the working directory, which is - * the repository root under {@code make test}. An operation with no runner - * below fails the suite rather than being skipped: a vector that does not - * run is a vector that proves nothing. + * The directory is DISCOVERED at run time rather than named in this source, so + * a vector file vendored in but never mentioned still runs, and an empty + * directory fails the run. A vector whose operation this runner cannot drive + * FAILS, and so does a vector naming a parameter it does not understand: a + * gate that skips what it does not understand reports green over a rule nobody + * checked. + * + * One STEP MACHINE drives both the single operation files and the sequence, + * object and message files. A single operation vector is a one or two step + * sequence built from the record's own parameters, so the sequence files + * cannot drift away from the operation files. + * + * An accepted vector must decode to the stated value and consume the stated + * bits. A vector carrying {@code writer = canonical} is additionally re-emitted + * through the write stream and compared byte for byte, flush included, which is + * where the trailing bits obligation bites. A vector carrying + * {@code measure_at_least} is run through the measure stream, as a floor rather + * than an equality, because a measure is a bound and not the packet size. + * + * A refused vector must be refused, must leave the caller's scalar destination + * holding the sentinel it was seeded with, and must leave the stream TERMINAL. + * Terminality is checked by BEHAVIOR rather than by an accessor, so the check + * is the same one every port in the family runs: every later step of a sequence + * must refuse too, and a further read the vector does not name must fail, + * consume no bits and write nothing to its own destination. STANDARD.md leaves + * a caller-owned BUFFER unspecified after a refusal — {@code bytes}, + * {@code string} and {@code wstring} — so those destinations are not checked. + * + * THE BUFFER CONTRACT. Every stream presented here carries the eight bytes of + * slack {@link serialize.BitReader} requires, filled with a non-zero pattern, + * so a decode that depends on memory past the end cannot pass by reading zeros. */ public final class ConformanceTests { @@ -44,15 +78,35 @@ private ConformanceTests() {} private static final Path CORPUS = Paths.get( "conformance" ); - /** A value no vector decodes to, so a written destination is visible after a refusal. */ - private static final int SENTINEL = 0x5E5E5E5E; + /** Non-zero, so a read that strays past the end of the data is visible. */ + private static final byte SLACK_FILL = (byte) 0xA5; + + /** The buffer contract: at least 8 bytes past the data. */ + private static final int SLACK = 8; - private static final Int128Value SENTINEL_128 = new Int128Value( 0x5E5E5E5E5E5E5E5EL, 0x5E5E5E5E5E5E5E5EL ); + /** + * The sentinel a bit pattern destination is seeded with. It fits 32 bits, + * so it survives the narrowing this runner performs on the way to float's + * own width and a destination the library correctly left alone still reads + * as written. + */ + private static final BigInteger SENTINEL_BITS = BigInteger.valueOf( 0xCAFEF00DL ); + + /** The sentinel a number destination is seeded with: small and negative, so it fits every width. */ + private static final BigInteger SENTINEL_NUMBER = BigInteger.valueOf( -1234567 ); private static final BigInteger MASK_64 = BigInteger.ONE.shiftLeft( 64 ).subtract( BigInteger.ONE ); private static final BigInteger MASK_128 = BigInteger.ONE.shiftLeft( 128 ).subtract( BigInteger.ONE ); + // what the run covered, reported at the end: a corpus whose writer or + // measure legs never ran is a corpus running at less than its own strength + private static int vectorsChecked; + + private static int writerChecks; + + private static int measureChecks; + /** * Runs this suite on its own ({@code java serialize.tests.ConformanceTests}), so * the interop job can hold this reader and the pinned C++ reader to the same @@ -73,18 +127,23 @@ static void run() for ( Path file : files ) { - List vectors = parse( file ); String fileName = file.getFileName().toString(); - test( "conformance " + fileName + ": the file holds vectors", () -> { + List vectors = new ArrayList<>(); + test( "conformance " + fileName + ": the file parses to vectors", () -> { + vectors.addAll( parse( file ) ); check( !vectors.isEmpty(), fileName + " parsed to no vectors" ); } ); for ( Vector vector : vectors ) { - test( "conformance " + fileName + ": " + vector.name, () -> vector.run() ); + test( "conformance " + fileName + ": " + vector.name, vector::run ); } } + + System.out.printf( " %d vectors from %d file(s): %d writer checks, %d measure checks%n", + vectorsChecked, files.size(), writerChecks, measureChecks ); } + // Discovery, not a list of filenames: a vendored file no one named still runs. private static List corpusFiles() { List files = new ArrayList<>(); @@ -107,8 +166,9 @@ private static List corpusFiles() return files; } - // STANDARD.md, "The vector format": text, `#` begins a comment, blank - // lines separate records, each record is `key` and value one per line. + // STANDARD.md, "The vector format" and its lexical rules: `#` begins a + // comment at the START of a line and nowhere else, blank lines separate + // records, and each record is a key and a value one per line. private static List parse( Path file ) { List lines; @@ -122,17 +182,20 @@ private static List parse( Path file ) } List vectors = new ArrayList<>(); - Vector current = new Vector(); + Vector current = new Vector( file ); for ( String rawLine : lines ) { - int comment = rawLine.indexOf( '#' ); - String line = ( comment >= 0 ? rawLine.substring( 0, comment ) : rawLine ).trim(); + if ( rawLine.startsWith( "#" ) ) + { + continue; + } + String line = rawLine.trim(); if ( line.isEmpty() ) { if ( current.started() ) { vectors.add( current ); - current = new Vector(); + current = new Vector( file ); } continue; } @@ -148,15 +211,431 @@ private static List parse( Path file ) return vectors; } + // --------------------------------------------------------------------- + // numbers + // + // A vector's number is signed decimal or 0x hexadecimal and a parser must + // accept values up to 128 bits wide, which is wider than any Java integral + // type, so they travel as BigInteger and land in the operation's own width + // at the call site. + + private static BigInteger parseNumber( String text ) + { + String body = text; + boolean negative = false; + if ( body.startsWith( "-" ) ) + { + negative = true; + body = body.substring( 1 ); + } + else if ( body.startsWith( "+" ) ) + { + body = body.substring( 1 ); + } + check( !body.isEmpty(), "malformed number: " + text ); + BigInteger value; + if ( body.startsWith( "0x" ) || body.startsWith( "0X" ) ) + { + String digits = body.substring( 2 ); + check( !digits.isEmpty(), "malformed number: " + text ); + value = new BigInteger( digits, 16 ); + } + else + { + value = new BigInteger( body, 10 ); + } + return negative ? value.negate() : value; + } + + /** The 128-bit two's complement pattern of a value, which is how every numeric expectation compares. */ + private static BigInteger pattern128( BigInteger value ) + { + return value.and( MASK_128 ); + } + + private static BigInteger unsigned64( long value ) + { + return BigInteger.valueOf( value ).and( MASK_64 ); + } + + private static Int128Value toInt128( BigInteger value ) + { + BigInteger bits = pattern128( value ); + return new Int128Value( bits.shiftRight( 64 ).longValue(), bits.and( MASK_64 ).longValue() ); + } + + private static BigInteger fromInt128( Int128Value value ) + { + return BigInteger.valueOf( value.hi ).shiftLeft( 64 ).add( unsigned64( value.lo ) ); + } + + private static UInt128Value toUInt128( BigInteger value ) + { + BigInteger bits = pattern128( value ); + return new UInt128Value( bits.shiftRight( 64 ).longValue(), bits.and( MASK_64 ).longValue() ); + } + + private static BigInteger fromUInt128( UInt128Value value ) + { + return unsigned64( value.hi ).shiftLeft( 64 ).or( unsigned64( value.lo ) ); + } + + // --------------------------------------------------------------------- + // the step machine + + private enum Kind + { + BITS, + BOOL, + UINT128, + ALIGN, + INT, + INT64, + INT128, + INT_RELATIVE, + FLOAT, + DOUBLE, + COMPRESSED_FLOAT, + BYTES, + STRING, + WSTRING, + FIXED, + OBJECT // opens a nested object over the steps that follow + } + + private static boolean valueIsABitPattern( Kind kind ) + { + return kind == Kind.BITS || kind == Kind.UINT128 || kind == Kind.FLOAT + || kind == Kind.DOUBLE || kind == Kind.COMPRESSED_FLOAT; + } + + private static boolean valueIsANumber( Kind kind ) + { + return kind == Kind.INT || kind == Kind.INT64 || kind == Kind.INT128 + || kind == Kind.INT_RELATIVE || kind == Kind.FIXED; + } + + private static final class Step + { + Kind kind; + int width; // bits, count, buffer_size, or the steps an object wraps + BigInteger min = BigInteger.ZERO; + BigInteger max = BigInteger.ZERO; + float floatMin; + float floatMax; + float resolution; + int integerBits; + int fractionBits; + int previous; + + // destinations + BigInteger bits = BigInteger.ZERO; // the decoded value where a bit pattern is what is pinned + BigInteger number = BigInteger.ZERO; + boolean booleanValue; + byte[] buffer = new byte[0]; + String text = ""; + } + + /** + * Runs one step against any stream. The step carries the value in and out, + * so the reader leg's decoded values are exactly what the writer and the + * measure legs are handed. The destination is written back whether the + * operation accepted or refused, which is what makes the sentinel check + * after a refusal meaningful. + */ + private static boolean runStep( BitStream stream, Step step ) + { + switch ( step.kind ) + { + case BITS: + { + LongRef value = new LongRef( step.bits.longValue() ); + boolean ok = stream.serializeBits64( value, step.width ); + step.bits = unsigned64( value.value ); + return ok; + } + case BOOL: + { + BoolRef value = new BoolRef( step.booleanValue ); + boolean ok = stream.serializeBool( value ); + step.booleanValue = value.value; + return ok; + } + case UINT128: + { + Ref value = new Ref<>( toUInt128( step.bits ) ); + boolean ok = stream.serializeUint128( value ); + step.bits = fromUInt128( value.value ); + return ok; + } + case ALIGN: + return stream.serializeAlign(); + + case INT: + { + IntRef value = new IntRef( step.number.intValue() ); + boolean ok = stream.serializeInt( value, step.min.intValue(), step.max.intValue() ); + step.number = BigInteger.valueOf( value.value ); + return ok; + } + case INT64: + { + LongRef value = new LongRef( step.number.longValue() ); + boolean ok = stream.serializeInt64( value, step.min.longValue(), step.max.longValue() ); + step.number = BigInteger.valueOf( value.value ); + return ok; + } + case INT128: + { + Ref value = new Ref<>( toInt128( step.number ) ); + boolean ok = stream.serializeInt128( value, toInt128( step.min ), toInt128( step.max ) ); + step.number = fromInt128( value.value ); + return ok; + } + case INT_RELATIVE: + { + IntRef value = new IntRef( step.number.intValue() ); + boolean ok = stream.serializeIntRelative( step.previous, value ); + step.number = BigInteger.valueOf( value.value ); + return ok; + } + case FLOAT: + { + FloatRef value = new FloatRef( Float.intBitsToFloat( step.bits.intValue() ) ); + boolean ok = stream.serializeFloat( value ); + step.bits = BigInteger.valueOf( Integer.toUnsignedLong( Float.floatToRawIntBits( value.value ) ) ); + return ok; + } + case DOUBLE: + { + DoubleRef value = new DoubleRef( Double.longBitsToDouble( step.bits.longValue() ) ); + boolean ok = stream.serializeDouble( value ); + step.bits = unsigned64( Double.doubleToRawLongBits( value.value ) ); + return ok; + } + case COMPRESSED_FLOAT: + { + FloatRef value = new FloatRef( Float.intBitsToFloat( step.bits.intValue() ) ); + boolean ok = stream.serializeCompressedFloat( value, step.floatMin, step.floatMax, step.resolution ); + step.bits = BigInteger.valueOf( Integer.toUnsignedLong( Float.floatToRawIntBits( value.value ) ) ); + return ok; + } + case BYTES: + { + if ( step.buffer.length != step.width ) + { + step.buffer = new byte[step.width]; + } + return stream.serializeBytes( step.buffer, step.width ); + } + case STRING: + { + Ref value = new Ref<>( step.text ); + boolean ok = stream.serializeString( value, step.width ); + step.text = value.value; + return ok; + } + case WSTRING: + { + Ref value = new Ref<>( step.text ); + boolean ok = stream.serializeWideString( value, step.width ); + step.text = value.value; + return ok; + } + case FIXED: + { + if ( step.integerBits + step.fractionBits == 128 ) + { + Ref value = new Ref<>( toInt128( step.number ) ); + boolean ok = stream.serializeFixed128( value, step.integerBits, step.fractionBits, + step.min.longValue(), step.max.longValue() ); + step.number = fromInt128( value.value ); + return ok; + } + LongRef value = new LongRef( step.number.longValue() ); + boolean ok = stream.serializeFixed( value, step.integerBits, step.fractionBits, + step.min.longValue(), step.max.longValue() ); + step.number = BigInteger.valueOf( value.value ); + return ok; + } + case OBJECT: + default: + // nesting is driven by runSteps, which owns the step range an object + // wraps; a bare object step reaching here is a runner bug + return false; + } + } + + // advances past the steps a nested object owns, so a walk over the top level sees one step per object + private static int stepSpan( List steps, int index ) + { + return steps.get( index ).kind == Kind.OBJECT ? 1 + steps.get( index ).width : 1; + } + + /** + * The run state a walk over the steps leaves behind: which top level step a + * refusal stopped on, and which primitive step it was that refused, which is + * the destination the sentinel check inspects. + */ + private static final class Walk + { + int stoppedAt = -1; + Step failedStep; + } + + /** + * Runs a range of steps against a stream. A step spelled {@code object } + * wraps the next n steps in a nested object, driven through the library's own + * {@code serializeObject} so what the vectors exercise is the composition the + * operation performs. + */ + private static boolean runSteps( BitStream stream, List steps, int from, int count, Walk walk ) + { + int end = from + count; + int i = from; + while ( i < end ) + { + Step step = steps.get( i ); + if ( step.kind == Kind.OBJECT ) + { + final int nested = i + 1; + final int nestedCount = step.width; + if ( !stream.serializeObject( inner -> runSteps( inner, steps, nested, nestedCount, walk ) ) ) + { + if ( walk.stoppedAt < 0 ) + { + walk.stoppedAt = i; + } + return false; + } + } + else if ( !runStep( stream, step ) ) + { + walk.failedStep = step; + if ( walk.stoppedAt < 0 ) + { + walk.stoppedAt = i; + } + return false; + } + i += stepSpan( steps, i ); + } + return true; + } + + // --------------------------------------------------------------------- + // rendering and comparison + // + // Numeric values — every integer width, and the float, double and + // compressed_float bit patterns — compare as 128-bit two's complement + // PATTERNS, so a hexadecimal expectation and its decimal twin are one + // expectation and nothing goes through a float. STANDARD.md requires that: + // NaN compares unequal to itself, -0.0 == 0.0, and a tolerance comparison + // cannot see a quieted signaling bit. + + private static String hexBytes( byte[] data, int count ) + { + StringBuilder text = new StringBuilder(); + for ( int i = 0; i < count; i++ ) + { + if ( text.length() > 0 ) + { + text.append( ' ' ); + } + text.append( String.format( "%02X", data[i] & 0xFF ) ); + } + return text.toString(); + } + + private static String hexCodeUnits( String value ) + { + StringBuilder text = new StringBuilder(); + for ( int i = 0; i < value.length(); i++ ) + { + if ( text.length() > 0 ) + { + text.append( ' ' ); + } + text.append( String.format( "%04X", (int) value.charAt( i ) ) ); + } + return text.toString(); + } + + private static BigInteger stepPattern( Step step ) + { + if ( valueIsABitPattern( step.kind ) ) + { + return pattern128( step.bits ); + } + if ( valueIsANumber( step.kind ) ) + { + return pattern128( step.number ); + } + return null; + } + + private static String renderStepValue( Step step ) + { + BigInteger value = stepPattern( step ); + if ( value != null ) + { + return "0x" + String.format( "%032X", value ); + } + switch ( step.kind ) + { + case BOOL: + return step.booleanValue ? "true" : "false"; + case BYTES: + return hexBytes( step.buffer, step.width ); + case STRING: + { + byte[] utf8 = step.text.getBytes( StandardCharsets.UTF_8 ); + return hexBytes( utf8, utf8.length ); + } + case WSTRING: + return hexCodeUnits( step.text ); + case ALIGN: + case OBJECT: + default: + // neither has a value of its own; for align the corpus states the + // padding it consumed, which a conforming read always finds zero + return "0"; + } + } + + private static boolean expectationMatches( Step step, String expected ) + { + BigInteger value = stepPattern( step ); + if ( value != null ) + { + return value.equals( pattern128( parseNumber( expected ) ) ); + } + return renderStepValue( step ).equals( expected ); + } + + // --------------------------------------------------------------------- + // one vector + private static final class Vector { + final String file; String operation; String name; final Map params = new LinkedHashMap<>(); + final List stepWords = new ArrayList<>(); byte[] bytes = new byte[0]; boolean refused; - String expected; + String expected = ""; long consumed; + boolean hasConsumed; + long measureAtLeast; + boolean hasMeasure; + boolean writerCanonical; + + Vector( Path file ) + { + this.file = file.getFileName().toString(); + } boolean started() { @@ -177,7 +656,16 @@ void put( String key, String value ) { int equals = value.indexOf( '=' ); check( equals > 0, "malformed param: " + value ); - params.put( value.substring( 0, equals ).trim(), value.substring( equals + 1 ).trim() ); + String paramName = value.substring( 0, equals ).trim(); + String paramValue = value.substring( equals + 1 ).trim(); + if ( paramName.equals( "step" ) ) + { + stepWords.add( paramValue ); + } + else + { + params.put( paramName, paramValue ); + } break; } case "bytes": @@ -192,11 +680,23 @@ void put( String key, String value ) { int equals = value.indexOf( '=' ); check( equals > 0, "malformed expect: " + value ); + String expectKind = value.substring( 0, equals ).trim(); + check( expectKind.equals( "bits" ) || expectKind.equals( "value" ), + "unknown expect kind '" + expectKind + "'" ); expected = value.substring( equals + 1 ).trim(); } break; case "consumed": consumed = Long.parseLong( value ); + hasConsumed = true; + break; + case "measure_at_least": + measureAtLeast = Long.parseLong( value ); + hasMeasure = true; + break; + case "writer": + check( value.equals( "canonical" ), "unknown writer mode '" + value + "'" ); + writerCanonical = true; break; default: check( false, "unknown vector key: " + key ); @@ -204,71 +704,456 @@ void put( String key, String value ) } } - String param( String key ) + private void fail( String detail ) + { + check( false, detail + " [" + file + "]" ); + } + + private String param( String key ) { String value = params.get( key ); - check( value != null, "vector " + name + " has no param " + key ); + if ( value == null ) + { + fail( "no param " + key + " on operation " + operation ); + } return value; } - /** The reader's buffer: the vector's bytes plus the 8 bytes of slack the reader loads through. */ - byte[] buffer() + private int intParam( String key ) + { + return parseNumber( param( key ) ).intValue(); + } + + /** + * A parameter this runner does not understand is a FAILURE and not a + * silent default: a vector whose declaration is not the one being + * exercised proves nothing. + */ + private static boolean operationTakesParam( String operation, String name ) + { + switch ( name ) + { + case "preceding_bits": + return operation.equals( "align" ) || operation.equals( "bytes" ); + case "bits": + return operation.equals( "bits" ); + case "count": + return operation.equals( "bytes" ); + case "buffer_size": + return operation.equals( "string" ) || operation.equals( "wstring" ); + case "previous": + return operation.equals( "int_relative" ); + case "res": + return operation.equals( "compressed_float" ); + case "integer_bits": + case "fraction_bits": + return operation.equals( "fixed" ); + case "min": + case "max": + return operation.equals( "int" ) || operation.equals( "int64" ) || operation.equals( "int128" ) + || operation.equals( "fixed" ) || operation.equals( "compressed_float" ); + default: + return false; + } + } + + /** The reader's buffer: the vector's bytes plus the slack the buffer contract requires, non-zero filled. */ + private byte[] readerBuffer() { - byte[] buffer = new byte[bytes.length + 8]; + byte[] buffer = new byte[bytes.length + SLACK]; + Arrays.fill( buffer, SLACK_FILL ); System.arraycopy( bytes, 0, buffer, 0, bytes.length ); return buffer; } - void run() + // A step spelling inside a sequence. The spellings are documented at the + // head of conformance/sequence.txt and conformance/object.txt. + private Step stepFromWords( String text ) + { + String[] words = text.trim().split( "\\s+" ); + Step step = new Step(); + switch ( words[0] ) + { + case "bits": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.BITS; + step.width = parseNumber( words[1] ).intValue(); + return step; + case "bool": + check( words.length == 1, "malformed step: " + text ); + step.kind = Kind.BOOL; + return step; + case "align": + check( words.length == 1, "malformed step: " + text ); + step.kind = Kind.ALIGN; + return step; + case "float": + check( words.length == 1, "malformed step: " + text ); + step.kind = Kind.FLOAT; + return step; + case "object": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.OBJECT; + step.width = parseNumber( words[1] ).intValue(); + return step; + case "bytes": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.BYTES; + step.width = parseNumber( words[1] ).intValue(); + return step; + case "string": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.STRING; + step.width = parseNumber( words[1] ).intValue(); + return step; + case "wstring": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.WSTRING; + step.width = parseNumber( words[1] ).intValue(); + return step; + case "int": + check( words.length == 3, "malformed step: " + text ); + step.kind = Kind.INT; + step.min = parseNumber( words[1] ); + step.max = parseNumber( words[2] ); + return step; + case "fixed": + check( words.length == 5, "malformed step: " + text ); + step.kind = Kind.FIXED; + step.integerBits = parseNumber( words[1] ).intValue(); + step.fractionBits = parseNumber( words[2] ).intValue(); + step.min = parseNumber( words[3] ); + step.max = parseNumber( words[4] ); + return step; + default: + fail( "no runner for step '" + text + "'" ); + return step; + } + } + + /** + * The step list for a vector. A single operation vector becomes a one or + * two step sequence: the operations whose interesting behavior only + * exists at a non-zero bit index take a {@code preceding_bits} + * parameter, which becomes a leading bits step. + */ + private List buildSteps() { + List steps = new ArrayList<>(); + if ( operation.equals( "sequence" ) ) + { + for ( String words : stepWords ) + { + steps.add( stepFromWords( words ) ); + } + check( !steps.isEmpty(), "a sequence with no steps" ); + return steps; + } + + if ( params.containsKey( "preceding_bits" ) ) + { + int precedingBits = intParam( "preceding_bits" ); + if ( precedingBits > 0 ) + { + Step leading = new Step(); + leading.kind = Kind.BITS; + leading.width = precedingBits; + steps.add( leading ); + } + } + + Step step = new Step(); switch ( operation ) { - case "int_relative": - runIntRelative(); + case "bits": + step.kind = Kind.BITS; + step.width = intParam( "bits" ); + break; + case "bool": + step.kind = Kind.BOOL; + break; + case "uint128": + step.kind = Kind.UINT128; + break; + case "align": + step.kind = Kind.ALIGN; + break; + case "int": + step.kind = Kind.INT; + step.min = parseNumber( param( "min" ) ); + step.max = parseNumber( param( "max" ) ); + break; + case "int64": + step.kind = Kind.INT64; + step.min = parseNumber( param( "min" ) ); + step.max = parseNumber( param( "max" ) ); break; case "int128": - runInt128(); + step.kind = Kind.INT128; + step.min = parseNumber( param( "min" ) ); + step.max = parseNumber( param( "max" ) ); + break; + case "int_relative": + step.kind = Kind.INT_RELATIVE; + step.previous = intParam( "previous" ); + break; + case "float": + step.kind = Kind.FLOAT; + break; + case "double": + step.kind = Kind.DOUBLE; + break; + case "compressed_float": + step.kind = Kind.COMPRESSED_FLOAT; + step.floatMin = floatParam( "min" ); + step.floatMax = floatParam( "max" ); + step.resolution = floatParam( "res" ); + break; + case "bytes": + step.kind = Kind.BYTES; + step.width = intParam( "count" ); + break; + case "string": + step.kind = Kind.STRING; + step.width = intParam( "buffer_size" ); + break; + case "wstring": + step.kind = Kind.WSTRING; + step.width = intParam( "buffer_size" ); + break; + case "fixed": + step.kind = Kind.FIXED; + step.integerBits = intParam( "integer_bits" ); + step.fractionBits = intParam( "fraction_bits" ); + step.min = parseNumber( param( "min" ) ); + step.max = parseNumber( param( "max" ) ); break; default: - check( false, "the corpus carries operation " + operation + ", which this suite does not run" ); + // a corpus file this runner cannot drive is a gap in the runner, not a pass + fail( "no runner for operation '" + operation + "'" ); break; } + steps.add( step ); + return steps; } - private void runIntRelative() + // res, min and max under compressed_float are float32 operands + private float floatParam( String key ) { - int previous = Integer.parseInt( param( "previous" ) ); - ReadStream reader = new ReadStream( buffer(), bytes.length ); - IntRef current = new IntRef( SENTINEL ); - boolean accepted = reader.serializeIntRelative( previous, current ); - if ( refused ) + return (float) Double.parseDouble( param( key ) ); + } + + void run() + { + vectorsChecked++; + for ( String paramName : params.keySet() ) { - check( !accepted, "must refuse" ); - checkEqual( current.value, SENTINEL, "destination unwritten after a refusal" ); - return; + if ( !operationTakesParam( operation, paramName ) ) + { + fail( "no runner for parameter '" + paramName + "' on operation '" + operation + "'" ); + } + } + if ( !stepWords.isEmpty() && !operation.equals( "sequence" ) ) + { + fail( "steps are only meaningful on a sequence" ); + } + + List steps = buildSteps(); + + runReader( steps ); + + // the writer and the measure are handed the values the reader decoded, + // so a canonical vector's round trip is decode then re-emit + if ( !refused ) + { + if ( writerCanonical ) + { + runWriter( steps ); + } + if ( hasMeasure ) + { + runMeasure( steps ); + } } - check( accepted, "must accept" ); - checkEqual( current.value, Long.parseLong( expected ), "decoded value" ); - checkEqual( reader.getBitsProcessed(), consumed, "bits consumed" ); } - private void runInt128() + private void runReader( List steps ) { - Int128Value min = toInt128( new BigInteger( param( "min" ) ) ); - Int128Value max = toInt128( new BigInteger( param( "max" ) ) ); - ReadStream reader = new ReadStream( buffer(), bytes.length ); - Ref value = new Ref<>( SENTINEL_128 ); - boolean accepted = reader.serializeInt128( value, min, max ); + ReadStream stream = new ReadStream( readerBuffer(), bytes.length ); + + for ( Step step : steps ) + { + step.bits = SENTINEL_BITS; + step.number = SENTINEL_NUMBER; + step.booleanValue = true; // a refused bool read must leave this alone + step.text = ""; + } + + Walk walk = new Walk(); + boolean accepted = runSteps( stream, steps, 0, steps.size(), walk ); + if ( refused ) { - check( !accepted, "must refuse" ); - check( value.value.equals( SENTINEL_128 ), "destination unwritten after a refusal" ); + if ( accepted ) + { + fail( "the read succeeded, the corpus requires refusal" ); + } + + // STANDARD.md, "A refused primitive read must leave its destination + // unwritten". The rule reaches the scalars only: a read into a + // caller-owned buffer — bytes, string and wstring — leaves that + // buffer unspecified after a refusal, so those are not checked. + Step failedStep = walk.failedStep; + if ( failedStep != null ) + { + if ( valueIsABitPattern( failedStep.kind ) && !failedStep.bits.equals( SENTINEL_BITS ) ) + { + fail( "the refused read wrote to the destination" ); + } + if ( valueIsANumber( failedStep.kind ) && !failedStep.number.equals( SENTINEL_NUMBER ) ) + { + fail( "the refused read wrote to the destination" ); + } + if ( failedStep.kind == Kind.BOOL && !failedStep.booleanValue ) + { + fail( "the refused read wrote to the destination" ); + } + } + + // Failure is terminal, and a sequence states its own successors: + // every step after the failing one must fail too, however many + // readable bits the stream still holds. + for ( int i = walk.stoppedAt + stepSpan( steps, walk.stoppedAt ); i < steps.size(); + i += stepSpan( steps, i ) ) + { + if ( runSteps( stream, steps, i, stepSpan( steps, i ), new Walk() ) ) + { + fail( "step " + ( i + 1 ) + " succeeded after step " + ( walk.stoppedAt + 1 ) + + " was refused; failure must be terminal" ); + } + } + + // and the same rule against a read the vector does not name, so + // every refused vector carries the terminality check and not only + // the sequences that spell a successor + failUnlessStreamIsTerminal( stream ); return; } - check( accepted, "must accept" ); - check( fromInt128( value.value ).equals( new BigInteger( expected ) ), - "decoded value: got " + fromInt128( value.value ) + ", expected " + expected ); - checkEqual( reader.getBitsProcessed(), consumed, "bits consumed" ); + + if ( !accepted ) + { + fail( "the read was refused, the corpus requires it to be accepted" ); + } + + String[] entries = expected.split( "\\|", -1 ); + // one expect entry per step, objects and aligns included, which state + // `-`. A leading preceding_bits step carries no entry of its own: it + // exists to place the stream, and the record states only the operation + // under test, so the entries align to the END of the step list. + int offset = steps.size() - entries.length; + if ( offset < 0 ) + { + fail( "the expect list states more values than the vector has steps" ); + } + for ( int i = 0; i < entries.length; i++ ) + { + String entry = entries[i].trim(); + if ( entry.equals( "-" ) ) + { + continue; + } + Step step = steps.get( offset + i ); + if ( !expectationMatches( step, entry ) ) + { + fail( "step " + ( offset + i + 1 ) + " decoded " + renderStepValue( step ) + + ", the corpus states " + entry ); + } + } + + if ( hasConsumed && stream.getBitsProcessed() != consumed ) + { + fail( "consumed " + stream.getBitsProcessed() + " bits, the corpus states " + consumed ); + } + } + + // Failure is terminal, and a refused vector is where that rule is + // testable. The stream is checked by BEHAVIOR rather than by an accessor, + // so the same check ports to every implementation in the family: a further + // read must fail, consume no bits and leave its destination alone. + private void failUnlessStreamIsTerminal( ReadStream stream ) + { + IntRef after = new IntRef( 0xFFFFFFFF ); + long bitsBefore = stream.getBitsProcessed(); + if ( stream.serializeBits( after, 8 ) ) + { + fail( "the stream accepted a read after the refusal: failure is not terminal" ); + } + if ( after.value != 0xFFFFFFFF ) + { + fail( "the read after the refusal wrote to its destination" ); + } + if ( stream.getBitsProcessed() != bitsBefore ) + { + fail( "the read after the refusal consumed bits" ); + } + } + + /** + * The writer leg. A vector marked {@code writer = canonical} states the + * bytes a conforming writer emits for its value, so the runner writes the + * decoded steps back and compares the WHOLE emitted stream. That is what + * pins the trailing bits obligation: the unused bits of the final byte + * must be zero, and a writer leaking anything into them produces a byte + * the vector does not carry. + */ + private void runWriter( List steps ) + { + writerChecks++; + byte[] scratch = new byte[512]; + Arrays.fill( scratch, SLACK_FILL ); + WriteStream stream = new WriteStream( scratch, scratch.length ); + + if ( !runSteps( stream, steps, 0, steps.size(), new Walk() ) ) + { + fail( "the writer refused a canonical vector" ); + } + stream.flush(); + + long written = stream.getBytesProcessed(); + if ( written != bytes.length ) + { + fail( "the writer emitted " + written + " bytes, the corpus states " + bytes.length ); + } + for ( int i = 0; i < written; i++ ) + { + if ( scratch[i] != bytes[i] ) + { + fail( "the writer emitted " + hexBytes( scratch, (int) written ) + + ", the corpus states " + hexBytes( bytes, bytes.length ) ); + } + } + } + + /** + * The measure leg. STANDARD.md makes a measure a BOUND and not the packet + * size, so the corpus states a floor and the check is an inequality. A + * measure that computes alignment from a running bit index starting at + * zero under-counts every unaligned start and falls below the floor, which + * is the non-conforming accounting the document names. + */ + private void runMeasure( List steps ) + { + measureChecks++; + MeasureStream stream = new MeasureStream(); + if ( !runSteps( stream, steps, 0, steps.size(), new Walk() ) ) + { + fail( "the measure refused a step; a measure refuses nothing at runtime" ); + } + if ( stream.getBitsProcessed() < measureAtLeast ) + { + fail( "measured " + stream.getBitsProcessed() + " bits, the corpus requires at least " + + measureAtLeast ); + } } } @@ -282,17 +1167,4 @@ private static byte[] hex( String text ) } return bytes; } - - /** The decimal vector value as the library's two's-complement pair. */ - private static Int128Value toInt128( BigInteger value ) - { - BigInteger bits = value.and( MASK_128 ); - return new Int128Value( bits.shiftRight( 64 ).longValue(), bits.and( MASK_64 ).longValue() ); - } - - /** The library's pair back to a signed decimal, for comparison against the vector. */ - private static BigInteger fromInt128( Int128Value value ) - { - return BigInteger.valueOf( value.hi ).shiftLeft( 64 ).add( BigInteger.valueOf( value.lo ).and( MASK_64 ) ); - } } diff --git a/test/serialize/tests/StreamTests.java b/test/serialize/tests/StreamTests.java index f231e38..bdc2106 100644 --- a/test/serialize/tests/StreamTests.java +++ b/test/serialize/tests/StreamTests.java @@ -293,6 +293,59 @@ static void run() check( reader.serializeBits( value, 4 ), "the stream reads again" ); checkEqual( value.value, 0x0F, "the value after reset" ); } ); + + // STANDARD.md bounds a bits field by value < 2^bits at every width in + // [1,64]. The write path narrows a 64-bit value to a 32-bit group at + // widths of 32 or fewer, so the bound must be checked on the caller's + // value FIRST: a check after the narrowing is handed a value the + // narrowing already made legal, and 2^32 + 5 arrives as 5, which fits + // four bits. Every writing stream carries the same check. + // The contract is an assert, so it exists only in the checked shape. + if ( assertionsEnabled() ) + { + test( "write contract: a bits64 value is bounded at the caller's width, before it narrows", () -> { + WriteStream writer = new WriteStream( new byte[16], 16 ); + check( fires( () -> writer.serializeBits64( new LongRef( ( 1L << 32 ) + 5 ), 4 ) ), + "the write stream must reject a value that does not fit four bits" ); + + MeasureStream measure = new MeasureStream(); + check( fires( () -> measure.serializeBits64( new LongRef( ( 1L << 32 ) + 5 ), 4 ) ), + "the measure stream must reject a value that does not fit four bits" ); + + // the negative control: the value one step inside the bound, which + // must not fire on either stream + WriteStream inside = new WriteStream( new byte[16], 16 ); + check( !fires( () -> inside.serializeBits64( new LongRef( 5 ), 4 ) ), + "a value that fits four bits must pass" ); + check( !fires( () -> new MeasureStream().serializeBits64( new LongRef( 5 ), 4 ) ), + "a value that fits four bits must pass the measure" ); + + // and the widest field, where every 64-bit value fits + check( !fires( () -> new MeasureStream().serializeBits64( new LongRef( -1L ), 64 ) ), + "every value fits 64 bits" ); + } ); + } + } + + private static boolean assertionsEnabled() + { + boolean enabled = false; + assert enabled = true; + return enabled; + } + + /** Did the body's debug contract fire? */ + private static boolean fires( Runnable body ) + { + try + { + body.run(); + return false; + } + catch ( AssertionError error ) + { + return true; + } } /** A value no read below decodes to, so a written destination is visible after a refusal. */