diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6183cde..a8007b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,24 +82,19 @@ jobs: # what a commit's CI proves. Find the newest tag with: # gh release list --repo mas-bandwidth/serialize --limit 1 # Bump family-wide, deliberately, in its own commit per repo. SERIALIZE_TAG - # below is the ONE place this repository names a RELEASE of the reference; the - # corpus commit beside it names the vendoring and is a different pin for a - # different job, explained where it is declared. + # below is the ONE place this repository names a version of the reference, and + # ONE pin is the whole point: the wire exchange and the conformance corpus are + # held to the same release, so the reference reader driving the corpus is the + # reference library the bytes are exchanged with. A release whose corpus this + # repository has not vendored fails the diff below rather than passing on a + # second, looser pin. # (The spec-sync job above deliberately stays on upstream main: drift DETECTION # is that job's whole purpose.) interop: name: interop with the C++ reference runs-on: ubuntu-latest env: - SERIALIZE_TAG: v1.16.0 - # TWO PINS, TWO JOBS FOR THEM. SERIALIZE_TAG above is the wire - # compatibility pin and is a released tag. SERIALIZE_CORPUS_COMMIT is the - # commit STANDARD.md and conformance/ in this repository are vendored - # from, and the reference conformance runner is built from it, because a - # runner older than the corpus cannot drive the operations the corpus - # carries. The spec-sync job proves this repository's vendored copies - # match upstream main, so this commit is upstream main at the vendoring. - SERIALIZE_CORPUS_COMMIT: 7e0515e952f3373d001ec1899adf9dffb823e1c5 + SERIALIZE_TAG: v1.16.2 steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -110,10 +105,13 @@ jobs: - name: Check out the C++ reference at the pinned release run: git clone --quiet --depth 1 --branch "$SERIALIZE_TAG" https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize - - name: Check out the C++ reference at the corpus commit - run: | - git clone --quiet https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize-corpus - git -C /tmp/cpp-serialize-corpus checkout --quiet "$SERIALIZE_CORPUS_COMMIT" + # The corpus vendored here must be the one the pinned release carries: the + # corpus and the runner that can drive it are one artifact, and a reference + # reader older than the corpus cannot drive vectors whose operations it has + # no code for. STANDARD.md requires that to be red rather than skipped, so + # it is checked here rather than assumed. + - name: The vendored corpus is the pinned release's corpus + run: diff -ru /tmp/cpp-serialize/conformance conformance # Asserts stay ON (no -DNDEBUG): they are the C++ half of "API misuse # panics", and the degenerate ranges the message carries must pass with the @@ -123,7 +121,7 @@ jobs: - name: Build the C++ halves run: | c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_interop interop/interop.cpp - c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize-corpus -o /tmp/cpp_conformance /tmp/cpp-serialize-corpus/conformance.cpp + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_conformance /tmp/cpp-serialize/conformance.cpp - name: Both halves write byte identical wire data run: | diff --git a/STANDARD.md b/STANDARD.md index 88cf1bd..04732c4 100644 --- a/STANDARD.md +++ b/STANDARD.md @@ -693,6 +693,75 @@ bits — four 32-bit groups). A decoder that assembles the groups in the wrong order, or puts the remainder anywhere but the most significant position, decodes the wrong values here. +### The whole message + +The fields above are three of the message's twenty-eight operations. Here is all +of it, in the step spelling the shared corpus uses, with the value each operation +carries. `conformance/message.txt` states the same message as a vector, so every +implementation in the family runs it. + +| # | operation | value | +|---|---|---| +| 1 | `bits 4` | 13 | +| 2 | `bits 11` | 1445 | +| 3 | `bits 24` | 11259375 | +| 4 | `bits 32` | 3735928559 (`0xDEADBEEF`) | +| 5 | `int -100 100` | -37 | +| 6 | `int -2147483648 2147483647` | -123456789 | +| 7 | `bool` | true | +| 8 | `float` | 3.1415926 (`0x40490FDA`) | +| 9 | `compressed_float 0 10 0.01` | 5.0 (`0x40A00000`) | +| 10 | `double` | 1/3 (`0x3FD5555555555555`) | +| 11 | `bits 8` | 127 | +| 12 | `bits 16` | 4660 | +| 13 | `bits 32` | 305419896 | +| 14 | `bits 64` | 1311768467463790320 | +| 15 | `int_relative 100` | 101 | +| 16 | `int_relative 100` | 2100 | +| 17 | `align` | - | +| 18 | `bytes 7` | `DE AD BE EF CA FE 01` | +| 19 | `string 16` | `golden` | +| 20 | `wstring 8` | `043C 0438 0440` | +| 21 | `align` | - | +| 22 | `fixed 8 8 -100 100` | -832 | +| 23 | `fixed 16 16 -2000 2000` | 80904192 | +| 24 | `fixed 48 16 -100000 100000` | -3559993401 | +| 25 | `fixed 16 16 0 30000` | 1966079999 | +| 26 | `align` | - | +| 27 | `fixed 112 16 -144115188075855872 144115188075855872` | -6472691358699745 | +| 28 | `fixed 64 64 -9223372036854775808 9223372036854775807` | 1512366075204170930115394234220888865 | + +A `-` marks an operation that produces no value of its own. A value written as a +hexadecimal pattern is stated that way because it is compared as a bit pattern +rather than as a number, which is what `float`, `double` and `compressed_float` +require. The `fixed` values are raw scaled integers, not +real units, because that is what the operation carries. + +The whole message is 112 bytes: + + 0x5D 0xDA 0xF7 0xE6 0xD5 0x77 0xDF 0x56 0xEF 0x9F 0x75 0x19 + 0x52 0xBC 0xDA 0x0F 0x49 0x40 0xF4 0x55 0x55 0x55 0x55 0x55 + 0x55 0x55 0xFF 0xFC 0xD1 0x48 0xE0 0x59 0xD1 0x48 0xC0 0x7B + 0xF3 0x6A 0xE2 0x59 0xD1 0x48 0x84 0xB7 0x06 0xDE 0xAD 0xBE + 0xEF 0xCA 0xFE 0x01 0x06 0x67 0x6F 0x6C 0x64 0x65 0x6E 0xE3 + 0x21 0x00 0x00 0xC0 0x21 0x00 0x00 0x00 0x22 0x00 0x00 0x00 + 0xC0 0x60 0x00 0x80 0xA2 0x7C 0xFC 0xEC 0x26 0xCB 0xFF 0xFF + 0x4B 0x1D 0x1F 0xEF 0xD2 0x1A 0x1F 0x01 0xE9 0xFF 0xFF 0x09 + 0x19 0x2A 0x3B 0x4C 0x5D 0x6E 0x7F 0x78 0x6F 0x5E 0x4D 0x3C + 0x2B 0x1A 0x09 0x04 + +It occupies 891 bits, so the final byte carries three bits of message and five bits +of zero padding, which is the writer obligation under Reader Obligations. A +conforming measure reports at least 895 bits for it, four more than the stream +occupies from an aligned start, because the three aligns, the byte block and the +string each cost more from some other starting position. + +These bytes are not copied out of an implementation. They are produced by encoding +the fields above with a writer written from this document alone, in +`tools/conformance`, and they are printed here because that writer, this byte block +and the C++ implementation's pinned emission agree byte for byte, which the same +tool checks on every run. + ## Read-only and write-only forms Every operation above has `read_` and `write_` variants — `read_string`, @@ -1020,8 +1089,8 @@ operations are: | `string.txt` | `string` | valid UTF-8 accepted, and invalid UTF-8, an interior NUL and an out-of-range length refused | | `wstring.txt` | `wstring` | no alignment anywhere, a surrogate pair accepted, and an unpaired surrogate, a group above `0xFFFF` and a zero group refused | | `object.txt` | `object` | that it adds no bytes of its own, each vector twinned with the same operations unnested | -| `sequence.txt` | sequences of operations | alignment after an odd width, a zero-bit field between wide ones, terminal failure, and the measure floor | -| `message.txt` | the Worked Example's message | three operations across an alignment boundary, byte for byte | +| `sequence.txt` | sequences of operations | alignment after an odd width, a zero-bit field between wide ones, terminal failure before a zero-bit ranged field and before a zero-bit fixed point one, and the measure floor | +| `message.txt` | the Worked Example's message | three operations across an alignment boundary, and the whole 112-byte golden message, byte for byte | Adding a file here is how a rule becomes a family obligation, and the table above moves with the directory. @@ -1051,7 +1120,6 @@ vector would pin. The tie-break sentence above stands until this list is empty. |---|---|---| | `fixed` | a negative tie value, which separates half away from zero from an arithmetic shift | the rounding happens where a value is quantized into a Q format or narrowed out of one, and no `serialize_fixed` call rounds: the wire round trip is exact, this document says as much, and no record whose input is a byte stream can reach the rule. It needs a record shape this format does not have | | `compressed_float` | the between-quanta writer inputs, and the fusion witness as a WRITE | a vector's input is a stream, so it can pin what a writer left on the wire but not the value a writer was handed. Both clamp declarations' widths, boundaries and refusals are pinned, and `8388608.0` is pinned as a decode in each of them; the writer inputs need a record shape this format does not have | -| Worked Example | the whole 112-byte golden message | this document prints 15 of its bytes and `message.txt` carries exactly those. The rest exist only inside an implementation, and a vector copied off the implementation it judges is the failure this section opens by naming | **The vector format.** A vector file is text. `#` begins a comment, blank lines separate records, and each record is `key` and value, one per line: @@ -1082,7 +1150,11 @@ produces for that value. **Values are typed by the operation's table.** A `param` takes the type its own section gives that parameter, so `min` and `max` under `fixed` are whole real units, `buffer_size` under `string` is a byte count, and `res` under -`compressed_float` is a `float32`. +`compressed_float` is a `float32`. A `wstring` value is stated as the transmitted +UTF-16 **code units**, four hexadecimal digits each, because that is what the +groups carry. A runtime whose `wchar_t` is wider recombines surrogate pairs into +code points on read, so its runner splits them again before comparing, and both +platforms are held to the same units and the same bytes. **Lexical rules.** `#` begins a comment at the start of a line and nowhere else, numbers are written as signed decimal or as `0x` hexadecimal, and a @@ -1093,7 +1165,9 @@ is how `float` and `double` vectors are stated. **A sequence vector states more than one operation.** Its `operation` is `sequence`, its steps are `param step = ` lines in order, and `expect value` lists one entry per step separated by ` | `, with `-` for a step that produces -no value of its own. Sequences carry what a single-operation record cannot: +no value of its own. There is one step spelling per operation this document +defines, listed at the head of `conformance/sequence.txt`, so any message the +format can carry can be stated as a sequence, the golden message included. Sequences carry what a single-operation record cannot: alignment cost, which depends on the bit index the previous operation left behind; a zero-bit field, which must leave that index exactly where it found it; a nested `object`, spelled `object ` to wrap the next `n` steps; and diff --git a/conformance/message.txt b/conformance/message.txt index d5f8596..b9ba6d6 100644 --- a/conformance/message.txt +++ b/conformance/message.txt @@ -22,15 +22,13 @@ # corpus where three different operations meet across an alignment boundary # with a byte-for-byte expectation the page prints in full. # -# It is a FRAGMENT of the 112-byte golden and not the whole of it, deliberately. -# The remaining fields' bytes appear nowhere in STANDARD.md; they exist only as -# an array inside serialize.h, and lifting them here would put an -# implementation's output into the corpus, which is the one thing the corpus -# must never contain — "no checker reimplements the codec and then checks that -# reimplementation against itself", and a vector copied off the implementation -# it is meant to judge is the same failure wearing a different coat. Shipping -# the full golden needs the document to print the rest of its bytes; that is -# filed as an underdetermination rather than worked around here. +# The first record below is that fragment. The second is the whole 112-byte +# message, whose bytes STANDARD.md now prints in full. Those bytes are not +# copied off an implementation: they are produced by encoding the message's +# fields with a writer written from the document alone, in tools/conformance, +# and they are carried here only because that writer and the C++ writer's +# pinned emission agree byte for byte. A vector copied off the implementation +# it is meant to judge would be the failure this corpus exists to prevent. # # The wide string ends at bit 99, three bits into byte 12, so the align pads # the remaining five bits of that byte — which is why the document's 13-byte @@ -46,3 +44,55 @@ expect value = 043C 0438 0440 | - | -832 consumed 120 measure_at_least 122 writer canonical + +# The whole golden message, every operation this format defines meeting every +# other one in a single stream: all four raw bit widths, a narrow and a full +# range ranged int, a bool, a float, a quantized compressed float, a double, +# four unsigned widths, both interesting tiers of the relative integer ladder, +# a byte block, a narrow string, a wide string, and six fixed point fields +# across three storage widths and three alignment boundaries. +# +# It ends at bit 891, three bits into byte 111, so the final byte carries five +# bits of zero padding: the writer obligation, pinned by writer = canonical +# over the whole stream. +# +# The measure floor is the true worst case over every starting bit position, +# which is four bits above the 891 this stream occupies from an aligned start, +# because the three aligns, the bytes and the string each cost more from some +# other start than they cost from this one. + +operation sequence +name message-golden-whole +param step = bits 4 +param step = bits 11 +param step = bits 24 +param step = bits 32 +param step = int -100 100 +param step = int -2147483648 2147483647 +param step = bool +param step = float +param step = compressed_float 0 10 0.01 +param step = double +param step = bits 8 +param step = bits 16 +param step = bits 32 +param step = bits 64 +param step = int_relative 100 +param step = int_relative 100 +param step = align +param step = bytes 7 +param step = string 16 +param step = wstring 8 +param step = align +param step = fixed 8 8 -100 100 +param step = fixed 16 16 -2000 2000 +param step = fixed 48 16 -100000 100000 +param step = fixed 16 16 0 30000 +param step = align +param step = fixed 112 16 -144115188075855872 144115188075855872 +param step = fixed 64 64 -9223372036854775808 9223372036854775807 +bytes 5D DA F7 E6 D5 77 DF 56 EF 9F 75 19 52 BC DA 0F 49 40 F4 55 55 55 55 55 55 55 FF FC D1 48 E0 59 D1 48 C0 7B F3 6A E2 59 D1 48 84 B7 06 DE AD BE EF CA FE 01 06 67 6F 6C 64 65 6E E3 21 00 00 C0 21 00 00 00 22 00 00 00 C0 60 00 80 A2 7C FC EC 26 CB FF FF 4B 1D 1F EF D2 1A 1F 01 E9 FF FF 09 19 2A 3B 4C 5D 6E 7F 78 6F 5E 4D 3C 2B 1A 09 04 +expect value = 13 | 1445 | 11259375 | 3735928559 | -37 | -123456789 | true | 0x40490FDA | 0x40A00000 | 0x3FD5555555555555 | 127 | 4660 | 305419896 | 1311768467463790320 | 101 | 2100 | - | DE AD BE EF CA FE 01 | 67 6F 6C 64 65 6E | 043C 0438 0440 | - | -832 | 80904192 | -3559993401 | 1966079999 | - | -6472691358699745 | 1512366075204170930115394234220888865 +consumed 891 +measure_at_least 895 +writer canonical diff --git a/conformance/sequence.txt b/conformance/sequence.txt index 33d7b0d..f80725c 100644 --- a/conformance/sequence.txt +++ b/conformance/sequence.txt @@ -16,10 +16,17 @@ # # bits bool # align float -# bytes int -# string wstring +# bytes double +# string uint128 +# wstring int +# int64 int128 +# int_relative +# compressed_float # fixed # +# There is one spelling per operation, so any message this format can carry can +# be written as a sequence, the golden message in message.txt included. +# # `expect value` lists one entry per step in order, separated by ` | `, with # `-` for a step that produces no value of its own. `consumed` is the total # for the whole sequence. @@ -188,6 +195,20 @@ param step = int 7 7 bytes FF expect refused +# The same vector with a degenerate FIXED POINT field as the successor. The two +# are twins on purpose: a zero-bit field is zero bits whatever operation +# declares it, so an implementation whose ranged int consults the failure state +# before its degenerate case while its fixed point path returns early passes +# the vector above and fails this one. Every read consults the failure state +# before it does anything else, zero-bit reads included. + +operation sequence +name sequence-refusal-is-terminal-before-a-zero-bit-fixed-field +param step = bits 16 +param step = fixed 16 16 7 7 +bytes FF +expect refused + # Terminal failure inside a composite: the string's payload runs past the end, # and the wstring after it must refuse rather than resynchronize. diff --git a/test/serialize/tests/ConformanceTests.java b/test/serialize/tests/ConformanceTests.java index 2406275..846978c 100644 --- a/test/serialize/tests/ConformanceTests.java +++ b/test/serialize/tests/ConformanceTests.java @@ -791,6 +791,26 @@ private Step stepFromWords( String text ) check( words.length == 1, "malformed step: " + text ); step.kind = Kind.FLOAT; return step; + case "double": + check( words.length == 1, "malformed step: " + text ); + step.kind = Kind.DOUBLE; + return step; + case "uint128": + check( words.length == 1, "malformed step: " + text ); + step.kind = Kind.UINT128; + return step; + case "int_relative": + check( words.length == 2, "malformed step: " + text ); + step.kind = Kind.INT_RELATIVE; + step.previous = parseNumber( words[1] ).intValue(); + return step; + case "compressed_float": + check( words.length == 4, "malformed step: " + text ); + step.kind = Kind.COMPRESSED_FLOAT; + step.floatMin = (float) Double.parseDouble( words[1] ); + step.floatMax = (float) Double.parseDouble( words[2] ); + step.resolution = (float) Double.parseDouble( words[3] ); + return step; case "object": check( words.length == 2, "malformed step: " + text ); step.kind = Kind.OBJECT; @@ -812,8 +832,11 @@ private Step stepFromWords( String text ) step.width = parseNumber( words[1] ).intValue(); return step; case "int": + case "int64": + case "int128": check( words.length == 3, "malformed step: " + text ); - step.kind = Kind.INT; + step.kind = words[0].equals( "int" ) ? Kind.INT + : words[0].equals( "int64" ) ? Kind.INT64 : Kind.INT128; step.min = parseNumber( words[1] ); step.max = parseNumber( words[2] ); return step;