From 4a37f2903dc6f4176708dc50170958e5fbc8ac21 Mon Sep 17 00:00:00 2001 From: Rowan Date: Fri, 4 Sep 2026 21:22:13 +1000 Subject: [PATCH] A live interop job: this port exchanges bytes with the pinned C++ reference The port exists to be wire compatible and nothing measured that against the reference itself. The new interop job builds the C++ library at a pinned release and runs it head to head with this port on every push and pull request. interop/interop.cpp and interop/serialize/interop/Interop.java serialize the same boundary message: every operation STANDARD.md defines, at the values where implementations disagree. Zero bit ranges on int, int64, int128 and fixed point; the domain edges of the ranged integers and of int_relative, every tier of its ladder at both ends; the maximum widths of bits, uint128 and the four group fixed point path; both sides of the alignment rule, including the align inside a zero length bytes; empty and full strings; and the wide string cases the surrogate rule governs, up to the largest code point. Floats are given and compared as raw bit patterns, so a value comparison cannot hide -0.0. Both halves write it and the files must be byte identical; each decodes the other's file, checks every value and re-encodes it byte for byte. Then the hostile half: every proper prefix of the other's stream is a truncated stream and both sides must refuse it without throwing, and both sides run the shared conformance corpus, where the refusal vectors are the point. ConformanceTests gains a main so the corpus can be run on its own, which is what lets both sides of that claim live in one job. The reference tag is named in ONE place, SERIALIZE_TAG in the workflow, and the version in one, SerializeUtil.VERSION. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/ci.yml | 76 +++ Makefile | 28 +- README.md | 13 +- interop/interop.cpp | 732 +++++++++++++++++++++ interop/serialize/interop/Interop.java | 640 ++++++++++++++++++ src/serialize/SerializeUtil.java | 2 +- test/serialize/tests/ConformanceTests.java | 13 +- 7 files changed, 1497 insertions(+), 7 deletions(-) create mode 100644 interop/interop.cpp create mode 100644 interop/serialize/interop/Interop.java diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a58330..2afa63d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,3 +59,79 @@ jobs: exit 1 fi echo "conformance/ matches upstream" + + # The port exists to be wire compatible; everything else is detail. This job is + # the only place that claim is MEASURED against the reference itself: the exact + # release candidate in this repository exchanges bytes with the exact C++ + # library, in both directions, on every push and pull request. Nothing here is a + # local one-off, and no expectation is regenerated from this port's own codec. + # + # What is exchanged: interop/serialize/interop/Interop.java and + # interop/interop.cpp serialize the SAME boundary message -- every operation + # STANDARD.md defines, at the values where implementations disagree. Both halves + # write it and the files must be byte identical; each half then decodes the + # other's file, checks every value and re-encodes it byte for byte. Then the + # hostile half: every proper prefix of the other's stream is a truncated stream + # and both must refuse it, and both run the shared conformance corpus, where the + # refusal vectors are the point. + # + # THE FAMILY PIN -- ONE POLICY, ONE VERSION. The C++ checkout is pinned to a + # RELEASED tag of mas-bandwidth/serialize, never main, and to the SAME tag in + # every port's wire-compat gate, so "compatible with C++" means the same thing + # in all of them. Never main, because an upstream merge mid-day must not change + # what a commit's CI proves. Find the newest tag with: + # gh release list --repo mas-bandwidth/serialize --limit 1 + # Bump family-wide, deliberately, in its own commit per repo. SERIALIZE_TAG + # below is the ONE place this repository names a version of the reference. + # (The spec-sync job above deliberately stays on upstream main: drift DETECTION + # is that job's whole purpose.) + interop: + name: interop with the C++ reference + runs-on: ubuntu-latest + env: + SERIALIZE_TAG: v1.16.0 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + + - name: Check out the C++ reference at the pinned release + run: git clone --quiet --depth 1 --branch "$SERIALIZE_TAG" https://github.com/mas-bandwidth/serialize.git /tmp/cpp-serialize + + # Asserts stay ON (no -DNDEBUG): they are the C++ half of "API misuse + # panics", and the degenerate ranges the message carries must pass with the + # library's own checks enabled rather than around them. No -ffp-contract=off + # either: the compressed float arithmetic is pinned in the library's source, + # and building with default flags proves that rather than masking it. + - name: Build the C++ halves against the pinned library + run: | + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_interop interop/interop.cpp + c++ -O2 -std=c++17 -Wall -I /tmp/cpp-serialize -o /tmp/cpp_conformance /tmp/cpp-serialize/conformance.cpp + + - name: Both halves write byte identical wire data + run: | + /tmp/cpp_interop write /tmp/cpp.bin + make interop JDK_HOME="$JAVA_HOME" MODE=write FILE=/tmp/java.bin + cmp /tmp/cpp.bin /tmp/java.bin + + - name: Each half decodes the other's bytes and re-encodes them exactly + run: | + make interop JDK_HOME="$JAVA_HOME" MODE=read FILE=/tmp/cpp.bin + /tmp/cpp_interop read /tmp/java.bin + + - name: Both halves refuse every truncation of the other's stream + run: | + make interop JDK_HOME="$JAVA_HOME" MODE=refuse FILE=/tmp/cpp.bin + /tmp/cpp_interop refuse /tmp/java.bin + + # The corpus is the family's one conformance instrument, and its refusal + # vectors are what this step is for: the pinned reference reader and this + # port's reader are held to the same accepts and the same refusals, from the + # same vendored files. The spec-sync job above proves those files match + # upstream. + - name: Both halves run the shared conformance corpus + run: | + /tmp/cpp_conformance conformance/*.txt + make conformance JDK_HOME="$JAVA_HOME" diff --git a/Makefile b/Makefile index 9cdad97..0d794bd 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,14 @@ JAVAC := $(JDK_HOME)/bin/javac JAVA := $(JDK_HOME)/bin/java SRC := $(wildcard src/serialize/*.java) -TEST_SRC := $(wildcard test/serialize/tests/*.java) +TEST_SRC := $(wildcard test/serialize/tests/*.java) +INTEROP_SRC := $(wildcard interop/serialize/interop/*.java) -CLASSES := build/classes -TEST_CLASSES := build/test-classes +CLASSES := build/classes +TEST_CLASSES := build/test-classes +INTEROP_CLASSES := build/interop-classes -.PHONY: all test test-release clean +.PHONY: all test test-release conformance interop clean all: test test-release @@ -37,5 +39,23 @@ test: $(TEST_CLASSES)/.stamp test-release: $(TEST_CLASSES)/.stamp $(JAVA) -da -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.AllTests --release +# the shared conformance corpus alone, so the interop job can hold this reader +# and the pinned C++ reader to the same vendored files in one place. make test +# runs it too, as one suite among many. +conformance: $(TEST_CLASSES)/.stamp + $(JAVA) -ea -cp $(CLASSES):$(TEST_CLASSES) serialize.tests.ConformanceTests + +# the interop harness: compiled against the library classes only, like the tests. +# One exchange with the C++ reference per invocation: +# make interop MODE=write FILE=/tmp/java.bin +$(INTEROP_CLASSES)/.stamp: $(INTEROP_SRC) $(CLASSES)/.stamp + $(JAVAC) --release 17 -Xlint:all -Werror -cp $(CLASSES) -d $(INTEROP_CLASSES) $(INTEROP_SRC) + @touch $@ + +# asserts on: the write-side contracts are asserts here as everywhere, and the +# degenerate ranges the message carries must pass with them enabled +interop: $(INTEROP_CLASSES)/.stamp + $(JAVA) -ea -cp $(CLASSES):$(INTEROP_CLASSES) serialize.interop.Interop $(MODE) $(FILE) + clean: rm -rf build diff --git a/README.md b/README.md index 8864acd..f5d77aa 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ specification in [mas-bandwidth/serialize](https://github.com/mas-bandwidth/serialize), which CI checks for drift — is the authority on every byte. -Version 1.1.0 (`SerializeUtil.VERSION`). +Version 1.1.1 (`SerializeUtil.VERSION`). ## The surface @@ -129,6 +129,17 @@ plus a sabotage sweep proving every consumed bit of the golden stream is load bearing, refusal and terminality proofs for hostile input, and the measure bound. +[`interop/`](interop) takes it further: the CI `interop` job builds the +C++ reference at a pinned release and runs it head to head with this +port. Both halves write the same boundary message — every operation the +standard defines, at its boundary values — and the files must be byte +identical; each then decodes the other's bytes and re-encodes them +exactly; both must refuse every truncation of the other's stream; and +both run the corpus. The release candidate in this repository exchanges +bytes with the reference on every push, so wire compatibility is +measured rather than asserted. `make interop MODE=write FILE=out.bin` +runs one exchange by hand. + Benchmarking for the serialize family lives in [mas-bandwidth/schema](https://github.com/mas-bandwidth/schema)'s data-driven bench, which measures the generated codecs across every language on one corpus. ## License diff --git a/interop/interop.cpp b/interop/interop.cpp new file mode 100644 index 0000000..5fbe0a1 --- /dev/null +++ b/interop/interop.cpp @@ -0,0 +1,732 @@ +/* + The C++ half of the cross language interop harness. + + Built in CI against the real C++ serialize library (github.com/mas-bandwidth/serialize) + at the release the workflow pins, and run head to head with this port's half. Neither + half is a local one-off: the exact release candidate in this repository exchanges bytes + with the exact reference, on every push and pull request. + + interop write write the boundary message and hand the bytes over + interop read decode the other half's bytes, check every value, re-encode + and require the result to be byte identical to the input + interop refuse every proper prefix of the other half's bytes is a truncated + stream, and every one of them must be REFUSED + + THE MESSAGE is the boundary set: every operation STANDARD.md defines, at the values where + implementations disagree. Zero bit ranges on all three ranged widths and on fixed point; + the domain edges of int, int64, int128 and int_relative; the maximum widths of bits, + uint128 and the four group fixed point path; both sides of the alignment rule, including + the align inside a zero length bytes; empty and full strings; and the wide string cases + the surrogate rule governs, up to the largest code unit. Each section is preceded by an + align, so a divergence localizes to a section instead of shifting every byte after it. + + WHAT IT DELIBERATELY DOES NOT CARRY: a NaN payload. STANDARD.md's bit transparency claim + covers it, but a NaN's payload bits do not survive every language's float type on the way + to the wire, so a difference here would say nothing about the wire format. Each port pins + its own NaN patterns in its own suite, where the claim can be tested honestly. + + Build with asserts ON (no -DNDEBUG): they are the C++ half of "API misuse panics", and the + degenerate ranges must pass with the library's own checks enabled rather than around them. + + The library tag is pinned in ONE place, .github/workflows/ci.yml, and this file names no + version. Any change to the sequence below must be mirrored in this port's half, and never + changes the wire format. +*/ + +#include "serialize.h" + +#include +#include +#include +#include + +// --------------------------------------------------------------------------------------- +// section 1: raw bit groups, every width boundary + +struct BitsVector { int bits; uint64_t value; }; + +static const BitsVector bits_vectors[] = +{ + { 1, 1 }, // the minimum width, at its maximum value + { 1, 0 }, // and at its minimum + { 7, 0x7F }, // a sub-byte width, all ones + { 31, 0x7FFFFFFF }, // one below the single group maximum + { 32, 0xFFFFFFFFULL }, // the widest single group, all ones + { 32, 0 }, // and all zeros + { 33, 0x1FFFFFFFFULL }, // the first width past the 32 bit split + { 64, 0xFFFFFFFFFFFFFFFFULL }, // the maximum width, all ones + { 64, 0 }, // and all zeros +}; +static const int bits_count = (int) ( sizeof( bits_vectors ) / sizeof( bits_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 3: the fixed width unsigned helpers at their domain edges + +static const uint32_t uint8_values[] = { 0x00u, 0xFFu }; +static const uint32_t uint16_values[] = { 0x0000u, 0xFFFFu }; +static const uint32_t uint32_values[] = { 0x00000000u, 0xFFFFFFFFu }; +static const uint64_t uint64_values[] = { 0ULL, 0xFFFFFFFFFFFFFFFFULL }; +static const int uint8_count = (int) ( sizeof( uint8_values ) / sizeof( uint8_values[0] ) ); +static const int uint16_count = (int) ( sizeof( uint16_values ) / sizeof( uint16_values[0] ) ); +static const int uint32_count = (int) ( sizeof( uint32_values ) / sizeof( uint32_values[0] ) ); +static const int uint64_count = (int) ( sizeof( uint64_values ) / sizeof( uint64_values[0] ) ); + +static const int uint128_count = 3; + +// --------------------------------------------------------------------------------------- +// sections 5 to 7: the ranged integers + +struct IntVector { int32_t min; int32_t max; int32_t value; }; + +static const IntVector int_vectors[] = +{ + { 42, 42, 42 }, // degenerate: zero bits, mid sequence + { -100, +100, -100 }, // the bottom of the range + { -100, +100, +100 }, // the top of the range + { INT32_MIN, INT32_MAX, INT32_MIN }, // the full domain, 32 bits on the wire + { INT32_MIN, INT32_MAX, INT32_MAX }, + { -100, +100, -37 }, // a live field after the degenerate one +}; +static const int int_count = (int) ( sizeof( int_vectors ) / sizeof( int_vectors[0] ) ); + +struct Int64Vector { int64_t min; int64_t max; int64_t value; }; + +static const Int64Vector int64_vectors[] = +{ + { 10000000000LL, 10000000000LL, 10000000000LL }, // degenerate, with bounds past 2^32 + { -5000000000LL, +5000000000LL, -5000000000LL }, // a range wider than 32 bits, bottom + { -5000000000LL, +5000000000LL, +5000000000LL }, // and top + { INT64_MIN, INT64_MAX, INT64_MIN }, // the full domain, 64 bits on the wire + { INT64_MIN, INT64_MAX, INT64_MAX }, +}; +static const int int64_count = (int) ( sizeof( int64_vectors ) / sizeof( int64_vectors[0] ) ); + +static const int int128_count = 4; + +// the 128 bit constants are built rather than tabulated: serialize::int128_t is native +// __int128 where the compiler has it and the emulated pair where it does not, and only one +// of those is a literal type. +static serialize::int128_t int128_min_value() +{ + return serialize::int128_t( serialize::uint128_t( 1 ) << 127 ); +} + +static serialize::int128_t int128_max_value() +{ + return ~int128_min_value(); +} + +// 2^100 + 7: a degenerate bound that no 64 bit path can carry +static serialize::int128_t int128_degenerate_value() +{ + return ( serialize::int128_t( 1 ) << 100 ) + serialize::int128_t( 7 ); +} + +// --------------------------------------------------------------------------------------- +// section 9: int_relative, every tier at both ends and the domain edges + +struct RelativeVector { int32_t previous; int32_t current; }; + +static const RelativeVector relative_vectors[] = +{ + { 0, 1 }, // one-bit + { 0, 2 }, { 0, 6 }, // bounded-3, both ends + { 0, 7 }, { 0, 23 }, // bounded-5 + { 0, 24 }, { 0, 280 }, // bounded-9 + { 0, 281 }, { 0, 4377 }, // bounded-13 + { 0, 4378 }, { 0, 69914 }, // bounded-17 + { 0, 69915 }, // absolute, at its smallest difference + { 2147483646, 2147483647 }, // one-bit, at the top of the domain + { 0, 2147483647 }, // absolute, at the top of the domain +}; +static const int relative_count = (int) ( sizeof( relative_vectors ) / sizeof( relative_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 10: float and double, given as bit patterns so no decimal literal is parsed twice + +static const uint32_t float_bits[] = +{ + 0x00000000u, // +0 + 0x80000000u, // -0 + 0x7F800000u, // +infinity + 0xFF800000u, // -infinity + 0x7F7FFFFFu, // the largest finite float32 + 0x00800000u, // the smallest normal + 0x00000001u, // the smallest subnormal + 0x3F800000u, // 1.0f + 0xBF800000u, // -1.0f +}; +static const int float_count = (int) ( sizeof( float_bits ) / sizeof( float_bits[0] ) ); + +static const uint64_t double_bits[] = +{ + 0x0000000000000000ULL, // +0 + 0x8000000000000000ULL, // -0 + 0x7FF0000000000000ULL, // +infinity + 0xFFF0000000000000ULL, // -infinity + 0x7FEFFFFFFFFFFFFFULL, // the largest finite float64 + 0x0010000000000000ULL, // the smallest normal + 0x0000000000000001ULL, // the smallest subnormal + 0x3FF0000000000000ULL, // 1.0 + 0xBFF0000000000000ULL, // -1.0 +}; +static const int double_count = (int) ( sizeof( double_bits ) / sizeof( double_bits[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 11: compressed_float + +struct CompressedFloatVector { float value; float min; float max; float res; }; + +static const CompressedFloatVector compressed_float_vectors[] = +{ + { 0.0f, 0.0f, 10.0f, 0.01f }, // the bottom of the range: integer 0 + { 10.0f, 0.0f, 10.0f, 0.01f }, // the top: the maximum integer + { 0.005f, 0.0f, 10.0f, 0.01f }, // between quanta: 1 under float32, 0 widened + { 0.025f, 0.0f, 10.0f, 0.01f }, // between quanta: 3 vs 2 + { 0.105f, 0.0f, 10.0f, 0.01f }, // between quanta: 11 vs 10 + { 9.995f, 0.0f, 10.0f, 0.01f }, // between quanta: 1000 vs 999 + { -100.0f, -100.0f, 100.0f, 0.01f }, // the bottom of a range with a non-zero min + { -42.573f, -100.0f, 100.0f, 0.01f }, // off quantum over a non-zero min + { 8388609.0f, 0.0f, 8388609.0f, 1.0f }, // clamp witness A: an unclamped writer emits + // a code its own reader rejects (schema#109) + { 16777215.0f, 0.0f, 16777215.0f, 1.0f }, // clamp witness B: one bit wider than the field + { 0.0f, 0.0f, 1.0f, 1.0f }, // a one bit field, both codes + { 1.0f, 0.0f, 1.0f, 1.0f }, +}; +static const int compressed_float_count = (int) ( sizeof( compressed_float_vectors ) / sizeof( compressed_float_vectors[0] ) ); + +// --------------------------------------------------------------------------------------- +// section 12: bytes. The block lengths, and the byte each block is filled with. + +struct BytesVector { int length; uint8_t fill; }; + +static const BytesVector bytes_vectors[] = +{ + { 0, 0x00 }, // zero length: the align happens anyway + { 8, 0x00 }, + { 8, 0xFF }, + { 1, 0x5A }, +}; +static const int bytes_count = (int) ( sizeof( bytes_vectors ) / sizeof( bytes_vectors[0] ) ); +static const int bytes_max = 8; + +// --------------------------------------------------------------------------------------- +// section 13: string. buffer_size 16 throughout, so the length field is four bits. + +static const int string_buffer_size = 16; +static const int string_count = 3; + +// --------------------------------------------------------------------------------------- +// section 14: wstring. buffer_size 8 throughout: at most seven UTF-16 code units. + +static const int wstring_buffer_size = 8; +static const int wstring_count = 6; + +// the wide strings as explicit code points, so no source file encoding can reach the wire. +// On a four byte wchar_t the astral entry is one wchar_t and two code units, and the library +// splits it into its surrogate pair at the boundary; on a two byte wchar_t it is already the +// pair. Both produce the same bytes, which is the rule this entry exists to hold. +static void init_wide_strings( wchar_t strings[wstring_count][wstring_buffer_size] ) +{ + memset( (void*) strings, 0, sizeof( wchar_t ) * wstring_count * wstring_buffer_size ); + + // 0: empty + // 1: cyrillic "мир", basic plane + strings[1][0] = (wchar_t) 0x043C; + strings[1][1] = (wchar_t) 0x0438; + strings[1][2] = (wchar_t) 0x0440; + // 2: the first code unit above the surrogate block + strings[2][0] = (wchar_t) 0xE000; + // 3: the largest code unit there is + strings[3][0] = (wchar_t) 0xFFFF; + // 4: an astral code point between two basic plane ones: four code units, one pair + strings[4][0] = (wchar_t) 0x0041; + strings[4][1] = (wchar_t) 0x1F600; + strings[4][2] = (wchar_t) 0x0042; + // 5: seven code units, the most buffer_size 8 carries + for ( int i = 0; i < 7; i++ ) + { + strings[5][i] = (wchar_t) ( 0x0061 + i ); + } +} + +// --------------------------------------------------------------------------------------- +// the message + +struct InteropData +{ + uint64_t bits_values[bits_count]; + bool bool_values[2]; + uint32_t uint8_values_data[uint8_count]; + uint32_t uint16_values_data[uint16_count]; + uint32_t uint32_values_data[uint32_count]; + uint64_t uint64_values_data[uint64_count]; + serialize::uint128_t uint128_values_data[uint128_count]; + int32_t int_values[int_count]; + int64_t int64_values[int64_count]; + serialize::int128_t int128_values[int128_count]; + int16_t fixed_q8_8_min; + int16_t fixed_q8_8_max; + int32_t fixed_q16_16_degenerate; + int64_t fixed_q48_16_min; + int64_t fixed_q48_16_max; + serialize::int128_t fixed_q112_16_max; + serialize::int128_t fixed_q64_64_degenerate; + serialize::int128_t fixed_q64_64_max; + int32_t relative_values[relative_count]; + float float_values[float_count]; + double double_values[double_count]; + float compressed_float_values[compressed_float_count]; + uint32_t filler; + uint8_t bytes_values[bytes_count][bytes_max]; + char strings[string_count][string_buffer_size]; + wchar_t wide_strings[wstring_count][wstring_buffer_size]; +}; + +static float float_from_bits( uint32_t bits ) +{ + float value; + memcpy( &value, &bits, sizeof( value ) ); + return value; +} + +static uint32_t bits_from_float( float value ) +{ + uint32_t bits; + memcpy( &bits, &value, sizeof( bits ) ); + return bits; +} + +static double double_from_bits( uint64_t bits ) +{ + double value; + memcpy( &value, &bits, sizeof( value ) ); + return value; +} + +static uint64_t bits_from_double( double value ) +{ + uint64_t bits; + memcpy( &bits, &value, sizeof( bits ) ); + return bits; +} + +static void InteropInit( InteropData & data ) +{ + memset( (void*) &data, 0, sizeof( data ) ); + + for ( int i = 0; i < bits_count; i++ ) + { + data.bits_values[i] = bits_vectors[i].value; + } + + data.bool_values[0] = true; + data.bool_values[1] = false; + + for ( int i = 0; i < uint8_count; i++ ) { data.uint8_values_data[i] = uint8_values[i]; } + for ( int i = 0; i < uint16_count; i++ ) { data.uint16_values_data[i] = uint16_values[i]; } + for ( int i = 0; i < uint32_count; i++ ) { data.uint32_values_data[i] = uint32_values[i]; } + for ( int i = 0; i < uint64_count; i++ ) { data.uint64_values_data[i] = uint64_values[i]; } + + data.uint128_values_data[0] = serialize::uint128_t( 0u ); + data.uint128_values_data[1] = ~serialize::uint128_t( 0u ); + data.uint128_values_data[2] = ( serialize::uint128_t( 0x0123456789ABCDEFULL ) << 64 ) + | serialize::uint128_t( 0x0FEDCBA987654321ULL ); + + for ( int i = 0; i < int_count; i++ ) { data.int_values[i] = int_vectors[i].value; } + for ( int i = 0; i < int64_count; i++ ) { data.int64_values[i] = int64_vectors[i].value; } + + data.int128_values[0] = int128_degenerate_value(); + data.int128_values[1] = serialize::int128_t( 5000000000LL ); + data.int128_values[2] = int128_min_value(); + data.int128_values[3] = int128_max_value(); + + data.fixed_q8_8_min = (int16_t) ( -100 * 256 ); // the bottom of the range + data.fixed_q8_8_max = (int16_t) ( 100 * 256 ); // the top + data.fixed_q16_16_degenerate = (int32_t) ( 7 * 65536 ); // min == max: zero bits + data.fixed_q48_16_min = -( (int64_t) 100000 * 65536 ); // 34 bits on the wire + data.fixed_q48_16_max = (int64_t) 100000 * 65536; + data.fixed_q112_16_max = serialize::int128_t( 144115188075855872LL ) << 16; // 75 bits, three groups + data.fixed_q64_64_degenerate = serialize::int128_t( 5 ) << 64; // zero bits at 128 bit storage + data.fixed_q64_64_max = serialize::int128_t( INT64_MAX ) << 64; // 128 bits, four groups + + for ( int i = 0; i < relative_count; i++ ) + { + data.relative_values[i] = relative_vectors[i].current; + } + + for ( int i = 0; i < float_count; i++ ) { data.float_values[i] = float_from_bits( float_bits[i] ); } + for ( int i = 0; i < double_count; i++ ) { data.double_values[i] = double_from_bits( double_bits[i] ); } + + for ( int i = 0; i < compressed_float_count; i++ ) + { + data.compressed_float_values[i] = compressed_float_vectors[i].value; + } + + data.filler = 5; + for ( int i = 0; i < bytes_count; i++ ) + { + memset( data.bytes_values[i], bytes_vectors[i].fill, (size_t) bytes_vectors[i].length ); + } + + serialize_copy_string( data.strings[0], "", string_buffer_size ); + serialize_copy_string( data.strings[1], "0123456789abcde", string_buffer_size ); // fifteen bytes: full + serialize_copy_string( data.strings[2], "\xD0\xBC\xD0\xB8\xD1\x80", string_buffer_size ); // "мир", six UTF-8 bytes + + init_wide_strings( data.wide_strings ); +} + +template bool InteropSerialize( Stream & stream, InteropData & data ) +{ + // ----- raw bit groups + for ( int i = 0; i < bits_count; i++ ) + { + serialize_bits( stream, data.bits_values[i], bits_vectors[i].bits ); + } + + // ----- bool, both codes + for ( int i = 0; i < 2; i++ ) + { + serialize_bool( stream, data.bool_values[i] ); + } + + // both sides of the alignment rule: the stream is unaligned here, so the first align + // pads, and the second must write nothing at all + serialize_align( stream ); + serialize_align( stream ); + + // ----- the fixed width unsigned helpers + for ( int i = 0; i < uint8_count; i++ ) { serialize_uint8( stream, data.uint8_values_data[i] ); } + for ( int i = 0; i < uint16_count; i++ ) { serialize_uint16( stream, data.uint16_values_data[i] ); } + for ( int i = 0; i < uint32_count; i++ ) { serialize_uint32( stream, data.uint32_values_data[i] ); } + for ( int i = 0; i < uint64_count; i++ ) { serialize_uint64( stream, data.uint64_values_data[i] ); } + for ( int i = 0; i < uint128_count; i++ ) { serialize_uint128( stream, data.uint128_values_data[i] ); } + + // ----- ranged integers + for ( int i = 0; i < int_count; i++ ) + { + serialize_int( stream, data.int_values[i], int_vectors[i].min, int_vectors[i].max ); + } + for ( int i = 0; i < int64_count; i++ ) + { + serialize_int64( stream, data.int64_values[i], int64_vectors[i].min, int64_vectors[i].max ); + } + { + const serialize::int128_t degenerate = int128_degenerate_value(); + const serialize::int128_t minimum = int128_min_value(); + const serialize::int128_t maximum = int128_max_value(); + serialize_int128( stream, data.int128_values[0], degenerate, degenerate ); + // bounds inside the 64 bit domain: the bytes are identical to serialize_int64 here + serialize_int128( stream, data.int128_values[1], serialize::int128_t( -5000000000LL ), serialize::int128_t( 5000000000LL ) ); + serialize_int128( stream, data.int128_values[2], minimum, maximum ); + serialize_int128( stream, data.int128_values[3], minimum, maximum ); + } + + // ----- fixed point, at the ends of its ranges and degenerate on two storage widths + serialize_align( stream ); + serialize_fixed( stream, data.fixed_q8_8_min, 8, 8, -100, +100 ); + serialize_fixed( stream, data.fixed_q8_8_max, 8, 8, -100, +100 ); + serialize_fixed( stream, data.fixed_q16_16_degenerate, 16, 16, 7, 7 ); + serialize_fixed( stream, data.fixed_q48_16_min, 48, 16, -100000, +100000 ); + serialize_fixed( stream, data.fixed_q48_16_max, 48, 16, -100000, +100000 ); + serialize_fixed( stream, data.fixed_q112_16_max, 112, 16, -144115188075855872LL, +144115188075855872LL ); + serialize_fixed( stream, data.fixed_q64_64_degenerate, 64, 64, 5, 5 ); + serialize_fixed( stream, data.fixed_q64_64_max, 64, 64, INT64_MIN, INT64_MAX ); + + // ----- int_relative + for ( int i = 0; i < relative_count; i++ ) + { + serialize_int_relative( stream, relative_vectors[i].previous, data.relative_values[i] ); + } + + // ----- float and double + for ( int i = 0; i < float_count; i++ ) { serialize_float( stream, data.float_values[i] ); } + for ( int i = 0; i < double_count; i++ ) { serialize_double( stream, data.double_values[i] ); } + + // ----- compressed_float + for ( int i = 0; i < compressed_float_count; i++ ) + { + serialize_compressed_float( stream, data.compressed_float_values[i], + compressed_float_vectors[i].min, + compressed_float_vectors[i].max, + compressed_float_vectors[i].res ); + } + + // ----- bytes. The three bit filler leaves the stream unaligned, so the align that + // begins the first block -- a ZERO LENGTH one -- is load bearing. + serialize_bits( stream, data.filler, 3 ); + for ( int i = 0; i < bytes_count; i++ ) + { + serialize_bytes( stream, data.bytes_values[i], bytes_vectors[i].length ); + } + + // ----- string + for ( int i = 0; i < string_count; i++ ) + { + serialize_string( stream, data.strings[i], string_buffer_size ); + } + + // ----- wstring + for ( int i = 0; i < wstring_count; i++ ) + { + serialize_wstring( stream, data.wide_strings[i], wstring_buffer_size ); + } + + return true; +} + +// What a conforming reader recovers. Everything is exact except the compressed floats, which +// are lossy by construction: the reader returns the nearest quantum, so they are compared +// within one resolution step. Floats compare by BIT PATTERN -- a value comparison cannot see +// -0.0, which is the whole point of half of this section. +static bool InteropCheck( const InteropData & data ) +{ + InteropData expected; + InteropInit( expected ); + + for ( int i = 0; i < bits_count; i++ ) + { + if ( data.bits_values[i] != expected.bits_values[i] ) { printf( "mismatch: bits[%d]\n", i ); return false; } + } + for ( int i = 0; i < 2; i++ ) + { + if ( data.bool_values[i] != expected.bool_values[i] ) { printf( "mismatch: bool[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint8_count; i++ ) + { + if ( data.uint8_values_data[i] != expected.uint8_values_data[i] ) { printf( "mismatch: uint8[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint16_count; i++ ) + { + if ( data.uint16_values_data[i] != expected.uint16_values_data[i] ) { printf( "mismatch: uint16[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint32_count; i++ ) + { + if ( data.uint32_values_data[i] != expected.uint32_values_data[i] ) { printf( "mismatch: uint32[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint64_count; i++ ) + { + if ( data.uint64_values_data[i] != expected.uint64_values_data[i] ) { printf( "mismatch: uint64[%d]\n", i ); return false; } + } + for ( int i = 0; i < uint128_count; i++ ) + { + if ( !( data.uint128_values_data[i] == expected.uint128_values_data[i] ) ) { printf( "mismatch: uint128[%d]\n", i ); return false; } + } + for ( int i = 0; i < int_count; i++ ) + { + if ( data.int_values[i] != expected.int_values[i] ) { printf( "mismatch: int[%d]\n", i ); return false; } + } + for ( int i = 0; i < int64_count; i++ ) + { + if ( data.int64_values[i] != expected.int64_values[i] ) { printf( "mismatch: int64[%d]\n", i ); return false; } + } + for ( int i = 0; i < int128_count; i++ ) + { + if ( !( data.int128_values[i] == expected.int128_values[i] ) ) { printf( "mismatch: int128[%d]\n", i ); return false; } + } + if ( data.fixed_q8_8_min != expected.fixed_q8_8_min ) { printf( "mismatch: fixed q8.8 min\n" ); return false; } + if ( data.fixed_q8_8_max != expected.fixed_q8_8_max ) { printf( "mismatch: fixed q8.8 max\n" ); return false; } + if ( data.fixed_q16_16_degenerate != expected.fixed_q16_16_degenerate ) { printf( "mismatch: fixed q16.16 degenerate\n" ); return false; } + if ( data.fixed_q48_16_min != expected.fixed_q48_16_min ) { printf( "mismatch: fixed q48.16 min\n" ); return false; } + if ( data.fixed_q48_16_max != expected.fixed_q48_16_max ) { printf( "mismatch: fixed q48.16 max\n" ); return false; } + if ( !( data.fixed_q112_16_max == expected.fixed_q112_16_max ) ) { printf( "mismatch: fixed q112.16 max\n" ); return false; } + if ( !( data.fixed_q64_64_degenerate == expected.fixed_q64_64_degenerate ) ) { printf( "mismatch: fixed q64.64 degenerate\n" ); return false; } + if ( !( data.fixed_q64_64_max == expected.fixed_q64_64_max ) ) { printf( "mismatch: fixed q64.64 max\n" ); return false; } + for ( int i = 0; i < relative_count; i++ ) + { + if ( data.relative_values[i] != expected.relative_values[i] ) { printf( "mismatch: int_relative[%d]\n", i ); return false; } + } + for ( int i = 0; i < float_count; i++ ) + { + if ( bits_from_float( data.float_values[i] ) != float_bits[i] ) { printf( "mismatch: float[%d]\n", i ); return false; } + } + for ( int i = 0; i < double_count; i++ ) + { + if ( bits_from_double( data.double_values[i] ) != double_bits[i] ) { printf( "mismatch: double[%d]\n", i ); return false; } + } + for ( int i = 0; i < compressed_float_count; i++ ) + { + const float difference = data.compressed_float_values[i] - compressed_float_vectors[i].value; + const float magnitude = difference < 0.0f ? -difference : difference; + if ( magnitude > compressed_float_vectors[i].res ) { printf( "mismatch: compressed_float[%d]\n", i ); return false; } + } + if ( data.filler != expected.filler ) { printf( "mismatch: filler\n" ); return false; } + for ( int i = 0; i < bytes_count; i++ ) + { + if ( memcmp( data.bytes_values[i], expected.bytes_values[i], (size_t) bytes_vectors[i].length ) != 0 ) { printf( "mismatch: bytes[%d]\n", i ); return false; } + } + for ( int i = 0; i < string_count; i++ ) + { + if ( strcmp( data.strings[i], expected.strings[i] ) != 0 ) { printf( "mismatch: string[%d]\n", i ); return false; } + } + for ( int i = 0; i < wstring_count; i++ ) + { + if ( wcscmp( data.wide_strings[i], expected.wide_strings[i] ) != 0 ) { printf( "mismatch: wstring[%d]\n", i ); return false; } + } + return true; +} + +// --------------------------------------------------------------------------------------- + +static const int buffer_size = 1024; + +// + 8: read buffer allocations extend eight bytes past the data, per the read allocation +// contract, and the writer shares the array so the two halves cannot disagree about size +static uint8_t write_buffer[buffer_size + 8]; +static uint8_t read_buffer[buffer_size + 8]; +static uint8_t reencode_buffer[buffer_size + 8]; + +static bool encode( uint8_t * buffer, InteropData & data, int64_t & bytes ) +{ + memset( buffer, 0, (size_t) buffer_size + 8 ); + serialize::WriteStream stream( buffer, buffer_size ); + if ( !InteropSerialize( stream, data ) ) + { + return false; + } + stream.Flush(); + bytes = stream.GetBytesProcessed(); + return true; +} + +static int Write( const char * path ) +{ + InteropData data; + InteropInit( data ); + + int64_t bytes = 0; + if ( !encode( write_buffer, data, bytes ) ) + { + fprintf( stderr, "interop cpp write: serialize failed\n" ); + return 1; + } + + FILE * file = fopen( path, "wb" ); + if ( !file ) + { + fprintf( stderr, "interop cpp write: could not open %s\n", path ); + return 1; + } + if ( fwrite( write_buffer, 1, (size_t) bytes, file ) != (size_t) bytes ) + { + fprintf( stderr, "interop cpp write: short write to %s\n", path ); + fclose( file ); + return 1; + } + fclose( file ); + + printf( "interop cpp: wrote %d bytes to %s\n", (int) bytes, path ); + return 0; +} + +static int64_t load( const char * path, uint8_t * buffer ) +{ + FILE * file = fopen( path, "rb" ); + if ( !file ) + { + fprintf( stderr, "interop cpp: could not open %s\n", path ); + return -1; + } + memset( buffer, 0, (size_t) buffer_size + 8 ); + const size_t bytes = fread( buffer, 1, (size_t) buffer_size, file ); + fclose( file ); + if ( bytes == 0 || bytes >= (size_t) buffer_size ) + { + fprintf( stderr, "interop cpp: unexpected size %d for %s\n", (int) bytes, path ); + return -1; + } + return (int64_t) bytes; +} + +static bool decode( uint8_t * buffer, int64_t bytes, InteropData & data ) +{ + memset( (void*) &data, 0, sizeof( data ) ); + serialize::ReadStream stream( buffer, bytes ); + return InteropSerialize( stream, data ); +} + +static int Read( const char * path ) +{ + const int64_t bytes = load( path, read_buffer ); + if ( bytes < 0 ) + { + return 1; + } + + InteropData data; + if ( !decode( read_buffer, bytes, data ) ) + { + fprintf( stderr, "interop cpp read: could not decode %s\n", path ); + return 1; + } + if ( !InteropCheck( data ) ) + { + fprintf( stderr, "interop cpp read: %s decoded to unexpected values\n", path ); + return 1; + } + + // re-encode what was decoded: the bytes must be identical to the input + int64_t reencoded = 0; + if ( !encode( reencode_buffer, data, reencoded ) ) + { + fprintf( stderr, "interop cpp read: re-encode failed\n" ); + return 1; + } + if ( reencoded != bytes || memcmp( reencode_buffer, read_buffer, (size_t) bytes ) != 0 ) + { + fprintf( stderr, "interop cpp read: re-encoded bytes differ from %s\n", path ); + return 1; + } + + printf( "interop cpp: decoded and re-encoded %d bytes from %s, byte identical\n", (int) bytes, path ); + return 0; +} + +// The hostile half: every proper prefix of a valid stream is a truncated stream, and a +// conforming reader refuses every one of them. Nothing here may crash, hang or accept. +static int Refuse( const char * path ) +{ + const int64_t bytes = load( path, read_buffer ); + if ( bytes < 0 ) + { + return 1; + } + + for ( int64_t length = 0; length < bytes; length++ ) + { + memset( reencode_buffer, 0, (size_t) buffer_size + 8 ); + memcpy( reencode_buffer, read_buffer, (size_t) length ); + InteropData data; + if ( decode( reencode_buffer, length, data ) ) + { + fprintf( stderr, "interop cpp refuse: the %d byte prefix of %s was ACCEPTED\n", (int) length, path ); + return 1; + } + } + + printf( "interop cpp: refused all %d truncated prefixes of %s\n", (int) bytes, path ); + return 0; +} + +int main( int argc, char ** argv ) +{ + if ( argc != 3 ) + { + fprintf( stderr, "usage: interop write|read|refuse \n" ); + return 2; + } + if ( strcmp( argv[1], "write" ) == 0 ) + { + return Write( argv[2] ); + } + if ( strcmp( argv[1], "read" ) == 0 ) + { + return Read( argv[2] ); + } + if ( strcmp( argv[1], "refuse" ) == 0 ) + { + return Refuse( argv[2] ); + } + fprintf( stderr, "usage: interop write|read|refuse \n" ); + return 2; +} diff --git a/interop/serialize/interop/Interop.java b/interop/serialize/interop/Interop.java new file mode 100644 index 0000000..771fe53 --- /dev/null +++ b/interop/serialize/interop/Interop.java @@ -0,0 +1,640 @@ +package serialize.interop; + +import serialize.BitStream; +import serialize.BoolRef; +import serialize.DoubleRef; +import serialize.FloatRef; +import serialize.Int128Value; +import serialize.IntRef; +import serialize.LongRef; +import serialize.ReadStream; +import serialize.Ref; +import serialize.UInt128Value; +import serialize.WriteStream; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * The Java half of the cross language interop harness. + * + * Its twin is {@code interop/interop.cpp}, built in CI against the real C++ + * serialize library at the release {@code .github/workflows/ci.yml} pins. The two + * halves run head to head on every push and pull request: each writes the boundary + * message, the two files must be byte identical, and each must decode the other's + * file to the exact values and re-encode it to the exact bytes. + * + * 
+ *   make interop MODE=write  FILE=/tmp/java.bin
+ *   make interop MODE=read   FILE=/tmp/cpp.bin
+ *   make interop MODE=refuse FILE=/tmp/cpp.bin
+ *
+ * + * THE MESSAGE is the boundary set: every operation STANDARD.md defines, at the + * values where implementations disagree. Zero bit ranges on all three ranged + * widths and on fixed point; the domain edges of int, int64, int128 and + * int_relative; the maximum widths of bits, uint128 and the four group fixed point + * path; both sides of the alignment rule, including the align inside a zero length + * bytes; empty and full strings; and the wide string cases the surrogate rule + * governs, up to the largest code unit. + * + * WHAT IT DELIBERATELY DOES NOT CARRY: a NaN payload. STANDARD.md's bit + * transparency claim covers it, but a NaN's payload bits do not survive every + * language's float type on the way to the wire, so a difference here would say + * nothing about the wire format. This port pins its own NaN patterns in its own + * suite, where the claim can be tested honestly. + * + * Any change to the sequence below must be mirrored in {@code interop/interop.cpp}, + * and never changes the wire format. + */ +public final class Interop +{ + private Interop() {} + + // ----------------------------------------------------------------------- + // the sequence, mirrored field for field from interop/interop.cpp + + // raw bit groups, every width boundary + private static final int[] BITS_WIDTHS = { 1, 1, 7, 31, 32, 32, 33, 64, 64 }; + private static final long[] BITS_VALUES = + { + 1, // the minimum width, at its maximum value + 0, // and at its minimum + 0x7F, // a sub-byte width, all ones + 0x7FFFFFFF, // one below the single group maximum + 0xFFFFFFFFL, // the widest single group, all ones + 0, // and all zeros + 0x1FFFFFFFFL, // the first width past the 32 bit split + 0xFFFFFFFFFFFFFFFFL, // the maximum width, all ones + 0, // and all zeros + }; + + private static final int[] UINT8_VALUES = { 0x00, 0xFF }; + private static final int[] UINT16_VALUES = { 0x0000, 0xFFFF }; + private static final int[] UINT32_VALUES = { 0x00000000, 0xFFFFFFFF }; + private static final long[] UINT64_VALUES = { 0L, 0xFFFFFFFFFFFFFFFFL }; + + private static UInt128Value[] uint128Values() + { + return new UInt128Value[] + { + UInt128Value.ZERO, + UInt128Value.ZERO.not(), // all ones + new UInt128Value( 0x0123456789ABCDEFL, 0x0FEDCBA987654321L ), + }; + } + + // ranged 32 bit integers: min, max, value + private static final int[] INT_MIN = + { 42, -100, -100, Integer.MIN_VALUE, Integer.MIN_VALUE, -100 }; + private static final int[] INT_MAX = + { 42, +100, +100, Integer.MAX_VALUE, Integer.MAX_VALUE, +100 }; + private static final int[] INT_VALUE = + { + 42, // degenerate: zero bits, mid sequence + -100, // the bottom of the range + +100, // the top of the range + Integer.MIN_VALUE, // the full domain, 32 bits on the wire + Integer.MAX_VALUE, + -37, // a live field after the degenerate one + }; + + private static final long[] INT64_MIN = + { 10000000000L, -5000000000L, -5000000000L, Long.MIN_VALUE, Long.MIN_VALUE }; + private static final long[] INT64_MAX = + { 10000000000L, +5000000000L, +5000000000L, Long.MAX_VALUE, Long.MAX_VALUE }; + private static final long[] INT64_VALUE = + { + 10000000000L, // degenerate, with bounds past 2^32 + -5000000000L, // a range wider than 32 bits, bottom + +5000000000L, // and top + Long.MIN_VALUE, // the full domain, 64 bits on the wire + Long.MAX_VALUE, + }; + + // 2^100 + 7: a degenerate bound no 64 bit path can carry + private static Int128Value int128Degenerate() + { + return Int128Value.fromLong( 1 ).shiftLeft( 100 ).add( Int128Value.fromLong( 7 ) ); + } + + private static Int128Value[] int128Bounds( boolean minimum ) + { + Int128Value degenerate = int128Degenerate(); + return new Int128Value[] + { + degenerate, + // bounds inside the 64 bit domain: the bytes are identical to serializeInt64 here + Int128Value.fromLong( minimum ? -5000000000L : +5000000000L ), + minimum ? Int128Value.MIN_VALUE : Int128Value.MAX_VALUE, + minimum ? Int128Value.MIN_VALUE : Int128Value.MAX_VALUE, + }; + } + + private static Int128Value[] int128Values() + { + return new Int128Value[] + { + int128Degenerate(), + Int128Value.fromLong( 5000000000L ), + Int128Value.MIN_VALUE, + Int128Value.MAX_VALUE, + }; + } + + // int_relative: every tier at both ends, and the domain edges + private static final int[] RELATIVE_PREVIOUS = + { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2147483646, 0 }; + private static final int[] RELATIVE_CURRENT = + { + 1, // one-bit + 2, 6, // bounded-3, both ends + 7, 23, // bounded-5 + 24, 280, // bounded-9 + 281, 4377, // bounded-13 + 4378, 69914, // bounded-17 + 69915, // absolute, at its smallest difference + 2147483647, // one-bit, at the top of the domain + 2147483647, // absolute, at the top of the domain + }; + + // floats are given as bit patterns, so no decimal literal is parsed twice + private static final int[] FLOAT_BITS = + { + 0x00000000, // +0 + 0x80000000, // -0 + 0x7F800000, // +infinity + 0xFF800000, // -infinity + 0x7F7FFFFF, // the largest finite float32 + 0x00800000, // the smallest normal + 0x00000001, // the smallest subnormal + 0x3F800000, // 1.0f + 0xBF800000, // -1.0f + }; + + private static final long[] DOUBLE_BITS = + { + 0x0000000000000000L, // +0 + 0x8000000000000000L, // -0 + 0x7FF0000000000000L, // +infinity + 0xFFF0000000000000L, // -infinity + 0x7FEFFFFFFFFFFFFFL, // the largest finite float64 + 0x0010000000000000L, // the smallest normal + 0x0000000000000001L, // the smallest subnormal + 0x3FF0000000000000L, // 1.0 + 0xBFF0000000000000L, // -1.0 + }; + + private static final float[] CF_VALUE = + { + 0.0f, // the bottom of the range: integer 0 + 10.0f, // the top: the maximum integer + 0.005f, // between quanta: 1 under float32, 0 widened + 0.025f, // between quanta: 3 vs 2 + 0.105f, // between quanta: 11 vs 10 + 9.995f, // between quanta: 1000 vs 999 + -100.0f, // the bottom of a range with a non-zero min + -42.573f, // off quantum over a non-zero min + 8388609.0f, // clamp witness A (schema#109) + 16777215.0f, // clamp witness B + 0.0f, // a one bit field, both codes + 1.0f, + }; + private static final float[] CF_MIN = + { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, -100.0f, -100.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + private static final float[] CF_MAX = + { 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 10.0f, 100.0f, 100.0f, 8388609.0f, 16777215.0f, 1.0f, 1.0f }; + private static final float[] CF_RES = + { 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 0.01f, 1.0f, 1.0f, 1.0f, 1.0f }; + + // a zero length block first: the align happens anyway + private static final int[] BYTES_LENGTH = { 0, 8, 8, 1 }; + private static final int[] BYTES_FILL = { 0x00, 0x00, 0xFF, 0x5A }; + + private static final int STRING_BUFFER_SIZE = 16; + private static final String[] STRINGS = + { + "", // empty + "0123456789abcde", // fifteen bytes: the most buffer size 16 carries + "\u043C\u0438\u0440", // six UTF-8 bytes, as explicit code points so no + // source file encoding can reach the wire + }; + + private static final int WSTRING_BUFFER_SIZE = 8; + private static final String[] WIDE_STRINGS = + { + "", // empty + "\u043C\u0438\u0440", // basic plane + "\uE000", // the first code unit above the surrogate block + "\uFFFF", // the largest code unit there is + "A\uD83D\uDE00B", // U+1F600 as its surrogate pair: four code units + "abcdefg", // seven code units, the most buffer size 8 carries + }; + + // ----------------------------------------------------------------------- + // the message + + /** + * Holders for every field. The raw bit groups keep both a narrow and a wide + * holder per entry, because the port splits raw bits at 32 where the wire does + * not; only the one the width selects is ever used. + */ + private static final class Data + { + final IntRef[] bitsNarrow = new IntRef[BITS_WIDTHS.length]; + final LongRef[] bitsWide = new LongRef[BITS_WIDTHS.length]; + final BoolRef[] bools = { new BoolRef(), new BoolRef() }; + final IntRef[] uint8 = new IntRef[UINT8_VALUES.length]; + final IntRef[] uint16 = new IntRef[UINT16_VALUES.length]; + final IntRef[] uint32 = new IntRef[UINT32_VALUES.length]; + final LongRef[] uint64 = new LongRef[UINT64_VALUES.length]; + final List> uint128 = new ArrayList<>(); + final IntRef[] ints = new IntRef[INT_VALUE.length]; + final LongRef[] int64s = new LongRef[INT64_VALUE.length]; + final List> int128s = new ArrayList<>(); + final LongRef fixedQ8_8Min = new LongRef(); + final LongRef fixedQ8_8Max = new LongRef(); + final LongRef fixedQ16_16Degenerate = new LongRef(); + final LongRef fixedQ48_16Min = new LongRef(); + final LongRef fixedQ48_16Max = new LongRef(); + final Ref fixedQ112_16Max = new Ref<>( Int128Value.ZERO ); + final Ref fixedQ64_64Degenerate = new Ref<>( Int128Value.ZERO ); + final Ref fixedQ64_64Max = new Ref<>( Int128Value.ZERO ); + final IntRef[] relative = new IntRef[RELATIVE_CURRENT.length]; + final FloatRef[] floats = new FloatRef[FLOAT_BITS.length]; + final DoubleRef[] doubles = new DoubleRef[DOUBLE_BITS.length]; + final FloatRef[] compressedFloats = new FloatRef[CF_VALUE.length]; + final IntRef filler = new IntRef(); + final byte[][] bytes = new byte[BYTES_LENGTH.length][]; + final List> strings = new ArrayList<>(); + final List> wideStrings = new ArrayList<>(); + + Data() + { + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + bitsNarrow[i] = new IntRef(); + bitsWide[i] = new LongRef(); + } + for ( int i = 0; i < UINT8_VALUES.length; i++ ) { uint8[i] = new IntRef(); } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) { uint16[i] = new IntRef(); } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) { uint32[i] = new IntRef(); } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) { uint64[i] = new LongRef(); } + for ( int i = 0; i < 3; i++ ) { uint128.add( new Ref<>( UInt128Value.ZERO ) ); } + for ( int i = 0; i < INT_VALUE.length; i++ ) { ints[i] = new IntRef(); } + for ( int i = 0; i < INT64_VALUE.length; i++ ) { int64s[i] = new LongRef(); } + for ( int i = 0; i < 4; i++ ) { int128s.add( new Ref<>( Int128Value.ZERO ) ); } + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) { relative[i] = new IntRef(); } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) { floats[i] = new FloatRef(); } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) { doubles[i] = new DoubleRef(); } + for ( int i = 0; i < CF_VALUE.length; i++ ) { compressedFloats[i] = new FloatRef(); } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) { bytes[i] = new byte[BYTES_LENGTH[i]]; } + for ( int i = 0; i < STRINGS.length; i++ ) { strings.add( new Ref<>( "" ) ); } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) { wideStrings.add( new Ref<>( "" ) ); } + } + } + + /** The write side: every holder carries the value the sequence pins. */ + private static Data boundaryData() + { + Data data = new Data(); + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + data.bitsNarrow[i].value = (int) BITS_VALUES[i]; + data.bitsWide[i].value = BITS_VALUES[i]; + } + data.bools[0].value = true; + data.bools[1].value = false; + for ( int i = 0; i < UINT8_VALUES.length; i++ ) { data.uint8[i].value = UINT8_VALUES[i]; } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) { data.uint16[i].value = UINT16_VALUES[i]; } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) { data.uint32[i].value = UINT32_VALUES[i]; } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) { data.uint64[i].value = UINT64_VALUES[i]; } + UInt128Value[] unsigned = uint128Values(); + for ( int i = 0; i < unsigned.length; i++ ) { data.uint128.get( i ).value = unsigned[i]; } + for ( int i = 0; i < INT_VALUE.length; i++ ) { data.ints[i].value = INT_VALUE[i]; } + for ( int i = 0; i < INT64_VALUE.length; i++ ) { data.int64s[i].value = INT64_VALUE[i]; } + Int128Value[] signed = int128Values(); + for ( int i = 0; i < signed.length; i++ ) { data.int128s.get( i ).value = signed[i]; } + data.fixedQ8_8Min.value = -100L * 256; // the bottom of the range + data.fixedQ8_8Max.value = 100L * 256; // the top + data.fixedQ16_16Degenerate.value = 7L * 65536; // min == max: zero bits + data.fixedQ48_16Min.value = -100000L * 65536; // 34 bits on the wire + data.fixedQ48_16Max.value = 100000L * 65536; + data.fixedQ112_16Max.value = Int128Value.fromLong( 144115188075855872L ).shiftLeft( 16 ); + data.fixedQ64_64Degenerate.value = Int128Value.fromLong( 5 ).shiftLeft( 64 ); + data.fixedQ64_64Max.value = Int128Value.fromLong( Long.MAX_VALUE ).shiftLeft( 64 ); + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) { data.relative[i].value = RELATIVE_CURRENT[i]; } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) { data.floats[i].value = Float.intBitsToFloat( FLOAT_BITS[i] ); } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) { data.doubles[i].value = Double.longBitsToDouble( DOUBLE_BITS[i] ); } + for ( int i = 0; i < CF_VALUE.length; i++ ) { data.compressedFloats[i].value = CF_VALUE[i]; } + data.filler.value = 5; + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) { Arrays.fill( data.bytes[i], (byte) BYTES_FILL[i] ); } + for ( int i = 0; i < STRINGS.length; i++ ) { data.strings.get( i ).value = STRINGS[i]; } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) { data.wideStrings.get( i ).value = WIDE_STRINGS[i]; } + return data; + } + + /** The message, operation for operation. The chain stops at the first refusal. */ + private static boolean interopSerialize( BitStream stream, Data d ) + { + // ----- raw bit groups + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + int width = BITS_WIDTHS[i]; + boolean ok = width <= 32 ? stream.serializeBits( d.bitsNarrow[i], width ) + : stream.serializeBits64( d.bitsWide[i], width ); + if ( !ok ) { return false; } + } + + // ----- bool, both codes + for ( BoolRef ref : d.bools ) + { + if ( !stream.serializeBool( ref ) ) { return false; } + } + + // both sides of the alignment rule: the stream is unaligned here, so the + // first align pads and the second must write nothing at all + if ( !stream.serializeAlign() ) { return false; } + if ( !stream.serializeAlign() ) { return false; } + + // ----- the fixed width unsigned helpers, at their domain edges + for ( IntRef ref : d.uint8 ) { if ( !stream.serializeUint8( ref ) ) { return false; } } + for ( IntRef ref : d.uint16 ) { if ( !stream.serializeUint16( ref ) ) { return false; } } + for ( IntRef ref : d.uint32 ) { if ( !stream.serializeUint32( ref ) ) { return false; } } + for ( LongRef ref : d.uint64 ) { if ( !stream.serializeUint64( ref ) ) { return false; } } + for ( Ref ref : d.uint128 ) { if ( !stream.serializeUint128( ref ) ) { return false; } } + + // ----- ranged integers + for ( int i = 0; i < INT_VALUE.length; i++ ) + { + if ( !stream.serializeInt( d.ints[i], INT_MIN[i], INT_MAX[i] ) ) { return false; } + } + for ( int i = 0; i < INT64_VALUE.length; i++ ) + { + if ( !stream.serializeInt64( d.int64s[i], INT64_MIN[i], INT64_MAX[i] ) ) { return false; } + } + Int128Value[] lower = int128Bounds( true ); + Int128Value[] upper = int128Bounds( false ); + for ( int i = 0; i < lower.length; i++ ) + { + if ( !stream.serializeInt128( d.int128s.get( i ), lower[i], upper[i] ) ) { return false; } + } + + // ----- fixed point, at the ends of its ranges and degenerate on two storage widths + if ( !stream.serializeAlign() ) { return false; } + if ( !stream.serializeFixed( d.fixedQ8_8Min, 8, 8, -100, +100 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ8_8Max, 8, 8, -100, +100 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ16_16Degenerate, 16, 16, 7, 7 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ48_16Min, 48, 16, -100000, +100000 ) ) { return false; } + if ( !stream.serializeFixed( d.fixedQ48_16Max, 48, 16, -100000, +100000 ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ112_16Max, 112, 16, -144115188075855872L, +144115188075855872L ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ64_64Degenerate, 64, 64, 5, 5 ) ) { return false; } + if ( !stream.serializeFixed128( d.fixedQ64_64Max, 64, 64, Long.MIN_VALUE, Long.MAX_VALUE ) ) { return false; } + + // ----- int_relative + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) + { + if ( !stream.serializeIntRelative( RELATIVE_PREVIOUS[i], d.relative[i] ) ) { return false; } + } + + // ----- float and double, bit transparent at the domain edges + for ( FloatRef ref : d.floats ) { if ( !stream.serializeFloat( ref ) ) { return false; } } + for ( DoubleRef ref : d.doubles ) { if ( !stream.serializeDouble( ref ) ) { return false; } } + + // ----- compressed_float + for ( int i = 0; i < CF_VALUE.length; i++ ) + { + if ( !stream.serializeCompressedFloat( d.compressedFloats[i], CF_MIN[i], CF_MAX[i], CF_RES[i] ) ) { return false; } + } + + // ----- bytes. The three bit filler leaves the stream unaligned, so the + // align that begins the first block -- a ZERO LENGTH one -- is load bearing. + if ( !stream.serializeBits( d.filler, 3 ) ) { return false; } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) + { + if ( !stream.serializeBytes( d.bytes[i], BYTES_LENGTH[i] ) ) { return false; } + } + + // ----- string: empty, full, and multi-byte UTF-8 + for ( Ref ref : d.strings ) + { + if ( !stream.serializeString( ref, STRING_BUFFER_SIZE ) ) { return false; } + } + + // ----- wstring: empty, basic plane, the code unit boundaries, a pair, full + for ( Ref ref : d.wideStrings ) + { + if ( !stream.serializeWideString( ref, WSTRING_BUFFER_SIZE ) ) { return false; } + } + + return true; + } + + /** + * What a conforming reader recovers. Everything is exact except the compressed + * floats, which are lossy by construction: the reader returns the nearest + * quantum, so they are compared within one resolution step. Floats compare by + * RAW BIT PATTERN -- a value comparison cannot see -0.0. + */ + private static List interopCheck( Data d ) + { + List problems = new ArrayList<>(); + Data expected = boundaryData(); + + for ( int i = 0; i < BITS_WIDTHS.length; i++ ) + { + long actual = BITS_WIDTHS[i] <= 32 ? ( d.bitsNarrow[i].value & 0xFFFFFFFFL ) : d.bitsWide[i].value; + long want = BITS_WIDTHS[i] <= 32 ? ( (int) BITS_VALUES[i] & 0xFFFFFFFFL ) : BITS_VALUES[i]; + if ( actual != want ) { problems.add( "bits[" + i + "]: got " + actual + ", expected " + want ); } + } + if ( !d.bools[0].value ) { problems.add( "bool[0]" ); } + if ( d.bools[1].value ) { problems.add( "bool[1]" ); } + for ( int i = 0; i < UINT8_VALUES.length; i++ ) + { + if ( d.uint8[i].value != UINT8_VALUES[i] ) { problems.add( "uint8[" + i + "]" ); } + } + for ( int i = 0; i < UINT16_VALUES.length; i++ ) + { + if ( d.uint16[i].value != UINT16_VALUES[i] ) { problems.add( "uint16[" + i + "]" ); } + } + for ( int i = 0; i < UINT32_VALUES.length; i++ ) + { + if ( d.uint32[i].value != UINT32_VALUES[i] ) { problems.add( "uint32[" + i + "]" ); } + } + for ( int i = 0; i < UINT64_VALUES.length; i++ ) + { + if ( d.uint64[i].value != UINT64_VALUES[i] ) { problems.add( "uint64[" + i + "]" ); } + } + for ( int i = 0; i < d.uint128.size(); i++ ) + { + if ( !d.uint128.get( i ).value.equals( expected.uint128.get( i ).value ) ) { problems.add( "uint128[" + i + "]" ); } + } + for ( int i = 0; i < INT_VALUE.length; i++ ) + { + if ( d.ints[i].value != INT_VALUE[i] ) { problems.add( "int[" + i + "]" ); } + } + for ( int i = 0; i < INT64_VALUE.length; i++ ) + { + if ( d.int64s[i].value != INT64_VALUE[i] ) { problems.add( "int64[" + i + "]" ); } + } + for ( int i = 0; i < d.int128s.size(); i++ ) + { + if ( !d.int128s.get( i ).value.equals( expected.int128s.get( i ).value ) ) { problems.add( "int128[" + i + "]" ); } + } + if ( d.fixedQ8_8Min.value != expected.fixedQ8_8Min.value ) { problems.add( "fixed q8.8 min" ); } + if ( d.fixedQ8_8Max.value != expected.fixedQ8_8Max.value ) { problems.add( "fixed q8.8 max" ); } + if ( d.fixedQ16_16Degenerate.value != expected.fixedQ16_16Degenerate.value ) { problems.add( "fixed q16.16 degenerate" ); } + if ( d.fixedQ48_16Min.value != expected.fixedQ48_16Min.value ) { problems.add( "fixed q48.16 min" ); } + if ( d.fixedQ48_16Max.value != expected.fixedQ48_16Max.value ) { problems.add( "fixed q48.16 max" ); } + if ( !d.fixedQ112_16Max.value.equals( expected.fixedQ112_16Max.value ) ) { problems.add( "fixed q112.16 max" ); } + if ( !d.fixedQ64_64Degenerate.value.equals( expected.fixedQ64_64Degenerate.value ) ) { problems.add( "fixed q64.64 degenerate" ); } + if ( !d.fixedQ64_64Max.value.equals( expected.fixedQ64_64Max.value ) ) { problems.add( "fixed q64.64 max" ); } + if ( d.filler.value != expected.filler.value ) { problems.add( "filler" ); } + for ( int i = 0; i < RELATIVE_CURRENT.length; i++ ) + { + if ( d.relative[i].value != RELATIVE_CURRENT[i] ) { problems.add( "int_relative[" + i + "]" ); } + } + for ( int i = 0; i < FLOAT_BITS.length; i++ ) + { + if ( Float.floatToRawIntBits( d.floats[i].value ) != FLOAT_BITS[i] ) { problems.add( "float[" + i + "]" ); } + } + for ( int i = 0; i < DOUBLE_BITS.length; i++ ) + { + if ( Double.doubleToRawLongBits( d.doubles[i].value ) != DOUBLE_BITS[i] ) { problems.add( "double[" + i + "]" ); } + } + for ( int i = 0; i < CF_VALUE.length; i++ ) + { + if ( !( Math.abs( d.compressedFloats[i].value - CF_VALUE[i] ) <= CF_RES[i] ) ) + { + problems.add( "compressed_float[" + i + "]: got " + d.compressedFloats[i].value ); + } + } + for ( int i = 0; i < BYTES_LENGTH.length; i++ ) + { + if ( !Arrays.equals( d.bytes[i], expected.bytes[i] ) ) { problems.add( "bytes[" + i + "]" ); } + } + for ( int i = 0; i < STRINGS.length; i++ ) + { + if ( !STRINGS[i].equals( d.strings.get( i ).value ) ) { problems.add( "string[" + i + "]: got " + d.strings.get( i ).value ); } + } + for ( int i = 0; i < WIDE_STRINGS.length; i++ ) + { + if ( !WIDE_STRINGS[i].equals( d.wideStrings.get( i ).value ) ) { problems.add( "wstring[" + i + "]" ); } + } + return problems; + } + + // ----------------------------------------------------------------------- + // the three modes + + private static final int BUFFER_BYTES = 1024; + + /** + * The read allocation contract: this port's reader requires the array to extend + * eight bytes past the data (BitReader). The slack is zero filled, so nothing in + * it can reach a decoded value -- STANDARD.md ruled that bytes past the stream + * end are never interpreted, and a truncated prefix must be refused for the bits + * it does not have, not accepted from what follows it. + */ + private static byte[] withSlack( byte[] input, int length ) + { + byte[] buffer = new byte[length + 8]; + System.arraycopy( input, 0, buffer, 0, length ); + return buffer; + } + + private static byte[] encode( Data data ) + { + WriteStream stream = new WriteStream( new byte[BUFFER_BYTES], BUFFER_BYTES ); + if ( !interopSerialize( stream, data ) ) + { + throw new IllegalStateException( "interop java: serialize failed" ); + } + stream.flush(); + return Arrays.copyOf( stream.getData(), (int) stream.getBytesProcessed() ); + } + + private static void write( String path ) throws IOException + { + byte[] bytes = encode( boundaryData() ); + Files.write( Path.of( path ), bytes ); + System.out.println( "interop java: wrote " + bytes.length + " bytes to " + path ); + } + + private static void read( String path ) throws IOException + { + byte[] input = Files.readAllBytes( Path.of( path ) ); + Data data = new Data(); + if ( !interopSerialize( new ReadStream( withSlack( input, input.length ), input.length ), data ) ) + { + throw new IllegalStateException( "interop java: could not decode " + path ); + } + List problems = interopCheck( data ); + if ( !problems.isEmpty() ) + { + throw new IllegalStateException( "interop java: " + path + " decoded to unexpected values: " + problems ); + } + // re-encode what was decoded: the bytes must be identical to the input + if ( !Arrays.equals( encode( data ), input ) ) + { + throw new IllegalStateException( "interop java: re-encoded bytes differ from " + path ); + } + System.out.println( "interop java: decoded and re-encoded " + input.length + " bytes from " + path + ", byte identical" ); + } + + /** + * The hostile half: every proper prefix of a valid stream is a truncated + * stream, and a conforming reader refuses every one of them without throwing. + */ + private static void refuse( String path ) throws IOException + { + byte[] input = Files.readAllBytes( Path.of( path ) ); + for ( int length = 0; length < input.length; length++ ) + { + byte[] truncated = withSlack( input, length ); + boolean accepted; + try + { + accepted = interopSerialize( new ReadStream( truncated, length ), new Data() ); + } + catch ( RuntimeException error ) + { + throw new IllegalStateException( "interop java refuse: the " + length + " byte prefix of " + path + " THREW", error ); + } + if ( accepted ) + { + throw new IllegalStateException( "interop java refuse: the " + length + " byte prefix of " + path + " was ACCEPTED" ); + } + } + System.out.println( "interop java: refused all " + input.length + " truncated prefixes of " + path ); + } + + public static void main( String[] args ) throws IOException + { + if ( args.length != 2 ) + { + System.err.println( "usage: Interop write|read|refuse " ); + System.exit( 2 ); + } + switch ( args[0] ) + { + case "write": + write( args[1] ); + break; + case "read": + read( args[1] ); + break; + case "refuse": + refuse( args[1] ); + break; + default: + System.err.println( "usage: Interop write|read|refuse " ); + System.exit( 2 ); + break; + } + } +} diff --git a/src/serialize/SerializeUtil.java b/src/serialize/SerializeUtil.java index 51ebcb7..a2e2f85 100644 --- a/src/serialize/SerializeUtil.java +++ b/src/serialize/SerializeUtil.java @@ -11,7 +11,7 @@ public final class SerializeUtil private SerializeUtil() {} /** The library version, matching the release tag. */ - public static final String VERSION = "1.1.0"; + public static final String VERSION = "1.1.1"; /** * The number of bits required to serialize an integer in [min,max]. diff --git a/test/serialize/tests/ConformanceTests.java b/test/serialize/tests/ConformanceTests.java index 7410780..0178be3 100644 --- a/test/serialize/tests/ConformanceTests.java +++ b/test/serialize/tests/ConformanceTests.java @@ -38,7 +38,7 @@ * below fails the suite rather than being skipped: a vector that does not * run is a vector that proves nothing. */ -final class ConformanceTests +public final class ConformanceTests { private ConformanceTests() {} @@ -53,6 +53,17 @@ private ConformanceTests() {} private static final BigInteger MASK_128 = BigInteger.ONE.shiftLeft( 128 ).subtract( BigInteger.ONE ); + /** + * Runs this suite on its own ({@code java serialize.tests.ConformanceTests}), so + * the interop job can hold this reader and the pinned C++ reader to the same + * corpus in one place. AllTests calls {@link #run} directly. + */ + public static void main( String[] args ) + { + run(); + System.exit( Harness.finish() ); + } + static void run() { List files = corpusFiles();