From 8e567ec582ba76f5694ed3a0f37d21a55490b927 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 4 Nov 2026 09:58:37 +0800 Subject: [PATCH] Fix C++ cross-language codecs and expand compatibility matrix to four scales Close the Java/C++ compatibility gaps so every Java-writable (type, encoding, compression) combination is readable by C++, and expand the shared fixture matrix to four data-scale tiers with page-size coverage. - Add C++ FLOAT/DOUBLE RLE and GORILLA_V1 encoders/decoders. - Wire DATE/TIMESTAMP into ZIGZAG/SPRINTZ and BOOLEAN into RLE. - Fix INT64 RLE decoder endianness: Java stores run values big-endian. - Fix SPRINTZ FIR predictor to match Java's 32/64-bit wrapping multiply. - Replace the table-only Java/C++ tests with a spec-driven EncodingCompressionCompatibilityTest consumed by both sides from compat/compatibility-matrix.csv; cover tree + table models, four scales (10/1k/100k/1M rows), and page-boundary sizes. - Narrow the compatibility workflow trigger to the binary-format surface and report the coverage/gap matrix in CI. --- .github/workflows/compatibility-test.yml | 82 +- compat/README.md | 130 ++ compat/compatibility-matrix.csv | 625 ++++++ cpp/src/encoding/decoder_factory.h | 20 + cpp/src/encoding/encoder_factory.h | 22 +- cpp/src/encoding/fire.h | 38 +- cpp/src/encoding/float_rle_decoder.h | 224 ++ cpp/src/encoding/float_rle_encoder.h | 321 +++ cpp/src/encoding/gorilla_v1_decoder.h | 289 +++ cpp/src/encoding/gorilla_v1_encoder.h | 276 +++ cpp/src/encoding/int64_rle_decoder.h | 8 +- ...encoding_compression_compatibility_test.cc | 1918 +++++++++++++++++ ...encoding_compression_compatibility_test.cc | 863 -------- .../EncodingCompressionCompatibilityTest.java | 1632 ++++++++++++++ ...lEncodingCompressionCompatibilityTest.java | 801 ------- 15 files changed, 5546 insertions(+), 1703 deletions(-) create mode 100644 compat/README.md create mode 100644 compat/compatibility-matrix.csv create mode 100644 cpp/src/encoding/float_rle_decoder.h create mode 100644 cpp/src/encoding/float_rle_encoder.h create mode 100644 cpp/src/encoding/gorilla_v1_decoder.h create mode 100644 cpp/src/encoding/gorilla_v1_encoder.h create mode 100644 cpp/test/reader/table_view/encoding_compression_compatibility_test.cc delete mode 100644 cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc create mode 100644 java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java delete mode 100644 java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java diff --git a/.github/workflows/compatibility-test.yml b/.github/workflows/compatibility-test.yml index 58a0cd7761..6210602dd1 100644 --- a/.github/workflows/compatibility-test.yml +++ b/.github/workflows/compatibility-test.yml @@ -26,10 +26,19 @@ on: paths: - '.github/workflows/compatibility-test.yml' - '.mvn/**' - - 'cpp/**' - - 'java/**' - 'mvnw' - 'pom.xml' + # The binary-format surface: the Java/C++ sources that define how data + # is encoded, compressed, and laid out on disk. Test/example/tool/doc + # changes do not affect the format and are deliberately excluded so a + # routine PR does not pay the ~30 minute cross-language matrix. + - 'compat/**' + - 'java/common/src/main/**' + - 'java/tsfile/src/main/**' + - 'cpp/src/**' + - 'cpp/CMakeLists.txt' + - 'cpp/cmake/**' + - 'cpp/third_party/**' pull_request: branches: - develop @@ -39,10 +48,19 @@ on: paths: - '.github/workflows/compatibility-test.yml' - '.mvn/**' - - 'cpp/**' - - 'java/**' - 'mvnw' - 'pom.xml' + # The binary-format surface: the Java/C++ sources that define how data + # is encoded, compressed, and laid out on disk. Test/example/tool/doc + # changes do not affect the format and are deliberately excluded so a + # routine PR does not pay the ~30 minute cross-language matrix. + - 'compat/**' + - 'java/common/src/main/**' + - 'java/tsfile/src/main/**' + - 'cpp/src/**' + - 'cpp/CMakeLists.txt' + - 'cpp/cmake/**' + - 'cpp/third_party/**' workflow_dispatch: inputs: forceUpdates: @@ -94,17 +112,26 @@ jobs: path: cpp/target/dependency-cache key: ${{ runner.os }}-tsfile-cpp-deps-${{ hashFiles('cpp/CMakeLists.txt', 'cpp/cmake/*Source.cmake', 'cpp/third_party/CMakeLists.txt') }} - - name: Generate Java table fixtures + - name: Generate Java compatibility fixtures if: matrix.language == 'java' shell: bash run: | mkdir -p compat-fixtures/java ./mvnw -P with-java \ - -Dtest=TableModelEncodingCompressionCompatibilityTest#generateFixtures \ + -Dtest=EncodingCompressionCompatibilityTest#generateFixtures \ -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtsfile.compat.spec="$PWD/compat/compatibility-matrix.csv" \ -Dtsfile.compat.generate.dir="$PWD/compat-fixtures/java" \ test + - name: Publish Java generation summary + if: matrix.language == 'java' && always() + shell: bash + run: | + if [ -f compat-fixtures/java/summary.md ]; then + cat compat-fixtures/java/summary.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Build C++ tests if: matrix.language == 'cpp' shell: bash @@ -115,23 +142,32 @@ jobs: -Dspotless.skip=true \ -Dtsfile.dependency.source=BUNDLED \ -Denable.antlr4=OFF \ - -Denable.snappy=OFF \ + -Denable.snappy=ON \ -Denable.lz4=ON \ -Denable.lzokay=OFF \ - -Denable.zlib=OFF \ + -Denable.zlib=ON \ -Denable.zstd=ON \ -Denable.lzma2=ON \ -Denable.simd=OFF \ package - - name: Generate C++ table fixtures + - name: Generate C++ compatibility fixtures if: matrix.language == 'cpp' shell: bash run: | mkdir -p compat-fixtures/cpp + TSFILE_COMPAT_SPEC="$PWD/compat/compatibility-matrix.csv" \ TSFILE_COMPAT_GENERATE_DIR="$PWD/compat-fixtures/cpp" \ ./cpp/target/build/test/lib/TsFile_Test \ - --gtest_filter=TableModelEncodingCompressionCompatibilityTest.GenerateFixtures + --gtest_filter=EncodingCompressionCompatibilityTest.GenerateFixtures + + - name: Publish C++ generation summary + if: matrix.language == 'cpp' && always() + shell: bash + run: | + if [ -f compat-fixtures/cpp/summary.md ]; then + cat compat-fixtures/cpp/summary.md >> "$GITHUB_STEP_SUMMARY" + fi - name: Upload ${{ matrix.language }} fixtures uses: actions/upload-artifact@v7 @@ -189,11 +225,20 @@ jobs: shell: bash run: | ./mvnw -P with-java \ - -Dtest=TableModelEncodingCompressionCompatibilityTest#validateFixtures \ + -Dtest=EncodingCompressionCompatibilityTest#validateFixtures \ -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtsfile.compat.spec="$PWD/compat/compatibility-matrix.csv" \ -Dtsfile.compat.validate.dir="$PWD/compat-fixtures/${{ matrix.writer }}" \ test + - name: Publish Java validation summary + if: matrix.reader == 'java' && always() + shell: bash + run: | + if [ -f compat-fixtures/${{ matrix.writer }}/summary.md.validate ]; then + cat compat-fixtures/${{ matrix.writer }}/summary.md.validate >> "$GITHUB_STEP_SUMMARY" + fi + - name: Build C++ tests if: matrix.reader == 'cpp' shell: bash @@ -204,10 +249,10 @@ jobs: -Dspotless.skip=true \ -Dtsfile.dependency.source=BUNDLED \ -Denable.antlr4=OFF \ - -Denable.snappy=OFF \ + -Denable.snappy=ON \ -Denable.lz4=ON \ -Denable.lzokay=OFF \ - -Denable.zlib=OFF \ + -Denable.zlib=ON \ -Denable.zstd=ON \ -Denable.lzma2=ON \ -Denable.simd=OFF \ @@ -217,6 +262,15 @@ jobs: if: matrix.reader == 'cpp' shell: bash run: | + TSFILE_COMPAT_SPEC="$PWD/compat/compatibility-matrix.csv" \ TSFILE_COMPAT_VALIDATE_DIR="$PWD/compat-fixtures/${{ matrix.writer }}" \ ./cpp/target/build/test/lib/TsFile_Test \ - --gtest_filter=TableModelEncodingCompressionCompatibilityTest.ValidateFixtures + --gtest_filter=EncodingCompressionCompatibilityTest.ValidateFixtures + + - name: Publish C++ validation summary + if: matrix.reader == 'cpp' && always() + shell: bash + run: | + if [ -f compat-fixtures/${{ matrix.writer }}/summary.md.validate ]; then + cat compat-fixtures/${{ matrix.writer }}/summary.md.validate >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/compat/README.md b/compat/README.md new file mode 100644 index 0000000000..56dd3006dc --- /dev/null +++ b/compat/README.md @@ -0,0 +1,130 @@ + + +# Java / C++ cross-language compatibility fixtures + +`compatibility-matrix.csv` is the single source of truth for the fixtures that +the Java and C++ implementations write for each other. Both test suites read +this file: + +- Java: `java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java` +- C++: `cpp/test/reader/table_view/encoding_compression_compatibility_test.cc` + +Each side writes every case it declares writable, reads back its own fixtures, +and validates fixtures produced by the other language. The CI workflow +`.github/workflows/compatibility-test.yml` runs generation in one job per +language and cross-validation in both directions, then publishes the coverage +summary. + +## Columns + +| Column | Meaning | +| --- | --- | +| `id` | Stable case id. Expanded cases get `.` + compression + `.` + null pattern. The fixture file is `.tsfile`. | +| `model` | `table` (Tag/Field table model) or `tree` (device/measurement model). | +| `dataType` | `BOOLEAN`, `INT32`, `INT64`, `TIMESTAMP`, `DATE`, `FLOAT`, `DOUBLE`, `TEXT`, `STRING`, `BLOB`. | +| `encoding` | `TSEncoding` enum name. | +| `compressions` | `ALL` (the six codecs both projects ship: UNCOMPRESSED+SNAPPY+GZIP+LZ4+ZSTD+LZMA2) or a `+`-separated subset. | +| `rowCount` | Number of timestamps per device. | +| `pageRows` | Points per page; `0` keeps the writer default. | +| `chunks` | Number of chunks to produce by flushing between sub-batches. | +| `devices` | Number of devices written into one file (one chunk group per device). | +| `valueProfile` | Deterministic value set, see below. | +| `nullPatterns` | `|`-separated `NONE`, `MIXED`, `ALL`. | +| `javaWrite`/`cppWrite` | Whether that language writes the fixture. | +| `javaRead`/`cppRead` | Whether that language validates the fixture when the other side wrote it. | +| `gapReason` | Required whenever any write/read cell is `no`; the reason is reported in every CI run. | + +A case with a `no` in a write column is an explicit declared gap. A case absent +from the file is not a gap: the completeness test in both suites enumerates the +cartesian product of every data type and encoding the local writer supports and +fails when the spec does not cover it. That is what keeps the matrix complete +as new encodings are added. + +## Data scale and page-size tiers + +Every (type, encoding, compression) combination is generated at four data +scales — 10, 1000, 100000 and 1000000 rows — with a page size that crosses +page boundaries at that scale: + +| Scale | Page size | Page structure it exercises | +| --- | --- | --- | +| 10 rows | 4 | 3 pages with a partial last page (4+4+2) | +| 1000 rows | 100 | 10 pages | +| 100000 rows | 10000 | 10 pages (writer default) | +| 1000000 rows | 10000 | 100 pages (writer default) | + +`UNCOMPRESSED` is dropped at the 1M tier: a million raw values (or long +repeated strings) is a multi-megabyte file per case for no extra coverage, and +it would blow the CI artifact budget. It is still fully covered at the three +smaller scales. + +A reduced encoding set (`PLAIN`, `TS_2DIFF`, `RLE`) also gets a "many pages" +variant — page 100 at 100000 rows, page 1000 at 1000000 rows — which produces +1000 pages and crosses the 256-entry index-node fanout, stressing the +multi-level metadata index. + +## Value profiles + +Values are a pure function of the profile and the row index, implemented +identically in Java and C++. Expected values apply the same quantization the +writer performs, so a fixture has one expected result no matter which language +wrote it. + +| Profile | Purpose | +| --- | --- | +| `default` | Type-specific values including numeric extremes and repeated values. | +| `ts2diff` | FLOAT/DOUBLE TS_2DIFF values covering scaled, scale-overflow and raw-bit forms (NaN, infinities) across the 129-value block boundary. | +| `ts2diff_mpn0` | Same values written with an explicit `max_point_number=0` property (Java only; the C++ writer has no property path). | +| `sprintz` | Values inside a range the SPRINTZ FIR predictor can encode (about ±1e8 for INT32 and ±1e12 for INT64). Outside it the predictor itself overflows, which is a codec limitation rather than a cross-language difference. | +| `runs` | Long runs for RLE encodings. | +| `strings` | Empty string, embedded NUL, multi-byte UTF-8, repeated values, long values. | +| `dict_high` | One distinct string per row to stress dictionary growth. | +| `blob` | Empty, all-zero, binary and text-like byte arrays. | + +## Running locally + +```bash +# Java writes, Java and C++ read +./mvnw -P with-java \ + -Dtest=EncodingCompressionCompatibilityTest#generateFixtures \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dtsfile.compat.spec="$PWD/compat/compatibility-matrix.csv" \ + -Dtsfile.compat.generate.dir="$PWD/compat-fixtures/java" test + +TSFILE_COMPAT_SPEC="$PWD/compat/compatibility-matrix.csv" \ +TSFILE_COMPAT_VALIDATE_DIR="$PWD/compat-fixtures/java" \ + ./cpp/target/build/test/lib/TsFile_Test \ + --gtest_filter=EncodingCompressionCompatibilityTest.ValidateFixtures +``` + +Swap the environments for the C++ → Java direction. Both suites also emit +`report.csv` next to the fixtures with a per-case status, which the CI job +summary aggregates. + +## Adding coverage + +1. Add a row (or edit an existing one) in `compatibility-matrix.csv`. +2. If a language cannot write or read the new combination, set the column to + `no` and write the reason. Link a tracking issue when the gap is a defect. +3. Run both directions locally; the generator fails when a declared case cannot + be produced and the validator fails when a declared-readable fixture cannot + be read. diff --git a/compat/compatibility-matrix.csv b/compat/compatibility-matrix.csv new file mode 100644 index 0000000000..696fb29d91 --- /dev/null +++ b/compat/compatibility-matrix.csv @@ -0,0 +1,625 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Apache TsFile Java/C++ cross-language compatibility matrix. +# +# This file is the single source of truth for the compatibility fixture matrix. +# Both java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java +# and cpp/test/reader/table_view/encoding_compression_compatibility_test.cc +# read it: each language writes every case it declares with Write=yes and +# verifies the fixtures of the other language for every case declared with +# Read=yes. See compat/README.md for the full semantics. +# +# Lines starting with '#' and blank lines are ignored. Column values are +# case-sensitive enum names shared by both implementations. +# +# Data scale: every (type, encoding, compression) combination is generated at +# four scales -- 10, 1000, 100000 and 1000000 rows -- with a page size that +# crosses page boundaries at each scale (see README). An extra "many pages" +# variant (page 100 at 100000 rows, page 1000 at 1000000 rows) stresses the +# index-node fanout for a reduced encoding set. +# +# compressions: 'ALL' expands to the six codecs both implementations ship +# (UNCOMPRESSED+SNAPPY+GZIP+LZ4+ZSTD+LZMA2). A '+' separated list restricts +# the expansion. +# nullPatterns: '|' separated NONE/MIXED/ALL values. +# pageRows: 0 keeps the writer default (10000). chunks: flush boundaries. +# devices: number of devices (and therefore chunk groups) in one file. +# valueProfile: default|ts2diff|ts2diff_mpn0|sprintz|runs|strings|dict_high|blob. +# A 'no' in any write/read column is a declared gap and gapReason must explain +# it. Known unsupported combinations that are not in this matrix at all: +# - DIFF, BITMAP, REGULAR, FREQ: deprecated and not writable by either side. +# - Compressions LZO, SDT, PAA, PLA: C++ enum placeholders with no Java +# counterpart. +# - VECTOR: no writer on either side. +# - max_point_number on the C++ side: declared per row below (C++ reads +# both forms). +# - Full-range INT32/INT64 extremes under SPRINTZ: the predictor overflows +# when one block spans INT_MIN and INT_MAX, so SPRINTZ rows use the +# 'sprintz' profile (values within +/-1e9). Streaming numeric extremes are +# covered under PLAIN, RLE, TS_2DIFF, GORILLA, CHIMP and RLBE. + +id,model,dataType,encoding,compressions,rowCount,pageRows,chunks,devices,valueProfile,nullPatterns,javaWrite,cppWrite,javaRead,cppRead,gapReason +table_int32_plain_r10,table,INT32,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_plain_r1000,table,INT32,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_plain_r100000,table,INT32,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_plain_r1000000,table,INT32,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rle_r10,table,INT32,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_int32_rle_r1000,table,INT32,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_int32_rle_r100000,table,INT32,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_int32_rle_r1000000,table,INT32,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_r10,table,INT32,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_r1000,table,INT32,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_r100000,table,INT32,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_r1000000,table,INT32,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_gorilla_r10,table,INT32,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_gorilla_r1000,table,INT32,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_gorilla_r100000,table,INT32,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_gorilla_r1000000,table,INT32,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_zigzag_r10,table,INT32,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_zigzag_r1000,table,INT32,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_zigzag_r100000,table,INT32,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_zigzag_r1000000,table,INT32,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_chimp_r10,table,INT32,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_chimp_r1000,table,INT32,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_chimp_r100000,table,INT32,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_chimp_r1000000,table,INT32,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_sprintz_r10,table,INT32,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int32_sprintz_r1000,table,INT32,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int32_sprintz_r100000,table,INT32,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int32_sprintz_r1000000,table,INT32,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int32_rlbe_r10,table,INT32,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rlbe_r1000,table,INT32,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rlbe_r100000,table,INT32,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rlbe_r1000000,table,INT32,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_r10,table,DATE,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_r1000,table,DATE,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_r100000,table,DATE,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_r1000000,table,DATE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_rle_r10,table,DATE,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_date_rle_r1000,table,DATE,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_date_rle_r100000,table,DATE,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_date_rle_r1000000,table,DATE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_date_ts_2diff_r10,table,DATE,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_ts_2diff_r1000,table,DATE,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_ts_2diff_r100000,table,DATE,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_ts_2diff_r1000000,table,DATE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_gorilla_r10,table,DATE,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_gorilla_r1000,table,DATE,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_gorilla_r100000,table,DATE,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_gorilla_r1000000,table,DATE,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_zigzag_r10,table,DATE,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_zigzag_r1000,table,DATE,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_zigzag_r100000,table,DATE,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_zigzag_r1000000,table,DATE,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_chimp_r10,table,DATE,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_chimp_r1000,table,DATE,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_chimp_r100000,table,DATE,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_chimp_r1000000,table,DATE,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_sprintz_r10,table,DATE,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_date_sprintz_r1000,table,DATE,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_date_sprintz_r100000,table,DATE,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_date_sprintz_r1000000,table,DATE,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_date_rlbe_r10,table,DATE,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_date_rlbe_r1000,table,DATE,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_rlbe_r100000,table,DATE,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_date_rlbe_r1000000,table,DATE,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_r10,table,INT64,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_r1000,table,INT64,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_r100000,table,INT64,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_r1000000,table,INT64,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rle_r10,table,INT64,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_int64_rle_r1000,table,INT64,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_int64_rle_r100000,table,INT64,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_int64_rle_r1000000,table,INT64,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_r10,table,INT64,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_r1000,table,INT64,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_r100000,table,INT64,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_r1000000,table,INT64,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_gorilla_r10,table,INT64,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_gorilla_r1000,table,INT64,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_gorilla_r100000,table,INT64,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_gorilla_r1000000,table,INT64,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_zigzag_r10,table,INT64,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_zigzag_r1000,table,INT64,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_zigzag_r100000,table,INT64,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_zigzag_r1000000,table,INT64,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_chimp_r10,table,INT64,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_chimp_r1000,table,INT64,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_chimp_r100000,table,INT64,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_chimp_r1000000,table,INT64,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_sprintz_r10,table,INT64,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int64_sprintz_r1000,table,INT64,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int64_sprintz_r100000,table,INT64,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int64_sprintz_r1000000,table,INT64,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_int64_rlbe_r10,table,INT64,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rlbe_r1000,table,INT64,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rlbe_r100000,table,INT64,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rlbe_r1000000,table,INT64,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_r10,table,TIMESTAMP,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_r1000,table,TIMESTAMP,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_r100000,table,TIMESTAMP,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_r1000000,table,TIMESTAMP,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rle_r10,table,TIMESTAMP,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_timestamp_rle_r1000,table,TIMESTAMP,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_timestamp_rle_r100000,table,TIMESTAMP,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_timestamp_rle_r1000000,table,TIMESTAMP,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_r10,table,TIMESTAMP,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_r1000,table,TIMESTAMP,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_r100000,table,TIMESTAMP,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_r1000000,table,TIMESTAMP,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_gorilla_r10,table,TIMESTAMP,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_gorilla_r1000,table,TIMESTAMP,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_gorilla_r100000,table,TIMESTAMP,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_gorilla_r1000000,table,TIMESTAMP,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_zigzag_r10,table,TIMESTAMP,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_zigzag_r1000,table,TIMESTAMP,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_zigzag_r100000,table,TIMESTAMP,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_zigzag_r1000000,table,TIMESTAMP,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_chimp_r10,table,TIMESTAMP,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_chimp_r1000,table,TIMESTAMP,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_chimp_r100000,table,TIMESTAMP,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_chimp_r1000000,table,TIMESTAMP,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_sprintz_r10,table,TIMESTAMP,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_timestamp_sprintz_r1000,table,TIMESTAMP,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_timestamp_sprintz_r100000,table,TIMESTAMP,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_timestamp_sprintz_r1000000,table,TIMESTAMP,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_timestamp_rlbe_r10,table,TIMESTAMP,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rlbe_r1000,table,TIMESTAMP,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rlbe_r100000,table,TIMESTAMP,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rlbe_r1000000,table,TIMESTAMP,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_r10,table,FLOAT,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_r1000,table,FLOAT,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_r100000,table,FLOAT,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_r1000000,table,FLOAT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_ts_2diff_r10,table,FLOAT,TS_2DIFF,ALL,10,4,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_float_ts_2diff_r1000,table,FLOAT,TS_2DIFF,ALL,1000,100,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_float_ts_2diff_r100000,table,FLOAT,TS_2DIFF,ALL,100000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_float_ts_2diff_r1000000,table,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_float_gorilla_r10,table,FLOAT,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_r1000,table,FLOAT,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_r100000,table,FLOAT,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_r1000000,table,FLOAT,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_chimp_r10,table,FLOAT,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_float_chimp_r1000,table,FLOAT,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_chimp_r100000,table,FLOAT,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_chimp_r1000000,table,FLOAT,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_sprintz_r10,table,FLOAT,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_float_sprintz_r1000,table,FLOAT,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_float_sprintz_r100000,table,FLOAT,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_float_sprintz_r1000000,table,FLOAT,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_float_rlbe_r10,table,FLOAT,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_float_rlbe_r1000,table,FLOAT,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_rlbe_r100000,table,FLOAT,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_rlbe_r1000000,table,FLOAT,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_rle_r10,table,FLOAT,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_float_rle_r1000,table,FLOAT,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_float_rle_r100000,table,FLOAT,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_float_rle_r1000000,table,FLOAT,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_float_gorilla_v1_r10,table,FLOAT,GORILLA_V1,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_v1_r1000,table,FLOAT,GORILLA_V1,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_v1_r100000,table,FLOAT,GORILLA_V1,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_gorilla_v1_r1000000,table,FLOAT,GORILLA_V1,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_float_ts2diff_mpn0_java_r10,table,FLOAT,TS_2DIFF,ALL,10,4,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_float_ts2diff_mpn0_java_r1000,table,FLOAT,TS_2DIFF,ALL,1000,100,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_float_ts2diff_mpn0_java_r100000,table,FLOAT,TS_2DIFF,ALL,100000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_float_ts2diff_mpn0_java_r1000000,table,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_double_plain_r10,table,DOUBLE,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_plain_r1000,table,DOUBLE,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_plain_r100000,table,DOUBLE,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_plain_r1000000,table,DOUBLE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_ts_2diff_r10,table,DOUBLE,TS_2DIFF,ALL,10,4,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_double_ts_2diff_r1000,table,DOUBLE,TS_2DIFF,ALL,1000,100,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_double_ts_2diff_r100000,table,DOUBLE,TS_2DIFF,ALL,100000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_double_ts_2diff_r1000000,table,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +table_double_gorilla_r10,table,DOUBLE,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_r1000,table,DOUBLE,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_r100000,table,DOUBLE,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_r1000000,table,DOUBLE,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_chimp_r10,table,DOUBLE,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_chimp_r1000,table,DOUBLE,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_chimp_r100000,table,DOUBLE,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_chimp_r1000000,table,DOUBLE,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_sprintz_r10,table,DOUBLE,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +table_double_sprintz_r1000,table,DOUBLE,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +table_double_sprintz_r100000,table,DOUBLE,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_double_sprintz_r1000000,table,DOUBLE,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +table_double_rlbe_r10,table,DOUBLE,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_rlbe_r1000,table,DOUBLE,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_rlbe_r100000,table,DOUBLE,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_rlbe_r1000000,table,DOUBLE,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_camel_r10,table,DOUBLE,CAMEL,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_camel_r1000,table,DOUBLE,CAMEL,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_camel_r100000,table,DOUBLE,CAMEL,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_camel_r1000000,table,DOUBLE,CAMEL,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_rle_r10,table,DOUBLE,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +table_double_rle_r1000,table,DOUBLE,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +table_double_rle_r100000,table,DOUBLE,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_double_rle_r1000000,table,DOUBLE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +table_double_gorilla_v1_r10,table,DOUBLE,GORILLA_V1,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_v1_r1000,table,DOUBLE,GORILLA_V1,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_v1_r100000,table,DOUBLE,GORILLA_V1,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_gorilla_v1_r1000000,table,DOUBLE,GORILLA_V1,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_double_ts2diff_mpn0_java_r10,table,DOUBLE,TS_2DIFF,ALL,10,4,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_double_ts2diff_mpn0_java_r1000,table,DOUBLE,TS_2DIFF,ALL,1000,100,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_double_ts2diff_mpn0_java_r100000,table,DOUBLE,TS_2DIFF,ALL,100000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_double_ts2diff_mpn0_java_r1000000,table,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +table_boolean_plain_r10,table,BOOLEAN,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_plain_r1000,table,BOOLEAN,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_plain_r100000,table,BOOLEAN,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_plain_r1000000,table,BOOLEAN,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_rle_r10,table,BOOLEAN,RLE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_rle_r1000,table,BOOLEAN,RLE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_rle_r100000,table,BOOLEAN,RLE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_boolean_rle_r1000000,table,BOOLEAN,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +table_text_plain_r10,table,TEXT,PLAIN,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +table_text_plain_r1000,table,TEXT,PLAIN,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +table_text_plain_r100000,table,TEXT,PLAIN,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_text_plain_r1000000,table,TEXT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_text_dictionary_r10,table,TEXT,DICTIONARY,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +table_text_dictionary_r1000,table,TEXT,DICTIONARY,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +table_text_dictionary_r100000,table,TEXT,DICTIONARY,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_text_dictionary_r1000000,table,TEXT,DICTIONARY,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_string_plain_r10,table,STRING,PLAIN,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +table_string_plain_r1000,table,STRING,PLAIN,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +table_string_plain_r100000,table,STRING,PLAIN,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_string_plain_r1000000,table,STRING,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_string_dictionary_r10,table,STRING,DICTIONARY,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +table_string_dictionary_r1000,table,STRING,DICTIONARY,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +table_string_dictionary_r100000,table,STRING,DICTIONARY,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_string_dictionary_r1000000,table,STRING,DICTIONARY,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +table_blob_plain_r10,table,BLOB,PLAIN,ALL,10,4,1,1,blob,NONE,yes,yes,yes,yes, +table_blob_plain_r1000,table,BLOB,PLAIN,ALL,1000,100,1,1,blob,NONE,yes,yes,yes,yes, +table_blob_plain_r100000,table,BLOB,PLAIN,ALL,100000,10000,1,1,blob,NONE,yes,yes,yes,yes, +table_blob_plain_r1000000,table,BLOB,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,blob,NONE,yes,yes,yes,yes, +table_int32_plain_p100_r100000,table,INT32,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_plain_p1000_r1000000,table,INT32,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_p100_r100000,table,INT32,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_ts_2diff_p1000_r1000000,table,INT32,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rle_p100_r100000,table,INT32,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int32_rle_p1000_r1000000,table,INT32,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_p100_r100000,table,DATE,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_plain_p1000_r1000000,table,DATE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_date_ts_2diff_p100_r100000,table,DATE,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_ts_2diff_p1000_r1000000,table,DATE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_date_rle_p100_r100000,table,DATE,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_date_rle_p1000_r1000000,table,DATE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_p100_r100000,table,INT64,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_plain_p1000_r1000000,table,INT64,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_p100_r100000,table,INT64,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_ts_2diff_p1000_r1000000,table,INT64,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rle_p100_r100000,table,INT64,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_int64_rle_p1000_r1000000,table,INT64,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_p100_r100000,table,TIMESTAMP,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_plain_p1000_r1000000,table,TIMESTAMP,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_p100_r100000,table,TIMESTAMP,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_ts_2diff_p1000_r1000000,table,TIMESTAMP,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rle_p100_r100000,table,TIMESTAMP,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_timestamp_rle_p1000_r1000000,table,TIMESTAMP,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_p100_r100000,table,FLOAT,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_plain_p1000_r1000000,table,FLOAT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_double_plain_p100_r100000,table,DOUBLE,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_plain_p1000_r1000000,table,DOUBLE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_float_ts_2diff_p100_r100000,table,FLOAT,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_ts_2diff_p1000_r1000000,table,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_double_ts_2diff_p100_r100000,table,DOUBLE,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_ts_2diff_p1000_r1000000,table,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_float_rle_p100_r100000,table,FLOAT,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_float_rle_p1000_r1000000,table,FLOAT,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_double_rle_p100_r100000,table,DOUBLE,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +table_double_rle_p1000_r1000000,table,DOUBLE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +table_int32_null,table,INT32,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_date_null,table,DATE,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_int64_null,table,INT64,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_timestamp_null,table,TIMESTAMP,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_float_null,table,FLOAT,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_double_null,table,DOUBLE,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_boolean_null,table,BOOLEAN,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED|ALL,yes,yes,yes,yes, +table_text_null,table,TEXT,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,strings,MIXED|ALL,yes,yes,yes,yes, +table_string_null,table,STRING,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,strings,MIXED|ALL,yes,yes,yes,yes, +table_int32_null_single_page,table,INT32,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,64,64,1,1,default,ALL,yes,yes,yes,yes, +table_row1,table,INT64,TS_2DIFF,ALL,1,0,1,1,default,NONE,yes,yes,yes,yes, +table_one_page,table,DOUBLE,GORILLA,ALL,100,100,1,1,default,NONE,yes,yes,yes,yes, +table_page_plus_one,table,DOUBLE,GORILLA,ALL,101,100,1,1,default,NONE,yes,yes,yes,yes, +table_two_pages,table,FLOAT,CHIMP,ALL,250,100,1,1,default,NONE,yes,yes,yes,yes, +table_multi_chunk,table,INT64,TS_2DIFF,ALL,300,0,3,1,default,NONE,yes,yes,yes,yes, +table_multi_chunk_group,table,INT32,RLE,ALL,50,0,1,3,runs,NONE,yes,yes,yes,yes, +table_long_runs,table,INT64,RLE,ALL,1024,0,1,1,runs,NONE,yes,yes,yes,yes, +tree_int32_plain_r10,tree,INT32,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_plain_r1000,tree,INT32,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_plain_r100000,tree,INT32,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_plain_r1000000,tree,INT32,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rle_r10,tree,INT32,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_int32_rle_r1000,tree,INT32,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_int32_rle_r100000,tree,INT32,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_int32_rle_r1000000,tree,INT32,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_r10,tree,INT32,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_r1000,tree,INT32,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_r100000,tree,INT32,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_r1000000,tree,INT32,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_gorilla_r10,tree,INT32,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_gorilla_r1000,tree,INT32,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_gorilla_r100000,tree,INT32,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_gorilla_r1000000,tree,INT32,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_zigzag_r10,tree,INT32,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_zigzag_r1000,tree,INT32,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_zigzag_r100000,tree,INT32,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_zigzag_r1000000,tree,INT32,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_chimp_r10,tree,INT32,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_chimp_r1000,tree,INT32,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_chimp_r100000,tree,INT32,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_chimp_r1000000,tree,INT32,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_sprintz_r10,tree,INT32,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int32_sprintz_r1000,tree,INT32,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int32_sprintz_r100000,tree,INT32,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int32_sprintz_r1000000,tree,INT32,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int32_rlbe_r10,tree,INT32,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rlbe_r1000,tree,INT32,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rlbe_r100000,tree,INT32,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rlbe_r1000000,tree,INT32,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_r10,tree,DATE,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_r1000,tree,DATE,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_r100000,tree,DATE,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_r1000000,tree,DATE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rle_r10,tree,DATE,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_date_rle_r1000,tree,DATE,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_date_rle_r100000,tree,DATE,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_date_rle_r1000000,tree,DATE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_r10,tree,DATE,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_r1000,tree,DATE,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_r100000,tree,DATE,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_r1000000,tree,DATE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_gorilla_r10,tree,DATE,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_gorilla_r1000,tree,DATE,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_gorilla_r100000,tree,DATE,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_gorilla_r1000000,tree,DATE,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_zigzag_r10,tree,DATE,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_zigzag_r1000,tree,DATE,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_zigzag_r100000,tree,DATE,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_zigzag_r1000000,tree,DATE,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_chimp_r10,tree,DATE,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_chimp_r1000,tree,DATE,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_chimp_r100000,tree,DATE,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_chimp_r1000000,tree,DATE,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_sprintz_r10,tree,DATE,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_date_sprintz_r1000,tree,DATE,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_date_sprintz_r100000,tree,DATE,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_date_sprintz_r1000000,tree,DATE,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_date_rlbe_r10,tree,DATE,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rlbe_r1000,tree,DATE,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rlbe_r100000,tree,DATE,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rlbe_r1000000,tree,DATE,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_r10,tree,INT64,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_r1000,tree,INT64,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_r100000,tree,INT64,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_r1000000,tree,INT64,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rle_r10,tree,INT64,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_int64_rle_r1000,tree,INT64,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_int64_rle_r100000,tree,INT64,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_int64_rle_r1000000,tree,INT64,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_r10,tree,INT64,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_r1000,tree,INT64,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_r100000,tree,INT64,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_r1000000,tree,INT64,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_gorilla_r10,tree,INT64,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_gorilla_r1000,tree,INT64,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_gorilla_r100000,tree,INT64,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_gorilla_r1000000,tree,INT64,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_zigzag_r10,tree,INT64,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_zigzag_r1000,tree,INT64,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_zigzag_r100000,tree,INT64,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_zigzag_r1000000,tree,INT64,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_chimp_r10,tree,INT64,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_chimp_r1000,tree,INT64,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_chimp_r100000,tree,INT64,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_chimp_r1000000,tree,INT64,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_sprintz_r10,tree,INT64,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int64_sprintz_r1000,tree,INT64,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int64_sprintz_r100000,tree,INT64,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int64_sprintz_r1000000,tree,INT64,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_int64_rlbe_r10,tree,INT64,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rlbe_r1000,tree,INT64,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rlbe_r100000,tree,INT64,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rlbe_r1000000,tree,INT64,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_r10,tree,TIMESTAMP,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_r1000,tree,TIMESTAMP,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_r100000,tree,TIMESTAMP,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_r1000000,tree,TIMESTAMP,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rle_r10,tree,TIMESTAMP,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_timestamp_rle_r1000,tree,TIMESTAMP,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_timestamp_rle_r100000,tree,TIMESTAMP,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_timestamp_rle_r1000000,tree,TIMESTAMP,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_r10,tree,TIMESTAMP,TS_2DIFF,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_r1000,tree,TIMESTAMP,TS_2DIFF,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_r100000,tree,TIMESTAMP,TS_2DIFF,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_r1000000,tree,TIMESTAMP,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_gorilla_r10,tree,TIMESTAMP,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_gorilla_r1000,tree,TIMESTAMP,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_gorilla_r100000,tree,TIMESTAMP,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_gorilla_r1000000,tree,TIMESTAMP,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_zigzag_r10,tree,TIMESTAMP,ZIGZAG,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_zigzag_r1000,tree,TIMESTAMP,ZIGZAG,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_zigzag_r100000,tree,TIMESTAMP,ZIGZAG,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_zigzag_r1000000,tree,TIMESTAMP,ZIGZAG,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_chimp_r10,tree,TIMESTAMP,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_chimp_r1000,tree,TIMESTAMP,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_chimp_r100000,tree,TIMESTAMP,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_chimp_r1000000,tree,TIMESTAMP,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_sprintz_r10,tree,TIMESTAMP,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_timestamp_sprintz_r1000,tree,TIMESTAMP,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_timestamp_sprintz_r100000,tree,TIMESTAMP,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_timestamp_sprintz_r1000000,tree,TIMESTAMP,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_timestamp_rlbe_r10,tree,TIMESTAMP,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rlbe_r1000,tree,TIMESTAMP,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rlbe_r100000,tree,TIMESTAMP,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rlbe_r1000000,tree,TIMESTAMP,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_r10,tree,FLOAT,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_r1000,tree,FLOAT,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_r100000,tree,FLOAT,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_r1000000,tree,FLOAT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_r10,tree,FLOAT,TS_2DIFF,ALL,10,4,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_r1000,tree,FLOAT,TS_2DIFF,ALL,1000,100,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_r100000,tree,FLOAT,TS_2DIFF,ALL,100000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_r1000000,tree,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_float_gorilla_r10,tree,FLOAT,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_r1000,tree,FLOAT,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_r100000,tree,FLOAT,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_r1000000,tree,FLOAT,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_chimp_r10,tree,FLOAT,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_float_chimp_r1000,tree,FLOAT,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_chimp_r100000,tree,FLOAT,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_chimp_r1000000,tree,FLOAT,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_sprintz_r10,tree,FLOAT,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_float_sprintz_r1000,tree,FLOAT,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_float_sprintz_r100000,tree,FLOAT,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_float_sprintz_r1000000,tree,FLOAT,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_float_rlbe_r10,tree,FLOAT,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rlbe_r1000,tree,FLOAT,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rlbe_r100000,tree,FLOAT,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rlbe_r1000000,tree,FLOAT,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rle_r10,tree,FLOAT,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_float_rle_r1000,tree,FLOAT,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_float_rle_r100000,tree,FLOAT,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_float_rle_r1000000,tree,FLOAT,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_float_gorilla_v1_r10,tree,FLOAT,GORILLA_V1,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_v1_r1000,tree,FLOAT,GORILLA_V1,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_v1_r100000,tree,FLOAT,GORILLA_V1,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_gorilla_v1_r1000000,tree,FLOAT,GORILLA_V1,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_ts2diff_mpn0_java_r10,tree,FLOAT,TS_2DIFF,ALL,10,4,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_float_ts2diff_mpn0_java_r1000,tree,FLOAT,TS_2DIFF,ALL,1000,100,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_float_ts2diff_mpn0_java_r100000,tree,FLOAT,TS_2DIFF,ALL,100000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_float_ts2diff_mpn0_java_r1000000,tree,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_double_plain_r10,tree,DOUBLE,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_plain_r1000,tree,DOUBLE,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_plain_r100000,tree,DOUBLE,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_plain_r1000000,tree,DOUBLE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_r10,tree,DOUBLE,TS_2DIFF,ALL,10,4,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_r1000,tree,DOUBLE,TS_2DIFF,ALL,1000,100,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_r100000,tree,DOUBLE,TS_2DIFF,ALL,100000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_r1000000,tree,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff,NONE,yes,yes,yes,yes, +tree_double_gorilla_r10,tree,DOUBLE,GORILLA,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_r1000,tree,DOUBLE,GORILLA,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_r100000,tree,DOUBLE,GORILLA,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_r1000000,tree,DOUBLE,GORILLA,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_chimp_r10,tree,DOUBLE,CHIMP,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_chimp_r1000,tree,DOUBLE,CHIMP,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_chimp_r100000,tree,DOUBLE,CHIMP,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_chimp_r1000000,tree,DOUBLE,CHIMP,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_sprintz_r10,tree,DOUBLE,SPRINTZ,ALL,10,4,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_double_sprintz_r1000,tree,DOUBLE,SPRINTZ,ALL,1000,100,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_double_sprintz_r100000,tree,DOUBLE,SPRINTZ,ALL,100000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_double_sprintz_r1000000,tree,DOUBLE,SPRINTZ,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,sprintz,NONE,yes,yes,yes,yes, +tree_double_rlbe_r10,tree,DOUBLE,RLBE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rlbe_r1000,tree,DOUBLE,RLBE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rlbe_r100000,tree,DOUBLE,RLBE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rlbe_r1000000,tree,DOUBLE,RLBE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_camel_r10,tree,DOUBLE,CAMEL,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_camel_r1000,tree,DOUBLE,CAMEL,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_camel_r100000,tree,DOUBLE,CAMEL,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_camel_r1000000,tree,DOUBLE,CAMEL,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rle_r10,tree,DOUBLE,RLE,ALL,10,4,1,1,runs,NONE,yes,yes,yes,yes, +tree_double_rle_r1000,tree,DOUBLE,RLE,ALL,1000,100,1,1,runs,NONE,yes,yes,yes,yes, +tree_double_rle_r100000,tree,DOUBLE,RLE,ALL,100000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_double_rle_r1000000,tree,DOUBLE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,runs,NONE,yes,yes,yes,yes, +tree_double_gorilla_v1_r10,tree,DOUBLE,GORILLA_V1,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_v1_r1000,tree,DOUBLE,GORILLA_V1,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_v1_r100000,tree,DOUBLE,GORILLA_V1,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_gorilla_v1_r1000000,tree,DOUBLE,GORILLA_V1,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_ts2diff_mpn0_java_r10,tree,DOUBLE,TS_2DIFF,ALL,10,4,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_double_ts2diff_mpn0_java_r1000,tree,DOUBLE,TS_2DIFF,ALL,1000,100,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_double_ts2diff_mpn0_java_r100000,tree,DOUBLE,TS_2DIFF,ALL,100000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_double_ts2diff_mpn0_java_r1000000,tree,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,ts2diff_mpn0,NONE,yes,no,yes,yes,C++ writer exposes no max_point_number property path for TS_2DIFF +tree_boolean_plain_r10,tree,BOOLEAN,PLAIN,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_plain_r1000,tree,BOOLEAN,PLAIN,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_plain_r100000,tree,BOOLEAN,PLAIN,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_plain_r1000000,tree,BOOLEAN,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_rle_r10,tree,BOOLEAN,RLE,ALL,10,4,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_rle_r1000,tree,BOOLEAN,RLE,ALL,1000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_rle_r100000,tree,BOOLEAN,RLE,ALL,100000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_boolean_rle_r1000000,tree,BOOLEAN,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,default,NONE,yes,yes,yes,yes, +tree_text_plain_r10,tree,TEXT,PLAIN,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_plain_r1000,tree,TEXT,PLAIN,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_plain_r100000,tree,TEXT,PLAIN,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_plain_r1000000,tree,TEXT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_dictionary_r10,tree,TEXT,DICTIONARY,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_dictionary_r1000,tree,TEXT,DICTIONARY,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_dictionary_r100000,tree,TEXT,DICTIONARY,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_text_dictionary_r1000000,tree,TEXT,DICTIONARY,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_plain_r10,tree,STRING,PLAIN,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_plain_r1000,tree,STRING,PLAIN,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_plain_r100000,tree,STRING,PLAIN,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_plain_r1000000,tree,STRING,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_dictionary_r10,tree,STRING,DICTIONARY,ALL,10,4,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_dictionary_r1000,tree,STRING,DICTIONARY,ALL,1000,100,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_dictionary_r100000,tree,STRING,DICTIONARY,ALL,100000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_string_dictionary_r1000000,tree,STRING,DICTIONARY,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,strings,NONE,yes,yes,yes,yes, +tree_blob_plain_r10,tree,BLOB,PLAIN,ALL,10,4,1,1,blob,NONE,yes,yes,yes,yes, +tree_blob_plain_r1000,tree,BLOB,PLAIN,ALL,1000,100,1,1,blob,NONE,yes,yes,yes,yes, +tree_blob_plain_r100000,tree,BLOB,PLAIN,ALL,100000,10000,1,1,blob,NONE,yes,yes,yes,yes, +tree_blob_plain_r1000000,tree,BLOB,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,10000,1,1,blob,NONE,yes,yes,yes,yes, +tree_int32_plain_p100_r100000,tree,INT32,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_plain_p1000_r1000000,tree,INT32,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_p100_r100000,tree,INT32,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_ts_2diff_p1000_r1000000,tree,INT32,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rle_p100_r100000,tree,INT32,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_rle_p1000_r1000000,tree,INT32,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_p100_r100000,tree,DATE,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_plain_p1000_r1000000,tree,DATE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_p100_r100000,tree,DATE,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_ts_2diff_p1000_r1000000,tree,DATE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rle_p100_r100000,tree,DATE,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_date_rle_p1000_r1000000,tree,DATE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_p100_r100000,tree,INT64,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_plain_p1000_r1000000,tree,INT64,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_p100_r100000,tree,INT64,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_ts_2diff_p1000_r1000000,tree,INT64,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rle_p100_r100000,tree,INT64,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_int64_rle_p1000_r1000000,tree,INT64,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_p100_r100000,tree,TIMESTAMP,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_plain_p1000_r1000000,tree,TIMESTAMP,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_p100_r100000,tree,TIMESTAMP,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_ts_2diff_p1000_r1000000,tree,TIMESTAMP,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rle_p100_r100000,tree,TIMESTAMP,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_timestamp_rle_p1000_r1000000,tree,TIMESTAMP,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_p100_r100000,tree,FLOAT,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_plain_p1000_r1000000,tree,FLOAT,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_plain_p100_r100000,tree,DOUBLE,PLAIN,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_plain_p1000_r1000000,tree,DOUBLE,PLAIN,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_p100_r100000,tree,FLOAT,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_ts_2diff_p1000_r1000000,tree,FLOAT,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_p100_r100000,tree,DOUBLE,TS_2DIFF,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_ts_2diff_p1000_r1000000,tree,DOUBLE,TS_2DIFF,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rle_p100_r100000,tree,FLOAT,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_float_rle_p1000_r1000000,tree,FLOAT,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rle_p100_r100000,tree,DOUBLE,RLE,ALL,100000,100,1,1,default,NONE,yes,yes,yes,yes, +tree_double_rle_p1000_r1000000,tree,DOUBLE,RLE,SNAPPY+GZIP+LZ4+ZSTD+LZMA2,1000000,1000,1,1,default,NONE,yes,yes,yes,yes, +tree_int32_null,tree,INT32,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_date_null,tree,DATE,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_int64_null,tree,INT64,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_timestamp_null,tree,TIMESTAMP,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_float_null,tree,FLOAT,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_double_null,tree,DOUBLE,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_boolean_null,tree,BOOLEAN,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,default,MIXED,yes,yes,yes,yes, +tree_text_null,tree,TEXT,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,strings,MIXED,yes,yes,yes,yes, +tree_string_null,tree,STRING,PLAIN,UNCOMPRESSED+LZ4+ZSTD+LZMA2,512,64,1,1,strings,MIXED,yes,yes,yes,yes, +tree_row1,tree,INT64,TS_2DIFF,ALL,1,0,1,1,default,NONE,yes,yes,yes,yes, +tree_one_page,tree,DOUBLE,GORILLA,ALL,100,100,1,1,default,NONE,yes,yes,yes,yes, +tree_page_plus_one,tree,DOUBLE,GORILLA,ALL,101,100,1,1,default,NONE,yes,yes,yes,yes, +tree_two_pages,tree,FLOAT,CHIMP,ALL,250,100,1,1,default,NONE,yes,yes,yes,yes, +tree_multi_chunk,tree,INT64,TS_2DIFF,ALL,300,0,3,1,default,NONE,yes,yes,yes,yes, +tree_multi_chunk_group,tree,INT32,RLE,ALL,50,0,1,3,runs,NONE,yes,yes,yes,yes, +tree_long_runs,tree,INT64,RLE,ALL,1024,0,1,1,runs,NONE,yes,yes,yes,yes, diff --git a/cpp/src/encoding/decoder_factory.h b/cpp/src/encoding/decoder_factory.h index 3247b93bac..ce62195940 100644 --- a/cpp/src/encoding/decoder_factory.h +++ b/cpp/src/encoding/decoder_factory.h @@ -28,7 +28,9 @@ #include "double_sprintz_decoder.h" #include "encoding/int32_rle_decoder.h" #include "encoding/int64_rle_decoder.h" +#include "float_rle_decoder.h" #include "float_sprintz_decoder.h" +#include "gorilla_v1_decoder.h" #include "gorilla_decoder.h" #include "int32_sprintz_decoder.h" #include "int64_sprintz_decoder.h" @@ -89,6 +91,10 @@ class DecoderFactory { case INT64: case TIMESTAMP: ALLOC_AND_RETURN_DECODER(Int64RleDecoder); + case FLOAT: + ALLOC_AND_RETURN_DECODER(FloatRLEDecoder); + case DOUBLE: + ALLOC_AND_RETURN_DECODER(DoubleRLEDecoder); default: return nullptr; } @@ -144,8 +150,10 @@ class DecoderFactory { case ZIGZAG: switch (data_type) { case INT32: + case DATE: ALLOC_AND_RETURN_DECODER(IntZigzagDecoder); case INT64: + case TIMESTAMP: ALLOC_AND_RETURN_DECODER(LongZigzagDecoder); default: return nullptr; @@ -154,8 +162,10 @@ class DecoderFactory { case SPRINTZ: switch (data_type) { case INT32: + case DATE: ALLOC_AND_RETURN_DECODER(Int32SprintzDecoder); case INT64: + case TIMESTAMP: ALLOC_AND_RETURN_DECODER(Int64SprintzDecoder); case FLOAT: ALLOC_AND_RETURN_DECODER(FloatSprintzDecoder); @@ -189,6 +199,16 @@ class DecoderFactory { return nullptr; } + case GORILLA_V1: + switch (data_type) { + case FLOAT: + ALLOC_AND_RETURN_DECODER(FloatGorillaV1Decoder); + case DOUBLE: + ALLOC_AND_RETURN_DECODER(DoubleGorillaV1Decoder); + default: + return nullptr; + } + default: // Not supported encoding return nullptr; diff --git a/cpp/src/encoding/encoder_factory.h b/cpp/src/encoding/encoder_factory.h index 052cbf8eed..d009f90ff8 100644 --- a/cpp/src/encoding/encoder_factory.h +++ b/cpp/src/encoding/encoder_factory.h @@ -28,7 +28,9 @@ #include "encoder.h" #include "encoding/int32_rle_encoder.h" #include "encoding/int64_rle_encoder.h" +#include "float_rle_encoder.h" #include "float_sprintz_encoder.h" +#include "gorilla_v1_encoder.h" #include "gorilla_encoder.h" #include "int32_sprintz_encoder.h" #include "int64_sprintz_encoder.h" @@ -95,12 +97,17 @@ class EncoderFactory { case RLE: switch (data_type) { + case BOOLEAN: case INT32: case DATE: ALLOC_AND_RETURN_ENCODER(Int32RleEncoder); case INT64: case TIMESTAMP: ALLOC_AND_RETURN_ENCODER(Int64RleEncoder); + case FLOAT: + ALLOC_AND_RETURN_ENCODER(FloatRLEEncoder); + case DOUBLE: + ALLOC_AND_RETURN_ENCODER(DoubleRLEEncoder); default: return nullptr; } @@ -156,8 +163,10 @@ class EncoderFactory { case ZIGZAG: switch (data_type) { case INT32: + case DATE: ALLOC_AND_RETURN_ENCODER(IntZigzagEncoder); case INT64: + case TIMESTAMP: ALLOC_AND_RETURN_ENCODER(LongZigzagEncoder); default: return nullptr; @@ -166,8 +175,10 @@ class EncoderFactory { case SPRINTZ: switch (data_type) { case INT32: + case DATE: ALLOC_AND_RETURN_ENCODER(Int32SprintzEncoder); case INT64: + case TIMESTAMP: ALLOC_AND_RETURN_ENCODER(Int64SprintzEncoder); case FLOAT: ALLOC_AND_RETURN_ENCODER(FloatSprintzEncoder); @@ -201,9 +212,18 @@ class EncoderFactory { return nullptr; } + case GORILLA_V1: + switch (data_type) { + case FLOAT: + ALLOC_AND_RETURN_ENCODER(FloatGorillaV1Encoder); + case DOUBLE: + ALLOC_AND_RETURN_ENCODER(DoubleGorillaV1Encoder); + default: + return nullptr; + } + case DIFF: case BITMAP: - case GORILLA_V1: case REGULAR: case FREQ: return nullptr; diff --git a/cpp/src/encoding/fire.h b/cpp/src/encoding/fire.h index 0ca3d16804..58ad5a4943 100644 --- a/cpp/src/encoding/fire.h +++ b/cpp/src/encoding/fire.h @@ -64,10 +64,15 @@ class IntFire : public Fire { int predict(int value) override { int alpha = accumulator_ >> learn_shift_; - int diff = static_cast((static_cast(alpha) * delta_)) >> - bit_width_; - - return value + diff; + // Mirror the Java/IoTDB IntFire exactly: a 32-bit wrapping multiply + // followed by an arithmetic shift. Using a wider intermediate product + // here would define a different predictor and make SPRINTZ streams + // unreadable by the other implementations. + int product = static_cast(static_cast(alpha) * + static_cast(delta_)); + int diff = product >> bit_width_; + return static_cast(static_cast(value) + + static_cast(diff)); } void train(int pre, int val, int err) override { @@ -92,8 +97,13 @@ class LongFire : public Fire { int64_t predict(int64_t value) override { int64_t alpha = accumulator_ >> learn_shift_; - int64_t diff = safe_mul_shift(alpha, delta_, bit_width_); - return value + diff; + // See IntFire::predict: match the Java/IoTDB LongFire's 64-bit + // wrapping multiply. A 128-bit product would diverge from the format. + int64_t product = static_cast(static_cast(alpha) * + static_cast(delta_)); + int64_t diff = product >> bit_width_; + return static_cast(static_cast(value) + + static_cast(diff)); } void train(int64_t pre, int64_t val, int64_t err) override { @@ -101,22 +111,6 @@ class LongFire : public Fire { accumulator_ -= gradient; delta_ = val - pre; } - - private: - /** (alpha * delta_) >> shift without signed overflow; both args are - * int64_t. */ - static int64_t safe_mul_shift(int64_t alpha, int64_t delta, int shift) { -#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ >= 16 - __int128 product = static_cast<__int128>(alpha) * delta; - return static_cast(product >> shift); -#else - /* Portable fallback: use double for product. Exact for |alpha|,|delta| - * < 2^53. */ - double prod = static_cast(alpha) * static_cast(delta); - double div = static_cast(1LL << shift); - return static_cast(prod / div); -#endif - } }; #endif // ENCODING_FIRE_H diff --git a/cpp/src/encoding/float_rle_decoder.h b/cpp/src/encoding/float_rle_decoder.h new file mode 100644 index 0000000000..331b0c124b --- /dev/null +++ b/cpp/src/encoding/float_rle_decoder.h @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_FLOAT_RLE_DECODER_H +#define ENCODING_FLOAT_RLE_DECODER_H + +#include +#include +#include + +#include "common/allocator/byte_stream.h" +#include "decoder.h" +#include "encoding/int32_rle_decoder.h" +#include "encoding/int64_rle_decoder.h" +#include "encoding/ts2diff_decoder.h" // page metadata helpers + +namespace storage { + +// Decoder for the Java FloatEncoder RLE layout; see FloatRLEEncoder. +class FloatRLEDecoder : public Decoder { + public: + FloatRLEDecoder() { reset(); } + ~FloatRLEDecoder() override = default; + + void reset() override { + int_decoder_.reset(); + page_meta_parsed_ = false; + max_point_value_ = 1.0; + page_pos_ = 0; + scaled_bm_.clear(); + raw_bm_.clear(); + } + + bool has_remaining(const common::ByteStream& buffer) override { + return int_decoder_.has_remaining(buffer); + } + + int read_boolean(bool& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int32(int32_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int64(int64_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_float(float& ret_value, common::ByteStream& in) override { + int ret = ensure_page_meta(in); + if (RET_FAIL(ret)) { + return ret; + } + int32_t value_int = 0; + if (RET_FAIL(int_decoder_.read_int32(value_int, in))) { + return ret; + } + ret_value = value_at(value_int); + page_pos_++; + return ret; + } + int read_double(double& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_String(common::String& /*ret_value*/, common::PageArena& /*pa*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + + private: + int ensure_page_meta(common::ByteStream& in) { + if (page_meta_parsed_) { + return common::E_OK; + } + ts2diff_java_detail::PageMeta meta; + int ret = ts2diff_java_detail::read_page_meta(in, meta); + if (RET_FAIL(ret)) { + return ret; + } + max_point_value_ = + meta.max_point_number <= 0 + ? 1.0 + : std::pow(10.0, static_cast(meta.max_point_number)); + scaled_bm_ = std::move(meta.scaled_bm); + raw_bm_ = std::move(meta.raw_bm); + page_pos_ = 0; + page_meta_parsed_ = true; + return common::E_OK; + } + + float value_at(int32_t value_int) const { + if (!raw_bm_.empty() && + ts2diff_java_detail::bitmap_marked(raw_bm_, page_pos_)) { + return common::int_to_float(value_int); + } + const bool use_scaled = + scaled_bm_.empty() || + ts2diff_java_detail::bitmap_marked(scaled_bm_, page_pos_); + const double divisor = use_scaled ? max_point_value_ : 1.0; + return static_cast(static_cast(value_int) / divisor); + } + + Int32RleDecoder int_decoder_; + double max_point_value_ = 1.0; + int page_pos_ = 0; + bool page_meta_parsed_ = false; + std::vector scaled_bm_; + std::vector raw_bm_; +}; + +class DoubleRLEDecoder : public Decoder { + public: + DoubleRLEDecoder() { reset(); } + ~DoubleRLEDecoder() override = default; + + void reset() override { + int_decoder_.reset(); + page_meta_parsed_ = false; + max_point_value_ = 1.0; + page_pos_ = 0; + scaled_bm_.clear(); + raw_bm_.clear(); + } + + bool has_remaining(const common::ByteStream& buffer) override { + return int_decoder_.has_remaining(buffer); + } + + int read_boolean(bool& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int32(int32_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int64(int64_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_float(float& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_double(double& ret_value, common::ByteStream& in) override { + int ret = ensure_page_meta(in); + if (RET_FAIL(ret)) { + return ret; + } + int64_t value_long = 0; + if (RET_FAIL(int_decoder_.read_int64(value_long, in))) { + return ret; + } + ret_value = value_at(value_long); + page_pos_++; + return ret; + } + int read_String(common::String& /*ret_value*/, common::PageArena& /*pa*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + + private: + int ensure_page_meta(common::ByteStream& in) { + if (page_meta_parsed_) { + return common::E_OK; + } + ts2diff_java_detail::PageMeta meta; + int ret = ts2diff_java_detail::read_page_meta(in, meta); + if (RET_FAIL(ret)) { + return ret; + } + max_point_value_ = + meta.max_point_number <= 0 + ? 1.0 + : std::pow(10.0, static_cast(meta.max_point_number)); + scaled_bm_ = std::move(meta.scaled_bm); + raw_bm_ = std::move(meta.raw_bm); + page_pos_ = 0; + page_meta_parsed_ = true; + return common::E_OK; + } + + double value_at(int64_t value_long) const { + if (!raw_bm_.empty() && + ts2diff_java_detail::bitmap_marked(raw_bm_, page_pos_)) { + return common::long_to_double(value_long); + } + const bool use_scaled = + scaled_bm_.empty() || + ts2diff_java_detail::bitmap_marked(scaled_bm_, page_pos_); + const double divisor = use_scaled ? max_point_value_ : 1.0; + return static_cast(value_long) / divisor; + } + + Int64RleDecoder int_decoder_; + double max_point_value_ = 1.0; + int page_pos_ = 0; + bool page_meta_parsed_ = false; + std::vector scaled_bm_; + std::vector raw_bm_; +}; + +} // namespace storage + +#endif // ENCODING_FLOAT_RLE_DECODER_H diff --git a/cpp/src/encoding/float_rle_encoder.h b/cpp/src/encoding/float_rle_encoder.h new file mode 100644 index 0000000000..ace4565ec2 --- /dev/null +++ b/cpp/src/encoding/float_rle_encoder.h @@ -0,0 +1,321 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_FLOAT_RLE_ENCODER_H +#define ENCODING_FLOAT_RLE_ENCODER_H + +#include +#include +#include +#include + +#include "common/allocator/byte_stream.h" +#include "encoder.h" +#include "encoding/int32_rle_encoder.h" +#include "encoding/int64_rle_encoder.h" +#include "encoding/ts2diff_encoder.h" // java_round +#include "encoding/ts2diff_wire_format.h" // overflow markers + +namespace storage { + +// FLOAT/DOUBLE RLE is the Java FloatEncoder wrapper: values are converted to +// scaled integers (with raw-bit fallbacks) and stored with the 32/64-bit RLE +// codec. The page layout is the same as the TS_2DIFF float wrapper: +// no overflow: [maxPointNumber varint][RLE stream] +// overflow: [marker varint][valueCount varint][scaled bitmap] +// [raw bitmap?][maxPointNumber varint][RLE stream] +class FloatRLEEncoder : public Encoder { + public: + FloatRLEEncoder() { reset(); } + ~FloatRLEEncoder() override = default; + + void reset() override { + int_encoder_.reset(); + underflow_flags_.clear(); + } + void destroy() override {} + + int encode(bool /*value*/, common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int32_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int64_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(float value, common::ByteStream& out_stream) override { + return int_encoder_.encode(convert_float_to_int(value), out_stream); + } + int encode(double /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(common::String /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + + int flush(common::ByteStream& out_stream) override { + int ret = common::E_OK; + if (underflow_flags_.empty()) { + return ret; + } + common::ByteStream data(1024, common::MOD_ENCODER_OBJ, false); + if (RET_FAIL(int_encoder_.flush(data))) { + return ret; + } + const int num_values = static_cast(underflow_flags_.size()); + if (has_overflow()) { + std::vector scaled_bitmap( + static_cast(num_values / 8 + 1), 0); + std::vector raw_bits_bitmap( + static_cast(num_values / 8 + 1), 0); + bool has_raw_bits = false; + for (int i = 0; i < num_values; i++) { + int8_t f = underflow_flags_[static_cast(i)]; + if (f == 1) { + scaled_bitmap[static_cast(i / 8)] |= + static_cast(1u << (i % 8)); + } else if (f == -1) { + has_raw_bits = true; + raw_bits_bitmap[static_cast(i / 8)] |= + static_cast(1u << (i % 8)); + } + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + has_raw_bits + ? ts2diff_java_detail::FLAG_ORIGINAL_VALUE_OVERFLOW + : ts2diff_java_detail::FLAG_SCALED_VALUE_OVERFLOW, + out_stream))) { + return ret; + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + static_cast(num_values), out_stream))) { + return ret; + } + const uint32_t bm_len = static_cast(num_values / 8 + 1); + if (RET_FAIL( + out_stream.write_buf(scaled_bitmap.data(), bm_len))) { + return ret; + } + if (has_raw_bits && + RET_FAIL( + out_stream.write_buf(raw_bits_bitmap.data(), bm_len))) { + return ret; + } + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + static_cast(max_point_number_), out_stream))) { + return ret; + } + if (RET_FAIL(common::merge_byte_stream(out_stream, data))) { + return ret; + } + underflow_flags_.clear(); + return ret; + } + + int get_max_byte_size() override { + return int_encoder_.get_max_byte_size() + 16 + + static_cast(underflow_flags_.size() / 4 + 2); + } + + private: + int32_t convert_float_to_int(float value) { + const double scaled = static_cast(value) * max_point_value_; + if (scaled > static_cast(std::numeric_limits::max()) || + scaled < static_cast(std::numeric_limits::min())) { + if (std::isnan(value) || + value > + static_cast(std::numeric_limits::max()) || + value < + static_cast(std::numeric_limits::min())) { + underflow_flags_.push_back(-1); + return common::float_to_int(value); + } + underflow_flags_.push_back(0); + return java_round(value); + } + if (std::isnan(value)) { + underflow_flags_.push_back(-1); + return common::float_to_int(value); + } + underflow_flags_.push_back(1); + return java_round(scaled); + } + + bool has_overflow() const { + for (int8_t f : underflow_flags_) { + if (f != 1) { + return true; + } + } + return false; + } + + Int32RleEncoder int_encoder_; + std::vector underflow_flags_; + // Java Rle builder default: TSFileConfig.floatPrecision, currently 2. + int max_point_number_ = 2; + double max_point_value_ = 100.0; +}; + +class DoubleRLEEncoder : public Encoder { + public: + DoubleRLEEncoder() { reset(); } + ~DoubleRLEEncoder() override = default; + + void reset() override { + int_encoder_.reset(); + underflow_flags_.clear(); + } + void destroy() override {} + + int encode(bool /*value*/, common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int32_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int64_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(float /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(double value, common::ByteStream& out_stream) override { + return int_encoder_.encode(convert_double_to_long(value), out_stream); + } + int encode(common::String /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + + int flush(common::ByteStream& out_stream) override { + int ret = common::E_OK; + if (underflow_flags_.empty()) { + return ret; + } + common::ByteStream data(1024, common::MOD_ENCODER_OBJ, false); + if (RET_FAIL(int_encoder_.flush(data))) { + return ret; + } + const int num_values = static_cast(underflow_flags_.size()); + if (has_overflow()) { + std::vector scaled_bitmap( + static_cast(num_values / 8 + 1), 0); + std::vector raw_bits_bitmap( + static_cast(num_values / 8 + 1), 0); + bool has_raw_bits = false; + for (int i = 0; i < num_values; i++) { + int8_t f = underflow_flags_[static_cast(i)]; + if (f == 1) { + scaled_bitmap[static_cast(i / 8)] |= + static_cast(1u << (i % 8)); + } else if (f == -1) { + has_raw_bits = true; + raw_bits_bitmap[static_cast(i / 8)] |= + static_cast(1u << (i % 8)); + } + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + has_raw_bits + ? ts2diff_java_detail::FLAG_ORIGINAL_VALUE_OVERFLOW + : ts2diff_java_detail::FLAG_SCALED_VALUE_OVERFLOW, + out_stream))) { + return ret; + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + static_cast(num_values), out_stream))) { + return ret; + } + const uint32_t bm_len = static_cast(num_values / 8 + 1); + if (RET_FAIL( + out_stream.write_buf(scaled_bitmap.data(), bm_len))) { + return ret; + } + if (has_raw_bits && + RET_FAIL( + out_stream.write_buf(raw_bits_bitmap.data(), bm_len))) { + return ret; + } + } + if (RET_FAIL(common::SerializationUtil::write_var_uint( + static_cast(max_point_number_), out_stream))) { + return ret; + } + if (RET_FAIL(common::merge_byte_stream(out_stream, data))) { + return ret; + } + underflow_flags_.clear(); + return ret; + } + + int get_max_byte_size() override { + return int_encoder_.get_max_byte_size() + 16 + + static_cast(underflow_flags_.size() / 4 + 2); + } + + private: + int64_t convert_double_to_long(double value) { + const double scaled = value * max_point_value_; + if (scaled > static_cast(std::numeric_limits::max()) || + scaled < static_cast(std::numeric_limits::min())) { + if (std::isnan(value) || + value > static_cast(std::numeric_limits::max()) || + value < + static_cast(std::numeric_limits::min())) { + underflow_flags_.push_back(-1); + return common::double_to_long(value); + } + underflow_flags_.push_back(0); + return java_round(value); + } + if (std::isnan(value)) { + underflow_flags_.push_back(-1); + return common::double_to_long(value); + } + underflow_flags_.push_back(1); + return java_round(scaled); + } + + bool has_overflow() const { + for (int8_t f : underflow_flags_) { + if (f != 1) { + return true; + } + } + return false; + } + + Int64RleEncoder int_encoder_; + std::vector underflow_flags_; + int max_point_number_ = 2; + double max_point_value_ = 100.0; +}; + +} // namespace storage + +#endif // ENCODING_FLOAT_RLE_ENCODER_H diff --git a/cpp/src/encoding/gorilla_v1_decoder.h b/cpp/src/encoding/gorilla_v1_decoder.h new file mode 100644 index 0000000000..01c53f28bc --- /dev/null +++ b/cpp/src/encoding/gorilla_v1_decoder.h @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_GORILLA_V1_DECODER_H +#define ENCODING_GORILLA_V1_DECODER_H + +#include +#include + +#include "common/allocator/byte_stream.h" +#include "decoder.h" +#include "encoding/encode_utils.h" + +namespace storage { + +// Decoder for the original Gorilla (GORILLA_V1) layout written by Java's +// SinglePrecisionEncoderV1/DoublePrecisionEncoderV1. Values are decoded one +// step ahead, exactly like the Java decoder, because the NaN sentinel that +// flush() appends is what marks the end of the stream. +class GorillaV1DecoderBase : public Decoder { + public: + void reset() override { reset_state(); } + + protected: + void reset_state() { + flag_ = false; + leading_zero_num_ = 0; + trailing_zero_num_ = 0; + is_end_ = false; + buffer_ = 0; + bits_left_ = 0; + } + + bool read_bit(common::ByteStream& in) { + if (bits_left_ == 0 && !is_end_) { + fill_buffer(in); + } + if (buffer_ < 0) { + return false; + } + --bits_left_; + return ((buffer_ >> bits_left_) & 1) == 1; + } + + void fill_buffer(common::ByteStream& in) { + uint8_t byte = 0; + uint32_t read_len = 0; + if (in.read_buf(&byte, 1, read_len) == common::E_OK && read_len == 1) { + buffer_ = byte; + bits_left_ = 8; + } else { + buffer_ = -1; + bits_left_ = -1; + } + } + + int read_bits_as_int(common::ByteStream& in, int len) { + int num = 0; + for (int i = 0; i < len; i++) { + num = (num << 1) | (read_bit(in) ? 1 : 0); + } + return num; + } + + bool flag_ = false; + int leading_zero_num_ = 0; + int trailing_zero_num_ = 0; + bool is_end_ = false; + int buffer_ = 0; + int bits_left_ = 0; +}; + +class FloatGorillaV1Decoder : public GorillaV1DecoderBase { + public: + bool has_remaining(const common::ByteStream& buffer) override { + return buffer.has_remaining(); + } + + int read_float(float& ret_value, common::ByteStream& in) override { + if (!flag_) { + flag_ = true; + uint8_t bytes[4] = {0, 0, 0, 0}; + uint32_t read_len = 0; + if (in.read_buf(bytes, 4, read_len) != common::E_OK || + read_len != 4) { + return common::E_DECODE_ERR; + } + pre_value_ = static_cast(bytes[0]) | + (static_cast(bytes[1]) << 8) | + (static_cast(bytes[2]) << 16) | + (static_cast(bytes[3]) << 24); + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + ret_value = common::int_to_float(static_cast(pre_value_)); + prefetch(in); + return common::E_OK; + } + ret_value = common::int_to_float(static_cast(pre_value_)); + prefetch(in); + return common::E_OK; + } + + int read_boolean(bool& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int32(int32_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int64(int64_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_double(double& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_String(common::String& /*ret_value*/, common::PageArena& /*pa*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + + private: + void prefetch(common::ByteStream& in) { + const bool next_flag1 = read_bit(in); + if (!next_flag1) { + return; + } + const bool next_flag2 = read_bit(in); + if (!next_flag2) { + const int len = 32 - leading_zero_num_ - trailing_zero_num_; + if (len < 0 || len > 32) { + is_end_ = true; + return; + } + uint32_t xor_value = 0; + for (int i = 0; i < len; i++) { + xor_value = (xor_value << 1) | (read_bit(in) ? 1u : 0u); + } + xor_value <<= trailing_zero_num_; + pre_value_ = xor_value ^ pre_value_; + } else { + const int leading = read_bits_as_int(in, 5); + const int len = read_bits_as_int(in, 6); + if (len < 0 || leading < 0 || leading + len > 32) { + is_end_ = true; + return; + } + uint32_t xor_value = 0; + for (int i = 0; i < len; i++) { + xor_value = (xor_value << 1) | (read_bit(in) ? 1u : 0u); + } + xor_value <<= (32 - leading - len); + pre_value_ = xor_value ^ pre_value_; + } + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + if (std::isnan(common::int_to_float(static_cast(pre_value_)))) { + is_end_ = true; + } + } + + uint32_t pre_value_ = 0; +}; + +class DoubleGorillaV1Decoder : public GorillaV1DecoderBase { + public: + bool has_remaining(const common::ByteStream& buffer) override { + return buffer.has_remaining(); + } + + int read_double(double& ret_value, common::ByteStream& in) override { + if (!flag_) { + flag_ = true; + uint8_t bytes[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + uint32_t read_len = 0; + if (in.read_buf(bytes, 8, read_len) != common::E_OK || + read_len != 8) { + return common::E_DECODE_ERR; + } + pre_value_ = 0; + for (int i = 0; i < 8; i++) { + pre_value_ |= static_cast(bytes[i]) << (i * 8); + } + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + ret_value = + common::long_to_double(static_cast(pre_value_)); + prefetch(in); + return common::E_OK; + } + ret_value = common::long_to_double(static_cast(pre_value_)); + prefetch(in); + return common::E_OK; + } + + int read_boolean(bool& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int32(int32_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_int64(int64_t& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_float(float& /*ret_value*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + int read_String(common::String& /*ret_value*/, common::PageArena& /*pa*/, + common::ByteStream& /*in*/) override { + return common::E_TYPE_NOT_MATCH; + } + + private: + void prefetch(common::ByteStream& in) { + const bool next_flag1 = read_bit(in); + if (!next_flag1) { + return; + } + const bool next_flag2 = read_bit(in); + if (!next_flag2) { + const int len = 64 - leading_zero_num_ - trailing_zero_num_; + if (len < 0 || len > 64) { + is_end_ = true; + return; + } + uint64_t xor_value = 0; + for (int i = 0; i < len; i++) { + xor_value = (xor_value << 1) | (read_bit(in) ? 1ULL : 0ULL); + } + xor_value <<= trailing_zero_num_; + pre_value_ = xor_value ^ pre_value_; + } else { + const int leading = read_bits_as_int(in, 6); + const int len = read_bits_as_int(in, 7); + if (len < 0 || leading < 0 || leading + len > 64) { + is_end_ = true; + return; + } + uint64_t xor_value = 0; + for (int i = 0; i < len; i++) { + xor_value = (xor_value << 1) | (read_bit(in) ? 1ULL : 0ULL); + } + xor_value <<= (64 - leading - len); + pre_value_ = xor_value ^ pre_value_; + } + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + if (std::isnan( + common::long_to_double(static_cast(pre_value_)))) { + is_end_ = true; + } + } + + uint64_t pre_value_ = 0; +}; + +} // namespace storage + +#endif // ENCODING_GORILLA_V1_DECODER_H diff --git a/cpp/src/encoding/gorilla_v1_encoder.h b/cpp/src/encoding/gorilla_v1_encoder.h new file mode 100644 index 0000000000..e77f08141e --- /dev/null +++ b/cpp/src/encoding/gorilla_v1_encoder.h @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#ifndef ENCODING_GORILLA_V1_ENCODER_H +#define ENCODING_GORILLA_V1_ENCODER_H + +#include +#include + +#include "common/allocator/byte_stream.h" +#include "encoder.h" +#include "encoding/encode_utils.h" + +namespace storage { + +// The original Gorilla (GORILLA_V1) bit layout used by Java's +// SinglePrecisionEncoderV1/DoublePrecisionEncoderV1: the first value is +// stored raw little-endian, later values are an MSB-first bit stream of +// reuse-XOR ('0' / '10') or explicit-XOR ('11') encodings, terminated by a +// NaN sentinel written by flush(). +class GorillaV1EncoderBase : public Encoder { + public: + void destroy() override {} + + protected: + void reset_bits() { + flag_ = false; + leading_zero_num_ = 0; + trailing_zero_num_ = 0; + buffer_ = 0; + bits_in_buffer_ = 0; + } + + void write_bit(bool bit, common::ByteStream& out_stream) { + buffer_ = static_cast(buffer_ << 1); + if (bit) { + buffer_ |= 1u; + } + if (++bits_in_buffer_ == 8) { + clear_buffer(out_stream); + } + } + + void write_bits(uint64_t num, common::ByteStream& out_stream, int start, + int end) { + for (int i = start; i >= end; i--) { + write_bit(((num >> i) & 1ULL) != 0, out_stream); + } + } + + void clear_buffer(common::ByteStream& out_stream) { + if (bits_in_buffer_ == 0) { + return; + } + buffer_ = static_cast(buffer_ << (8 - bits_in_buffer_)); + out_stream.write_buf(&buffer_, 1); + bits_in_buffer_ = 0; + buffer_ = 0; + } + + bool flag_ = false; + int leading_zero_num_ = 0; + int trailing_zero_num_ = 0; + uint8_t buffer_ = 0; + int bits_in_buffer_ = 0; +}; + +class FloatGorillaV1Encoder : public GorillaV1EncoderBase { + public: + void reset() override { reset_bits(); } + + int encode(float value, common::ByteStream& out_stream) override { + uint32_t next_value = float_bits(value); + if (!flag_) { + flag_ = true; + pre_value_ = next_value; + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + uint8_t bytes[4]; + for (int i = 0; i < 4; i++) { + bytes[i] = static_cast(pre_value_ >> (i * 8)); + } + return out_stream.write_buf(bytes, 4); + } + const uint32_t xor_value = next_value ^ pre_value_; + if (xor_value == 0) { + write_bit(false, out_stream); + } else { + const int leading = number_of_leading_zeros( + static_cast(xor_value)); + const int trailing = number_of_trailing_zeros( + static_cast(xor_value)); + if (leading >= leading_zero_num_ && + trailing >= trailing_zero_num_) { + write_bit(true, out_stream); + write_bit(false, out_stream); + write_bits(xor_value, out_stream, 31 - leading_zero_num_, + trailing_zero_num_); + } else { + write_bit(true, out_stream); + write_bit(true, out_stream); + write_bits(static_cast(leading), out_stream, 4, 0); + write_bits( + static_cast(32 - leading - trailing), + out_stream, 5, 0); + write_bits(xor_value, out_stream, 31 - leading, trailing); + } + } + pre_value_ = next_value; + leading_zero_num_ = + number_of_leading_zeros(static_cast(pre_value_)); + trailing_zero_num_ = + number_of_trailing_zeros(static_cast(pre_value_)); + return common::E_OK; + } + + int encode(bool /*value*/, common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int32_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int64_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(double /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(common::String /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + + int flush(common::ByteStream& out_stream) override { + int ret = encode(std::nanf(""), out_stream); + if (RET_FAIL(ret)) { + return ret; + } + clear_buffer(out_stream); + reset(); + return common::E_OK; + } + + int get_max_byte_size() override { return 12; } + + private: + static uint32_t float_bits(float value) { + // Java uses Float.floatToIntBits, which canonicalizes every NaN. + if (std::isnan(value)) { + return 0x7fc00000U; + } + return static_cast(common::float_to_int(value)); + } + + uint32_t pre_value_ = 0; +}; + +class DoubleGorillaV1Encoder : public GorillaV1EncoderBase { + public: + void reset() override { reset_bits(); } + + int encode(double value, common::ByteStream& out_stream) override { + uint64_t next_value = double_bits(value); + if (!flag_) { + flag_ = true; + pre_value_ = next_value; + leading_zero_num_ = number_of_leading_zeros( + static_cast(pre_value_)); + trailing_zero_num_ = number_of_trailing_zeros( + static_cast(pre_value_)); + uint8_t bytes[8]; + for (int i = 0; i < 8; i++) { + bytes[i] = static_cast(pre_value_ >> (i * 8)); + } + return out_stream.write_buf(bytes, 8); + } + const uint64_t xor_value = next_value ^ pre_value_; + if (xor_value == 0) { + write_bit(false, out_stream); + } else { + const int leading = number_of_leading_zeros( + static_cast(xor_value)); + const int trailing = number_of_trailing_zeros( + static_cast(xor_value)); + if (leading >= leading_zero_num_ && + trailing >= trailing_zero_num_) { + write_bit(true, out_stream); + write_bit(false, out_stream); + write_bits(xor_value, out_stream, 63 - leading_zero_num_, + trailing_zero_num_); + } else { + write_bit(true, out_stream); + write_bit(true, out_stream); + write_bits(static_cast(leading), out_stream, 5, 0); + write_bits( + static_cast(64 - leading - trailing), + out_stream, 6, 0); + write_bits(xor_value, out_stream, 63 - leading, trailing); + } + } + pre_value_ = next_value; + leading_zero_num_ = number_of_leading_zeros( + static_cast(pre_value_)); + trailing_zero_num_ = number_of_trailing_zeros( + static_cast(pre_value_)); + return common::E_OK; + } + + int encode(bool /*value*/, common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int32_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(int64_t /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(float /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + int encode(common::String /*value*/, + common::ByteStream& /*out_stream*/) override { + return common::E_TYPE_NOT_MATCH; + } + + int flush(common::ByteStream& out_stream) override { + int ret = encode(std::nan(""), out_stream); + if (RET_FAIL(ret)) { + return ret; + } + clear_buffer(out_stream); + reset(); + return common::E_OK; + } + + int get_max_byte_size() override { return 20; } + + private: + static uint64_t double_bits(double value) { + // Java uses Double.doubleToLongBits, which canonicalizes every NaN. + if (std::isnan(value)) { + return UINT64_C(0x7ff8000000000000); + } + return static_cast(common::double_to_long(value)); + } + + uint64_t pre_value_ = 0; +}; + +} // namespace storage + +#endif // ENCODING_GORILLA_V1_ENCODER_H diff --git a/cpp/src/encoding/int64_rle_decoder.h b/cpp/src/encoding/int64_rle_decoder.h index df8e17838a..bcbfe6f6ec 100644 --- a/cpp/src/encoding/int64_rle_decoder.h +++ b/cpp/src/encoding/int64_rle_decoder.h @@ -150,14 +150,18 @@ class Int64RleDecoder : public Decoder { return common::E_DECODE_ERR; } int byte_width = (bit_width_ + 7) / 8; - // Read the repeated value (stored as byte_width bytes, little-endian) + // The repeated value is stored big-endian (most significant byte + // first), matching the Java/IoTDB LongRleEncoder + // (BytesUtils.longToBytes(value, byteWidth)). This differs from the + // INT32 RLE path, which stores the repeated value little-endian. int64_t value = 0; for (int i = 0; i < byte_width; i++) { uint8_t b; if (RET_FAIL(common::SerializationUtil::read_ui8(b, byte_cache_))) { return ret; } - value |= ((int64_t)b) << (i * 8); + value = static_cast( + (static_cast(value) << 8) | b); } rle_value_ = value; is_rle_run_ = true; diff --git a/cpp/test/reader/table_view/encoding_compression_compatibility_test.cc b/cpp/test/reader/table_view/encoding_compression_compatibility_test.cc new file mode 100644 index 0000000000..17eadb5686 --- /dev/null +++ b/cpp/test/reader/table_view/encoding_compression_compatibility_test.cc @@ -0,0 +1,1918 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#include +#endif +#include + +#include "common/config/config.h" +#include "common/db_common.h" +#include "common/record.h" +#include "common/row_record.h" +#include "common/schema.h" +#include "common/tablet.h" +#include "compress/compressor_factory.h" +#include "encoding/decoder_factory.h" +#include "encoding/encoder_factory.h" +#include "file/local_random_access_read_file.h" +#include "file/write_file.h" +#include "reader/result_set.h" +#include "reader/table_result_set.h" +#include "reader/tsfile_reader.h" +#include "reader/tsfile_tree_reader.h" +#include "writer/tsfile_table_writer.h" +#include "writer/tsfile_tree_writer.h" + +using namespace common; +using namespace storage; + +namespace { + +const char* const kSpecEnv = "TSFILE_COMPAT_SPEC"; +const char* const kGenerateDirEnv = "TSFILE_COMPAT_GENERATE_DIR"; +const char* const kValidateDirEnv = "TSFILE_COMPAT_VALIDATE_DIR"; +const char* const kDefaultSpec = "compat/compatibility-matrix.csv"; +const char* const kLanguage = "cpp"; + +const char* const kManifestFile = "manifest.csv"; +const char* const kReportFile = "report.csv"; +const char* const kSummaryFile = "summary.md"; + +const char* const kSpecHeader = + "id,model,dataType,encoding,compressions,rowCount,pageRows,chunks,devices," + "valueProfile,nullPatterns,javaWrite,cppWrite,javaRead,cppRead,gapReason"; +const char* const kManifestHeader = + "producer,id,model,dataType,encoding,compression,rowCount,pageRows,chunks," + "devices,valueProfile,nullPattern"; + +const char* const kTableName = "compat_table"; +const char* const kTagColumn = "device"; +const char* const kValueColumn = "value"; +const char* const kAnchorColumn = "anchor"; +const char* const kDevicePrefix = "compat_table_device_"; + +const char* const kAllCompressions[] = {"UNCOMPRESSED", "SNAPPY", "GZIP", + "LZ4", "ZSTD", "LZMA2"}; + +// Concrete storage types covered by the matrix; see the Java counterpart. +const TSDataType kMatrixDataTypes[] = {BOOLEAN, INT32, INT64, TIMESTAMP, + DATE, FLOAT, DOUBLE, TEXT, + STRING, BLOB}; + +enum class Model { TABLE, TREE }; +enum class NullPattern { NONE, MIXED, ALL }; +enum class ValueProfile { + DEFAULT, + TS2DIFF, + TS2DIFF_MPN0, + SPRINTZ, + RUNS, + STRINGS, + DICT_HIGH, + BLOB +}; + +const char* ModelName(Model model) { + return model == Model::TABLE ? "TABLE" : "TREE"; +} + +const char* NullPatternName(NullPattern pattern) { + switch (pattern) { + case NullPattern::NONE: + return "NONE"; + case NullPattern::MIXED: + return "MIXED"; + case NullPattern::ALL: + return "ALL"; + } + return "NONE"; +} + +const char* ProfileName(ValueProfile profile) { + switch (profile) { + case ValueProfile::DEFAULT: + return "DEFAULT"; + case ValueProfile::TS2DIFF: + return "TS2DIFF"; + case ValueProfile::TS2DIFF_MPN0: + return "TS2DIFF_MPN0"; + case ValueProfile::SPRINTZ: + return "SPRINTZ"; + case ValueProfile::RUNS: + return "RUNS"; + case ValueProfile::STRINGS: + return "STRINGS"; + case ValueProfile::DICT_HIGH: + return "DICT_HIGH"; + case ValueProfile::BLOB: + return "BLOB"; + } + return "DEFAULT"; +} + +std::string Upper(std::string value) { + for (char& c : value) { + c = static_cast(std::toupper(static_cast(c))); + } + return value; +} + +std::string Lower(std::string value) { + for (char& c : value) { + c = static_cast(std::tolower(static_cast(c))); + } + return value; +} + +struct FixtureCase { + std::string spec_id; + std::string id; + Model model = Model::TABLE; + TSDataType data_type = INVALID_DATATYPE; + TSEncoding encoding = INVALID_ENCODING; + CompressionType compression = INVALID_COMPRESSION; + int row_count = 0; + int page_rows = 0; + int chunks = 1; + int devices = 1; + ValueProfile profile = ValueProfile::DEFAULT; + NullPattern null_pattern = NullPattern::NONE; + bool java_write = false; + bool cpp_write = false; + bool java_read = false; + bool cpp_read = false; + std::string gap_reason; + + bool Writes(const std::string& language) const { + return language == "java" ? java_write : cpp_write; + } + bool Reads(const std::string& language) const { + return language == "java" ? java_read : cpp_read; + } + std::string FileName() const { return id + ".tsfile"; } + std::string Describe() const { + return id + " (" + ModelName(model) + "/" + + get_data_type_name(data_type) + "/" + + get_encoding_name(encoding) + "/" + + get_compression_name(compression) + ")"; + } +}; + +// --------------------------------------------------------------------------- +// Deterministic value profiles, shared with the Java test +// --------------------------------------------------------------------------- + +const int32_t kIntValues[] = { + 0, + 1, + -1, + 7, + -8, + 1024, + -1024, + 123456, + -654321, + std::numeric_limits::max() - 1024, + std::numeric_limits::min() + 1024, + 42, + 42, + 0, + -8, + 2048, +}; + +const int64_t kLongValues[] = { + 0LL, + 1LL, + -1LL, + 7LL, + -8LL, + 1234567890LL, + -987654321LL, + 4294967296LL, + -4294967296LL, + 1234567890123LL, + -1234567890123LL, + std::numeric_limits::max() - 4096, + std::numeric_limits::min() + 4096, + 42LL, + 42LL, + 0LL, +}; + +const uint32_t kFloatBits[] = { + 0x00000000U, 0x80000000U, 0x3f800000U, 0xbf800000U, + 0x41280000U, 0xc0700000U, 0x44801000U, 0xc5002000U, + 0x40490fdbU, 0xc02df854U, 0x3f800000U, 0x3f800000U, + 0x00000000U, 0x80000000U, 0x42f6e979U, 0xc2f6e979U, +}; + +const uint64_t kDoubleBits[] = { + UINT64_C(0x0000000000000000), UINT64_C(0x8000000000000000), + UINT64_C(0x3ff0000000000000), UINT64_C(0xbff0000000000000), + UINT64_C(0x4029000000000000), UINT64_C(0xc00c000000000000), + UINT64_C(0x4090008000000000), UINT64_C(0xc0a0010000000000), + UINT64_C(0x400921fb54442d18), UINT64_C(0xc005bf0a8b145769), + UINT64_C(0x3ff0000000000000), UINT64_C(0x3ff0000000000000), + UINT64_C(0x0000000000000000), UINT64_C(0x8000000000000000), + UINT64_C(0x405edd2f1a9fbe77), UINT64_C(0xc05edd2f1a9fbe77), +}; + +const uint32_t kTs2DiffFloatBits[] = { + 0x00000000U, 0x3f800000U, 0xbf800000U, 0x461c4000U, + 0xc61c4000U, 0x4eb2d05eU, 0x7f800000U, 0xff800000U, + 0x7fc00000U, 0x3f800000U, 0x00000000U, 0x4a742400U, + 0xca742400U, 0x4e6e6b28U, 0x3f800000U, 0x00000000U, +}; + +const uint64_t kTs2DiffDoubleBits[] = { + UINT64_C(0x0000000000000000), UINT64_C(0x3ff0000000000000), + UINT64_C(0xbff0000000000000), UINT64_C(0x40c3880000000000), + UINT64_C(0xc0c3880000000000), UINT64_C(0x43abc16d674ec800), + UINT64_C(0x7ff0000000000000), UINT64_C(0xfff0000000000000), + UINT64_C(0x7ff8000000000000), UINT64_C(0x3ff0000000000000), + UINT64_C(0x0000000000000000), UINT64_C(0x41f0000000000000), + UINT64_C(0xc1cd6f3458800000), UINT64_C(0x430c6bf526340000), + UINT64_C(0x3ff0000000000000), UINT64_C(0x0000000000000000), +}; + +// SPRINTZ predicts from first-order differences of the stored integers; a +// block spanning INT_MIN and INT_MAX overflows the predictor, so this profile +// stays inside a safe range. Streaming numeric extremes are covered under the +// other encodings. +const int32_t kSprintzIntValues[] = { + 0, 1, -1, 7, -8, 1024, -1024, 123456, + -654321, 100000000, -100000000, 42, 42, 0, -8, 2048, +}; + +const int64_t kSprintzLongValues[] = { + 0LL, 1LL, -1LL, 7LL, -8LL, 1234567890LL, -987654321LL, 4294967296LL, + -4294967296LL, 1000000000000LL, -1000000000000LL, 42LL, 42LL, + 0LL, -8LL, 2048LL, +}; + +const uint32_t kSprintzFloatBits[] = { + 0x3f800000U, 0x3fa00000U, 0x3fc00000U, 0x3fe00000U, + 0x40000000U, 0x3fc00000U, 0x3fa00000U, 0x3f800000U, + 0x3fc00000U, 0x3fe00000U, 0x40000000U, 0x3fe00000U, + 0x3fc00000U, 0x3f800000U, 0x3fa00000U, 0x3fc00000U, +}; + +const uint64_t kSprintzDoubleBits[] = { + UINT64_C(0x3ff0000000000000), UINT64_C(0x3ff4000000000000), + UINT64_C(0x3ff8000000000000), UINT64_C(0x3ffc000000000000), + UINT64_C(0x4000000000000000), UINT64_C(0x3ff8000000000000), + UINT64_C(0x3ff4000000000000), UINT64_C(0x3ff0000000000000), + UINT64_C(0x3ff8000000000000), UINT64_C(0x3ffc000000000000), + UINT64_C(0x4000000000000000), UINT64_C(0x3ffc000000000000), + UINT64_C(0x3ff8000000000000), UINT64_C(0x3ff0000000000000), + UINT64_C(0x3ff4000000000000), UINT64_C(0x3ff8000000000000), +}; + +const int32_t kDateValues[] = { + 19700101, 19991231, 20000229, 20240229, + 20380119, 20500615, 19690720, 19800106, +}; + +std::string Repeat(char c, size_t count) { return std::string(count, c); } + +std::vector MakeStringValues() { + std::vector values; + values.emplace_back(""); + values.emplace_back("hello"); + values.emplace_back( + "\xE4\xBD\xA0\xE5\xA5\xBD\xEF\xBC\x8C\xE4\xB8\x96\xE7\x95\x8C"); + values.emplace_back(std::string("nul\0inside", 10)); + values.emplace_back( + "\xC3\x84\xC3\x96\xC3\x9C \xC3\x9F " + "\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"); + values.emplace_back("value"); + values.emplace_back("value"); + values.emplace_back(Repeat('a', 1024)); + values.emplace_back("\xF0\x9F\x98\x80 emoji \xF0\x9F\x8C\x8D"); + return values; +} + +const std::vector& StringValues() { + static const std::vector values = MakeStringValues(); + return values; +} + +std::vector MakeBlobValues() { + std::vector values; + values.emplace_back(""); + values.emplace_back(1, '\0'); + values.emplace_back(4, '\0'); + values.emplace_back("\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a", 10); + values.emplace_back("blob-as-text"); + values.emplace_back("\xff\xfe\x80\x00\x7f", 5); + values.emplace_back(""); + values.emplace_back("blob-as-text"); + return values; +} + +const std::vector& BlobValues() { + static const std::vector values = MakeBlobValues(); + return values; +} + +template +int32_t IntAt(const int32_t (&values)[N], int row) { + return values[static_cast(row) % N]; +} + +// Mirrors Java Math.round(x) == floor(x + 0.5). +double JavaRound(double x) { return std::floor(x + 0.5); } + +uint32_t FloatBits(float value) { + uint32_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +float BitsToFloat(uint32_t bits) { + float value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +uint64_t DoubleBits(double value) { + uint64_t bits; + std::memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +double BitsToDouble(uint64_t bits) { + double value; + std::memcpy(&value, &bits, sizeof(value)); + return value; +} + +template +uint32_t FloatBitsAt(const uint32_t (&values)[N], int row) { + return values[static_cast(row) % N]; +} + +template +uint64_t DoubleBitsAt(const uint64_t (&values)[N], int row) { + return values[static_cast(row) % N]; +} + +bool IsFloatEncoderCase(const FixtureCase& fixture_case) { + return (fixture_case.encoding == TS_2DIFF || + fixture_case.encoding == RLE) && + (fixture_case.data_type == FLOAT || + fixture_case.data_type == DOUBLE); +} + +double MaxPointValue(const FixtureCase& fixture_case) { + return fixture_case.profile == ValueProfile::TS2DIFF_MPN0 ? 1.0 : 100.0; +} + +bool BooleanValue(int row) { + return row % 192 < 64 ? row % 2 == 0 : row % 192 < 128; +} + +int32_t IntValue(const FixtureCase& fixture_case, int row) { + if (fixture_case.profile == ValueProfile::RUNS) { + return (row / 256) % 4 - 1; + } + if (fixture_case.profile == ValueProfile::SPRINTZ) { + return IntAt(kSprintzIntValues, row); + } + return IntAt(kIntValues, row); +} + +int64_t LongValue(const FixtureCase& fixture_case, int row) { + if (fixture_case.profile == ValueProfile::RUNS) { + return static_cast((row / 256) % 4 - 1) * 1000000000LL; + } + if (fixture_case.profile == ValueProfile::SPRINTZ) { + const size_t count = + sizeof(kSprintzLongValues) / sizeof(kSprintzLongValues[0]); + return kSprintzLongValues[static_cast(row) % count]; + } + const size_t count = sizeof(kLongValues) / sizeof(kLongValues[0]); + return kLongValues[static_cast(row) % count]; +} + +int32_t DateValue(const FixtureCase& fixture_case, int row) { + const size_t count = sizeof(kDateValues) / sizeof(kDateValues[0]); + if (fixture_case.profile == ValueProfile::RUNS) { + return kDateValues[static_cast(row / 256) % count]; + } + return kDateValues[static_cast(row) % count]; +} + +float FloatValue(const FixtureCase& fixture_case, int row) { + switch (fixture_case.profile) { + case ValueProfile::TS2DIFF: + case ValueProfile::TS2DIFF_MPN0: + return BitsToFloat(FloatBitsAt(kTs2DiffFloatBits, row)); + case ValueProfile::RUNS: + return static_cast((row / 256) % 4 - 1); + case ValueProfile::SPRINTZ: + return BitsToFloat(FloatBitsAt(kSprintzFloatBits, row)); + default: + return BitsToFloat(FloatBitsAt(kFloatBits, row)); + } +} + +double DoubleValue(const FixtureCase& fixture_case, int row) { + switch (fixture_case.profile) { + case ValueProfile::TS2DIFF: + case ValueProfile::TS2DIFF_MPN0: + return BitsToDouble(DoubleBitsAt(kTs2DiffDoubleBits, row)); + case ValueProfile::RUNS: + return static_cast((row / 256) % 4 - 1); + case ValueProfile::SPRINTZ: + return BitsToDouble(DoubleBitsAt(kSprintzDoubleBits, row)); + default: + return BitsToDouble(DoubleBitsAt(kDoubleBits, row)); + } +} + +std::string StringValue(const FixtureCase& fixture_case, int row) { + if (fixture_case.profile == ValueProfile::DICT_HIGH) { + return "key_" + std::to_string(row); + } + const std::vector& values = StringValues(); + return values[static_cast(row) % values.size()]; +} + +const std::string& BlobValue(const FixtureCase& fixture_case, int row) { + (void)fixture_case; + const std::vector& values = BlobValues(); + return values[static_cast(row) % values.size()]; +} + +float ExpectedFloatValue(const FixtureCase& fixture_case, int row) { + if (!IsFloatEncoderCase(fixture_case)) { + return FloatValue(fixture_case, row); + } + float value = FloatValue(fixture_case, row); + if (std::isnan(value)) { + return BitsToFloat(0x7fc00000U); + } + const double mpv = MaxPointValue(fixture_case); + const double scaled = static_cast(value) * mpv; + const bool scaled_overflow = + scaled > static_cast(std::numeric_limits::max()) || + scaled < static_cast(std::numeric_limits::min()); + if (scaled_overflow) { + const bool value_overflows = + value > static_cast(std::numeric_limits::max()) || + value < static_cast(std::numeric_limits::min()); + if (value_overflows) { + return value; + } + return static_cast(JavaRound(value)) / 1.0f; + } + return static_cast(JavaRound(scaled) / mpv); +} + +double ExpectedDoubleValue(const FixtureCase& fixture_case, int row) { + if (!IsFloatEncoderCase(fixture_case)) { + return DoubleValue(fixture_case, row); + } + double value = DoubleValue(fixture_case, row); + if (std::isnan(value)) { + return BitsToDouble(UINT64_C(0x7ff8000000000000)); + } + const double mpv = MaxPointValue(fixture_case); + const double scaled = value * mpv; + const bool scaled_overflow = + scaled > static_cast(std::numeric_limits::max()) || + scaled < static_cast(std::numeric_limits::min()); + if (scaled_overflow) { + const bool value_overflows = + value > static_cast(std::numeric_limits::max()) || + value < static_cast(std::numeric_limits::min()); + if (value_overflows) { + return value; + } + return JavaRound(value) / 1.0; + } + return JavaRound(scaled) / mpv; +} + +// --------------------------------------------------------------------------- +// Spec parsing +// --------------------------------------------------------------------------- + +std::vector Split(const std::string& line, char delimiter) { + std::vector parts; + std::stringstream stream(line); + std::string part; + while (std::getline(stream, part, delimiter)) { + parts.push_back(part); + } + if (!line.empty() && line.back() == delimiter) { + parts.emplace_back(); + } + return parts; +} + +bool ParseEncodingName(const std::string& value, TSEncoding& out) { + for (int encoding = PLAIN; encoding <= CAMEL; ++encoding) { + TSEncoding candidate = static_cast(encoding); + if (value == get_encoding_name(candidate)) { + out = candidate; + return true; + } + } + return false; +} + +bool ParseCompressionName(const std::string& value, CompressionType& out) { + for (int compression = UNCOMPRESSED; compression <= LZMA2; ++compression) { + CompressionType candidate = static_cast(compression); + if (value == get_compression_name(candidate)) { + out = candidate; + return true; + } + } + return false; +} + +bool ParseModel(const std::string& value, Model& out) { + std::string upper = Upper(value); + if (upper == "TABLE") { + out = Model::TABLE; + return true; + } + if (upper == "TREE") { + out = Model::TREE; + return true; + } + return false; +} + +bool ParseNullPattern(const std::string& value, NullPattern& out) { + std::string upper = Upper(value); + if (upper == "NONE") { + out = NullPattern::NONE; + return true; + } + if (upper == "MIXED") { + out = NullPattern::MIXED; + return true; + } + if (upper == "ALL") { + out = NullPattern::ALL; + return true; + } + return false; +} + +bool ParseProfile(const std::string& value, ValueProfile& out) { + std::string upper = Upper(value); + static const ValueProfile kProfiles[] = { + ValueProfile::DEFAULT, ValueProfile::TS2DIFF, + ValueProfile::TS2DIFF_MPN0, ValueProfile::SPRINTZ, + ValueProfile::RUNS, ValueProfile::STRINGS, + ValueProfile::DICT_HIGH, ValueProfile::BLOB, + }; + for (ValueProfile profile : kProfiles) { + if (upper == ProfileName(profile)) { + out = profile; + return true; + } + } + return false; +} + +bool ParseYesNo(const std::string& value, bool& out) { + if (value == "yes") { + out = true; + return true; + } + if (value == "no") { + out = false; + return true; + } + return false; +} + +std::string JoinPath(const std::string& directory, const std::string& file) { + if (directory.empty()) { + return file; + } + char last = directory[directory.size() - 1]; + if (last == '/' || last == '\\') { + return directory + file; + } + return directory + "/" + file; +} + +std::string ResolveSpecPath() { + const char* configured = std::getenv(kSpecEnv); + if (configured != nullptr && std::strlen(configured) > 0) { + return configured; + } + std::string directory = "."; + for (int depth = 0; depth < 8; depth++) { + std::string candidate = JoinPath(directory, kDefaultSpec); + std::ifstream input(candidate.c_str()); + if (input.good()) { + return candidate; + } + directory = JoinPath(directory, ".."); + } + ADD_FAILURE() << "Cannot locate " << kDefaultSpec; + return kDefaultSpec; +} + +int ColumnIndex(const std::map& columns, + const std::string& name) { + std::map::const_iterator it = columns.find(name); + if (it == columns.end()) { + ADD_FAILURE() << "Missing column " << name; + return -1; + } + return it->second; +} + +std::map IndexColumns(const std::string& header, + const std::string& expected, + const std::string& path) { + EXPECT_EQ(expected, header) << "Unexpected header in " << path; + std::map columns; + std::vector names = Split(header, ','); + for (size_t i = 0; i < names.size(); i++) { + columns[names[i]] = static_cast(i); + } + return columns; +} + +std::vector ReadSpec(const std::string& path) { + std::vector cases; + std::ifstream input(path.c_str()); + if (!input.good()) { + ADD_FAILURE() << "Cannot read compatibility spec " << path; + return cases; + } + std::map columns; + bool header_seen = false; + std::string line; + std::unordered_set ids; + while (std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty() || line[0] == '#') { + continue; + } + std::vector parts = Split(line, ','); + if (!header_seen) { + columns = IndexColumns(line, kSpecHeader, path); + header_seen = true; + continue; + } + if (columns.empty() || parts.size() != columns.size()) { + ADD_FAILURE() << "Bad spec line: " << line; + continue; + } + const std::string spec_id = parts[ColumnIndex(columns, "id")]; + std::string model_name = parts[ColumnIndex(columns, "model")]; + std::string data_type_name = parts[ColumnIndex(columns, "dataType")]; + std::string encoding_name = parts[ColumnIndex(columns, "encoding")]; + std::string compressions = parts[ColumnIndex(columns, "compressions")]; + int row_count = std::atoi(parts[ColumnIndex(columns, "rowCount")].c_str()); + int page_rows = std::atoi(parts[ColumnIndex(columns, "pageRows")].c_str()); + int chunks = std::atoi(parts[ColumnIndex(columns, "chunks")].c_str()); + int devices = std::atoi(parts[ColumnIndex(columns, "devices")].c_str()); + std::string profile_name = parts[ColumnIndex(columns, "valueProfile")]; + std::string patterns = parts[ColumnIndex(columns, "nullPatterns")]; + bool java_write = false; + bool cpp_write = false; + bool java_read = false; + bool cpp_read = false; + if (!ParseYesNo(parts[ColumnIndex(columns, "javaWrite")], java_write) || + !ParseYesNo(parts[ColumnIndex(columns, "cppWrite")], cpp_write) || + !ParseYesNo(parts[ColumnIndex(columns, "javaRead")], java_read) || + !ParseYesNo(parts[ColumnIndex(columns, "cppRead")], cpp_read)) { + ADD_FAILURE() << "Bad yes/no in spec line: " << line; + continue; + } + std::string gap_reason = parts[ColumnIndex(columns, "gapReason")]; + + Model model; + TSDataType data_type; + TSEncoding encoding; + ValueProfile profile; + if (!ParseModel(model_name, model) || + !parse_data_type_name(data_type_name, data_type) || + !ParseEncodingName(encoding_name, encoding) || + !ParseProfile(profile_name, profile)) { + ADD_FAILURE() << "Bad enum value in spec line: " << line; + continue; + } + + std::vector compression_list; + if (compressions == "ALL") { + compression_list.assign(std::begin(kAllCompressions), + std::end(kAllCompressions)); + } else { + compression_list = Split(compressions, '+'); + } + std::vector pattern_list = Split(patterns, '|'); + for (const std::string& compression_name : compression_list) { + CompressionType compression; + if (!ParseCompressionName(compression_name, compression)) { + ADD_FAILURE() << "Bad compression in spec line: " << line; + continue; + } + for (const std::string& pattern_name : pattern_list) { + NullPattern pattern; + if (!ParseNullPattern(pattern_name, pattern)) { + ADD_FAILURE() << "Bad null pattern in spec line: " << line; + continue; + } + FixtureCase fixture_case; + fixture_case.spec_id = spec_id; + fixture_case.id = spec_id + "." + Lower(compression_name) + + "." + Lower(pattern_name); + if (!ids.insert(fixture_case.id).second) { + ADD_FAILURE() << "Duplicate expanded case id " + << fixture_case.id; + } + fixture_case.model = model; + fixture_case.data_type = data_type; + fixture_case.encoding = encoding; + fixture_case.compression = compression; + fixture_case.row_count = row_count; + fixture_case.page_rows = page_rows; + fixture_case.chunks = chunks; + fixture_case.devices = devices; + fixture_case.profile = profile; + fixture_case.null_pattern = pattern; + fixture_case.java_write = java_write; + fixture_case.cpp_write = cpp_write; + fixture_case.java_read = java_read; + fixture_case.cpp_read = cpp_read; + fixture_case.gap_reason = gap_reason; + cases.push_back(fixture_case); + } + } + } + if (!header_seen) { + ADD_FAILURE() << "Empty compatibility spec " << path; + } + return cases; +} + +// --------------------------------------------------------------------------- +// Manifest and reports +// --------------------------------------------------------------------------- + +struct ManifestEntry { + std::string producer; + FixtureCase fixture_case; +}; + +void ExpectSameCase(const FixtureCase& expected, const FixtureCase& produced) { + SCOPED_TRACE("manifest case " + produced.id); + EXPECT_EQ(expected.id, produced.id); + EXPECT_EQ(static_cast(expected.model), + static_cast(produced.model)); + EXPECT_EQ(expected.data_type, produced.data_type); + EXPECT_EQ(expected.encoding, produced.encoding); + EXPECT_EQ(expected.compression, produced.compression); + EXPECT_EQ(expected.row_count, produced.row_count); + EXPECT_EQ(expected.page_rows, produced.page_rows); + EXPECT_EQ(expected.chunks, produced.chunks); + EXPECT_EQ(expected.devices, produced.devices); + EXPECT_EQ(static_cast(expected.profile), + static_cast(produced.profile)); + EXPECT_EQ(static_cast(expected.null_pattern), + static_cast(produced.null_pattern)); +} + +void WriteManifest(const std::string& directory, + const std::vector& cases) { + std::ofstream out(JoinPath(directory, kManifestFile).c_str(), + std::ios::out | std::ios::trunc); + ASSERT_TRUE(out.good()); + out << kManifestHeader << "\n"; + for (const FixtureCase& fixture_case : cases) { + out << kLanguage << "," << fixture_case.id << "," + << ModelName(fixture_case.model) << "," + << get_data_type_name(fixture_case.data_type) << "," + << get_encoding_name(fixture_case.encoding) << "," + << get_compression_name(fixture_case.compression) << "," + << fixture_case.row_count << "," << fixture_case.page_rows << "," + << fixture_case.chunks << "," << fixture_case.devices << "," + << ProfileName(fixture_case.profile) << "," + << NullPatternName(fixture_case.null_pattern) << "\n"; + } +} + +std::vector ReadManifest(const std::string& directory) { + std::vector entries; + std::ifstream input(JoinPath(directory, kManifestFile).c_str()); + if (!input.good()) { + ADD_FAILURE() << "Missing manifest in " << directory; + return entries; + } + std::map columns; + bool header_seen = false; + std::string line; + while (std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + if (line.empty()) { + continue; + } + if (!header_seen) { + columns = IndexColumns(line, kManifestHeader, + JoinPath(directory, kManifestFile)); + header_seen = true; + continue; + } + std::vector parts = Split(line, ','); + if (columns.empty() || parts.size() != columns.size()) { + ADD_FAILURE() << "Bad manifest line: " << line; + continue; + } + ManifestEntry entry; + entry.producer = parts[ColumnIndex(columns, "producer")]; + FixtureCase& fixture_case = entry.fixture_case; + fixture_case.id = parts[ColumnIndex(columns, "id")]; + if (!ParseModel(parts[ColumnIndex(columns, "model")], + fixture_case.model)) { + ADD_FAILURE() << "Bad model in manifest line: " << line; + } + if (!parse_data_type_name(parts[ColumnIndex(columns, "dataType")], + fixture_case.data_type)) { + ADD_FAILURE() << "Bad data type in manifest line: " << line; + } + if (!ParseEncodingName(parts[ColumnIndex(columns, "encoding")], + fixture_case.encoding)) { + ADD_FAILURE() << "Bad encoding in manifest line: " << line; + } + if (!ParseCompressionName(parts[ColumnIndex(columns, "compression")], + fixture_case.compression)) { + ADD_FAILURE() << "Bad compression in manifest line: " << line; + } + fixture_case.row_count = + std::atoi(parts[ColumnIndex(columns, "rowCount")].c_str()); + fixture_case.page_rows = + std::atoi(parts[ColumnIndex(columns, "pageRows")].c_str()); + fixture_case.chunks = + std::atoi(parts[ColumnIndex(columns, "chunks")].c_str()); + fixture_case.devices = + std::atoi(parts[ColumnIndex(columns, "devices")].c_str()); + if (!ParseProfile(parts[ColumnIndex(columns, "valueProfile")], + fixture_case.profile)) { + ADD_FAILURE() << "Bad value profile in manifest line: " << line; + } + if (!ParseNullPattern(parts[ColumnIndex(columns, "nullPattern")], + fixture_case.null_pattern)) { + ADD_FAILURE() << "Bad null pattern in manifest line: " << line; + } + entries.push_back(entry); + } + return entries; +} + +struct ReportRow { + std::string id; + std::string status; + std::string detail; +}; + +void WriteReport(const std::string& directory, const std::string& name, + const std::vector& report) { + std::ofstream out(JoinPath(directory, name).c_str(), + std::ios::out | std::ios::trunc); + ASSERT_TRUE(out.good()); + out << "id,status,detail\n"; + for (const ReportRow& row : report) { + out << row.id << "," << row.status << "," << row.detail << "\n"; + } +} + +std::string FormatCounts(const std::map& counts) { + std::ostringstream out; + bool first = true; + for (std::map::const_iterator it = counts.begin(); + it != counts.end(); ++it) { + if (!first) { + out << ", "; + } + first = false; + out << it->first << "=" << it->second; + } + return out.str(); +} + +void WriteSummary(const std::string& directory, const std::string& name, + const std::string& title, + const std::vector& cases, + const std::vector& report) { + int generated = 0; + int validated = 0; + int skipped = 0; + std::map by_status; + std::map by_model; + std::map by_type; + std::vector gaps; + for (const ReportRow& row : report) { + by_status[row.status]++; + if (row.status.find("generated") == 0 || row.status == "written") { + generated++; + } + if (row.status.find("validated") != std::string::npos) { + validated++; + } + if (row.status.find("skipped") == 0) { + skipped++; + gaps.push_back(row.id + ": " + row.detail); + } + } + for (const FixtureCase& fixture_case : cases) { + by_model[ModelName(fixture_case.model)]++; + by_type[get_data_type_name(fixture_case.data_type)]++; + } + + std::ostringstream out; + out << "### " << title << "\n\n"; + out << "- Spec cases: " << cases.size() << " (" << FormatCounts(by_model) + << ")\n"; + out << "- Status: " << FormatCounts(by_status) << "\n"; + out << "- Generated: " << generated << ", validated: " << validated + << ", skipped: " << skipped << "\n"; + out << "- Types: " << FormatCounts(by_type) << "\n"; + if (!gaps.empty()) { + out << "\nDeclared/observed gaps:\n\n"; + for (const std::string& gap : gaps) { + out << "- " << gap << "\n"; + } + } + std::ofstream file(JoinPath(directory, name).c_str(), + std::ios::out | std::ios::trunc); + if (file.good()) { + file << out.str(); + } + std::cout << out.str() << std::flush; +} + +void MakeDirectory(const std::string& directory) { + if (directory.empty()) { + return; + } +#ifdef _WIN32 + int ret = _mkdir(directory.c_str()); +#else + int ret = mkdir(directory.c_str(), 0777); +#endif + if (ret != 0 && errno != EEXIST) { + ADD_FAILURE() << "Failed to create directory " << directory << ": " + << std::strerror(errno); + } +} + +void EnsureDirectory(const std::string& directory) { + size_t pos = 0; + while ((pos = directory.find_first_of("/\\", pos + 1)) != + std::string::npos) { + if (pos > 0) { + MakeDirectory(directory.substr(0, pos)); + } + } + MakeDirectory(directory); +} + +// --------------------------------------------------------------------------- +// Fixture writing +// --------------------------------------------------------------------------- + +bool IsNull(const FixtureCase& fixture_case, int row) { + if (fixture_case.null_pattern == NullPattern::ALL) { + return true; + } + if (fixture_case.null_pattern == NullPattern::NONE) { + return false; + } + int page_rows = fixture_case.page_rows > 0 ? fixture_case.page_rows : 64; + int page = row / page_rows; + return page == 0 || page == 2 || page == 3 || page >= 6 || + (page == 4 && row % 2 == 0); +} + +// Restore process-wide settings even when a fatal assertion returns early. +class ScopedPageConfig { + public: + explicit ScopedPageConfig(const FixtureCase& fixture_case) + : old_rows_(g_config_value_.page_writer_max_point_num_), + old_bytes_(g_config_value_.page_writer_max_memory_bytes_) { + if (fixture_case.page_rows > 0) { + g_config_value_.page_writer_max_point_num_ = + static_cast(fixture_case.page_rows); + g_config_value_.page_writer_max_memory_bytes_ = 16 * 1024 * 1024; + } + } + ~ScopedPageConfig() { + g_config_value_.page_writer_max_point_num_ = old_rows_; + g_config_value_.page_writer_max_memory_bytes_ = old_bytes_; + } + + private: + uint32_t old_rows_; + uint32_t old_bytes_; +}; + +std::vector> Segments(const FixtureCase& fixture_case) { + int chunks = std::max(1, fixture_case.chunks); + std::vector> segments; + int base = fixture_case.row_count / chunks; + int extra = fixture_case.row_count % chunks; + int start = 0; + for (int i = 0; i < chunks; i++) { + int length = base + (i < extra ? 1 : 0); + segments.emplace_back(start, start + length); + start += length; + } + return segments; +} + +std::string DeviceId(int device) { + return std::string(kDevicePrefix) + std::to_string(device); +} + +TableSchema* CreateTableSchema(const FixtureCase& fixture_case) { + std::vector measurement_schemas; + std::vector column_categories; + measurement_schemas.emplace_back( + new MeasurementSchema(kTagColumn, STRING, PLAIN, UNCOMPRESSED)); + column_categories.emplace_back(ColumnCategory::TAG); + measurement_schemas.emplace_back(new MeasurementSchema( + kValueColumn, fixture_case.data_type, fixture_case.encoding, + fixture_case.compression)); + column_categories.emplace_back(ColumnCategory::FIELD); + if (fixture_case.null_pattern != NullPattern::NONE) { + measurement_schemas.emplace_back(new MeasurementSchema( + kAnchorColumn, INT64, PLAIN, fixture_case.compression)); + column_categories.emplace_back(ColumnCategory::FIELD); + } + return new TableSchema(kTableName, measurement_schemas, + column_categories); +} + +void AddTabletValue(Tablet& tablet, const FixtureCase& fixture_case, + uint32_t index, int row) { + switch (fixture_case.data_type) { + case BOOLEAN: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + BooleanValue(row))); + break; + case INT32: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + IntValue(fixture_case, row))); + break; + case DATE: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + DateValue(fixture_case, row))); + break; + case INT64: + case TIMESTAMP: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + LongValue(fixture_case, row))); + break; + case FLOAT: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + FloatValue(fixture_case, row))); + break; + case DOUBLE: + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + DoubleValue(fixture_case, row))); + break; + case TEXT: + case STRING: { + std::string value = StringValue(fixture_case, row); + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + common::String(value))); + break; + } + case BLOB: { + const std::string& value = BlobValue(fixture_case, row); + EXPECT_EQ(E_OK, tablet.add_value(index, kValueColumn, + common::String(value))); + break; + } + default: + FAIL() << "Unsupported data type: " + << get_data_type_name(fixture_case.data_type); + } +} + +Tablet CreateTablet(TableSchema* table_schema, const FixtureCase& fixture_case, + int device, int start, int end) { + Tablet tablet(table_schema->get_table_name(), + table_schema->get_measurement_names(), + table_schema->get_data_types(), + table_schema->get_column_categories(), + static_cast(std::max(1, end - start))); + uint32_t index = 0; + for (int row = start; row < end; row++, index++) { + EXPECT_EQ(E_OK, tablet.add_timestamp(index, row)); + EXPECT_EQ(E_OK, tablet.add_value(index, kTagColumn, + common::String(DeviceId(device)))); + if (!IsNull(fixture_case, row)) { + AddTabletValue(tablet, fixture_case, index, row); + } + if (fixture_case.null_pattern != NullPattern::NONE) { + EXPECT_EQ(E_OK, tablet.add_value(index, kAnchorColumn, + static_cast(row))); + } + } + return tablet; +} + +void CreateWriteFile(const std::string& path, WriteFile& write_file) { + remove(path.c_str()); + int flags = O_WRONLY | O_CREAT | O_TRUNC; +#ifdef _WIN32 + flags |= O_BINARY; +#endif + ASSERT_EQ(E_OK, write_file.create(path, flags, 0666)); +} + +// The tree write path stores a borrowed pointer to the value bytes, so the +// caller keeps the string alive through writer.write(). +void AddRecordPoint(TsRecord& record, std::string& keep_alive, + const FixtureCase& fixture_case, int row) { + switch (fixture_case.data_type) { + case BOOLEAN: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, BooleanValue(row))); + break; + case INT32: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, IntValue(fixture_case, row))); + break; + case DATE: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, + DateValue(fixture_case, row))); + break; + case INT64: + case TIMESTAMP: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, + LongValue(fixture_case, row))); + break; + case FLOAT: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, + FloatValue(fixture_case, row))); + break; + case DOUBLE: + EXPECT_EQ(E_OK, + record.add_point(kValueColumn, + DoubleValue(fixture_case, row))); + break; + case TEXT: + case STRING: + keep_alive = StringValue(fixture_case, row); + EXPECT_EQ(E_OK, record.add_point(kValueColumn, + common::String(keep_alive))); + break; + case BLOB: + keep_alive = BlobValue(fixture_case, row); + EXPECT_EQ(E_OK, record.add_point(kValueColumn, + common::String(keep_alive))); + break; + default: + FAIL() << "Unsupported data type: " + << get_data_type_name(fixture_case.data_type); + } +} + +void WriteTableFixture(const std::string& directory, + const FixtureCase& fixture_case) { + ScopedPageConfig page_config(fixture_case); + std::string path = JoinPath(directory, fixture_case.FileName()); + WriteFile write_file; + CreateWriteFile(path, write_file); + TableSchema* table_schema = CreateTableSchema(fixture_case); + TsFileTableWriter writer(&write_file, table_schema); + for (int device = 0; device < fixture_case.devices; device++) { + for (const std::pair& segment : Segments(fixture_case)) { + Tablet tablet = CreateTablet(table_schema, fixture_case, device, + segment.first, segment.second); + ASSERT_EQ(E_OK, writer.write_table(tablet)); + ASSERT_EQ(E_OK, writer.flush()); + } + } + ASSERT_EQ(E_OK, writer.close()); + delete table_schema; +} + +void WriteTreeFixture(const std::string& directory, + const FixtureCase& fixture_case) { + ScopedPageConfig page_config(fixture_case); + std::string path = JoinPath(directory, fixture_case.FileName()); + WriteFile write_file; + CreateWriteFile(path, write_file); + TsFileTreeWriter writer(&write_file); + for (int device = 0; device < fixture_case.devices; device++) { + std::string device_id = DeviceId(device); + MeasurementSchema* value_schema = new MeasurementSchema( + kValueColumn, fixture_case.data_type, fixture_case.encoding, + fixture_case.compression); + ASSERT_EQ(E_OK, writer.register_timeseries(device_id, value_schema)); + delete value_schema; + if (fixture_case.null_pattern != NullPattern::NONE) { + MeasurementSchema* anchor_schema = new MeasurementSchema( + kAnchorColumn, INT64, PLAIN, fixture_case.compression); + ASSERT_EQ(E_OK, + writer.register_timeseries(device_id, anchor_schema)); + delete anchor_schema; + } + } + for (int device = 0; device < fixture_case.devices; device++) { + std::string device_id = DeviceId(device); + for (const std::pair& segment : Segments(fixture_case)) { + for (int row = segment.first; row < segment.second; row++) { + TsRecord record(device_id, row); + std::string keep_alive; + int points = 0; + if (!IsNull(fixture_case, row)) { + AddRecordPoint(record, keep_alive, fixture_case, row); + points++; + } + if (fixture_case.null_pattern != NullPattern::NONE) { + EXPECT_EQ(E_OK, record.add_point( + kAnchorColumn, + static_cast(row))); + points++; + } + if (points > 0) { + ASSERT_EQ(E_OK, writer.write(record)); + } + } + ASSERT_EQ(E_OK, writer.flush()); + } + } + ASSERT_EQ(E_OK, writer.close()); +} + +void AssertOnWireCodec(const std::string& directory, + const FixtureCase& fixture_case) { + // Prove that the declared encoding and compression are what the file + // actually carries. Reading values back cannot tell a PLAIN column from an + // RLE column that happens to decode identically, nor whether a codec + // request was silently replaced by a default. + std::string path = JoinPath(directory, fixture_case.FileName()); + TsFileIOReader io_reader; + ASSERT_EQ(E_OK, io_reader.init(path)); + TsFileMeta* tsfile_meta = io_reader.get_tsfile_meta(); + ASSERT_NE(nullptr, tsfile_meta); + + std::vector> devices; + if (fixture_case.model == Model::TABLE) { + MetaIndexNode* table_root = nullptr; + ASSERT_EQ(E_OK, + tsfile_meta->get_table_metaindex_node(kTableName, table_root)); + ASSERT_NE(nullptr, table_root); + for (const std::shared_ptr& entry : + table_root->children_) { + std::shared_ptr device_id = entry->get_device_id(); + if (device_id != nullptr) { + devices.push_back(device_id); + } + } + } else { + for (int device = 0; device < fixture_case.devices; device++) { + devices.push_back( + std::make_shared(DeviceId(device))); + } + } + ASSERT_EQ(static_cast(fixture_case.devices), devices.size()); + + LocalRandomAccessReadFile read_file; + ASSERT_EQ(E_OK, read_file.open(path)); + common::PageArena arena; + arena.init(512, common::MOD_TSFILE_READER); + for (const std::shared_ptr& device_id : devices) { + std::vector indexes(1, nullptr); + std::unordered_set measurements = {kValueColumn}; + ASSERT_EQ(E_OK, io_reader.get_timeseries_indexes(device_id, measurements, + indexes, arena)); + ASSERT_NE(nullptr, indexes.front()); + common::SimpleList* chunks = + indexes.front()->get_value_chunk_meta_list(); + if (chunks == nullptr) { + chunks = indexes.front()->get_chunk_meta_list(); + } + ASSERT_NE(nullptr, chunks); + ASSERT_EQ(static_cast(fixture_case.chunks), chunks->size()); + for (auto cursor = chunks->begin(); cursor != chunks->end(); cursor++) { + ChunkMeta* chunk = cursor.get(); + ASSERT_NE(nullptr, chunk); + char header_buf[1024]; + int32_t read_len = 0; + ASSERT_EQ(E_OK, read_file.read(chunk->offset_of_chunk_header_, + header_buf, sizeof(header_buf), + read_len)); + ASSERT_GE(read_len, 7); + common::ByteStream stream; + stream.wrap_from(header_buf, read_len); + ChunkHeader chunk_header; + ASSERT_EQ(E_OK, chunk_header.deserialize_from(stream)); + EXPECT_EQ(fixture_case.encoding, chunk_header.encoding_type_) + << fixture_case.Describe() << " encoding on wire"; + EXPECT_EQ(fixture_case.compression, chunk_header.compression_type_) + << fixture_case.Describe() << " compression on wire"; + } + } +} + +void WriteFixture(const std::string& directory, + const FixtureCase& fixture_case) { + if (fixture_case.model == Model::TABLE) { + WriteTableFixture(directory, fixture_case); + } else { + WriteTreeFixture(directory, fixture_case); + } + std::string path = JoinPath(directory, fixture_case.FileName()); + std::ifstream input(path.c_str(), std::ios::binary | std::ios::ate); + ASSERT_TRUE(input.good()) << "fixture was not created: " << path; + ASSERT_GT(input.tellg(), 0) << "fixture is empty: " << path; + AssertOnWireCodec(directory, fixture_case); +} + +// --------------------------------------------------------------------------- +// Fixture validation +// --------------------------------------------------------------------------- + +std::string StringOf(const common::String* value) { + if (value == nullptr || value->buf_ == nullptr || value->len_ == 0) { + return std::string(); + } + return std::string(value->buf_, value->len_); +} + +// Verifies one value cell of a TableResultSet against the fixture's expected +// value. Primitive types use the direct buffer accessors (get_*_at), which +// read straight from the column vector without materializing a RowRecord/Field +// or doing a column-name lookup per row; only the variable-length string/binary +// columns fall back to get_value(). +void ExpectTableValue(const FixtureCase& fixture_case, int row, + ResultSet* result_set) { + SCOPED_TRACE(::testing::Message() << fixture_case.id << " row=" << row); + switch (fixture_case.data_type) { + case BOOLEAN: + EXPECT_EQ(BooleanValue(row), result_set->get_bool_at(3)); + break; + case INT32: + EXPECT_EQ(IntValue(fixture_case, row), + result_set->get_int32_at(3)); + break; + case DATE: + EXPECT_EQ(DateValue(fixture_case, row), + result_set->get_int32_at(3)); + break; + case INT64: + case TIMESTAMP: + EXPECT_EQ(LongValue(fixture_case, row), + result_set->get_int64_at(3)); + break; + case FLOAT: + EXPECT_EQ(FloatBits(ExpectedFloatValue(fixture_case, row)), + FloatBits(result_set->get_float_at(3))); + break; + case DOUBLE: + EXPECT_EQ(DoubleBits(ExpectedDoubleValue(fixture_case, row)), + DoubleBits(result_set->get_double_at(3))); + break; + case TEXT: + case STRING: + EXPECT_EQ(StringValue(fixture_case, row), + StringOf(result_set->get_value(3))); + break; + case BLOB: + EXPECT_EQ(BlobValue(fixture_case, row), + StringOf(result_set->get_value(3))); + break; + default: + FAIL() << "Unsupported data type: " + << get_data_type_name(fixture_case.data_type); + } +} + +void ExpectFieldValue(const FixtureCase& fixture_case, int row, Field* field) { + SCOPED_TRACE(::testing::Message() << fixture_case.id << " row=" << row); + EXPECT_EQ(fixture_case.data_type, field->type_); + switch (fixture_case.data_type) { + case BOOLEAN: + EXPECT_EQ(BooleanValue(row), field->get_value()); + break; + case INT32: + EXPECT_EQ(IntValue(fixture_case, row), + field->get_value()); + break; + case DATE: + EXPECT_EQ(DateValue(fixture_case, row), + field->get_value()); + break; + case INT64: + case TIMESTAMP: + EXPECT_EQ(LongValue(fixture_case, row), + field->get_value()); + break; + case FLOAT: + EXPECT_EQ(FloatBits(ExpectedFloatValue(fixture_case, row)), + FloatBits(field->get_value())); + break; + case DOUBLE: + EXPECT_EQ(DoubleBits(ExpectedDoubleValue(fixture_case, row)), + DoubleBits(field->get_value())); + break; + case TEXT: + case STRING: + EXPECT_EQ(StringValue(fixture_case, row), + StringOf(field->get_string_value())); + break; + case BLOB: + EXPECT_EQ(BlobValue(fixture_case, row), + StringOf(field->get_string_value())); + break; + default: + FAIL() << "Unsupported data type: " + << get_data_type_name(fixture_case.data_type); + } +} + +struct Range { + int start; + int end; +}; + +std::vector QueryRanges(const FixtureCase& fixture_case) { + std::vector ranges; + ranges.push_back({0, fixture_case.row_count - 1}); + int page_rows = fixture_case.page_rows; + if (fixture_case.null_pattern != NullPattern::NONE && page_rows > 0 && + fixture_case.row_count > 2 * page_rows) { + ranges.push_back( + {page_rows - 1, + std::min(fixture_case.row_count - 1, 2 * page_rows + 1)}); + ranges.push_back( + {page_rows, std::min(fixture_case.row_count - 1, 2 * page_rows - 1)}); + } + return ranges; +} + +void ValidateTableRange(const std::string& directory, + const FixtureCase& fixture_case, const Range& range, + bool with_anchor) { + SCOPED_TRACE(::testing::Message() + << fixture_case.Describe() << " table range=" << range.start + << ":" << range.end << " anchor=" << with_anchor); + std::string path = JoinPath(directory, fixture_case.FileName()); + TsFileReader reader; + ASSERT_EQ(E_OK, reader.open(path)); + std::vector columns = {kTagColumn, kValueColumn}; + if (with_anchor) { + columns.push_back(kAnchorColumn); + } + ResultSet* raw_result_set = nullptr; + ASSERT_EQ(E_OK, reader.query(kTableName, columns, range.start, range.end, + raw_result_set)); + ASSERT_NE(nullptr, raw_result_set); + auto cleanup = [&reader](ResultSet* result) { + reader.destroy_query_data_set(result); + }; + std::unique_ptr owned_result( + raw_result_set, cleanup); + std::shared_ptr metadata = owned_result->get_metadata(); + ASSERT_EQ(with_anchor ? 4U : 3U, metadata->get_column_count()); + ASSERT_EQ("time", metadata->get_column_name(1)); + ASSERT_EQ(INT64, metadata->get_column_type(1)); + ASSERT_EQ(kTagColumn, metadata->get_column_name(2)); + ASSERT_EQ(STRING, metadata->get_column_type(2)); + ASSERT_EQ(kValueColumn, metadata->get_column_name(3)); + ASSERT_EQ(fixture_case.data_type, metadata->get_column_type(3)); + + // Rows arrive grouped by device, each device's rows time-ordered, so the + // tag only changes at device boundaries. Verify it once per device rather + // than materializing a string Field for every row; primitive columns use + // the direct buffer accessors. This keeps a 1M-row fixture's validation + // from doing a million string copies and map inserts. + const int range_len = range.end - range.start + 1; + int device = 0; + int expected_time = range.start; + int rows_in_device = 0; + int total_rows = 0; + bool tag_verified = false; + bool has_next = false; + while (true) { + ASSERT_EQ(E_OK, owned_result->next(has_next)); + if (!has_next) { + break; + } + total_rows++; + SCOPED_TRACE(::testing::Message() + << "device=" << device << " t=" << expected_time); + + if (!tag_verified) { + common::String* tag = owned_result->get_value(2); + ASSERT_NE(nullptr, tag); + EXPECT_EQ(DeviceId(device), StringOf(tag)); + tag_verified = true; + } + + EXPECT_EQ(static_cast(expected_time), + owned_result->get_int64_at(1)); + + const bool expected_null = IsNull(fixture_case, expected_time); + EXPECT_EQ(expected_null, owned_result->is_null(3)); + if (!expected_null) { + ASSERT_NO_FATAL_FAILURE( + ExpectTableValue(fixture_case, expected_time, + owned_result.get())); + } + + if (with_anchor) { + EXPECT_EQ(static_cast(expected_time), + owned_result->get_int64_at(4)); + } + + expected_time++; + if (++rows_in_device == range_len) { + device++; + expected_time = range.start; + rows_in_device = 0; + tag_verified = false; + } + } + EXPECT_EQ(fixture_case.devices, device); + EXPECT_EQ(fixture_case.devices * range_len, total_rows); + owned_result.reset(); + ASSERT_EQ(E_OK, reader.close()); +} + +void ValidateTreeRange(const std::string& directory, + const FixtureCase& fixture_case, int device, + const Range& range, bool with_anchor) { + SCOPED_TRACE(::testing::Message() + << fixture_case.Describe() << " tree device=" << device + << " range=" << range.start << ":" << range.end + << " anchor=" << with_anchor); + std::string path = JoinPath(directory, fixture_case.FileName()); + TsFileTreeReader reader; + ASSERT_EQ(E_OK, reader.open(path)); + std::vector device_ids = {DeviceId(device)}; + std::vector measurements = {kValueColumn}; + if (with_anchor) { + measurements.push_back(kAnchorColumn); + } + ResultSet* result = nullptr; + ASSERT_EQ(E_OK, reader.query(device_ids, measurements, range.start, + range.end, result)); + ASSERT_NE(nullptr, result); + auto cleanup = [&reader](ResultSet* owned) { + reader.destroy_query_data_set(owned); + }; + std::unique_ptr owned_result(result, cleanup); + ResultSetIterator iter = owned_result->iterator(); + int expected_time = range.start; + while (iter.hasNext()) { + RowRecord* record = iter.next(); + ASSERT_NE(nullptr, record); + if (!with_anchor) { + while (expected_time <= range.end && + IsNull(fixture_case, expected_time)) { + expected_time++; + } + } + ASSERT_LE(expected_time, range.end); + EXPECT_EQ(static_cast(expected_time), + record->get_timestamp()); + Field* value_field = record->get_field(1); + ASSERT_NE(nullptr, value_field); + bool is_null = value_field->type_ == NULL_TYPE; + EXPECT_EQ(IsNull(fixture_case, expected_time), is_null); + if (!is_null) { + ExpectFieldValue(fixture_case, expected_time, value_field); + } + if (with_anchor) { + Field* anchor_field = record->get_field(2); + ASSERT_NE(nullptr, anchor_field); + EXPECT_EQ(static_cast(expected_time), + anchor_field->get_value()); + } + expected_time++; + } + if (!with_anchor) { + while (expected_time <= range.end && + IsNull(fixture_case, expected_time)) { + expected_time++; + } + } + EXPECT_EQ(range.end + 1, expected_time); + owned_result.reset(); + ASSERT_EQ(E_OK, reader.close()); +} + +void ValidateFixture(const std::string& directory, + const FixtureCase& fixture_case) { + SCOPED_TRACE(fixture_case.Describe()); + for (const Range& range : QueryRanges(fixture_case)) { + if (fixture_case.model == Model::TABLE) { + ValidateTableRange(directory, fixture_case, range, false); + if (fixture_case.null_pattern != NullPattern::NONE) { + ValidateTableRange(directory, fixture_case, range, true); + } + } else { + for (int device = 0; device < fixture_case.devices; device++) { + ValidateTreeRange(directory, fixture_case, device, range, + false); + if (fixture_case.null_pattern != NullPattern::NONE) { + ValidateTreeRange(directory, fixture_case, device, range, + true); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Capability probes for the spec integrity test +// --------------------------------------------------------------------------- + +bool EncoderSupported(TSEncoding encoding, TSDataType data_type) { + Encoder* encoder = EncoderFactory::alloc_value_encoder(encoding, data_type); + if (encoder == nullptr) { + return false; + } + EncoderFactory::free(encoder); + return true; +} + +bool DecoderSupported(TSEncoding encoding, TSDataType data_type) { + Decoder* decoder = DecoderFactory::alloc_value_decoder(encoding, data_type); + if (decoder == nullptr) { + return false; + } + DecoderFactory::free(decoder); + return true; +} + +bool CompressionSupported(CompressionType compression) { + Compressor* compressor = CompressorFactory::alloc_compressor(compression); + if (compressor == nullptr) { + return false; + } + CompressorFactory::free(compressor); + return true; +} + +CompressionType ParseCompressionOrFail(const std::string& name) { + CompressionType compression = INVALID_COMPRESSION; + if (!ParseCompressionName(name, compression)) { + ADD_FAILURE() << "Bad compression name " << name; + } + return compression; +} + +std::string CapabilityKey(Model model, TSDataType data_type, + TSEncoding encoding, CompressionType compression) { + return std::string(ModelName(model)) + "/" + + get_data_type_name(data_type) + "/" + get_encoding_name(encoding) + + "/" + get_compression_name(compression); +} + +} // namespace + +class EncodingCompressionCompatibilityTest : public ::testing::Test { + protected: + void SetUp() override { ASSERT_EQ(E_OK, libtsfile_init()); } + void TearDown() override { libtsfile_destroy(); } +}; + +// Guards the spec against drift on the C++ side: every combination the C++ +// encoder/decoder factory supports must be declared, every declared C++ case +// must be constructible, and every declared gap must carry a reason. Runs +// without any environment configuration. +TEST_F(EncodingCompressionCompatibilityTest, SpecMatchesCppCapabilities) { + std::vector cases = ReadSpec(ResolveSpecPath()); + std::unordered_set write_coverage; + std::unordered_set read_coverage; + for (const FixtureCase& fixture_case : cases) { + if (fixture_case.gap_reason.empty()) { + EXPECT_TRUE(fixture_case.java_write && fixture_case.cpp_write && + fixture_case.java_read && fixture_case.cpp_read) + << "gapReason is required when a case declares a gap: " + << fixture_case.Describe(); + } + if (fixture_case.cpp_write) { + EXPECT_TRUE(fixture_case.cpp_read) + << "a C++-written case must also be C++-readable: " + << fixture_case.Describe(); + EXPECT_TRUE(EncoderSupported(fixture_case.encoding, + fixture_case.data_type)) + << "C++ cannot build an encoder for " + << fixture_case.Describe(); + write_coverage.insert(CapabilityKey( + fixture_case.model, fixture_case.data_type, + fixture_case.encoding, fixture_case.compression)); + } + if (fixture_case.cpp_read) { + read_coverage.insert(CapabilityKey( + fixture_case.model, fixture_case.data_type, + fixture_case.encoding, fixture_case.compression)); + } + } + + std::vector available_compressions; + for (const char* name : kAllCompressions) { + CompressionType compression = ParseCompressionOrFail(name); + if (CompressionSupported(compression)) { + available_compressions.push_back(name); + } + } + ASSERT_FALSE(available_compressions.empty()); + + int missing = 0; + for (Model model : {Model::TABLE, Model::TREE}) { + for (TSDataType data_type : kMatrixDataTypes) { + for (int e = PLAIN; e <= CAMEL; e++) { + TSEncoding encoding = static_cast(e); + const bool can_write = EncoderSupported(encoding, data_type); + const bool can_read = DecoderSupported(encoding, data_type); + if (!can_write && !can_read) { + continue; + } + for (const std::string& compression_name : + available_compressions) { + CompressionType compression = + ParseCompressionOrFail(compression_name); + std::string key = CapabilityKey(model, data_type, encoding, + compression); + if (can_write && write_coverage.count(key) == 0) { + ADD_FAILURE() + << "compatibility-matrix.csv does not declare a " + "C++-writable case for " + << key; + missing++; + } + if (can_read && read_coverage.count(key) == 0) { + ADD_FAILURE() + << "compatibility-matrix.csv does not declare a " + "C++-readable case for " + << key; + missing++; + } + } + } + } + } + EXPECT_EQ(0, missing); +} + +TEST_F(EncodingCompressionCompatibilityTest, GenerateFixtures) { + const char* output_dir = std::getenv(kGenerateDirEnv); + if (output_dir == nullptr || std::strlen(output_dir) == 0) { + GTEST_SKIP() << kGenerateDirEnv << " is not set"; + } + EnsureDirectory(output_dir); + std::vector cases = ReadSpec(ResolveSpecPath()); + std::vector written; + std::vector report; + for (const FixtureCase& fixture_case : cases) { + if (!fixture_case.cpp_write) { + report.push_back( + {fixture_case.id, "not-mine", fixture_case.gap_reason}); + continue; + } + ASSERT_NO_FATAL_FAILURE(WriteFixture(output_dir, fixture_case)); + ASSERT_NO_FATAL_FAILURE(ValidateFixture(output_dir, fixture_case)); + written.push_back(fixture_case); + report.push_back({fixture_case.id, "generated-and-validated", ""}); + } + WriteManifest(output_dir, written); + WriteReport(output_dir, kReportFile, report); + WriteSummary(output_dir, kSummaryFile, "C++ fixture generation", cases, + report); +} + +TEST_F(EncodingCompressionCompatibilityTest, ValidateFixtures) { + const char* input_dir = std::getenv(kValidateDirEnv); + if (input_dir == nullptr || std::strlen(input_dir) == 0) { + GTEST_SKIP() << kValidateDirEnv << " is not set"; + } + std::vector manifest = ReadManifest(input_dir); + ASSERT_FALSE(manifest.empty()); + const std::string producer = manifest.front().producer; + ASSERT_TRUE(producer == "java" || producer == "cpp") + << "unexpected producer " << producer; + + std::map spec; + for (const FixtureCase& fixture_case : ReadSpec(ResolveSpecPath())) { + spec[fixture_case.id] = fixture_case; + } + + std::unordered_set expected_ids; + for (const auto& entry : spec) { + if (entry.second.Writes(producer)) { + expected_ids.insert(entry.first); + } + } + std::unordered_set manifest_ids; + for (const ManifestEntry& entry : manifest) { + manifest_ids.insert(entry.fixture_case.id); + } + for (const std::string& id : expected_ids) { + if (manifest_ids.count(id) == 0) { + ADD_FAILURE() << "manifest for " << producer + << " is missing declared case " << id; + } + } + for (const std::string& id : manifest_ids) { + if (expected_ids.count(id) == 0) { + ADD_FAILURE() << "manifest for " << producer + << " has undeclared case " << id; + } + } + + std::vector report; + for (const ManifestEntry& entry : manifest) { + const FixtureCase& produced = entry.fixture_case; + std::map::const_iterator it = + spec.find(produced.id); + if (it == spec.end()) { + ADD_FAILURE() << "manifest case absent from the spec: " + << produced.id; + continue; + } + const FixtureCase& fixture_case = it->second; + ASSERT_NO_FATAL_FAILURE(ExpectSameCase(fixture_case, produced)); + if (!fixture_case.Reads(kLanguage)) { + report.push_back({produced.id, "skipped-declared-gap", + fixture_case.gap_reason}); + continue; + } + std::string path = JoinPath(input_dir, produced.FileName()); + std::ifstream input(path.c_str(), std::ios::binary); + ASSERT_TRUE(input.good()) + << "missing fixture declared as readable: " << path; + ASSERT_NO_FATAL_FAILURE(ValidateFixture(input_dir, fixture_case)); + report.push_back({produced.id, "validated", ""}); + } + WriteReport(input_dir, std::string(kReportFile) + ".validate", report); + std::vector all_cases; + for (const auto& entry : spec) { + all_cases.push_back(entry.second); + } + WriteSummary(input_dir, std::string(kSummaryFile) + ".validate", + "C++ validation of " + producer + " fixtures", all_cases, + report); +} diff --git a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc b/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc deleted file mode 100644 index 71958cd674..0000000000 --- a/cpp/test/reader/table_view/table_model_encoding_compression_compatibility_test.cc +++ /dev/null @@ -1,863 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * License); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#include -#else -#include -#include -#include -#endif -#include - -#include "common/config/config.h" -#include "common/db_common.h" -#include "common/schema.h" -#include "common/tablet.h" -#include "file/local_random_access_read_file.h" -#include "file/tsfile_io_reader.h" -#include "file/write_file.h" -#include "reader/table_result_set.h" -#include "reader/tsfile_reader.h" -#include "writer/tsfile_table_writer.h" - -using namespace common; -using namespace storage; - -namespace { - -const char* const kGenerateDirEnv = "TSFILE_COMPAT_GENERATE_DIR"; -const char* const kValidateDirEnv = "TSFILE_COMPAT_VALIDATE_DIR"; -const char* const kManifestFile = "manifest.csv"; -const char* const kManifestHeader = - "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount," - "nullPattern"; -const char* const kTableName = "compat_table"; -const char* const kTagColumn = "device"; -const char* const kValueColumn = "value"; -const char* const kTagValue = "compat_device"; -const char* const kAnchorColumn = "anchor"; -const int kNullPageRows = 64; -// Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so -// page-wide bitmaps must survive block transitions. Applied to every case. -constexpr int kRowCount = 300; - -const int32_t kIntValues[] = { - 0, - 1, - -1, - 7, - -8, - 1024, - -1024, - 123456, - -654321, - std::numeric_limits::max() - 1024, - std::numeric_limits::min() + 1024, - 42, - 42, - 0, - -8, - 2048, -}; - -const int64_t kLongValues[] = { - 0LL, - 1LL, - -1LL, - 7LL, - -8LL, - 1234567890LL, - -987654321LL, - 4294967296LL, - -4294967296LL, - 1234567890123LL, - -1234567890123LL, - std::numeric_limits::max() - 4096, - std::numeric_limits::min() + 4096, - 42LL, - 42LL, - 0LL, -}; - -const uint32_t kFloatBits[] = { - 0x00000000U, 0x80000000U, 0x3f800000U, 0xbf800000U, - 0x41280000U, 0xc0700000U, 0x44801000U, 0xc5002000U, - 0x40490fdbU, 0xc02df854U, 0x3f800000U, 0x3f800000U, - 0x00000000U, 0x80000000U, 0x42f6e979U, 0xc2f6e979U, -}; - -const uint64_t kDoubleBits[] = { - UINT64_C(0x0000000000000000), UINT64_C(0x8000000000000000), - UINT64_C(0x3ff0000000000000), UINT64_C(0xbff0000000000000), - UINT64_C(0x4029000000000000), UINT64_C(0xc00c000000000000), - UINT64_C(0x4090008000000000), UINT64_C(0xc0a0010000000000), - UINT64_C(0x400921fb54442d18), UINT64_C(0xc005bf0a8b145769), - UINT64_C(0x3ff0000000000000), UINT64_C(0x3ff0000000000000), - UINT64_C(0x0000000000000000), UINT64_C(0x8000000000000000), - UINT64_C(0x405edd2f1a9fbe77), UINT64_C(0xc05edd2f1a9fbe77), -}; - -// FLOAT/DOUBLE + TS_2DIFF converts values to fixed-point with -// maxPointNumber (10^mpn). The expected value is computed by applying the -// same tri-state conversion the writer performs and then decoding it back, -// so expectations reflect the encoding rules rather than the raw input -// bits. Every chosen value has the same expected value whether the writer -// used maxPointNumber 0 (explicit property; the "*.mpn0" Java fixtures) or -// 2 (the TSFileConfig.floatPrecision default shared with the C++ writer), -// so the validating reader never needs to know which writer produced the -// file. The values cover all page layouts the writers can produce: -// - plain integers (scaled form; at mpn = 0 every finite in-range value -// is scaled, so pages written at mpn = 0 contain no bitmap) -// - 1.5e9f / 1e18: the scaled product overflows while round(v) still -// fits -> scale-overflow form, single page-wide bitmap (reachable only -// at mpn > 0, i.e. the default writer configuration) -// - NaN and +/-Infinity (stored as raw IEEE bits -> two page-wide -// bitmaps; Java floatToIntBits canonicalizes NaN) -const uint32_t kTs2DiffFloatBits[] = { - 0x00000000U, 0x3f800000U, 0xbf800000U, 0x461c4000U, - 0xc61c4000U, 0x4eb2d05eU, 0x7f800000U, 0xff800000U, - 0x7fc00000U, 0x3f800000U, 0x00000000U, 0x4a742400U, - 0xca742400U, 0x4e6e6b28U, 0x3f800000U, 0x00000000U, -}; - -const uint64_t kTs2DiffDoubleBits[] = { - UINT64_C(0x0000000000000000), UINT64_C(0x3ff0000000000000), - UINT64_C(0xbff0000000000000), UINT64_C(0x40c3880000000000), - UINT64_C(0xc0c3880000000000), UINT64_C(0x43abc16d674ec800), - UINT64_C(0x7ff0000000000000), UINT64_C(0xfff0000000000000), - UINT64_C(0x7ff8000000000000), UINT64_C(0x3ff0000000000000), - UINT64_C(0x0000000000000000), UINT64_C(0x41f0000000000000), - UINT64_C(0xc1cd6f3458800000), UINT64_C(0x430c6bf526340000), - UINT64_C(0x3ff0000000000000), UINT64_C(0x0000000000000000), -}; - -// maxPointNumber used by the C++ FloatTS2DIFFEncoder/DoubleTS2DIFFEncoder, -// matching the standard Java schema write path (TSEncodingBuilder.Ts2Diff -// initFromProps -> TSFileConfig.floatPrecision, current default 2). -constexpr int kTs2DiffMaxPointNumber = 2; -constexpr double kTs2DiffMaxPointValue = 100.0; - -const int32_t kDateValues[] = { - 19700101, 19991231, 20000229, 20240229, - 20380119, 20500615, 19690720, 19800106, -}; - -struct FixtureCase { - std::string file_name; - std::string table_name; - std::string tag_column; - std::string value_column; - TSDataType data_type; - TSEncoding encoding; - CompressionType compression; - int row_count; - std::string null_pattern = "NONE"; - - FixtureCase() - : table_name(kTableName), - tag_column(kTagColumn), - value_column(kValueColumn), - data_type(INVALID_DATATYPE), - encoding(INVALID_ENCODING), - compression(INVALID_COMPRESSION), - row_count(0) {} - - FixtureCase(TSDataType data_type, TSEncoding encoding, - CompressionType compression, int row_count) - : file_name(FileName(data_type, encoding, compression)), - table_name(kTableName), - tag_column(kTagColumn), - value_column(kValueColumn), - data_type(data_type), - encoding(encoding), - compression(compression), - row_count(row_count) {} - - static std::string Lower(std::string value) { - for (char& c : value) { - c = static_cast(std::tolower(static_cast(c))); - } - return value; - } - - static std::string FileName(TSDataType data_type, TSEncoding encoding, - CompressionType compression) { - return Lower(get_encoding_name(encoding)) + "_" + - Lower(get_data_type_name(data_type)) + "_" + - Lower(get_compression_name(compression)) + ".tsfile"; - } - - std::string ToManifestLine() const { - std::ostringstream out; - out << file_name << "," << table_name << "," << tag_column << "," - << value_column << "," << get_data_type_name(data_type) << "," - << get_encoding_name(encoding) << "," - << get_compression_name(compression) << "," << row_count << "," - << null_pattern; - return out.str(); - } -}; - -std::vector Split(const std::string& line, char delimiter) { - std::vector parts; - std::stringstream stream(line); - std::string part; - while (std::getline(stream, part, delimiter)) { - parts.push_back(part); - } - if (!line.empty() && line.back() == delimiter) { - parts.emplace_back(); - } - return parts; -} - -bool ParseEncodingName(const std::string& value, TSEncoding& out) { - for (int encoding = PLAIN; encoding <= CAMEL; ++encoding) { - TSEncoding candidate = static_cast(encoding); - if (value == get_encoding_name(candidate)) { - out = candidate; - return true; - } - } - return false; -} - -bool ParseCompressionName(const std::string& value, CompressionType& out) { - for (int compression = UNCOMPRESSED; compression <= LZMA2; ++compression) { - CompressionType candidate = static_cast(compression); - if (value == get_compression_name(candidate)) { - out = candidate; - return true; - } - } - return false; -} - -FixtureCase ParseManifestLine(const std::string& line) { - FixtureCase fixture_case; - std::vector parts = Split(line, ','); - if (parts.size() != 9) { - ADD_FAILURE() << "Bad manifest line: " << line; - return fixture_case; - } - fixture_case.file_name = parts[0]; - fixture_case.table_name = parts[1]; - fixture_case.tag_column = parts[2]; - fixture_case.value_column = parts[3]; - if (!parse_data_type_name(parts[4], fixture_case.data_type)) { - ADD_FAILURE() << "Bad data type in manifest line: " << line; - } - if (!ParseEncodingName(parts[5], fixture_case.encoding)) { - ADD_FAILURE() << "Bad encoding in manifest line: " << line; - } - if (!ParseCompressionName(parts[6], fixture_case.compression)) { - ADD_FAILURE() << "Bad compression in manifest line: " << line; - } - fixture_case.row_count = std::atoi(parts[7].c_str()); - fixture_case.null_pattern = parts[8]; - EXPECT_TRUE(fixture_case.null_pattern == "NONE" || - fixture_case.null_pattern == "MIXED" || - fixture_case.null_pattern == "ALL"); - return fixture_case; -} - -std::string JoinPath(const std::string& directory, const std::string& file) { - if (directory.empty()) { - return file; - } - char last = directory[directory.size() - 1]; - if (last == '/' || last == '\\') { - return directory + file; - } - return directory + "/" + file; -} - -void MakeDirectory(const std::string& directory) { - if (directory.empty()) { - return; - } -#ifdef _WIN32 - int ret = _mkdir(directory.c_str()); -#else - int ret = mkdir(directory.c_str(), 0777); -#endif - if (ret != 0 && errno != EEXIST) { - ADD_FAILURE() << "Failed to create directory " << directory << ": " - << std::strerror(errno); - } -} - -void EnsureDirectory(const std::string& directory) { - size_t pos = 0; - while ((pos = directory.find_first_of("/\\", pos + 1)) != - std::string::npos) { - if (pos > 0) { - MakeDirectory(directory.substr(0, pos)); - } - } - MakeDirectory(directory); -} - -int32_t IntValue(int row) { - return kIntValues[row % (sizeof(kIntValues) / sizeof(kIntValues[0]))]; -} - -int64_t LongValue(int row) { - return kLongValues[row % (sizeof(kLongValues) / sizeof(kLongValues[0]))]; -} - -uint32_t FloatBits(float value) { - uint32_t bits; - std::memcpy(&bits, &value, sizeof(bits)); - return bits; -} - -float BitsToFloat(uint32_t bits) { - float value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -float FloatValue(int row) { - uint32_t bits = - kFloatBits[row % (sizeof(kFloatBits) / sizeof(kFloatBits[0]))]; - float value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -float Ts2DiffFloatValue(int row) { - uint32_t bits = kTs2DiffFloatBits[row % (sizeof(kTs2DiffFloatBits) / - sizeof(kTs2DiffFloatBits[0]))]; - float value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -uint64_t DoubleBits(double value) { - uint64_t bits; - std::memcpy(&bits, &value, sizeof(bits)); - return bits; -} - -double BitsToDouble(uint64_t bits) { - double value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -double DoubleValue(int row) { - uint64_t bits = - kDoubleBits[row % (sizeof(kDoubleBits) / sizeof(kDoubleBits[0]))]; - double value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -double Ts2DiffDoubleValue(int row) { - uint64_t bits = kTs2DiffDoubleBits[row % (sizeof(kTs2DiffDoubleBits) / - sizeof(kTs2DiffDoubleBits[0]))]; - double value; - std::memcpy(&value, &bits, sizeof(value)); - return value; -} - -bool IsTs2DiffFloatCase(const FixtureCase& fixture_case) { - return fixture_case.encoding == TS_2DIFF && - (fixture_case.data_type == FLOAT || - fixture_case.data_type == DOUBLE); -} - -bool StringEndsWith(const std::string& value, const std::string& suffix) { - return value.size() >= suffix.size() && - value.compare(value.size() - suffix.size(), suffix.size(), suffix) == - 0; -} - -// maxPointValue of the writer configuration for this fixture: the -// "*.mpn0" Java fixtures carry an explicit max_point_number=0 property, -// all others use the shared default (see kTs2DiffMaxPointNumber). -double Ts2DiffMaxPointValue(const FixtureCase& fixture_case) { - return StringEndsWith(fixture_case.file_name, ".mpn0") - ? 1.0 - : kTs2DiffMaxPointValue; -} - -// Java Math.round(x) == floor(x + 0.5) (ties towards +infinity). -static double JavaRound(double x) { return std::floor(x + 0.5); } - -float ExpectedFloatValue(const FixtureCase& fixture_case, int row) { - if (!IsTs2DiffFloatCase(fixture_case)) { - return FloatValue(row); - } - float value = Ts2DiffFloatValue(row); - if (std::isnan(value)) { - // Java FloatEncoder normalizes any NaN to the canonical pattern - // via floatToIntBits. - return BitsToFloat(0x7fc00000U); - } - // Mirror FloatTS2DIFFEncoder::convert_float_to_int, then - // FloatDecoder::readFloat: scaled -> stored / mpv, scale-overflow -> - // stored / 1, raw bits -> intBitsToFloat. - const double mpv = Ts2DiffMaxPointValue(fixture_case); - const double scaled = static_cast(value) * mpv; - const bool scaled_overflow = - scaled > static_cast(std::numeric_limits::max()) || - scaled < static_cast(std::numeric_limits::min()); - if (scaled_overflow) { - const bool value_overflows = - value > static_cast(std::numeric_limits::max()) || - value < static_cast(std::numeric_limits::min()); - if (value_overflows) { - // Raw IEEE bits, restored losslessly (infinite values here). - return value; - } - return static_cast(JavaRound(value)) / 1.0f; - } - return static_cast(JavaRound(scaled) / mpv); -} - -double ExpectedDoubleValue(const FixtureCase& fixture_case, int row) { - if (!IsTs2DiffFloatCase(fixture_case)) { - return DoubleValue(row); - } - double value = Ts2DiffDoubleValue(row); - if (std::isnan(value)) { - return BitsToDouble(UINT64_C(0x7ff8000000000000)); - } - const double mpv = Ts2DiffMaxPointValue(fixture_case); - const double scaled = value * mpv; - const bool scaled_overflow = - scaled > static_cast(std::numeric_limits::max()) || - scaled < static_cast(std::numeric_limits::min()); - if (scaled_overflow) { - const bool value_overflows = - value > static_cast(std::numeric_limits::max()) || - value < static_cast(std::numeric_limits::min()); - if (value_overflows) { - return value; - } - return JavaRound(value) / 1.0; - } - return JavaRound(scaled) / mpv; -} - -int32_t DateValue(int row) { - return kDateValues[row % (sizeof(kDateValues) / sizeof(kDateValues[0]))]; -} - -std::vector BuildMatrix() { - std::vector cases; - CompressionType compressions[] = {UNCOMPRESSED, ZSTD, LZMA2}; - TSDataType data_types[] = {INT32, DATE, INT64, TIMESTAMP, FLOAT, DOUBLE}; - for (CompressionType compression : compressions) { - for (TSDataType data_type : data_types) { - cases.emplace_back(data_type, CHIMP, compression, kRowCount); - cases.emplace_back(data_type, RLBE, compression, kRowCount); - if (data_type == FLOAT || data_type == DOUBLE) { - // The value set exercises scaled, scale-overflow, and - // raw-bit forms, including the page-wide bitmaps across - // the 129-value TS_2DIFF block boundary. - cases.emplace_back(data_type, TS_2DIFF, compression, kRowCount); - } - } - cases.emplace_back(DOUBLE, CAMEL, compression, kRowCount); - } - for (CompressionType compression : {UNCOMPRESSED, LZ4, ZSTD, LZMA2}) { - for (TSDataType data_type : data_types) { - for (const char* pattern : {"MIXED", "ALL"}) { - FixtureCase fixture(data_type, PLAIN, compression, - kNullPageRows * 8); - fixture.null_pattern = pattern; - fixture.file_name += "." + FixtureCase::Lower(pattern) + ".512"; - cases.push_back(fixture); - } - FixtureCase fixture(data_type, PLAIN, compression, kNullPageRows); - fixture.null_pattern = "ALL"; - fixture.file_name += ".all.64"; - cases.push_back(fixture); - } - } - return cases; -} - -// Shared contract with the Java test: empty first/middle/consecutive/trailing -// pages, non-empty pages 1/5, and alternating null values inside page 4. -bool IsNull(const FixtureCase& fixture_case, int row) { - if (fixture_case.null_pattern == "ALL") return true; - if (fixture_case.null_pattern == "NONE") return false; - int page = row / kNullPageRows; - return page == 0 || page == 2 || page == 3 || page >= 6 || - (page == 4 && row % 2 == 0); -} - -// Restore process-wide settings even when a fatal assertion returns early. -class ScopedNullPageSize { - public: - explicit ScopedNullPageSize(const FixtureCase& fixture_case) - : old_rows_(g_config_value_.page_writer_max_point_num_), - old_bytes_(g_config_value_.page_writer_max_memory_bytes_) { - if (fixture_case.null_pattern != "NONE") { - g_config_value_.page_writer_max_point_num_ = kNullPageRows; - g_config_value_.page_writer_max_memory_bytes_ = 1024 * 1024; - } - } - ~ScopedNullPageSize() { - g_config_value_.page_writer_max_point_num_ = old_rows_; - g_config_value_.page_writer_max_memory_bytes_ = old_bytes_; - } - - private: - uint32_t old_rows_; - uint32_t old_bytes_; -}; - -TableSchema* CreateTableSchema(const FixtureCase& fixture_case) { - std::vector measurement_schemas; - std::vector column_categories; - measurement_schemas.emplace_back(new MeasurementSchema( - fixture_case.tag_column, STRING, PLAIN, UNCOMPRESSED)); - column_categories.emplace_back(ColumnCategory::TAG); - measurement_schemas.emplace_back( - new MeasurementSchema(fixture_case.value_column, fixture_case.data_type, - fixture_case.encoding, fixture_case.compression)); - column_categories.emplace_back(ColumnCategory::FIELD); - if (fixture_case.null_pattern != "NONE") { - measurement_schemas.emplace_back(new MeasurementSchema( - kAnchorColumn, INT64, PLAIN, fixture_case.compression)); - column_categories.emplace_back(ColumnCategory::FIELD); - } - return new TableSchema(fixture_case.table_name, measurement_schemas, - column_categories); -} - -void AddValue(Tablet& tablet, const FixtureCase& fixture_case, int row) { - switch (fixture_case.data_type) { - case INT32: - ASSERT_EQ(E_OK, tablet.add_value(row, kValueColumn, IntValue(row))); - break; - case DATE: - ASSERT_EQ(E_OK, - tablet.add_value(row, kValueColumn, DateValue(row))); - break; - case INT64: - case TIMESTAMP: - ASSERT_EQ(E_OK, - tablet.add_value(row, kValueColumn, LongValue(row))); - break; - case FLOAT: - ASSERT_EQ(E_OK, - tablet.add_value(row, kValueColumn, - ExpectedFloatValue(fixture_case, row))); - break; - case DOUBLE: - ASSERT_EQ(E_OK, - tablet.add_value(row, kValueColumn, - ExpectedDoubleValue(fixture_case, row))); - break; - default: - FAIL() << "Unsupported data type: " - << get_data_type_name(fixture_case.data_type); - } -} - -Tablet CreateTablet(TableSchema* table_schema, - const FixtureCase& fixture_case) { - Tablet tablet( - table_schema->get_table_name(), table_schema->get_measurement_names(), - table_schema->get_data_types(), table_schema->get_column_categories(), - fixture_case.row_count); - for (int row = 0; row < fixture_case.row_count; ++row) { - EXPECT_EQ(E_OK, tablet.add_timestamp(row, row)); - EXPECT_EQ(E_OK, tablet.add_value(row, kTagColumn, kTagValue)); - // Tablet entries remain null until add_value is called. - if (!IsNull(fixture_case, row)) { - AddValue(tablet, fixture_case, row); - } - if (fixture_case.null_pattern != "NONE") { - EXPECT_EQ(E_OK, tablet.add_value(row, kAnchorColumn, - static_cast(row))); - } - } - return tablet; -} - -void WriteFixture(const std::string& directory, - const FixtureCase& fixture_case) { - ScopedNullPageSize page_size(fixture_case); - std::string file_name = JoinPath(directory, fixture_case.file_name); - remove(file_name.c_str()); - WriteFile write_file; - int flags = O_WRONLY | O_CREAT | O_TRUNC; -#ifdef _WIN32 - flags |= O_BINARY; -#endif - ASSERT_EQ(E_OK, write_file.create(file_name, flags, 0666)); - TableSchema* table_schema = CreateTableSchema(fixture_case); - TsFileTableWriter writer(&write_file, table_schema); - Tablet tablet = CreateTablet(table_schema, fixture_case); - ASSERT_EQ(E_OK, writer.write_table(tablet)); - ASSERT_EQ(E_OK, writer.flush()); - ASSERT_EQ(E_OK, writer.close()); - delete table_schema; -} - -void WriteManifest(const std::string& directory, - const std::vector& cases) { - std::ofstream out(JoinPath(directory, kManifestFile).c_str(), - std::ios::out | std::ios::trunc); - ASSERT_TRUE(out.good()); - out << kManifestHeader << "\n"; - for (const FixtureCase& fixture_case : cases) { - out << fixture_case.ToManifestLine() << "\n"; - } -} - -std::vector ReadManifest(const std::string& directory) { - std::ifstream in(JoinPath(directory, kManifestFile).c_str()); - EXPECT_TRUE(in.good()); - std::vector cases; - std::string line; - if (!std::getline(in, line)) { - ADD_FAILURE() << "Empty manifest"; - return cases; - } - EXPECT_EQ(kManifestHeader, line); - while (std::getline(in, line)) { - if (!line.empty() && line.back() == '\r') { - line.pop_back(); - } - if (!line.empty()) { - cases.push_back(ParseManifestLine(line)); - } - } - return cases; -} - -void AssertValue(const FixtureCase& fixture_case, int row, - TableResultSet* result_set) { - switch (fixture_case.data_type) { - case BOOLEAN: - ASSERT_EQ(row % 192 < 64 ? row % 2 == 0 : row % 192 < 128, - result_set->get_value(3)); - break; - case INT32: - ASSERT_EQ(IntValue(row), result_set->get_value(3)); - break; - case DATE: - ASSERT_EQ(DateValue(row), result_set->get_value(3)); - break; - case INT64: - case TIMESTAMP: - ASSERT_EQ(LongValue(row), result_set->get_value(3)); - break; - case FLOAT: - ASSERT_EQ(FloatBits(ExpectedFloatValue(fixture_case, row)), - FloatBits(result_set->get_value(3))); - break; - case DOUBLE: - ASSERT_EQ(DoubleBits(ExpectedDoubleValue(fixture_case, row)), - DoubleBits(result_set->get_value(3))); - break; - default: - FAIL() << "Unsupported data type: " - << get_data_type_name(fixture_case.data_type); - } -} - -void AssertOnWireCodec(const std::string& directory, - const FixtureCase& fixture_case) { - TsFileIOReader io_reader; - ASSERT_EQ(E_OK, - io_reader.init(JoinPath(directory, fixture_case.file_name))); - - MetaIndexNode* table_root = nullptr; - ASSERT_EQ(E_OK, io_reader.get_tsfile_meta()->get_table_metaindex_node( - fixture_case.table_name, table_root)); - ASSERT_NE(nullptr, table_root); - ASSERT_EQ(1U, table_root->children_.size()); - std::shared_ptr device_id = - table_root->children_.front()->get_device_id(); - ASSERT_NE(nullptr, device_id); - - common::PageArena arena; - arena.init(512, common::MOD_TSFILE_READER); - std::vector indexes(1, nullptr); - std::unordered_set measurements = {fixture_case.value_column}; - ASSERT_EQ(E_OK, io_reader.get_timeseries_indexes(device_id, measurements, - indexes, arena)); - ASSERT_NE(nullptr, indexes.front()); - auto* aligned_index = - dynamic_cast(indexes.front()); - ASSERT_NE(nullptr, aligned_index); - - auto* value_chunks = aligned_index->get_value_chunk_meta_list(); - ASSERT_NE(nullptr, value_chunks); - ASSERT_GT(value_chunks->size(), 0U); - - LocalRandomAccessReadFile read_file; - ASSERT_EQ(E_OK, - read_file.open(JoinPath(directory, fixture_case.file_name))); - for (auto cursor = value_chunks->begin(); cursor != value_chunks->end(); - cursor++) { - ChunkMeta* chunk = cursor.get(); - ASSERT_NE(nullptr, chunk); - char header_buf[1024]; - int32_t read_len = 0; - ASSERT_EQ(E_OK, - read_file.read(chunk->offset_of_chunk_header_, header_buf, - sizeof(header_buf), read_len)); - ASSERT_GE(read_len, 7); - common::ByteStream stream; - stream.wrap_from(header_buf, read_len); - ChunkHeader chunk_header; - ASSERT_EQ(E_OK, chunk_header.deserialize_from(stream)); - ASSERT_EQ(fixture_case.encoding, chunk_header.encoding_type_); - ASSERT_EQ(fixture_case.compression, chunk_header.compression_type_); - } -} - -void ValidateQuery(const std::string& directory, - const FixtureCase& fixture_case, bool with_anchor, int start, - int end) { - SCOPED_TRACE(::testing::Message() - << fixture_case.ToManifestLine() << " anchor=" << with_anchor - << " range=" << start << ":" << end); - TsFileReader reader; - ASSERT_EQ(E_OK, reader.open(JoinPath(directory, fixture_case.file_name))); - std::vector columns = {fixture_case.tag_column, - fixture_case.value_column}; - if (with_anchor) columns.push_back(kAnchorColumn); - ResultSet* raw_result_set = nullptr; - ASSERT_EQ(E_OK, reader.query(fixture_case.table_name, columns, start, end, - raw_result_set)); - ASSERT_NE(nullptr, raw_result_set); - auto* result_set = static_cast(raw_result_set); - // Assertion failures must release the query before its reader is destroyed. - auto cleanup = [&reader](TableResultSet* result) { - reader.destroy_query_data_set(result); - }; - std::unique_ptr owned_result(result_set, - cleanup); - std::shared_ptr metadata = result_set->get_metadata(); - ASSERT_EQ(with_anchor ? 4U : 3U, metadata->get_column_count()); - ASSERT_EQ("time", metadata->get_column_name(1)); - ASSERT_EQ(INT64, metadata->get_column_type(1)); - ASSERT_EQ(fixture_case.tag_column, metadata->get_column_name(2)); - ASSERT_EQ(STRING, metadata->get_column_type(2)); - ASSERT_EQ(fixture_case.value_column, metadata->get_column_name(3)); - ASSERT_EQ(fixture_case.data_type, metadata->get_column_type(3)); - - bool has_next = false; - int row = start; - while (true) { - ASSERT_EQ(E_OK, result_set->next(has_next)); - if (!has_next) break; - SCOPED_TRACE(::testing::Message() << "row=" << row); - ASSERT_LE(row, end); - ASSERT_EQ(row, result_set->get_value(1)); - ASSERT_FALSE(result_set->is_null(2)); - ASSERT_EQ(kTagValue, - result_set->get_value(2)->to_std_string()); - ASSERT_EQ(IsNull(fixture_case, row), result_set->is_null(3)); - if (!IsNull(fixture_case, row)) { - ASSERT_NO_FATAL_FAILURE(AssertValue(fixture_case, row, result_set)); - } - if (with_anchor) { - ASSERT_FALSE(result_set->is_null(4)); - ASSERT_EQ(row, result_set->get_value(4)); - } - ++row; - } - ASSERT_EQ(end + 1, row); - owned_result.reset(); - ASSERT_EQ(E_OK, reader.close()); -} - -void ValidateFixture(const std::string& directory, - const FixtureCase& fixture_case) { - SCOPED_TRACE(fixture_case.ToManifestLine()); - ASSERT_NO_FATAL_FAILURE(AssertOnWireCodec(directory, fixture_case)); - ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, false, 0, - fixture_case.row_count - 1)); - if (fixture_case.null_pattern != "NONE") { - ASSERT_NO_FATAL_FAILURE(ValidateQuery(directory, fixture_case, true, 0, - fixture_case.row_count - 1)); - if (fixture_case.row_count > kNullPageRows) { - ASSERT_NO_FATAL_FAILURE( - ValidateQuery(directory, fixture_case, false, 63, 321)); - ASSERT_NO_FATAL_FAILURE( - ValidateQuery(directory, fixture_case, true, 128, 255)); - ASSERT_NO_FATAL_FAILURE( - ValidateQuery(directory, fixture_case, true, 320, 383)); - } - } -} - -} // namespace - -class TableModelEncodingCompressionCompatibilityTest : public ::testing::Test { - protected: - void SetUp() override { ASSERT_EQ(E_OK, libtsfile_init()); } - void TearDown() override { libtsfile_destroy(); } -}; - -TEST_F(TableModelEncodingCompressionCompatibilityTest, GenerateFixtures) { - const char* output_dir = std::getenv(kGenerateDirEnv); - if (output_dir == nullptr || std::strlen(output_dir) == 0) { - GTEST_SKIP() << kGenerateDirEnv << " is not set"; - } - EnsureDirectory(output_dir); - std::vector cases = BuildMatrix(); - for (const FixtureCase& fixture_case : cases) { - WriteFixture(output_dir, fixture_case); - AssertOnWireCodec(output_dir, fixture_case); - } - WriteManifest(output_dir, cases); -} - -TEST_F(TableModelEncodingCompressionCompatibilityTest, ValidateFixtures) { - const char* input_dir = std::getenv(kValidateDirEnv); - if (input_dir == nullptr || std::strlen(input_dir) == 0) { - GTEST_SKIP() << kValidateDirEnv << " is not set"; - } - std::vector cases = ReadManifest(input_dir); - for (const FixtureCase& fixture_case : cases) { - ASSERT_NO_FATAL_FAILURE(ValidateFixture(input_dir, fixture_case)); - } -} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java new file mode 100644 index 0000000000..e32d4671cf --- /dev/null +++ b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/EncodingCompressionCompatibilityTest.java @@ -0,0 +1,1632 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.compatibility; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.encoding.encoder.Encoder; +import org.apache.tsfile.encoding.encoder.TSEncodingBuilder; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.header.ChunkHeader; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileReader; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.read.expression.IExpression; +import org.apache.tsfile.read.expression.QueryExpression; +import org.apache.tsfile.read.expression.impl.BinaryExpression; +import org.apache.tsfile.read.expression.impl.GlobalTimeExpression; +import org.apache.tsfile.read.filter.factory.TimeFilterApi; +import org.apache.tsfile.read.query.dataset.QueryDataSet; +import org.apache.tsfile.read.query.dataset.ResultSet; +import org.apache.tsfile.read.query.dataset.ResultSetMetadata; +import org.apache.tsfile.read.v4.ITsFileReader; +import org.apache.tsfile.read.v4.TsFileReaderBuilder; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.chunk.ChunkWriterImpl; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; +import org.apache.tsfile.write.record.datapoint.DataPoint; +import org.apache.tsfile.write.record.datapoint.DateDataPoint; +import org.apache.tsfile.write.record.datapoint.DoubleDataPoint; +import org.apache.tsfile.write.record.datapoint.FloatDataPoint; +import org.apache.tsfile.write.record.datapoint.IntDataPoint; +import org.apache.tsfile.write.record.datapoint.LongDataPoint; +import org.apache.tsfile.write.record.datapoint.StringDataPoint; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import org.junit.Assume; +import org.junit.Test; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Writes and validates the Java side of the cross-language compatibility matrix in {@code + * compat/compatibility-matrix.csv}. The C++ counterpart is {@code + * cpp/test/reader/table_view/encoding_compression_compatibility_test.cc}; both read the same spec, + * write every case they declare writable, and validate the fixtures written by the other language. + * + *

See {@code compat/README.md} for the spec format and the value profile contract. + */ +public class EncodingCompressionCompatibilityTest { + + private static final String SPEC_PROPERTY = "tsfile.compat.spec"; + private static final String GENERATE_DIR_PROPERTY = "tsfile.compat.generate.dir"; + private static final String VALIDATE_DIR_PROPERTY = "tsfile.compat.validate.dir"; + private static final String DEFAULT_SPEC = "compat/compatibility-matrix.csv"; + + private static final String MANIFEST_FILE = "manifest.csv"; + private static final String REPORT_FILE = "report.csv"; + private static final String SUMMARY_FILE = "summary.md"; + private static final String LANGUAGE = "java"; + + private static final String SPEC_HEADER = + "id,model,dataType,encoding,compressions,rowCount,pageRows,chunks,devices,valueProfile," + + "nullPatterns,javaWrite,cppWrite,javaRead,cppRead,gapReason"; + private static final String MANIFEST_HEADER = + "producer,id,model,dataType,encoding,compression,rowCount,pageRows,chunks,devices," + + "valueProfile,nullPattern"; + + private static final String[] ALL_COMPRESSIONS = { + "UNCOMPRESSED", "SNAPPY", "GZIP", "LZ4", "ZSTD", "LZMA2" + }; + + // Concrete storage types covered by the matrix. OBJECT is a pseudo type in + // TYPE_SUPPORTED_ENCODINGS and VECTOR has no writer on either side. + private static final List MATRIX_DATA_TYPES = + Collections.unmodifiableList( + Arrays.asList( + TSDataType.BOOLEAN, + TSDataType.INT32, + TSDataType.INT64, + TSDataType.TIMESTAMP, + TSDataType.DATE, + TSDataType.FLOAT, + TSDataType.DOUBLE, + TSDataType.TEXT, + TSDataType.STRING, + TSDataType.BLOB)); + + private static final String TABLE_NAME = "compat_table"; + private static final String TAG_COLUMN = "device"; + private static final String VALUE_COLUMN = "value"; + private static final String ANCHOR_COLUMN = "anchor"; + private static final String DEVICE_PREFIX = "compat_table_device_"; + + private enum Model { + TABLE, + TREE + } + + private enum NullPattern { + NONE, + MIXED, + ALL + } + + private enum ValueProfile { + DEFAULT, + TS2DIFF, + TS2DIFF_MPN0, + SPRINTZ, + RUNS, + STRINGS, + DICT_HIGH, + BLOB + } + + private static final class FixtureCase { + final String specId; + final String id; + final Model model; + final TSDataType dataType; + final TSEncoding encoding; + final CompressionType compression; + final int rowCount; + final int pageRows; + final int chunks; + final int devices; + final ValueProfile profile; + final NullPattern nullPattern; + final boolean javaWrite; + final boolean cppWrite; + final boolean javaRead; + final boolean cppRead; + final String gapReason; + + FixtureCase( + String specId, + String id, + Model model, + TSDataType dataType, + TSEncoding encoding, + CompressionType compression, + int rowCount, + int pageRows, + int chunks, + int devices, + ValueProfile profile, + NullPattern nullPattern, + boolean javaWrite, + boolean cppWrite, + boolean javaRead, + boolean cppRead, + String gapReason) { + this.specId = specId; + this.id = id; + this.model = model; + this.dataType = dataType; + this.encoding = encoding; + this.compression = compression; + this.rowCount = rowCount; + this.pageRows = pageRows; + this.chunks = chunks; + this.devices = devices; + this.profile = profile; + this.nullPattern = nullPattern; + this.javaWrite = javaWrite; + this.cppWrite = cppWrite; + this.javaRead = javaRead; + this.cppRead = cppRead; + this.gapReason = gapReason; + } + + String fileName() { + return id + ".tsfile"; + } + + boolean writes(String language) { + return "java".equals(language) ? javaWrite : cppWrite; + } + + boolean reads(String language) { + return "java".equals(language) ? javaRead : cppRead; + } + + String describe() { + return id + + " (" + + model + + "/" + + dataType + + "/" + + encoding + + "/" + + compression + + ")"; + } + } + + // --------------------------------------------------------------------------- + // Entry points + // --------------------------------------------------------------------------- + + /** + * Guards the spec itself: every (type, encoding, compression) combination the Java writer + * supports must be covered, every declared Java case must be constructible, and every declared + * gap must carry a reason. Runs without any environment configuration, so spec drift fails the + * normal Java test suite. + */ + @Test + public void specMatchesJavaCapabilities() throws IOException { + List cases = readSpec(resolveSpecPath()); + Set covered = new LinkedHashSet<>(); + for (FixtureCase fixtureCase : cases) { + if (fixtureCase.gapReason.isEmpty()) { + assertTrue( + "gapReason is required when a case declares a gap: " + fixtureCase.describe(), + fixtureCase.javaWrite && fixtureCase.cppWrite + && fixtureCase.javaRead + && fixtureCase.cppRead); + } + if (fixtureCase.javaWrite) { + assertTrue( + "a Java-written case must also be Java-readable: " + fixtureCase.describe(), + fixtureCase.javaRead); + Encoder encoder = + TSEncodingBuilder.getEncodingBuilder(fixtureCase.encoding) + .getEncoder(fixtureCase.dataType); + assertNotNull( + "Java cannot build an encoder for " + fixtureCase.describe(), encoder); + covered.add(capabilityKey(fixtureCase)); + } + } + + List missing = new ArrayList<>(); + for (Model model : Model.values()) { + for (TSDataType dataType : MATRIX_DATA_TYPES) { + Set supported = TSEncoding.TYPE_SUPPORTED_ENCODINGS.get(dataType); + assertNotNull("no supported encodings recorded for " + dataType, supported); + for (TSEncoding encoding : supported) { + for (String compression : ALL_COMPRESSIONS) { + String key = model + "/" + dataType + "/" + encoding + "/" + compression; + if (!covered.contains(key)) { + missing.add(key); + } + } + } + } + } + if (!missing.isEmpty()) { + fail( + "compatibility-matrix.csv does not declare Java-writable cases for: " + + String.join(", ", missing)); + } + } + + @Test + public void generateFixtures() throws Exception { + String outputDir = System.getProperty(GENERATE_DIR_PROPERTY); + Assume.assumeTrue( + GENERATE_DIR_PROPERTY + " is not set", + outputDir != null && !outputDir.isEmpty()); + Path directory = Paths.get(outputDir); + Files.createDirectories(directory); + + List cases = readSpec(resolveSpecPath()); + List written = new ArrayList<>(); + List report = new ArrayList<>(); + for (FixtureCase fixtureCase : cases) { + if (!fixtureCase.javaWrite) { + report.add( + new String[] {fixtureCase.id, "not-mine", fixtureCase.gapReason}); + continue; + } + writeFixture(directory, fixtureCase); + // A writer must be able to read back what it wrote; declared-unreadable + // cases are rejected by specMatchesJavaCapabilities. + validateFixture(directory, fixtureCase); + written.add(fixtureCase); + report.add(new String[] {fixtureCase.id, "generated-and-validated", ""}); + } + writeManifest(directory, written); + writeReport(directory, REPORT_FILE, report); + writeSummary(directory, SUMMARY_FILE, "Java fixture generation", cases, report); + } + + @Test + public void validateFixtures() throws Exception { + String inputDir = System.getProperty(VALIDATE_DIR_PROPERTY); + Assume.assumeTrue( + VALIDATE_DIR_PROPERTY + " is not set", inputDir != null && !inputDir.isEmpty()); + Path directory = Paths.get(inputDir); + + List manifest = readManifest(directory); + String producer = manifest.isEmpty() ? "" : manifest.get(0).producer; + assertFalse("empty manifest in " + directory, manifest.isEmpty()); + assertTrue("unexpected producer " + producer, "java".equals(producer) || "cpp".equals(producer)); + + Map spec = new LinkedHashMap<>(); + for (FixtureCase fixtureCase : readSpec(resolveSpecPath())) { + spec.put(fixtureCase.id, fixtureCase); + } + + Set expectedIds = new LinkedHashSet<>(); + for (FixtureCase fixtureCase : spec.values()) { + if (fixtureCase.writes(producer)) { + expectedIds.add(fixtureCase.id); + } + } + Set manifestIds = new LinkedHashSet<>(); + for (ManifestEntry entry : manifest) { + manifestIds.add(entry.id); + } + List missing = new ArrayList<>(expectedIds); + missing.removeAll(manifestIds); + List extra = new ArrayList<>(manifestIds); + extra.removeAll(expectedIds); + if (!missing.isEmpty() || !extra.isEmpty()) { + fail( + "manifest for " + + producer + + " does not match the spec; missing=" + + missing + + " extra=" + + extra); + } + + List report = new ArrayList<>(); + for (ManifestEntry entry : manifest) { + FixtureCase fixtureCase = spec.get(entry.id); + assertNotNull("manifest case absent from the spec: " + entry.id, fixtureCase); + entry.assertMatches(fixtureCase); + if (!fixtureCase.reads(LANGUAGE)) { + report.add(new String[] {entry.id, "skipped-declared-gap", fixtureCase.gapReason}); + continue; + } + Path file = directory.resolve(entry.id + ".tsfile"); + assertTrue("missing fixture declared as readable: " + file, Files.isRegularFile(file)); + validateFixture(directory, fixtureCase); + report.add(new String[] {entry.id, "validated", ""}); + } + writeReport(directory, REPORT_FILE + ".validate", report); + writeSummary( + directory, + SUMMARY_FILE + ".validate", + "Java validation of " + producer + " fixtures", + new ArrayList<>(spec.values()), + report); + } + + // --------------------------------------------------------------------------- + // Spec reading and expansion + // --------------------------------------------------------------------------- + + private static Path resolveSpecPath() { + String configured = System.getProperty(SPEC_PROPERTY); + if (configured != null && !configured.isEmpty()) { + Path path = Paths.get(configured); + if (!Files.isRegularFile(path)) { + throw new IllegalStateException(SPEC_PROPERTY + " does not name a file: " + path); + } + return path; + } + Path directory = Paths.get("").toAbsolutePath(); + for (int depth = 0; depth < 8 && directory != null; depth++, directory = directory.getParent()) { + Path candidate = directory.resolve(DEFAULT_SPEC); + if (Files.isRegularFile(candidate)) { + return candidate; + } + } + throw new IllegalStateException( + "Cannot locate " + DEFAULT_SPEC + " above " + Paths.get("").toAbsolutePath()); + } + + private static List readSpec(Path specPath) throws IOException { + List lines = Files.readAllLines(specPath, StandardCharsets.UTF_8); + Map columns = null; + List cases = new ArrayList<>(); + Set ids = new LinkedHashSet<>(); + for (String raw : lines) { + String line = raw.trim(); + if (line.isEmpty() || line.startsWith("#")) { + continue; + } + String[] parts = line.split(",", -1); + if (columns == null) { + columns = indexColumns(parts, SPEC_HEADER, specPath); + continue; + } + if (parts.length != columns.size()) { + throw new IllegalArgumentException("Bad spec line (" + parts.length + " fields): " + line); + } + String id = part(parts, columns, "id"); + String model = part(parts, columns, "model"); + String dataType = part(parts, columns, "dataType"); + String encoding = part(parts, columns, "encoding"); + String compressions = part(parts, columns, "compressions"); + int rowCount = Integer.parseInt(part(parts, columns, "rowCount")); + int pageRows = Integer.parseInt(part(parts, columns, "pageRows")); + int chunks = Integer.parseInt(part(parts, columns, "chunks")); + int devices = Integer.parseInt(part(parts, columns, "devices")); + String profile = part(parts, columns, "valueProfile"); + String patterns = part(parts, columns, "nullPatterns"); + boolean javaWrite = parseYesNo(part(parts, columns, "javaWrite")); + boolean cppWrite = parseYesNo(part(parts, columns, "cppWrite")); + boolean javaRead = parseYesNo(part(parts, columns, "javaRead")); + boolean cppRead = parseYesNo(part(parts, columns, "cppRead")); + String gapReason = part(parts, columns, "gapReason"); + + List compressionList = + "ALL".equals(compressions) + ? Arrays.asList(ALL_COMPRESSIONS) + : Arrays.asList(compressions.split("\\+")); + List patternList = Arrays.asList(patterns.split("\\|")); + for (String compression : compressionList) { + for (String pattern : patternList) { + String expandedId = + id + + "." + + compression.toLowerCase(Locale.ROOT) + + "." + + pattern.toLowerCase(Locale.ROOT); + if (!ids.add(expandedId)) { + throw new IllegalArgumentException("Duplicate expanded case id: " + expandedId); + } + cases.add( + new FixtureCase( + id, + expandedId, + Model.valueOf(model.toUpperCase(Locale.ROOT)), + TSDataType.valueOf(dataType), + TSEncoding.valueOf(encoding), + CompressionType.valueOf(compression), + rowCount, + pageRows, + chunks, + devices, + ValueProfile.valueOf(profile.toUpperCase(Locale.ROOT)), + NullPattern.valueOf(pattern.toUpperCase(Locale.ROOT)), + javaWrite, + cppWrite, + javaRead, + cppRead, + gapReason)); + } + } + } + if (columns == null) { + throw new IllegalArgumentException("Empty compatibility spec: " + specPath); + } + return cases; + } + + private static Map indexColumns( + String[] header, String expected, Path specPath) { + Map columns = new LinkedHashMap<>(); + for (int i = 0; i < header.length; i++) { + columns.put(header[i], i); + } + if (!String.join(",", header).equals(expected)) { + throw new IllegalArgumentException( + "Unexpected header in " + specPath + ": " + String.join(",", header)); + } + return columns; + } + + private static String part(String[] parts, Map columns, String name) { + Integer index = columns.get(name); + if (index == null) { + throw new IllegalArgumentException("Missing column " + name); + } + return parts[index]; + } + + private static boolean parseYesNo(String value) { + if ("yes".equals(value)) { + return true; + } + if ("no".equals(value)) { + return false; + } + throw new IllegalArgumentException("Expected yes/no, got: " + value); + } + + private static String capabilityKey(FixtureCase fixtureCase) { + return fixtureCase.model + + "/" + + fixtureCase.dataType + + "/" + + fixtureCase.encoding + + "/" + + fixtureCase.compression; + } + + // --------------------------------------------------------------------------- + // Manifest and reports + // --------------------------------------------------------------------------- + + private static final class ManifestEntry { + final String producer; + final String id; + final Model model; + final TSDataType dataType; + final TSEncoding encoding; + final CompressionType compression; + final int rowCount; + final int pageRows; + final int chunks; + final int devices; + final ValueProfile profile; + final NullPattern nullPattern; + + ManifestEntry( + String producer, + String id, + Model model, + TSDataType dataType, + TSEncoding encoding, + CompressionType compression, + int rowCount, + int pageRows, + int chunks, + int devices, + ValueProfile profile, + NullPattern nullPattern) { + this.producer = producer; + this.id = id; + this.model = model; + this.dataType = dataType; + this.encoding = encoding; + this.compression = compression; + this.rowCount = rowCount; + this.pageRows = pageRows; + this.chunks = chunks; + this.devices = devices; + this.profile = profile; + this.nullPattern = nullPattern; + } + + void assertMatches(FixtureCase fixtureCase) { + assertEquals("spec id " + id, fixtureCase.id, id); + assertEquals("model " + id, fixtureCase.model, model); + assertEquals("dataType " + id, fixtureCase.dataType, dataType); + assertEquals("encoding " + id, fixtureCase.encoding, encoding); + assertEquals("compression " + id, fixtureCase.compression, compression); + assertEquals("rowCount " + id, fixtureCase.rowCount, rowCount); + assertEquals("pageRows " + id, fixtureCase.pageRows, pageRows); + assertEquals("chunks " + id, fixtureCase.chunks, chunks); + assertEquals("devices " + id, fixtureCase.devices, devices); + assertEquals("valueProfile " + id, fixtureCase.profile, profile); + assertEquals("nullPattern " + id, fixtureCase.nullPattern, nullPattern); + } + } + + private static void writeManifest(Path directory, List cases) throws IOException { + try (BufferedWriter writer = + Files.newBufferedWriter(directory.resolve(MANIFEST_FILE), StandardCharsets.UTF_8)) { + writer.write(MANIFEST_HEADER); + writer.newLine(); + for (FixtureCase fixtureCase : cases) { + writer.write( + String.join( + ",", + LANGUAGE, + fixtureCase.id, + fixtureCase.model.name(), + fixtureCase.dataType.name(), + fixtureCase.encoding.name(), + fixtureCase.compression.name(), + String.valueOf(fixtureCase.rowCount), + String.valueOf(fixtureCase.pageRows), + String.valueOf(fixtureCase.chunks), + String.valueOf(fixtureCase.devices), + fixtureCase.profile.name(), + fixtureCase.nullPattern.name())); + writer.newLine(); + } + } + } + + private static List readManifest(Path directory) throws IOException { + Path manifest = directory.resolve(MANIFEST_FILE); + if (!Files.isRegularFile(manifest)) { + throw new IllegalStateException("Missing manifest: " + manifest); + } + List lines = Files.readAllLines(manifest, StandardCharsets.UTF_8); + Map columns = null; + List entries = new ArrayList<>(); + for (String raw : lines) { + String line = raw.trim(); + if (line.isEmpty()) { + continue; + } + String[] parts = line.split(",", -1); + if (columns == null) { + columns = indexColumns(parts, MANIFEST_HEADER, manifest); + continue; + } + if (parts.length != columns.size()) { + throw new IllegalArgumentException("Bad manifest line: " + line); + } + entries.add( + new ManifestEntry( + part(parts, columns, "producer"), + part(parts, columns, "id"), + Model.valueOf(part(parts, columns, "model")), + TSDataType.valueOf(part(parts, columns, "dataType")), + TSEncoding.valueOf(part(parts, columns, "encoding")), + CompressionType.valueOf(part(parts, columns, "compression")), + Integer.parseInt(part(parts, columns, "rowCount")), + Integer.parseInt(part(parts, columns, "pageRows")), + Integer.parseInt(part(parts, columns, "chunks")), + Integer.parseInt(part(parts, columns, "devices")), + ValueProfile.valueOf(part(parts, columns, "valueProfile")), + NullPattern.valueOf(part(parts, columns, "nullPattern")))); + } + return entries; + } + + private static void writeReport(Path directory, String name, List report) + throws IOException { + try (BufferedWriter writer = + Files.newBufferedWriter(directory.resolve(name), StandardCharsets.UTF_8)) { + writer.write("id,status,detail"); + writer.newLine(); + for (String[] row : report) { + writer.write(String.join(",", row)); + writer.newLine(); + } + } + } + + private static void writeSummary( + Path directory, String name, String title, List cases, List report) + throws IOException { + int generated = 0; + int validated = 0; + int skipped = 0; + Map byStatus = new LinkedHashMap<>(); + List gaps = new ArrayList<>(); + for (String[] row : report) { + byStatus.merge(row[1], 1, Integer::sum); + if (row[1].startsWith("generated") || "written".equals(row[1])) { + generated++; + } + if (row[1].startsWith("validated") || row[1].endsWith("and-validated")) { + validated++; + } + if (row[1].startsWith("skipped")) { + skipped++; + gaps.add(row[0] + ": " + row[2]); + } + } + Map byModel = new LinkedHashMap<>(); + Map byType = new LinkedHashMap<>(); + for (FixtureCase fixtureCase : cases) { + byModel.merge(fixtureCase.model.name(), 1, Integer::sum); + byType.merge(fixtureCase.dataType.name(), 1, Integer::sum); + } + StringBuilder out = new StringBuilder(); + out.append("### ").append(title).append("\n\n"); + out.append("- Spec cases: ").append(cases.size()).append(" ("); + out.append(formatCounts(byModel)).append(")\n"); + out.append("- Status: ").append(formatCounts(byStatus)).append("\n"); + out.append("- Generated: ").append(generated).append(", validated: ").append(validated); + out.append(", skipped: ").append(skipped).append("\n"); + out.append("- Types: ").append(formatCounts(byType)).append("\n"); + if (!gaps.isEmpty()) { + out.append("\nDeclared/observed gaps:\n\n"); + for (String gap : gaps) { + out.append("- ").append(gap).append("\n"); + } + } + Files.write( + directory.resolve(name), + out.toString().getBytes(StandardCharsets.UTF_8)); + System.out.println(out); + } + + private static String formatCounts(Map counts) { + StringBuilder out = new StringBuilder(); + for (Map.Entry entry : counts.entrySet()) { + if (out.length() > 0) { + out.append(", "); + } + out.append(entry.getKey()).append('=').append(entry.getValue()); + } + return out.toString(); + } + + // --------------------------------------------------------------------------- + // Deterministic value profiles (shared contract with the C++ test) + // --------------------------------------------------------------------------- + + private static final int[] INT_VALUES = { + 0, 1, -1, 7, -8, 1024, -1024, 123456, -654321, + Integer.MAX_VALUE - 1024, Integer.MIN_VALUE + 1024, 42, 42, 0, -8, 2048 + }; + + private static final long[] LONG_VALUES = { + 0L, 1L, -1L, 7L, -8L, 1_234_567_890L, -987_654_321L, 4_294_967_296L, + -4_294_967_296L, 1_234_567_890_123L, -1_234_567_890_123L, + Long.MAX_VALUE - 4096, Long.MIN_VALUE + 4096, 42L, 42L, 0L + }; + + private static final int[] FLOAT_BITS = { + 0x00000000, 0x80000000, 0x3f800000, 0xbf800000, 0x41280000, 0xc0700000, + 0x44801000, 0xc5002000, 0x40490fdb, 0xc02df854, 0x3f800000, 0x3f800000, + 0x00000000, 0x80000000, 0x42f6e979, 0xc2f6e979 + }; + + private static final long[] DOUBLE_BITS = { + 0x0000000000000000L, 0x8000000000000000L, 0x3ff0000000000000L, + 0xbff0000000000000L, 0x4029000000000000L, 0xc00c000000000000L, + 0x4090008000000000L, 0xc0a0010000000000L, 0x400921fb54442d18L, + 0xc005bf0a8b145769L, 0x3ff0000000000000L, 0x3ff0000000000000L, + 0x0000000000000000L, 0x8000000000000000L, 0x405edd2f1a9fbe77L, + 0xc05edd2f1a9fbe77L + }; + + private static final int[] TS_2DIFF_FLOAT_BITS = { + 0x00000000, 0x3f800000, 0xbf800000, 0x461c4000, 0xc61c4000, 0x4eb2d05e, + 0x7f800000, 0xff800000, 0x7fc00000, 0x3f800000, 0x00000000, 0x4a742400, + 0xca742400, 0x4e6e6b28, 0x3f800000, 0x00000000 + }; + + private static final long[] TS_2DIFF_DOUBLE_BITS = { + 0x0000000000000000L, 0x3ff0000000000000L, 0xbff0000000000000L, + 0x40c3880000000000L, 0xc0c3880000000000L, 0x43abc16d674ec800L, + 0x7ff0000000000000L, 0xfff0000000000000L, 0x7ff8000000000000L, + 0x3ff0000000000000L, 0x0000000000000000L, 0x41f0000000000000L, + 0xc1cd6f3458800000L, 0x430c6bf526340000L, 0x3ff0000000000000L, + 0x0000000000000000L + }; + + // SPRINTZ predicts from first-order differences of the stored integers; a + // block that spans INT_MIN and INT_MAX overflows the predictor, so these + // values stay inside a safe range (see compat/README.md). + private static final int[] SPRINTZ_INT_VALUES = { + 0, 1, -1, 7, -8, 1024, -1024, 123456, -654321, + 100_000_000, -100_000_000, 42, 42, 0, -8, 2048 + }; + + private static final long[] SPRINTZ_LONG_VALUES = { + 0L, 1L, -1L, 7L, -8L, 1_234_567_890L, -987_654_321L, 4_294_967_296L, + -4_294_967_296L, 1_000_000_000_000L, -1_000_000_000_000L, + 42L, 42L, 0L, -8L, 2048L + }; + + private static final int[] SPRINTZ_FLOAT_BITS = { + 0x3f800000, 0x3fa00000, 0x3fc00000, 0x3fe00000, 0x40000000, 0x3fc00000, + 0x3fa00000, 0x3f800000, 0x3fc00000, 0x3fe00000, 0x40000000, 0x3fe00000, + 0x3fc00000, 0x3f800000, 0x3fa00000, 0x3fc00000 + }; + + private static final long[] SPRINTZ_DOUBLE_BITS = { + 0x3ff0000000000000L, 0x3ff4000000000000L, 0x3ff8000000000000L, + 0x3ffc000000000000L, 0x4000000000000000L, 0x3ff8000000000000L, + 0x3ff4000000000000L, 0x3ff0000000000000L, 0x3ff8000000000000L, + 0x3ffc000000000000L, 0x4000000000000000L, 0x3ffc000000000000L, + 0x3ff8000000000000L, 0x3ff0000000000000L, 0x3ff4000000000000L, + 0x3ff8000000000000L + }; + + private static final LocalDate[] DATE_VALUES = { + LocalDate.of(1970, 1, 1), + LocalDate.of(1999, 12, 31), + LocalDate.of(2000, 2, 29), + LocalDate.of(2024, 2, 29), + LocalDate.of(2038, 1, 19), + LocalDate.of(2050, 6, 15), + LocalDate.of(1969, 7, 20), + LocalDate.of(1980, 1, 6) + }; + + private static final String[] STRING_VALUES = { + "", + "hello", + "你好，世界", + "nul\u0000inside", + "ÄÖÜ ß 日本語", + "value", + "value", + repeat('a', 1024), + "😀 emoji 🌍" + }; + + private static final byte[][] BLOB_VALUES = { + {}, + {0}, + {0, 0, 0, 0}, + {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + "blob-as-text".getBytes(StandardCharsets.UTF_8), + {(byte) 0xff, (byte) 0xfe, (byte) 0x80, (byte) 0x00, (byte) 0x7f}, + {}, + "blob-as-text".getBytes(StandardCharsets.UTF_8) + }; + + // The TS_2DIFF builder and the RLE float/double builder both default + // maxPointNumber to TSFileConfig.floatPrecision, currently 2. + private static final int DEFAULT_MAX_POINT_NUMBER = 2; + private static final double DEFAULT_MAX_POINT_VALUE = 100.0; + + private static String repeat(char c, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, c); + return new String(chars); + } + + private static boolean isNull(FixtureCase fixtureCase, int row) { + if (fixtureCase.nullPattern == NullPattern.ALL) { + return true; + } + if (fixtureCase.nullPattern == NullPattern.NONE) { + return false; + } + int pageRows = nullPatternPageRows(fixtureCase); + int page = row / pageRows; + // Empty first page, a consecutive empty pair in the middle, empty tail + // pages, and alternating nulls inside one interior page. Both + // implementations share this contract. + return page == 0 || page == 2 || page == 3 || page >= 6 || (page == 4 && row % 2 == 0); + } + + private static Object valueFor(FixtureCase fixtureCase, int row) { + switch (fixtureCase.dataType) { + case BOOLEAN: + return booleanValue(row); + case INT32: + return intValue(fixtureCase, row); + case DATE: + return dateValue(fixtureCase, row); + case INT64: + case TIMESTAMP: + return longValue(fixtureCase, row); + case FLOAT: + return floatValue(fixtureCase, row); + case DOUBLE: + return doubleValue(fixtureCase, row); + case TEXT: + case STRING: + return stringValue(fixtureCase, row); + case BLOB: + return blobValue(fixtureCase, row); + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } + + private static Object expectedValue(FixtureCase fixtureCase, int row) { + if (isFloatEncoderCase(fixtureCase)) { + if (fixtureCase.dataType == TSDataType.FLOAT) { + return expectedFloatValue(fixtureCase, row); + } + return expectedDoubleValue(fixtureCase, row); + } + return valueFor(fixtureCase, row); + } + + // TS_2DIFF and RLE for FLOAT/DOUBLE scale by 10^maxPointNumber before + // storing an integer and divide on the way out. + private static boolean isFloatEncoderCase(FixtureCase fixtureCase) { + return (fixtureCase.encoding == TSEncoding.TS_2DIFF + || fixtureCase.encoding == TSEncoding.RLE) + && (fixtureCase.dataType == TSDataType.FLOAT + || fixtureCase.dataType == TSDataType.DOUBLE); + } + + private static double maxPointValue(FixtureCase fixtureCase) { + return fixtureCase.profile == ValueProfile.TS2DIFF_MPN0 ? 1.0 : DEFAULT_MAX_POINT_VALUE; + } + + private static boolean booleanValue(int row) { + return row % 192 < 64 ? row % 2 == 0 : row % 192 < 128; + } + + private static int intValue(FixtureCase fixtureCase, int row) { + if (fixtureCase.profile == ValueProfile.RUNS) { + return (row / 256) % 4 - 1; + } + if (fixtureCase.profile == ValueProfile.SPRINTZ) { + return SPRINTZ_INT_VALUES[row % SPRINTZ_INT_VALUES.length]; + } + return INT_VALUES[row % INT_VALUES.length]; + } + + private static long longValue(FixtureCase fixtureCase, int row) { + if (fixtureCase.profile == ValueProfile.RUNS) { + return (long) ((row / 256) % 4 - 1) * 1_000_000_000L; + } + if (fixtureCase.profile == ValueProfile.SPRINTZ) { + return SPRINTZ_LONG_VALUES[row % SPRINTZ_LONG_VALUES.length]; + } + return LONG_VALUES[row % LONG_VALUES.length]; + } + + private static LocalDate dateValue(FixtureCase fixtureCase, int row) { + if (fixtureCase.profile == ValueProfile.RUNS) { + return DATE_VALUES[(row / 256) % DATE_VALUES.length]; + } + return DATE_VALUES[row % DATE_VALUES.length]; + } + + private static float floatValue(FixtureCase fixtureCase, int row) { + switch (fixtureCase.profile) { + case TS2DIFF: + case TS2DIFF_MPN0: + return Float.intBitsToFloat( + TS_2DIFF_FLOAT_BITS[row % TS_2DIFF_FLOAT_BITS.length]); + case RUNS: + return (float) ((row / 256) % 4 - 1); + case SPRINTZ: + return Float.intBitsToFloat( + SPRINTZ_FLOAT_BITS[row % SPRINTZ_FLOAT_BITS.length]); + default: + return Float.intBitsToFloat(FLOAT_BITS[row % FLOAT_BITS.length]); + } + } + + private static double doubleValue(FixtureCase fixtureCase, int row) { + switch (fixtureCase.profile) { + case TS2DIFF: + case TS2DIFF_MPN0: + return Double.longBitsToDouble( + TS_2DIFF_DOUBLE_BITS[row % TS_2DIFF_DOUBLE_BITS.length]); + case RUNS: + return (double) ((row / 256) % 4 - 1); + case SPRINTZ: + return Double.longBitsToDouble( + SPRINTZ_DOUBLE_BITS[row % SPRINTZ_DOUBLE_BITS.length]); + default: + return Double.longBitsToDouble(DOUBLE_BITS[row % DOUBLE_BITS.length]); + } + } + + private static String stringValue(FixtureCase fixtureCase, int row) { + if (fixtureCase.profile == ValueProfile.DICT_HIGH) { + return "key_" + row; + } + return STRING_VALUES[row % STRING_VALUES.length]; + } + + private static byte[] blobValue(FixtureCase fixtureCase, int row) { + return BLOB_VALUES[row % BLOB_VALUES.length]; + } + + // Mirror FloatEncoder/DoubleEncoder at the effective maxPointNumber: + // scaled -> round(v * mpv) / mpv, scale-overflow -> round(v) / 1, + // raw bits -> back from the IEEE pattern (NaN is canonicalized on write). + private static float expectedFloatValue(FixtureCase fixtureCase, int row) { + if (!isFloatEncoderCase(fixtureCase)) { + return floatValue(fixtureCase, row); + } + float value = floatValue(fixtureCase, row); + if (Float.isNaN(value)) { + return Float.intBitsToFloat(0x7fc00000); + } + double mpv = maxPointValue(fixtureCase); + double scaled = (double) value * mpv; + if (scaled > Integer.MAX_VALUE || scaled < Integer.MIN_VALUE) { + if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { + return Float.intBitsToFloat(Float.floatToIntBits(value)); + } + return (float) ((double) Math.round(value) / 1.0); + } + return (float) ((double) Math.round(scaled) / mpv); + } + + private static double expectedDoubleValue(FixtureCase fixtureCase, int row) { + if (!isFloatEncoderCase(fixtureCase)) { + return doubleValue(fixtureCase, row); + } + double value = doubleValue(fixtureCase, row); + if (Double.isNaN(value)) { + return Double.longBitsToDouble(0x7ff8000000000000L); + } + double mpv = maxPointValue(fixtureCase); + double scaled = value * mpv; + if (scaled > Long.MAX_VALUE || scaled < Long.MIN_VALUE) { + if (value > Long.MAX_VALUE || value < Long.MIN_VALUE) { + return Double.longBitsToDouble(Double.doubleToLongBits(value)); + } + return (double) Math.round(value) / 1.0; + } + return (double) Math.round(scaled) / mpv; + } + + // --------------------------------------------------------------------------- + // Fixture writing + // --------------------------------------------------------------------------- + + private static void writeFixture(Path directory, FixtureCase fixtureCase) throws Exception { + File file = directory.resolve(fixtureCase.fileName()).toFile(); + Files.deleteIfExists(file.toPath()); + if (fixtureCase.model == Model.TABLE) { + writeTableFixture(file, fixtureCase); + } else { + writeTreeFixture(file, fixtureCase); + } + assertTrue("fixture was not created: " + file, file.isFile() && file.length() > 0); + assertOnWireCodec(file, fixtureCase); + } + + // Prove that the declared encoding and compression are what the file + // actually carries. Reading the values back cannot tell a PLAIN column from + // an RLE column that happens to decode to the same values, nor whether a + // codec request was silently ignored in favor of a default. + private static void assertOnWireCodec(File file, FixtureCase fixtureCase) throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { + List devices = reader.getAllDevices(); + assertEquals( + fixtureCase.id + " device count on wire", fixtureCase.devices, devices.size()); + for (IDeviceID device : devices) { + List chunks = + reader.getChunkMetadataList(device, VALUE_COLUMN, false); + assertEquals( + fixtureCase.id + " chunk count on wire", fixtureCase.chunks, chunks.size()); + for (ChunkMetadata chunk : chunks) { + ChunkHeader header = reader.readChunkHeader(chunk.getOffsetOfChunkHeader()); + assertEquals( + fixtureCase.id + " encoding on wire", + fixtureCase.encoding, + header.getEncodingType()); + assertEquals( + fixtureCase.id + " compression on wire", + fixtureCase.compression, + header.getCompressionType()); + } + } + } + } + + private static void writeTableFixture(File file, FixtureCase fixtureCase) throws Exception { + TableSchema tableSchema = tableSchema(fixtureCase); + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int oldPageRows = config.getMaxNumberOfPointsInPage(); + int oldPageBytes = config.getPageSizeInByte(); + try { + applyPageConfiguration(config, fixtureCase); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.setGenerateTableSchema(true); + writer.registerTableSchema(tableSchema); + for (int device = 0; device < fixtureCase.devices; device++) { + for (int[] segment : segments(fixtureCase)) { + writer.writeTable(tablet(tableSchema, fixtureCase, device, segment[0], segment[1])); + writer.flush(); + } + } + } + } finally { + config.setMaxNumberOfPointsInPage(oldPageRows); + config.setPageSizeInByte(oldPageBytes); + } + } + + private static void writeTreeFixture(File file, FixtureCase fixtureCase) throws Exception { + TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); + int oldPageRows = config.getMaxNumberOfPointsInPage(); + int oldPageBytes = config.getPageSizeInByte(); + try { + applyPageConfiguration(config, fixtureCase); + try (TsFileWriter writer = new TsFileWriter(file)) { + for (int device = 0; device < fixtureCase.devices; device++) { + org.apache.tsfile.read.common.Path devicePath = + new org.apache.tsfile.read.common.Path(deviceId(device)); + writer.registerTimeseries(devicePath, valueSchema(fixtureCase)); + if (fixtureCase.nullPattern != NullPattern.NONE) { + writer.registerTimeseries( + devicePath, + new MeasurementSchema( + ANCHOR_COLUMN, + TSDataType.INT64, + TSEncoding.PLAIN, + fixtureCase.compression)); + } + } + for (int device = 0; device < fixtureCase.devices; device++) { + String deviceId = deviceId(device); + for (int[] segment : segments(fixtureCase)) { + for (int row = segment[0]; row < segment[1]; row++) { + TSRecord record = new TSRecord(deviceId, row); + int points = 0; + if (!isNull(fixtureCase, row)) { + record.addTuple(dataPoint(fixtureCase, row)); + points++; + } + if (fixtureCase.nullPattern != NullPattern.NONE) { + record.addTuple(new LongDataPoint(ANCHOR_COLUMN, row)); + points++; + } + if (points > 0) { + writer.writeRecord(record); + } + } + writer.flush(); + } + } + } + } finally { + config.setMaxNumberOfPointsInPage(oldPageRows); + config.setPageSizeInByte(oldPageBytes); + } + } + + private static void applyPageConfiguration(TSFileConfig config, FixtureCase fixtureCase) { + if (fixtureCase.pageRows > 0) { + config.setMaxNumberOfPointsInPage(fixtureCase.pageRows); + // Keep the byte-size trigger from splitting pages described by pageRows. + config.setPageSizeInByte(16 * 1024 * 1024); + } + } + + private static int[][] segments(FixtureCase fixtureCase) { + int chunks = Math.max(1, fixtureCase.chunks); + int[][] segments = new int[chunks][]; + int base = fixtureCase.rowCount / chunks; + int extra = fixtureCase.rowCount % chunks; + int start = 0; + for (int i = 0; i < chunks; i++) { + int length = base + (i < extra ? 1 : 0); + segments[i] = new int[] {start, start + length}; + start += length; + } + return segments; + } + + private static String deviceId(int device) { + return DEVICE_PREFIX + device; + } + + private static MeasurementSchema valueSchema(FixtureCase fixtureCase) { + Map props = + fixtureCase.profile == ValueProfile.TS2DIFF_MPN0 + ? Collections.singletonMap(Encoder.MAX_POINT_NUMBER, "0") + : null; + return new MeasurementSchema( + VALUE_COLUMN, + fixtureCase.dataType, + fixtureCase.encoding, + fixtureCase.compression, + props); + } + + private static TableSchema tableSchema(FixtureCase fixtureCase) { + List schemas = new ArrayList<>(); + List categories = new ArrayList<>(); + schemas.add( + new MeasurementSchema( + TAG_COLUMN, + TSDataType.STRING, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED)); + categories.add(ColumnCategory.TAG); + schemas.add(valueSchema(fixtureCase)); + categories.add(ColumnCategory.FIELD); + if (fixtureCase.nullPattern != NullPattern.NONE) { + schemas.add( + new MeasurementSchema( + ANCHOR_COLUMN, + TSDataType.INT64, + TSEncoding.PLAIN, + fixtureCase.compression)); + categories.add(ColumnCategory.FIELD); + } + return new TableSchema(TABLE_NAME, schemas, categories); + } + + private static Tablet tablet( + TableSchema tableSchema, FixtureCase fixtureCase, int device, int start, int end) { + Tablet tablet = + new Tablet( + tableSchema.getTableName(), + IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas()), + IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas()), + tableSchema.getColumnTypes(), + Math.max(1, end - start)); + int index = 0; + for (int row = start; row < end; row++, index++) { + tablet.addTimestamp(index, row); + tablet.addValue(TAG_COLUMN, index, deviceId(device)); + if (isNull(fixtureCase, row)) { + tablet.addValue(VALUE_COLUMN, index, null); + } else { + addTabletValue(tablet, fixtureCase, index, row); + } + if (fixtureCase.nullPattern != NullPattern.NONE) { + tablet.addValue(ANCHOR_COLUMN, index, (long) row); + } + } + return tablet; + } + + private static void addTabletValue( + Tablet tablet, FixtureCase fixtureCase, int index, int row) { + Object value = valueFor(fixtureCase, row); + switch (fixtureCase.dataType) { + case BOOLEAN: + tablet.addValue(index, VALUE_COLUMN, (Boolean) value); + break; + case INT32: + tablet.addValue(index, VALUE_COLUMN, (Integer) value); + break; + case DATE: + tablet.addValue(index, VALUE_COLUMN, (LocalDate) value); + break; + case INT64: + case TIMESTAMP: + tablet.addValue(index, VALUE_COLUMN, (Long) value); + break; + case FLOAT: + tablet.addValue(index, VALUE_COLUMN, (Float) value); + break; + case DOUBLE: + tablet.addValue(index, VALUE_COLUMN, (Double) value); + break; + case TEXT: + case STRING: + tablet.addValue(index, VALUE_COLUMN, (String) value); + break; + case BLOB: + tablet.addValue(index, VALUE_COLUMN, (byte[]) value); + break; + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } + + private static DataPoint dataPoint(FixtureCase fixtureCase, int row) { + Object value = valueFor(fixtureCase, row); + switch (fixtureCase.dataType) { + case BOOLEAN: + return new BooleanDataPoint(VALUE_COLUMN, (Boolean) value); + case INT32: + return new IntDataPoint(VALUE_COLUMN, (Integer) value); + case DATE: + return new DateDataPoint(VALUE_COLUMN, (LocalDate) value); + case INT64: + case TIMESTAMP: + return new LongDataPoint(VALUE_COLUMN, (Long) value); + case FLOAT: + return new FloatDataPoint(VALUE_COLUMN, (Float) value); + case DOUBLE: + return new DoubleDataPoint(VALUE_COLUMN, (Double) value); + case TEXT: + case STRING: + return new StringDataPoint( + VALUE_COLUMN, new Binary((String) value, StandardCharsets.UTF_8)); + case BLOB: + return new BlobDataPoint(VALUE_COLUMN, new Binary((byte[]) value)); + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } + + /** BLOB value for the tree write path, which has no built-in BLOB data point. */ + private static final class BlobDataPoint extends DataPoint { + private Binary value; + + BlobDataPoint(String measurementId, Binary value) { + super(TSDataType.BLOB, measurementId); + this.value = value; + } + + @Override + public void writeTo(long time, ChunkWriterImpl writer) { + writer.write(time, value); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public void setString(Binary value) { + this.value = value; + } + } + + // --------------------------------------------------------------------------- + // Fixture validation + // --------------------------------------------------------------------------- + + private static int nullPatternPageRows(FixtureCase fixtureCase) { + return fixtureCase.pageRows > 0 ? fixtureCase.pageRows : 64; + } + + private static void validateFixture(Path directory, FixtureCase fixtureCase) throws Exception { + File file = directory.resolve(fixtureCase.fileName()).toFile(); + assertTrue("missing fixture " + file, file.isFile()); + if (fixtureCase.model == Model.TABLE) { + validateTableFixture(file, fixtureCase); + } else { + validateTreeFixture(file, fixtureCase); + } + } + + private static List queryRanges(FixtureCase fixtureCase) { + List ranges = new ArrayList<>(); + ranges.add(new long[] {0, fixtureCase.rowCount - 1}); + int pageRows = fixtureCase.pageRows; + if (fixtureCase.nullPattern != NullPattern.NONE + && pageRows > 0 + && fixtureCase.rowCount > 2L * pageRows) { + ranges.add(new long[] {pageRows - 1L, Math.min(fixtureCase.rowCount - 1, 2L * pageRows + 1)}); + ranges.add(new long[] {pageRows, Math.min(fixtureCase.rowCount - 1L, 2L * pageRows - 1L)}); + } + return ranges; + } + + private static void validateTableFixture(File file, FixtureCase fixtureCase) throws Exception { + boolean nulls = fixtureCase.nullPattern != NullPattern.NONE; + for (long[] range : queryRanges(fixtureCase)) { + validateTableRange( + file, fixtureCase, (int) range[0], (int) range[1], false); + if (nulls) { + validateTableRange(file, fixtureCase, (int) range[0], (int) range[1], true); + } + } + } + + private static void validateTableRange( + File file, FixtureCase fixtureCase, int start, int end, boolean withAnchor) + throws Exception { + List columns = new ArrayList<>(Arrays.asList(TAG_COLUMN, VALUE_COLUMN)); + if (withAnchor) { + columns.add(ANCHOR_COLUMN); + } + try (ITsFileReader reader = new TsFileReaderBuilder().file(file).build(); + ResultSet resultSet = reader.query(TABLE_NAME, columns, start, end)) { + ResultSetMetadata metadata = resultSet.getMetadata(); + assertEquals("Time", metadata.getColumnName(1)); + assertEquals(TSDataType.INT64, metadata.getColumnType(1)); + assertEquals(TAG_COLUMN, metadata.getColumnName(2)); + assertEquals(TSDataType.STRING, metadata.getColumnType(2)); + assertEquals(VALUE_COLUMN, metadata.getColumnName(3)); + assertEquals(fixtureCase.dataType, metadata.getColumnType(3)); + if (withAnchor) { + assertEquals(ANCHOR_COLUMN, metadata.getColumnName(4)); + assertEquals(TSDataType.INT64, metadata.getColumnType(4)); + } + + Map> rowsByDevice = new LinkedHashMap<>(); + Map> valuesByDevice = new LinkedHashMap<>(); + while (resultSet.next()) { + long time = resultSet.getLong(1); + String device = resultSet.getString(2); + rowsByDevice.computeIfAbsent(device, key -> new ArrayList<>()).add(time); + boolean isNull = resultSet.isNull(3); + Object value = isNull ? null : readResultValue(fixtureCase, resultSet, 3); + Long anchor = null; + if (withAnchor) { + assertFalse("anchor null at " + fixtureCase.id + " t=" + time, resultSet.isNull(4)); + anchor = resultSet.getLong(4); + } + valuesByDevice + .computeIfAbsent(device, key -> new LinkedHashMap<>()) + .put(time, new Object[] {isNull, value, anchor}); + } + + assertEquals( + fixtureCase.id + " devices", fixtureCase.devices, rowsByDevice.size()); + for (int device = 0; device < fixtureCase.devices; device++) { + String deviceKey = deviceId(device); + List times = rowsByDevice.get(deviceKey); + assertNotNull("missing device " + deviceKey + " in " + fixtureCase.id, times); + assertEquals(fixtureCase.id + " rows for " + deviceKey, end - start + 1, times.size()); + Map values = valuesByDevice.get(deviceKey); + long expectedTime = start; + for (Long time : times) { + assertEquals(fixtureCase.id + " timestamp", expectedTime, time.longValue()); + Object[] entry = values.get(time); + assertNotNull(entry); + boolean expectedNull = isNull(fixtureCase, (int) expectedTime); + assertEquals( + fixtureCase.id + " null at t=" + expectedTime, expectedNull, entry[0]); + if (!expectedNull) { + assertValue( + fixtureCase, expectedValue(fixtureCase, (int) expectedTime), entry[1], expectedTime); + } + if (withAnchor) { + assertEquals( + fixtureCase.id + " anchor at t=" + expectedTime, + expectedTime, + ((Long) entry[2]).longValue()); + } + expectedTime++; + } + } + } + } + + private static void validateTreeFixture(File file, FixtureCase fixtureCase) throws Exception { + boolean nulls = fixtureCase.nullPattern != NullPattern.NONE; + for (long[] range : queryRanges(fixtureCase)) { + for (int device = 0; device < fixtureCase.devices; device++) { + validateTreeRange( + file, fixtureCase, device, (int) range[0], (int) range[1], false); + if (nulls) { + validateTreeRange( + file, fixtureCase, device, (int) range[0], (int) range[1], true); + } + } + } + } + + private static void validateTreeRange( + File file, + FixtureCase fixtureCase, + int device, + int start, + int end, + boolean withAnchor) + throws Exception { + List paths = new ArrayList<>(); + paths.add(new org.apache.tsfile.read.common.Path(deviceId(device), VALUE_COLUMN, true)); + if (withAnchor) { + paths.add(new org.apache.tsfile.read.common.Path(deviceId(device), ANCHOR_COLUMN, true)); + } + IExpression filter = + BinaryExpression.and( + new GlobalTimeExpression(TimeFilterApi.gtEq(start)), + new GlobalTimeExpression(TimeFilterApi.ltEq(end))); + QueryExpression expression = QueryExpression.create(paths, filter); + try (TsFileSequenceReader sequenceReader = new TsFileSequenceReader(file.getPath())) { + try (TsFileReader reader = new TsFileReader(sequenceReader)) { + QueryDataSet dataSet = reader.query(expression); + long expectedTime = start; + while (dataSet.hasNext()) { + RowRecord record = dataSet.next(); + if (!withAnchor) { + // Without the anchor, timestamps whose value is missing are not + // returned at all; skip to the next present value. + while (expectedTime <= end && isNull(fixtureCase, (int) expectedTime)) { + expectedTime++; + } + } + assertEquals( + fixtureCase.id + " tree timestamp for device " + device, + expectedTime, + record.getTimestamp()); + Field valueField = record.getFields().get(0); + boolean isNull = valueField == null || valueField.getDataType() == null; + assertEquals( + fixtureCase.id + " tree null at t=" + expectedTime, + isNull(fixtureCase, (int) expectedTime), + isNull); + if (!isNull) { + assertFieldValue( + fixtureCase, + expectedValue(fixtureCase, (int) expectedTime), + valueField, + expectedTime); + } + if (withAnchor) { + Field anchorField = record.getFields().get(1); + assertNotNull(anchorField); + assertEquals( + fixtureCase.id + " tree anchor at t=" + expectedTime, + expectedTime, + anchorField.getLongV()); + } + expectedTime++; + } + if (!withAnchor) { + while (expectedTime <= end && isNull(fixtureCase, (int) expectedTime)) { + expectedTime++; + } + } + assertEquals( + fixtureCase.id + " tree row count for device " + device, end + 1, expectedTime); + } + } + } + + private static Object readResultValue(FixtureCase fixtureCase, ResultSet resultSet, int index) { + switch (fixtureCase.dataType) { + case BOOLEAN: + return resultSet.getBoolean(index); + case INT32: + return resultSet.getInt(index); + case DATE: + return resultSet.getDate(index); + case INT64: + case TIMESTAMP: + return resultSet.getLong(index); + case FLOAT: + return resultSet.getFloat(index); + case DOUBLE: + return resultSet.getDouble(index); + case TEXT: + case STRING: + return resultSet.getString(index); + case BLOB: + return resultSet.getBinary(index); + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } + + private static void assertValue( + FixtureCase fixtureCase, Object expected, Object actual, long time) { + String message = fixtureCase.id + " value at t=" + time; + switch (fixtureCase.dataType) { + case BOOLEAN: + assertEquals(message, expected, actual); + break; + case INT32: + assertEquals(message, expected, actual); + break; + case DATE: + assertEquals(message, expected, actual); + break; + case INT64: + case TIMESTAMP: + assertEquals(message, expected, actual); + break; + case FLOAT: + assertEquals( + message, + Float.floatToIntBits((Float) expected), + Float.floatToIntBits((Float) actual)); + break; + case DOUBLE: + assertEquals( + message, + Double.doubleToLongBits((Double) expected), + Double.doubleToLongBits((Double) actual)); + break; + case TEXT: + case STRING: + assertEquals(message, expected, actual); + break; + case BLOB: + assertTrue(message, Arrays.equals((byte[]) expected, (byte[]) actual)); + break; + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } + + private static void assertFieldValue( + FixtureCase fixtureCase, Object expected, Field field, long time) { + String message = fixtureCase.id + " value at t=" + time; + assertEquals(message + " type", fixtureCase.dataType, field.getDataType()); + switch (fixtureCase.dataType) { + case BOOLEAN: + assertEquals(message, expected, field.getBoolV()); + break; + case INT32: + assertEquals(message, expected, field.getIntV()); + break; + case DATE: + assertEquals(message, expected, field.getDateV()); + break; + case INT64: + case TIMESTAMP: + assertEquals(message, expected, field.getLongV()); + break; + case FLOAT: + assertEquals( + message, + Float.floatToIntBits((Float) expected), + Float.floatToIntBits(field.getFloatV())); + break; + case DOUBLE: + assertEquals( + message, + Double.doubleToLongBits((Double) expected), + Double.doubleToLongBits(field.getDoubleV())); + break; + case TEXT: + case STRING: + assertEquals(message, expected, field.getBinaryV().getStringValue(StandardCharsets.UTF_8)); + break; + case BLOB: + assertTrue( + message, Arrays.equals((byte[]) expected, field.getBinaryV().getValues())); + break; + default: + throw new IllegalArgumentException("Unsupported data type " + fixtureCase.dataType); + } + } +} diff --git a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java b/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java deleted file mode 100644 index 5f55fd9563..0000000000 --- a/java/tsfile/src/test/java/org/apache/tsfile/compatibility/TableModelEncodingCompressionCompatibilityTest.java +++ /dev/null @@ -1,801 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tsfile.compatibility; - -import org.apache.tsfile.common.conf.TSFileConfig; -import org.apache.tsfile.common.conf.TSFileDescriptor; -import org.apache.tsfile.encoding.encoder.Encoder; -import org.apache.tsfile.enums.ColumnCategory; -import org.apache.tsfile.enums.TSDataType; -import org.apache.tsfile.file.header.PageHeader; -import org.apache.tsfile.file.metadata.ChunkMetadata; -import org.apache.tsfile.file.metadata.TableSchema; -import org.apache.tsfile.file.metadata.enums.CompressionType; -import org.apache.tsfile.file.metadata.enums.TSEncoding; -import org.apache.tsfile.read.TsFileSequenceReader; -import org.apache.tsfile.read.common.Chunk; -import org.apache.tsfile.read.query.dataset.ResultSet; -import org.apache.tsfile.read.query.dataset.ResultSetMetadata; -import org.apache.tsfile.read.v4.ITsFileReader; -import org.apache.tsfile.read.v4.TsFileReaderBuilder; -import org.apache.tsfile.write.TsFileWriter; -import org.apache.tsfile.write.record.Tablet; -import org.apache.tsfile.write.schema.IMeasurementSchema; -import org.apache.tsfile.write.schema.MeasurementSchema; - -import org.junit.Assume; -import org.junit.Test; - -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -public class TableModelEncodingCompressionCompatibilityTest { - - private static final String GENERATE_DIR_PROPERTY = "tsfile.compat.generate.dir"; - private static final String VALIDATE_DIR_PROPERTY = "tsfile.compat.validate.dir"; - private static final String MANIFEST_FILE = "manifest.csv"; - private static final String MANIFEST_HEADER = - "file,table,tagColumn,valueColumn,dataType,encoding,compression,rowCount,nullPattern"; - private static final String TABLE_NAME = "compat_table"; - private static final String TAG_COLUMN = "device"; - private static final String VALUE_COLUMN = "value"; - private static final String TAG_VALUE = "compat_device"; - // Crosses the 129-value TS_2DIFF block boundary (300 -> 129+129+42), so - // page-wide bitmaps must survive block transitions. Applied to every case. - private static final int ROW_COUNT = 300; - private static final int NULL_PAGE_ROWS = 64; - private static final String ANCHOR_COLUMN = "anchor"; - - // Keep this contract in sync with the C++ compatibility test. MIXED has - // empty pages at the start, in the middle (consecutive), and at the end; - // page 4 mixes null/non-null rows and pages 1/5 are entirely non-null. - private enum NullPattern { - NONE, - MIXED, - ALL - } - - private static boolean isNull(FixtureCase fixtureCase, int row) { - if (fixtureCase.nullPattern == NullPattern.ALL) { - return true; - } - if (fixtureCase.nullPattern == NullPattern.NONE) { - return false; - } - int page = row / NULL_PAGE_ROWS; - return page == 0 || page == 2 || page == 3 || page >= 6 || (page == 4 && row % 2 == 0); - } - - private static final int[] INT_VALUES = { - 0, - 1, - -1, - 7, - -8, - 1024, - -1024, - 123456, - -654321, - Integer.MAX_VALUE - 1024, - Integer.MIN_VALUE + 1024, - 42, - 42, - 0, - -8, - 2048 - }; - - private static final long[] LONG_VALUES = { - 0L, - 1L, - -1L, - 7L, - -8L, - 1_234_567_890L, - -987_654_321L, - 4_294_967_296L, - -4_294_967_296L, - 1_234_567_890_123L, - -1_234_567_890_123L, - Long.MAX_VALUE - 4096, - Long.MIN_VALUE + 4096, - 42L, - 42L, - 0L - }; - - private static final int[] FLOAT_BITS = { - 0x00000000, - 0x80000000, - 0x3f800000, - 0xbf800000, - 0x41280000, - 0xc0700000, - 0x44801000, - 0xc5002000, - 0x40490fdb, - 0xc02df854, - 0x3f800000, - 0x3f800000, - 0x00000000, - 0x80000000, - 0x42f6e979, - 0xc2f6e979 - }; - - private static final long[] DOUBLE_BITS = { - 0x0000000000000000L, - 0x8000000000000000L, - 0x3ff0000000000000L, - 0xbff0000000000000L, - 0x4029000000000000L, - 0xc00c000000000000L, - 0x4090008000000000L, - 0xc0a0010000000000L, - 0x400921fb54442d18L, - 0xc005bf0a8b145769L, - 0x3ff0000000000000L, - 0x3ff0000000000000L, - 0x0000000000000000L, - 0x8000000000000000L, - 0x405edd2f1a9fbe77L, - 0xc05edd2f1a9fbe77L - }; - - // FLOAT/DOUBLE + TS_2DIFF converts values to fixed-point with - // maxPointNumber (10^mpn). The expected value is computed by applying the - // same tri-state conversion the writer performs and then decoding it back, - // so expectations reflect the encoding rules rather than the raw input - // bits. Every chosen value has the same expected value whether the writer - // used maxPointNumber 0 (explicit max_point_number property) or 2 (the - // TSFileConfig.floatPrecision default shared with the C++ writer), so the - // validating reader never needs to know which writer produced the file. - // The values cover all page layouts the writers can produce: - // - plain integers (scaled form; at mpn = 0 every finite in-range value - // is scaled, so pages written at mpn = 0 contain no bitmap) - // - 1.5e9f / 1e18: the scaled product overflows while round(v) still - // fits -> scale-overflow form, single page-wide bitmap (reachable only - // at mpn > 0, i.e. the default writer configuration) - // - NaN and +/-Infinity (stored as raw IEEE bits -> two page-wide - // bitmaps; Java floatToIntBits canonicalizes NaN) - private static final int[] TS_2DIFF_FLOAT_BITS = { - 0x00000000, 0x3f800000, 0xbf800000, 0x461c4000, 0xc61c4000, 0x4eb2d05e, - 0x7f800000, 0xff800000, 0x7fc00000, 0x3f800000, 0x00000000, 0x4a742400, - 0xca742400, 0x4e6e6b28, 0x3f800000, 0x00000000 - }; - - private static final long[] TS_2DIFF_DOUBLE_BITS = { - 0x0000000000000000L, 0x3ff0000000000000L, 0xbff0000000000000L, - 0x40c3880000000000L, 0xc0c3880000000000L, 0x43abc16d674ec800L, - 0x7ff0000000000000L, 0xfff0000000000000L, 0x7ff8000000000000L, - 0x3ff0000000000000L, 0x0000000000000000L, 0x41f0000000000000L, - 0xc1cd6f3458800000L, 0x430c6bf526340000L, 0x3ff0000000000000L, - 0x0000000000000000L - }; - - // maxPointNumber used by the standard Java schema write path - // (TSEncodingBuilder.Ts2Diff initFromProps -> TSFileConfig.floatPrecision, - // current default 2); the C++ FloatTS2DIFFEncoder default matches. - private static final int TS_2DIFF_MAX_POINT_NUMBER = 2; - private static final double TS_2DIFF_MAX_POINT_VALUE = - TS_2DIFF_MAX_POINT_NUMBER <= 0 ? 1.0 : Math.pow(10, TS_2DIFF_MAX_POINT_NUMBER); - - private static final LocalDate[] DATE_VALUES = { - LocalDate.of(1970, 1, 1), - LocalDate.of(1999, 12, 31), - LocalDate.of(2000, 2, 29), - LocalDate.of(2024, 2, 29), - LocalDate.of(2038, 1, 19), - LocalDate.of(2050, 6, 15), - LocalDate.of(1969, 7, 20), - LocalDate.of(1980, 1, 6) - }; - - @Test - public void generateFixtures() throws Exception { - String outputDir = System.getProperty(GENERATE_DIR_PROPERTY); - Assume.assumeTrue( - GENERATE_DIR_PROPERTY + " is not set", outputDir != null && !outputDir.isEmpty()); - - Path directory = Paths.get(outputDir); - Files.createDirectories(directory); - - List cases = buildMatrix(); - for (FixtureCase fixtureCase : cases) { - writeFixture(directory, fixtureCase); - } - writeManifest(directory, cases); - } - - @Test - public void validateFixtures() throws Exception { - String inputDir = System.getProperty(VALIDATE_DIR_PROPERTY); - Assume.assumeTrue( - VALIDATE_DIR_PROPERTY + " is not set", inputDir != null && !inputDir.isEmpty()); - - Path directory = Paths.get(inputDir); - for (FixtureCase fixtureCase : readManifest(directory)) { - validateFixture(directory, fixtureCase); - } - } - - private static List buildMatrix() { - List cases = new ArrayList<>(); - List compressions = - Arrays.asList(CompressionType.UNCOMPRESSED, CompressionType.ZSTD, CompressionType.LZMA2); - for (CompressionType compression : compressions) { - for (TSDataType dataType : - Arrays.asList( - TSDataType.INT32, - TSDataType.DATE, - TSDataType.INT64, - TSDataType.TIMESTAMP, - TSDataType.FLOAT, - TSDataType.DOUBLE)) { - cases.add(new FixtureCase(dataType, TSEncoding.CHIMP, compression, ROW_COUNT)); - cases.add(new FixtureCase(dataType, TSEncoding.RLBE, compression, ROW_COUNT)); - if (dataType == TSDataType.FLOAT || dataType == TSDataType.DOUBLE) { - // The value set exercises scaled, scale-overflow, and raw-bit - // forms, including the page-wide bitmaps across the 129-value - // TS_2DIFF block boundary. - cases.add(new FixtureCase(dataType, TSEncoding.TS_2DIFF, compression, ROW_COUNT)); - // Explicit max_point_number=0 fixture: a canonical page starting - // with the 0x00 maxPointNumber byte must enter the Form 1 - // parsing path. - cases.add( - new FixtureCase( - FixtureCase.fileName(dataType, TSEncoding.TS_2DIFF, compression) + ".mpn0", - TABLE_NAME, - TAG_COLUMN, - VALUE_COLUMN, - dataType, - TSEncoding.TS_2DIFF, - compression, - ROW_COUNT)); - } - } - cases.add(new FixtureCase(TSDataType.DOUBLE, TSEncoding.CAMEL, compression, ROW_COUNT)); - } - for (CompressionType compression : - Arrays.asList( - CompressionType.UNCOMPRESSED, - CompressionType.LZ4, - CompressionType.ZSTD, - CompressionType.LZMA2)) { - for (TSDataType dataType : - Arrays.asList( - TSDataType.INT32, - TSDataType.DATE, - TSDataType.INT64, - TSDataType.TIMESTAMP, - TSDataType.FLOAT, - TSDataType.DOUBLE)) { - for (NullPattern pattern : Arrays.asList(NullPattern.MIXED, NullPattern.ALL)) { - cases.add(new FixtureCase(dataType, compression, NULL_PAGE_ROWS * 8, pattern)); - } - cases.add(new FixtureCase(dataType, compression, NULL_PAGE_ROWS, NullPattern.ALL)); - } - } - // Java-only fixtures: exercise the default BOOLEAN encoding without requiring - // the C++ writer to support RLE booleans as well. - for (CompressionType compression : - Arrays.asList( - CompressionType.UNCOMPRESSED, - CompressionType.LZ4, - CompressionType.ZSTD, - CompressionType.LZMA2)) { - for (NullPattern pattern : NullPattern.values()) { - cases.add( - new FixtureCase( - FixtureCase.fileName(TSDataType.BOOLEAN, TSEncoding.RLE, compression) - + ".default." - + pattern.name().toLowerCase(Locale.ROOT), - TABLE_NAME, - TAG_COLUMN, - VALUE_COLUMN, - TSDataType.BOOLEAN, - TSEncoding.RLE, - compression, - NULL_PAGE_ROWS * 8, - pattern)); - } - } - return cases; - } - - private static void writeFixture(Path directory, FixtureCase fixtureCase) throws Exception { - TableSchema tableSchema = tableSchema(fixtureCase); - File file = directory.resolve(fixtureCase.fileName).toFile(); - TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); - int oldPageRows = config.getMaxNumberOfPointsInPage(); - int oldPageBytes = config.getPageSizeInByte(); - try { - if (fixtureCase.nullPattern != NullPattern.NONE) { - config.setMaxNumberOfPointsInPage(NULL_PAGE_ROWS); - config.setPageSizeInByte(1024 * 1024); - } - try (TsFileWriter writer = new TsFileWriter(file)) { - writer.setGenerateTableSchema(true); - writer.registerTableSchema(tableSchema); - writer.writeTable(tablet(tableSchema, fixtureCase)); - } - } finally { - config.setMaxNumberOfPointsInPage(oldPageRows); - config.setPageSizeInByte(oldPageBytes); - } - if (fixtureCase.nullPattern == NullPattern.MIXED) { - assertEmptyPages(file, fixtureCase); - } - } - - // Prove that the regression fixture actually contains Java's one-byte empty - // page headers, rather than merely null values inside a non-empty page. - private static void assertEmptyPages(File file, FixtureCase fixtureCase) throws IOException { - try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getPath())) { - List chunks = - reader.getChunkMetadataList( - reader.getAllDevices().get(0), fixtureCase.valueColumn, false); - assertEquals(1, chunks.size()); - Chunk chunk = reader.readMemChunk(chunks.get(0)); - ByteBuffer data = chunk.getData(); - int page = 0; - while (data.hasRemaining()) { - int start = data.position(); - PageHeader header = PageHeader.deserializeFrom(data, fixtureCase.dataType); - boolean empty = page == 0 || page == 2 || page == 3 || page == 6 || page == 7; - assertEquals("empty page " + page, empty, header.getUncompressedSize() == 0); - if (empty) { - assertEquals(1, data.position() - start); - } - data.position(data.position() + header.getCompressedSize()); - page++; - } - assertEquals(8, page); - } - } - - private static void validateFixture(Path directory, FixtureCase fixtureCase) throws Exception { - validateQuery(directory, fixtureCase, false, 0, fixtureCase.rowCount - 1); - if (fixtureCase.nullPattern != NullPattern.NONE) { - validateQuery(directory, fixtureCase, true, 0, fixtureCase.rowCount - 1); - if (fixtureCase.rowCount > NULL_PAGE_ROWS) { - // Start inside/at page boundaries and query an entirely null interval. - validateQuery(directory, fixtureCase, false, 63, 321); - validateQuery(directory, fixtureCase, true, 128, 255); - validateQuery(directory, fixtureCase, true, 320, 383); - } - } - } - - private static void validateQuery( - Path directory, FixtureCase fixtureCase, boolean withAnchor, int start, int end) - throws Exception { - File file = directory.resolve(fixtureCase.fileName).toFile(); - List columns = - new ArrayList<>(Arrays.asList(fixtureCase.tagColumn, fixtureCase.valueColumn)); - if (withAnchor) { - columns.add(ANCHOR_COLUMN); - } - try (ITsFileReader reader = new TsFileReaderBuilder().file(file).build(); - ResultSet resultSet = reader.query(fixtureCase.tableName, columns, start, end)) { - ResultSetMetadata metadata = resultSet.getMetadata(); - assertEquals("Time", metadata.getColumnName(1)); - assertEquals(TSDataType.INT64, metadata.getColumnType(1)); - assertEquals(fixtureCase.tagColumn, metadata.getColumnName(2)); - assertEquals(TSDataType.STRING, metadata.getColumnType(2)); - assertEquals(fixtureCase.valueColumn, metadata.getColumnName(3)); - assertEquals(fixtureCase.dataType, metadata.getColumnType(3)); - int row = start; - while (resultSet.next()) { - assertTrue("unexpected row in " + fixtureCase.fileName, row <= end); - assertEquals("time at row " + row, row, resultSet.getLong(1)); - assertFalse("tag is null at row " + row, resultSet.isNull(2)); - assertEquals(TAG_VALUE, resultSet.getString(2)); - assertEquals( - fixtureCase.fileName + " null at row " + row, - isNull(fixtureCase, row), - resultSet.isNull(3)); - if (!isNull(fixtureCase, row)) { - assertValue(fixtureCase, row, resultSet); - } - if (withAnchor) { - assertFalse(resultSet.isNull(4)); - assertEquals(row, resultSet.getLong(4)); - } - row++; - } - assertEquals(fixtureCase.fileName + " row count", end + 1, row); - } - } - - private static TableSchema tableSchema(FixtureCase fixtureCase) { - Map props = null; - if (isTs2DiffFloatCase(fixtureCase) && fixtureCase.fileName.endsWith(".mpn0")) { - props = Collections.singletonMap(Encoder.MAX_POINT_NUMBER, "0"); - } - MeasurementSchema valueSchema; - if (fixtureCase.dataType == TSDataType.BOOLEAN) { - valueSchema = new MeasurementSchema(fixtureCase.valueColumn, TSDataType.BOOLEAN); - assertEquals("Java default BOOLEAN encoding", TSEncoding.RLE, valueSchema.getEncodingType()); - valueSchema.setCompressionType(fixtureCase.compression); - } else { - valueSchema = - new MeasurementSchema( - fixtureCase.valueColumn, - fixtureCase.dataType, - fixtureCase.encoding, - fixtureCase.compression, - props); - } - List schemas = - Arrays.asList( - new MeasurementSchema( - fixtureCase.tagColumn, - TSDataType.STRING, - TSEncoding.PLAIN, - CompressionType.UNCOMPRESSED), - valueSchema); - List categories = - new ArrayList<>(Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD)); - if (fixtureCase.nullPattern != NullPattern.NONE) { - schemas = new ArrayList<>(schemas); - schemas.add( - new MeasurementSchema( - ANCHOR_COLUMN, TSDataType.INT64, TSEncoding.PLAIN, fixtureCase.compression)); - categories.add(ColumnCategory.FIELD); - } - return new TableSchema(fixtureCase.tableName, schemas, categories); - } - - private static Tablet tablet(TableSchema tableSchema, FixtureCase fixtureCase) { - Tablet tablet = - new Tablet( - tableSchema.getTableName(), - IMeasurementSchema.getMeasurementNameList(tableSchema.getColumnSchemas()), - IMeasurementSchema.getDataTypeList(tableSchema.getColumnSchemas()), - tableSchema.getColumnTypes(), - fixtureCase.rowCount); - for (int row = 0; row < fixtureCase.rowCount; row++) { - tablet.addTimestamp(row, row); - tablet.addValue(TAG_COLUMN, row, TAG_VALUE); - if (isNull(fixtureCase, row)) { - tablet.addValue(VALUE_COLUMN, row, null); - } else { - addValue(tablet, fixtureCase, row); - } - if (fixtureCase.nullPattern != NullPattern.NONE) { - tablet.addValue(ANCHOR_COLUMN, row, (long) row); - } - } - return tablet; - } - - private static void addValue(Tablet tablet, FixtureCase fixtureCase, int row) { - TSDataType dataType = fixtureCase.dataType; - switch (dataType) { - case BOOLEAN: - tablet.addValue(row, VALUE_COLUMN, booleanValue(row)); - break; - case INT32: - tablet.addValue(row, VALUE_COLUMN, intValue(row)); - break; - case DATE: - tablet.addValue(row, VALUE_COLUMN, dateValue(row)); - break; - case INT64: - case TIMESTAMP: - tablet.addValue(row, VALUE_COLUMN, longValue(row)); - break; - case FLOAT: - tablet.addValue(row, VALUE_COLUMN, expectedFloatValue(fixtureCase, row)); - break; - case DOUBLE: - tablet.addValue(row, VALUE_COLUMN, expectedDoubleValue(fixtureCase, row)); - break; - default: - throw new IllegalArgumentException("Unsupported data type: " + dataType); - } - } - - private static void assertValue(FixtureCase fixtureCase, int row, ResultSet resultSet) { - switch (fixtureCase.dataType) { - case BOOLEAN: - assertEquals("BOOLEAN at row " + row, booleanValue(row), resultSet.getBoolean(3)); - break; - case INT32: - assertEquals("INT32 at row " + row, intValue(row), resultSet.getInt(3)); - break; - case DATE: - assertEquals("DATE at row " + row, dateValue(row), resultSet.getDate(3)); - break; - case INT64: - case TIMESTAMP: - assertEquals("INT64/TIMESTAMP at row " + row, longValue(row), resultSet.getLong(3)); - break; - case FLOAT: - assertEquals( - "FLOAT bits at row " + row, - Float.floatToIntBits(expectedFloatValue(fixtureCase, row)), - Float.floatToIntBits(resultSet.getFloat(3))); - break; - case DOUBLE: - assertEquals( - "DOUBLE bits at row " + row, - Double.doubleToLongBits(expectedDoubleValue(fixtureCase, row)), - Double.doubleToLongBits(resultSet.getDouble(3))); - break; - default: - throw new IllegalArgumentException("Unsupported data type: " + fixtureCase.dataType); - } - } - - // Alternate bits and long true/false runs to cover both RLE packing modes. - private static boolean booleanValue(int row) { - return row % 192 < 64 ? row % 2 == 0 : row % 192 < 128; - } - - private static int intValue(int row) { - return INT_VALUES[row % INT_VALUES.length]; - } - - private static long longValue(int row) { - return LONG_VALUES[row % LONG_VALUES.length]; - } - - private static float floatValue(int row) { - return Float.intBitsToFloat(FLOAT_BITS[row % FLOAT_BITS.length]); - } - - private static double doubleValue(int row) { - return Double.longBitsToDouble(DOUBLE_BITS[row % DOUBLE_BITS.length]); - } - - private static boolean isTs2DiffFloatCase(FixtureCase fixtureCase) { - return fixtureCase.encoding == TSEncoding.TS_2DIFF - && (fixtureCase.dataType == TSDataType.FLOAT || fixtureCase.dataType == TSDataType.DOUBLE); - } - - // maxPointValue of the writer configuration for this fixture: the - // "*.mpn0" fixtures carry an explicit max_point_number=0 property, all - // others use the shared default (see TS_2DIFF_MAX_POINT_NUMBER). - private static double ts2DiffMaxPointValue(FixtureCase fixtureCase) { - return fixtureCase.fileName.endsWith(".mpn0") ? 1.0 : TS_2DIFF_MAX_POINT_VALUE; - } - - // Mirror FloatEncoder/FloatDecoder at the writer's maxPointNumber: - // scaled -> round(v * mpv) / mpv, scale-overflow -> round(v) / 1, - // raw bits -> intBitsToFloat (NaN is canonicalized by floatToIntBits). - private static float expectedFloatValue(FixtureCase fixtureCase, int row) { - if (!isTs2DiffFloatCase(fixtureCase)) { - return floatValue(row); - } - float value = Float.intBitsToFloat(TS_2DIFF_FLOAT_BITS[row % TS_2DIFF_FLOAT_BITS.length]); - if (Float.isNaN(value)) { - return Float.intBitsToFloat(0x7fc00000); - } - double mpv = ts2DiffMaxPointValue(fixtureCase); - double scaled = (double) value * mpv; - if (scaled > Integer.MAX_VALUE || scaled < Integer.MIN_VALUE) { - // value itself stays in int range for the values used here - // (Infinity is caught below), so this is the scale-overflow form. - if (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE) { - return Float.intBitsToFloat(Float.floatToIntBits(value)); - } - return (float) ((double) Math.round(value) / 1.0); - } - return (float) ((double) Math.round(scaled) / mpv); - } - - private static double expectedDoubleValue(FixtureCase fixtureCase, int row) { - if (!isTs2DiffFloatCase(fixtureCase)) { - return doubleValue(row); - } - double value = Double.longBitsToDouble(TS_2DIFF_DOUBLE_BITS[row % TS_2DIFF_DOUBLE_BITS.length]); - if (Double.isNaN(value)) { - return Double.longBitsToDouble(0x7ff8000000000000L); - } - double mpv = ts2DiffMaxPointValue(fixtureCase); - double scaled = value * mpv; - if (scaled > Long.MAX_VALUE || scaled < Long.MIN_VALUE) { - if (value > Long.MAX_VALUE || value < Long.MIN_VALUE) { - return Double.longBitsToDouble(Double.doubleToLongBits(value)); - } - return (double) Math.round(value) / 1.0; - } - return (double) Math.round(scaled) / mpv; - } - - private static LocalDate dateValue(int row) { - return DATE_VALUES[row % DATE_VALUES.length]; - } - - private static void writeManifest(Path directory, List cases) throws IOException { - try (BufferedWriter writer = - Files.newBufferedWriter(directory.resolve(MANIFEST_FILE), StandardCharsets.UTF_8)) { - writer.write(MANIFEST_HEADER); - writer.newLine(); - for (FixtureCase fixtureCase : cases) { - writer.write(fixtureCase.toManifestLine()); - writer.newLine(); - } - } - } - - private static List readManifest(Path directory) throws IOException { - Path manifest = directory.resolve(MANIFEST_FILE); - List lines = Files.readAllLines(manifest, StandardCharsets.UTF_8); - if (lines.isEmpty()) { - return Collections.emptyList(); - } - assertEquals(MANIFEST_HEADER, lines.get(0)); - List cases = new ArrayList<>(); - for (int i = 1; i < lines.size(); i++) { - String line = lines.get(i).trim(); - if (line.isEmpty()) { - continue; - } - cases.add(FixtureCase.fromManifestLine(line)); - } - return cases; - } - - private static class FixtureCase { - private final String fileName; - private final String tableName; - private final String tagColumn; - private final String valueColumn; - private final TSDataType dataType; - private final TSEncoding encoding; - private final CompressionType compression; - private final int rowCount; - private final NullPattern nullPattern; - - private FixtureCase( - TSDataType dataType, TSEncoding encoding, CompressionType compression, int rowCount) { - this( - fileName(dataType, encoding, compression), - TABLE_NAME, - TAG_COLUMN, - VALUE_COLUMN, - dataType, - encoding, - compression, - rowCount); - } - - private FixtureCase( - String fileName, - String tableName, - String tagColumn, - String valueColumn, - TSDataType dataType, - TSEncoding encoding, - CompressionType compression, - int rowCount) { - this( - fileName, - tableName, - tagColumn, - valueColumn, - dataType, - encoding, - compression, - rowCount, - NullPattern.NONE); - } - - private FixtureCase( - TSDataType dataType, CompressionType compression, int rowCount, NullPattern nullPattern) { - this( - fileName(dataType, TSEncoding.PLAIN, compression) - + "." - + nullPattern.name().toLowerCase(Locale.ROOT) - + "." - + rowCount, - TABLE_NAME, - TAG_COLUMN, - VALUE_COLUMN, - dataType, - TSEncoding.PLAIN, - compression, - rowCount, - nullPattern); - } - - private FixtureCase( - String fileName, - String tableName, - String tagColumn, - String valueColumn, - TSDataType dataType, - TSEncoding encoding, - CompressionType compression, - int rowCount, - NullPattern nullPattern) { - this.fileName = fileName; - this.tableName = tableName; - this.tagColumn = tagColumn; - this.valueColumn = valueColumn; - this.dataType = dataType; - this.encoding = encoding; - this.compression = compression; - this.rowCount = rowCount; - this.nullPattern = nullPattern; - } - - private static FixtureCase fromManifestLine(String line) { - String[] parts = line.split(",", -1); - if (parts.length != 9) { - throw new IllegalArgumentException("Bad manifest line: " + line); - } - return new FixtureCase( - parts[0], - parts[1], - parts[2], - parts[3], - TSDataType.valueOf(parts[4]), - TSEncoding.valueOf(parts[5]), - CompressionType.valueOf(parts[6]), - Integer.parseInt(parts[7]), - NullPattern.valueOf(parts[8])); - } - - private String toManifestLine() { - return String.join( - ",", - fileName, - tableName, - tagColumn, - valueColumn, - dataType.name(), - encoding.name(), - compression.name(), - String.valueOf(rowCount), - nullPattern.name()); - } - - private static String fileName( - TSDataType dataType, TSEncoding encoding, CompressionType compression) { - return encoding.name().toLowerCase(Locale.ROOT) - + "_" - + dataType.name().toLowerCase(Locale.ROOT) - + "_" - + compression.name().toLowerCase(Locale.ROOT) - + ".tsfile"; - } - } -}