diff --git a/cpp/src/encoding/double_sprintz_decoder.h b/cpp/src/encoding/double_sprintz_decoder.h index 57f4b0b23..1d0ac5316 100644 --- a/cpp/src/encoding/double_sprintz_decoder.h +++ b/cpp/src/encoding/double_sprintz_decoder.h @@ -100,9 +100,8 @@ class DoubleSprintzDecoder : public SprintzDecoder { } bool has_remaining(const common::ByteStream& input) override { - uint32_t min_length = sizeof(uint32_t) + 1; return (is_block_read_ && current_count_ < decode_size_) || - input.remaining_size() >= min_length; + input.has_remaining(); } protected: @@ -126,6 +125,14 @@ class DoubleSprintzDecoder : public SprintzDecoder { return ret; } } + // Java's Gorilla decoder eagerly consumes a byte after a + // byte-aligned terminator. Match it before the next Sprintz block. + if (decoder.bits_left_ == 0) { + uint8_t padding; + uint32_t read_len; + if (RET_FAIL(input.read_buf(&padding, 1, read_len))) return ret; + } + } else { decode_size_ = block_size_ + 1; common::SerializationUtil::read_double(pre_value_, input); @@ -156,15 +163,17 @@ class DoubleSprintzDecoder : public SprintzDecoder { if (predict_scheme_ == "delta") { uint64_t prev_bits; std::memcpy(&prev_bits, ¤t_buffer_[0], sizeof(prev_bits)); - int64_t corrected0 = - convert_buffer_[0] + static_cast(prev_bits); + int64_t corrected0 = static_cast( + static_cast(convert_buffer_[0]) + prev_bits); convert_buffer_[0] = corrected0; double d0; std::memcpy(&d0, &corrected0, sizeof(corrected0)); current_buffer_[1] = d0; for (int i = 1; i < block_size_; ++i) { - convert_buffer_[i] += convert_buffer_[i - 1]; + convert_buffer_[i] = static_cast( + static_cast(convert_buffer_[i]) + + static_cast(convert_buffer_[i - 1])); int64_t bits = convert_buffer_[i]; double di; std::memcpy(&di, &bits, sizeof(bits)); @@ -177,7 +186,8 @@ class DoubleSprintzDecoder : public SprintzDecoder { std::memcpy(&prev_bits, ¤t_buffer_[0], sizeof(prev_bits)); int64_t p = fire_pred_.predict(prev_bits); int64_t e0 = convert_buffer_[0]; - int64_t corrected0 = p + e0; + int64_t corrected0 = static_cast( + static_cast(p) + static_cast(e0)); convert_buffer_[0] = corrected0; double d0; std::memcpy(&d0, &corrected0, sizeof(corrected0)); @@ -190,7 +200,8 @@ class DoubleSprintzDecoder : public SprintzDecoder { sizeof(prev_bits_i)); int64_t err = convert_buffer_[i]; int64_t pred = fire_pred_.predict(prev_bits_i); - int64_t corrected = pred + err; + int64_t corrected = static_cast( + static_cast(pred) + static_cast(err)); convert_buffer_[i] = corrected; double di; std::memcpy(&di, &corrected, sizeof(corrected)); diff --git a/cpp/src/encoding/double_sprintz_encoder.h b/cpp/src/encoding/double_sprintz_encoder.h index 08d498c61..e79f45b1e 100644 --- a/cpp/src/encoding/double_sprintz_encoder.h +++ b/cpp/src/encoding/double_sprintz_encoder.h @@ -79,13 +79,31 @@ class DoubleSprintzEncoder : public SprintzEncoder { values_.push_back(value); if (values_.size() == (size_t)block_size_ + 1) { + const std::vector raw_values(values_); + unsafe_residual_ = false; fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { convert_buffer_[i - 1] = predict(values_[i], values_[i - 1]); } - bit_pack(); - is_first_cached_ = false; - values_.clear(); + if (unsafe_residual_) { + // Reuse the existing raw fallback format; retain the ninth + // value as the start of the next block. + common::SerializationUtil:: + write_int_little_endian_padded_on_bit_width( + block_size_ | 0x80, byte_cache_, 1); + DoubleGorillaEncoder encoder; + for (int i = 0; i < block_size_; ++i) { + if (RET_FAIL(encoder.encode(raw_values[i], byte_cache_))) + return ret; + } + if (RET_FAIL(encoder.flush(byte_cache_))) return ret; + values_.assign(1, raw_values.back()); + is_first_cached_ = true; + } else { + bit_pack(); + is_first_cached_ = false; + values_.clear(); + } group_num_++; if (group_num_ == group_max_) { if (RET_FAIL(flush(out_stream))) return ret; @@ -151,19 +169,28 @@ class DoubleSprintzEncoder : public SprintzEncoder { int64_t prev_bits = common::double_to_long(prev); int64_t raw_pred = 0; if (predict_method_ == "delta") { - raw_pred = curr_bits - prev_bits; + raw_pred = static_cast(static_cast(curr_bits) - + static_cast(prev_bits)); } else if (predict_method_ == "fire") { int64_t pred = fire_pred_.predict(prev_bits); - int64_t err = curr_bits - pred; + int64_t err = static_cast( + static_cast(curr_bits) - static_cast(pred)); fire_pred_.train(prev_bits, curr_bits, err); raw_pred = err; } else { ASSERT(false); } + // The historical mapping is only reversible within this range. + const int64_t limit = int64_t(1) << 62; + if (raw_pred <= -limit || raw_pred >= limit) { + unsafe_residual_ = true; + return 0; + } return (raw_pred <= 0) ? -2 * raw_pred : 2 * raw_pred - 1; } private: + bool unsafe_residual_ = false; std::vector values_; std::vector convert_buffer_; std::shared_ptr packer_; diff --git a/cpp/src/encoding/fire.h b/cpp/src/encoding/fire.h index 0ca3d1680..e31ab41a3 100644 --- a/cpp/src/encoding/fire.h +++ b/cpp/src/encoding/fire.h @@ -64,16 +64,21 @@ class IntFire : public Fire { int predict(int value) override { int alpha = accumulator_ >> learn_shift_; - int diff = static_cast((static_cast(alpha) * delta_)) >> + int diff = static_cast(static_cast(alpha) * + static_cast(delta_)) >> bit_width_; - - return value + diff; + return static_cast(static_cast(value) + + static_cast(diff)); } void train(int pre, int val, int err) override { - int gradient = err > 0 ? -delta_ : delta_; - accumulator_ -= gradient; - delta_ = val - pre; + uint32_t gradient = err > 0 + ? uint32_t(0) - static_cast(delta_) + : static_cast(delta_); + accumulator_ = static_cast( + static_cast(accumulator_) - gradient); + delta_ = static_cast(static_cast(val) - + static_cast(pre)); } }; @@ -92,30 +97,23 @@ class LongFire : public Fire { int64_t predict(int64_t value) override { int64_t alpha = accumulator_ >> learn_shift_; - int64_t diff = safe_mul_shift(alpha, delta_, bit_width_); - return value + diff; + // Java wraps the product to 64 bits before the arithmetic shift. + int64_t diff = static_cast(static_cast(alpha) * + static_cast(delta_)) >> + bit_width_; + return static_cast(static_cast(value) + + static_cast(diff)); } void train(int64_t pre, int64_t val, int64_t err) override { - int64_t gradient = err > 0 ? -delta_ : delta_; - accumulator_ -= gradient; - delta_ = val - pre; - } - - private: - /** (alpha * delta_) >> shift without signed overflow; both args are - * int64_t. */ - static int64_t safe_mul_shift(int64_t alpha, int64_t delta, int shift) { -#if defined(__SIZEOF_INT128__) && __SIZEOF_INT128__ >= 16 - __int128 product = static_cast<__int128>(alpha) * delta; - return static_cast(product >> shift); -#else - /* Portable fallback: use double for product. Exact for |alpha|,|delta| - * < 2^53. */ - double prod = static_cast(alpha) * static_cast(delta); - double div = static_cast(1LL << shift); - return static_cast(prod / div); -#endif + uint64_t gradient = err > 0 + ? uint64_t(0) - static_cast(delta_) + : static_cast(delta_); + accumulator_ = + static_cast(static_cast(accumulator_) - + static_cast(gradient)); + delta_ = static_cast(static_cast(val) - + static_cast(pre)); } }; diff --git a/cpp/src/encoding/float_sprintz_decoder.h b/cpp/src/encoding/float_sprintz_decoder.h index e3842eaaf..46469df3d 100644 --- a/cpp/src/encoding/float_sprintz_decoder.h +++ b/cpp/src/encoding/float_sprintz_decoder.h @@ -86,9 +86,8 @@ class FloatSprintzDecoder : public SprintzDecoder { } bool has_remaining(const common::ByteStream& input) override { - size_t min_length = sizeof(uint32_t) + 1; return (is_block_read_ && current_count_ < decode_size_) || - input.remaining_size() >= min_length; + input.has_remaining(); } int read_float(float& ret_value, common::ByteStream& input) override { @@ -127,6 +126,14 @@ class FloatSprintzDecoder : public SprintzDecoder { return ret; } } + // Java's Gorilla decoder eagerly consumes a byte after a + // byte-aligned terminator. Match it before the next Sprintz block. + if (decoder.bits_left_ == 0) { + uint8_t padding; + uint32_t read_len; + if (RET_FAIL(input.read_buf(&padding, 1, read_len))) return ret; + } + } else { // packed block decode_size_ = block_size_ + 1; @@ -162,8 +169,8 @@ class FloatSprintzDecoder : public SprintzDecoder { std::memcpy(&prev_bits, ¤t_buffer_[0], sizeof(prev_bits)); // Java: convertBuffer[0] = convertBuffer[0] + // Float.floatToIntBits(preValue); - int32_t corrected0 = - convert_buffer_[0] + static_cast(prev_bits); + int32_t corrected0 = static_cast( + static_cast(convert_buffer_[0]) + prev_bits); convert_buffer_[0] = corrected0; // Java: currentBuffer[1] = Float.intBitsToFloat(convertBuffer[0]); float f0; @@ -172,7 +179,9 @@ class FloatSprintzDecoder : public SprintzDecoder { for (int i = 1; i < block_size_; ++i) { // Java: convertBuffer[i] += convertBuffer[i - 1]; - convert_buffer_[i] += convert_buffer_[i - 1]; + convert_buffer_[i] = static_cast( + static_cast(convert_buffer_[i]) + + static_cast(convert_buffer_[i - 1])); int32_t bits = convert_buffer_[i]; float fi; std::memcpy(&fi, &bits, sizeof(bits)); @@ -186,7 +195,8 @@ class FloatSprintzDecoder : public SprintzDecoder { // Java: int p = firePred.predict(Float.floatToIntBits(preValue)); int32_t p = fire_pred_.predict(prev_bits); int32_t e0 = convert_buffer_[0]; - int32_t corrected0 = p + e0; + int32_t corrected0 = static_cast( + static_cast(p) + static_cast(e0)); convert_buffer_[0] = corrected0; float f0; std::memcpy(&f0, &corrected0, sizeof(corrected0)); @@ -201,7 +211,8 @@ class FloatSprintzDecoder : public SprintzDecoder { sizeof(prev_bits_i)); int32_t err = convert_buffer_[i]; int32_t pred = fire_pred_.predict(prev_bits_i); - int32_t corrected = pred + err; + int32_t corrected = static_cast( + static_cast(pred) + static_cast(err)); convert_buffer_[i] = corrected; float fi; std::memcpy(&fi, &corrected, sizeof(corrected)); diff --git a/cpp/src/encoding/float_sprintz_encoder.h b/cpp/src/encoding/float_sprintz_encoder.h index 73b55a669..aacb399c5 100644 --- a/cpp/src/encoding/float_sprintz_encoder.h +++ b/cpp/src/encoding/float_sprintz_encoder.h @@ -76,13 +76,31 @@ class FloatSprintzEncoder : public SprintzEncoder { values_.push_back(value); if (values_.size() == (size_t)block_size_ + 1) { + const std::vector raw_values(values_); + unsafe_residual_ = false; fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { convert_buffer_[i - 1] = predict(values_[i], values_[i - 1]); } - bit_pack(); - is_first_cached_ = false; - values_.clear(); + if (unsafe_residual_) { + // Reuse the existing raw fallback format; retain the ninth + // value as the start of the next block. + common::SerializationUtil:: + write_int_little_endian_padded_on_bit_width( + block_size_ | 0x80, byte_cache_, 1); + FloatGorillaEncoder encoder; + for (int i = 0; i < block_size_; ++i) { + if (RET_FAIL(encoder.encode(raw_values[i], byte_cache_))) + return ret; + } + if (RET_FAIL(encoder.flush(byte_cache_))) return ret; + values_.assign(1, raw_values.back()); + is_first_cached_ = true; + } else { + bit_pack(); + is_first_cached_ = false; + values_.clear(); + } group_num_++; if (group_num_ == group_max_) { if (RET_FAIL(flush(out_stream))) return ret; @@ -149,20 +167,29 @@ class FloatSprintzEncoder : public SprintzEncoder { int32_t prev_bits = common::float_to_int(prev_value); int32_t raw_pred = 0; if (predict_method_ == "delta") { - raw_pred = curr_bits - prev_bits; + raw_pred = static_cast(static_cast(curr_bits) - + static_cast(prev_bits)); } else if (predict_method_ == "fire") { int32_t pred = fire_pred_.predict(prev_bits); - int32_t err = curr_bits - pred; + int32_t err = static_cast( + static_cast(curr_bits) - static_cast(pred)); fire_pred_.train(prev_bits, curr_bits, err); raw_pred = err; } else { // unsupported ASSERT(false); } + // The historical mapping is only reversible within this range. + const int32_t limit = int32_t(1) << 30; + if (raw_pred <= -limit || raw_pred >= limit) { + unsafe_residual_ = true; + return 0; + } return (raw_pred <= 0) ? -2 * raw_pred : 2 * raw_pred - 1; } private: + bool unsafe_residual_ = false; std::vector values_; std::vector convert_buffer_; std::shared_ptr packer_; diff --git a/cpp/src/encoding/int32_sprintz_decoder.h b/cpp/src/encoding/int32_sprintz_decoder.h index a7c92eede..4ee6738be 100644 --- a/cpp/src/encoding/int32_sprintz_decoder.h +++ b/cpp/src/encoding/int32_sprintz_decoder.h @@ -56,9 +56,8 @@ class Int32SprintzDecoder : public SprintzDecoder { } bool has_remaining(const common::ByteStream& in) override { - uint32_t min_len = sizeof(int32_t) + 1; - return (is_block_read_ && current_count_ < block_size_) || - in.remaining_size() >= min_len; + return (is_block_read_ && current_count_ < decode_size_) || + in.has_remaining(); } int read_boolean(bool& ret_value, common::ByteStream& in) override { @@ -103,9 +102,8 @@ class Int32SprintzDecoder : public SprintzDecoder { } bool has_next(common::ByteStream& input) { - uint32_t min_length = sizeof(int32_t) + 1; - return (is_block_read_ && current_count_ < block_size_) || - input.remaining_size() >= min_length; + return (is_block_read_ && current_count_ < decode_size_) || + input.has_remaining(); } protected: @@ -166,14 +164,17 @@ class Int32SprintzDecoder : public SprintzDecoder { if (predict_scheme_ == "delta") { for (size_t i = 1; i < current_buffer_.size(); ++i) { - current_buffer_[i] += current_buffer_[i - 1]; + current_buffer_[i] = static_cast( + static_cast(current_buffer_[i]) + + static_cast(current_buffer_[i - 1])); } } else if (predict_scheme_ == "fire") { fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { int32_t pred = fire_pred_.predict(current_buffer_[i - 1]); int32_t err = current_buffer_[i]; - current_buffer_[i] = pred + err; + current_buffer_[i] = static_cast( + static_cast(pred) + static_cast(err)); fire_pred_.train(current_buffer_[i - 1], current_buffer_[i], err); } diff --git a/cpp/src/encoding/int32_sprintz_encoder.h b/cpp/src/encoding/int32_sprintz_encoder.h index e92f25c3e..992c024b2 100644 --- a/cpp/src/encoding/int32_sprintz_encoder.h +++ b/cpp/src/encoding/int32_sprintz_encoder.h @@ -89,6 +89,8 @@ class Int32SprintzEncoder : public SprintzEncoder { if (values_.size() == (size_t)block_size_ + 1) { int32_t prev = values_[0]; + const std::vector raw_values(values_); + unsafe_residual_ = false; fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { int32_t temp = values_[i]; @@ -96,9 +98,25 @@ class Int32SprintzEncoder : public SprintzEncoder { prev = temp; } - bit_pack(); - is_first_cached_ = false; - values_.clear(); + if (unsafe_residual_) { + // Reuse the existing raw fallback format; retain the ninth + // value as the start of the next block. + common::SerializationUtil:: + write_int_little_endian_padded_on_bit_width( + block_size_ | 0x80, byte_cache_, 1); + Int32RleEncoder encoder; + for (int i = 0; i < block_size_; ++i) { + if (RET_FAIL(encoder.encode(raw_values[i], byte_cache_))) + return ret; + } + if (RET_FAIL(encoder.flush(byte_cache_))) return ret; + values_.assign(1, raw_values.back()); + is_first_cached_ = true; + } else { + bit_pack(); + is_first_cached_ = false; + values_.clear(); + } group_num_++; if (group_num_ == group_max_) { @@ -168,19 +186,30 @@ class Int32SprintzEncoder : public SprintzEncoder { ASSERT(false); } + // The historical mapping is only reversible within this range. + const int32_t limit = int32_t(1) << 30; + if (pred <= -limit || pred >= limit) { + unsafe_residual_ = true; + return 0; + } return (pred <= 0) ? -2 * pred : 2 * pred - 1; } - int32_t delta(int32_t value, int32_t prev) { return value - prev; } + int32_t delta(int32_t value, int32_t prev) { + return static_cast(static_cast(value) - + static_cast(prev)); + } int32_t fire(int32_t value, int32_t prev) { int32_t pred = fire_pred_.predict(prev); - int32_t err = value - pred; + int32_t err = static_cast(static_cast(value) - + static_cast(pred)); fire_pred_.train(prev, value, err); return err; } private: + bool unsafe_residual_ = false; std::vector values_; std::shared_ptr packer_; IntFire fire_pred_; diff --git a/cpp/src/encoding/int64_sprintz_decoder.h b/cpp/src/encoding/int64_sprintz_decoder.h index 7b0827688..20192e701 100644 --- a/cpp/src/encoding/int64_sprintz_decoder.h +++ b/cpp/src/encoding/int64_sprintz_decoder.h @@ -62,12 +62,12 @@ class Int64SprintzDecoder : public SprintzDecoder { } bool has_remaining(const common::ByteStream& in) override { - return (is_block_read_ && current_count_ < block_size_) || + return (is_block_read_ && current_count_ < decode_size_) || in.has_remaining(); } bool has_next(common::ByteStream& input) { - return (is_block_read_ && current_count_ < block_size_) || + return (is_block_read_ && current_count_ < decode_size_) || input.remaining_size() > 0; } @@ -167,14 +167,17 @@ class Int64SprintzDecoder : public SprintzDecoder { if (predict_scheme_ == "delta") { for (int i = 1; i <= block_size_; ++i) { - current_buffer_[i] += current_buffer_[i - 1]; + current_buffer_[i] = static_cast( + static_cast(current_buffer_[i]) + + static_cast(current_buffer_[i - 1])); } } else if (predict_scheme_ == "fire") { fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { int64_t pred = fire_pred_.predict(current_buffer_[i - 1]); int64_t err = current_buffer_[i]; - current_buffer_[i] = pred + err; + current_buffer_[i] = static_cast( + static_cast(pred) + static_cast(err)); fire_pred_.train(current_buffer_[i - 1], current_buffer_[i], err); } diff --git a/cpp/src/encoding/int64_sprintz_encoder.h b/cpp/src/encoding/int64_sprintz_encoder.h index 34151a6cb..6383ce0e1 100644 --- a/cpp/src/encoding/int64_sprintz_encoder.h +++ b/cpp/src/encoding/int64_sprintz_encoder.h @@ -78,6 +78,8 @@ class Int64SprintzEncoder : public SprintzEncoder { if (values_.size() == (size_t)block_size_ + 1) { int64_t prev = values_[0]; + const std::vector raw_values(values_); + unsafe_residual_ = false; fire_pred_.reset(); for (int i = 1; i <= block_size_; ++i) { int64_t temp = values_[i]; @@ -85,9 +87,25 @@ class Int64SprintzEncoder : public SprintzEncoder { prev = temp; } - bit_pack(); - is_first_cached_ = false; - values_.clear(); + if (unsafe_residual_) { + // Reuse the existing raw fallback format; retain the ninth + // value as the start of the next block. + common::SerializationUtil:: + write_int_little_endian_padded_on_bit_width( + block_size_ | 0x80, byte_cache_, 1); + Int64RleEncoder encoder; + for (int i = 0; i < block_size_; ++i) { + if (RET_FAIL(encoder.encode(raw_values[i], byte_cache_))) + return ret; + } + if (RET_FAIL(encoder.flush(byte_cache_))) return ret; + values_.assign(1, raw_values.back()); + is_first_cached_ = true; + } else { + bit_pack(); + is_first_cached_ = false; + values_.clear(); + } group_num_++; if (group_num_ == group_max_) { @@ -167,19 +185,30 @@ class Int64SprintzEncoder : public SprintzEncoder { ASSERT(false); } + // The historical mapping is only reversible within this range. + const int64_t limit = int64_t(1) << 62; + if (pred <= -limit || pred >= limit) { + unsafe_residual_ = true; + return 0; + } return (pred <= 0) ? -2 * pred : 2 * pred - 1; } - int64_t delta(int64_t value, int64_t prev) { return value - prev; } + int64_t delta(int64_t value, int64_t prev) { + return static_cast(static_cast(value) - + static_cast(prev)); + } int64_t fire(int64_t value, int64_t prev) { int64_t pred = fire_pred_.predict(prev); - int64_t err = value - pred; + int64_t err = static_cast(static_cast(value) - + static_cast(pred)); fire_pred_.train(prev, value, err); return err; } private: + bool unsafe_residual_ = false; std::vector values_; std::shared_ptr packer_; LongFire fire_pred_; diff --git a/cpp/test/encoding/sprintz_codec_test.cc b/cpp/test/encoding/sprintz_codec_test.cc index ec43ff406..5aef9199d 100644 --- a/cpp/test/encoding/sprintz_codec_test.cc +++ b/cpp/test/encoding/sprintz_codec_test.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include "common/allocator/byte_stream.h" #include "encoding/double_sprintz_decoder.h" @@ -79,6 +80,240 @@ class SprintzCodecTest : public ::testing::Test { } }; +// Produced by Java Sprintz encoders with FIRE: one packed block, then +// extreme transitions requiring the existing raw fallback representation. +const char* java_sprintz_golden[] = { + "0100ff882320030800000000800000007fffffffffffffff00000001c00000003fffffff00" + "00002a810401030100", + "010000000000000000ff8843400308000000000000000080000000000000007fffffffffff" + "ffffffffffffffffffff0000000000000001c0000000000000003fffffffffffffff000000" + "000000002a810401030100", + "1f00000000fdfffffe03fffffc03fffff807fffff007ffffe00fffffc01fffff803fffff88" + "00000000c00e0fffbfffffd000000017fbfffff5000000015fc00000d00000001602000000" + "81422a0000c4cf7a80", + "3f0000000000000000ffbffffffffffffe007ffffffffffffc007ffffffffffff800ffffff" + "fffffff000ffffffffffffe001ffffffffffffc003ffffffffffff8007ffffffffffff8800" + "00000000000000c00381ffff7ffffffffffffd00000000000000017ff7ffffffffffff5000" + "00000000000015ff8000000000000d00000000000000016004000000000000008140454000" + "00000000c23ffbd4", +}; + +std::string StreamHex(ByteStream& stream) { + static const char digits[] = "0123456789abcdef"; + std::string hex; + std::vector bytes; + uint8_t byte; + uint32_t read_len; + while (stream.has_remaining()) { + if (stream.read_buf(&byte, 1, read_len) != E_OK) break; + bytes.push_back(byte); + hex += digits[byte >> 4]; + hex += digits[byte & 15]; + } + stream.reset(); + stream.write_buf(bytes.data(), bytes.size()); + return hex; +} + +TEST_F(SprintzCodecTest, FireMatchesJavaWrappingArithmetic) { + IntFire int_fire(2); + LongFire long_fire(3); + const int32_t ints[] = {0, INT32_MIN, 123456789, INT32_MAX, + -42, 1 << 30, 23}; + const int64_t longs[] = {0, + int64_t(1) << 40, + (int64_t(1) << 40) + (int64_t(1) << 30), + int64_t(1) << 55, + INT64_MAX, + -42, + 23}; + const int32_t java_int_predictions[] = { + 0, INT32_MIN, 129748245, 2145006038, 6291414, 1077936126, 7340055}; + const int64_t java_long_predictions[] = {0, + 1099511627776LL, + 1100585369600LL, + 36026597995708416LL, + INT64_MAX, + -140737488355370LL, + 22}; + for (int i = 0; i < 7; ++i) { + ASSERT_EQ(int_fire.predict(ints[i]), java_int_predictions[i]); + ASSERT_EQ(long_fire.predict(longs[i]), java_long_predictions[i]); + int_fire.train(ints[i], ints[(i + 1) % 7], 1); + long_fire.train(longs[i], longs[(i + 1) % 7], 1); + } +} + +TEST_F(SprintzCodecTest, Int32FullBlocksAndExtremeTransitions) { + const std::vector edge_values = { + 0, INT32_MIN, INT32_MAX, -1, 1, INT32_MIN / 2, INT32_MAX / 2, 42, 0}; + for (const std::string method : {"fire", "delta"}) { + for (int count : {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + SCOPED_TRACE(method + ":" + std::to_string(count)); + Int32SprintzEncoder encoder; + encoder.set_predict_method(method); + for (int round = 0; round < 2; ++round) { + ByteStream stream(128, MOD_ENCODER_OBJ); + for (int i = 0; i < count; ++i) { + // Also exercise a safe packed block before raw fallbacks. + int32_t value = i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(encoder.encode(value, stream), E_OK); + } + ASSERT_EQ(encoder.flush(stream), E_OK); + if (method == "fire" && count == 18) { + ASSERT_EQ(StreamHex(stream), java_sprintz_golden[0]); + } + Int32SprintzDecoder decoder; + decoder.set_predict_method(method); + for (int i = 0; i < count; ++i) { + SCOPED_TRACE(i); + ASSERT_TRUE(decoder.has_remaining(stream)); + int32_t actual; + ASSERT_EQ(decoder.read_int32(actual, stream), E_OK); + int32_t expected = + i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(actual, expected); + } + ASSERT_FALSE(decoder.has_remaining(stream)); + } + } + } +} + +TEST_F(SprintzCodecTest, Int64FullBlocksAndExtremeTransitions) { + const std::vector edge_values = { + 0, INT64_MIN, INT64_MAX, -1, 1, INT64_MIN / 2, INT64_MAX / 2, 42, 0}; + for (const std::string method : {"fire", "delta"}) { + for (int count : {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + SCOPED_TRACE(method + ":" + std::to_string(count)); + Int64SprintzEncoder encoder; + encoder.set_predict_method(method); + for (int round = 0; round < 2; ++round) { + ByteStream stream(128, MOD_ENCODER_OBJ); + for (int i = 0; i < count; ++i) { + // Also exercise a safe packed block before raw fallbacks. + int64_t value = i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(encoder.encode(value, stream), E_OK); + } + ASSERT_EQ(encoder.flush(stream), E_OK); + if (method == "fire" && count == 18) { + ASSERT_EQ(StreamHex(stream), java_sprintz_golden[1]); + } + Int64SprintzDecoder decoder; + decoder.set_predict_method(method); + for (int i = 0; i < count; ++i) { + SCOPED_TRACE(i); + ASSERT_TRUE(decoder.has_remaining(stream)); + int64_t actual; + ASSERT_EQ(decoder.read_int64(actual, stream), E_OK); + int64_t expected = + i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(actual, expected); + } + ASSERT_FALSE(decoder.has_remaining(stream)); + } + } + } +} + +TEST_F(SprintzCodecTest, FloatFullBlocksAndExtremeTransitions) { + const std::vector edge_values = { + 0.0f, + -0.0f, + FLT_MAX, + -FLT_MAX, + std::numeric_limits::denorm_min(), + -std::numeric_limits::denorm_min(), + 1.0f, + -1.0f, + 42.5f}; + for (const std::string method : {"fire", "delta"}) { + for (int count : {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + SCOPED_TRACE(method + ":" + std::to_string(count)); + FloatSprintzEncoder encoder; + encoder.set_predict_method(method); + for (int round = 0; round < 2; ++round) { + ByteStream stream(128, MOD_ENCODER_OBJ); + for (int i = 0; i < count; ++i) { + // Also exercise a safe packed block before raw fallbacks. + float value = i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(encoder.encode(value, stream), E_OK); + } + ASSERT_EQ(encoder.flush(stream), E_OK); + if (method == "fire" && count == 18) { + ASSERT_EQ(StreamHex(stream), java_sprintz_golden[2]); + } + FloatSprintzDecoder decoder; + decoder.set_predict_method(method); + for (int i = 0; i < count; ++i) { + SCOPED_TRACE(i); + ASSERT_TRUE(decoder.has_remaining(stream)); + float actual; + ASSERT_EQ(decoder.read_float(actual, stream), E_OK); + float expected = i < 9 + ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(common::float_to_int(actual), + common::float_to_int(expected)); + } + ASSERT_FALSE(decoder.has_remaining(stream)); + } + } + } +} + +TEST_F(SprintzCodecTest, DoubleFullBlocksAndExtremeTransitions) { + const std::vector edge_values = { + 0.0, + -0.0, + DBL_MAX, + -DBL_MAX, + std::numeric_limits::denorm_min(), + -std::numeric_limits::denorm_min(), + 1.0, + -1.0, + 42.5}; + for (const std::string method : {"fire", "delta"}) { + for (int count : {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + SCOPED_TRACE(method + ":" + std::to_string(count)); + DoubleSprintzEncoder encoder; + encoder.set_predict_method(method); + for (int round = 0; round < 2; ++round) { + ByteStream stream(128, MOD_ENCODER_OBJ); + for (int i = 0; i < count; ++i) { + // Also exercise a safe packed block before raw fallbacks. + double value = i < 9 ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(encoder.encode(value, stream), E_OK); + } + ASSERT_EQ(encoder.flush(stream), E_OK); + if (method == "fire" && count == 18) { + ASSERT_EQ(StreamHex(stream), java_sprintz_golden[3]); + } + DoubleSprintzDecoder decoder; + decoder.set_predict_method(method); + for (int i = 0; i < count; ++i) { + SCOPED_TRACE(i); + ASSERT_TRUE(decoder.has_remaining(stream)); + double actual; + ASSERT_EQ(decoder.read_double(actual, stream), E_OK); + double expected = i < 9 + ? static_cast(i) + : edge_values[i % edge_values.size()]; + ASSERT_EQ(common::double_to_long(actual), + common::double_to_long(expected)); + } + ASSERT_FALSE(decoder.has_remaining(stream)); + } + } + } +} + TEST_F(SprintzCodecTest, Int32SingleValue) { Int32SprintzEncoder encoder; ByteStream stream(128, MOD_ENCODER_OBJ); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleSprintzDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleSprintzDecoder.java index c6c7327a5..eeb1a21bf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleSprintzDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/DoubleSprintzDecoder.java @@ -50,7 +50,7 @@ public DoubleSprintzDecoder() { @Override public boolean hasNext(ByteBuffer buffer) throws IOException { // int minLenth = Long.BYTES + 1; - return (isBlockReaded && currentCount < Block_size) || buffer.remaining() > 0; + return (isBlockReaded && currentCount < decodeSize) || buffer.hasRemaining(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatSprintzDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatSprintzDecoder.java index 2ac1e41e4..c1562acae 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatSprintzDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/FloatSprintzDecoder.java @@ -50,8 +50,7 @@ public FloatSprintzDecoder() { @Override public boolean hasNext(ByteBuffer buffer) throws IOException { - int minLenth = Float.BYTES + 1; - return (isBlockReaded && currentCount < Block_size) || buffer.remaining() >= minLenth; + return (isBlockReaded && currentCount < decodeSize) || buffer.hasRemaining(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntSprintzDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntSprintzDecoder.java index a647935cf..497f5a007 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntSprintzDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/IntSprintzDecoder.java @@ -48,8 +48,7 @@ public IntSprintzDecoder() { @Override public boolean hasNext(ByteBuffer buffer) throws IOException { - int minLenth = Integer.BYTES + 1; - return (isBlockReaded && currentCount < Block_size) || buffer.remaining() >= minLenth; + return (isBlockReaded && currentCount < decodeSize) || buffer.hasRemaining(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java index 5780a38fe..f89c15bc7 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/decoder/LongSprintzDecoder.java @@ -48,7 +48,7 @@ public LongSprintzDecoder() { @Override public boolean hasNext(ByteBuffer buffer) throws IOException { // int minLenth = Long.BYTES + 1; - return (isBlockReaded && currentCount < Block_size) || buffer.remaining() > 0; + return (isBlockReaded && currentCount < decodeSize) || buffer.hasRemaining(); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleSprintzEncoder.java index 7775f72a5..cd7edcba2 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DoubleSprintzEncoder.java @@ -44,6 +44,8 @@ public class DoubleSprintzEncoder extends SprintzEncoder { // convert to Long Buffer long[] convertBuffer; + private boolean unsafeResidual; + public DoubleSprintzEncoder() { super(); values = new Vector<>(); @@ -77,10 +79,16 @@ protected long predict(Double value, Double preVlaue) throws TsFileEncodingExcep throw new TsFileEncodingException( Messages.get("error.encoding.sprintz_encoder_unsupported_predict_method")); } + // Larger residuals overflow the historical signed mapping. + long limit = 1L << 62; + if (pred <= -limit || pred >= limit) { + unsafeResidual = true; + return 0L; + } if (pred <= 0) { pred = -2 * pred; } else { - pred = 2 * pred - 1; // TODO:overflow + pred = 2 * pred - 1; } return pred; } @@ -144,13 +152,29 @@ public void encode(double value, ByteArrayOutputStream out) { } if (values.size() == Block_size + 1) { try { + Vector rawValues = new Vector<>(values); + unsafeResidual = false; firePred.reset(); for (int i = 1; i <= Block_size; i++) { convertBuffer[i - 1] = predict(values.get(i), values.get(i - 1)); } - bitPack(); - isFirstCached = false; - values.clear(); + if (unsafeResidual) { + // Use the existing raw fallback format and retain the ninth value. + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth( + Block_size | 0x80, byteCache, 1); + DoublePrecisionEncoderV2 encoder = new DoublePrecisionEncoderV2(); + for (int i = 0; i < Block_size; i++) { + encoder.encode(rawValues.get(i), byteCache); + } + encoder.flush(byteCache); + values.clear(); + values.add(rawValues.lastElement()); + isFirstCached = true; + } else { + bitPack(); + isFirstCached = false; + values.clear(); + } groupNum++; if (groupNum == groupMax) { flush(out); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatSprintzEncoder.java index 3259c1ff6..6b988520f 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/FloatSprintzEncoder.java @@ -44,6 +44,8 @@ public class FloatSprintzEncoder extends SprintzEncoder { /** convert to integer Buffer * */ int[] convertBuffer; + private boolean unsafeResidual; + public FloatSprintzEncoder() { super(); values = new Vector(); @@ -76,8 +78,14 @@ protected int predict(Float value, Float preVlaue) throws TsFileEncodingExceptio throw new TsFileEncodingException( Messages.get("error.encoding.sprintz_encoder_unsupported_predict_method")); } + // Larger residuals overflow the historical signed mapping. + int limit = 1 << 30; + if (pred <= -limit || pred >= limit) { + unsafeResidual = true; + return 0; + } if (pred <= 0) pred = -2 * pred; - else pred = 2 * pred - 1; // TODO:overflow + else pred = 2 * pred - 1; return pred; } @@ -138,13 +146,29 @@ public void encode(float value, ByteArrayOutputStream out) { } if (values.size() == Block_size + 1) { try { + Vector rawValues = new Vector<>(values); + unsafeResidual = false; firePred.reset(); for (int i = 1; i <= Block_size; i++) { convertBuffer[i - 1] = predict(values.get(i), values.get(i - 1)); } - bitPack(); - isFirstCached = false; - values.clear(); + if (unsafeResidual) { + // Use the existing raw fallback format and retain the ninth value. + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth( + Block_size | 0x80, byteCache, 1); + SinglePrecisionEncoderV2 encoder = new SinglePrecisionEncoderV2(); + for (int i = 0; i < Block_size; i++) { + encoder.encode(rawValues.get(i), byteCache); + } + encoder.flush(byteCache); + values.clear(); + values.add(rawValues.lastElement()); + isFirstCached = true; + } else { + bitPack(); + isFirstCached = false; + values.clear(); + } groupNum++; if (groupNum == groupMax) { flush(out); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntSprintzEncoder.java index 97430c19d..e96b84640 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntSprintzEncoder.java @@ -40,6 +40,8 @@ public class IntSprintzEncoder extends SprintzEncoder { // we save all value in a list and calculate its bitwidth. protected Vector values; + private boolean unsafeResidual; + public IntSprintzEncoder() { super(); values = new Vector<>(); @@ -72,10 +74,16 @@ protected Integer predict(Integer value, Integer preVlaue) throws TsFileEncoding throw new TsFileEncodingException( Messages.get("error.encoding.sprintz_encoder_unsupported_predict_method")); } + // Larger residuals overflow the historical signed mapping. + int limit = 1 << 30; + if (pred <= -limit || pred >= limit) { + unsafeResidual = true; + return 0; + } if (pred <= 0) { pred = -2 * pred; } else { - pred = 2 * pred - 1; // TODO:overflow + pred = 2 * pred - 1; } return pred; } @@ -138,15 +146,31 @@ public void encode(int value, ByteArrayOutputStream out) { if (values.size() == Block_size + 1) { try { int pre = values.get(0); + Vector rawValues = new Vector<>(values); + unsafeResidual = false; firePred.reset(); for (int i = 1; i <= Block_size; i++) { int tmp = values.get(i); values.set(i, predict(values.get(i), pre)); pre = tmp; } - bitPack(); - isFirstCached = false; - values.clear(); + if (unsafeResidual) { + // Use the existing raw fallback format and retain the ninth value. + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth( + Block_size | 0x80, byteCache, 1); + IntRleEncoder encoder = new IntRleEncoder(); + for (int i = 0; i < Block_size; i++) { + encoder.encode(rawValues.get(i), byteCache); + } + encoder.flush(byteCache); + values.clear(); + values.add(rawValues.lastElement()); + isFirstCached = true; + } else { + bitPack(); + isFirstCached = false; + values.clear(); + } groupNum++; if (groupNum == groupMax) { flush(out); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java index 2a1b91b4c..60f2167fe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongSprintzEncoder.java @@ -41,6 +41,8 @@ public class LongSprintzEncoder extends SprintzEncoder { // we save all value in a list and calculate its bitwidth. protected Vector values; + private boolean unsafeResidual; + public LongSprintzEncoder() { super(); values = new Vector<>(); @@ -73,10 +75,16 @@ protected Long predict(Long value, Long preVlaue) throws TsFileEncodingException throw new TsFileEncodingException( Messages.get("error.encoding.sprintz_encoder_unsupported_predict_method")); } + // Larger residuals overflow the historical signed mapping. + long limit = 1L << 62; + if (pred <= -limit || pred >= limit) { + unsafeResidual = true; + return 0L; + } if (pred <= 0) { pred = -2 * pred; } else { - pred = 2 * pred - 1; // TODO:overflow + pred = 2 * pred - 1; } return pred; } @@ -139,15 +147,31 @@ public void encode(long value, ByteArrayOutputStream out) { if (values.size() == Block_size + 1) { try { long pre = values.get(0); + Vector rawValues = new Vector<>(values); + unsafeResidual = false; firePred.reset(); for (int i = 1; i <= Block_size; i++) { long tmp = values.get(i); values.set(i, predict(values.get(i), pre)); pre = tmp; } - bitPack(); - isFirstCached = false; - values.clear(); + if (unsafeResidual) { + // Use the existing raw fallback format and retain the ninth value. + ReadWriteForEncodingUtils.writeIntLittleEndianPaddedOnBitWidth( + Block_size | 0x80, byteCache, 1); + LongRleEncoder encoder = new LongRleEncoder(); + for (int i = 0; i < Block_size; i++) { + encoder.encode(rawValues.get(i), byteCache); + } + encoder.flush(byteCache); + values.clear(); + values.add(rawValues.lastElement()); + isFirstCached = true; + } else { + bitPack(); + isFirstCached = false; + values.clear(); + } groupNum++; if (groupNum == groupMax) { flush(out); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/SprintzDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/SprintzDecoderTest.java index 34f2b9b79..548d71a22 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/SprintzDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/SprintzDecoderTest.java @@ -35,6 +35,8 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public class SprintzDecoderTest { @@ -49,6 +51,167 @@ public class SprintzDecoderTest { private static final List iterations = new ArrayList<>(); + // Byte-for-byte identical to the C++ encoder's output for the regression sequence. + private static final String[] CPP_SPRINTZ_GOLDEN = { + "0100ff882320030800000000800000007fffffffffffffff00000001c00000003fffffff0000002a810401030100", + "010000000000000000ff8843400308000000000000000080000000000000007fffffffffffffffffffffffffffffff0000000000000001c0000000000000003fffffffffffffff000000000000002a810401030100", + "1f00000000fdfffffe03fffffc03fffff807fffff007ffffe00fffffc01fffff803fffff8800000000c00e0fffbfffffd000000017fbfffff5000000015fc00000d0000000160200000081422a0000c4cf7a80", + "3f0000000000000000ffbffffffffffffe007ffffffffffffc007ffffffffffff800fffffffffffff000ffffffffffffe001ffffffffffffc003ffffffffffff8007ffffffffffff880000000000000000c00381ffff7ffffffffffffd00000000000000017ff7ffffffffffff500000000000000015ff8000000000000d0000000000000001600400000000000000814045400000000000c23ffbd4", + }; + + private static String toHex(byte[] bytes) { + StringBuilder hex = new StringBuilder(); + for (byte value : bytes) { + hex.append(Character.forDigit((value >>> 4) & 15, 16)); + hex.append(Character.forDigit(value & 15, 16)); + } + return hex.toString(); + } + + @Test + public void testIntFullBlocksAndExtremeTransitions() throws IOException { + String method = "fire"; + int[] edgeValues = { + 0, + Integer.MIN_VALUE, + Integer.MAX_VALUE, + -1, + 1, + Integer.MIN_VALUE / 2, + Integer.MAX_VALUE / 2, + 42, + 0 + }; + for (int count : new int[] {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + IntSprintzEncoder encoder = new IntSprintzEncoder(); + for (int round = 0; round < 2; round++) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int i = 0; i < count; i++) { + encoder.encode(i < 9 ? (int) i : edgeValues[i % edgeValues.length], out); + } + encoder.flush(out); + if (count == 18) { + assertEquals(CPP_SPRINTZ_GOLDEN[0], toHex(out.toByteArray())); + } + ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray()); + IntSprintzDecoder decoder = new IntSprintzDecoder(); + for (int i = 0; i < count; i++) { + assertTrue(method + ":" + count + ":" + i, decoder.hasNext(buffer)); + int actual = decoder.readInt(buffer); + int expected = i < 9 ? (int) i : edgeValues[i % edgeValues.length]; + assertEquals(expected, actual); + } + assertFalse(decoder.hasNext(buffer)); + } + } + } + + @Test + public void testLongFullBlocksAndExtremeTransitions() throws IOException { + String method = "fire"; + long[] edgeValues = { + 0, Long.MIN_VALUE, Long.MAX_VALUE, -1, 1, Long.MIN_VALUE / 2, Long.MAX_VALUE / 2, 42, 0 + }; + for (int count : new int[] {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + LongSprintzEncoder encoder = new LongSprintzEncoder(); + for (int round = 0; round < 2; round++) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int i = 0; i < count; i++) { + encoder.encode(i < 9 ? (long) i : edgeValues[i % edgeValues.length], out); + } + encoder.flush(out); + if (count == 18) { + assertEquals(CPP_SPRINTZ_GOLDEN[1], toHex(out.toByteArray())); + } + ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray()); + LongSprintzDecoder decoder = new LongSprintzDecoder(); + for (int i = 0; i < count; i++) { + assertTrue(method + ":" + count + ":" + i, decoder.hasNext(buffer)); + long actual = decoder.readLong(buffer); + long expected = i < 9 ? (long) i : edgeValues[i % edgeValues.length]; + assertEquals(expected, actual); + } + assertFalse(decoder.hasNext(buffer)); + } + } + } + + @Test + public void testFloatFullBlocksAndExtremeTransitions() throws IOException { + String method = "fire"; + float[] edgeValues = { + 0.0f, + -0.0f, + Float.MAX_VALUE, + -Float.MAX_VALUE, + Float.MIN_VALUE, + -Float.MIN_VALUE, + 1, + -1, + 42.5f + }; + for (int count : new int[] {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + FloatSprintzEncoder encoder = new FloatSprintzEncoder(); + for (int round = 0; round < 2; round++) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int i = 0; i < count; i++) { + encoder.encode(i < 9 ? (float) i : edgeValues[i % edgeValues.length], out); + } + encoder.flush(out); + if (count == 18) { + assertEquals(CPP_SPRINTZ_GOLDEN[2], toHex(out.toByteArray())); + } + ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray()); + FloatSprintzDecoder decoder = new FloatSprintzDecoder(); + for (int i = 0; i < count; i++) { + assertTrue(method + ":" + count + ":" + i, decoder.hasNext(buffer)); + float actual = decoder.readFloat(buffer); + float expected = i < 9 ? (float) i : edgeValues[i % edgeValues.length]; + assertEquals(Float.floatToIntBits(expected), Float.floatToIntBits(actual)); + } + assertFalse(decoder.hasNext(buffer)); + } + } + } + + @Test + public void testDoubleFullBlocksAndExtremeTransitions() throws IOException { + String method = "fire"; + double[] edgeValues = { + 0.0, + -0.0, + Double.MAX_VALUE, + -Double.MAX_VALUE, + Double.MIN_VALUE, + -Double.MIN_VALUE, + 1, + -1, + 42.5 + }; + for (int count : new int[] {1, 8, 9, 10, 16, 18, 32, 143, 144, 145, 4096}) { + DoubleSprintzEncoder encoder = new DoubleSprintzEncoder(); + for (int round = 0; round < 2; round++) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (int i = 0; i < count; i++) { + encoder.encode(i < 9 ? (double) i : edgeValues[i % edgeValues.length], out); + } + encoder.flush(out); + if (count == 18) { + assertEquals(CPP_SPRINTZ_GOLDEN[3], toHex(out.toByteArray())); + } + ByteBuffer buffer = ByteBuffer.wrap(out.toByteArray()); + DoubleSprintzDecoder decoder = new DoubleSprintzDecoder(); + for (int i = 0; i < count; i++) { + assertTrue(method + ":" + count + ":" + i, decoder.hasNext(buffer)); + double actual = decoder.readDouble(buffer); + double expected = i < 9 ? (double) i : edgeValues[i % edgeValues.length]; + assertEquals(Double.doubleToLongBits(expected), Double.doubleToLongBits(actual)); + } + assertFalse(decoder.hasNext(buffer)); + } + } + } + @BeforeClass public static void setUp() { int hybridCount = 11;