From 1e8b2a44705081fc28bca14eca9ac96dedf46613 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Fri, 6 Nov 2026 17:14:38 -0500 Subject: [PATCH 1/3] Return nothing for a repeated sync or full flush, as zlib does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sync or full flush with nothing compressed since the previous flush made ISA-L write an empty block: 6 bytes at levels 1 to 3, and 117 bytes at level 0, where each block starts with a Huffman table. Each such flush also allocated a 16 KiB output buffer and called isal_deflate. zlib returns b"" in this case. Compress objects now remember the mode of their last flush. A sync flush after a sync or full flush, and a full flush after a full flush, return b"" when no compress() call came in between. A full flush after a sync flush still writes a block, because it resets the history. compress() forgets the mode even for empty data, as deflate() does in zlib, so compress(b"") + flush(Z_SYNC_FLUSH) still ends in 00 00 ff ff, which permessage-deflate needs for empty messages. Z_FINISH forgets it too, so flushes after finishing behave as before. pyperf timeit on Linux, Python 3.11 / 3.14, mean of 2 runs: - repeated flush(Z_SYNC_FLUSH), level 2: 1.49 / 1.38 µs -> 47 / 51 ns - repeated flush(Z_SYNC_FLUSH), level 0: 160 / 162 ns -> 46 / 50 ns - repeated IGzipFile.flush(): 1.63 / 1.66 µs -> 229 / 323 ns One write followed by 100 IGzipFile.flush() calls now gives 29 instead of 623 bytes at levels 1 to 3 (gzip gives 30), and 253 instead of 11836 bytes at level 0. A flush after new data costs 8 more instructions, which is within timing noise. The output changes only for back-to-back flushes, and the stream stays valid. --- CHANGELOG.rst | 7 ++++ src/isal/isal_zlibmodule.c | 14 ++++++++ tests/test_compat.py | 57 +++++++++++++++++++++++++++++++++ tests/test_dictionary_header.py | 6 +++- tests/test_igzip.py | 13 ++++++++ 5 files changed, 96 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 481b7c5f..488308ee 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,13 @@ Changelog version 1.10.0-dev ------------------ ++ Make ``isal_zlib.Compress.flush()`` return ``b""`` for ``Z_SYNC_FLUSH`` + when nothing was compressed since a sync or full flush, and for + ``Z_FULL_FLUSH`` when nothing was compressed since a full flush, as zlib + does. It wrote an empty block of 6 bytes, or 117 bytes at level 0. On + Linux, such a flush is now 20 times faster at levels 1 to 3. Flushing an + ``igzip`` file again without a write in between no longer grows the file + and is 4 times faster. + Fix the gzip header that ``igzip_threaded.open`` writes when ``threads`` is not 0. It had the XFL and OS bytes swapped, so the ``file`` command reported the output as coming from VM/CMS or a FAT file system. diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 3ee12d83..b6eecfbf 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -584,6 +584,8 @@ typedef struct PyObject *zdict; struct isal_zlib_header zlib_header; int is_initialised; + /* Mode of the last flush; Z_NO_FLUSH after compress() or Z_FINISH. */ + int last_flush; PyThread_type_lock lock; // isal_zstream should be at the bottom as it contains buffers inside the struct. struct isal_zstream zst; @@ -609,6 +611,7 @@ newcompobject() if (self == NULL) return NULL; self->is_initialised = 0; + self->last_flush = Z_NO_FLUSH; self->zdict = NULL; self->zlib_header.dict_flag = 0; self->level_buf = NULL; @@ -898,6 +901,8 @@ isal_zlib_Compress_compress_impl(compobject *self, Py_buffer *data) ENTER_ZLIB(self); + /* Also for b"", as in zlib: permessage-deflate needs the sync marker. */ + self->last_flush = Z_NO_FLUSH; self->zst.next_in = data->buf; ibuflen = data->len; @@ -1117,6 +1122,13 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) ENTER_ZLIB(self); + /* As in zlib, skip the empty block after an equal or stronger flush. */ + if ((mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH) && + self->last_flush >= mode) { + LEAVE_ZLIB(self); + return PyBytes_FromStringAndSize(NULL, 0); + } + if (mode == Z_FINISH) { self->zst.flush = FULL_FLUSH; self->zst.end_of_stream = 1; @@ -1167,9 +1179,11 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) self->zst.level_buf_size = 0; self->zst.level_buf = NULL; self->is_initialised = 0; + self->last_flush = Z_NO_FLUSH; } else { // reset the flush mode back so compressobject can be used again. self->zst.flush = NO_FLUSH; + self->last_flush = mode; } if (_PyBytes_Resize(&RetVal, self->zst.next_out - diff --git a/tests/test_compat.py b/tests/test_compat.py index 3f339851..4db3765c 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -174,6 +174,63 @@ def test_compressobj_level_0_flush_after_finish(): assert zlib.decompress(compressed) == b"data" +@pytest.mark.parametrize("level", range(4)) +@pytest.mark.parametrize("wbits", [-15, 15, 31]) +@pytest.mark.parametrize(["first_mode", "second_mode"], + [(isal_zlib.Z_SYNC_FLUSH, isal_zlib.Z_SYNC_FLUSH), + (isal_zlib.Z_FULL_FLUSH, isal_zlib.Z_FULL_FLUSH), + (isal_zlib.Z_FULL_FLUSH, isal_zlib.Z_SYNC_FLUSH)]) +def test_compressobj_repeated_flush(level, wbits, first_mode, second_mode): + compressor = isal_zlib.compressobj(level, wbits=wbits) + decompressor = zlib.decompressobj(wbits=wbits) + compressed = (compressor.compress(DATA[:4096]) + + compressor.flush(first_mode)) + assert compressed.endswith(b"\x00\x00\xff\xff") + assert decompressor.decompress(compressed) == DATA[:4096] + assert compressor.flush(second_mode) == b"" + assert compressor.flush(second_mode) == b"" + compressed = compressor.compress(DATA[4096:8192]) + compressor.flush() + assert decompressor.decompress(compressed) == DATA[4096:8192] + assert decompressor.eof + + +@pytest.mark.parametrize("level", range(4)) +@pytest.mark.parametrize("wbits", [-15, 15, 31]) +def test_compressobj_repeated_flush_before_data(level, wbits): + compressor = isal_zlib.compressobj(level, wbits=wbits) + compressed = compressor.flush(isal_zlib.Z_SYNC_FLUSH) + assert compressed.endswith(b"\x00\x00\xff\xff") + assert compressor.flush(isal_zlib.Z_SYNC_FLUSH) == b"" + compressed += compressor.compress(DATA[:4096]) + compressor.flush() + assert zlib.decompress(compressed, wbits=wbits) == DATA[:4096] + + +@pytest.mark.parametrize("level", range(4)) +def test_compressobj_full_flush_after_sync_flush(level): + compressor = isal_zlib.compressobj(level, wbits=-15) + compressor.compress(DATA[:4096]) + compressor.flush(isal_zlib.Z_SYNC_FLUSH) + # A full flush after a sync flush still resets the history. + assert compressor.flush(isal_zlib.Z_FULL_FLUSH).endswith( + b"\x00\x00\xff\xff") + compressed = compressor.compress(DATA[:4096]) + compressor.flush() + assert zlib.decompress(compressed, wbits=-15) == DATA[:4096] + + +@pytest.mark.parametrize("level", range(4)) +@pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH, + isal_zlib.Z_FULL_FLUSH]) +def test_compressobj_flush_after_empty_compress(level, flush_mode): + # permessage-deflate (RFC 7692) needs 00 00 ff ff after empty messages. + compressor = isal_zlib.compressobj(level, wbits=-15) + decompressor = zlib.decompressobj(wbits=-15) + for message in [b"hello", b"", b"", DATA[:4096], b""]: + compressed = (compressor.compress(message) + + compressor.flush(flush_mode)) + assert compressed.endswith(b"\x00\x00\xff\xff") + assert decompressor.decompress(compressed) == message + + @pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH, isal_zlib.Z_FULL_FLUSH]) @pytest.mark.parametrize(["strategy", "btype"], diff --git a/tests/test_dictionary_header.py b/tests/test_dictionary_header.py index 028b88c7..c2adbd1a 100644 --- a/tests/test_dictionary_header.py +++ b/tests/test_dictionary_header.py @@ -11,7 +11,8 @@ @pytest.mark.parametrize('wbits', [9, 15, -15]) @pytest.mark.parametrize('dictionary_size', [11, 32768, 40000]) @pytest.mark.parametrize( - 'first_operation', ['compress', 'empty', 'sync', 'finish']) + 'first_operation', + ['compress', 'empty', 'sync', 'repeated sync', 'finish']) def test_dictionary_cross_decode(level, wbits, dictionary_size, first_operation): dictionary = random.Random(21).randbytes(dictionary_size) @@ -22,6 +23,9 @@ def test_dictionary_cross_decode(level, wbits, dictionary_size, packed += compressor.compress(b'') elif first_operation == 'sync': packed += compressor.flush(isal_zlib.Z_SYNC_FLUSH) + elif first_operation == 'repeated sync': + packed += compressor.flush(isal_zlib.Z_SYNC_FLUSH) + assert compressor.flush(isal_zlib.Z_SYNC_FLUSH) == b'' elif first_operation == 'finish': data = b'' for pos in range(0, len(data), 17): diff --git a/tests/test_igzip.py b/tests/test_igzip.py index 457f13f6..25a000d1 100644 --- a/tests/test_igzip.py +++ b/tests/test_igzip.py @@ -771,6 +771,19 @@ def test_small_writes_tell_and_flush(): assert gzip.decompress(output.getvalue()) == b"abcdefgh" +def test_repeated_flush_writes_nothing(): + output = io.BytesIO() + with igzip.IGzipFile(fileobj=output, mode="wb") as gzip_file: + gzip_file.write(b"abc") + gzip_file.flush() + size = len(output.getvalue()) + gzip_file.flush() + assert len(output.getvalue()) == size + gzip_file.write(b"def") + gzip_file.flush() + assert gzip.decompress(output.getvalue()) == b"abcdef" + + @pytest.mark.skipif(sys.platform != "linux", reason="bash required") def test_no_broken_pipe(): subprocess.run( From aeff64cfccbf0d855acbe7937d7e1d0dc508df35 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Tue, 17 Nov 2026 22:18:30 -0500 Subject: [PATCH 2/3] Record the mode of skipped flushes too, as deflate() does zlib's deflate() stores the flush mode of every call, including the calls it skips, so after a full flush and a skipped sync flush, another full flush writes a block. Compress.flush() kept the mode of the full flush instead and returned b"" for the second full flush. It now also stores the mode of a skipped flush, so it returns b"" for exactly the flushes for which zlib writes nothing. A new test compares isal_zlib with zlib for every sequence of up to four compress() and flush() calls, at every level and for raw, zlib and gzip streams. --- CHANGELOG.rst | 11 +++++------ src/isal/isal_zlibmodule.c | 9 +++++++-- tests/test_compat.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c688c461..f125d54c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -10,12 +10,11 @@ Changelog version 1.10.0-dev ------------------ + Make ``isal_zlib.Compress.flush()`` return ``b""`` for ``Z_SYNC_FLUSH`` - when nothing was compressed since a sync or full flush, and for - ``Z_FULL_FLUSH`` when nothing was compressed since a full flush, as zlib - does. It wrote an empty block of 6 bytes, or 117 bytes at level 0. On - Linux, such a flush is now 20 times faster at levels 1 to 3. Flushing an - ``igzip`` file again without a write in between no longer grows the file - and is 4 times faster. + right after a sync or full flush, and for ``Z_FULL_FLUSH`` right after a + full flush, as zlib does. It wrote an empty block of 6 bytes, or 117 bytes + at level 0. On Linux, such a flush is now 20 times faster at levels 1 to 3. + Flushing an ``igzip`` file again without a write in between no longer grows + the file and is 4 times faster. + Fix ``igzip_threaded`` files that stopped working when the thread that opened them ended, for example a ``ThreadPoolExecutor`` thread after ``shutdown()``. Reads returned truncated data without an error, and writes diff --git a/src/isal/isal_zlibmodule.c b/src/isal/isal_zlibmodule.c index 5bae8f9a..4cb9bf40 100644 --- a/src/isal/isal_zlibmodule.c +++ b/src/isal/isal_zlibmodule.c @@ -584,7 +584,8 @@ typedef struct PyObject *zdict; struct isal_zlib_header zlib_header; int is_initialised; - /* Mode of the last flush; Z_NO_FLUSH after compress() or Z_FINISH. */ + /* Mode of the last flush() call, skipped or not, as zlib keeps it. + Z_NO_FLUSH after compress() or Z_FINISH. */ int last_flush; PyThread_type_lock lock; // isal_zstream should be at the bottom as it contains buffers inside the struct. @@ -1122,9 +1123,13 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode) ENTER_ZLIB(self); - /* As in zlib, skip the empty block after an equal or stronger flush. */ + /* isal_deflate() writes an empty block for each sync or full flush, even + without new input (see igzip_lib.h). Like deflate() in zlib, write + nothing when the previous call was an equal or stronger flush, and + remember the mode of a skipped flush as well. */ if ((mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH) && self->last_flush >= mode) { + self->last_flush = mode; LEAVE_ZLIB(self); return PyBytes_FromStringAndSize(NULL, 0); } diff --git a/tests/test_compat.py b/tests/test_compat.py index 4db3765c..18b8b469 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -231,6 +231,34 @@ def test_compressobj_flush_after_empty_compress(level, flush_mode): assert decompressor.decompress(compressed) == message +@pytest.mark.parametrize("level", range(4)) +@pytest.mark.parametrize("wbits", [-15, 15, 31]) +def test_compressobj_skipped_flushes_match_zlib(level, wbits): + # zlib writes nothing for a sync flush right after a sync or full flush, + # or for a full flush right after a full flush. A skipped flush counts + # as the previous call too: in full, sync, full, the last flush writes. + modes = {"sync": zlib.Z_SYNC_FLUSH, "full": zlib.Z_FULL_FLUSH} + for length in range(1, 5): + for steps in itertools.product(["data", "empty", "sync", "full"], + repeat=length): + isal_compressor = isal_zlib.compressobj(level, wbits=wbits) + zlib_compressor = zlib.compressobj(wbits=wbits) + compressed = b"" + for step in steps: + if step in modes: + isal_output = isal_compressor.flush(modes[step]) + zlib_output = zlib_compressor.flush(modes[step]) + assert (isal_output == b"") == (zlib_output == b""), steps + compressed += isal_output + else: + data = DATA[:1000] if step == "data" else b"" + compressed += isal_compressor.compress(data) + zlib_compressor.compress(data) + compressed += isal_compressor.flush() + assert (zlib.decompress(compressed, wbits=wbits) == + DATA[:1000] * steps.count("data")) + + @pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH, isal_zlib.Z_FULL_FLUSH]) @pytest.mark.parametrize(["strategy", "btype"], From 588bcf3ba6e0c8ef9f6a8c97f91481d691143156 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Tue, 17 Nov 2026 22:36:21 -0500 Subject: [PATCH 3/3] Drop the flush tests that the zlib comparison covers The comparison with zlib runs every sequence of up to four compress() and flush() calls at every level, for raw, zlib and gzip streams. That covers what the separate tests checked: repeated sync and full flushes, a flush before any data, a full flush after a sync flush and a flush after compress(b""). Remove those tests, and the repeated sync case in test_dictionary_header.py, since a dictionary does not change which flushes are skipped. --- tests/test_compat.py | 58 +-------------------------------- tests/test_dictionary_header.py | 6 +--- 2 files changed, 2 insertions(+), 62 deletions(-) diff --git a/tests/test_compat.py b/tests/test_compat.py index 18b8b469..edaafff8 100644 --- a/tests/test_compat.py +++ b/tests/test_compat.py @@ -174,69 +174,13 @@ def test_compressobj_level_0_flush_after_finish(): assert zlib.decompress(compressed) == b"data" -@pytest.mark.parametrize("level", range(4)) -@pytest.mark.parametrize("wbits", [-15, 15, 31]) -@pytest.mark.parametrize(["first_mode", "second_mode"], - [(isal_zlib.Z_SYNC_FLUSH, isal_zlib.Z_SYNC_FLUSH), - (isal_zlib.Z_FULL_FLUSH, isal_zlib.Z_FULL_FLUSH), - (isal_zlib.Z_FULL_FLUSH, isal_zlib.Z_SYNC_FLUSH)]) -def test_compressobj_repeated_flush(level, wbits, first_mode, second_mode): - compressor = isal_zlib.compressobj(level, wbits=wbits) - decompressor = zlib.decompressobj(wbits=wbits) - compressed = (compressor.compress(DATA[:4096]) + - compressor.flush(first_mode)) - assert compressed.endswith(b"\x00\x00\xff\xff") - assert decompressor.decompress(compressed) == DATA[:4096] - assert compressor.flush(second_mode) == b"" - assert compressor.flush(second_mode) == b"" - compressed = compressor.compress(DATA[4096:8192]) + compressor.flush() - assert decompressor.decompress(compressed) == DATA[4096:8192] - assert decompressor.eof - - -@pytest.mark.parametrize("level", range(4)) -@pytest.mark.parametrize("wbits", [-15, 15, 31]) -def test_compressobj_repeated_flush_before_data(level, wbits): - compressor = isal_zlib.compressobj(level, wbits=wbits) - compressed = compressor.flush(isal_zlib.Z_SYNC_FLUSH) - assert compressed.endswith(b"\x00\x00\xff\xff") - assert compressor.flush(isal_zlib.Z_SYNC_FLUSH) == b"" - compressed += compressor.compress(DATA[:4096]) + compressor.flush() - assert zlib.decompress(compressed, wbits=wbits) == DATA[:4096] - - -@pytest.mark.parametrize("level", range(4)) -def test_compressobj_full_flush_after_sync_flush(level): - compressor = isal_zlib.compressobj(level, wbits=-15) - compressor.compress(DATA[:4096]) - compressor.flush(isal_zlib.Z_SYNC_FLUSH) - # A full flush after a sync flush still resets the history. - assert compressor.flush(isal_zlib.Z_FULL_FLUSH).endswith( - b"\x00\x00\xff\xff") - compressed = compressor.compress(DATA[:4096]) + compressor.flush() - assert zlib.decompress(compressed, wbits=-15) == DATA[:4096] - - -@pytest.mark.parametrize("level", range(4)) -@pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH, - isal_zlib.Z_FULL_FLUSH]) -def test_compressobj_flush_after_empty_compress(level, flush_mode): - # permessage-deflate (RFC 7692) needs 00 00 ff ff after empty messages. - compressor = isal_zlib.compressobj(level, wbits=-15) - decompressor = zlib.decompressobj(wbits=-15) - for message in [b"hello", b"", b"", DATA[:4096], b""]: - compressed = (compressor.compress(message) + - compressor.flush(flush_mode)) - assert compressed.endswith(b"\x00\x00\xff\xff") - assert decompressor.decompress(compressed) == message - - @pytest.mark.parametrize("level", range(4)) @pytest.mark.parametrize("wbits", [-15, 15, 31]) def test_compressobj_skipped_flushes_match_zlib(level, wbits): # zlib writes nothing for a sync flush right after a sync or full flush, # or for a full flush right after a full flush. A skipped flush counts # as the previous call too: in full, sync, full, the last flush writes. + # compress(b"") counts as a call as well, which permessage-deflate needs. modes = {"sync": zlib.Z_SYNC_FLUSH, "full": zlib.Z_FULL_FLUSH} for length in range(1, 5): for steps in itertools.product(["data", "empty", "sync", "full"], diff --git a/tests/test_dictionary_header.py b/tests/test_dictionary_header.py index c2adbd1a..028b88c7 100644 --- a/tests/test_dictionary_header.py +++ b/tests/test_dictionary_header.py @@ -11,8 +11,7 @@ @pytest.mark.parametrize('wbits', [9, 15, -15]) @pytest.mark.parametrize('dictionary_size', [11, 32768, 40000]) @pytest.mark.parametrize( - 'first_operation', - ['compress', 'empty', 'sync', 'repeated sync', 'finish']) + 'first_operation', ['compress', 'empty', 'sync', 'finish']) def test_dictionary_cross_decode(level, wbits, dictionary_size, first_operation): dictionary = random.Random(21).randbytes(dictionary_size) @@ -23,9 +22,6 @@ def test_dictionary_cross_decode(level, wbits, dictionary_size, packed += compressor.compress(b'') elif first_operation == 'sync': packed += compressor.flush(isal_zlib.Z_SYNC_FLUSH) - elif first_operation == 'repeated sync': - packed += compressor.flush(isal_zlib.Z_SYNC_FLUSH) - assert compressor.flush(isal_zlib.Z_SYNC_FLUSH) == b'' elif first_operation == 'finish': data = b'' for pos in range(0, len(data), 17):