From 6460ffc79fcaad28d635c4bf5aed519ca1f5a5e3 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Wed, 18 Nov 2026 10:33:40 -0500 Subject: [PATCH] Keep the unread input when IgzipDecompressor stops at max_length decompress_buf() passes the input to isal_inflate() in parts of at most UINT32_MAX bytes. When the output reached max_length, only the inner loop ended. The outer loop then set avail_in to the next part and dropped the bytes of the current part that isal_inflate() had not read. With more than 4 GiB of input in one call, later calls returned only part of the data. Leave both loops instead, so that the code after them adds avail_in back to avail_in_real. --- CHANGELOG.rst | 4 ++++ src/isal/igzip_libmodule.c | 6 +++++- tests/test_igzip_lib.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 372b8223..1fa5eb70 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,10 @@ Changelog version 1.10.0-dev ------------------ ++ Fix ``igzip_lib.IgzipDecompressor.decompress()`` losing input when one call + gets more than 4 GiB of data and the output reaches ``max_length``. Later + calls returned only part of the data, and ``needs_input`` became ``True`` + before the end of the stream. + Make ``isal_zlib.Compress.flush()`` return ``b""`` for ``Z_SYNC_FLUSH`` right after a sync or full flush, and for ``Z_FULL_FLUSH`` right after a full flush, as zlib does. It wrote an empty block of 6 bytes, or 117 bytes diff --git a/src/isal/igzip_libmodule.c b/src/isal/igzip_libmodule.c index 16c041fd..853fe897 100644 --- a/src/isal/igzip_libmodule.c +++ b/src/isal/igzip_libmodule.c @@ -129,7 +129,10 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length) goto error; } else if (obuflen == -2) - break; + /* Do not go back to the outer loop: it would set avail_in + to the next part of the input and drop the bytes that + isal_inflate() has not read. */ + goto max_length_reached; Py_BEGIN_ALLOW_THREADS; err = isal_inflate(&(self->state)); @@ -160,6 +163,7 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length) } while(self->avail_in_real != 0 && self->state.block_state != ISAL_BLOCK_FINISH); +max_length_reached: if (self->state.block_state == ISAL_BLOCK_FINISH) self->eof = 1; diff --git a/tests/test_igzip_lib.py b/tests/test_igzip_lib.py index f0cc434d..a26aaece 100644 --- a/tests/test_igzip_lib.py +++ b/tests/test_igzip_lib.py @@ -129,6 +129,27 @@ def testDecompress4G(self): compressed = None decompressed = None + @pytest.mark.skip(reason="Causes memory issues on CI systems.") + def testDecompress4GMaxLength(self): + # The input is larger than one isal_inflate() call takes (4 GiB - 1 + # bytes), and the output reaches max_length during the first call. + size = 4 * 1024 ** 3 + 100 # 4 GiB + 100 + blocksize = 10 * 1024 * 1024 + block = os.urandom(blocksize) + try: + data = block * (size // blocksize + 1) + compressed = igzip_lib.compress(data) + igzd = IgzipDecompressor() + decompressed = igzd.decompress(compressed, max_length=blocksize) + assert not igzd.needs_input + decompressed += igzd.decompress(b"") + assert igzd.eof + assert decompressed == data + finally: + data = None + compressed = None + decompressed = None + @pytest.mark.skipif(sys.implementation.name == "pypy", reason="Pickling is not a requirement, and certainly " "not a blocker for PyPy.")