Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,10 @@ Changelog

version 1.10.0-dev
------------------
+ Fix ``igzip_lib.IgzipDecompressor.decompress()`` losing input when one call
gets more than 4 GiB of data and the output reaches ``max_length``. Later
calls returned only part of the data, and ``needs_input`` became ``True``
before the end of the stream.
+ Make ``isal_zlib.Compress.flush()`` return ``b""`` for ``Z_SYNC_FLUSH``
right after a sync or full flush, and for ``Z_FULL_FLUSH`` right after a
full flush, as zlib does. It wrote an empty block of 6 bytes, or 117 bytes
Expand Down
6 changes: 5 additions & 1 deletion src/isal/igzip_libmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,10 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length)
goto error;
}
else if (obuflen == -2)
break;
/* Do not go back to the outer loop: it would set avail_in
to the next part of the input and drop the bytes that
isal_inflate() has not read. */
goto max_length_reached;

Py_BEGIN_ALLOW_THREADS;
err = isal_inflate(&(self->state));
Expand Down Expand Up @@ -160,6 +163,7 @@ decompress_buf(IgzipDecompressor *self, Py_ssize_t max_length)
} while(self->avail_in_real != 0 &&
self->state.block_state != ISAL_BLOCK_FINISH);

max_length_reached:
if (self->state.block_state == ISAL_BLOCK_FINISH)
self->eof = 1;

Expand Down
21 changes: 21 additions & 0 deletions tests/test_igzip_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,27 @@ def testDecompress4G(self):
compressed = None
decompressed = None

@pytest.mark.skip(reason="Causes memory issues on CI systems.")
def testDecompress4GMaxLength(self):
# The input is larger than one isal_inflate() call takes (4 GiB - 1
# bytes), and the output reaches max_length during the first call.
size = 4 * 1024 ** 3 + 100 # 4 GiB + 100
blocksize = 10 * 1024 * 1024
block = os.urandom(blocksize)
try:
data = block * (size // blocksize + 1)
compressed = igzip_lib.compress(data)
igzd = IgzipDecompressor()
decompressed = igzd.decompress(compressed, max_length=blocksize)
assert not igzd.needs_input
decompressed += igzd.decompress(b"")
assert igzd.eof
assert decompressed == data
finally:
data = None
compressed = None
decompressed = None

@pytest.mark.skipif(sys.implementation.name == "pypy",
reason="Pickling is not a requirement, and certainly "
"not a blocker for PyPy.")
Expand Down
Loading