Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@ Changelog

version 1.10.0-dev
------------------
+ Make ``isal_zlib.Compress.flush()`` return ``b""`` for ``Z_SYNC_FLUSH``
right after a sync or full flush, and for ``Z_FULL_FLUSH`` right after a
full flush, as zlib does. It wrote an empty block of 6 bytes, or 117 bytes
at level 0. On Linux, such a flush is now 20 times faster at levels 1 to 3.
Flushing an ``igzip`` file again without a write in between no longer grows
the file and is 4 times faster.
+ Speed up ``import isal`` by about 0.1 ms in programs that have not imported
``__future__`` yet. The version module that is generated when isal is built
no longer imports ``__future__``. Building isal now requires setuptools-scm
Expand Down
19 changes: 19 additions & 0 deletions src/isal/isal_zlibmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -584,6 +584,9 @@ typedef struct
PyObject *zdict;
struct isal_zlib_header zlib_header;
int is_initialised;
/* Mode of the last flush() call, skipped or not, as zlib keeps it.
Z_NO_FLUSH after compress() or Z_FINISH. */
int last_flush;
PyThread_type_lock lock;
// isal_zstream should be at the bottom as it contains buffers inside the struct.
struct isal_zstream zst;
Expand All @@ -609,6 +612,7 @@ newcompobject()
if (self == NULL)
return NULL;
self->is_initialised = 0;
self->last_flush = Z_NO_FLUSH;
self->zdict = NULL;
self->zlib_header.dict_flag = 0;
self->level_buf = NULL;
Expand Down Expand Up @@ -898,6 +902,8 @@ isal_zlib_Compress_compress_impl(compobject *self, Py_buffer *data)

ENTER_ZLIB(self);

/* Also for b"", as in zlib: permessage-deflate needs the sync marker. */
self->last_flush = Z_NO_FLUSH;
self->zst.next_in = data->buf;
ibuflen = data->len;

Expand Down Expand Up @@ -1117,6 +1123,17 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode)

ENTER_ZLIB(self);

/* isal_deflate() writes an empty block for each sync or full flush, even
without new input (see igzip_lib.h). Like deflate() in zlib, write
nothing when the previous call was an equal or stronger flush, and
remember the mode of a skipped flush as well. */
if ((mode == Z_SYNC_FLUSH || mode == Z_FULL_FLUSH) &&
self->last_flush >= mode) {
self->last_flush = mode;
LEAVE_ZLIB(self);
return PyBytes_FromStringAndSize(NULL, 0);
}

if (mode == Z_FINISH) {
self->zst.flush = FULL_FLUSH;
self->zst.end_of_stream = 1;
Expand Down Expand Up @@ -1167,9 +1184,11 @@ isal_zlib_Compress_flush_impl(compobject *self, int mode)
self->zst.level_buf_size = 0;
self->zst.level_buf = NULL;
self->is_initialised = 0;
self->last_flush = Z_NO_FLUSH;
} else {
// reset the flush mode back so compressobject can be used again.
self->zst.flush = NO_FLUSH;
self->last_flush = mode;
}

if (_PyBytes_Resize(&RetVal, self->zst.next_out -
Expand Down
29 changes: 29 additions & 0 deletions tests/test_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,35 @@ def test_compressobj_level_0_flush_after_finish():
assert zlib.decompress(compressed) == b"data"


@pytest.mark.parametrize("level", range(4))
@pytest.mark.parametrize("wbits", [-15, 15, 31])
def test_compressobj_skipped_flushes_match_zlib(level, wbits):
# zlib writes nothing for a sync flush right after a sync or full flush,
# or for a full flush right after a full flush. A skipped flush counts
# as the previous call too: in full, sync, full, the last flush writes.
# compress(b"") counts as a call as well, which permessage-deflate needs.
modes = {"sync": zlib.Z_SYNC_FLUSH, "full": zlib.Z_FULL_FLUSH}
for length in range(1, 5):
for steps in itertools.product(["data", "empty", "sync", "full"],
repeat=length):
isal_compressor = isal_zlib.compressobj(level, wbits=wbits)
zlib_compressor = zlib.compressobj(wbits=wbits)
compressed = b""
for step in steps:
if step in modes:
isal_output = isal_compressor.flush(modes[step])
zlib_output = zlib_compressor.flush(modes[step])
assert (isal_output == b"") == (zlib_output == b""), steps
compressed += isal_output
else:
data = DATA[:1000] if step == "data" else b""
compressed += isal_compressor.compress(data)
zlib_compressor.compress(data)
compressed += isal_compressor.flush()
assert (zlib.decompress(compressed, wbits=wbits) ==
DATA[:1000] * steps.count("data"))


@pytest.mark.parametrize("flush_mode", [isal_zlib.Z_SYNC_FLUSH,
isal_zlib.Z_FULL_FLUSH])
@pytest.mark.parametrize(["strategy", "btype"],
Expand Down
13 changes: 13 additions & 0 deletions tests/test_igzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -821,6 +821,19 @@ def test_small_writes_tell_and_flush():
assert gzip.decompress(output.getvalue()) == b"abcdefgh"


def test_repeated_flush_writes_nothing():
output = io.BytesIO()
with igzip.IGzipFile(fileobj=output, mode="wb") as gzip_file:
gzip_file.write(b"abc")
gzip_file.flush()
size = len(output.getvalue())
gzip_file.flush()
assert len(output.getvalue()) == size
gzip_file.write(b"def")
gzip_file.flush()
assert gzip.decompress(output.getvalue()) == b"abcdef"


@pytest.mark.skipif(sys.platform != "linux", reason="bash required")
def test_no_broken_pipe():
subprocess.run(
Expand Down
Loading