Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions Lib/test/test_zlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -1195,6 +1195,35 @@ def testDecompress4G(self, size):
compressed = None
decompressed = None

@support.skip_if_pgo_task
@bigmemtest(size=_4G + 100, memuse=2.5)
def testDecompressMaxLength4G(self, size):
# gh-162153: a decompress() call whose input exceeds UINT_MAX
# bytes must not silently drop the unconsumed remainder when
# max_length is reached in the same call.
# Peak memory is about 2x size: the >4 GiB compressed input and
# the decompressor's internal copy of the unconsumed input tail
# coexist across the first decompress() call.
blocksize = min(10 * 1024 * 1024, size)
block = random.randbytes(blocksize)
nblocks = (size - 1) // blocksize + 1
# Level 0 (stored blocks) keeps the compressed input above
# 4 GiB, so arrange_input_buffer() has to chunk it.
c = zlib.compressobj(0)
compressed = bytearray()
try:
for _ in range(nblocks):
compressed += c.compress(block)
compressed += c.flush()
zlibd = zlib._ZlibDecompressor()
out = len(zlibd.decompress(compressed, 65536))
finally:
block = compressed = None
while not zlibd.eof and not zlibd.needs_input:
out += len(zlibd.decompress(b"", 65536))
self.assertTrue(zlibd.eof)
self.assertEqual(out, blocksize * nblocks)

def testPickle(self):
for proto in range(pickle.HIGHEST_PROTOCOL + 1):
with self.assertRaises(TypeError):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
Fix silent input loss in :meth:`!zlib._ZlibDecompressor.decompress` when one
call receives more than 4 GiB of input and reaches *max_length*: the
unconsumed remainder of an internal input chunk was dropped from the
bookkeeping, so decompression continued at the wrong stream position.
7 changes: 7 additions & 0 deletions Modules/zlibmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -1541,6 +1541,13 @@ decompress_buf(ZlibDecompressor *self, Py_ssize_t max_length)
}
}
} while (self->zst.avail_out == 0);
/* Stop when the output limit is reached: no more input is needed,
and re-running arrange_input_buffer() would overwrite
zst.avail_in without crediting its unconsumed remainder to
avail_in_real, silently dropping that input (see gh-162153). */
if (obuflen == -2) {
break;
}
} while(err != Z_STREAM_END && self->avail_in_real != 0);

if (err == Z_STREAM_END) {
Expand Down
Loading