Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ Changelog

version 1.10.0-dev
------------------
+ Speed up ``import isal.igzip`` and ``import isal.igzip_threaded`` by not
importing ``typing``.
+ Fix the gzip header that ``igzip_threaded.open`` writes when ``threads`` is
not 0. It had the XFL and OS bytes swapped, so the ``file`` command reported
the output as coming from VM/CMS or a FAT file system.
Expand Down
5 changes: 2 additions & 3 deletions src/isal/igzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import struct
import sys
import time
from typing import Optional, SupportsInt

from . import igzip_lib, isal_zlib
from .isal_zlib import _GzipReader
Expand Down Expand Up @@ -250,12 +249,12 @@ def seek(self, offset, whence=io.SEEK_SET):


def compress(data, compresslevel: int = _COMPRESS_LEVEL_BEST, *,
mtime: Optional[SupportsInt] = None) -> bytes:
mtime: int | None = None) -> bytes:
"""Compress data in one shot and return the compressed string.
Optional argument is the compression level, in range of 0-3.
"""
if mtime is None:
mtime = time.time()
mtime = int(time.time())
# There is no best compression level. ISA-L only provides algorithms for
# fast and medium levels.
xfl = 4 if compresslevel == _COMPRESS_LEVEL_FAST else 0
Expand Down
13 changes: 6 additions & 7 deletions src/isal/igzip_threaded.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import queue
import struct
import threading
from typing import List, Optional, Tuple

from . import igzip, isal_zlib

Expand Down Expand Up @@ -235,7 +234,7 @@ def __init__(self,
mode += "b"
self.lock = threading.Lock()
self._calling_thread = threading.current_thread()
self.exception: Optional[Exception] = None
self.exception: Exception | None = None
self.level = level
self.previous_block = b""
# Deflating random data results in an output a little larger than the
Expand All @@ -250,17 +249,17 @@ def __init__(self,
else:
compress_buffer_size = block_size + max(block_size // 10, 500)
self.block_size = block_size
self.compressors: List[isal_zlib._ParallelCompress] = [
self.compressors: list[isal_zlib._ParallelCompress] = [
isal_zlib._ParallelCompress(buffersize=compress_buffer_size,
level=level) for _ in range(threads)
]
if threads > 1:
# None is the sentinel from stop()
self.input_queues: List[
queue.Queue[Optional[Tuple[bytes, memoryview]]]] = [
self.input_queues: list[
queue.Queue[tuple[bytes, memoryview] | None]] = [
queue.Queue(queue_size) for _ in range(threads)]
self.output_queues: List[
queue.Queue[Optional[Tuple[bytes, int, int]]]] = [
self.output_queues: list[
queue.Queue[tuple[bytes, int, int] | None]] = [
queue.Queue(queue_size) for _ in range(threads)]
self.output_worker = threading.Thread(target=self._write)
self.compression_workers = [
Expand Down
5 changes: 3 additions & 2 deletions tests/test_igzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -419,7 +419,7 @@ def test_compress_infile_out_file_no_name(monkeypatch, tmp_path, capsysbinary):
assert output[4:8] == b"\x00\x00\x00\x00" # No timestamp set.


def test_import_does_not_load_cli_modules():
def test_import_does_not_load_slow_modules():
# Each of these modules takes milliseconds to import
code = ("import sys; before = set(sys.modules); "
"import isal.igzip, isal.igzip_threaded; "
Expand All @@ -428,7 +428,8 @@ def test_import_does_not_load_cli_modules():
capture_output=True, text=True)
new_modules = set(result.stdout.split())
assert "isal.igzip_threaded" in new_modules
assert not new_modules & {"argparse", "shutil", "multiprocessing"}
assert not new_modules & {"argparse", "shutil", "multiprocessing",
"typing"}


def test_decompress():
Expand Down
Loading