Repository navigation
Read gzip files with python-isal on x86-64 only, and use it for writing on macOS arm64 #188
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -55,6 +55,16 @@ | |
| gzip_ng_threaded = None | ||
| zlib_ng = None | ||
|
|
||
| # ISA-L decompresses faster than zlib-ng and zlib on x86-64. On ARM (aarch64 on | ||
| # Linux, arm64 on macOS), zlib-ng decompresses faster than ISA-L, and so does | ||
| # the zlib that Apple ships with macOS. platform.machine() is not used here | ||
| # because it takes tens of milliseconds on Windows. | ||
| if sys.platform == "win32": | ||
| _MACHINE = os.environ.get("PROCESSOR_ARCHITECTURE", "AMD64") | ||
| else: | ||
| _MACHINE = os.uname().machine | ||
| _ISAL_DECOMPRESSES_FASTEST = _MACHINE.lower() in ("x86_64", "amd64") | ||
|
Comment on lines
+58
to
+66
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This code can just be in |
||
|
|
||
| if sys.version_info >= (3, 14): | ||
| from compression import zstd | ||
| else: | ||
|
|
@@ -567,6 +577,15 @@ def _open_lz4( | |
| ) | ||
|
|
||
|
|
||
| def _use_isal(mode: str, compresslevel: int) -> bool: | ||
| """Return whether ISA-L is the preferred gzip backend.""" | ||
| if "r" in mode: | ||
| return _ISAL_DECOMPRESSES_FASTEST | ||
| # Igzip level 0 does not output uncompressed deflate blocks as zlib does | ||
| # and level 3 is slower but does not compress better than level 1 and 2. | ||
| return compresslevel in (1, 2) | ||
|
|
||
|
|
||
| def _open_gz( | ||
| filename: FileOrPath, | ||
| mode: str, | ||
|
|
@@ -575,7 +594,8 @@ def _open_gz( | |
| ): | ||
| """ | ||
| Open a gzip file. The ISA-L library is preferred when applicable because | ||
| it is the fastest. Then zlib-ng which is not as fast, but supports all | ||
| it is the fastest: for compression at levels 1 and 2, and for decompression | ||
| on x86-64. Then zlib-ng which is not as fast, but supports all | ||
| compression levels. After that comes pigz, which can utilize multiple | ||
| threads and is more efficient than gzip, even on one core. gzip is chosen | ||
| when none of the alternatives are available. Despite it being able to use | ||
|
|
@@ -595,9 +615,7 @@ def _open_gz( | |
| ) | ||
|
|
||
| if threads != 0: | ||
| # Igzip level 0 does not output uncompressed deflate blocks as zlib does | ||
| # and level 3 is slower but does not compress better than level 1 and 2. | ||
| if igzip_threaded and (compresslevel in (1, 2) or "r" in mode): | ||
| if igzip_threaded and _use_isal(mode, compresslevel): | ||
| return igzip_threaded.open( # type: ignore | ||
| filename, | ||
| mode, | ||
|
|
@@ -647,9 +665,7 @@ def _open_reproducible_gzip(filename, mode: str, compresslevel: int): | |
| mode=mode, | ||
| mtime=0, | ||
| ) | ||
| # Igzip level 0 does not output uncompressed deflate blocks as zlib does | ||
| # and level 3 is slower but does not compress better than level 1 and 2. | ||
| if igzip is not None and (compresslevel in (1, 2) or "r" in mode): | ||
| if igzip is not None and _use_isal(mode, compresslevel): | ||
| gzip_file = igzip.IGzipFile(**kwargs, compresslevel=compresslevel) | ||
| elif gzip_ng is not None: | ||
| # Zlib-ng level 1 creates much bigger files than zlib level 1. | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -432,6 +432,57 @@ def test_write_gzip_no_threads_no_isal(tmp_path, xopen_without_igzip): | |
| assert isinstance(f.raw, gzip.GzipFile) | ||
|
|
||
|
|
||
| def _gzip_backend(f) -> str: | ||
| """Return the top-level module of the gzip file object that xopen returned""" | ||
| if isinstance(f, (io.BufferedReader, io.BufferedWriter)): | ||
| f = f.raw | ||
| return type(f).__module__.split(".")[0] | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("threads", [0, 1]) | ||
| @pytest.mark.parametrize( | ||
| ["isal_decompresses_fastest", "expected"], [(True, "isal"), (False, "zlib_ng")] | ||
| ) | ||
| def test_gzip_read_backend_depends_on_architecture( | ||
| monkeypatch, threads, isal_decompresses_fastest, expected | ||
| ): | ||
| import xopen # xopen local overrides xopen global variable | ||
|
|
||
| if xopen.igzip is None or xopen.gzip_ng is None: | ||
| pytest.skip("requires python-isal and python-zlib-ng") | ||
| monkeypatch.setattr(xopen, "_ISAL_DECOMPRESSES_FASTEST", isal_decompresses_fastest) | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Rather than monkey patching global variables from the code it is better to monkeypatch the things that are checked in the _use_isal function, such as the sys.platorm variables. |
||
| with xopen.xopen(TEST_DIR / "file.txt.gz", "rb", threads=threads) as f: | ||
| assert _gzip_backend(f) == expected | ||
| assert f.read() == CONTENT.encode() | ||
|
|
||
|
|
||
| def test_gzip_read_without_zlib_ng_where_isal_is_slower(monkeypatch): | ||
| import xopen # xopen local overrides xopen global variable | ||
|
|
||
| monkeypatch.setattr(xopen, "_ISAL_DECOMPRESSES_FASTEST", False) | ||
| monkeypatch.setattr(xopen, "gzip_ng", None) | ||
| with xopen.xopen(TEST_DIR / "file.txt.gz", "rb", threads=0) as f: | ||
| assert type(f) is gzip.GzipFile | ||
| assert f.read() == CONTENT.encode() | ||
|
|
||
|
|
||
| @pytest.mark.parametrize("threads", [0, 1]) | ||
| @pytest.mark.parametrize("isal_decompresses_fastest", [True, False]) | ||
| def test_gzip_write_backend_at_level_1_is_isal( | ||
| tmp_path, monkeypatch, threads, isal_decompresses_fastest | ||
| ): | ||
| import xopen # xopen local overrides xopen global variable | ||
|
|
||
| if xopen.igzip is None: | ||
| pytest.skip("requires python-isal") | ||
| monkeypatch.setattr(xopen, "_ISAL_DECOMPRESSES_FASTEST", isal_decompresses_fastest) | ||
| path = tmp_path / "out.gz" | ||
| with xopen.xopen(path, "wb", compresslevel=1, threads=threads) as f: | ||
| assert _gzip_backend(f) == "isal" | ||
| f.write(b"hello") | ||
| assert gzip.decompress(path.read_bytes()) == b"hello" | ||
|
|
||
|
|
||
| def test_write_stdout(): | ||
| f = xopen("-", mode="w") | ||
| print("Hello", file=f) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Getting this info from the environment seems awfully flaky... Isn't there some winapi call for this? Or some python function?