diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8f110d1..1a622e2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,12 +6,15 @@ version: 2 build: os: ubuntu-24.04 tools: + # Python 3.15 is not available yet on Read the Docs: this one is only used to + # bootstrap uv, which installs the Python 3.15 hatch then runs the build on python: '3.14' # custom commands to run mkdocs build within hatch, as suggested by maintainer in # https://github.com/readthedocs/readthedocs.org/issues/10706 commands: - - pip install hatch - - hatch run docs:build + - pip install hatch uv + - uv python install 3.15 + - HATCH_PYTHON="$(uv python find 3.15)" hatch run docs:build - mkdir --parents $READTHEDOCS_OUTPUT - mv site $READTHEDOCS_OUTPUT/html diff --git a/CHANGELOG.md b/CHANGELOG.md index f1a786f..44ec35f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- **Breaking**: support only Python 3.15 (#374) +- **Breaking**: require libzim 4, whose API changes reach `Creator` since it subclasses libzim's: camelCase arguments renamed (`config_nbworkers(nb_workers=)`, `set_mainpath(main_path=)`, `add_redirection(target_path=)`, `add_alias(target_path=)`), `add_illustration(content, width, height=None, scale=1.0)` signature (#374) +- **Breaking**: `Archive.get_search_results` yields libzim 4 `SearchResult` objects (with `path`, `title`, `snippet` and `archive`) instead of entry paths (#374) + +### Fixed + +- Depend on `pymupdf-lite` instead of `pymupdf`: same `pymupdf` module, without pymupdf 2.0 layout engine and the heavy dependencies (onnxruntime, numpy, ...) it pulled into every scraper; a scraper also depending on the full `pymupdf` must keep both at the same version (#372) + ## [5.5.0] - 2026-10-29 ### Added diff --git a/javascript/package.json b/javascript/package.json index bf1c47e..19e2ec0 100644 --- a/javascript/package.json +++ b/javascript/package.json @@ -1,7 +1,7 @@ { "name": "@openzim/wombat-setup", "type": "module", - "version": "5.5.1-dev0", + "version": "6.0.0-dev0", "license": "GPL-3.0-or-later", "author": "openZIM", "devDependencies": { diff --git a/pyproject.toml b/pyproject.toml index 5e65102..dec26a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,12 +1,12 @@ [build-system] # jinja2 is required to generate JS and Python rules at build time # PyYAML is used to parse fuzzy rules and generate Python/JS code -requires = ["hatchling", "hatch-openzim>=0.2", "jinja2==3.1.6", "PyYAML==6.0.3"] +requires = ["hatchling", "hatch-openzim>=0.4", "jinja2==3.1.6", "PyYAML==6.0.3"] build-backend = "hatchling.build" [project] name = "zimscraperlib" -requires-python = ">=3.14,<3.15" +requires-python = ">=3.15,<3.16" description = "Collection of python tools to re-use common code across scrapers" readme = "README.md" dependencies = [ @@ -16,7 +16,7 @@ dependencies = [ "python-resize-image>=1.1.19,<1.2", "Babel>=2.9,<3.0", "python-magic>=0.4.3,<0.5", - "libzim>=3.8.0,<4.0", + "libzim>=4.0.0,<5.0", "beautifulsoup4>=4.9.3,<5.0", "lxml>=4.6.3,<7.0", "optimize-images>=1.3.6,<3.0", @@ -24,7 +24,9 @@ dependencies = [ # promise is that they will never (or always) break the API, and the API is very # limited and we use only a very small subset of it. "regex>=2020.7.14", - "pymupdf>=1.24.0,<3.0", + # pymupdf-lite is the official build of pymupdf without its layout engine, and + # without the heavy dependencies (onnxruntime, numpy, ...) it brings since 2.0 + "pymupdf-lite>=2.0,<3.0", "CairoSVG>=2.2.0,<3.0", "beartype>=0.19,<0.24", # youtube-dl should be updated as frequently as possible @@ -155,7 +157,7 @@ serve = "inv docs-serve --args '{args}'" build = "inv docs-build --args '{args}'" [tool.ruff] -target-version = "py314" +target-version = "py315" line-length = 88 src = ["src", "contrib"] @@ -288,6 +290,6 @@ exclude_lines = [ include = ["contrib", "src", "tests", "tasks.py"] exclude = [".env/**", ".venv/**", ".hatch/**", "site/**"] extraPaths = ["src"] -pythonVersion = "3.14" +pythonVersion = "3.15" typeCheckingMode="strict" disableBytesTypePromotions = true diff --git a/src/zimscraperlib/__about__.py b/src/zimscraperlib/__about__.py index 1029f3b..36dab20 100644 --- a/src/zimscraperlib/__about__.py +++ b/src/zimscraperlib/__about__.py @@ -1 +1 @@ -__version__ = "5.5.1-dev0" +__version__ = "6.0.0-dev0" diff --git a/src/zimscraperlib/rewriting/js_ast.py b/src/zimscraperlib/rewriting/js_ast.py index 2f66254..96259b0 100644 --- a/src/zimscraperlib/rewriting/js_ast.py +++ b/src/zimscraperlib/rewriting/js_ast.py @@ -100,7 +100,11 @@ def parse_top_level(text: str) -> TopLevel | None: here may throw into a scrape.""" try: source = text.encode("utf-8") - root = _PARSER.parse(source).root_node + # tree-sitter's stub types `source` as collections.abc.ByteString, which + # typeshed no longer has since Python 3.14 + root = _PARSER.parse( # pyright: ignore[reportUnknownMemberType] + source + ).root_node declarations: list[Declaration] = [] has_document_write = False for node in root.named_children: diff --git a/src/zimscraperlib/zim/archive.py b/src/zimscraperlib/zim/archive.py index 2e45676..948b3a9 100644 --- a/src/zimscraperlib/zim/archive.py +++ b/src/zimscraperlib/zim/archive.py @@ -57,10 +57,6 @@ def get_text_metadata(self, name: str) -> str: """Decoded value of a text metadata""" return super().get_metadata(name).decode("UTF-8") - def get_entry_by_id(self, id_: int) -> libzim.reader.Entry: - """Entry from its Id in ZIM""" - return self._get_entry_by_id(id_) - def get_item(self, path: str) -> libzim.reader.Item: """Item from a path""" return self.get_entry_by_path(path).get_item() @@ -75,31 +71,31 @@ def get_suggestions( """paths iterator over suggestion matches for query""" suggestion = libzim.suggestion.SuggestionSearcher(self).suggest(query) if end is None: - end = suggestion.getEstimatedMatches() - return suggestion.getResults(start, end) + end = suggestion.get_estimated_matches() + return suggestion.get_results(start, end) def get_suggestions_count(self, query: str) -> int: """Estimated number of suggestion matches for query""" suggestion = libzim.suggestion.SuggestionSearcher(self).suggest(query) - return suggestion.getEstimatedMatches() + return suggestion.get_estimated_matches() def get_search_results( self, query: str, start: int = 0, end: int | None = None - ) -> Iterable[str]: - """paths iterator over search results for query""" + ) -> Iterable[libzim.search.SearchResult]: + """SearchResult iterator over search results for query""" search = libzim.search.Searcher(self).search( libzim.search.Query().set_query(query) ) if end is None: - end = search.getEstimatedMatches() - return search.getResults(start, end) + end = search.get_estimated_matches() + return search.get_results(start, end) def get_search_results_count(self, query: str) -> int: """Estimated number of search results for query""" search = libzim.search.Searcher(self).search( libzim.search.Query().set_query(query) ) - return search.getEstimatedMatches() + return search.get_estimated_matches() @property def counters(self) -> CounterMap: diff --git a/src/zimscraperlib/zim/creator.py b/src/zimscraperlib/zim/creator.py index c6dfd5b..6deb06b 100644 --- a/src/zimscraperlib/zim/creator.py +++ b/src/zimscraperlib/zim/creator.py @@ -245,7 +245,11 @@ def start(self): for metadata in self._metadata.values(): if isinstance(metadata, IllustrationBasedMetadata): - self.add_illustration(metadata.illustration_size, metadata.libzim_value) + self.add_illustration( + metadata.libzim_value, + width=metadata.illustration_size, + height=metadata.illustration_size, + ) else: self.add_metadata(metadata) self._metadata.clear() diff --git a/src/zimscraperlib/zim/dedup.py b/src/zimscraperlib/zim/dedup.py index 7f9f2cf..8dd43e2 100644 --- a/src/zimscraperlib/zim/dedup.py +++ b/src/zimscraperlib/zim/dedup.py @@ -69,7 +69,7 @@ def add_item_for( if existing_item := self.added_items.get(digest): self.creator.add_alias( path, - targetPath=existing_item, + target_path=existing_item, title=title or path, hints={Hint.FRONT_ARTICLE: True} if kwargs.get("is_front") else {}, ) diff --git a/src/zimscraperlib/zim/indexing.py b/src/zimscraperlib/zim/indexing.py index e1b0c01..24e0412 100644 --- a/src/zimscraperlib/zim/indexing.py +++ b/src/zimscraperlib/zim/indexing.py @@ -4,12 +4,7 @@ import pathlib import libzim.writer # pyright: ignore[reportMissingModuleSource] - -try: - import pymupdf # pyright: ignore[reportMissingTypeStubs] -except ImportError: # pragma: no cover - # pymupdf main module was named fitz before 1.24.3 - import fitz as pymupdf # pyright: ignore[reportMissingTypeStubs] +import pymupdf # pyright: ignore[reportMissingTypeStubs] from zimscraperlib import logger diff --git a/tests/zim/test_archive.py b/tests/zim/test_archive.py index fbeed26..095c051 100644 --- a/tests/zim/test_archive.py +++ b/tests/zim/test_archive.py @@ -64,16 +64,20 @@ def test_search_no_fti(small_zim_file: pathlib.Path): def test_search(real_zim_file: pathlib.Path): with Archive(real_zim_file) as zim: assert zim.get_search_results_count("test") > 0 - assert "A/Diesel_emissions_scandal" in list(zim.get_search_results("test")) + results = list(zim.get_search_results("test")) + assert "A/Diesel_emissions_scandal" in [result.path for result in results] + result = next(r for r in results if r.path == "A/Diesel_emissions_scandal") + assert result.title + assert result.archive.filename == zim.filename @pytest.mark.slow def test_search_end_index(real_zim_file: pathlib.Path): with Archive(real_zim_file) as zim: assert list(zim.get_search_results("test", end=0)) == [] - assert "A/Diesel_emissions_scandal" in list( - zim.get_search_results("test", end=1) - ) + assert "A/Diesel_emissions_scandal" in [ + result.path for result in zim.get_search_results("test", end=1) + ] def test_counters(small_zim_file: pathlib.Path):