From 00785656393cfe9b373662a7c98b0766c55eb38d Mon Sep 17 00:00:00 2001 From: benoit74 Date: Fri, 30 Oct 2026 10:10:46 +0000 Subject: [PATCH] Add Archive iterators and prefix search Add Archive.iter_by_path(), iter_by_title(), iter_efficient(), __iter__() (alias for iter_efficient()), find_by_path() and find_by_title(), wrapping libzim's Archive::iterByPath/iterByTitle/iterEfficient/findByPath/findByTitle. This removes the need to loop over entries via get_entry_by_id() and all_entry_count, as previously required (fixes #94). --- CHANGELOG.md | 1 + README.md | 4 ++ libzim/libwrapper.h | 39 ++++++++++++++ libzim/libzim.pyx | 101 ++++++++++++++++++++++++++++++++++++ libzim/reader.pyi | 7 +++ libzim/zim.pxd | 45 ++++++++++++++++ tests/test_libzim_reader.py | 80 ++++++++++++++++++++++++++++ 7 files changed, 277 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34fb361b..8f1e40e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Add Android wheels support via cibuildwheel for arm64_v8a and x86_64 with CPython 3.13+; these wheels are reader-only since libzim is compiled without writer support on Android (#167) +- Add `Archive.iter_by_path()`, `Archive.iter_by_title()`, `Archive.iter_efficient()`, `Archive.__iter__()` (alias for `iter_efficient()`), `Archive.find_by_path()` and `Archive.find_by_title()` to iterate over entries without `get_entry_by_id()` (#94, #294) ### Changed diff --git a/README.md b/README.md index d44132d9..468f6052 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,10 @@ entry = zim.get_entry_by_path("home/fr") print(f"Entry {entry.title} at {entry.path} is {entry.get_item().size}b.") print(bytes(entry.get_item().content).decode("UTF-8")) +# iterating over all entries (in the most efficient order to read them) +for entry in zim: + print(entry.path, entry.title) + # searching using full-text index search_string = "Welcome" query = Query().set_query(search_string) diff --git a/libzim/libwrapper.h b/libzim/libwrapper.h index 92a89144..96265515 100644 --- a/libzim/libwrapper.h +++ b/libzim/libwrapper.h @@ -138,6 +138,40 @@ class Entry : public Wrapper FORWARD(zim::entry_index_type, getIndex) }; +// Wrapping of the templated zim::Archive::iterator / zim::Archive::EntryRange. +// Both lack a default constructor, so (as explained above) they must go through Wrapper. +// The template parameter (a zim::EntryOrder, i.e. a non-type parameter) is resolved at +// compile time; we only need one concrete type per order, named below for Cython to pick up. +template +class EntryIterator : public Wrapper> +{ + public: + EntryIterator() = default; + EntryIterator(const zim::Archive::iterator& o) : Wrapper>(o) {}; + bool operator==(const EntryIterator& it) const { return *this->mp_base == *it.mp_base; } + bool operator!=(const EntryIterator& it) const { return *this->mp_base != *it.mp_base; } + EntryIterator& operator++() { ++(*this->mp_base); return *this; } + wrapper::Entry getEntry() const { return this->mp_base->operator*(); } +}; + +template +class EntryRange : public Wrapper> +{ + public: + EntryRange() = default; + EntryRange(const zim::Archive::EntryRange& o) : Wrapper>(o) {}; + EntryIterator begin() const { return this->mp_base->begin(); } + EntryIterator end() const { return this->mp_base->end(); } + int size() const { return this->mp_base->size(); } +}; + +typedef EntryIterator PathEntryIterator; +typedef EntryRange PathEntryRange; +typedef EntryIterator TitleEntryIterator; +typedef EntryRange TitleEntryRange; +typedef EntryIterator EfficientEntryIterator; +typedef EntryRange EfficientEntryRange; + class Archive : public Wrapper { public: @@ -152,6 +186,11 @@ class Archive : public Wrapper FORWARD(wrapper::Entry, getEntryByTitle) FORWARD(wrapper::Entry, getMainEntry) FORWARD(wrapper::Entry, getRandomEntry) + FORWARD(wrapper::PathEntryRange, iterByPath) + FORWARD(wrapper::TitleEntryRange, iterByTitle) + FORWARD(wrapper::EfficientEntryRange, iterEfficient) + FORWARD(wrapper::PathEntryRange, findByPath) + FORWARD(wrapper::TitleEntryRange, findByTitle) FORWARD(wrapper::Item, getIllustrationItem) FORWARD(std::set, getIllustrationSizes) FORWARD(zim::Archive::IllustrationInfos, getIllustrationInfos) diff --git a/libzim/libzim.pyx b/libzim/libzim.pyx index e8f61da9..5f60b50b 100644 --- a/libzim/libzim.pyx +++ b/libzim/libzim.pyx @@ -1326,6 +1326,107 @@ cdef class Archive: cdef zim.Entry entry = move(self.c_archive.getEntryByPath(entry_id)) return Entry.from_entry(move(entry)) + def iter_by_path(self) -> Iterator[Entry]: + """Iterate over all user entries, in path order. + + If Archive doesn't support “user entries” then this iterates on all + entries in the ZIM file. + + Yields: + Entry: Each entry, in path order. + """ + cdef zim.PathEntryRange c_range = move(self.c_archive.iterByPath()) + cdef zim.PathEntryIterator current = c_range.begin() + cdef zim.PathEntryIterator end = c_range.end() + while current != end: + yield Entry.from_entry(move(current.getEntry())) + preincrement(current) + + def iter_by_title(self) -> Iterator[Entry]: + """Iterate over all articles, in title order. + + The definition of "article" depends on the ZIM archive: on recent + archives this is all entries marked as `FRONT_ARTICLE` at creation + time; on old archives this is all entries in the `A` namespace. + + Yields: + Entry: Each article, in title order. + """ + cdef zim.TitleEntryRange c_range = move(self.c_archive.iterByTitle()) + cdef zim.TitleEntryIterator current = c_range.begin() + cdef zim.TitleEntryIterator end = c_range.end() + while current != end: + yield Entry.from_entry(move(current.getEntry())) + preincrement(current) + + def iter_efficient(self) -> Iterator[Entry]: + """Iterate over all user entries, in the most efficient order to read them. + + If Archive doesn't support “user entries” then this iterates on all + entries in the ZIM file. + + Yields: + Entry: Each entry, in an unspecified but efficient-to-read order. + """ + cdef zim.EfficientEntryRange c_range = move(self.c_archive.iterEfficient()) + cdef zim.EfficientEntryIterator current = c_range.begin() + cdef zim.EfficientEntryIterator end = c_range.end() + while current != end: + yield Entry.from_entry(move(current.getEntry())) + preincrement(current) + + def __iter__(self) -> Iterator[Entry]: + """Iterate over all user entries, in the most efficient order to read them. + + Equivalent to :meth:`iter_efficient`. + + Yields: + Entry: Each entry, in an unspecified but efficient-to-read order. + """ + return self.iter_efficient() + + def find_by_path(self, path: str) -> Iterator[Entry]: + """Iterate over all entries whose path starts with `path`. + + Unlike `get_entry_by_path`, there is no compatibility layer: `path` + must follow the archive's namespace scheme (see `has_new_namespace_scheme`). + + Args: + path: The path prefix to search for. + + Yields: + Entry: Each entry whose path starts with `path`, in path order. + """ + cdef zim.PathEntryRange c_range = move( + self.c_archive.findByPath(path.encode('UTF-8')) + ) + cdef zim.PathEntryIterator current = c_range.begin() + cdef zim.PathEntryIterator end = c_range.end() + while current != end: + yield Entry.from_entry(move(current.getEntry())) + preincrement(current) + + def find_by_title(self, title: str) -> Iterator[Entry]: + """Iterate over all articles whose title starts with `title`. + + Case and diacritic sensitive; cannot find a pattern in the middle of + a title (only prefixes). + + Args: + title: The title prefix to search for. + + Yields: + Entry: Each article whose title starts with `title`, in title order. + """ + cdef zim.TitleEntryRange c_range = move( + self.c_archive.findByTitle(title.encode('UTF-8')) + ) + cdef zim.TitleEntryIterator current = c_range.begin() + cdef zim.TitleEntryIterator end = c_range.end() + while current != end: + yield Entry.from_entry(move(current.getEntry())) + preincrement(current) + @property def has_main_entry(self) -> pybool: """Whether Archive has a main entry set. diff --git a/libzim/reader.pyi b/libzim/reader.pyi index 54ef4079..f7afc4f5 100644 --- a/libzim/reader.pyi +++ b/libzim/reader.pyi @@ -1,6 +1,7 @@ from __future__ import annotations import pathlib +from collections.abc import Iterator from uuid import UUID from typing_extensions import disjoint_base @@ -63,6 +64,12 @@ class Archive: def get_metadata_item(self, name: str) -> Item: ... def get_metadata(self, name: str) -> bytes: ... def get_entry_by_id(self, entry_id: int) -> Entry: ... + def iter_by_path(self) -> Iterator[Entry]: ... + def iter_by_title(self) -> Iterator[Entry]: ... + def iter_efficient(self) -> Iterator[Entry]: ... + def __iter__(self) -> Iterator[Entry]: ... + def find_by_path(self, path: str) -> Iterator[Entry]: ... + def find_by_title(self, title: str) -> Iterator[Entry]: ... @property def has_main_entry(self) -> bool: ... @property diff --git a/libzim/zim.pxd b/libzim/zim.pxd index 26bbb68c..73a6420d 100644 --- a/libzim/zim.pxd +++ b/libzim/zim.pxd @@ -145,6 +145,45 @@ cdef extern from "libwrapper.h" namespace "wrapper": int getIndex() except + + cdef cppclass PathEntryIterator: + PathEntryIterator() + PathEntryIterator operator++() + bint operator==(PathEntryIterator) + bint operator!=(PathEntryIterator) + Entry getEntry() except + + + cdef cppclass PathEntryRange: + PathEntryRange() + PathEntryIterator begin() + PathEntryIterator end() + int size() + + cdef cppclass TitleEntryIterator: + TitleEntryIterator() + TitleEntryIterator operator++() + bint operator==(TitleEntryIterator) + bint operator!=(TitleEntryIterator) + Entry getEntry() except + + + cdef cppclass TitleEntryRange: + TitleEntryRange() + TitleEntryIterator begin() + TitleEntryIterator end() + int size() + + cdef cppclass EfficientEntryIterator: + EfficientEntryIterator() + EfficientEntryIterator operator++() + bint operator==(EfficientEntryIterator) + bint operator!=(EfficientEntryIterator) + Entry getEntry() except + + + cdef cppclass EfficientEntryRange: + EfficientEntryRange() + EfficientEntryIterator begin() + EfficientEntryIterator end() + int size() + cdef cppclass Item: string getTitle() except + string getPath() except + @@ -168,6 +207,12 @@ cdef extern from "libwrapper.h" namespace "wrapper": Entry getEntryByPath(entry_index_type idx) except + Entry getEntryByTitle(string title) except + + PathEntryRange iterByPath() except + + TitleEntryRange iterByTitle() except + + EfficientEntryRange iterEfficient() except + + PathEntryRange findByPath(string path) except + + TitleEntryRange findByTitle(string title) except + + string getMetadata(string name) except + Item getMetadataItem(string name) except + vector[string] getMetadataKeys() except + diff --git a/tests/test_libzim_reader.py b/tests/test_libzim_reader.py index b813be2c..7c6ee262 100644 --- a/tests/test_libzim_reader.py +++ b/tests/test_libzim_reader.py @@ -596,6 +596,86 @@ def test_reader_by_id(all_zims, filename): assert zim.get_entry_by_id(index).get_item()._index >= 0 +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename"])) +def test_reader_iter_by_path(all_zims, filename): + zim = Archive(all_zims / filename) + + entries = list(zim.iter_by_path()) + assert len(entries) == zim.entry_count + paths = [entry.path for entry in entries] + assert paths == sorted(paths) + assert len(set(paths)) == len(paths) + + assert {zim.get_entry_by_id(index).path for index in range(zim.entry_count)} == set( + paths + ) + + +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename"])) +def test_reader_iter_by_title(all_zims, filename): + zim = Archive(all_zims / filename) + + entries = list(zim.iter_by_title()) + titles = [entry.title for entry in entries] + assert titles == sorted(titles) + assert len({entry.path for entry in entries}) == len(entries) + + +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename"])) +def test_reader_iter_efficient(all_zims, filename): + zim = Archive(all_zims / filename) + + entries = list(zim.iter_efficient()) + assert len(entries) == zim.entry_count + assert {entry.path for entry in entries} == { + entry.path for entry in zim.iter_by_path() + } + + +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename"])) +def test_reader_dunder_iter(all_zims, filename): + zim = Archive(all_zims / filename) + + assert [entry.path for entry in zim] == [ + entry.path for entry in zim.iter_efficient() + ] + + +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename", "test_path"])) +def test_reader_find_by_path(all_zims, filename, test_path): + zim = Archive(all_zims / filename) + + assert list(zim.find_by_path("___missing_prefix___")) == [] + + if test_path: + prefix = test_path[:3] + found = list(zim.find_by_path(prefix)) + assert found + assert all(entry.path.startswith(prefix) for entry in found) + assert test_path in {entry.path for entry in found} + + +@skip_if_offline +@pytest.mark.parametrize(*parametrize_for(["filename", "test_title"])) +def test_reader_find_by_title(all_zims, filename, test_title): + zim = Archive(all_zims / filename) + + assert list(zim.find_by_title("___missing_prefix___")) == [] + + # example.zim cannot be queried by title, see test_reader_get_entries + if test_title and filename != "example.zim": + prefix = test_title[:3] + found = list(zim.find_by_title(prefix)) + assert found + assert all(entry.title.startswith(prefix) for entry in found) + assert test_title in {entry.title for entry in found} + + @skip_if_offline def test_archive_equality(all_zims): class Different: