Add Archive iterators and prefix search

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Add Android wheels support via cibuildwheel for arm64_v8a and x86_64 with CPython 3.13+; these wheels are reader-only since libzim is compiled without writer support on Android (#167)
+- Add `Archive.iter_by_path()`, `Archive.iter_by_title()`, `Archive.iter_efficient()`, `Archive.__iter__()` (alias for `iter_efficient()`), `Archive.find_by_path()` and `Archive.find_by_title()` to iterate over entries without `get_entry_by_id()` (#94, #294)
 
 ### Changed
 

README.md

@@ -61,6 +61,10 @@ entry = zim.get_entry_by_path("home/fr")
 print(f"Entry {entry.title} at {entry.path} is {entry.get_item().size}b.")
 print(bytes(entry.get_item().content).decode("UTF-8"))
 
+# iterating over all entries (in the most efficient order to read them)
+for entry in zim:
+    print(entry.path, entry.title)
+
 # searching using full-text index
 search_string = "Welcome"
 query = Query().set_query(search_string)

libzim/libwrapper.h

@@ -138,6 +138,40 @@ class Entry : public Wrapper<zim::Entry>
     FORWARD(zim::entry_index_type, getIndex)
 };
 
+// Wrapping of the templated zim::Archive::iterator<order> / zim::Archive::EntryRange<order>.
+// Both lack a default constructor, so (as explained above) they must go through Wrapper.
+// The template parameter (a zim::EntryOrder, i.e. a non-type parameter) is resolved at
+// compile time; we only need one concrete type per order, named below for Cython to pick up.
+template<zim::EntryOrder order>
+class EntryIterator : public Wrapper<zim::Archive::iterator<order>>
+{
+  public:
+    EntryIterator() = default;
+    EntryIterator(const zim::Archive::iterator<order>& o) : Wrapper<zim::Archive::iterator<order>>(o) {};
+    bool operator==(const EntryIterator& it) const { return *this->mp_base == *it.mp_base; }
+    bool operator!=(const EntryIterator& it) const { return *this->mp_base != *it.mp_base; }
+    EntryIterator& operator++() { ++(*this->mp_base); return *this; }
+    wrapper::Entry getEntry() const { return this->mp_base->operator*(); }
+};
+
+template<zim::EntryOrder order>
+class EntryRange : public Wrapper<zim::Archive::EntryRange<order>>
+{
+  public:
+    EntryRange() = default;
+    EntryRange(const zim::Archive::EntryRange<order>& o) : Wrapper<zim::Archive::EntryRange<order>>(o) {};
+    EntryIterator<order> begin() const { return this->mp_base->begin(); }
+    EntryIterator<order> end() const { return this->mp_base->end(); }
+    int size() const { return this->mp_base->size(); }
+};
+
+typedef EntryIterator<zim::EntryOrder::pathOrder> PathEntryIterator;
+typedef EntryRange<zim::EntryOrder::pathOrder> PathEntryRange;
+typedef EntryIterator<zim::EntryOrder::titleOrder> TitleEntryIterator;
+typedef EntryRange<zim::EntryOrder::titleOrder> TitleEntryRange;
+typedef EntryIterator<zim::EntryOrder::efficientOrder> EfficientEntryIterator;
+typedef EntryRange<zim::EntryOrder::efficientOrder> EfficientEntryRange;
+
 class Archive : public Wrapper<zim::Archive>
 {
   public:
@@ -152,6 +186,11 @@ class Archive : public Wrapper<zim::Archive>
     FORWARD(wrapper::Entry, getEntryByTitle)
     FORWARD(wrapper::Entry, getMainEntry)
     FORWARD(wrapper::Entry, getRandomEntry)
+    FORWARD(wrapper::PathEntryRange, iterByPath)
+    FORWARD(wrapper::TitleEntryRange, iterByTitle)
+    FORWARD(wrapper::EfficientEntryRange, iterEfficient)
+    FORWARD(wrapper::PathEntryRange, findByPath)
+    FORWARD(wrapper::TitleEntryRange, findByTitle)
     FORWARD(wrapper::Item, getIllustrationItem)
     FORWARD(std::set<unsigned int>, getIllustrationSizes)
     FORWARD(zim::Archive::IllustrationInfos, getIllustrationInfos)

libzim/libzim.pyx

@@ -1326,6 +1326,107 @@ cdef class Archive:
         cdef zim.Entry entry = move(self.c_archive.getEntryByPath(<zim.entry_index_type>entry_id))
         return Entry.from_entry(move(entry))
 
+    def iter_by_path(self) -> Iterator[Entry]:
+        """Iterate over all user entries, in path order.
+
+        If Archive doesn't support “user entries” then this iterates on all
+        entries in the ZIM file.
+
+        Yields:
+            Entry: Each entry, in path order.
+        """
+        cdef zim.PathEntryRange c_range = move(self.c_archive.iterByPath())
+        cdef zim.PathEntryIterator current = c_range.begin()
+        cdef zim.PathEntryIterator end = c_range.end()
+        while current != end:
+            yield Entry.from_entry(move(current.getEntry()))
+            preincrement(current)
+
+    def iter_by_title(self) -> Iterator[Entry]:
+        """Iterate over all articles, in title order.
+
+        The definition of "article" depends on the ZIM archive: on recent
+        archives this is all entries marked as `FRONT_ARTICLE` at creation
+        time; on old archives this is all entries in the `A` namespace.
+
+        Yields:
+            Entry: Each article, in title order.
+        """
+        cdef zim.TitleEntryRange c_range = move(self.c_archive.iterByTitle())
+        cdef zim.TitleEntryIterator current = c_range.begin()
+        cdef zim.TitleEntryIterator end = c_range.end()
+        while current != end:
+            yield Entry.from_entry(move(current.getEntry()))
+            preincrement(current)
+
+    def iter_efficient(self) -> Iterator[Entry]:
+        """Iterate over all user entries, in the most efficient order to read them.
+
+        If Archive doesn't support “user entries” then this iterates on all
+        entries in the ZIM file.
+
+        Yields:
+            Entry: Each entry, in an unspecified but efficient-to-read order.
+        """
+        cdef zim.EfficientEntryRange c_range = move(self.c_archive.iterEfficient())
+        cdef zim.EfficientEntryIterator current = c_range.begin()
+        cdef zim.EfficientEntryIterator end = c_range.end()
+        while current != end:
+            yield Entry.from_entry(move(current.getEntry()))
+            preincrement(current)
+
+    def __iter__(self) -> Iterator[Entry]:
+        """Iterate over all user entries, in the most efficient order to read them.
+
+        Equivalent to :meth:`iter_efficient`.
+
+        Yields:
+            Entry: Each entry, in an unspecified but efficient-to-read order.
+        """
+        return self.iter_efficient()
+
+    def find_by_path(self, path: str) -> Iterator[Entry]:
+        """Iterate over all entries whose path starts with `path`.
+
+        Unlike `get_entry_by_path`, there is no compatibility layer: `path`
+        must follow the archive's namespace scheme (see `has_new_namespace_scheme`).
+
+        Args:
+            path: The path prefix to search for.
+
+        Yields:
+            Entry: Each entry whose path starts with `path`, in path order.
+        """
+        cdef zim.PathEntryRange c_range = move(
+            self.c_archive.findByPath(<string>path.encode('UTF-8'))
+        )
+        cdef zim.PathEntryIterator current = c_range.begin()
+        cdef zim.PathEntryIterator end = c_range.end()
+        while current != end:
+            yield Entry.from_entry(move(current.getEntry()))
+            preincrement(current)
+
+    def find_by_title(self, title: str) -> Iterator[Entry]:
+        """Iterate over all articles whose title starts with `title`.
+
+        Case and diacritic sensitive; cannot find a pattern in the middle of
+        a title (only prefixes).
+
+        Args:
+            title: The title prefix to search for.
+
+        Yields:
+            Entry: Each article whose title starts with `title`, in title order.
+        """
+        cdef zim.TitleEntryRange c_range = move(
+            self.c_archive.findByTitle(<string>title.encode('UTF-8'))
+        )
+        cdef zim.TitleEntryIterator current = c_range.begin()
+        cdef zim.TitleEntryIterator end = c_range.end()
+        while current != end:
+            yield Entry.from_entry(move(current.getEntry()))
+            preincrement(current)
+
     @property
     def has_main_entry(self) -> pybool:
         """Whether Archive has a main entry set.

libzim/reader.pyi

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pathlib
+from collections.abc import Iterator
 from uuid import UUID
 
 from typing_extensions import disjoint_base
@@ -63,6 +64,12 @@ class Archive:
     def get_metadata_item(self, name: str) -> Item: ...
     def get_metadata(self, name: str) -> bytes: ...
     def get_entry_by_id(self, entry_id: int) -> Entry: ...
+    def iter_by_path(self) -> Iterator[Entry]: ...
+    def iter_by_title(self) -> Iterator[Entry]: ...
+    def iter_efficient(self) -> Iterator[Entry]: ...
+    def __iter__(self) -> Iterator[Entry]: ...
+    def find_by_path(self, path: str) -> Iterator[Entry]: ...
+    def find_by_title(self, title: str) -> Iterator[Entry]: ...
     @property
     def has_main_entry(self) -> bool: ...
     @property

libzim/zim.pxd

@@ -145,6 +145,45 @@ cdef extern from "libwrapper.h" namespace "wrapper":
 
         int getIndex() except +
 
+    cdef cppclass PathEntryIterator:
+        PathEntryIterator()
+        PathEntryIterator operator++()
+        bint operator==(PathEntryIterator)
+        bint operator!=(PathEntryIterator)
+        Entry getEntry() except +
+
+    cdef cppclass PathEntryRange:
+        PathEntryRange()
+        PathEntryIterator begin()
+        PathEntryIterator end()
+        int size()
+
+    cdef cppclass TitleEntryIterator:
+        TitleEntryIterator()
+        TitleEntryIterator operator++()
+        bint operator==(TitleEntryIterator)
+        bint operator!=(TitleEntryIterator)
+        Entry getEntry() except +
+
+    cdef cppclass TitleEntryRange:
+        TitleEntryRange()
+        TitleEntryIterator begin()
+        TitleEntryIterator end()
+        int size()
+
+    cdef cppclass EfficientEntryIterator:
+        EfficientEntryIterator()
+        EfficientEntryIterator operator++()
+        bint operator==(EfficientEntryIterator)
+        bint operator!=(EfficientEntryIterator)
+        Entry getEntry() except +
+
+    cdef cppclass EfficientEntryRange:
+        EfficientEntryRange()
+        EfficientEntryIterator begin()
+        EfficientEntryIterator end()
+        int size()
+
     cdef cppclass Item:
         string getTitle() except +
         string getPath() except +
@@ -168,6 +207,12 @@ cdef extern from "libwrapper.h" namespace "wrapper":
         Entry getEntryByPath(entry_index_type idx) except +
         Entry getEntryByTitle(string title) except +
 
+        PathEntryRange iterByPath() except +
+        TitleEntryRange iterByTitle() except +
+        EfficientEntryRange iterEfficient() except +
+        PathEntryRange findByPath(string path) except +
+        TitleEntryRange findByTitle(string title) except +
+
         string getMetadata(string name) except +
         Item getMetadataItem(string name) except +
         vector[string] getMetadataKeys() except +

tests/test_libzim_reader.py

@@ -596,6 +596,86 @@ def test_reader_by_id(all_zims, filename):
         assert zim.get_entry_by_id(index).get_item()._index >= 0
 
 
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename"]))
+def test_reader_iter_by_path(all_zims, filename):
+    zim = Archive(all_zims / filename)
+
+    entries = list(zim.iter_by_path())
+    assert len(entries) == zim.entry_count
+    paths = [entry.path for entry in entries]
+    assert paths == sorted(paths)
+    assert len(set(paths)) == len(paths)
+
+    assert {zim.get_entry_by_id(index).path for index in range(zim.entry_count)} == set(
+        paths
+    )
+
+
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename"]))
+def test_reader_iter_by_title(all_zims, filename):
+    zim = Archive(all_zims / filename)
+
+    entries = list(zim.iter_by_title())
+    titles = [entry.title for entry in entries]
+    assert titles == sorted(titles)
+    assert len({entry.path for entry in entries}) == len(entries)
+
+
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename"]))
+def test_reader_iter_efficient(all_zims, filename):
+    zim = Archive(all_zims / filename)
+
+    entries = list(zim.iter_efficient())
+    assert len(entries) == zim.entry_count
+    assert {entry.path for entry in entries} == {
+        entry.path for entry in zim.iter_by_path()
+    }
+
+
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename"]))
+def test_reader_dunder_iter(all_zims, filename):
+    zim = Archive(all_zims / filename)
+
+    assert [entry.path for entry in zim] == [
+        entry.path for entry in zim.iter_efficient()
+    ]
+
+
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename", "test_path"]))
+def test_reader_find_by_path(all_zims, filename, test_path):
+    zim = Archive(all_zims / filename)
+
+    assert list(zim.find_by_path("___missing_prefix___")) == []
+
+    if test_path:
+        prefix = test_path[:3]
+        found = list(zim.find_by_path(prefix))
+        assert found
+        assert all(entry.path.startswith(prefix) for entry in found)
+        assert test_path in {entry.path for entry in found}
+
+
+@skip_if_offline
+@pytest.mark.parametrize(*parametrize_for(["filename", "test_title"]))
+def test_reader_find_by_title(all_zims, filename, test_title):
+    zim = Archive(all_zims / filename)
+
+    assert list(zim.find_by_title("___missing_prefix___")) == []
+
+    # example.zim cannot be queried by title, see test_reader_get_entries
+    if test_title and filename != "example.zim":
+        prefix = test_title[:3]
+        found = list(zim.find_by_title(prefix))
+        assert found
+        assert all(entry.title.startswith(prefix) for entry in found)
+        assert test_title in {entry.title for entry in found}
+
+
 @skip_if_offline
 def test_archive_equality(all_zims):
     class Different: