From 07c9371f8718e7d647b49cb5b110cc223c20e68f Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 10:40:36 +0200 Subject: [PATCH 1/2] Make Jobs.iter return all jobs, not only the first 1000 --- scrapinghub/client/jobs.py | 39 +++++++++++++++++++++++++++++--------- tests/client/test_jobs.py | 21 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 tests/client/test_jobs.py diff --git a/scrapinghub/client/jobs.py b/scrapinghub/client/jobs.py index 75d0acbc..a7e2b6c8 100644 --- a/scrapinghub/client/jobs.py +++ b/scrapinghub/client/jobs.py @@ -17,6 +17,9 @@ from .utils import get_tags_for_update, parse_job_key, update_kwargs +_PAGE_SIZE = 1000 + + class Jobs(object): """Class representing a collection of jobs for a project/spider. @@ -142,6 +145,9 @@ def iter(self, count=None, start=None, spider=None, state=None, meta=None, **params): """Iterate over jobs collection for a given set of params. + .. versionchanged:: VERSION + Returns all matching jobs, instead of at most 1000. + :param count: (optional) limit amount of returned jobs. :param start: (optional) number of jobs to skip in the beginning. :param spider: (optional) filter by spider name. @@ -186,11 +192,6 @@ def iter(self, count=None, start=None, spider=None, state=None, >>> jobs_summary = project.jobs.iter(meta=['scheduled_by', ]) - - by default :meth:`Jobs.iter` returns maximum last 1000 results. - Pagination is available using start parameter:: - - >>> jobs_summary = spider.jobs.iter(start=1000) - - get jobs filtered by tags (list of tags has ``OR`` power):: >>> jobs_summary = project.jobs.iter( @@ -201,12 +202,32 @@ def iter(self, count=None, start=None, spider=None, state=None, >>> jobs_summary = project.jobs.iter( ... spider='spider2', state='finished', count=3) """ - update_kwargs(params, count=count, start=start, jobmeta=meta, - spider=spider, state=state, has_tag=has_tag, - lacks_tag=lacks_tag, startts=startts, endts=endts) + update_kwargs(params, jobmeta=meta, spider=spider, state=state, + has_tag=has_tag, lacks_tag=lacks_tag, startts=startts, + endts=endts) if self.spider: params['spider'] = self.spider.name - return self._project.jobq.list(**params) + return self._iter_pages(count, start, params) + + def _iter_pages(self, count, start, params): + if start is not None: + params['start'] = start + if count is not None: + params['count'] = min(count, _PAGE_SIZE) + while True: + received = 0 + for job in self._project.jobq.list(**params): + received += 1 + yield job + if count is not None: + count -= received + if count <= 0: + return + if received < _PAGE_SIZE: + return + params['start'] = params.get('start', 0) + received + params['count'] = (_PAGE_SIZE if count is None + else min(count, _PAGE_SIZE)) def list(self, count=None, start=None, spider=None, state=None, has_tag=None, lacks_tag=None, startts=None, endts=None, diff --git a/tests/client/test_jobs.py b/tests/client/test_jobs.py new file mode 100644 index 00000000..e9af3555 --- /dev/null +++ b/tests/client/test_jobs.py @@ -0,0 +1,21 @@ +import mock + +from scrapinghub.client.jobs import Jobs + + +def _mock_jobs(total): + def list_(start=0, count=None, **params): + end = total if count is None else min(total, start + count) + return iter(range(start, end)) + + jobs = Jobs(mock.Mock(), '1') + jobs._project.jobq.list.side_effect = list_ + return jobs + + +def test_iter_paginates(): + jobs = _mock_jobs(2500) + assert list(jobs.iter()) == list(range(2500)) + assert list(jobs.iter(start=500, count=1200)) == list(range(500, 1700)) + assert list(jobs.iter(count=3)) == [0, 1, 2] + assert list(_mock_jobs(1000).iter()) == list(range(1000)) From 77a73a363b34e1738474028f381a7c37884069ee Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 12:09:18 +0200 Subject: [PATCH 2/2] Drop the outdated 1000-job limit note from the overview docs --- docs/use/overview.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/use/overview.rst b/docs/use/overview.rst index 528ad213..ca6c1269 100644 --- a/docs/use/overview.rst +++ b/docs/use/overview.rst @@ -232,12 +232,6 @@ even default ones:: >>> job_summary.get('spider', 'missing') missing -By default ``.jobs.iter()`` returns the last 1000 jobs at most. -To get more than the last 1000, you need to paginate through results -in batches, using the ``start`` parameter:: - - >>> jobs_summary = spider.jobs.iter(start=1000) - There are several filters like ``spider``, ``state``, ``has_tag``, ``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more details).