Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions docs/use/overview.rst
Original file line number Diff line number Diff line change
Expand Up @@ -232,12 +232,6 @@ even default ones::
>>> job_summary.get('spider', 'missing')
missing

By default ``.jobs.iter()`` returns the last 1000 jobs at most.
To get more than the last 1000, you need to paginate through results
in batches, using the ``start`` parameter::

>>> jobs_summary = spider.jobs.iter(start=1000)

There are several filters like ``spider``, ``state``, ``has_tag``,
``lacks_tag``, ``startts`` and ``endts`` (check `list endpoint`_ for more details).

Expand Down
39 changes: 30 additions & 9 deletions scrapinghub/client/jobs.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
from .utils import get_tags_for_update, parse_job_key, update_kwargs


_PAGE_SIZE = 1000


class Jobs(object):
"""Class representing a collection of jobs for a project/spider.

Expand Down Expand Up @@ -142,6 +145,9 @@ def iter(self, count=None, start=None, spider=None, state=None,
meta=None, **params):
"""Iterate over jobs collection for a given set of params.

.. versionchanged:: VERSION
Returns all matching jobs, instead of at most 1000.

:param count: (optional) limit amount of returned jobs.
:param start: (optional) number of jobs to skip in the beginning.
:param spider: (optional) filter by spider name.
Expand Down Expand Up @@ -186,11 +192,6 @@ def iter(self, count=None, start=None, spider=None, state=None,

>>> jobs_summary = project.jobs.iter(meta=['scheduled_by', ])

- by default :meth:`Jobs.iter` returns maximum last 1000 results.
Pagination is available using start parameter::

>>> jobs_summary = spider.jobs.iter(start=1000)

- get jobs filtered by tags (list of tags has ``OR`` power)::

>>> jobs_summary = project.jobs.iter(
Expand All @@ -201,12 +202,32 @@ def iter(self, count=None, start=None, spider=None, state=None,
>>> jobs_summary = project.jobs.iter(
... spider='spider2', state='finished', count=3)
"""
update_kwargs(params, count=count, start=start, jobmeta=meta,
spider=spider, state=state, has_tag=has_tag,
lacks_tag=lacks_tag, startts=startts, endts=endts)
update_kwargs(params, jobmeta=meta, spider=spider, state=state,
has_tag=has_tag, lacks_tag=lacks_tag, startts=startts,
endts=endts)
if self.spider:
params['spider'] = self.spider.name
return self._project.jobq.list(**params)
return self._iter_pages(count, start, params)

def _iter_pages(self, count, start, params):
if start is not None:
params['start'] = start
if count is not None:
params['count'] = min(count, _PAGE_SIZE)
while True:
received = 0
for job in self._project.jobq.list(**params):
received += 1
yield job
if count is not None:
count -= received
if count <= 0:
return
if received < _PAGE_SIZE:
return
params['start'] = params.get('start', 0) + received
params['count'] = (_PAGE_SIZE if count is None
else min(count, _PAGE_SIZE))

def list(self, count=None, start=None, spider=None, state=None,
has_tag=None, lacks_tag=None, startts=None, endts=None,
Expand Down
21 changes: 21 additions & 0 deletions tests/client/test_jobs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
import mock

from scrapinghub.client.jobs import Jobs


def _mock_jobs(total):
def list_(start=0, count=None, **params):
end = total if count is None else min(total, start + count)
return iter(range(start, end))

jobs = Jobs(mock.Mock(), '1')
jobs._project.jobq.list.side_effect = list_
return jobs


def test_iter_paginates():
jobs = _mock_jobs(2500)
assert list(jobs.iter()) == list(range(2500))
assert list(jobs.iter(start=500, count=1200)) == list(range(500, 1700))
assert list(jobs.iter(count=3)) == [0, 1, 2]
assert list(_mock_jobs(1000).iter()) == list(range(1000))
Loading