From f81ccf3613515d0140dd9e0d1cb6be79eb0800c3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sat, 26 Sep 2026 00:12:21 +0200 Subject: [PATCH 1/2] Add strict mypy checking and type hints --- .github/workflows/main.yml | 3 + MANIFEST.in | 1 + pyproject.toml | 7 + scrapinghub/__init__.py | 6 +- scrapinghub/client/__init__.py | 28 ++-- scrapinghub/client/activity.py | 13 +- scrapinghub/client/collections.py | 67 ++++++---- scrapinghub/client/exceptions.py | 21 ++- scrapinghub/client/frontiers.py | 82 +++++++----- scrapinghub/client/items.py | 14 +- scrapinghub/client/jobs.py | 104 +++++++++------ scrapinghub/client/logs.py | 21 +-- scrapinghub/client/projects.py | 25 ++-- scrapinghub/client/proxy.py | 66 ++++++---- scrapinghub/client/requests.py | 10 +- scrapinghub/client/samples.py | 5 +- scrapinghub/client/spiders.py | 24 ++-- scrapinghub/client/utils.py | 42 +++--- scrapinghub/hubstorage/__init__.py | 2 +- scrapinghub/hubstorage/activity.py | 11 +- scrapinghub/hubstorage/batchuploader.py | 67 ++++++---- scrapinghub/hubstorage/client.py | 68 ++++++---- scrapinghub/hubstorage/collectionsrt.py | 78 ++++++++---- scrapinghub/hubstorage/frontier.py | 37 ++++-- scrapinghub/hubstorage/job.py | 48 ++++--- scrapinghub/hubstorage/jobq.py | 47 +++++-- scrapinghub/hubstorage/project.py | 42 ++++-- scrapinghub/hubstorage/resourcetype.py | 131 ++++++++++++------- scrapinghub/hubstorage/serialization.py | 18 ++- scrapinghub/hubstorage/utils.py | 36 ++++-- scrapinghub/legacy.py | 162 +++++++++++++----------- scrapinghub/py.typed | 0 setup.py | 2 +- tests/client/conftest.py | 56 +++++--- tests/client/test_activity.py | 12 +- tests/client/test_client.py | 6 +- tests/client/test_collections.py | 37 +++--- tests/client/test_frontiers.py | 15 ++- tests/client/test_items.py | 19 ++- tests/client/test_job.py | 46 +++---- tests/client/test_logs.py | 14 +- tests/client/test_projects.py | 41 +++--- tests/client/test_proxy.py | 19 +-- tests/client/test_requests.py | 7 +- tests/client/test_samples.py | 9 +- tests/client/test_spiders.py | 31 ++--- tests/client/test_utils.py | 67 +++++----- tests/client/utils.py | 12 +- tests/conftest.py | 24 ++-- tests/hubstorage/conftest.py | 66 ++++++---- tests/hubstorage/test_activity.py | 8 +- tests/hubstorage/test_batchuploader.py | 33 +++-- tests/hubstorage/test_client.py | 15 ++- tests/hubstorage/test_collections.py | 34 +++-- tests/hubstorage/test_frontier.py | 16 ++- tests/hubstorage/test_jobq.py | 65 ++++++---- tests/hubstorage/test_jobsmeta.py | 22 +++- tests/hubstorage/test_project.py | 41 +++--- tests/hubstorage/test_retry.py | 55 +++++--- tests/hubstorage/test_serialization.py | 8 +- tests/hubstorage/test_system.py | 38 ++++-- tests/hubstorage/test_utils.py | 2 +- tests/hubstorage/testutil.py | 18 ++- tests/legacy/conftest.py | 10 +- tests/legacy/test_connection.py | 80 ++++++------ tests/legacy/test_job.py | 67 +++++----- tests/legacy/test_jobset.py | 44 ++++--- tests/legacy/test_project.py | 41 +++--- tests/legacy/test_proxymixin.py | 19 ++- tests/test_jobq_client.py | 20 ++- tox.ini | 12 ++ 71 files changed, 1504 insertions(+), 913 deletions(-) create mode 100644 scrapinghub/py.typed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5bd464b0..f6a506b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,6 +42,9 @@ jobs: - python-version: "3.14" env: TOXENV: "docs" + - python-version: "3.14" + env: + TOXENV: "mypy" # https://github.com/kevin1024/vcrpy/issues/885 # - python-version: "pypy3.11-v7.3.20" diff --git a/MANIFEST.in b/MANIFEST.in index f8fc798b..84ec0301 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,3 +2,4 @@ include README.rst include LICENSE include MANIFEST.in include scrapinghub/VERSION +include scrapinghub/py.typed diff --git a/pyproject.toml b/pyproject.toml index e9210475..2002d319 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,10 @@ [tool.sphinx-scrapy] # Keep in sync with .github/workflows/main.yml python-version = "3.14" + +[tool.mypy] +strict = true + +[[tool.mypy.overrides]] +module = ["msgpack", "retrying", "vcr"] +ignore_missing_imports = true diff --git a/scrapinghub/__init__.py b/scrapinghub/__init__.py index c1240ce7..89f69b7d 100644 --- a/scrapinghub/__init__.py +++ b/scrapinghub/__init__.py @@ -3,10 +3,8 @@ "DuplicateJobError", "BadRequest", "NotFound", "Unauthorized", "ValueTooLarge", "ServerError"] -import pkgutil -__version__ = pkgutil.get_data(__package__, 'VERSION') -__version__ = str(__version__.decode('ascii').strip()) -del pkgutil +from importlib.metadata import version as _version +__version__ = _version(__name__) from .legacy import * diff --git a/scrapinghub/client/__init__.py b/scrapinghub/client/__init__.py index 473ac6f3..98078ecd 100644 --- a/scrapinghub/client/__init__.py +++ b/scrapinghub/client/__init__.py @@ -1,8 +1,16 @@ +from __future__ import annotations + +import os +from typing import Any + +from requests import Response + from scrapinghub import Connection as _Connection from scrapinghub import HubstorageClient as _HubstorageClient from .exceptions import _wrap_http_errors -from .projects import Projects +from .jobs import Job +from .projects import Project, Projects from .utils import parse_auth from .utils import parse_project_id, parse_job_key @@ -15,14 +23,14 @@ class Connection(_Connection): @_wrap_http_errors - def _request(self, *args, **kwargs): + def _request(self, *args: Any, **kwargs: Any) -> Any: return super(Connection, self)._request(*args, **kwargs) class HubstorageClient(_HubstorageClient): @_wrap_http_errors - def request(self, *args, **kwargs): + def request(self, *args: Any, **kwargs: Any) -> Response: return super(HubstorageClient, self).request(*args, **kwargs) @@ -62,9 +70,11 @@ class ScrapinghubClient(object): """ - def __init__(self, auth=None, dash_endpoint=None, - connection_timeout=DEFAULT_CONNECTION_TIMEOUT, - dotenv_path=None, **kwargs): + def __init__(self, auth: str | tuple[str, str] | None = None, + dash_endpoint: str | None = None, + connection_timeout: float = DEFAULT_CONNECTION_TIMEOUT, + dotenv_path: str | os.PathLike[str] | None = None, + **kwargs: Any) -> None: self.projects = Projects(self) login, password = parse_auth(auth, dotenv_path=dotenv_path) timeout = connection_timeout or DEFAULT_CONNECTION_TIMEOUT @@ -75,7 +85,7 @@ def __init__(self, auth=None, dash_endpoint=None, self._hsclient = HubstorageClient(auth=(login, password), connection_timeout=timeout, **kwargs) - def get_project(self, project_id): + def get_project(self, project_id: int | str) -> Project: """Get :class:`scrapinghub.client.projects.Project` instance with a given project id. @@ -93,7 +103,7 @@ def get_project(self, project_id): """ return self.projects.get(parse_project_id(project_id)) - def get_job(self, job_key): + def get_job(self, job_key: str) -> Job: """Get :class:`~scrapinghub.client.jobs.Job` with a given job key. :param job_key: job key string in format ``project_id/spider_id/job_id``, @@ -110,7 +120,7 @@ def get_job(self, job_key): project_id = parse_job_key(job_key).project_id return self.projects.get(project_id).jobs.get(job_key) - def close(self, timeout=None): + def close(self, timeout: float | None = None) -> None: """Close client instance. :param timeout: (optional) float timeout secs to stop gracefully. diff --git a/scrapinghub/client/activity.py b/scrapinghub/client/activity.py index b5d1777f..f82488f4 100644 --- a/scrapinghub/client/activity.py +++ b/scrapinghub/client/activity.py @@ -1,10 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations +from collections.abc import Iterable, Iterator +from typing import Any + +from ..hubstorage.activity import Activity as _Activity from .proxy import _Proxy from .utils import parse_job_key, update_kwargs -class Activity(_Proxy): +class Activity(_Proxy[_Activity]): """Representation of collection of job activity events. Not a public constructor: use :class:`~scrapinghub.client.projects.Project` @@ -44,7 +48,8 @@ class Activity(_Proxy): >>> project.activity.add(events) """ - def iter(self, count=None, **params): + def iter(self, count: int | None = None, + **params: Any) -> Iterator[Any]: """Iterate over activity events. :param count: limit amount of elements. @@ -55,7 +60,7 @@ def iter(self, count=None, **params): params = self._modify_iter_params(params) return self._origin.list(**params) - def add(self, values, **kwargs): + def add(self, values: Iterable[dict[str, Any]], **kwargs: Any) -> None: """Add new event to the project activity. :param values: a single event or a list of events, where event is diff --git a/scrapinghub/client/collections.py b/scrapinghub/client/collections.py index 9cdc061b..2608e7c7 100644 --- a/scrapinghub/client/collections.py +++ b/scrapinghub/client/collections.py @@ -1,15 +1,24 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations +from collections.abc import Callable, Iterable, Iterator +from typing import TYPE_CHECKING, Any + +import requests from six import string_types from six.moves import collections_abc from ..hubstorage.collectionsrt import Collection as _Collection +from ..hubstorage.collectionsrt import Collections as _Collections from .proxy import _Proxy from .utils import update_kwargs +if TYPE_CHECKING: + from ..hubstorage.batchuploader import _BatchWriter + from . import ScrapinghubClient + -class Collections(_Proxy): +class Collections(_Proxy[_Collections]): """Access to project collections. Not a public constructor: use :class:`~scrapinghub.client.projects.Project` @@ -24,7 +33,7 @@ class Collections(_Proxy): >>> foo_store = collections.get_store('foo_store') """ - def get(self, type_, name): + def get(self, type_: str, name: str) -> Collection: """Base method to get a collection with a given type and name. :param `type_`: a collection type string. @@ -35,7 +44,7 @@ def get(self, type_, name): self._origin._validate_collection(type_, name) return Collection(self._client, self, type_, name) - def get_store(self, name): + def get_store(self, name: str) -> Collection: """Method to get a store collection by name. :param name: a collection name string. @@ -44,7 +53,7 @@ def get_store(self, name): """ return self.get('s', name) - def get_cached_store(self, name): + def get_cached_store(self, name: str) -> Collection: """Method to get a cashed-store collection by name. The collection type means that items expire after a month. @@ -55,7 +64,7 @@ def get_cached_store(self, name): """ return self.get('cs', name) - def get_versioned_store(self, name): + def get_versioned_store(self, name: str) -> Collection: """Method to get a versioned-store collection by name. The collection type retains up to 3 copies of each item. @@ -66,7 +75,7 @@ def get_versioned_store(self, name): """ return self.get('vs', name) - def get_versioned_cached_store(self, name): + def get_versioned_cached_store(self, name: str) -> Collection: """Method to get a versioned-cached-store collection by name. Multiple copies are retained, and each one expires after a month. @@ -77,7 +86,7 @@ def get_versioned_cached_store(self, name): """ return self.get('vcs', name) - def iter(self): + def iter(self) -> Iterator[Any]: """Iterate through collections of a project. :return: an iterator over collections list where each collection is @@ -86,7 +95,7 @@ def iter(self): """ return self._origin.apiget('list') - def list(self): + def list(self) -> list[Any]: """List collections of a project. :return: a list of collections where each collection is @@ -151,12 +160,13 @@ class Collection(object): >>> foo_store.truncate() """ - def __init__(self, client, collections, type_, name): + def __init__(self, client: ScrapinghubClient, collections: Collections, + type_: str, name: str) -> None: self._client = client self._collections = collections self._origin = _Collection(type_, name, collections._origin) - def get(self, key, **params): + def get(self, key: str, **params: Any) -> Any: """Get item from collection by key. :param key: string item key. @@ -168,7 +178,7 @@ def get(self, key, **params): raise ValueError("key cannot be None") return self._origin.get(key, **params) - def set(self, value): + def set(self, value: Any) -> None: """Set item to collection by key. :param value: a dict representing a collection item. @@ -177,7 +187,7 @@ def set(self, value): """ self._origin.set(value) - def delete(self, keys): + def delete(self, keys: str | Iterable[str]) -> None: """Delete item(s) from collection by key(s). :param keys: a single key or a list of keys. @@ -190,14 +200,14 @@ def delete(self, keys): "object providing string keys") self._origin.delete(keys) - def truncate(self): + def truncate(self) -> None: """Remove the entire collection with a single API call. The method returns ``None`` (original method returns an empty generator). """ self._origin.truncate() - def count(self, *args, **kwargs): + def count(self, *args: Any, **kwargs: Any) -> int: """Count collection items with a given filters. :return: amount of elements in collection. @@ -207,8 +217,12 @@ def count(self, *args, **kwargs): return self._origin._collections.count( self._origin.coltype, self._origin.colname, *args, **kwargs) - def iter(self, key=None, prefix=None, prefixcount=None, startts=None, - endts=None, requests_params=None, **params): + def iter(self, key: str | Iterable[str] | None = None, + prefix: str | Iterable[str] | None = None, + prefixcount: int | None = None, startts: int | None = None, + endts: int | None = None, + requests_params: dict[str, Any] | None = None, + **params: Any) -> Iterator[Any]: """A method to iterate through collection items. :param key: a string key or a list of keys to filter with. @@ -228,8 +242,12 @@ def iter(self, key=None, prefix=None, prefixcount=None, startts=None, return self._origin._collections.iter_values( self._origin.coltype, self._origin.colname, **params) - def list(self, key=None, prefix=None, prefixcount=None, startts=None, - endts=None, requests_params=None, **params): + def list(self, key: str | Iterable[str] | None = None, + prefix: str | Iterable[str] | None = None, + prefixcount: int | None = None, startts: int | None = None, + endts: int | None = None, + requests_params: dict[str, Any] | None = None, + **params: Any) -> list[Any]: """Convenient shortcut to list iter results. Please note that :meth:`list` method can use a lot of memory and for a @@ -251,9 +269,12 @@ def list(self, key=None, prefix=None, prefixcount=None, startts=None, startts=startts, endts=endts) return list(self.iter(requests_params=requests_params, **params)) - def create_writer(self, start=0, auth=None, size=1000, interval=15, - qsize=None, content_encoding='identity', - maxitemsize=1024 ** 2, callback=None): + def create_writer( + self, start: int = 0, auth: str | tuple[str, str] | None = None, + size: int = 1000, interval: float = 15, qsize: int | None = None, + content_encoding: str = 'identity', maxitemsize: int = 1024 ** 2, + callback: Callable[[requests.Response | None], object] | None = None, + ) -> _BatchWriter: """Create a new writer for a collection. :param start: (optional) initial offset for writer thread. @@ -270,7 +291,7 @@ def create_writer(self, start=0, auth=None, size=1000, interval=15, If provided - calllback shouldn't try to inject more items in the queue, otherwise it can lead to deadlocks. """ - kwargs = {} + kwargs: dict[str, Any] = {} update_kwargs(kwargs, start=start, auth=auth, size=size, interval=interval, qsize=qsize, content_encoding=content_encoding, maxitemsize=maxitemsize, callback=callback) diff --git a/scrapinghub/client/exceptions.py b/scrapinghub/client/exceptions.py index d804a796..1329b674 100644 --- a/scrapinghub/client/exceptions.py +++ b/scrapinghub/client/exceptions.py @@ -1,20 +1,25 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, annotations +from collections.abc import Callable from functools import wraps +from typing import ParamSpec, TypeVar from requests import HTTPError from ..legacy import APIError +_P = ParamSpec('_P') +_T = TypeVar('_T') -def _get_http_error_msg(exc): - if isinstance(exc, HTTPError): + +def _get_http_error_msg(exc: BaseException | None) -> str: + if isinstance(exc, HTTPError) and exc.response is not None: try: payload = exc.response.json() except ValueError: payload = None if payload and isinstance(payload, dict): - message = payload.get('message') + message: str | None = payload.get('message') if message: return message elif exc.response.text: @@ -25,7 +30,8 @@ def _get_http_error_msg(exc): class ScrapinghubAPIError(Exception): """Base exception class.""" - def __init__(self, message=None, http_error=None): + def __init__(self, message: object = None, + http_error: BaseException | None = None) -> None: self.http_error = http_error if not message: message = _get_http_error_msg(http_error) @@ -61,13 +67,14 @@ class ServerError(ScrapinghubAPIError): """Indicates some server error: something unexpected has happened.""" -def _wrap_http_errors(method): +def _wrap_http_errors(method: Callable[_P, _T]) -> Callable[_P, _T]: """Internal helper to handle exceptions gracefully.""" @wraps(method) - def wrapped(*args, **kwargs): + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T: try: return method(*args, **kwargs) except HTTPError as exc: + assert exc.response is not None status_code = exc.response.status_code if status_code == 400: raise BadRequest(http_error=exc) diff --git a/scrapinghub/client/frontiers.py b/scrapinghub/client/frontiers.py index 38298a0b..1156bc1f 100644 --- a/scrapinghub/client/frontiers.py +++ b/scrapinghub/client/frontiers.py @@ -1,6 +1,10 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations from functools import partial from collections import defaultdict +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING, Any + +import requests from six import string_types @@ -10,15 +14,21 @@ from .proxy import _Proxy from .utils import update_kwargs +if TYPE_CHECKING: + from ..hubstorage.batchuploader import _BatchWriter + from . import ScrapinghubClient + class _HSFrontier(_Frontier): """Modified hubstorage Frontier with newcount per slot.""" - def __init__(self, *args, **kwargs): + newcount: defaultdict[tuple[str, str], int] # type: ignore[assignment] + + def __init__(self, *args: Any, **kwargs: Any) -> None: super(_HSFrontier, self).__init__(*args, **kwargs) self.newcount = defaultdict(int) - def _get_writer(self, frontier, slot): + def _get_writer(self, frontier: str, slot: str) -> _BatchWriter: """Modified helper method to create a batchuploader writer with updated callback to write newcount data per slot. @@ -41,12 +51,15 @@ def _get_writer(self, frontier, slot): self._writers[key] = writer return writer - def _writer_callback(self, key, response): + def _writer_callback( # type: ignore[override] + self, key: tuple[str, str], response: requests.Response | None, + ) -> None: """Writer callback function when new batch is added.""" + assert response is not None self.newcount[key] += response.json()["newcount"] -class Frontiers(_Proxy): +class Frontiers(_Proxy[_HSFrontier]): """Frontiers collection for a project. Not a public constructor: use :class:`~scrapinghub.client.projects.Project` @@ -83,10 +96,10 @@ class Frontiers(_Proxy): >>> project.frontiers.close() """ - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super(Frontiers, self).__init__(*args, **kwargs) - def get(self, name): + def get(self, name: str) -> Frontier: """Get a frontier by name. :param name: a frontier name string. @@ -95,7 +108,7 @@ def get(self, name): """ return Frontier(self._client, self, name) - def iter(self): + def iter(self) -> Iterator[Any]: """Iterate through frontiers. :return: an iterator over frontiers names. @@ -103,7 +116,7 @@ def iter(self): """ return iter(self.list()) - def list(self): + def list(self) -> Any: """List frontiers names. :return: a list of frontiers names. @@ -112,15 +125,15 @@ def list(self): return next(self._origin.apiget('list')) @property - def newcount(self): + def newcount(self) -> int: """Integer amount of new entries added to all frontiers.""" return sum(self._origin.newcount.values()) - def flush(self): + def flush(self) -> None: """Flush data in all frontiers writer threads.""" self._origin.flush() - def close(self): + def close(self) -> None: """Close frontier writer threads one-by-one.""" self._origin.close() @@ -157,12 +170,13 @@ class Frontier(object): >>> frontier.newcount 3 """ - def __init__(self, client, frontiers, name): + def __init__(self, client: ScrapinghubClient, frontiers: Frontiers, + name: str) -> None: self.key = name self._client = client self._frontiers = frontiers - def get(self, slot): + def get(self, slot: str) -> FrontierSlot: """Get a slot by name. :return: a frontier slot instance. @@ -170,7 +184,7 @@ def get(self, slot): """ return FrontierSlot(self._client, self, slot) - def iter(self): + def iter(self) -> Iterator[Any]: """Iterate through slots. :return: an iterator over frontier slots names. @@ -178,7 +192,7 @@ def iter(self): """ return iter(self.list()) - def list(self): + def list(self) -> Any: """List all slots. :return: a list of frontier slots names. @@ -186,7 +200,7 @@ def list(self): """ return next(self._frontiers._origin.apiget((self.key, 'list'))) - def flush(self): + def flush(self) -> None: """Flush data for a whole frontier.""" writers = self._frontiers._origin._writers for (fname, _), writer in writers.items(): @@ -194,7 +208,7 @@ def flush(self): writer.flush() @property - def newcount(self): + def newcount(self) -> int: """Integer amount of new entries added to frontier.""" newcount_values = self._frontiers._origin.newcount return sum(v for (frontier, _), v in newcount_values.items() @@ -250,7 +264,8 @@ class FrontierSlot(object): >>> slot.delete() """ - def __init__(self, client, frontier, slot): + def __init__(self, client: ScrapinghubClient, frontier: Frontier, + slot: str) -> None: self.key = slot self._client = client self._frontier = frontier @@ -258,7 +273,7 @@ def __init__(self, client, frontier, slot): self.queue = FrontierSlotQueue(self) @property - def f(self): + def f(self) -> FrontierSlotFingerprints: """Shortcut to have quick access to slot fingerprints. :return: fingerprints collection for the slot. @@ -267,7 +282,7 @@ def f(self): return self.fingerprints @property - def q(self): + def q(self) -> FrontierSlotQueue: """Shortcut to have quick access to a slot queue. :return: queue instance for the slot. @@ -275,13 +290,13 @@ def q(self): """ return self.queue - def delete(self): + def delete(self) -> None: """Delete the slot.""" origin = self._frontier._frontiers._origin origin.delete_slot(self._frontier.key, self.key) origin.newcount.pop((self._frontier.key, self.key), None) - def flush(self): + def flush(self) -> None: """Flush data for the slot.""" writers = self._frontier._frontiers._origin._writers writer = writers.get((self._frontier.key, self.key)) @@ -289,7 +304,7 @@ def flush(self): writer.flush() @property - def newcount(self): + def newcount(self) -> int: """Integer amount of new entries added to slot.""" newcount_values = self._frontier._frontiers._origin.newcount return newcount_values.get((self._frontier.key, self.key), 0) @@ -298,12 +313,12 @@ def newcount(self): class FrontierSlotFingerprints(object): """Representation of request fingerprints collection stored in slot.""" - def __init__(self, slot): + def __init__(self, slot: FrontierSlot) -> None: self.key = slot.key self._frontier = slot._frontier self._slot = slot - def add(self, fps): + def add(self, fps: Iterable[str]) -> None: """Add new fingerprints to slot. :param fps: a list of string fingerprints to add. @@ -316,7 +331,7 @@ def add(self, fps): for fp in fps: writer.write({'fp': fp}) - def iter(self, **params): + def iter(self, **params: Any) -> Iterator[str]: """Iterate through fingerprints in the slot. :param params: (optional) additional query params for the request. @@ -328,7 +343,7 @@ def iter(self, **params): for fp in origin.apiget(path, params=params): yield fp.get('fp') - def list(self, **params): + def list(self, **params: Any) -> list[str]: """List fingerprints in the slot. :param params: (optional) additional query params for the request. @@ -341,17 +356,18 @@ def list(self, **params): class FrontierSlotQueue(object): """Representation of request batches queue stored in slot.""" - def __init__(self, slot): + def __init__(self, slot: FrontierSlot) -> None: self.key = slot.key self._frontier = slot._frontier self._slot = slot - def add(self, fps): + def add(self, fps: Iterable[Any]) -> None: """Add requests to the queue.""" origin = self._frontier._frontiers._origin return origin.add(self._frontier.key, self.key, fps) - def iter(self, mincount=None, **params): + def iter(self, mincount: int | None = None, + **params: Any) -> Iterator[Any]: """Iterate through batches in the queue. :param mincount: (optional) limit results with min amount of requests. @@ -365,7 +381,7 @@ def iter(self, mincount=None, **params): update_kwargs(params, mincount=mincount) return origin.apiget(path, params=params) - def list(self, mincount=None, **params): + def list(self, mincount: int | None = None, **params: Any) -> list[Any]: """List request batches in the queue. :param mincount: (optional) limit results with min amount of requests. @@ -376,7 +392,7 @@ def list(self, mincount=None, **params): """ return list(self.iter(mincount=mincount, **params)) - def delete(self, ids): + def delete(self, ids: Iterable[str]) -> None: """Delete request batches from the queue.""" origin = self._frontier._frontiers._origin return origin.delete(self._frontier.key, self.key, ids) diff --git a/scrapinghub/client/items.py b/scrapinghub/client/items.py index e9be38c0..0494cd4c 100644 --- a/scrapinghub/client/items.py +++ b/scrapinghub/client/items.py @@ -1,11 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations import sys +from collections.abc import Iterator +from typing import Any +from ..hubstorage.job import Items as _Items from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin -class Items(_DownloadableProxyMixin, _ItemsResourceProxy): +class Items(_DownloadableProxyMixin[_Items], _ItemsResourceProxy[_Items]): # type: ignore[misc] """Representation of collection of job items. Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` @@ -78,7 +81,7 @@ class Items(_DownloadableProxyMixin, _ItemsResourceProxy): }] """ - def _modify_iter_params(self, params): + def _modify_iter_params(self, params: dict[str, Any]) -> dict[str, Any]: """Modify iter filter to convert offset to start parameter. :return: a dict with updated set of params. @@ -90,7 +93,8 @@ def _modify_iter_params(self, params): params['start'] = '{}/{}'.format(self.key, offset) return params - def list_iter(self, chunksize=1000, *args, **kwargs): + def list_iter(self, chunksize: int = 1000, *args: Any, + **kwargs: Any) -> Iterator[list[Any]]: """An alternative interface for reading items by returning them as a generator which yields lists of items sized as `chunksize`. @@ -120,7 +124,7 @@ def list_iter(self, chunksize=1000, *args, **kwargs): chunksize = count - processed items = [ item for item in self.iter( - count=chunksize, start=next_key, *args, **kwargs) + count=chunksize, start=next_key, *args, **kwargs) # type: ignore[misc] ] yield items processed += len(items) diff --git a/scrapinghub/client/jobs.py b/scrapinghub/client/jobs.py index 75d0acbc..ce8d8e21 100644 --- a/scrapinghub/client/jobs.py +++ b/scrapinghub/client/jobs.py @@ -1,6 +1,8 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations import json +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any from ..hubstorage.job import JobMeta as _JobMeta from ..hubstorage.job import Items as _Items @@ -16,6 +18,13 @@ from .proxy import _MappingProxy from .utils import get_tags_for_update, parse_job_key, update_kwargs +if TYPE_CHECKING: + from . import ScrapinghubClient + from .spiders import Spider + +_Tags = str | Sequence[str] | None +_TagList = list[str] + class Jobs(object): """Class representing a collection of jobs for a project/spider. @@ -37,14 +46,17 @@ class Jobs(object): """ - def __init__(self, client, project_id, spider=None): + def __init__(self, client: ScrapinghubClient, project_id: str, + spider: Spider | None = None) -> None: self.project_id = project_id self.spider = spider self._client = client self._project = client._hsclient.get_project(project_id) - def count(self, spider=None, state=None, has_tag=None, lacks_tag=None, - startts=None, endts=None, **params): + def count(self, spider: str | None = None, state: _Tags = None, + has_tag: _Tags = None, lacks_tag: _Tags = None, + startts: int | None = None, endts: int | None = None, + **params: Any) -> Any: """Count jobs with a given set of filters. :param spider: (optional) filter by spider name. @@ -79,7 +91,8 @@ def count(self, spider=None, state=None, has_tag=None, lacks_tag=None, params['spider'] = self.spider.name return next(self._project.jobq.apiget(('count',), params=params)) - def cancel(self, keys=None, count=None, **params): + def cancel(self, keys: list[str] | None = None, count: int | None = None, + **params: Any) -> Any: """Cancel a list of jobs using the keys provided. :param keys: (optional) a list of strings containing the job keys in @@ -117,15 +130,15 @@ def cancel(self, keys=None, count=None, **params): raise ValueError("keys should be a list") # it raises ValueError if invalid - keys = [parse_job_key(k) for k in keys] + job_keys = [parse_job_key(k) for k in keys] - if not all([key.project_id == self.project_id for key in keys]): + if not all([key.project_id == self.project_id for key in job_keys]): raise ValueError( "all keys should belong to project: %s" % self.project_id ) # change it to the format in which JobQ expects. - data = [{"key": str(k)} for k in keys] + data = [{"key": str(k)} for k in job_keys] # may raise BadRequest if JobQ doesn't validate return list(self._project.jobq.apipost("cancel", @@ -137,9 +150,11 @@ def cancel(self, keys=None, count=None, **params): # may raise Forbidden return self._project.jobq.apipost("cancel?count=%s" % count) - def iter(self, count=None, start=None, spider=None, state=None, - has_tag=None, lacks_tag=None, startts=None, endts=None, - meta=None, **params): + def iter(self, count: int | None = None, start: int | None = None, + spider: str | None = None, state: _Tags = None, + has_tag: _Tags = None, lacks_tag: _Tags = None, + startts: int | None = None, endts: int | None = None, + meta: _Tags = None, **params: Any) -> Iterator[Any]: """Iterate over jobs collection for a given set of params. :param count: (optional) limit amount of returned jobs. @@ -208,9 +223,11 @@ def iter(self, count=None, start=None, spider=None, state=None, params['spider'] = self.spider.name return self._project.jobq.list(**params) - def list(self, count=None, start=None, spider=None, state=None, - has_tag=None, lacks_tag=None, startts=None, endts=None, - meta=None, **params): + def list(self, count: int | None = None, start: int | None = None, + spider: str | None = None, state: _Tags = None, + has_tag: _Tags = None, lacks_tag: _Tags = None, + startts: int | None = None, endts: int | None = None, + meta: _Tags = None, **params: Any) -> list[Any]: """Convenient shortcut to list iter results. :param count: (optional) limit amount of returned jobs. @@ -247,9 +264,13 @@ def list(self, count=None, start=None, spider=None, state=None, lacks_tag=lacks_tag, startts=startts, endts=endts) return list(self.iter(**params)) - def run(self, spider=None, units=None, priority=None, meta=None, - add_tag=None, job_args=None, job_settings=None, cmd_args=None, - environment=None, **params): + def run(self, spider: str | None = None, units: int | None = None, + priority: int | None = None, meta: dict[str, Any] | None = None, + add_tag: _Tags = None, job_args: dict[str, Any] | None = None, + job_settings: dict[str, Any] | None = None, + cmd_args: str | None = None, + environment: dict[str, str] | None = None, + **params: Any) -> Job: """Schedule a new job and returns its job key. :param spider: a spider name string @@ -275,7 +296,8 @@ def run(self, spider=None, units=None, priority=None, meta=None, >>> job.key '123/1/1' """ - if not spider and not self.spider: + spider = spider or (self.spider.name if self.spider else None) + if not spider: raise ValueError('Please provide `spider` name') if job_args: if not isinstance(job_args, dict): @@ -287,7 +309,7 @@ def run(self, spider=None, units=None, priority=None, meta=None, raise ValueError("environment should be a dictionary") params['project'] = self.project_id - params['spider'] = spider or self.spider.name + params['spider'] = spider update_kwargs(params, units=units, priority=priority, add_tag=add_tag, cmd_args=cmd_args, job_settings=job_settings, meta=meta, @@ -302,7 +324,7 @@ def run(self, spider=None, units=None, priority=None, meta=None, raise return Job(self._client, response['jobid']) - def get(self, job_key): + def get(self, job_key: str | tuple[int | str, ...]) -> Job: """Get a :class:`Job` with a given job_key. :param job_key: a string job key. @@ -321,14 +343,15 @@ def get(self, job_key): >>> job.key '123/1/2' """ - job_key = parse_job_key(job_key) - if job_key.project_id != self.project_id: + parsed_key = parse_job_key(job_key) + if parsed_key.project_id != self.project_id: raise ValueError('Please use same project id') - if self.spider and job_key.spider_id != self.spider._id: + if self.spider and parsed_key.spider_id != self.spider._id: raise ValueError('Please use same spider id') - return Job(self._client, str(job_key)) + return Job(self._client, str(parsed_key)) - def summary(self, state=None, spider=None, **params): + def summary(self, state: str | None = None, spider: str | None = None, + **params: Any) -> Any: """Get jobs summary (optionally by state). :param state: (optional) a string state to filter jobs. @@ -353,8 +376,10 @@ def summary(self, state=None, spider=None, **params): return self._project.jobq.summary( state, spiderid=spider_id, **params) - def iter_last(self, start=None, start_after=None, count=None, - spider=None, **params): + def iter_last(self, start: int | None = None, + start_after: str | None = None, count: int | None = None, + spider: str | None = None, + **params: Any) -> Iterator[Any]: """Iterate through last jobs for each spider. :param start: (optional) @@ -394,7 +419,7 @@ def iter_last(self, start=None, start_after=None, count=None, update_kwargs(params, start=start, startafter=start_after, count=count) return self._project.spiders.lastjobsummary(spider_id, **params) - def _extract_spider_id(self, spider): + def _extract_spider_id(self, spider: str | None) -> str | None: if not spider and self.spider: return self.spider._id if spider: @@ -405,7 +430,9 @@ def _extract_spider_id(self, spider): return spider_id return None - def update_tags(self, add=None, remove=None, spider=None): + def update_tags(self, add: _TagList | None = None, + remove: _TagList | None = None, + spider: str | None = None) -> Any: """Update tags for all existing spider jobs. :param add: (optional) list of tags to add to selected jobs. @@ -469,7 +496,7 @@ class Job(object): >>> job.metadata.get('state') 'finished' """ - def __init__(self, client, job_key): + def __init__(self, client: ScrapinghubClient, job_key: str) -> None: self.project_id = parse_job_key(job_key).project_id self.key = job_key @@ -485,7 +512,8 @@ def __init__(self, client, job_key): self.metadata = JobMeta(_JobMeta, client, job_key) - def update_tags(self, add=None, remove=None): + def update_tags(self, add: list[str] | None = None, + remove: list[str] | None = None) -> None: """Partially update job tags. It provides a convenient way to mark specific jobs (for better search, @@ -502,14 +530,14 @@ def update_tags(self, add=None, remove=None): params.update({'project': self.project_id, 'job': self.key}) self._client._connection._post('jobs_update', 'json', params) - def close_writers(self): + def close_writers(self) -> None: """Stop job batch writers threads gracefully. Called on :meth:`ScrapinghubClient.close` method. """ self._job.close_writers() - def start(self, **params): + def start(self, **params: Any) -> Any: """Move job to running state. :param params: (optional) keyword meta parameters to update. @@ -523,7 +551,7 @@ def start(self, **params): """ return self.update(state='running', **params) - def finish(self, **params): + def finish(self, **params: Any) -> Any: """Move running job to finished state. :param params: (optional) keyword meta parameters to update. @@ -537,7 +565,7 @@ def finish(self, **params): """ return self.update(state='finished', **params) - def delete(self, **params): + def delete(self, **params: Any) -> Any: """Mark finished job for deletion. :param params: (optional) keyword meta parameters to update. @@ -551,7 +579,7 @@ def delete(self, **params): """ return self.update(state='deleted', **params) - def update(self, state, **params): + def update(self, state: str, **params: Any) -> Any: """Update job state. :param state: a new job state. @@ -570,7 +598,7 @@ def update(self, state, **params): except StopIteration: raise NotFound("Job {} doesn't exist".format(self.key)) - def cancel(self): + def cancel(self) -> None: """Schedule a running job for cancellation. Usage:: @@ -582,7 +610,7 @@ def cancel(self): self._project.jobq.request_cancel(self) -class JobMeta(_MappingProxy): +class JobMeta(_MappingProxy[_JobMeta]): """Class representing job metadata. Not a public constructor: use :class:`Job` instance to get a diff --git a/scrapinghub/client/logs.py b/scrapinghub/client/logs.py index 3ab37e96..091eb9b9 100644 --- a/scrapinghub/client/logs.py +++ b/scrapinghub/client/logs.py @@ -1,13 +1,15 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations import json import logging +from typing import Any +from ..hubstorage.job import Logs as _Logs from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin from .utils import LogLevel -class Logs(_DownloadableProxyMixin, _ItemsResourceProxy): +class Logs(_DownloadableProxyMixin[_Logs], _ItemsResourceProxy[_Logs]): # type: ignore[misc] """Representation of collection of job logs. Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance @@ -49,7 +51,8 @@ class Logs(_DownloadableProxyMixin, _ItemsResourceProxy): 'time': 1486375511188, }] """ - def log(self, message, level=logging.INFO, ts=None, **other): + def log(self, message: str, level: int = logging.INFO, + ts: int | None = None, **other: Any) -> None: """Base method to write a log entry. :param message: a string message. @@ -59,28 +62,28 @@ def log(self, message, level=logging.INFO, ts=None, **other): """ self._origin.log(message, level=level, ts=ts, **other) - def debug(self, message, **other): + def debug(self, message: str, **other: Any) -> None: """Log a message with DEBUG level.""" self._origin.debug(message, **other) - def info(self, message, **other): + def info(self, message: str, **other: Any) -> None: """Log a message with INFO level.""" self._origin.info(message, **other) - def warn(self, message, **other): + def warn(self, message: str, **other: Any) -> None: """Log a message with WARN level.""" self._origin.warn(message, **other) warning = warn - def error(self, message, **other): + def error(self, message: str, **other: Any) -> None: """Log a message with ERROR level.""" self._origin.error(message, **other) - def batch_write_start(self): + def batch_write_start(self) -> int: """Override to set a start parameter when commencing writing.""" return self._origin.batch_write_start() - def _modify_iter_params(self, params): + def _modify_iter_params(self, params: dict[str, Any]) -> dict[str, Any]: """Modify iter() filters on-the-fly. - convert offset to start parameter diff --git a/scrapinghub/client/projects.py b/scrapinghub/client/projects.py index a67f3266..f034486e 100644 --- a/scrapinghub/client/projects.py +++ b/scrapinghub/client/projects.py @@ -1,4 +1,7 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations + +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any from ..hubstorage.activity import Activity as _Activity from ..hubstorage.collectionsrt import Collections as _Collections @@ -12,6 +15,9 @@ from .spiders import Spiders from .utils import parse_project_id +if TYPE_CHECKING: + from . import ScrapinghubClient + class Projects(object): """Collection of projects available to current user. @@ -26,10 +32,10 @@ class Projects(object): """ - def __init__(self, client): + def __init__(self, client: ScrapinghubClient) -> None: self._client = client - def get(self, project_id): + def get(self, project_id: int | str) -> Project: """Get project for a given project id. :param project_id: integer or string numeric project id. @@ -44,7 +50,7 @@ def get(self, project_id): """ return Project(self._client, parse_project_id(project_id)) - def list(self): + def list(self) -> Any: """Get list of projects available to current user. :return: a list of project ids. @@ -57,7 +63,7 @@ def list(self): """ return self._client._connection.project_ids() - def iter(self): + def iter(self) -> Iterator[Any]: """Iterate through list of projects available to current user. Provided for the sake of API consistency. @@ -67,7 +73,8 @@ def iter(self): """ return iter(self.list()) - def summary(self, state=None, **params): + def summary(self, state: str | Sequence[str] | None = None, + **params: Any) -> Any: """Get short summaries for all available user projects. :param state: a string state or a list of states. @@ -120,7 +127,7 @@ class Project(object): '123' """ - def __init__(self, client, project_id): + def __init__(self, client: ScrapinghubClient, project_id: str) -> None: self.key = str(project_id) self._client = client @@ -135,7 +142,7 @@ def __init__(self, client, project_id): self.settings = Settings(_Settings, client, project_id) -class Settings(_MappingProxy): +class Settings(_MappingProxy[_Settings]): """Class representing job metadata. Not a public constructor: use :class:`Project` instance to get a @@ -176,7 +183,7 @@ class Settings(_MappingProxy): >>> project.settings.delete('job_runtime_limit') """ - def set(self, key, value): + def set(self, key: str, value: Any) -> None: """Update project setting value by key. :param key: a string setting key. diff --git a/scrapinghub/client/proxy.py b/scrapinghub/client/proxy.py index f7d10b4d..66cfa2ac 100644 --- a/scrapinghub/client/proxy.py +++ b/scrapinghub/client/proxy.py @@ -1,14 +1,32 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations import six import json +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, Any, Generic, TypeVar from ..hubstorage import ValueTooLarge as _ValueTooLarge +from ..hubstorage.resourcetype import ( + DownloadableResource, + ItemsResourceType, + MappingResourceType, + ResourceType, +) from .utils import update_kwargs from .exceptions import ValueTooLarge +if TYPE_CHECKING: + from ..hubstorage import HubstorageClient + from . import ScrapinghubClient -class _Proxy(object): +_OriginT = TypeVar('_OriginT', bound=ResourceType) +_ItemsOriginT = TypeVar('_ItemsOriginT', bound=ItemsResourceType) +_DownloadableOriginT = TypeVar('_DownloadableOriginT', + bound=DownloadableResource) +_MappingOriginT = TypeVar('_MappingOriginT', bound=MappingResourceType) + + +class _Proxy(Generic[_OriginT]): """A helper to create a class instance and proxy its methods to origin. The internal proxy class is useful to link class attributes from its @@ -23,12 +41,13 @@ class _Proxy(object): msgpack support. """ - def __init__(self, cls, client, key): + def __init__(self, cls: Callable[[HubstorageClient, str], _OriginT], + client: ScrapinghubClient, key: str) -> None: self.key = key self._client = client self._origin = cls(client._hsclient, key) - def list(self, *args, **kwargs): + def list(self, *args: Any, **kwargs: Any) -> list[Any]: """Convenient shortcut to list iter results. Please note that :meth:`list` method can use a lot of memory and for a @@ -36,9 +55,9 @@ def list(self, *args, **kwargs): :meth:`iter` method (all params and available filters are same for both methods). """ - return list(self.iter(*args, **kwargs)) + return list(self.iter(*args, **kwargs)) # type: ignore[attr-defined] - def _modify_iter_params(self, params): + def _modify_iter_params(self, params: dict[str, Any]) -> dict[str, Any]: """A helper to modify iter*() params on-the-fly. The method is internal and should be redefined in subclasses. @@ -50,9 +69,9 @@ def _modify_iter_params(self, params): return _format_iter_filters(params) -class _ItemsResourceProxy(_Proxy): +class _ItemsResourceProxy(_Proxy[_ItemsOriginT]): - def get(self, key, **params): + def get(self, key: str | int, **params: Any) -> Any: """Get element from collection. :param key: element key. @@ -61,7 +80,7 @@ def get(self, key, **params): """ return self._origin.get(key, **params) - def write(self, item): + def write(self, item: Any) -> int: """Write new element to collection. :param item: element data dict to write. @@ -71,7 +90,8 @@ def write(self, item): except _ValueTooLarge as exc: raise ValueTooLarge(str(exc)) - def iter(self, _key=None, count=None, **params): + def iter(self, _key: str | None = None, count: int | None = None, + **params: Any) -> Iterator[Any]: """Iterate over elements in collection. :param count: limit amount of elements. @@ -82,11 +102,11 @@ def iter(self, _key=None, count=None, **params): params = self._modify_iter_params(params) return self._origin.list(_key, **params) - def flush(self): + def flush(self) -> None: """Flush data from writer threads.""" self._origin.flush() - def stats(self): + def stats(self) -> Any: """Get resource stats. :return: a dictionary with stats data. @@ -94,14 +114,16 @@ def stats(self): """ return self._origin.stats() - def close(self, block=True): + def close(self, block: bool = True) -> None: """Close writers one-by-one.""" self._origin.close(block) -class _DownloadableProxyMixin(object): +class _DownloadableProxyMixin(_Proxy[_DownloadableOriginT]): - def iter(self, _path=None, count=None, requests_params=None, **apiparams): + def iter(self, _path: str | None = None, count: int | None = None, + requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[Any]: """A general method to iterate through elements. :param count: limit amount of elements. @@ -119,19 +141,19 @@ def iter(self, _path=None, count=None, requests_params=None, **apiparams): yield entry -class _MappingProxy(_Proxy): +class _MappingProxy(_Proxy[_MappingOriginT]): """A helper class to support basic get/set interface for dict-like collections of elements. """ - def get(self, key): + def get(self, key: str) -> Any: """Get element value by key. :param key: a string key """ return next(self._origin.apiget(key)) - def set(self, key, value): + def set(self, key: str, value: Any) -> None: """Set element value. :param key: a string key @@ -139,7 +161,7 @@ def set(self, key, value): """ self._origin.apipost(key, data=json.dumps(value), is_idempotent=True) - def update(self, values): + def update(self, values: dict[str, Any]) -> None: """Update multiple elements at once. The method provides convenient interface for partial updates. @@ -154,14 +176,14 @@ def update(self, values): if k not in self._origin.ignore_fields}, is_idempotent=True) - def delete(self, key): + def delete(self, key: str) -> None: """Delete element by key. :param key: a string key """ self._origin.apidelete(key) - def iter(self): + def iter(self) -> Iterator[tuple[str, Any]]: """Iterate through key/value pairs. :return: an iterator over key/value pairs. @@ -170,7 +192,7 @@ def iter(self): return six.iteritems(next(self._origin.apiget())) -def _format_iter_filters(params): +def _format_iter_filters(params: dict[str, Any]) -> dict[str, Any]: """Format iter() filter param on-the-fly. Support passing multiple filters at once as a list with tuples. diff --git a/scrapinghub/client/requests.py b/scrapinghub/client/requests.py index 1ff227ea..70811051 100644 --- a/scrapinghub/client/requests.py +++ b/scrapinghub/client/requests.py @@ -1,9 +1,12 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations +from ..hubstorage.job import Requests as _Requests from .proxy import _ItemsResourceProxy, _DownloadableProxyMixin -class Requests(_DownloadableProxyMixin, _ItemsResourceProxy): +class Requests( # type: ignore[misc] + _DownloadableProxyMixin[_Requests], _ItemsResourceProxy[_Requests], +): """Representation of collection of job requests. Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance @@ -41,7 +44,8 @@ class Requests(_DownloadableProxyMixin, _ItemsResourceProxy): 'url': 'https://example.com' }] """ - def add(self, url, status, method, rs, duration, ts, parent=None, fp=None): + def add(self, url: str, status: int, method: str, rs: int, duration: int, + ts: int, parent: int | None = None, fp: str | None = None) -> int: """ Add a new requests. :param url: string url for the request. diff --git a/scrapinghub/client/samples.py b/scrapinghub/client/samples.py index 87a8e9bc..21db73db 100644 --- a/scrapinghub/client/samples.py +++ b/scrapinghub/client/samples.py @@ -1,9 +1,10 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations +from ..hubstorage.job import Samples as _Samples from .proxy import _ItemsResourceProxy -class Samples(_ItemsResourceProxy): +class Samples(_ItemsResourceProxy[_Samples]): """Representation of collection of job samples. Not a public constructor: use :class:`~scrapinghub.client.jobs.Job` instance diff --git a/scrapinghub/client/spiders.py b/scrapinghub/client/spiders.py index 1d665801..bf3dda1d 100644 --- a/scrapinghub/client/spiders.py +++ b/scrapinghub/client/spiders.py @@ -1,4 +1,7 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations + +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from requests.compat import urljoin @@ -6,6 +9,9 @@ from .jobs import Jobs from .utils import get_tags_for_update +if TYPE_CHECKING: + from . import ScrapinghubClient + class Spiders(object): """Class to work with a collection of project spiders. @@ -22,11 +28,11 @@ class Spiders(object): """ - def __init__(self, client, project_id): + def __init__(self, client: ScrapinghubClient, project_id: str) -> None: self.project_id = project_id self._client = client - def get(self, spider, **params): + def get(self, spider: str, **params: Any) -> Spider: """Get a spider object for a given spider name. The method gets/sets spider id (and checks if spider exists). @@ -48,7 +54,7 @@ def get(self, spider, **params): raise NotFound("Spider {} doesn't exist.".format(spider)) return Spider(self._client, self.project_id, spider_id, spider) - def list(self): + def list(self) -> Any: """Get a list of spiders for a project. :return: a list of dictionaries with spiders metadata. @@ -63,7 +69,7 @@ def list(self): project = self._client._connection[self.project_id] return project.spiders() - def iter(self): + def iter(self) -> Iterator[Any]: """Iterate through a list of spiders for a project. :return: an iterator over spiders list where each spider is represented @@ -95,7 +101,8 @@ class Spider(object): 'spider1' """ - def __init__(self, client, project_id, spider_id, spider): + def __init__(self, client: ScrapinghubClient, project_id: str, + spider_id: int | str, spider: str) -> None: self.project_id = project_id self.key = '{}/{}'.format(str(project_id), str(spider_id)) self._id = str(spider_id) @@ -104,7 +111,8 @@ def __init__(self, client, project_id, spider_id, spider): self._client = client @_wrap_http_errors - def update_tags(self, add=None, remove=None): + def update_tags(self, add: list[str] | None = None, + remove: list[str] | None = None) -> None: """Update tags for the spider. :param add: (optional) a list of string tags to add. @@ -118,7 +126,7 @@ def update_tags(self, add=None, remove=None): response.raise_for_status() @_wrap_http_errors - def list_tags(self): + def list_tags(self) -> Any: """List spider tags. :return: a list of spider tags. diff --git a/scrapinghub/client/utils.py b/scrapinghub/client/utils.py index dc89e2fb..278d8f63 100644 --- a/scrapinghub/client/utils.py +++ b/scrapinghub/client/utils.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import absolute_import, annotations import os import json @@ -6,6 +6,8 @@ import binascii import warnings from codecs import decode +from collections.abc import Sequence +from typing import Any import six from dotenv import dotenv_values, find_dotenv @@ -18,7 +20,9 @@ _DOTENV_AUTH_VARS = _APIKEY_VARS + ('SHUB_JOBAUTH',) -def _read_dotenv_auth(dotenv_path=None): +def _read_dotenv_auth( + dotenv_path: str | os.PathLike[str] | None = None, +) -> dict[str, str]: """Read Scrapy Cloud auth credentials from a ``.env`` file. Only the ``SH_APIKEY``, ``SHUB_APIKEY`` and ``SHUB_JOBAUTH`` variables are @@ -30,7 +34,8 @@ def _read_dotenv_auth(dotenv_path=None): or its parents is used. """ values = dotenv_values(dotenv_path or find_dotenv(usecwd=True)) - return {var: values[var] for var in _DOTENV_AUTH_VARS if values.get(var)} + return {var: value for var in _DOTENV_AUTH_VARS + if (value := values.get(var))} class LogLevel(object): @@ -44,16 +49,16 @@ class LogLevel(object): class JobKey(object): - def __init__(self, project_id, spider_id, job_id): + def __init__(self, project_id: str, spider_id: str, job_id: str) -> None: self.project_id = project_id self.spider_id = spider_id self.job_id = job_id - def __str__(self): + def __str__(self) -> str: return '{}/{}/{}'.format(self.project_id, self.spider_id, self.job_id) -def parse_project_id(project_id): +def parse_project_id(project_id: int | str) -> str: """Simple check for project id. :param project_id: a numeric project id, int or string. @@ -67,13 +72,14 @@ def parse_project_id(project_id): return str(project_id) -def parse_job_key(job_key): +def parse_job_key(job_key: str | tuple[int | str, ...]) -> JobKey: """Inner helper to parse job key. :param job_key: a job key (str or tuple of 3 ints). :return: parsed job key. :rtype: :class:`JobKey` """ + parts: Sequence[int | str] if isinstance(job_key, tuple): parts = job_key elif isinstance(job_key, six.string_types): @@ -93,9 +99,9 @@ def parse_job_key(job_key): return JobKey(*map(str, parts)) -def get_tags_for_update(**kwargs): +def get_tags_for_update(**kwargs: list[str] | None) -> dict[str, Any]: """Helper to check tags changes""" - params = {} + params: dict[str, Any] = {} for k, v in kwargs.items(): if not v: continue @@ -105,13 +111,16 @@ def get_tags_for_update(**kwargs): return params -def update_kwargs(kwargs, **params): +def update_kwargs(kwargs: dict[str, Any], **params: Any) -> None: """Update kwargs dict with non-empty params with json-encoded values.""" kwargs.update({k: json.dumps(v) if isinstance(v, dict) else v for k, v in params.items() if v is not None}) -def parse_auth(auth, dotenv_path=None): +def parse_auth( + auth: str | tuple[str, str] | None, + dotenv_path: str | os.PathLike[str] | None = None, +) -> tuple[str, str]: """Parse authentication token. When ``auth`` is None, the credentials are read from the ``SH_APIKEY`` (or @@ -153,7 +162,7 @@ def parse_auth(auth, dotenv_path=None): if jobauth: warnings.warn("You are using the SHUB_JOBAUTH credentials which " "may not work for some API endpoints") - return _search_for_jwt_credentials(jobauth) + return _search_for_jwt_credentials(jobauth) # type: ignore[return-value] raise RuntimeError("No API key provided and neither SH_APIKEY, " "SHUB_APIKEY nor SHUB_JOBAUTH environment variables " @@ -176,16 +185,15 @@ def parse_auth(auth, dotenv_path=None): return login, password -def _search_for_jwt_credentials(auth): +def _search_for_jwt_credentials(auth: str) -> tuple[str, str] | None: try: decoded_auth = decode(auth, 'hex_codec') except (binascii.Error, TypeError): - return + return None try: - if not isinstance(decoded_auth, six.string_types): - decoded_auth = decoded_auth.decode('ascii') - login, _, password = decoded_auth.partition(':') + login, _, password = decoded_auth.decode('ascii').partition(':') if password and parse_job_key(login): return login, password except (UnicodeDecodeError, ValueError): pass + return None diff --git a/scrapinghub/hubstorage/__init__.py b/scrapinghub/hubstorage/__init__.py index cb011905..9edb84a2 100644 --- a/scrapinghub/hubstorage/__init__.py +++ b/scrapinghub/hubstorage/__init__.py @@ -1,7 +1,7 @@ """ HubStorage client library """ -__all__ = ["HubstorageClient"] +__all__ = ["HubstorageClient", "ValueTooLarge"] from .client import HubstorageClient from .batchuploader import ValueTooLarge diff --git a/scrapinghub/hubstorage/activity.py b/scrapinghub/hubstorage/activity.py index acacfdac..a26e09bb 100644 --- a/scrapinghub/hubstorage/activity.py +++ b/scrapinghub/hubstorage/activity.py @@ -1,3 +1,8 @@ +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + from .resourcetype import ResourceType # TODO: remove backwards compatible methods @@ -7,13 +12,13 @@ class Activity(ResourceType): resource_type = 'activity' - def list(self, **params): + def list(self, **params: Any) -> Iterator[Any]: return self.apiget(params=params) get = list - def post(self, _value, **params): + def post(self, _value: Any, **params: Any) -> Iterator[Any]: return self.apipost(jl=_value, params=params) - def add(self, *args, **kwargs): + def add(self, *args: Any, **kwargs: Any) -> Iterator[Any]: entry = dict(*args, **kwargs) return self.post(entry) diff --git a/scrapinghub/hubstorage/batchuploader.py b/scrapinghub/hubstorage/batchuploader.py index c0601cd6..20ca9bd0 100644 --- a/scrapinghub/hubstorage/batchuploader.py +++ b/scrapinghub/hubstorage/batchuploader.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import time import socket import random @@ -12,9 +14,14 @@ import requests from collections import deque from threading import Thread, Event -from .utils import xauth, iterqueue, sizeof_fmt +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING, Any +from .utils import _Auth, xauth, iterqueue, sizeof_fmt from .serialization import jsonencode +if TYPE_CHECKING: + from .client import HubstorageClient + logger = logging.getLogger('hubstorage.batchuploader') @@ -38,18 +45,21 @@ class BatchUploader(object): # - standard deviation = approx. 40m, which means that 95% of the time the # total delay will be within 2*std = 1h20m of the average. - def __init__(self, client): + def __init__(self, client: HubstorageClient) -> None: self.client = client self.closed = False self._wait_event = Event() - self._writers = deque() + self._writers: deque[_BatchWriter] = deque() self._thread = Thread(target=self._worker) self._thread.daemon = True self._thread.start() - def create_writer(self, url, start=0, auth=None, size=1000, interval=15, - qsize=None, content_encoding='identity', - maxitemsize=1024 ** 2, callback=None): + def create_writer( + self, url: str, start: int = 0, auth: _Auth = None, size: int = 1000, + interval: float = 15, qsize: int | None = None, + content_encoding: str = 'identity', maxitemsize: int = 1024 ** 2, + callback: Callable[[requests.Response | None], object] | None = None, + ) -> _BatchWriter: # callback shouldn't try to inject more items in the queue # otherwise it can lead to deadlock on _checkpoint step assert not self.closed, 'Can not create new writers when closed' @@ -67,24 +77,24 @@ def create_writer(self, url, start=0, auth=None, size=1000, interval=15, self._writers.append(w) return w - def close(self, timeout=None): + def close(self, timeout: float | None = None) -> None: self.closed = True self.interrupt() self._thread.join(timeout) - def interrupt(self): + def interrupt(self) -> None: self._wait_event.set() - def __del__(self): + def __del__(self) -> None: if not self.closed: warnings.warn("%r not closed properly, some items may have been " "lost!: %r" % (self.__class__.__name__, self._writers)) - def _interruptable_sleep(self): + def _interruptable_sleep(self) -> None: self._wait_event.wait(self.worker_loop_delay) self._wait_event.clear() - def _worker(self): + def _worker(self) -> None: ctr = count() while True: if not self._writers: @@ -116,7 +126,7 @@ def _worker(self): if not (w.closed and w.itemsq.empty()): self._writers.append(w) - def _checkpoint(self, w): + def _checkpoint(self, w: _BatchWriter) -> None: q = w.itemsq qiter = iterqueue(q, w.size) data = self._content_encode(qiter, w) @@ -137,7 +147,7 @@ def _checkpoint(self, w): for _ in range(qiter.count): q.task_done() - def _content_encode(self, qiter, w): + def _content_encode(self, qiter: iterqueue, w: _BatchWriter) -> bytes: ce = w.content_encoding if ce == 'identity': return _encode_identity(qiter) @@ -146,7 +156,7 @@ def _content_encode(self, qiter, w): else: raise ValueError('Writer using unknown content encoding: %s' % ce) - def _tryupload(self, batch): + def _tryupload(self, batch: dict[str, Any]) -> requests.Response | None: """Retry uploads in case of server failures Use polinomial backoff with 10 minutes maximum interval that accounts @@ -168,7 +178,7 @@ def _tryupload(self, batch): r.status_code, r.reason, r.text.rstrip()) return r except (socket.error, requests.RequestException) as e: - if isinstance(e, requests.HTTPError): + if isinstance(e, requests.HTTPError) and e.response is not None: emsg = "[HTTP error {0}] {1}".format(e.response.status_code, e.response.text.rstrip()) else: @@ -182,8 +192,9 @@ def _tryupload(self, batch): backoff = min(max(retryn ** 2, self.worker_min_interval), self.worker_max_interval) time.sleep(backoff * (0.5 + random.random())) + return None - def _upload(self, batch): + def _upload(self, batch: dict[str, Any]) -> requests.Response: params = {'start': batch['offset']} headers = {'content-encoding': batch['content-encoding']} return self.client.session.request( @@ -205,8 +216,12 @@ class _BatchWriter(object): #: Truncate overly big items to that many bytes for the error message. ERRMSG_DATA_TRUNCATION_LEN = 1024 - def __init__(self, url, start, auth, size, interval, qsize, - maxitemsize, content_encoding, uploader, callback=None): + def __init__( + self, url: str, start: int, auth: tuple[str, str] | None, size: int, + interval: float, qsize: int | None, maxitemsize: int, + content_encoding: str, uploader: BatchUploader, + callback: Callable[[requests.Response | None], object] | None = None, + ) -> None: self.url = url self.offset = start self._nextid = count(start) @@ -216,13 +231,13 @@ def __init__(self, url, start, auth, size, interval, qsize, self.maxitemsize = maxitemsize self.content_encoding = content_encoding self.checkpoint = time.time() - self.itemsq = Queue(size * 2 if qsize is None else qsize) + self.itemsq: Queue[str] = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False self.uploader = uploader self.callback = callback - def write(self, item): + def write(self, item: Any) -> int: assert not self.closed, 'attempting writes to a closed writer' data = jsonencode(item) if len(data) > self.maxitemsize: @@ -236,25 +251,25 @@ def write(self, item): self.uploader.interrupt() return next(self._nextid) - def flush(self): + def flush(self) -> None: self.flushme = True self._waitforq() self.flushme = False - def close(self, block=True): + def close(self, block: bool = True) -> None: self.closed = True if block: self._waitforq() - def _waitforq(self): + def _waitforq(self) -> None: self.uploader.interrupt() self.itemsq.join() - def __str__(self): + def __str__(self) -> str: return self.url -def _encode_identity(iterable): +def _encode_identity(iterable: Iterable[str | bytes]) -> bytes: data = BytesIO() for item in iterable: if isinstance(item, six.text_type): @@ -264,7 +279,7 @@ def _encode_identity(iterable): return data.getvalue() -def _encode_gzip(iterable): +def _encode_gzip(iterable: Iterable[str | bytes]) -> bytes: data = BytesIO() with GzipFile(fileobj=data, mode='w') as gzo: for item in iterable: diff --git a/scrapinghub/hubstorage/client.py b/scrapinghub/hubstorage/client.py index 72d2a556..31e2ab1e 100644 --- a/scrapinghub/hubstorage/client.py +++ b/scrapinghub/hubstorage/client.py @@ -1,12 +1,15 @@ """ High level Hubstorage client """ +from __future__ import annotations + import logging import os +from typing import Any -from requests import session, HTTPError, ConnectionError, Timeout +from requests import session, HTTPError, ConnectionError, Response, Session, Timeout from retrying import Retrying -from .utils import xauth, urlpathjoin +from .utils import _Auth, _Part, xauth, urlpathjoin from .project import Project from .job import Job from .jobq import JobQ @@ -22,9 +25,10 @@ _HTTP_ERROR_CODES_TO_RETRY = (408, 429, 502, 503, 504) -def _hc_retry_on_exception(err): +def _hc_retry_on_exception(err: BaseException) -> bool: """Callback used by the client to restrict the retry to acceptable errors""" - if isinstance(err, HTTPError) and err.response.status_code in _HTTP_ERROR_CODES_TO_RETRY: + if (isinstance(err, HTTPError) and err.response is not None + and err.response.status_code in _HTTP_ERROR_CODES_TO_RETRY): logger.warning("Server failed with %d status code, retrying (maybe)", err.response.status_code) return True @@ -39,7 +43,7 @@ def _hc_retry_on_exception(err): return False -def _get_package_version(): +def _get_package_version() -> str: """Small helper to avoid circular imports""" from scrapinghub import __version__ return __version__ @@ -47,7 +51,7 @@ def _get_package_version(): class _JobQClientProxy: - def __init__(self, client, endpoint): + def __init__(self, client: HubstorageClient, endpoint: str) -> None: self.auth = client.auth self.endpoint = endpoint self.use_msgpack = False @@ -67,9 +71,12 @@ class HubstorageClient(object): RETRY_DEFAULT_JITTER_MS = 500 RETRY_DEFAULT_EXPONENTIAL_BACKOFF_MS = 500 - def __init__(self, auth=None, endpoint=None, connection_timeout=None, - max_retries=None, max_retry_time=None, user_agent=None, - use_msgpack=True, *, jobq_endpoint=None): + def __init__(self, auth: _Auth = None, endpoint: str | None = None, + connection_timeout: float | None = None, + max_retries: int | None = None, + max_retry_time: float | None = None, + user_agent: str | None = None, use_msgpack: bool = True, *, + jobq_endpoint: str | None = None) -> None: """ Note: max_retries and max_retry_time change how the client attempt to retry failing requests that are @@ -93,7 +100,7 @@ def __init__(self, auth=None, endpoint=None, connection_timeout=None, value of ``endpoint``. """ self.auth = xauth(auth) - self.endpoint = endpoint or os.getenv("SHUB_STORAGE", self.DEFAULT_ENDPOINT) + self.endpoint = endpoint or os.environ.get("SHUB_STORAGE", self.DEFAULT_ENDPOINT) self._jobq_endpoint = ( jobq_endpoint or os.getenv("SHUB_JOBQ") or @@ -106,14 +113,14 @@ def __init__(self, auth=None, endpoint=None, connection_timeout=None, self._jobq_client = _JobQClientProxy(self, self._jobq_endpoint) self.jobq = JobQ(self._jobq_client, None) self.projects = Projects(self, None) - self.root = ResourceType(self, None) - self._batchuploader = None - self.use_msgpack = MSGPACK_AVAILABLE and use_msgpack + self.root: ResourceType = ResourceType(self, None) + self._batchuploader: BatchUploader | None = None + self.use_msgpack: bool = MSGPACK_AVAILABLE and use_msgpack if use_msgpack != self.use_msgpack: logger.warning('Messagepack is not available, please ensure that ' 'msgpack library is properly installed.') - def request(self, is_idempotent=False, **kwargs): + def request(self, is_idempotent: bool = False, **kwargs: Any) -> Response: """ Execute an HTTP request with the current client session. @@ -121,7 +128,7 @@ def request(self, is_idempotent=False, **kwargs): """ kwargs.setdefault('timeout', self.connection_timeout) - def invoke_request(): + def invoke_request() -> Response: r = self.session.request(**kwargs) try: @@ -132,11 +139,13 @@ def invoke_request(): raise if is_idempotent: - return self.retrier.call(invoke_request) + response: Response = self.retrier.call(invoke_request) + return response else: return invoke_request() - def _create_retrier(self, max_retries, max_retry_time): + def _create_retrier(self, max_retries: int | None, + max_retry_time: float | None) -> Retrying: """ Create the Retrier object used to process idempotent client requests. @@ -151,9 +160,9 @@ def _create_retrier(self, max_retries, max_retry_time): # Client sets max_retries only if max_retries is not None and max_retry_time is None: - stop_max_delay = None + stop_max_delay: float | None = None stop_max_attempt_number = max_retries + 1 - wait_exponential_multiplier = self.RETRY_DEFAULT_EXPONENTIAL_BACKOFF_MS + wait_exponential_multiplier: float = self.RETRY_DEFAULT_EXPONENTIAL_BACKOFF_MS else: stop_max_delay = (max_retry_time or self.RETRY_DEFAUT_MAX_RETRY_TIME_S) * 1000.0 stop_max_attempt_number = (max_retries or self.RETRY_DEFAULT_MAX_RETRIES) + 1 @@ -169,32 +178,33 @@ def _create_retrier(self, max_retries, max_retry_time): wait_exponential_multiplier=wait_exponential_multiplier, wait_jitter_max=self.RETRY_DEFAULT_JITTER_MS) - def _create_session(self): + def _create_session(self) -> Session: s = session() s.headers.update({'User-Agent': self.user_agent}) return s @property - def batchuploader(self): + def batchuploader(self) -> BatchUploader: if self._batchuploader is None: self._batchuploader = BatchUploader(self) return self._batchuploader - def get_job(self, *args, **kwargs): + def get_job(self, *args: Any, **kwargs: Any) -> Job: return Job(self, *args, **kwargs) - def push_job(self, projectid, spidername, auth=None, **jobparams): + def push_job(self, projectid: _Part, spidername: str, auth: _Auth = None, + **jobparams: Any) -> Job: project = self.projects.get(projectid, auth=auth) return project.push_job(spidername, **jobparams) - def get_project(self, *args, **kwargs): + def get_project(self, *args: Any, **kwargs: Any) -> Project: return self.projects.get(*args, **kwargs) - def server_timestamp(self): + def server_timestamp(self) -> Any: tsurl = urlpathjoin(self.endpoint, 'system/ts') return self.session.get(tsurl).json() - def close(self, timeout=None): + def close(self, timeout: float | None = None) -> None: if self._batchuploader is not None: self.batchuploader.close(timeout) @@ -203,9 +213,11 @@ class Projects(ResourceType): resource_type = 'projects' - def get(self, *args, **kwargs): + client: HubstorageClient + + def get(self, *args: Any, **kwargs: Any) -> Project: return Project(self.client, *args, **kwargs) - def jobsummaries(self, auth=None, **params): + def jobsummaries(self, auth: _Auth = None, **params: Any) -> Any: auth = xauth(auth) or self.auth return next(self.apiget('jobsummaries', params=params, auth=auth)) diff --git a/scrapinghub/hubstorage/collectionsrt.py b/scrapinghub/hubstorage/collectionsrt.py index 4818b19a..89eeceac 100644 --- a/scrapinghub/hubstorage/collectionsrt.py +++ b/scrapinghub/hubstorage/collectionsrt.py @@ -1,9 +1,16 @@ +from __future__ import annotations + import re +from collections.abc import Callable, Iterable, Iterator +from typing import TYPE_CHECKING, Any from requests.exceptions import HTTPError from .resourcetype import DownloadableResource -from .utils import urlpathjoin +from .utils import _Part, urlpathjoin + +if TYPE_CHECKING: + from .batchuploader import _BatchWriter COLLECTIONS_MSGPACK_REGEX = re.compile( @@ -23,7 +30,7 @@ class Collections(DownloadableResource): resource_type = 'collections' - def _allows_mpack(self, path=None): + def _allows_mpack(self, path: _Part | None = None) -> bool: """Check if request can be served with msgpack data. Collection scan and get requests for keys are able to return msgpack data. @@ -38,11 +45,13 @@ def _allows_mpack(self, path=None): # count endpoint doesn't support msgpack return bool(match and match.group('key') != 'count') - def get(self, _type, _name, _key=None, **params): + def get(self, _type: str, _name: str, _key: str | None = None, + **params: Any) -> Any: try: r = self.apiget((_type, _name, _key), params=params) return r if _key is None else next(r) except HTTPError as exc: + assert exc.response is not None if exc.response.status_code == 404: raise KeyError(_key) elif exc.response.status_code == 400: @@ -50,32 +59,41 @@ def get(self, _type, _name, _key=None, **params): else: raise - def set(self, _type, _name, _values): + def set(self, _type: str, _name: str, _values: Any) -> Iterator[Any]: try: return self.apipost((_type, _name), is_idempotent=True, jl=_values) except HTTPError as exc: + assert exc.response is not None if exc.response.status_code in (400, 413): raise ValueError(exc.response.text) else: raise - def delete(self, _type, _name, _keys): + def delete(self, _type: str, _name: str, + _keys: str | Iterable[str]) -> Iterator[Any]: return self.apipost((_type, _name, 'deleted'), is_idempotent=True, jl=_keys) - def truncate(self, _name): + def truncate(self, _name: str) -> Iterator[Any]: return self.apipost('delete', params={'name': _name}, is_idempotent=True) - def iter_json(self, _type, _name, requests_params=None, **apiparams): + def iter_json( # type: ignore[override] + self, _type: str, _name: str, + requests_params: dict[str, Any] | None = None, **apiparams: Any, + ) -> Iterator[str]: return DownloadableResource.iter_json( self, (_type, _name), requests_params=requests_params, **apiparams ) - def iter_msgpack(self, _type, _name, requests_params=None, **apiparams): + def iter_msgpack( # type: ignore[override] + self, _type: str, _name: str, + requests_params: dict[str, Any] | None = None, **apiparams: Any, + ) -> Iterator[bytes]: return DownloadableResource.iter_msgpack( self, (_type, _name), requests_params=requests_params, **apiparams ) - def create_writer(self, coltype, colname, **writer_kwargs): + def create_writer(self, coltype: str, colname: str, + **writer_kwargs: Any) -> _BatchWriter: self._validate_collection(coltype, colname) kwargs = dict(writer_kwargs) kwargs.setdefault('content_encoding', 'gzip') @@ -83,26 +101,26 @@ def create_writer(self, coltype, colname, **writer_kwargs): url = urlpathjoin(self.url, coltype, colname) return self.client.batchuploader.create_writer(url, **kwargs) - def new_collection(self, coltype, colname): + def new_collection(self, coltype: str, colname: str) -> Collection: self._validate_collection(coltype, colname) return Collection(coltype, colname, self) - def new_store(self, colname): + def new_store(self, colname: str) -> Collection: return self.new_collection('s', colname) - def new_cached_store(self, colname): + def new_cached_store(self, colname: str) -> Collection: return self.new_collection('cs', colname) - def new_versioned_store(self, colname): + def new_versioned_store(self, colname: str) -> Collection: return self.new_collection('vs', colname) - def new_versioned_cached_store(self, colname): + def new_versioned_cached_store(self, colname: str) -> Collection: return self.new_collection('vcs', colname) - def count(self, _type, _name, **params): + def count(self, _type: str, _name: str, **params: Any) -> int: return self._batch('GET', (_type, _name, 'count'), 'count', **params) - def _validate_collection(self, coltype, colname): + def _validate_collection(self, coltype: str, colname: str) -> None: if coltype not in {'s', 'cs', 'vs', 'vcs'}: raise ValueError('Invalid collection type: {}'.format(coltype)) @@ -110,8 +128,10 @@ def _validate_collection(self, coltype, colname): raise ValueError('Invalid collection name {!r}, only alphanumeric ' 'characters'.format(colname)) - def _batch(self, method, path, total_param, progress=None, **params): - total = 0 + def _batch(self, method: str, path: _Part, total_param: str, + progress: Callable[[int, Any], object] | None = None, + **params: Any) -> int: + total: int = 0 getparams = dict(params) try: while True: @@ -128,6 +148,7 @@ def _batch(self, method, path, total_param, progress=None, **params): progress(total, next_start) return total except HTTPError as exc: + assert exc.response is not None if exc.response.status_code == 400: raise ValueError(exc.response.text) else: @@ -136,12 +157,13 @@ def _batch(self, method, path, total_param, progress=None, **params): class Collection(object): - def __init__(self, coltype, colname, collections): + def __init__(self, coltype: str, colname: str, + collections: Collections) -> None: self.coltype = coltype self.colname = colname self._collections = collections - def create_writer(self, **kwargs): + def create_writer(self, **kwargs: Any) -> _BatchWriter: """Create a writer for async writing of bulk data kwargs are passed to batchuploader.create_writer, but auth and gzip @@ -149,27 +171,29 @@ def create_writer(self, **kwargs): """ return self._collections.create_writer(self.coltype, self.colname, **kwargs) - def get(self, *args, **kwargs): + def get(self, *args: Any, **kwargs: Any) -> Any: return self._collections.get(self.coltype, self.colname, *args, **kwargs) - def set(self, *args, **kwargs): + def set(self, *args: Any, **kwargs: Any) -> Iterator[Any]: return self._collections.set(self.coltype, self.colname, *args, **kwargs) - def delete(self, *args, **kwargs): + def delete(self, *args: Any, **kwargs: Any) -> Iterator[Any]: return self._collections.delete(self.coltype, self.colname, *args, **kwargs) - def truncate(self): + def truncate(self) -> Iterator[Any]: return self._collections.truncate(self.colname) - def count(self, *args, **kwargs): + def count(self, *args: Any, **kwargs: Any) -> int: return self._collections.count(self.coltype, self.colname, *args, **kwargs) - def iter_json(self, requests_params=None, **apiparams): + def iter_json(self, requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[str]: return self._collections.iter_json( self.coltype, self.colname, requests_params=requests_params, **apiparams ) - def iter_values(self, requests_params=None, **apiparams): + def iter_values(self, requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[Any]: return self._collections.iter_values( self.coltype, self.colname, requests_params=requests_params, **apiparams ) diff --git a/scrapinghub/hubstorage/frontier.py b/scrapinghub/hubstorage/frontier.py index e82f2d90..088a2c06 100644 --- a/scrapinghub/hubstorage/frontier.py +++ b/scrapinghub/hubstorage/frontier.py @@ -1,12 +1,25 @@ +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING, Any + +import requests + from .resourcetype import ResourceType from .utils import urlpathjoin +if TYPE_CHECKING: + from .batchuploader import _BatchWriter + from .client import HubstorageClient + class Frontier(ResourceType): resource_type = 'hcf' + client: HubstorageClient + batch_size = 5000 batch_qsize = 6000 # defaults to twice batch_size if None batch_start = 0 @@ -14,12 +27,12 @@ class Frontier(ResourceType): batch_append = False batch_content_encoding = 'identity' - def __init__(self, *a, **kw): - self._writers = {} # dict of writers indexed by (frontier, slot) + def __init__(self, *a: Any, **kw: Any) -> None: + self._writers: dict[tuple[str, str], _BatchWriter] = {} # dict of writers indexed by (frontier, slot) self.newcount = 0 super(Frontier, self).__init__(*a, **kw) - def _get_writer(self, frontier, slot): + def _get_writer(self, frontier: str, slot: str) -> _BatchWriter: key = (frontier, slot) writer = self._writers.get(key) if not writer: @@ -36,30 +49,32 @@ def _get_writer(self, frontier, slot): self._writers[key] = writer return writer - def _writer_callback(self, response): + def _writer_callback(self, response: requests.Response | None) -> None: + assert response is not None self.newcount += response.json()["newcount"] - def close(self, block=True): + def close(self, block: bool = True) -> None: for writer in self._writers.values(): writer.close(block=block) - def flush(self): + def flush(self) -> None: for writer in self._writers.values(): writer.flush() - def add(self, frontier, slot, fps): + def add(self, frontier: str, slot: str, fps: Iterable[Any]) -> None: writer = self._get_writer(frontier, slot) for fp in fps: writer.write(fp) - def read(self, frontier, slot, mincount=None): - params = {} + def read(self, frontier: str, slot: str, + mincount: int | None = None) -> Iterator[Any]: + params: dict[str, Any] = {} if mincount is not None: params['mincount'] = mincount return self.apiget((frontier, 's', slot, 'q'), params=params) - def delete(self, frontier, slot, ids): + def delete(self, frontier: str, slot: str, ids: Iterable[str]) -> None: self.apipost((frontier, 's', slot, 'q/deleted'), jl=ids, is_idempotent=True) - def delete_slot(self, frontier, slot): + def delete_slot(self, frontier: str, slot: str) -> None: self.apidelete((frontier, 's', slot)) diff --git a/scrapinghub/hubstorage/job.py b/scrapinghub/hubstorage/job.py index f1c99c95..4835327d 100644 --- a/scrapinghub/hubstorage/job.py +++ b/scrapinghub/hubstorage/job.py @@ -1,13 +1,22 @@ +from __future__ import annotations + import logging +from typing import TYPE_CHECKING, Any, NoReturn + from .resourcetype import (ItemsResourceType, DownloadableResource, MappingResourceType) -from .utils import millitime, urlpathjoin +from .utils import _Auth, _Part, millitime, urlpathjoin from .jobq import JobQ +if TYPE_CHECKING: + from .client import HubstorageClient + class Job(object): - def __init__(self, client, key, auth=None, jobauth=None, metadata=None): + def __init__(self, client: HubstorageClient, key: _Part, auth: _Auth = None, + jobauth: _Auth = None, + metadata: dict[str, Any] | None = None) -> None: self.key = urlpathjoin(key) assert len(self.key.split('/')) == 3, \ 'Jobkey must be projectid/spiderid/jobid: %s' % self.key @@ -21,7 +30,7 @@ def __init__(self, client, key, auth=None, jobauth=None, metadata=None): jobq_client = getattr(client, '_jobq_client', client) self.jobq = JobQ(jobq_client, self.key.split('/')[0], auth) - def close_writers(self): + def close_writers(self) -> None: wl = [self.items, self.logs, self.samples, self.requests] # close all resources that use background writers for w in wl: @@ -30,15 +39,15 @@ def close_writers(self): for w in wl: w.close(block=True) - def update_metadata(self, *args, **kwargs): + def update_metadata(self, *args: Any, **kwargs: Any) -> None: self.metadata.update(*args, **kwargs) self.metadata.save() self.metadata.expire() - def request_cancel(self): + def request_cancel(self) -> None: self.jobq.request_cancel(self) - def purged(self): + def purged(self) -> None: self.jobq.delete(self) self.metadata.expire() @@ -48,7 +57,7 @@ class JobMeta(MappingResourceType): resource_type = 'jobs' ignore_fields = set(('auth', '_key', 'state')) - def authtoken(self): + def authtoken(self) -> Any: return self.liveget('auth') @@ -57,16 +66,19 @@ class Logs(ItemsResourceType, DownloadableResource): resource_type = 'logs' batch_content_encoding = 'gzip' - def __init__(self, client, key, auth=None, appendmode=False): + def __init__(self, client: HubstorageClient, key: _Part, + auth: _Auth = None, appendmode: bool = False) -> None: ItemsResourceType.__init__(self, client, key, auth) self.batch_append = appendmode - def batch_write_start(self): + def batch_write_start(self) -> int: if self.batch_append: - return self.stats().get('totals', {}).get('input_values', 0) + start: int = self.stats().get('totals', {}).get('input_values', 0) + return start return 0 - def log(self, message, level=logging.INFO, ts=None, **other): + def log(self, message: str, level: int = logging.INFO, + ts: int | None = None, **other: Any) -> None: other.update(message=message, level=level, time=ts or millitime()) # legacy support for an appendmode argument. This should be set at # object initialization time. @@ -74,17 +86,17 @@ def log(self, message, level=logging.INFO, ts=None, **other): self.batch_append = True self.write(other) - def debug(self, message, **other): + def debug(self, message: str, **other: Any) -> None: self.log(message, level=logging.DEBUG, **other) - def info(self, message, **other): + def info(self, message: str, **other: Any) -> None: self.log(message, level=logging.INFO, **other) - def warn(self, message, **other): + def warn(self, message: str, **other: Any) -> None: self.log(message, level=logging.WARNING, **other) warning = warn - def error(self, message, **other): + def error(self, message: str, **other: Any) -> None: self.log(message, level=logging.ERROR, **other) @@ -98,7 +110,7 @@ class Samples(ItemsResourceType): resource_type = 'samples' - def stats(self): + def stats(self) -> NoReturn: raise NotImplementedError('Resource does not expose stats') @@ -107,7 +119,9 @@ class Requests(ItemsResourceType, DownloadableResource): resource_type = 'requests' batch_content_encoding = 'gzip' - def add(self, url, status, method, rs, parent, duration, ts, fp=None): + def add(self, url: str, status: int, method: str, rs: int, + parent: int | None, duration: int, ts: int, + fp: str | None = None) -> int: return self.write({ 'url': url, 'status': int(status), diff --git a/scrapinghub/hubstorage/jobq.py b/scrapinghub/hubstorage/jobq.py index bae12a21..abe6252f 100644 --- a/scrapinghub/hubstorage/jobq.py +++ b/scrapinghub/hubstorage/jobq.py @@ -1,8 +1,21 @@ +from __future__ import annotations + import json +from collections.abc import Iterator +from typing import Any, Protocol, Union + from requests.exceptions import HTTPError from .resourcetype import ResourceType +class _HasKey(Protocol): + @property + def key(self) -> str: ... + + +_JobRef = Union[_HasKey, dict[str, Any], str, list["_JobRef"]] + + class DuplicateJobError(Exception): """Raised when a job with same unique is pushed""" @@ -17,7 +30,7 @@ class JobQ(ResourceType): PRIO_HIGH = 3 PRIO_HIGHEST = 4 - def push(self, spider, **jobparams): + def push(self, spider: str, **jobparams: Any) -> Any: jobparams['spider'] = spider try: for o in self.apipost('push', jl=jobparams): @@ -31,15 +44,18 @@ def push(self, spider, **jobparams): raise DuplicateJobError() raise - def jobsummary(self, jobkeys, jobmeta): + def jobsummary(self, jobkeys: list[str] | tuple[str, ...], + jobmeta: Any) -> Iterator[Any]: """Fetch selected job metadata fields for selected jobs.""" if not isinstance(jobkeys, (list, tuple)): raise TypeError("jobkeys must be a list or a tuple") return self.apiget(('jobsummary',), params={'key': jobkeys, 'jobmeta': jobmeta}) - def summary(self, _queuename=None, spiderid=None, count=None, start=None, jobmeta=None): - params = {} + def summary(self, _queuename: str | None = None, + spiderid: str | None = None, count: int | None = None, + start: int | None = None, jobmeta: Any = None) -> Any: + params: dict[str, Any] = {} if count is not None: params['count'] = count if start is not None: @@ -50,9 +66,12 @@ def summary(self, _queuename=None, spiderid=None, count=None, start=None, jobmet r = list(self.apiget((spiderid, 'summary', _queuename), params=params)) return (r and r[0] or None) if _queuename else r - def list(self, spider=None, count=None, stop=None, state=None, - has_tag=None, lacks_tag=None, startts=None, endts=None, - **params): + def list(self, spider: str | None = None, count: int | None = None, + stop: str | None = None, state: str | list[str] | None = None, + has_tag: str | list[str] | None = None, + lacks_tag: str | list[str] | None = None, + startts: int | None = None, endts: int | None = None, + **params: Any) -> Iterator[Any]: if 'filter' in params: return self._legacy_list_with_filter(params) @@ -74,7 +93,7 @@ def list(self, spider=None, count=None, stop=None, state=None, params['lacks_tag'] = lacks_tag return self.apiget(('list',), params=params) - def _legacy_list_with_filter(self, params): + def _legacy_list_with_filter(self, params: dict[str, Any]) -> Iterator[Any]: only_finished_outcome = False for row in params['filter']: field, matchdecider, value = json.loads(row) @@ -95,7 +114,7 @@ def _legacy_list_with_filter(self, params): return (x for x in jobs if x.get('close_reason') == 'finished') return jobs - def start(self, job=None, **start_params): + def start(self, job: _JobRef | None = None, **start_params: Any) -> Any: """Start a new job If a job is passed, it is changed to the started state and metadata @@ -112,17 +131,17 @@ def start(self, job=None, **start_params): for o in self.apipost('startjob', jl=start_params): return o - def request_cancel(self, job): + def request_cancel(self, job: _HasKey) -> None: """Cancel a running job""" self.apipost("%s/cancel" % job.key[job.key.index('/') + 1:]) - def finish(self, job, **params): + def finish(self, job: _JobRef, **params: Any) -> Iterator[Any]: return self.update(job, state='finished', **params) - def delete(self, job, **params): + def delete(self, job: _JobRef, **params: Any) -> Iterator[Any]: return self.update(job, state='deleted', **params) - def _jobkeys(self, job): + def _jobkeys(self, job: _JobRef) -> Iterator[str]: if isinstance(job, list): for x in job: for k in self._jobkeys(x): @@ -134,6 +153,6 @@ def _jobkeys(self, job): else: yield job - def update(self, job, **extra_args): + def update(self, job: _JobRef, **extra_args: Any) -> Iterator[Any]: data = [dict(extra_args, key=k) for k in self._jobkeys(job)] return self.apipost('update', jl=data) diff --git a/scrapinghub/hubstorage/project.py b/scrapinghub/hubstorage/project.py index 62801605..e8b71bc8 100644 --- a/scrapinghub/hubstorage/project.py +++ b/scrapinghub/hubstorage/project.py @@ -1,4 +1,8 @@ +from __future__ import annotations + import warnings +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any from .job import Job from .jobq import JobQ @@ -6,12 +10,16 @@ from .collectionsrt import Collections from .frontier import Frontier from .resourcetype import ResourceType, MappingResourceType -from .utils import urlpathjoin, xauth +from .utils import _Auth, _Part, urlpathjoin, xauth + +if TYPE_CHECKING: + from .client import HubstorageClient class Project(object): - def __init__(self, client, projectid, auth=None): + def __init__(self, client: HubstorageClient, projectid: _Part, + auth: _Auth = None) -> None: self.client = client self.projectid = urlpathjoin(projectid) assert len(self.projectid.split('/')) == 1, \ @@ -31,11 +39,12 @@ def __init__(self, client, projectid, auth=None): self.reports = Reports(client, self.projectid, auth=auth) self.spiders = Spiders(client, self.projectid, auth=auth) - def get_job(self, _key, *args, **kwargs): + def get_job(self, _key: _Part, *args: Any, **kwargs: Any) -> Job: key = urlpathjoin(_key) parts = key.split('/') + jobkey: _Part = key if len(parts) == 2: - key = (self.projectid, key) + jobkey = (self.projectid, key) elif len(parts) == 3 and parts[0] == self.projectid: pass else: @@ -43,21 +52,21 @@ def get_job(self, _key, *args, **kwargs): % (key, self.projectid)) kwargs.setdefault('auth', self.auth) - return self.client.get_job(key, *args, **kwargs) + return self.client.get_job(jobkey, *args, **kwargs) - def get_jobs(self, **kwargs): + def get_jobs(self, **kwargs: Any) -> Iterator[Job]: warnings.warn('Method `project.get_jobs()` is deprecated, ' 'use `project.jobq.list()` instead', Warning) for metadata in self.jobq.list(**kwargs): key = metadata.pop('key') yield self.get_job(key, metadata=metadata) - def push_job(self, spidername, **jobparams): + def push_job(self, spidername: str, **jobparams: Any) -> Job: data = self.jobq.push(spidername, **jobparams) key = data['key'] return Job(self.client, key, auth=self.auth) - def jobsummary(self, **params): + def jobsummary(self, **params: Any) -> Any: uri = ('projects', self.projectid, 'jobsummary') return next(self.client.root.apiget(uri, auth=self.auth, params=params)) @@ -66,7 +75,8 @@ class Jobs(ResourceType): resource_type = 'jobs' - def list(self, _key=None, **params): + def list(self, _key: _Part | None = None, + **params: Any) -> Iterator[Any]: return self.apiget(_key, params=params) @@ -74,7 +84,8 @@ class Items(ResourceType): resource_type = 'items' - def list(self, _key=None, **params): + def list(self, _key: _Part | None = None, + **params: Any) -> Iterator[Any]: return self.apiget(_key, params=params) @@ -82,7 +93,8 @@ class Logs(ResourceType): resource_type = 'logs' - def list(self, _key=None, **params): + def list(self, _key: _Part | None = None, + **params: Any) -> Iterator[Any]: return self.apiget(_key, params=params) @@ -90,7 +102,8 @@ class Samples(ResourceType): resource_type = 'samples' - def list(self, _key=None, **params): + def list(self, _key: _Part | None = None, + **params: Any) -> Iterator[Any]: return self.apiget(_key, params=params) @@ -98,7 +111,7 @@ class Ids(ResourceType): resource_type = 'ids' - def spider(self, spidername, **params): + def spider(self, spidername: str, **params: Any) -> Any: r = self.apiget(('spider', spidername), params=params) return next(r) @@ -119,5 +132,6 @@ class Spiders(ResourceType): resource_type = 'spiders' - def lastjobsummary(self, spiderid=None, **params): + def lastjobsummary(self, spiderid: str | None = None, + **params: Any) -> Iterator[Any]: return self.apiget((spiderid, 'lastjobsummary'), params=params) diff --git a/scrapinghub/hubstorage/resourcetype.py b/scrapinghub/hubstorage/resourcetype.py index 3ad8f545..0e17d15e 100644 --- a/scrapinghub/hubstorage/resourcetype.py +++ b/scrapinghub/hubstorage/resourcetype.py @@ -1,33 +1,52 @@ +from __future__ import annotations + import json import logging import socket import time +from collections.abc import Callable, Collection, Iterable, Iterator +from typing import TYPE_CHECKING, Any, Protocol import six +import requests import requests.exceptions as rexc from six.moves import range, collections_abc -from .utils import urlpathjoin, xauth +from .utils import _Auth, _Part, urlpathjoin, xauth from .serialization import jlencode, jldecode, mpdecode +if TYPE_CHECKING: + from .batchuploader import _BatchWriter + from .client import HubstorageClient + logger = logging.getLogger('hubstorage.resourcetype') CHUNK_SIZE = 512 STATS_CHUNK_SIZE = 512 * 1024 +class _Client(Protocol): + auth: tuple[str, str] | None + endpoint: str + use_msgpack: bool + + def request(self, is_idempotent: bool = False, + **kwargs: Any) -> requests.Response: ... + + class ResourceType(object): - resource_type = None - key_suffix = None + resource_type: str | None = None + key_suffix: str | None = None - def __init__(self, client, key, auth=None): + def __init__(self, client: _Client, key: _Part | None, + auth: _Auth = None) -> None: self.client = client self.key = urlpathjoin(self.resource_type, key, self.key_suffix) self.auth = xauth(auth) or client.auth self.url = urlpathjoin(client.endpoint, self.key) - def _allows_mpack(self, path=None): + def _allows_mpack(self, path: _Part | None = None) -> bool: """Check if request can be served with msgpack data. Currently, items, logs and samples endpoints are able to @@ -46,17 +65,19 @@ def _allows_mpack(self, path=None): ) @staticmethod - def _enforce_msgpack(**kwargs): + def _enforce_msgpack(**kwargs: Any) -> dict[str, Any]: kwargs.setdefault('headers', {}) kwargs['headers']['Accept'] = 'application/x-msgpack' return kwargs - def _iter_content(self, _path, **kwargs): + def _iter_content(self, _path: _Part | None, + **kwargs: Any) -> Iterator[bytes]: kwargs['url'] = urlpathjoin(self.url, _path) kwargs.setdefault('auth', self.auth) return self.client.request(**kwargs).iter_content(CHUNK_SIZE) - def _iter_lines(self, _path, **kwargs): + def _iter_lines(self, _path: _Part | None, + **kwargs: Any) -> Iterator[str]: kwargs['url'] = urlpathjoin(self.url, _path) kwargs.setdefault('auth', self.auth) chunk_size = kwargs.pop('chunk_size', CHUNK_SIZE) @@ -72,20 +93,24 @@ def _iter_lines(self, _path, **kwargs): return (l.decode(r.encoding or 'utf8') for l in lines) return lines - def apirequest(self, _path=None, **kwargs): - if self._allows_mpack(_path) and kwargs.get('method').upper() == 'GET': + def apirequest(self, _path: _Part | None = None, + **kwargs: Any) -> Iterator[Any]: + if self._allows_mpack(_path) and kwargs['method'].upper() == 'GET': kwargs = self._enforce_msgpack(**kwargs) return mpdecode(self._iter_content(_path=_path, **kwargs)) return jldecode(self._iter_lines(_path, **kwargs)) - def apipost(self, _path=None, **kwargs): + def apipost(self, _path: _Part | None = None, + **kwargs: Any) -> Iterator[Any]: return self.apirequest(_path, method='POST', **kwargs) - def apiget(self, _path=None, **kwargs): + def apiget(self, _path: _Part | None = None, + **kwargs: Any) -> Iterator[Any]: kwargs.setdefault('is_idempotent', True) return self.apirequest(_path, method='GET', **kwargs) - def apidelete(self, _path=None, **kwargs): + def apidelete(self, _path: _Part | None = None, + **kwargs: Any) -> Iterator[Any]: kwargs.setdefault('is_idempotent', True) return self.apirequest(_path, method='DELETE', **kwargs) @@ -94,8 +119,10 @@ class DownloadableResource(ResourceType): MAX_RETRIES = 180 RETRY_INTERVAL = 60 + client: HubstorageClient + @staticmethod - def _add_key_meta(params): + def _add_key_meta(params: dict[str, Any]) -> dict[str, Any]: """Adds meta=_key to ensure a key is returned""" meta = params.get('meta', []) if '_key' not in meta: @@ -104,7 +131,8 @@ def _add_key_meta(params): params['meta'] = meta return params - def _add_resume_param(self, lastline, offset, params): + def _add_resume_param(self, lastline: str | bytes | None, offset: int, + params: dict[str, Any]) -> None: """Adds a startafter=LASTKEY parameter if there was a lastvalue""" if lastline is not None: lastvalue = json.loads(lastline) @@ -112,7 +140,7 @@ def _add_resume_param(self, lastline, offset, params): if 'start' in params: del params['start'] - def iter_values(self, *args, **kwargs): + def iter_values(self, *args: Any, **kwargs: Any) -> Iterator[Any]: """Reliably iterate through all data as python objects calls either iter_json or iter_msgpack, decoding the results @@ -121,7 +149,10 @@ def iter_values(self, *args, **kwargs): return mpdecode(self.iter_msgpack(*args, **kwargs)) return jldecode(self.iter_json(*args, **kwargs)) - def _retry(self, iter_callback, resume=False, _path=None, requests_params=None, **apiparams): + def _retry(self, iter_callback: Callable[..., Iterable[Any]], + resume: bool = False, _path: _Part | None = None, + requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[Any]: """Reliable iterate through all data calling iter_callback""" self._add_key_meta(apiparams) lastexc = None @@ -132,7 +163,7 @@ def _retry(self, iter_callback, resume=False, _path=None, requests_params=None, self._add_resume_param(chunk, offset, apiparams) try: for chunk in iter_callback(_path=_path, params=apiparams, - **requests_params): + **(requests_params or {})): yield chunk offset += 1 break @@ -152,7 +183,9 @@ def _retry(self, iter_callback, resume=False, _path=None, requests_params=None, "last error was: %s", self.MAX_RETRIES, url, apiparams, lastexc) - def iter_msgpack(self, _path=None, requests_params=None, **apiparams): + def iter_msgpack(self, _path: _Part | None = None, + requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[bytes]: """Reliably iterate through all data as msgpack""" requests_params = dict(requests_params or {}) requests_params.setdefault('method', 'GET') @@ -163,7 +196,9 @@ def iter_msgpack(self, _path=None, requests_params=None, **apiparams): requests_params, **apiparams): yield chunk - def iter_json(self, _path=None, requests_params=None, **apiparams): + def iter_json(self, _path: _Part | None = None, + requests_params: dict[str, Any] | None = None, + **apiparams: Any) -> Iterator[str]: """Reliably iterate through all data as json strings""" requests_params = dict(requests_params or {}) requests_params.setdefault('method', 'GET') @@ -176,24 +211,26 @@ def iter_json(self, _path=None, requests_params=None, **apiparams): class ItemsResourceType(ResourceType): + client: HubstorageClient + batch_size = 1000 - batch_qsize = None # defaults to twice batch_size if None + batch_qsize: int | None = None # defaults to twice batch_size if None batch_start = 0 batch_interval = 15.0 batch_content_encoding = 'identity' # batch writer reference in case of used - _writer = None + _writer: _BatchWriter | None = None # TODO override _add_resume_param - can avoid requestomg _key by # deriving from project, spider, job and offset - def batch_write_start(self): + def batch_write_start(self) -> int: """Override to set a start parameter when commencing writing""" return 0 @property - def writer(self): + def writer(self) -> _BatchWriter: if self._writer is None: self._writer = self.client.batchuploader.create_writer( url=self.url, @@ -206,47 +243,49 @@ def writer(self): ) return self._writer - def flush(self): + def flush(self) -> None: if self._writer is not None: self._writer.flush() - def close(self, block=True): + def close(self, block: bool = True) -> None: if self._writer is not None: self._writer.close(block=block) - def write(self, item): + def write(self, item: Any) -> int: return self.writer.write(item) - def list(self, _key=None, **params): + def list(self, _key: _Part | None = None, + **params: Any) -> Iterator[Any]: return self.apiget(_key, params=params) - def get(self, _key, **params): + def get(self, _key: _Part | None, **params: Any) -> Any: """Return first matching result""" for o in self.list(_key, params=params): return o - def stats(self): + def stats(self) -> Any: return next(self.apiget('stats', chunk_size=STATS_CHUNK_SIZE)) -class MappingResourceType(ResourceType, collections_abc.MutableMapping): +class MappingResourceType(ResourceType, + collections_abc.MutableMapping[str, Any]): - _cached = None - ignore_fields = () + _cached: dict[str, Any] | None = None + ignore_fields: Collection[str] = () - def __init__(self, *a, **kw): + def __init__(self, *a: Any, **kw: Any) -> None: self._cached = kw.pop('cached', None) - self._deleted = set() + self._deleted: set[str] = set() super(MappingResourceType, self).__init__(*a, **kw) - def __str__(self): + def __str__(self) -> str: return str(self._data) - def __repr__(self): + def __repr__(self) -> str: return '{}({})'.format(self.__class__.__name__, repr(self._data)) @property - def _data(self): + def _data(self) -> dict[str, Any]: if self._cached is None: r = self.apiget() try: @@ -256,10 +295,10 @@ def _data(self): return self._cached - def expire(self): + def expire(self) -> None: self._cached = None - def save(self): + def save(self) -> None: for key in self._deleted: self.apidelete(key) self._deleted.clear() @@ -271,23 +310,23 @@ def save(self): if k not in self.ignore_fields}, is_idempotent=True) - def __getitem__(self, key): + def __getitem__(self, key: str) -> Any: return self._data[key] - def __setitem__(self, key, value): + def __setitem__(self, key: str, value: Any) -> None: self._data[key] = value self._deleted.discard(key) - def __delitem__(self, key): + def __delitem__(self, key: str) -> None: del self._data[key] self._deleted.add(key) - def __iter__(self): + def __iter__(self) -> Iterator[str]: return iter(self._data) - def __len__(self): + def __len__(self) -> int: return len(self._data) - def liveget(self, key): + def liveget(self, key: str) -> Any: for o in self.apiget(key): return o diff --git a/scrapinghub/hubstorage/serialization.py b/scrapinghub/hubstorage/serialization.py index 70567805..412d0d4a 100644 --- a/scrapinghub/hubstorage/serialization.py +++ b/scrapinghub/hubstorage/serialization.py @@ -1,5 +1,9 @@ -from datetime import datetime +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from datetime import datetime, timedelta from json import dumps, loads +from typing import Any, cast import six @@ -15,18 +19,18 @@ MSGPACK_AVAILABLE = False -def jlencode(iterable): +def jlencode(iterable: Any) -> str: if isinstance(iterable, (dict, six.string_types)): iterable = [iterable] return u'\n'.join(jsonencode(o) for o in iterable) -def jldecode(lineiterable): +def jldecode(lineiterable: Iterable[str | bytes]) -> Iterator[Any]: for line in lineiterable: yield loads(line) -def mpdecode(iterable): +def mpdecode(iterable: Iterable[bytes]) -> Iterator[Any]: unpacker = Unpacker() for chunk in iterable: unpacker.feed(chunk) @@ -36,15 +40,15 @@ def mpdecode(iterable): yield obj -def jsonencode(o): +def jsonencode(o: Any) -> str: return dumps(o, default=jsondefault) -def jsondefault(o): +def jsondefault(o: object) -> float | str: if isinstance(o, datetime): # convert TZ-aware datetime object to POSIX timestamp if o.tzinfo: - o = o.replace(tzinfo=None) - o.utcoffset() + o = o.replace(tzinfo=None) - cast(timedelta, o.utcoffset()) delta = o - EPOCH u = delta.microseconds s = delta.seconds diff --git a/scrapinghub/hubstorage/utils.py b/scrapinghub/hubstorage/utils.py index 9ac2e3c9..a2518e6c 100644 --- a/scrapinghub/hubstorage/utils.py +++ b/scrapinghub/hubstorage/utils.py @@ -1,9 +1,27 @@ -import six +from __future__ import annotations + import time +from collections.abc import Callable, Iterator +from typing import TYPE_CHECKING, Any, overload + +import six from six.moves.queue import Empty +if TYPE_CHECKING: + from queue import Queue + + +_Part = str | int | tuple[Any, ...] +_Auth = str | tuple[str, str] | None + + +@overload +def urlpathjoin(first: _Part, /, *parts: _Part | None) -> str: ... +@overload +def urlpathjoin(*parts: _Part | None) -> str | None: ... + -def urlpathjoin(*parts): +def urlpathjoin(*parts: _Part | None) -> str | None: """Join multiple paths into a single url >>> urlpathjoin('https://storage.scrapinghub.com:8002/', 'jobs', '1/2/3') @@ -30,7 +48,7 @@ def urlpathjoin(*parts): 'http://localhost:8003/jobs/1111111/2/1' """ - url = None + url: str | None = None for p in parts: if p is None: continue @@ -44,7 +62,7 @@ def urlpathjoin(*parts): return url -def xauth(auth): +def xauth(auth: _Auth) -> tuple[str, str] | None: """Expand authentication token >>> xauth(None) @@ -63,7 +81,7 @@ def xauth(auth): return u, p -def millitime(*a, **kw): +def millitime(*a: Any, **kw: Any) -> int: """The difference, measured in milliseconds, between the current time and midnight, January 1, 1970 UTC. @@ -99,12 +117,12 @@ class iterqueue(object): 4 """ - def __init__(self, queue, maxcount=None): + def __init__(self, queue: Queue[Any], maxcount: int | None = None) -> None: self.queue = queue self.maxcount = maxcount self.count = 0 - def __iter__(self): + def __iter__(self) -> Iterator[Any]: while (self.maxcount is None) or (self.count < self.maxcount): try: yield self.queue.get_nowait() @@ -113,7 +131,7 @@ def __iter__(self): break -def apipoll(endpoint, *args, **kwargs): +def apipoll(endpoint: Callable[..., Any], *args: Any, **kwargs: Any) -> Any: """Poll an api endpoint until there is a result that is not None poll_wait and max_poll can be specified in kwargs to set the polling @@ -132,7 +150,7 @@ def apipoll(endpoint, *args, **kwargs): return result -def sizeof_fmt(num): +def sizeof_fmt(num: float) -> str: """Little helper to get size in human readable form. Size is rounded to a closest integer value (for simplicity). diff --git a/scrapinghub/legacy.py b/scrapinghub/legacy.py index 30789185..5ea9693c 100644 --- a/scrapinghub/legacy.py +++ b/scrapinghub/legacy.py @@ -2,25 +2,18 @@ Scrapinghub API Client Library """ -from __future__ import division, print_function, absolute_import +from __future__ import annotations, division, print_function, absolute_import import os -import sys import json import logging import socket import time import warnings +import http.client as httplib +from collections.abc import Iterable, Iterator, Sequence +from typing import IO, TYPE_CHECKING, Any - -# Python 2/3 compatibility -_IS_PYTHON2 = sys.version_info < (3,) -if _IS_PYTHON2: - import httplib - _BINARY_TYPE = str - range = xrange -else: - import http.client as httplib - _BINARY_TYPE = bytes +import requests logger = logging.getLogger('scrapinghub') @@ -53,8 +46,9 @@ class Connection(object): 'reports_add': 'reports/add', } - def __init__(self, apikey=None, password='', _old_passwd='', - url=None, connection_timeout=None): + def __init__(self, apikey: str | None = None, password: str = '', + _old_passwd: str = '', url: str | None = None, + connection_timeout: float | None = None) -> None: if apikey is None: apikey = os.environ.get('SH_APIKEY') if apikey is None: @@ -66,20 +60,20 @@ def __init__(self, apikey=None, password='', _old_passwd='', warnings.warn("A lot of endpoints support authentication only via apikey.", stacklevel=2) self.apikey = apikey self.password = password or '' - self.url = url or os.getenv("SHUB_APIURL", self.DEFAULT_ENDPOINT) + self.url = url or os.environ.get("SHUB_APIURL", self.DEFAULT_ENDPOINT) self._session = self._create_session() self._connection_timeout = connection_timeout - def __repr__(self): + def __repr__(self) -> str: return "Connection(%r)" % self.apikey @property - def auth(self): + def auth(self) -> tuple[str, str]: warnings.warn("'auth' connection attribute is deprecated, " "use 'apikey' attribute instead", stacklevel=2) return self.apikey, self.password - def _create_session(self): + def _create_session(self) -> requests.Session: from requests import session from scrapinghub import __version__ s = session() @@ -90,10 +84,10 @@ def _create_session(self): # For python-requests >= 1.x s.stream = True # For python-requests < 1.x - s.prefetch = False + s.prefetch = False # type: ignore[attr-defined] return s - def _build_url(self, method, format): + def _build_url(self, method: str, format: str) -> str: """Returns full url for given method and format""" from requests.compat import urljoin # TODO: verify method's format support @@ -106,7 +100,9 @@ def _build_url(self, method, format): path = "{0}.{1}".format(base_path, format) return urljoin(self.url, path) - def _get(self, method, format, params=None, headers=None, raw=False): + def _get(self, method: str, format: str, + params: dict[str, Any] | Sequence[tuple[str, Any]] | None = None, + headers: dict[str, str] | None = None, raw: bool = False) -> Any: """Performs GET request""" from requests.compat import urlencode url = self._build_url(method, format) @@ -114,12 +110,17 @@ def _get(self, method, format, params=None, headers=None, raw=False): url = "{0}?{1}".format(url, urlencode(params, True)) return self._request(url, None, headers, format, raw) - def _post(self, method, format, params=None, headers=None, raw=False, files=None): + def _post(self, method: str, format: str, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, raw: bool = False, + files: dict[str, Any] | None = None) -> Any: """Performs POST request""" url = self._build_url(method, format) return self._request(url, params, headers, format, raw, files) - def _request(self, url, data, headers, format, raw, files=None): + def _request(self, url: str, data: Any, + headers: dict[str, str] | None, format: str, raw: bool, + files: dict[str, Any] | None = None) -> Any: """Performs the request using and returns the content deserialized, based on given `format`. @@ -142,7 +143,8 @@ def _request(self, url, data, headers, format, raw, files=None): timeout=self._connection_timeout) return self._decode_response(response, format, raw) - def _decode_response(self, response, format, raw): + def _decode_response(self, response: requests.Response, format: str, + raw: bool) -> Any: if response.status_code == 404: raise APIError("Not found", _type=APIError.ERR_NOT_FOUND) elif 500 <= response.status_code < 600: @@ -169,93 +171,108 @@ def _decode_response(self, response, format, raw): raise APIError("JSON response does not contain status") else: # jl return (json.loads(line.decode('utf-8') - if isinstance(line, _BINARY_TYPE) else line) + if isinstance(line, bytes) else line) for line in response.iter_lines()) ## ## public methods ## - def __getitem__(self, key): + def __getitem__(self, key: int | str) -> Project: """Returns `Project` instance for given key. Does not verify if project exists. """ return Project(self, key) - def project_ids(self): + def project_ids(self) -> Any: """Returns a list of projects available for this connection and crendentials. """ result = self._get('listprojects', 'json') return result['projects'] - def project_names(self): + def project_names(self) -> Any: warnings.warn("scrapinghub.Connection.project_names() method is deprecated, use project_ids() method instead", stacklevel=2) return self.project_ids() class RequestProxyMixin(object): - def _add_params(self, params): + if TYPE_CHECKING: + @property + def _request_proxy(self) -> Connection | RequestProxyMixin: ... + + def _add_params(self, params: dict[str, Any]) -> dict[str, Any]: return params - def _get(self, method, format, params=None, headers=None, raw=False): + def _get(self, method: str, format: str, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, raw: bool = False) -> Any: params = self._add_params(params or {}) return self._request_proxy._get(method, format, params, headers, raw) - def _post(self, method, format, params=None, headers=None, raw=False, files=None): + def _post(self, method: str, format: str, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, raw: bool = False, + files: dict[str, Any] | None = None) -> Any: params = self._add_params(params or {}) return self._request_proxy._post(method, format, params, headers, raw, files) class Project(RequestProxyMixin): - def __init__(self, connection, projectid): + def __init__(self, connection: Connection, projectid: int | str) -> None: self.connection = connection self.id = projectid - def __repr__(self): + def __repr__(self) -> str: return "Project({0.connection!r}, {0.id})".format(self) @property - def name(self): + def name(self) -> int | str: warnings.warn("Project.name is deprecated, use Project.id instead", stacklevel=2) return self.id - def schedule(self, spider, **params): + def schedule(self, spider: str, **params: Any) -> Any: params['spider'] = spider result = self._post('schedule', 'json', params) return result['jobid'] - def jobs(self, **params): + def jobs(self, **params: Any) -> JobSet: return JobSet(self, **params) - def job(self, id): + def job(self, id: str) -> Job | None: for x in self.jobs(job=id, count=1): return x + return None - def spiders(self, **params): + def spiders(self, **params: Any) -> Any: result = self._get('spiders', 'json', params) return result['spiders'] @property - def _request_proxy(self): + def _request_proxy(self) -> Connection: return self.connection - def _add_params(self, params): + def _add_params(self, params: dict[str, Any]) -> dict[str, Any]: # force project param params.update(project=self.id) return params - def autoscraping_project_slybot(self, spiders=(), outputfile=None): + def autoscraping_project_slybot( + self, spiders: Sequence[str] = (), + outputfile: IO[bytes] | None = None, + ) -> Any: from shutil import copyfileobj - params = {} + params: dict[str, Any] = {} if spiders: params['spider'] = spiders r = self._get('as_project_slybot', 'zip', params, raw=True) return r if outputfile is None else copyfileobj(r, outputfile) - def autoscraping_spider_properties(self, spider, start_urls=None): - params = {'spider': spider} + def autoscraping_spider_properties( + self, spider: str, start_urls: Sequence[str] | None = None, + ) -> Any: + params: dict[str, Any] = {'spider': spider} if start_urls: params['start_url'] = start_urls return self._post('as_spider_properties', 'json', params) @@ -264,42 +281,43 @@ def autoscraping_spider_properties(self, spider, start_urls=None): class JobSet(RequestProxyMixin): - def __init__(self, project, **params): + def __init__(self, project: Project, **params: Any) -> None: self.project = project self.params = params # jobs one-shot iterator - self._jobs = None + self._jobs: Iterable[Any] | None = None - def __repr__(self): + def __repr__(self) -> str: params = ', '.join("{0}={1}".format(*i) for i in self.params.items()) return "JobSet({0.project!r}, {1})".format(self, params) - def __iter__(self): + def __iter__(self) -> Iterator[Job]: self._load_jobs() + assert self._jobs is not None return (Job(self.project, info['id'], info) for info in self._jobs) - def count(self): + def count(self) -> Any: """Returns total results count of current filters. Does not inclue `count` neither `offset`. """ result = self._get('jobs_count', 'json') return result['total'] - def update(self, **modifiers): + def update(self, **modifiers: Any) -> Any: params = self.params.copy() params.update(modifiers) result = self._post('jobs_update', 'json', params) return result['count'] - def stop(self): + def stop(self) -> None: for job in self: job.stop() - def delete(self): + def delete(self) -> None: for job in self: job.delete() - def _load_jobs(self): + def _load_jobs(self) -> None: # only load once if self._jobs is None: result = self._get('jobs_list', 'jl', self.params) @@ -316,10 +334,10 @@ def _load_jobs(self): self._jobs = result @property - def _request_proxy(self): + def _request_proxy(self) -> Project: return self.project - def _add_params(self, params): + def _add_params(self, params: dict[str, Any]) -> dict[str, Any]: # default to JobSet's params params2 = self.params.copy() # update with user params @@ -332,21 +350,21 @@ class Job(RequestProxyMixin): MAX_RETRIES = 180 RETRY_INTERVAL = 60 - def __init__(self, project, id, info): + def __init__(self, project: Project, id: str, info: dict[str, Any]) -> None: self.project = project self._id = id self.info = info @property - def id(self): + def id(self) -> str: return self._id - def __repr__(self): + def __repr__(self) -> str: return "Job({0.project!r}, {0.id})".format(self) - def items(self, offset=0, count=None, meta=None): - import requests - params = {'offset': offset} + def items(self, offset: int = 0, count: int | None = None, + meta: Sequence[str] | None = None) -> Iterator[Any]: + params: dict[str, Any] = {'offset': offset} if meta is not None: params['meta'] = meta if count is not None: @@ -374,21 +392,23 @@ def items(self, offset=0, count=None, meta=None): logger.error('Failed %d times reading items from %s, last error was: %s', self.MAX_RETRIES, self._id, lastexc) - def update(self, **modifiers): + def update(self, **modifiers: Any) -> Any: # XXX: only allow add_tag/remove_tag result = self._post('jobs_update', 'json', modifiers) return result['count'] - def stop(self): + def stop(self) -> bool: result = self._post('jobs_stop', 'json') - return result['status'] == 'ok' + status: str = result['status'] + return status == 'ok' - def delete(self): + def delete(self) -> Any: result = self._post('jobs_delete', 'json') return result['count'] - def add_report(self, key, content, content_type='text/plain'): - from requests.compat import StringIO + def add_report(self, key: str, content: str, + content_type: str = 'text/plain') -> None: + from io import StringIO params = { 'project': self.project.id, 'job': self.id, @@ -398,14 +418,14 @@ def add_report(self, key, content, content_type='text/plain'): files = {'content': ('report', StringIO(content))} self._post('reports_add', 'json', params, files=files) - def log(self, **params): + def log(self, **params: Any) -> Any: return self._get('log', 'jl', params) @property - def _request_proxy(self): + def _request_proxy(self) -> Project: return self.project - def _add_params(self, params): + def _add_params(self, params: dict[str, Any]) -> dict[str, Any]: params['job'] = self.id return params @@ -419,6 +439,6 @@ class APIError(Exception): ERR_AUTH_ERROR = "err_auth_error" ERR_SERVER_ERROR = "err_server_error" - def __init__(self, message, _type=None): + def __init__(self, message: str, _type: str | None = None) -> None: super(APIError, self).__init__(message) self._type = _type or self.ERR_DEFAULT diff --git a/scrapinghub/py.typed b/scrapinghub/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/setup.py b/setup.py index 315ea184..161fdf6e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ url='http://github.com/scrapinghub/python-scrapinghub', platforms=['Any'], packages=['scrapinghub', 'scrapinghub.client', 'scrapinghub.hubstorage'], - package_data={'scrapinghub': ['VERSION']}, + package_data={'scrapinghub': ['VERSION', 'py.typed']}, install_requires=['python-dotenv>=1.0.0', 'requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'], extras_require={'msgpack': mpack_required}, diff --git a/tests/client/conftest.py b/tests/client/conftest.py index 8d253c0d..b8adeac6 100644 --- a/tests/client/conftest.py +++ b/tests/client/conftest.py @@ -1,4 +1,5 @@ import os +from collections.abc import Iterator import vcr import pytest @@ -7,6 +8,10 @@ from scrapinghub import ScrapinghubClient from scrapinghub.client.exceptions import NotFound from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE +from scrapinghub.client.collections import Collection +from scrapinghub.client.frontiers import Frontier +from scrapinghub.client.projects import Project +from scrapinghub.client.spiders import Spider from ..conftest import request_accept_header_matcher from ..conftest import VCRGzipSerializer @@ -34,7 +39,7 @@ 'path', 'query', 'accept_header') -def pytest_configure(config): +def pytest_configure(config: pytest.Config) -> None: if config.option.update_cassettes: # there's vcr `all` mode to update cassettes but it doesn't delete # or clear existing records, so its size will always only grow @@ -49,26 +54,26 @@ def pytest_configure(config): my_vcr.before_record_request = lambda request: None -def is_using_real_services(request): - return (request.config.option.update_cassettes or - request.config.option.ignore_cassettes) +def is_using_real_services(request: pytest.FixtureRequest) -> bool: + return bool(request.config.option.update_cassettes or + request.config.option.ignore_cassettes) @pytest.fixture(scope='session') -def client(): +def client() -> ScrapinghubClient: return ScrapinghubClient(auth=TEST_ADMIN_AUTH, endpoint=TEST_ENDPOINT, dash_endpoint=TEST_DASH_ENDPOINT) @pytest.fixture(scope='session') -def project(client): +def project(client: ScrapinghubClient) -> Project: return client.get_project(TEST_PROJECT_ID) @pytest.fixture(scope='session') -@my_vcr.use_cassette() -def spider(project, request): +@my_vcr.use_cassette() # type: ignore[untyped-decorator] +def spider(project: Project, request: pytest.FixtureRequest) -> Spider: # on normal conditions you can't create a new spider this way: # it can only be created on project deploy as usual spider = project.spiders.get(TEST_SPIDER_NAME, create=True) @@ -80,7 +85,9 @@ def spider(project, request): @pytest.fixture(scope='session') -def collection(project, request): +def collection( + project: Project, request: pytest.FixtureRequest, +) -> Iterator[Collection]: collection = get_test_collection(project) if is_using_real_services(request): clean_collection(collection) @@ -88,7 +95,9 @@ def collection(project, request): @pytest.fixture(scope='function') -def frontier(project, request, frontier_name): +def frontier( + project: Project, request: pytest.FixtureRequest, frontier_name: str, +) -> Iterator[Frontier]: frontier = project.frontiers.get(frontier_name) if is_using_real_services(request): clean_frontier_slot(frontier) @@ -96,7 +105,10 @@ def frontier(project, request, frontier_name): @pytest.fixture(autouse=True, scope='session') -def setup_session(client, project, collection, request): +def setup_session( + client: ScrapinghubClient, project: Project, collection: Collection, + request: pytest.FixtureRequest, +) -> Iterator[None]: if is_using_real_services(request): remove_all_jobs(project) yield @@ -104,16 +116,22 @@ def setup_session(client, project, collection, request): @pytest.fixture(params=['json', 'msgpack']) -def json_and_msgpack(client, monkeypatch, request): +def json_and_msgpack( + client: ScrapinghubClient, monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +) -> str: if request.param == 'json': monkeypatch.setattr(client._hsclient, 'use_msgpack', False) elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"): pytest.skip("messagepack-based tests are disabled") - return request.param + param: str = request.param + return param @pytest.fixture(autouse=True) -def setup_vcrpy(request, project): +def setup_vcrpy( + request: pytest.FixtureRequest, project: Project, +) -> Iterator[None]: # generates names like "test_module/test_function{-json}.yaml" # otherwise it uses current function name (setup_vcrpy) for all tests # other option is to add vcr decorator to each test separately @@ -138,7 +156,7 @@ def setup_vcrpy(request, project): # Clean environment section -def remove_all_jobs(project): +def remove_all_jobs(project: Project) -> None: for k, _ in project.settings.iter(): if k != 'botgroups': project.settings.delete(k) @@ -150,7 +168,7 @@ def remove_all_jobs(project): _remove_job(project, summary['key']) -def _remove_job(project, jobkey): +def _remove_job(project: Project, jobkey: str) -> None: job = project.jobs.get(jobkey) job.finish() job.delete() @@ -162,11 +180,11 @@ def _remove_job(project, jobkey): # Collection helpers section -def get_test_collection(project): +def get_test_collection(project: Project) -> Collection: return project.collections.get_store(TEST_COLLECTION_NAME) -def clean_collection(collection): +def clean_collection(collection: Collection) -> None: try: for item in collection.iter(): collection.delete(item['_key']) @@ -176,5 +194,5 @@ def clean_collection(collection): # Frontier helpers section -def clean_frontier_slot(frontier): +def clean_frontier_slot(frontier: Frontier) -> None: frontier.get(TEST_FRONTIER_SLOT).delete() diff --git a/tests/client/test_activity.py b/tests/client/test_activity.py index 5593ec54..473ce883 100644 --- a/tests/client/test_activity.py +++ b/tests/client/test_activity.py @@ -2,10 +2,12 @@ import pytest +from scrapinghub.client.projects import Project + from ..conftest import TEST_PROJECT_ID -def _add_test_activity(project): +def _add_test_activity(project: Project) -> None: activity = project.activity jobkey = TEST_PROJECT_ID + '/2/3' events = [{'event': 'job:completed', 'job': jobkey, 'user': 'jobrunner'}, @@ -13,13 +15,13 @@ def _add_test_activity(project): activity.add(events) -def test_activity_wrong_project(project): +def test_activity_wrong_project(project: Project) -> None: event = {'event': 'job:completed', 'job': '123/1/1', 'user': 'user'} with pytest.raises(ValueError): - project.activity.add(event) + project.activity.add(event) # type: ignore[arg-type] -def test_activity_iter(project): +def test_activity_iter(project: Project) -> None: _add_test_activity(project) activity = project.activity.iter() assert isinstance(activity, types.GeneratorType) @@ -29,7 +31,7 @@ def test_activity_iter(project): 'user': 'john'} -def test_activity_list(project): +def test_activity_list(project: Project) -> None: _add_test_activity(project) activity = project.activity.list(count=2) assert isinstance(activity, list) diff --git a/tests/client/test_client.py b/tests/client/test_client.py index 8bc64aeb..9a449d4a 100644 --- a/tests/client/test_client.py +++ b/tests/client/test_client.py @@ -12,7 +12,7 @@ # ScrapinghubClient class tests -def test_client_base(client): +def test_client_base(client: ScrapinghubClient) -> None: """Base tests for client instance""" assert isinstance(client, ScrapinghubClient) assert client._hsclient @@ -24,11 +24,11 @@ def test_client_base(client): assert isinstance(client.projects, Projects) -def test_client_get_project(client): +def test_client_get_project(client: ScrapinghubClient) -> None: project = client.get_project(TEST_PROJECT_ID) assert isinstance(project, Project) -def test_client_get_job(client): +def test_client_get_job(client: ScrapinghubClient) -> None: fake_job = client.get_job('1/2/3') assert isinstance(fake_job, Job) diff --git a/tests/client/test_collections.py b/tests/client/test_collections.py index 775d8fb8..3b77bb21 100644 --- a/tests/client/test_collections.py +++ b/tests/client/test_collections.py @@ -1,4 +1,6 @@ from contextlib import closing +from collections.abc import Callable +from typing import Any import pytest from six.moves import range @@ -6,16 +8,18 @@ from scrapinghub.client.exceptions import BadRequest from scrapinghub.client.exceptions import NotFound from scrapinghub.client.exceptions import ValueTooLarge +from scrapinghub.client.collections import Collection +from scrapinghub.client.projects import Project from ..conftest import TEST_COLLECTION_NAME -def _mkitem(): +def _mkitem() -> dict[str, Any]: return dict(field1='value1', field2=['value2a', 'value2b'], field3=3, field4={'v4k': 'v4v'}) -def test_collections_list(project): +def test_collections_list(project: Project) -> None: # create/check test collections project.collections.get_store(TEST_COLLECTION_NAME), project.collections.get_cached_store(TEST_COLLECTION_NAME), @@ -28,7 +32,7 @@ def test_collections_list(project): assert {'name': TEST_COLLECTION_NAME, 'type': coltype} in collections -def test_simple_count(project, collection): +def test_simple_count(project: Project, collection: Collection) -> None: test_item = dict(_mkitem()) test_item['_key'] = 'a' @@ -36,7 +40,7 @@ def test_simple_count(project, collection): assert collection.count() == 1 -def test_post_get_delete(project, json_and_msgpack): +def test_post_get_delete(project: Project, json_and_msgpack: str) -> None: test_item = _mkitem() item_to_send = dict(test_item) item_to_send['_key'] = test_key = 'insert_test_key' @@ -61,10 +65,12 @@ def test_post_get_delete(project, json_and_msgpack): col.get(test_key) -def test_post_scan(project, collection, json_and_msgpack): +def test_post_scan( + project: Project, collection: Collection, json_and_msgpack: str, +) -> None: # populate with 20 items test_item = _mkitem() - last_key = None + last_key = '' with closing(collection.create_writer()) as writer: for i in range(20): test_item['_key'] = last_key = "post_scan_test%d" % i @@ -78,7 +84,7 @@ def test_post_scan(project, collection, json_and_msgpack): # get requires key field with pytest.raises(TypeError): - collection.get() + collection.get() # type: ignore[call-arg] result = collection.get('post_scan_test2') assert isinstance(result, dict) @@ -105,7 +111,7 @@ def test_post_scan(project, collection, json_and_msgpack): collection.get(last_key) -def test_errors_bad_key(collection, json_and_msgpack): +def test_errors_bad_key(collection: Collection, json_and_msgpack: str) -> None: with pytest.raises(NotFound): collection.get('does_not_exist') @@ -114,17 +120,17 @@ def test_errors_bad_key(collection, json_and_msgpack): {'foo': 42}, {'_key': []}, ]) -def test_errors(collection, testarg): +def test_errors(collection: Collection, testarg: dict[str, Any]) -> None: with pytest.raises(BadRequest): collection.set(testarg) -def test_entity_too_large(collection): +def test_entity_too_large(collection: Collection) -> None: with pytest.raises(ValueTooLarge): collection.set({'_key': 'large_test', 'value': 'x' * 1024 ** 2}) -def test_data_download(project, collection): +def test_data_download(project: Project, collection: Collection) -> None: items = [] with closing(collection.create_writer()) as writer: for i in range(20): @@ -142,18 +148,19 @@ def test_data_download(project, collection): assert len(downloaded) == 19 -def test_invalid_collection_name(project): +def test_invalid_collection_name(project: Project) -> None: cols = project.collections - for method, args in [ + cases: list[tuple[Callable[..., Any], tuple[str, ...]]] = [ (cols.get, ('invalidtype', 'n')), (cols.get_store, ('foo-bar',)), (cols.get_store, ('foo/bar',)), - (cols.get_store, ('/foo',))]: + (cols.get_store, ('/foo',))] + for method, args in cases: with pytest.raises(ValueError): method(*args) -def test_truncate(collection): +def test_truncate(collection: Collection) -> None: # populate with 20 items test_item = _mkitem() with closing(collection.create_writer()) as writer: diff --git a/tests/client/test_frontiers.py b/tests/client/test_frontiers.py index b20cffb7..e58251b9 100644 --- a/tests/client/test_frontiers.py +++ b/tests/client/test_frontiers.py @@ -5,16 +5,17 @@ from six.moves import collections_abc from scrapinghub.client.frontiers import Frontiers, Frontier, FrontierSlot +from scrapinghub.client.projects import Project from ..conftest import TEST_FRONTIER_SLOT -def _add_test_requests_to_frontier(frontier): +def _add_test_requests_to_frontier(frontier: Frontier) -> None: slot = frontier.get(TEST_FRONTIER_SLOT) slot.q.add([{'fp': '/some/path.html'}, {'fp': '/other/path.html'}]) slot.flush() -def _clean_project_frontiers(project): +def _clean_project_frontiers(project: Project) -> None: """Helper to clean slots of all frontiers for a project. frontier fixture cleans a test slot before each test, but for some tests @@ -26,7 +27,9 @@ def _clean_project_frontiers(project): frontier.get(slot_name).delete() -def test_frontiers(project, frontier, frontier_name): +def test_frontiers( + project: Project, frontier: Frontier, frontier_name: str, +) -> None: # reset a test slot and add some requests to init it frontier.get(TEST_FRONTIER_SLOT).delete() _add_test_requests_to_frontier(frontier) @@ -52,7 +55,7 @@ def test_frontiers(project, frontier, frontier_name): assert isinstance(frontiers.newcount, int) -def test_frontier(project, frontier): +def test_frontier(project: Project, frontier: Frontier) -> None: # add some requests to test frontier to init a test slot frontier.get(TEST_FRONTIER_SLOT).delete() _add_test_requests_to_frontier(frontier) @@ -70,7 +73,7 @@ def test_frontier(project, frontier): frontier.flush() -def test_frontier_slot(project, frontier): +def test_frontier_slot(project: Project, frontier: Frontier) -> None: # add some requests to test frontier to init a test slot frontier.get(TEST_FRONTIER_SLOT).delete() _add_test_requests_to_frontier(frontier) @@ -110,7 +113,7 @@ def test_frontier_slot(project, frontier): assert TEST_FRONTIER_SLOT not in frontier.list() -def test_frontier_newcount(project, frontier): +def test_frontier_newcount(project: Project, frontier: Frontier) -> None: _clean_project_frontiers(project) first_slot = frontier.get(TEST_FRONTIER_SLOT) diff --git a/tests/client/test_items.py b/tests/client/test_items.py index 79f68c33..962597cc 100644 --- a/tests/client/test_items.py +++ b/tests/client/test_items.py @@ -1,17 +1,20 @@ import pytest from six.moves import range +from scrapinghub.client.jobs import Job +from scrapinghub.client.spiders import Spider + from .utils import normalize_job_for_tests -def _add_test_items(job, size=3): +def _add_test_items(job: Job, size: int = 3) -> None: for i in range(size): job.items.write({'id': i, 'data': 'data' + str(i)}) job.items.flush() job.items.close() -def test_items_iter(spider, json_and_msgpack): +def test_items_iter(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run(meta={'state': 'running'}) _add_test_items(job) @@ -28,7 +31,7 @@ def test_items_iter(spider, json_and_msgpack): next(o) -def test_items_list(spider, json_and_msgpack): +def test_items_list(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run(meta={'state': 'running'}) job = normalize_job_for_tests(job) _add_test_items(job) @@ -41,7 +44,7 @@ def test_items_list(spider, json_and_msgpack): assert o[2] == {'id': 2, 'data': 'data2'} -def test_items_list_iter(spider, json_and_msgpack): +def test_items_list_iter(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run(meta={'state': 'running'}) job = normalize_job_for_tests(job) _add_test_items(job) @@ -59,7 +62,9 @@ def test_items_list_iter(spider, json_and_msgpack): next(o) -def test_items_list_iter_with_start_and_count(spider, json_and_msgpack): +def test_items_list_iter_with_start_and_count( + spider: Spider, json_and_msgpack: str, +) -> None: job = spider.jobs.run(meta={'state': 'running'}) job = normalize_job_for_tests(job) _add_test_items(job, size=10) @@ -83,7 +88,9 @@ def test_items_list_iter_with_start_and_count(spider, json_and_msgpack): next(o) -def test_items_list_iter_with_start_and_count_2(spider, json_and_msgpack): +def test_items_list_iter_with_start_and_count_2( + spider: Spider, json_and_msgpack: str, +) -> None: """2nd version from the test above but this case makes sure that the total number of items returned would be equal to `count`. """ diff --git a/tests/client/test_job.py b/tests/client/test_job.py index 7c36d12a..e20fbe2b 100644 --- a/tests/client/test_job.py +++ b/tests/client/test_job.py @@ -8,12 +8,14 @@ from scrapinghub.client.requests import Requests from scrapinghub.client.samples import Samples from scrapinghub.client.exceptions import BadRequest +from scrapinghub import ScrapinghubClient +from scrapinghub.client.spiders import Spider from ..conftest import TEST_PROJECT_ID from ..conftest import TEST_SPIDER_NAME -def test_job_base(client, spider): +def test_job_base(client: ScrapinghubClient, spider: Spider) -> None: job = spider.jobs.run() assert isinstance(job, Job) assert job.project_id == TEST_PROJECT_ID @@ -26,7 +28,7 @@ def test_job_base(client, spider): assert isinstance(job.metadata, JobMeta) -def test_job_update_tags(spider): +def test_job_update_tags(spider: Spider) -> None: job1 = spider.jobs.run(job_args={'subid': 'tags-1'}, add_tag=['tag1']) job2 = spider.jobs.run(job_args={'subid': 'tags-2'}, @@ -44,7 +46,7 @@ def test_job_update_tags(spider): assert job2.metadata.get('tags') == ['tag2'] -def test_cancel_jobs_validation(spider): +def test_cancel_jobs_validation(spider: Spider) -> None: with pytest.raises(ValueError) as err: spider.jobs.cancel() @@ -56,12 +58,12 @@ def test_cancel_jobs_validation(spider): assert "keys and count can't be defined simultaneously" in str(err) with pytest.raises(ValueError) as err: - spider.jobs.cancel(keys="testing") + spider.jobs.cancel(keys="testing") # type: ignore[arg-type] assert 'keys should be a list' in str(err) with pytest.raises(ValueError) as err: - spider.jobs.cancel(count=[1, 2]) + spider.jobs.cancel(count=[1, 2]) # type: ignore[arg-type] assert 'count should be an int' in str(err) @@ -71,7 +73,7 @@ def test_cancel_jobs_validation(spider): assert 'all keys should belong to project' in str(err) -def test_cancel_jobs(spider): +def test_cancel_jobs(spider: Spider) -> None: job1 = spider.jobs.run(job_args={'subid': 'tags-1'}, add_tag=['tag1']) job2 = spider.jobs.run(job_args={'subid': 'tags-2'}, add_tag=['tag2']) assert job1.metadata.get('state') == 'pending' @@ -84,7 +86,7 @@ def test_cancel_jobs(spider): assert output == {'count': 2} -def test_cancel_jobs_non_existent(spider): +def test_cancel_jobs_non_existent(spider: Spider) -> None: job1 = spider.jobs.run(job_args={'subid': 'tags-1'}, add_tag=['tag1']) assert job1.metadata.get('state') == 'pending' @@ -94,7 +96,7 @@ def test_cancel_jobs_non_existent(spider): assert job1.metadata.get('state') == 'pending' -def test_job_start(spider): +def test_job_start(spider: Spider) -> None: job = spider.jobs.run() assert job.metadata.get('state') == 'pending' job.start() @@ -105,15 +107,15 @@ def test_job_start(spider): assert job.metadata.get('priority') == 2 -def test_job_start_with_environment(spider): +def test_job_start_with_environment(spider: Spider) -> None: with pytest.raises(ValueError): - spider.jobs.run(environment='wrong-env') + spider.jobs.run(environment='wrong-env') # type: ignore[arg-type] env = {'VAR1': 'VAL1', 'VAR2': 'VAL2'} job = spider.jobs.run(environment=env) assert job.metadata.get('environment') == env -def test_job_start_extras(spider): +def test_job_start_extras(spider: Spider) -> None: job = spider.jobs.run() extras = { 'string': 'foo', @@ -135,7 +137,7 @@ def test_job_start_extras(spider): assert job.metadata.get(k) == v -def test_job_update(spider): +def test_job_update(spider: Spider) -> None: job = spider.jobs.run() assert job.metadata.get('state') == 'pending' job.update(state='running', foo='bar') @@ -145,14 +147,14 @@ def test_job_update(spider): assert job.metadata.get('foo') == 'bar' -def test_job_cancel_pending(spider): +def test_job_cancel_pending(spider: Spider) -> None: job = spider.jobs.run() assert job.metadata.get('state') == 'pending' job.cancel() assert job.metadata.get('state') == 'finished' -def test_job_cancel_running(spider): +def test_job_cancel_running(spider: Spider) -> None: job = spider.jobs.run() job.start() assert job.metadata.get('state') == 'running' @@ -161,14 +163,14 @@ def test_job_cancel_running(spider): assert job.metadata.get('state') == 'running' -def test_job_finish(spider): +def test_job_finish(spider: Spider) -> None: job = spider.jobs.run() assert job.metadata.get('state') == 'pending' job.finish() assert job.metadata.get('state') == 'finished' -def test_job_finish_with_metadata(spider): +def test_job_finish_with_metadata(spider: Spider) -> None: job = spider.jobs.run(meta={'meta1': 'val1', 'meta2': 'val3'}) assert job.metadata.get('state') == 'pending' job.finish(meta2='val2', meta3='val3') @@ -178,14 +180,14 @@ def test_job_finish_with_metadata(spider): assert job.metadata.get('meta3') == 'val3' -def test_job_delete(spider): +def test_job_delete(spider: Spider) -> None: job = spider.jobs.run(meta={'state': 'finished'}) assert job.metadata.get('state') == 'finished' job.delete() assert job.metadata.get('state') == 'deleted' -def test_job_delete_with_metadata(spider): +def test_job_delete_with_metadata(spider: Spider) -> None: meta = {'state': 'finished', 'meta1': 'val1', 'meta2': 'val3'} job = spider.jobs.run(meta=meta) assert job.metadata.get('state') == 'finished' @@ -196,7 +198,7 @@ def test_job_delete_with_metadata(spider): assert job.metadata.get('meta3') == 'val3' -def test_metadata_update(spider): +def test_metadata_update(spider: Spider) -> None: job = spider.jobs.run(meta={'meta1': 'data1'}) assert job.metadata.get('meta1') == 'data1' job.metadata.update({'meta1': 'data2', 'meta2': 'data3'}) @@ -204,7 +206,7 @@ def test_metadata_update(spider): assert job.metadata.get('meta2') == 'data3' -def test_metadata_set(spider): +def test_metadata_set(spider: Spider) -> None: job = spider.jobs.run(meta={'meta1': 'data1'}) assert job.metadata.get('meta1') == 'data1' job.metadata.set('meta1', 'data2') @@ -213,14 +215,14 @@ def test_metadata_set(spider): assert job.metadata.get('meta2') == 123 -def test_metadata_delete(spider): +def test_metadata_delete(spider: Spider) -> None: job = spider.jobs.run(meta={'meta1': 'data1', 'meta2': 'data2'}) job.metadata.delete('meta1') assert job.metadata.get('meta1') is None assert job.metadata.get('meta2') == 'data2' -def test_metadata_iter_list(spider): +def test_metadata_iter_list(spider: Spider) -> None: job = spider.jobs.run(meta={'meta1': 'data1', 'meta2': 'data2'}) meta_iter = job.metadata.iter() assert isinstance(meta_iter, collections_abc.Iterator) diff --git a/tests/client/test_logs.py b/tests/client/test_logs.py index e6c2874b..9563fb78 100644 --- a/tests/client/test_logs.py +++ b/tests/client/test_logs.py @@ -4,11 +4,13 @@ import pytest from scrapinghub.client.utils import LogLevel +from scrapinghub.client.jobs import Job +from scrapinghub.client.spiders import Spider from .conftest import TEST_TS -def _add_test_logs(job): +def _add_test_logs(job: Job) -> None: job.logs.log('simple-msg1') job.logs.log('simple-msg2', ts=TEST_TS) job.logs.log('simple-msg3', level=LogLevel.DEBUG) @@ -26,7 +28,7 @@ def _add_test_logs(job): job.logs.flush() -def test_logs_base(spider, json_and_msgpack): +def test_logs_base(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run() assert list(job.logs.iter()) == [] assert job.logs.batch_write_start() == 0 @@ -34,7 +36,7 @@ def test_logs_base(spider, json_and_msgpack): log1 = job.logs.get(0) assert log1['level'] == 20 assert log1['message'] == 'simple-msg1' - assert isinstance(log1['time'], Integral) and log1['time'] > 0 + assert isinstance(log1['time'], Integral) and log1['time'] > 0 # type: ignore[operator] assert job.logs.stats() == { 'counts': {'10': 2, '20': 4, '30': 2, '40': 1}, 'totals': {'input_bytes': 91, 'input_values': 9} @@ -42,7 +44,7 @@ def test_logs_base(spider, json_and_msgpack): job.logs.close() -def test_logs_iter(spider, json_and_msgpack): +def test_logs_iter(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run() _add_test_logs(job) @@ -64,7 +66,7 @@ def test_logs_iter(spider, json_and_msgpack): next(logs3).get('message') -def test_logs_list(spider, json_and_msgpack): +def test_logs_list(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run() _add_test_logs(job) @@ -87,7 +89,7 @@ def test_logs_list(spider, json_and_msgpack): assert logs3[0].get('message') == 'error-msg' -def test_logs_list_filter(spider, json_and_msgpack): +def test_logs_list_filter(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run() _add_test_logs(job) diff --git a/tests/client/test_projects.py b/tests/client/test_projects.py index 65eb831e..b3442420 100644 --- a/tests/client/test_projects.py +++ b/tests/client/test_projects.py @@ -1,5 +1,6 @@ import types from collections import defaultdict +from typing import Any import pytest import responses @@ -25,7 +26,7 @@ # Projects class tests -def test_projects_get(client): +def test_projects_get(client: ScrapinghubClient) -> None: projects = client.projects # testing with int project id p1 = projects.get(int(TEST_PROJECT_ID)) @@ -36,7 +37,7 @@ def test_projects_get(client): assert p1.key == p2.key -def test_projects_list(client): +def test_projects_list(client: ScrapinghubClient) -> None: projects = client.projects.list() assert client.projects.list() == [] @@ -48,7 +49,7 @@ def test_projects_list(client): @responses.activate -def test_projects_list_server_error(client): +def test_projects_list_server_error(client: ScrapinghubClient) -> None: url = urljoin(TEST_DASH_ENDPOINT, 'scrapyd/listprojects.json') responses.add(responses.GET, url, body='some error body', status=500) with pytest.raises(ServerError): @@ -56,11 +57,11 @@ def test_projects_list_server_error(client): assert len(responses.calls) == 1 -def test_projects_summary(client, project): +def test_projects_summary(client: ScrapinghubClient, project: Project) -> None: # add at least one running or pending job to ensure summary is returned project.jobs.run(TEST_SPIDER_NAME, meta={'state': 'running'}) - def _get_summary(): + def _get_summary() -> Any: summaries = {str(js['project']): js for js in client.projects.summary()} return summaries.get(TEST_PROJECT_ID) @@ -71,7 +72,7 @@ def _get_summary(): # Project class tests -def test_project_base(project): +def test_project_base(project: Project) -> None: assert project.key == TEST_PROJECT_ID assert isinstance(project.collections, Collections) assert isinstance(project.jobs, Jobs) @@ -81,13 +82,13 @@ def test_project_base(project): assert isinstance(project.settings, Settings) -def test_project_jobs(project): +def test_project_jobs(project: Project) -> None: jobs = project.jobs assert jobs.project_id == TEST_PROJECT_ID assert jobs.spider is None -def test_project_jobs_count(project): +def test_project_jobs_count(project: Project) -> None: assert project.jobs.count() == 0 assert project.jobs.count(state=['pending', 'running', 'finished']) == 0 @@ -113,7 +114,7 @@ def test_project_jobs_count(project): assert project.jobs.count() == 3 -def test_project_jobs_iter(project): +def test_project_jobs_iter(project: Project) -> None: project.jobs.run(TEST_SPIDER_NAME, meta={'state': 'running'}) # no finished jobs @@ -132,7 +133,7 @@ def test_project_jobs_iter(project): assert isinstance(running_time, int) and running_time > 0 elapsed = job.get('elapsed') assert isinstance(elapsed, int) and elapsed > 0 - assert job.get('key').startswith(TEST_PROJECT_ID) + assert job['key'].startswith(TEST_PROJECT_ID) assert job.get('spider') == TEST_SPIDER_NAME assert job.get('state') == 'running' @@ -140,7 +141,7 @@ def test_project_jobs_iter(project): next(jobs1) -def test_project_jobs_list(project): +def test_project_jobs_list(project: Project) -> None: project.jobs.run(TEST_SPIDER_NAME, meta={'state': 'running'}) # no finished jobs @@ -159,12 +160,12 @@ def test_project_jobs_list(project): assert isinstance(running_time, int) and running_time > 0 elapsed = job.get('elapsed') assert isinstance(elapsed, int) and elapsed > 0 - assert job.get('key').startswith(TEST_PROJECT_ID) + assert job['key'].startswith(TEST_PROJECT_ID) assert job.get('spider') == TEST_SPIDER_NAME assert job.get('state') == 'running' -def test_project_jobs_run(project): +def test_project_jobs_run(project: Project) -> None: # scheduling on project level requires spidername with pytest.raises(ValueError): project.jobs.run() @@ -196,7 +197,7 @@ def test_project_jobs_run(project): assert meta.get('started_by') -def test_project_jobs_get(project): +def test_project_jobs_get(project: Project) -> None: # error when using different project id in jobkey with pytest.raises(ValueError): project.jobs.get('1/2/3') @@ -205,7 +206,7 @@ def test_project_jobs_get(project): assert isinstance(fake_job, Job) -def test_project_jobs_summary(project): +def test_project_jobs_summary(project: Project) -> None: expected_summary = [{'count': 0, 'name': state, 'summary': []} for state in ['pending', 'running', 'finished']] assert project.jobs.summary() == expected_summary @@ -243,7 +244,7 @@ def test_project_jobs_summary(project): assert summary4['summary'][0].get('units') == 1 -def test_project_jobs_iter_last(project): +def test_project_jobs_iter_last(project: Project) -> None: lastsumm0 = project.jobs.iter_last() assert isinstance(lastsumm0, types.GeneratorType) assert list(lastsumm0) == [] @@ -267,25 +268,25 @@ def test_project_jobs_iter_last(project): assert lastsumm2[0].get('key') == job2.key -def test_settings_get_set(project): +def test_settings_get_set(project: Project) -> None: project.settings.set('job_runtime_limit', 20) assert project.settings.get('job_runtime_limit') == 20 project.settings.set('job_runtime_limit', 24) assert project.settings.get('job_runtime_limit') == 24 -def test_settings_update(project): +def test_settings_update(project: Project) -> None: project.settings.set('job_runtime_limit', 20) project.settings.update({'job_runtime_limit': 24}) assert project.settings.get('job_runtime_limit') == 24 -def test_settings_delete(project): +def test_settings_delete(project: Project) -> None: project.settings.delete('job_runtime_limit') assert not project.settings.get('job_runtime_limit') -def test_settings_iter_list(project): +def test_settings_iter_list(project: Project) -> None: project.settings.set('job_runtime_limit', 24) settings_iter = project.settings.iter() assert isinstance(settings_iter, collections_abc.Iterator) diff --git a/tests/client/test_proxy.py b/tests/client/test_proxy.py index dfde07f3..56b11f5e 100644 --- a/tests/client/test_proxy.py +++ b/tests/client/test_proxy.py @@ -1,3 +1,6 @@ + +from typing import Any + import mock import pytest @@ -5,12 +8,12 @@ from scrapinghub.client.proxy import _ItemsResourceProxy -def test_format_iter_filters(): +def test_format_iter_filters() -> None: # work with empty params assert _format_iter_filters({}) == {} # doesn't affect other params - params = {'a': 123, 'b': 456} + params: dict[str, Any] = {'a': 123, 'b': 456} assert _format_iter_filters(params) == params # pass filter as-is if not list @@ -40,23 +43,23 @@ def test_format_iter_filters(): _format_iter_filters({'filter': ['test', 123]}) -def test_item_resource_iter_no_params(): +def test_item_resource_iter_no_params() -> None: class MockClient: - def __init__(self): + def __init__(self) -> None: self._hsclient = object() - items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') + items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') # type: ignore[arg-type] items_proxy._origin = mock.Mock() items_proxy.iter(count=123) assert items_proxy._origin.list.call_args == mock.call(None, count=123) -def test_item_resource_iter_with_params(): +def test_item_resource_iter_with_params() -> None: class MockClient: - def __init__(self): + def __init__(self) -> None: self._hsclient = object() - items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') + items_proxy = _ItemsResourceProxy(mock.Mock, MockClient(), 'mocked_key') # type: ignore[arg-type] items_proxy._origin = mock.Mock() items_proxy.iter(count=123, startts=12345) assert (items_proxy._origin.list.call_args == diff --git a/tests/client/test_requests.py b/tests/client/test_requests.py index 7a397bc2..0bf8aafb 100644 --- a/tests/client/test_requests.py +++ b/tests/client/test_requests.py @@ -1,9 +1,12 @@ import pytest +from scrapinghub.client.jobs import Job +from scrapinghub.client.spiders import Spider + from .conftest import TEST_TS -def _add_test_requests(job): +def _add_test_requests(job: Job) -> None: r1 = job.requests.add( url='http://test.com/', status=200, method='GET', rs=1337, duration=5, ts=TEST_TS) @@ -16,7 +19,7 @@ def _add_test_requests(job): job.requests.flush() -def test_requests_iter(spider): +def test_requests_iter(spider: Spider) -> None: job = spider.jobs.run(meta={'state': 'running'}) _add_test_requests(job) job.requests.close() diff --git a/tests/client/test_samples.py b/tests/client/test_samples.py index 17798ca6..a1128b24 100644 --- a/tests/client/test_samples.py +++ b/tests/client/test_samples.py @@ -1,16 +1,19 @@ import pytest +from scrapinghub.client.jobs import Job +from scrapinghub.client.spiders import Spider + from .conftest import TEST_TS -def _add_test_samples(job): +def _add_test_samples(job: Job) -> None: job.samples.write([TEST_TS, 1, 2, 3]) job.samples.write([TEST_TS + 1, 5, 9, 4]) job.samples.flush() job.samples.close() -def test_samples_iter(spider, json_and_msgpack): +def test_samples_iter(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run(meta={'state': 'running'}) assert list(job.samples.iter()) == [] _add_test_samples(job) @@ -22,7 +25,7 @@ def test_samples_iter(spider, json_and_msgpack): next(o) -def test_samples_list(spider, json_and_msgpack): +def test_samples_list(spider: Spider, json_and_msgpack: str) -> None: job = spider.jobs.run(meta={'state': 'running'}) _add_test_samples(job) o = job.samples.list() diff --git a/tests/client/test_spiders.py b/tests/client/test_spiders.py index 48ca2832..130dbe96 100644 --- a/tests/client/test_spiders.py +++ b/tests/client/test_spiders.py @@ -11,12 +11,13 @@ from scrapinghub.client.jobs import Jobs, Job from scrapinghub.client.spiders import Spider from scrapinghub.client.utils import JobKey +from scrapinghub.client.projects import Project from ..conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME from .utils import validate_default_meta -def test_spiders_get(project): +def test_spiders_get(project: Project) -> None: spider = project.spiders.get(TEST_SPIDER_NAME) assert isinstance(spider, Spider) assert isinstance(spider.jobs, Jobs) @@ -25,13 +26,13 @@ def test_spiders_get(project): project.spiders.get('non-existing') -def test_spiders_list(project): +def test_spiders_list(project: Project) -> None: assert project.spiders.list() == [ {'id': 'hs-test-spider', 'tags': [], 'type': 'manual', 'version': None}] -def test_spider_base(project, spider): +def test_spider_base(project: Project, spider: Spider) -> None: assert isinstance(spider._id, string_types) assert isinstance(spider.key, string_types) assert spider.key == spider.project_id + '/' + spider._id @@ -40,7 +41,7 @@ def test_spider_base(project, spider): assert isinstance(project.jobs, Jobs) -def test_spider_list_update_tags(project, spider): +def test_spider_list_update_tags(project: Project, spider: Spider) -> None: with pytest.raises(BadRequest): spider.update_tags() @@ -52,13 +53,13 @@ def test_spider_list_update_tags(project, spider): assert spider.list_tags() == [] -def test_spider_jobs(spider): +def test_spider_jobs(spider: Spider) -> None: jobs = spider.jobs assert jobs.project_id == TEST_PROJECT_ID assert jobs.spider is spider -def test_spider_jobs_count(spider): +def test_spider_jobs_count(spider: Spider) -> None: jobs = spider.jobs assert jobs.count() == 0 assert jobs.count(state=['pending', 'running', 'finished']) == 0 @@ -83,7 +84,7 @@ def test_spider_jobs_count(spider): assert jobs.count() == 3 -def test_spider_jobs_iter(spider): +def test_spider_jobs_iter(spider: Spider) -> None: spider.jobs.run(meta={'state': 'running'}) # no finished jobs @@ -102,14 +103,14 @@ def test_spider_jobs_iter(spider): assert isinstance(running_time, int) and running_time > 0 elapsed = job.get('elapsed') assert isinstance(elapsed, int) and elapsed > 0 - assert job.get('key').startswith(TEST_PROJECT_ID) + assert job['key'].startswith(TEST_PROJECT_ID) assert job.get('spider') == TEST_SPIDER_NAME assert job.get('state') == 'running' with pytest.raises(StopIteration): next(jobs1) -def test_spider_jobs_list(spider): +def test_spider_jobs_list(spider: Spider) -> None: spider.jobs.run(meta={'state': 'running'}) # no finished jobs @@ -128,12 +129,12 @@ def test_spider_jobs_list(spider): assert isinstance(running_time, int) and running_time > 0 elapsed = job.get('elapsed') assert isinstance(elapsed, int) and elapsed > 0 - assert job.get('key').startswith(TEST_PROJECT_ID) + assert job['key'].startswith(TEST_PROJECT_ID) assert job.get('spider') == TEST_SPIDER_NAME assert job.get('state') == 'running' -def test_spider_jobs_run(spider): +def test_spider_jobs_run(spider: Spider) -> None: job0 = spider.jobs.run() assert isinstance(job0, Job) validate_default_meta(job0.metadata, state='pending') @@ -159,7 +160,7 @@ def test_spider_jobs_run(spider): assert job1.metadata.get('started_by') -def test_spider_jobs_get(spider): +def test_spider_jobs_get(spider: Spider) -> None: # error on wrong jobkey format with pytest.raises(ValueError): spider.jobs.get('wrongg') @@ -172,12 +173,12 @@ def test_spider_jobs_get(spider): with pytest.raises(ValueError): spider.jobs.get(TEST_PROJECT_ID + '/2/3') - fake_job_id = str(JobKey(spider.project_id, spider._id, 3)) + fake_job_id = str(JobKey(spider.project_id, spider._id, '3')) fake_job = spider.jobs.get(fake_job_id) assert isinstance(fake_job, Job) -def test_spider_jobs_summary(spider): +def test_spider_jobs_summary(spider: Spider) -> None: summary = spider.jobs.summary() expected_summary = [{'count': 0, 'name': state, 'summary': []} for state in ['pending', 'running', 'finished']] @@ -215,7 +216,7 @@ def test_spider_jobs_summary(spider): assert summary4['summary'][0].get('units') == 1 -def test_spider_jobs_iter_last(spider): +def test_spider_jobs_iter_last(spider: Spider) -> None: lastsumm0 = spider.jobs.iter_last() assert isinstance(lastsumm0, types.GeneratorType) assert list(lastsumm0) == [] diff --git a/tests/client/test_utils.py b/tests/client/test_utils.py index 63d1b160..2a662153 100644 --- a/tests/client/test_utils.py +++ b/tests/client/test_utils.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import pytest from codecs import encode @@ -10,7 +11,7 @@ @pytest.fixture(autouse=True) -def isolated_auth_env(tmp_path, monkeypatch): +def isolated_auth_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Keep auth resolution hermetic: drop any ambient auth env vars and run from an empty directory so ``find_dotenv()`` can't pick up a stray ``.env`` from the developer's working tree.""" @@ -19,67 +20,67 @@ def isolated_auth_env(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) -def test_parse_auth_none(): +def test_parse_auth_none() -> None: with pytest.raises(RuntimeError): parse_auth(None) @mock.patch.dict(os.environ, {'SH_APIKEY': 'testkey'}) -def test_parse_auth_none_with_env(): +def test_parse_auth_none_with_env() -> None: assert parse_auth(None) == ('testkey', '') @mock.patch.dict(os.environ, {'SH_APIKEY': 'testkey', 'SHUB_JOBAUTH': 'jwt'}) -def test_parse_auth_none_with_multiple_env(): +def test_parse_auth_none_with_multiple_env() -> None: assert parse_auth(None) == ('testkey', '') @mock.patch.dict(os.environ, {'SHUB_APIKEY': 'aliaskey'}) -def test_parse_auth_none_with_shub_apikey_alias(): +def test_parse_auth_none_with_shub_apikey_alias() -> None: assert parse_auth(None) == ('aliaskey', '') @mock.patch.dict(os.environ, {'SH_APIKEY': 'primary', 'SHUB_APIKEY': 'alias'}) -def test_parse_auth_sh_apikey_takes_precedence_over_alias(): +def test_parse_auth_sh_apikey_takes_precedence_over_alias() -> None: assert parse_auth(None) == ('primary', '') -def test_parse_auth_tuple(): +def test_parse_auth_tuple() -> None: assert parse_auth(('test', 'test')) == ('test', 'test') assert parse_auth(('apikey', '')) == ('apikey', '') with pytest.raises(ValueError): - parse_auth(('user', 'pass', 'bad-param')) + parse_auth(('user', 'pass', 'bad-param')) # type: ignore[arg-type] with pytest.raises(ValueError): - parse_auth((None, None)) + parse_auth((None, None)) # type: ignore[arg-type] with pytest.raises(ValueError): - parse_auth((1234, '')) + parse_auth((1234, '')) # type: ignore[arg-type] -def test_parse_auth_not_string(): +def test_parse_auth_not_string() -> None: with pytest.raises(ValueError): - parse_auth(12345) + parse_auth(12345) # type: ignore[arg-type] -def test_parse_auth_simple(): +def test_parse_auth_simple() -> None: assert parse_auth('user:pass') == ('user', 'pass') -def test_parse_auth_apikey(): +def test_parse_auth_apikey() -> None: apikey = 'c3a3c298c2b8c3a6c291c284c3a9' assert parse_auth(apikey) == (apikey, '') -def test_parse_auth_jwt_token(): +def test_parse_auth_jwt_token() -> None: test_job, test_token = '1/2/3', 'some.jwt.token' raw_token = (test_job + ':' + test_token).encode('utf8') encoded_token = encode(raw_token, 'hex_codec').decode('ascii') assert parse_auth(encoded_token) == (test_job, test_token) -def test_parse_auth_jwt_token_with_jwt_token_env(): +def test_parse_auth_jwt_token_with_jwt_token_env() -> None: dummy_test_job, dummy_test_token = '1/2/3', 'some.dummy.jwt.token' raw_token = (dummy_test_job + ':' + dummy_test_token).encode('utf8') dummy_encoded_token = encode(raw_token, 'hex_codec').decode('ascii') @@ -92,7 +93,7 @@ def test_parse_auth_jwt_token_with_jwt_token_env(): assert parse_auth(encoded_token) == (test_job, test_token) -def test_parse_auth_none_with_jwt_token_env(): +def test_parse_auth_none_with_jwt_token_env() -> None: test_job, test_token = '1/2/3', 'some.jwt.token' raw_token = (test_job + ':' + test_token).encode('utf8') encoded_token = encode(raw_token, 'hex_codec').decode('ascii') @@ -101,14 +102,16 @@ def test_parse_auth_none_with_jwt_token_env(): assert parse_auth(None) == (test_job, test_token) -def test_read_dotenv_auth_default_path(tmp_path): +def test_read_dotenv_auth_default_path(tmp_path: Path) -> None: (tmp_path / '.env').write_text('SH_APIKEY=FROMDOTENV\n') assert _read_dotenv_auth() == {'SH_APIKEY': 'FROMDOTENV'} assert 'SH_APIKEY' not in os.environ # reading the file must not touch env -def test_read_dotenv_auth_parent_dir(tmp_path, monkeypatch): +def test_read_dotenv_auth_parent_dir( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: (tmp_path / '.env').write_text('SHUB_APIKEY=FROMPARENT\n') subdir = tmp_path / 'project' / 'subdir' subdir.mkdir(parents=True) @@ -117,7 +120,7 @@ def test_read_dotenv_auth_parent_dir(tmp_path, monkeypatch): assert _read_dotenv_auth() == {'SHUB_APIKEY': 'FROMPARENT'} -def test_read_dotenv_auth_custom_path(tmp_path): +def test_read_dotenv_auth_custom_path(tmp_path: Path) -> None: env_file = tmp_path / 'custom.env' env_file.write_text('SH_APIKEY=CUSTOMKEY\nSHUB_JOBAUTH=CUSTOMJWT\n') @@ -126,7 +129,7 @@ def test_read_dotenv_auth_custom_path(tmp_path): } -def test_read_dotenv_auth_only_reads_auth_vars(tmp_path): +def test_read_dotenv_auth_only_reads_auth_vars(tmp_path: Path) -> None: env_file = tmp_path / 'custom.env' env_file.write_text('SH_APIKEY=ONLYTHIS\nOTHER_VAR=ignored\n') @@ -134,25 +137,27 @@ def test_read_dotenv_auth_only_reads_auth_vars(tmp_path): assert 'OTHER_VAR' not in os.environ -def test_read_dotenv_auth_missing_file(tmp_path): +def test_read_dotenv_auth_missing_file(tmp_path: Path) -> None: assert _read_dotenv_auth(str(tmp_path / 'does-not-exist.env')) == {} -def test_parse_auth_none_reads_dotenv(tmp_path): +def test_parse_auth_none_reads_dotenv(tmp_path: Path) -> None: env_file = tmp_path / 'custom.env' env_file.write_text('SH_APIKEY=DOTENVKEY\n') assert parse_auth(None, dotenv_path=str(env_file)) == ('DOTENVKEY', '') -def test_parse_auth_none_reads_shub_apikey_alias_from_dotenv(tmp_path): +def test_parse_auth_none_reads_shub_apikey_alias_from_dotenv( + tmp_path: Path, +) -> None: env_file = tmp_path / 'custom.env' env_file.write_text('SHUB_APIKEY=ALIASFROMFILE\n') assert parse_auth(None, dotenv_path=str(env_file)) == ('ALIASFROMFILE', '') -def test_parse_auth_none_reads_jobauth_from_dotenv(tmp_path): +def test_parse_auth_none_reads_jobauth_from_dotenv(tmp_path: Path) -> None: test_job, test_token = '1/2/3', 'some.jwt.token' raw_token = (test_job + ':' + test_token).encode('utf8') encoded_token = encode(raw_token, 'hex_codec').decode('ascii') @@ -163,7 +168,9 @@ def test_parse_auth_none_reads_jobauth_from_dotenv(tmp_path): assert parse_auth(None, dotenv_path=str(env_file)) == (test_job, test_token) -def test_parse_auth_env_takes_precedence_over_dotenv(tmp_path, monkeypatch): +def test_parse_auth_env_takes_precedence_over_dotenv( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv('SH_APIKEY', 'FROMENV') env_file = tmp_path / 'custom.env' env_file.write_text('SH_APIKEY=FROMDOTENV\n') @@ -171,7 +178,7 @@ def test_parse_auth_env_takes_precedence_over_dotenv(tmp_path, monkeypatch): assert parse_auth(None, dotenv_path=str(env_file)) == ('FROMENV', '') -def test_parse_auth_none_does_not_mutate_environ(tmp_path): +def test_parse_auth_none_does_not_mutate_environ(tmp_path: Path) -> None: env_file = tmp_path / 'custom.env' env_file.write_text('SH_APIKEY=DOTENVKEY\nSHUB_JOBAUTH=JWT\n') @@ -181,18 +188,18 @@ def test_parse_auth_none_does_not_mutate_environ(tmp_path): assert 'SHUB_JOBAUTH' not in os.environ -def test_parse_job_key(): +def test_parse_job_key() -> None: job_key = parse_job_key('123/10/11') assert job_key.project_id == '123' assert job_key.spider_id == '10' assert job_key.job_id == '11' -def test_parse_job_key_non_numeric(): +def test_parse_job_key_non_numeric() -> None: with pytest.raises(ValueError): parse_job_key('123/a/6') -def test_parse_job_key_incorrect_length(): +def test_parse_job_key_incorrect_length() -> None: with pytest.raises(ValueError): parse_job_key('123/1') diff --git a/tests/client/utils.py b/tests/client/utils.py index 927193a6..252d0b95 100644 --- a/tests/client/utils.py +++ b/tests/client/utils.py @@ -1,9 +1,15 @@ +from typing import Any + +from scrapinghub.client.jobs import Job + from ..conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME from ..conftest import TEST_DASH_ENDPOINT -def validate_default_meta(meta, state='pending', units=1, - priority=2, tags=None): +def validate_default_meta( + meta: Any, state: str = 'pending', units: int = 1, priority: int = 2, + tags: list[str] | None = None, +) -> None: assert meta.get('project') == int(TEST_PROJECT_ID) assert meta.get('spider') == TEST_SPIDER_NAME assert meta.get('state') == state @@ -15,7 +21,7 @@ def validate_default_meta(meta, state='pending', units=1, assert meta.get('portia_url') -def normalize_job_for_tests(job): +def normalize_job_for_tests(job: Job) -> Job: """A temporary workaround to deal with VCR.py cassettes(snapshots). The existing tests highly rely on VCR.py which creates snapshots of real diff --git a/tests/conftest.py b/tests/conftest.py index 59ad94f2..d812d624 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ import base64 import os import pickle +from typing import Any import pytest import re import sys @@ -32,7 +33,7 @@ # https://github.com/kevin1024/vcrpy/issues/719#issuecomment-1811544263 -def upgrade_cassette(cassette): +def upgrade_cassette(cassette: dict[str, Any]) -> None: for interaction in cassette['interactions']: response = interaction.get('response', {}) headers = response.get('headers', {}) @@ -46,7 +47,7 @@ def upgrade_cassette(cassette): class VCRGzipSerializer(object): """Custom ZIP serializer for VCR.py.""" - def serialize(self, cassette_dict): + def serialize(self, cassette_dict: dict[str, Any]) -> str: # receives a dict, must return a string # there can be binary data inside some of the requests, # so it's impossible to use json for serialization to string @@ -54,20 +55,20 @@ def serialize(self, cassette_dict): compressed = zlib.compress(pickle.dumps(cassette_dict, protocol=2)) return base64.b64encode(compressed).decode('utf8') - def deserialize(self, cassette_string): + def deserialize(self, cassette_string: str) -> dict[str, Any]: # receives a string, must return a dict decoded = base64.b64decode(cassette_string.encode('utf8')) - cassette = pickle.loads(zlib.decompress(decoded)) + cassette: dict[str, Any] = pickle.loads(zlib.decompress(decoded)) if sys.version_info >= (3, 10): upgrade_cassette(cassette) return cassette -def normalize_endpoint(uri, endpoint, default_endpoint): +def normalize_endpoint(uri: str, endpoint: str, default_endpoint: str) -> str: return uri.replace(endpoint.rstrip('/'), default_endpoint.rstrip('/')) -def normalize_cassette(cassette_dict): +def normalize_cassette(cassette_dict: dict[str, Any]) -> dict[str, Any]: """ This function normalizes the cassette dict trying to make sure we are always making API requests with the same variables: @@ -104,7 +105,7 @@ def normalize_cassette(cassette_dict): return cassette_dict -def pytest_addoption(parser): +def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--update-cassettes", action="store_true", default=False, help="test with real services rewriting existing vcr cassettes") @@ -116,16 +117,17 @@ def pytest_addoption(parser): help="disable messagepack-serialization based tests") -def request_accept_header_matcher(r1, r2): +def request_accept_header_matcher(r1: Any, r2: Any) -> bool: """Custom VCR.py matcher by Accept header.""" - def _get_accept_header(request): - return request.headers.get('Accept', '').lower() + def _get_accept_header(request: Any) -> str: + accept: str = request.headers.get('Accept', '') + return accept.lower() return _get_accept_header(r1) == _get_accept_header(r2) @pytest.fixture -def frontier_name(request): +def frontier_name(request: pytest.FixtureRequest) -> str: """Provide a name for test-unique HS frontier.""" return re.sub(r'\W+', '-', request.node.nodeid) diff --git a/tests/hubstorage/conftest.py b/tests/hubstorage/conftest.py index 6ede3546..57836b89 100644 --- a/tests/hubstorage/conftest.py +++ b/tests/hubstorage/conftest.py @@ -1,4 +1,6 @@ import os +from collections.abc import Iterator +from typing import Any import vcr import pytest @@ -9,6 +11,9 @@ from scrapinghub import HubstorageClient from scrapinghub.hubstorage.utils import urlpathjoin from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE +from scrapinghub.hubstorage.collectionsrt import Collection +from scrapinghub.hubstorage.job import Job +from scrapinghub.hubstorage.project import Project from ..conftest import request_accept_header_matcher from ..conftest import VCRGzipSerializer @@ -32,7 +37,7 @@ 'path', 'query', 'accept_header') -def pytest_configure(config): +def pytest_configure(config: pytest.Config) -> None: if config.option.update_cassettes: # there's vcr `all` mode to update cassettes but it doesn't delete # or clear existing records, so its size will always only grow @@ -47,29 +52,31 @@ def pytest_configure(config): my_vcr.before_record_request = lambda request: None -def is_using_real_services(request): - return (request.config.option.update_cassettes or - request.config.option.ignore_cassettes) +def is_using_real_services(request: pytest.FixtureRequest) -> bool: + return bool(request.config.option.update_cassettes or + request.config.option.ignore_cassettes) @pytest.fixture(scope='session') -def hsclient(): +def hsclient() -> HubstorageClient: return HubstorageClient(auth=TEST_AUTH, endpoint=TEST_ENDPOINT) @pytest.fixture(scope='session') -def hsproject(hsclient): +def hsproject(hsclient: HubstorageClient) -> Project: return hsclient.get_project(TEST_PROJECT_ID) @pytest.fixture(scope='session') -@my_vcr.use_cassette() -def hsspiderid(hsproject): +@my_vcr.use_cassette() # type: ignore[untyped-decorator] +def hsspiderid(hsproject: Project) -> str: return str(hsproject.ids.spider(TEST_SPIDER_NAME, create=1)) @pytest.fixture(scope='session') -def hscollection(hsproject, request): +def hscollection( + hsproject: Project, request: pytest.FixtureRequest, +) -> Iterator[Collection]: collection = get_test_collection(hsproject) if is_using_real_services(request): clean_collection(collection) @@ -77,7 +84,10 @@ def hscollection(hsproject, request): @pytest.fixture(autouse=True, scope='session') -def setup_session(hsclient, hsproject, hscollection, request): +def setup_session( + hsclient: HubstorageClient, hsproject: Project, hscollection: Collection, + request: pytest.FixtureRequest, +) -> Iterator[None]: if is_using_real_services(request): set_testbotgroup(hsproject) remove_all_jobs(hsproject) @@ -86,16 +96,22 @@ def setup_session(hsclient, hsproject, hscollection, request): @pytest.fixture(params=['json', 'msgpack']) -def json_and_msgpack(hsclient, monkeypatch, request): +def json_and_msgpack( + hsclient: HubstorageClient, monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +) -> str: if request.param == 'json': monkeypatch.setattr(hsclient, 'use_msgpack', False) elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"): pytest.skip("messagepack-based tests are disabled") - return request.param + param: str = request.param + return param @pytest.fixture(autouse=True) -def setup_vcrpy(request, hsproject): +def setup_vcrpy( + request: pytest.FixtureRequest, hsproject: Project, +) -> Iterator[None]: # generates names like "test_module/test_function.yaml" # otherwise it uses current function name (setup_vcrpy) for all tests # other option is to add vcr decorator to each test separately @@ -117,18 +133,18 @@ def setup_vcrpy(request, hsproject): # ---------------------------------------------------------------------------- -def start_job(hsproject, **startparams): +def start_job(hsproject: Project, **startparams: Any) -> Job: jobdata = hsproject.jobq.start(**startparams) - if jobdata: - jobkey = jobdata.pop('key') - jobauth = (jobkey, jobdata['auth']) - return hsproject.get_job(jobkey, jobauth=jobauth, metadata=jobdata) + assert jobdata + jobkey = jobdata.pop('key') + jobauth = (jobkey, jobdata['auth']) + return hsproject.get_job(jobkey, jobauth=jobauth, metadata=jobdata) # Clean environment section -def remove_all_jobs(hsproject): +def remove_all_jobs(hsproject: Project) -> None: for k in list(hsproject.settings.keys()): if k != 'botgroups': del hsproject.settings[k] @@ -141,7 +157,7 @@ def remove_all_jobs(hsproject): _remove_job(hsproject, summary['key']) -def _remove_job(hsproject, jobkey): +def _remove_job(hsproject: Project, jobkey: str) -> None: hsproject.jobq.finish(jobkey) hsproject.jobq.delete(jobkey) # delete job @@ -151,24 +167,24 @@ def _remove_job(hsproject, jobkey): # Collection helpers section -def get_test_collection(project): +def get_test_collection(project: Project) -> Collection: return project.collections.new_store(TEST_COLLECTION_NAME) -def clean_collection(collection): +def clean_collection(collection: Collection) -> None: try: for item in collection.iter_values(): collection.delete(item['_key']) except HTTPError as e: # if collection doesn't exist yet service responds 404 - if e.response.status_code != 404: + if e.response is None or e.response.status_code != 404: raise # Botgroups helpers section -def set_testbotgroup(hsproject): +def set_testbotgroup(hsproject: Project) -> None: hsproject.settings.apipost(jl={'botgroups': [TEST_BOTGROUP]}) # Additional step to populate JobQ's botgroups table url = urlpathjoin(TEST_ENDPOINT, 'botgroups', TEST_BOTGROUP, 'max_running') @@ -176,7 +192,7 @@ def set_testbotgroup(hsproject): hsproject.settings.expire() -def unset_testbotgroup(hsproject): +def unset_testbotgroup(hsproject: Project) -> None: hsproject.settings.apidelete('botgroups') hsproject.settings.expire() # Additional step to delete botgroups in JobQ diff --git a/tests/hubstorage/test_activity.py b/tests/hubstorage/test_activity.py index d874f922..b8783f7b 100644 --- a/tests/hubstorage/test_activity.py +++ b/tests/hubstorage/test_activity.py @@ -3,8 +3,10 @@ """ from six.moves import range +from scrapinghub.hubstorage.project import Project -def test_post_and_reverse_get(hsproject): + +def test_post_and_reverse_get(hsproject: Project) -> None: # make some sample data orig_data = [{u'foo': 42, u'counter': i} for i in range(20)] data1 = orig_data[:10] @@ -20,13 +22,13 @@ def test_post_and_reverse_get(hsproject): assert orig_data[::-1] == result -def test_filters(hsproject): +def test_filters(hsproject: Project) -> None: hsproject.activity.post({'c': i} for i in range(10)) r = list(hsproject.activity.list(filter='["c", ">", [5]]', count=2)) assert r == [{'c': 9}, {'c': 8}] -def test_timestamp(hsproject): +def test_timestamp(hsproject: Project) -> None: hsproject.activity.add({'foo': 'bar'}, baz='qux') entry = next(hsproject.activity.list(count=1, meta='_ts')) assert entry.pop('_ts', None) diff --git a/tests/hubstorage/test_batchuploader.py b/tests/hubstorage/test_batchuploader.py index 435bee24..7ec971eb 100644 --- a/tests/hubstorage/test_batchuploader.py +++ b/tests/hubstorage/test_batchuploader.py @@ -2,16 +2,23 @@ Test Project """ import time +from typing import Any import pytest from six.moves import range from collections import defaultdict from scrapinghub.hubstorage import ValueTooLarge +from scrapinghub.hubstorage import HubstorageClient +from scrapinghub.hubstorage.batchuploader import _BatchWriter +from scrapinghub.hubstorage.job import Job +from scrapinghub.hubstorage.project import Project from ..conftest import TEST_SPIDER_NAME, TEST_AUTH from .conftest import start_job -def _job_and_writer(hsclient, hsproject, **writerargs): +def _job_and_writer( + hsclient: HubstorageClient, hsproject: Project, **writerargs: Any, +) -> tuple[Job, _BatchWriter]: hsproject.push_job(TEST_SPIDER_NAME) job = start_job(hsproject) batch_uploader = hsclient.batchuploader @@ -20,7 +27,9 @@ def _job_and_writer(hsclient, hsproject, **writerargs): return job, writer -def test_writer_batchsize(hsclient, hsproject, json_and_msgpack): +def test_writer_batchsize( + hsclient: HubstorageClient, hsproject: Project, json_and_msgpack: str, +) -> None: job, writer = _job_and_writer(hsclient, hsproject, size=10) for x in range(111): writer.write({'x': x}) @@ -28,14 +37,16 @@ def test_writer_batchsize(hsclient, hsproject, json_and_msgpack): # this works only for small batches (previous size=10 and small data) # as internally HS may commit a single large request as many smaller # commits, each with different timestamps - groups = defaultdict(int) + groups: defaultdict[Any, int] = defaultdict(int) for doc in job.items.list(meta=['_ts']): groups[doc['_ts']] += 1 assert len(groups) == 12 -def test_writer_maxitemsize(hsclient, hsproject): +def test_writer_maxitemsize( + hsclient: HubstorageClient, hsproject: Project, +) -> None: _, writer = _job_and_writer(hsclient, hsproject) max_size = writer.maxitemsize with pytest.raises(ValueTooLarge) as excinfo1: @@ -57,7 +68,9 @@ def test_writer_maxitemsize(hsclient, hsproject): ' \'{"b+\\.\\.\\.\'') -def test_writer_maxitemsize_custom(hsclient, hsproject): +def test_writer_maxitemsize_custom( + hsclient: HubstorageClient, hsproject: Project, +) -> None: _, writer = _job_and_writer(hsclient, hsproject, maxitemsize=512*1024) with pytest.raises(ValueTooLarge) as excinfo: writer.write({'b': 'x' * writer.maxitemsize}) @@ -66,7 +79,9 @@ def test_writer_maxitemsize_custom(hsclient, hsproject): ' \'{"b": "x+\\.\\.\\.\'') -def test_writer_contentencoding(hsclient, hsproject): +def test_writer_contentencoding( + hsclient: HubstorageClient, hsproject: Project, +) -> None: for ce in ('identity', 'gzip'): job, writer = _job_and_writer(hsclient, hsproject, content_encoding=ce) @@ -76,7 +91,9 @@ def test_writer_contentencoding(hsclient, hsproject): assert job.items.stats()['totals']['input_values'] == 111 -def test_writer_interval(hsclient, hsproject, json_and_msgpack): +def test_writer_interval( + hsclient: HubstorageClient, hsproject: Project, json_and_msgpack: str, +) -> None: job, writer = _job_and_writer(hsclient, hsproject, size=1000, interval=1) for x in range(111): @@ -85,7 +102,7 @@ def test_writer_interval(hsclient, hsproject, json_and_msgpack): time.sleep(2) writer.close() - groups = defaultdict(int) + groups: defaultdict[Any, int] = defaultdict(int) for doc in job.items.list(meta=['_ts']): groups[doc['_ts']] += 1 diff --git a/tests/hubstorage/test_client.py b/tests/hubstorage/test_client.py index d6533c31..9871b72e 100644 --- a/tests/hubstorage/test_client.py +++ b/tests/hubstorage/test_client.py @@ -1,26 +1,29 @@ """ Test Client """ +from typing import Any + from scrapinghub import HubstorageClient from scrapinghub.hubstorage.utils import apipoll +from scrapinghub.hubstorage.project import Project from ..conftest import TEST_AUTH, TEST_ENDPOINT from ..conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME from .conftest import start_job -def test_default_ua(hsclient): +def test_default_ua(hsclient: HubstorageClient) -> None: assert hsclient.user_agent == HubstorageClient.DEFAULT_USER_AGENT -def test_custom_ua(): +def test_custom_ua() -> None: client = HubstorageClient(auth=TEST_AUTH, endpoint=TEST_ENDPOINT, user_agent='testUA') assert client.user_agent == 'testUA' -def test_push_job(hsclient, hsproject): +def test_push_job(hsclient: HubstorageClient, hsproject: Project) -> None: hsclient.push_job( TEST_PROJECT_ID, TEST_SPIDER_NAME, priority=hsproject.jobq.PRIO_LOW, @@ -39,11 +42,11 @@ def test_push_job(hsclient, hsproject): assert meta.get('foo') == u'baz' -def test_jobsummaries(hsclient): +def test_jobsummaries(hsclient: HubstorageClient) -> None: # add at least one running or pending job to ensure summary is returned hsclient.push_job(TEST_PROJECT_ID, TEST_SPIDER_NAME, state='running') - def _get_summary(): + def _get_summary() -> Any: jss = hsclient.projects.jobsummaries() mjss = dict((str(js['project']), js) for js in jss) return mjss.get(TEST_PROJECT_ID) @@ -52,7 +55,7 @@ def _get_summary(): assert summary is not None -def test_timestamp(hsclient): +def test_timestamp(hsclient: HubstorageClient) -> None: ts1 = hsclient.server_timestamp() ts2 = hsclient.server_timestamp() assert ts1 > 0 diff --git a/tests/hubstorage/test_collections.py b/tests/hubstorage/test_collections.py index bb26c8c7..b860b8e1 100644 --- a/tests/hubstorage/test_collections.py +++ b/tests/hubstorage/test_collections.py @@ -3,21 +3,25 @@ """ import random from contextlib import closing +from collections.abc import Callable +from typing import Any import pytest from scrapinghub import HubstorageClient +from scrapinghub.hubstorage.collectionsrt import Collection +from scrapinghub.hubstorage.project import Project from six.moves import range from ..conftest import TEST_COLLECTION_NAME from .testutil import failing_downloader -def _mkitem(): +def _mkitem() -> dict[str, Any]: return dict(field1='value1', field2=['value2a', 'value2b'], field3=3, field4={'v4k': 'v4v'}) -def test_simple_count(hsproject, hscollection): +def test_simple_count(hsproject: Project, hscollection: Collection) -> None: test_item = dict(_mkitem()) test_item['_key'] = 'a' @@ -25,7 +29,7 @@ def test_simple_count(hsproject, hscollection): assert hscollection.count() == 1 -def post_get_delete_test(hsproject): +def post_get_delete_test(hsproject: Project) -> None: test_item = _mkitem() item_to_send = dict(test_item) item_to_send['_key'] = test_key = 'insert_test_key' @@ -50,7 +54,7 @@ def post_get_delete_test(hsproject): col.get(test_key) -def post_scan_test(hsproject, hscollection): +def post_scan_test(hsproject: Project, hscollection: Collection) -> None: # populate with 20 items test_item = _mkitem() last_key = None @@ -86,7 +90,9 @@ def post_scan_test(hsproject, hscollection): hscollection.get(last_key) -def test_errors_bad_key(hscollection, json_and_msgpack): +def test_errors_bad_key( + hscollection: Collection, json_and_msgpack: str, +) -> None: with pytest.raises(KeyError): hscollection.get('does_not_exist') @@ -96,12 +102,12 @@ def test_errors_bad_key(hscollection, json_and_msgpack): {'_key': []}, {'_key': 'large_test', 'value': 'x' * 1024 ** 2}, ]) -def test_errors(hscollection, testarg): +def test_errors(hscollection: Collection, testarg: dict[str, Any]) -> None: with pytest.raises(ValueError): hscollection.set(testarg) -def test_data_download(hsproject, hscollection): +def test_data_download(hsproject: Project, hscollection: Collection) -> None: items = [] with closing(hscollection.create_writer()) as writer: for i in range(20): @@ -121,15 +127,16 @@ def test_data_download(hsproject, hscollection): assert len(downloaded) == 19 -def test_invalid_collection_name(hsproject): +def test_invalid_collection_name(hsproject: Project) -> None: cols = hsproject.collections - for method, args in [ + cases: list[tuple[Callable[..., Any], tuple[str, ...]]] = [ (cols.new_collection, ('invalidtype', 'n')), (cols.new_store, ('foo-bar',)), (cols.new_store, ('foo/bar',)), (cols.new_store, ('/foo',)), (cols.create_writer, ('invalidtype', 'n')), - (cols.create_writer, ('s', 'foo-bar'))]: + (cols.create_writer, ('s', 'foo-bar'))] + for method, args in cases: with pytest.raises(ValueError): method(*args) @@ -153,12 +160,15 @@ def test_invalid_collection_name(hsproject): ('list', False), (None, False), ]) -def test_allows_msgpack(hsclient, path, expected_result, json_and_msgpack): +def test_allows_msgpack( + hsclient: HubstorageClient, path: str | None, expected_result: bool, + json_and_msgpack: str, +) -> None: collections = hsclient.get_project(2222000).collections assert collections._allows_mpack(path) is (hsclient.use_msgpack and expected_result) -def test_truncate(hscollection): +def test_truncate(hscollection: Collection) -> None: # populate with 20 items test_item = _mkitem() with closing(hscollection.create_writer()) as writer: diff --git a/tests/hubstorage/test_frontier.py b/tests/hubstorage/test_frontier.py index fedc154f..9160ab34 100644 --- a/tests/hubstorage/test_frontier.py +++ b/tests/hubstorage/test_frontier.py @@ -1,22 +1,26 @@ """ Test Frontier """ +from typing import Any + import pytest +from scrapinghub.hubstorage.project import Project + from ..conftest import TEST_FRONTIER_SLOT @pytest.fixture(autouse=True) -def delete_frontier_slot(hsproject, frontier_name): +def delete_frontier_slot(hsproject: Project, frontier_name: str) -> None: frontier = hsproject.frontier frontier.delete_slot(frontier_name, TEST_FRONTIER_SLOT) -def _get_urls(batch): +def _get_urls(batch: Any) -> list[Any]: return [r[0] for r in batch['requests']] -def test_add_read(hsproject, frontier_name): +def test_add_read(hsproject: Project, frontier_name: str) -> None: frontier = hsproject.frontier fps = [{'fp': '/'}] @@ -31,7 +35,7 @@ def test_add_read(hsproject, frontier_name): assert urls == expected_urls -def test_add_multiple_chunks(hsproject, frontier_name): +def test_add_multiple_chunks(hsproject: Project, frontier_name: str) -> None: frontier = hsproject.frontier old_count = frontier.newcount @@ -75,7 +79,7 @@ def test_add_multiple_chunks(hsproject, frontier_name): assert urls == expected_urls -def test_add_big_chunk(hsproject, frontier_name): +def test_add_big_chunk(hsproject: Project, frontier_name: str) -> None: frontier = hsproject.frontier batch_size = 300 @@ -113,7 +117,7 @@ def test_add_big_chunk(hsproject, frontier_name): assert urls == expected_urls -def test_add_extra_params(hsproject, frontier_name): +def test_add_extra_params(hsproject: Project, frontier_name: str) -> None: frontier = hsproject.frontier qdata = {"a": 1, "b": 2, "c": 3} diff --git a/tests/hubstorage/test_jobq.py b/tests/hubstorage/test_jobq.py index 8bc41278..a8010073 100644 --- a/tests/hubstorage/test_jobq.py +++ b/tests/hubstorage/test_jobq.py @@ -2,22 +2,26 @@ Test JobQ """ import os +from collections.abc import Iterable, Iterator +from typing import Any import six import pytest from six.moves import range from scrapinghub.hubstorage.jobq import DuplicateJobError from scrapinghub.hubstorage.utils import apipoll +from scrapinghub.hubstorage import HubstorageClient +from scrapinghub.hubstorage.project import Project from ..conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME from .conftest import hsspiderid -def _keys(lst): +def _keys(lst: Iterable[Any]) -> list[str]: return [x['key'] for x in lst] -def test_push(hsclient, hsproject): +def test_push(hsclient: HubstorageClient, hsproject: Project) -> None: jobq = hsproject.jobq qjob = jobq.push(TEST_SPIDER_NAME) assert 'key' in qjob, qjob @@ -41,21 +45,27 @@ def test_push(hsclient, hsproject): assert job.metadata.get('state') == u'deleted' -def test_push_with_extras(hsclient, hsproject): +def test_push_with_extras( + hsclient: HubstorageClient, hsproject: Project, +) -> None: qjob = hsproject.jobq.push(TEST_SPIDER_NAME, foo='bar', baz='fuu') job = hsclient.get_job(qjob['key']) assert job.metadata.get('foo') == u'bar' assert job.metadata.get('baz') == u'fuu' -def test_push_with_priority(hsclient, hsproject): +def test_push_with_priority( + hsclient: HubstorageClient, hsproject: Project, +) -> None: jobq = hsproject.jobq qjob = jobq.push(TEST_SPIDER_NAME, priority=jobq.PRIO_HIGHEST) assert 'key' in qjob, qjob assert 'auth' in qjob, qjob -def test_push_with_state(hsclient, hsproject): +def test_push_with_state( + hsclient: HubstorageClient, hsproject: Project, +) -> None: qjob = hsproject.jobq.push(TEST_SPIDER_NAME, state='running') assert 'key' in qjob, qjob assert 'auth' in qjob, qjob @@ -63,7 +73,7 @@ def test_push_with_state(hsclient, hsproject): assert job.metadata.get('state') == u'running' -def test_push_with_unique(hsproject): +def test_push_with_unique(hsproject: Project) -> None: jobq = hsproject.jobq # no unique key jobq.push(TEST_SPIDER_NAME) @@ -82,7 +92,7 @@ def test_push_with_unique(hsproject): jobq.push(TEST_SPIDER_NAME, unique='h1') -def test_startjob(hsproject): +def test_startjob(hsproject: Project) -> None: jobq = hsproject.jobq qj = jobq.push(TEST_SPIDER_NAME) nj = jobq.start() @@ -95,7 +105,7 @@ def test_startjob(hsproject): assert nj[u'priority'] == jobq.PRIO_NORMAL -def test_startjob_with_extras(hsproject): +def test_startjob_with_extras(hsproject: Project) -> None: jobq = hsproject.jobq pushextras = { 'string': 'foo', @@ -110,7 +120,8 @@ def test_startjob_with_extras(hsproject): 'nil': None, } qj = jobq.push(TEST_SPIDER_NAME, **pushextras) - startextras = dict(('s_' + k, v) for k, v in six.iteritems(pushextras)) + startextras: dict[str, Any] = dict( + ('s_' + k, v) for k, v in six.iteritems(pushextras)) nj = jobq.start(**startextras) assert qj['key'] == nj['key'] for k, v in six.iteritems(dict(pushextras, **startextras)): @@ -120,7 +131,7 @@ def test_startjob_with_extras(hsproject): assert nj.get(k) == v -def test_startjob_order(hsproject): +def test_startjob_order(hsproject: Project) -> None: jobq = hsproject.jobq q1 = jobq.push(TEST_SPIDER_NAME) q2 = jobq.push(TEST_SPIDER_NAME) @@ -130,7 +141,7 @@ def test_startjob_order(hsproject): assert jobq.start()['key'] == q3['key'] -def test_summary(hsproject): +def test_summary(hsproject: Project) -> None: jobq = hsproject.jobq # push at least one job per state jobq.push(TEST_SPIDER_NAME) @@ -143,7 +154,7 @@ def test_summary(hsproject): assert jobq.summary('finished') -def test_summary_jobmeta(hsproject): +def test_summary_jobmeta(hsproject: Project) -> None: jobq = hsproject.jobq jobq.push(TEST_SPIDER_NAME, foo='bar', caz='fuu') pendings = jobq.summary('pending', jobmeta='foo')['summary'] @@ -157,7 +168,7 @@ def test_summary_jobmeta(hsproject): assert p1.get('caz') == 'fuu' -def test_summary_countstart(hsproject): +def test_summary_countstart(hsproject: Project) -> None: # push more than 5 jobs into same queue N = 6 jobq = hsproject.jobq @@ -177,7 +188,9 @@ def test_summary_countstart(hsproject): [o['key'] for o in s2['summary'][-6:-3]]) -def test_summaries_and_state_changes(hsproject, hsspiderid): +def test_summaries_and_state_changes( + hsproject: Project, hsspiderid: str, +) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME) j2 = jobq.push(TEST_SPIDER_NAME) @@ -207,7 +220,7 @@ def test_summaries_and_state_changes(hsproject, hsspiderid): _assert_queue(hsproject, hsspiderid, 'finished', []) -def test_list_with_state(hsproject): +def test_list_with_state(hsproject: Project) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME, state='finished') j2 = jobq.push(TEST_SPIDER_NAME, state='running') @@ -220,7 +233,7 @@ def test_list_with_state(hsproject): assert _keys(jobq.list(state=['running', 'pending'])) == _keys([j3, j2]) -def test_list_with_count(hsproject): +def test_list_with_count(hsproject: Project) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME, state='finished') # NOQA j2 = jobq.push(TEST_SPIDER_NAME, state='finished') # NOQA @@ -230,7 +243,7 @@ def test_list_with_count(hsproject): assert _keys(jobq.list(count=2)) == _keys([j4, j3]) -def test_list_with_stop(hsproject): +def test_list_with_stop(hsproject: Project) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME, state='finished') j2 = jobq.push(TEST_SPIDER_NAME, state='finished') @@ -241,7 +254,7 @@ def test_list_with_stop(hsproject): assert _keys(jobq.list(stop=j1['key'])) == _keys([j4, j3, j2]) -def test_list_with_tags(hsproject): +def test_list_with_tags(hsproject: Project) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME, state='finished', tags=['t1']) j2 = jobq.push(TEST_SPIDER_NAME, state='finished', tags=['t2']) @@ -255,7 +268,7 @@ def test_list_with_tags(hsproject): # endts is not implemented @pytest.mark.xfail -def test_list_with_startts_endts(hsproject): +def test_list_with_startts_endts(hsproject: Project) -> None: jobq = hsproject.jobq j1 = jobq.push(TEST_SPIDER_NAME, state='finished') # NOQA j2 = jobq.push(TEST_SPIDER_NAME, state='finished') @@ -269,11 +282,11 @@ def test_list_with_startts_endts(hsproject): assert _keys(jobs) == _keys([j3, j2]) -def test_spider_updates(hsproject, hsspiderid): +def test_spider_updates(hsproject: Project, hsspiderid: str) -> None: jobq = hsproject.jobq spiderkey = '%s/%s' % (TEST_PROJECT_ID, hsspiderid) - def finish_and_delete_jobs(): + def finish_and_delete_jobs() -> Iterator[Any]: for job in jobq.finish(spiderkey): yield job jobq.delete(spiderkey) @@ -293,7 +306,7 @@ def finish_and_delete_jobs(): assert not list(jobq.delete(spiderkey)) -def test_multiple_job_update(hsproject): +def test_multiple_job_update(hsproject: Project) -> None: jobq = hsproject.jobq q1 = jobq.push(TEST_SPIDER_NAME) q2 = jobq.push(TEST_SPIDER_NAME) @@ -307,7 +320,7 @@ def test_multiple_job_update(hsproject): ['finished', 'finished', 'finished']) -def test_update(hsproject): +def test_update(hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) assert job.metadata['state'] == 'pending' hsproject.jobq.update(job, state='running', foo='bar') @@ -316,7 +329,7 @@ def test_update(hsproject): assert job.metadata['foo'] == 'bar' -def test_jobsummary(hsproject): +def test_jobsummary(hsproject: Project) -> None: jobs = [hsproject.push_job(TEST_SPIDER_NAME, foo=i) for i in range(5)] jobmetas = list(hsproject.jobq.jobsummary( @@ -328,7 +341,9 @@ def test_jobsummary(hsproject): } -def _assert_queue(hsproject, hsspiderid, qname, jobs): +def _assert_queue( + hsproject: Project, hsspiderid: str, qname: str, jobs: list[Any], +) -> None: summary = hsproject.jobq.summary(qname, spiderid=hsspiderid) assert summary['name'] == qname assert summary['count'] == len(jobs) diff --git a/tests/hubstorage/test_jobsmeta.py b/tests/hubstorage/test_jobsmeta.py index 6f5ff7db..4fa54b88 100644 --- a/tests/hubstorage/test_jobsmeta.py +++ b/tests/hubstorage/test_jobsmeta.py @@ -3,12 +3,20 @@ System tests for operations on stored job metadata """ +from collections.abc import Mapping +from typing import Any + +from scrapinghub.hubstorage import HubstorageClient +from scrapinghub.hubstorage.project import Project + from ..conftest import TEST_SPIDER_NAME from .conftest import start_job -def _assertMetadata(meta1, meta2): - def _clean(m): +def _assertMetadata( + meta1: Mapping[str, Any], meta2: Mapping[str, Any], +) -> None: + def _clean(m: Mapping[str, Any]) -> dict[str, Any]: return dict((k, v) for k, v in m.items() if k != 'updated_time') meta1 = _clean(meta1) @@ -16,7 +24,7 @@ def _clean(m): assert meta1 == meta2 -def test_basic(hsclient, hsproject): +def test_basic(hsclient: HubstorageClient, hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) assert 'auth' not in job.metadata assert 'state' in job.metadata @@ -60,7 +68,7 @@ def test_basic(hsclient, hsproject): _assertMetadata(job.metadata, job2.metadata) -def test_updating(hsproject): +def test_updating(hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) assert job.metadata.get('foo') is None job.update_metadata({'foo': 'bar'}) @@ -78,7 +86,7 @@ def test_updating(hsproject): assert job.metadata['state'] == state -def test_representation(hsproject): +def test_representation(hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) meta = job.metadata assert str(meta) != repr(meta) @@ -87,7 +95,7 @@ def test_representation(hsproject): assert meta.__class__.__name__ not in str(meta) -def test_jobauth(hsclient, hsproject): +def test_jobauth(hsclient: HubstorageClient, hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) assert job.jobauth is None assert job.auth == hsproject.auth @@ -99,7 +107,7 @@ def test_jobauth(hsclient, hsproject): assert samejob.items.auth == hsproject.auth -def test_authtoken(hsproject): +def test_authtoken(hsproject: Project) -> None: pendingjob = hsproject.push_job(TEST_SPIDER_NAME) runningjob = start_job(hsproject) assert pendingjob.key == runningjob.key diff --git a/tests/hubstorage/test_project.py b/tests/hubstorage/test_project.py index 4d849e4b..ae802cb3 100644 --- a/tests/hubstorage/test_project.py +++ b/tests/hubstorage/test_project.py @@ -1,6 +1,9 @@ """ Test Project """ +from collections.abc import Iterator +from typing import Any + import six import json import pytest @@ -8,6 +11,7 @@ from requests.exceptions import HTTPError from scrapinghub import HubstorageClient +from scrapinghub.hubstorage.project import Project from ..conftest import TEST_PROJECT_ID, TEST_SPIDER_NAME from .conftest import hsspiderid @@ -16,7 +20,7 @@ from .testutil import failing_downloader -def test_projectid(hsclient): +def test_projectid(hsclient: HubstorageClient) -> None: p1 = hsclient.get_project(int(TEST_PROJECT_ID)) p2 = hsclient.get_project(str(TEST_PROJECT_ID)) assert p1.projectid == p2.projectid @@ -26,7 +30,9 @@ def test_projectid(hsclient): hsclient.get_project('111/3') -def test_get_job_from_key(hsclient, hsproject, hsspiderid): +def test_get_job_from_key( + hsclient: HubstorageClient, hsproject: Project, hsspiderid: str, +) -> None: job = hsproject.push_job(TEST_SPIDER_NAME) parts = tuple(job.key.split('/')) assert len(parts) == 3 @@ -39,7 +45,7 @@ def test_get_job_from_key(hsclient, hsproject, hsspiderid): assert samejob3.key == job.key -def test_get_jobs(hsproject): +def test_get_jobs(hsproject: Project) -> None: p = hsproject j1 = p.push_job(TEST_SPIDER_NAME, testid=0) j2 = p.push_job(TEST_SPIDER_NAME, testid=1) @@ -51,7 +57,7 @@ def test_get_jobs(hsproject): assert [j.key for j in r] == [j3.key, j2.key, j1.key] -def test_get_jobs_with_legacy_filter(hsproject): +def test_get_jobs_with_legacy_filter(hsproject: Project) -> None: p = hsproject j1 = p.push_job(TEST_SPIDER_NAME, state='finished', close_reason='finished', tags=['t2']) @@ -73,7 +79,7 @@ def test_get_jobs_with_legacy_filter(hsproject): assert [j.key for j in jobs] == [j2.key], jobs -def test_push_job(hsproject): +def test_push_job(hsproject: Project) -> None: job = hsproject.push_job(TEST_SPIDER_NAME, state='running', priority=hsproject.jobq.PRIO_HIGH, foo=u'bar') @@ -86,7 +92,7 @@ def test_push_job(hsproject): assert job.metadata.get('foo') == u'bar' -def test_auth(hsclient, json_and_msgpack): +def test_auth(hsclient: HubstorageClient, json_and_msgpack: str) -> None: # client without global auth set hsc = HubstorageClient(endpoint=hsclient.endpoint, use_msgpack=hsclient.use_msgpack) @@ -131,7 +137,9 @@ def test_auth(hsclient, json_and_msgpack): assert samejob.key == job.key -def test_broad(hsproject, hsspiderid, json_and_msgpack): +def test_broad( + hsproject: Project, hsspiderid: str, json_and_msgpack: str, +) -> None: # populate project with at least one job job = hsproject.push_job(TEST_SPIDER_NAME) assert job.metadata.get('state') == 'pending' @@ -158,13 +166,13 @@ def test_broad(hsproject, hsspiderid, json_and_msgpack): @pytest.fixture -def unset_botgroup(hsproject): +def unset_botgroup(hsproject: Project) -> Iterator[None]: unset_testbotgroup(hsproject) yield set_testbotgroup(hsproject) -def test_settings(hsproject, unset_botgroup): +def test_settings(hsproject: Project, unset_botgroup: None) -> None: settings = dict(hsproject.settings) assert settings == {} # use some fixed timestamp to represent current time @@ -180,7 +188,7 @@ def test_settings(hsproject, unset_botgroup): } -def test_requests(hsproject): +def test_requests(hsproject: Project) -> None: # use some fixed timestamp to represent current time ts = 1476803148638 job = hsproject.push_job(TEST_SPIDER_NAME, state='running') @@ -216,7 +224,7 @@ def test_requests(hsproject): next(rr) -def test_samples(hsproject, json_and_msgpack): +def test_samples(hsproject: Project, json_and_msgpack: str) -> None: # use some fixed timestamp to represent current time ts = 1476803148638 # no samples stored @@ -247,7 +255,7 @@ def test_samples(hsproject, json_and_msgpack): assert r1 == r2 -def test_jobsummary(hsproject): +def test_jobsummary(hsproject: Project) -> None: js = hsproject.jobsummary() assert js.get('project') == int(hsproject.projectid), js assert js.get('has_capacity') is True, js @@ -255,7 +263,7 @@ def test_jobsummary(hsproject): assert 'running' in js, js -def test_bulkdata(hsproject, json_and_msgpack): +def test_bulkdata(hsproject: Project, json_and_msgpack: str) -> None: j = hsproject.push_job(TEST_SPIDER_NAME, state='running') for i in range(20): j.logs.info("log line %d" % i) @@ -271,7 +279,7 @@ def test_bulkdata(hsproject, json_and_msgpack): assert len(downloaded) == 20 -def test_output_string(hsclient, hsproject): +def test_output_string(hsclient: HubstorageClient, hsproject: Project) -> None: hsproject.push_job(TEST_SPIDER_NAME) job = start_job(hsproject) job.items.write({'foo': 'bar'}) @@ -291,7 +299,10 @@ def test_output_string(hsclient, hsproject): ('33/1/stats/', False), ((33, 1, 'stats'), False), ]) -def test_allows_msgpack(hsclient, path, expected_result, json_and_msgpack): +def test_allows_msgpack( + hsclient: HubstorageClient, path: Any, expected_result: bool, + json_and_msgpack: str, +) -> None: job = hsclient.get_job('2222000/1/1') for resource in [job.items, job.logs, job.samples]: assert resource._allows_mpack(path) is (hsclient.use_msgpack and expected_result) diff --git a/tests/hubstorage/test_retry.py b/tests/hubstorage/test_retry.py index 1db07d3d..d1a9c1a9 100644 --- a/tests/hubstorage/test_retry.py +++ b/tests/hubstorage/test_retry.py @@ -3,11 +3,13 @@ """ import json import re +from collections.abc import Callable +from typing import Any import pytest import responses -from mock import patch -from requests import HTTPError, ConnectionError +from mock import MagicMock, patch +from requests import HTTPError, ConnectionError, PreparedRequest from scrapinghub import HubstorageClient from six.moves.http_client import BadStatusLine @@ -19,11 +21,15 @@ POST = responses.POST DELETE = responses.DELETE +_Callback = Callable[[PreparedRequest], tuple[int, dict[str, str], str]] + @patch.multiple('scrapinghub.HubstorageClient', RETRY_DEFAULT_JITTER_MS=1, RETRY_DEFAULT_EXPONENTIAL_BACKOFF_MS=1) -def hsclient_with_retries(max_retries=3, max_retry_time=1): +def hsclient_with_retries( + max_retries: int = 3, max_retry_time: float | None = 1, +) -> HubstorageClient: return HubstorageClient( auth=TEST_AUTH, endpoint=TEST_ENDPOINT, max_retries=max_retries, max_retry_time=max_retry_time, @@ -31,7 +37,7 @@ def hsclient_with_retries(max_retries=3, max_retry_time=1): @patch('scrapinghub.hubstorage.client.Retrying') -def test_hsclient_with_retries_no_wait(retrying_class): +def test_hsclient_with_retries_no_wait(retrying_class: MagicMock) -> None: hsclient_with_retries() assert retrying_class.call_count == 1 call = retrying_class.call_args_list[0] @@ -39,7 +45,8 @@ def test_hsclient_with_retries_no_wait(retrying_class): assert int(call[1]['wait_exponential_multiplier']) == 33 -def mock_api(method=GET, callback=None, url_match='/.*'): +def mock_api(method: str = GET, *, callback: _Callback, + url_match: str = '/.*') -> None: """ Mock an API URL using the responses library. @@ -55,8 +62,10 @@ def mock_api(method=GET, callback=None, url_match='/.*'): ) -def make_request_callback(timeout_count, body_on_success, - http_error_status=504): +def make_request_callback( + timeout_count: int, body_on_success: Any, + http_error_status: int = 504, +) -> tuple[_Callback, list[int]]: """Make a request callback that timeout a couple of time before returning body_on_success. @@ -67,7 +76,9 @@ def make_request_callback(timeout_count, body_on_success, # use a list for nonlocal mutability used in request_callback attempts = [0] - def request_callback(request): + def request_callback( + request: PreparedRequest, + ) -> tuple[int, dict[str, str], str]: attempts[0] += 1 if attempts[0] <= timeout_count: @@ -79,7 +90,7 @@ def request_callback(request): return request_callback, attempts -def test_delete_on_hubstorage_api_does_not_404(): +def test_delete_on_hubstorage_api_does_not_404() -> None: """Delete a non-existing resource without exceptions. The current Hubstorage API does not raise 404 errors on deleting resources @@ -107,7 +118,7 @@ def test_delete_on_hubstorage_api_does_not_404(): @responses.activate -def test_retrier_does_not_catch_unwanted_exception(hsspiderid): +def test_retrier_does_not_catch_unwanted_exception(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries(max_retries=2) job_metadata = { @@ -136,7 +147,9 @@ def test_retrier_does_not_catch_unwanted_exception(hsspiderid): @pytest.mark.parametrize('err_code', [408, 429, 502, 503, 504]) @responses.activate -def test_retrier_catches_badstatusline_and_selected_http_errors(hsspiderid, err_code): +def test_retrier_catches_badstatusline_and_selected_http_errors( + hsspiderid: str, err_code: int, +) -> None: # Prepare client = hsclient_with_retries() job_metadata = { @@ -148,7 +161,9 @@ def test_retrier_catches_badstatusline_and_selected_http_errors(hsspiderid, err_ # use a list for nonlocal mutability used in request_callback attempts_count = [0] - def request_callback(request): + def request_callback( + request: PreparedRequest, + ) -> tuple[int, dict[str, str], str]: attempts_count[0] += 1 if attempts_count[0] <= 2: @@ -170,7 +185,7 @@ def request_callback(request): @responses.activate -def test_api_delete_can_be_set_to_non_idempotent(hsspiderid): +def test_api_delete_can_be_set_to_non_idempotent(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries() job_metadata = { @@ -199,7 +214,7 @@ def test_api_delete_can_be_set_to_non_idempotent(hsspiderid): @responses.activate -def test_collection_store_and_delete_are_retried(): +def test_collection_store_and_delete_are_retried() -> None: # Prepare client = hsclient_with_retries() callback_post, attempts_count_post = make_request_callback(2, []) @@ -219,7 +234,7 @@ def test_collection_store_and_delete_are_retried(): @responses.activate -def test_delete_requests_are_retried(hsspiderid): +def test_delete_requests_are_retried(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries() job_metadata = { @@ -247,7 +262,7 @@ def test_delete_requests_are_retried(hsspiderid): @responses.activate -def test_metadata_save_does_retry(hsspiderid): +def test_metadata_save_does_retry(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries() job_metadata = { @@ -271,7 +286,7 @@ def test_metadata_save_does_retry(hsspiderid): @responses.activate -def test_push_job_does_not_retry(): +def test_push_job_does_not_retry() -> None: # Prepare client = hsclient_with_retries() callback, attempts_count = make_request_callback(2, {'key': '1/2/3'}) @@ -293,7 +308,7 @@ def test_push_job_does_not_retry(): @responses.activate -def test_get_job_does_retry(hsspiderid): +def test_get_job_does_retry(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries() job_metadata = { @@ -314,7 +329,7 @@ def test_get_job_does_retry(hsspiderid): @responses.activate -def test_get_job_does_fails_if_no_retries(hsspiderid): +def test_get_job_does_fails_if_no_retries(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries(max_retries=0, max_retry_time=None) job_metadata = { @@ -342,7 +357,7 @@ def test_get_job_does_fails_if_no_retries(hsspiderid): @responses.activate -def test_get_job_does_fails_on_too_many_retries(hsspiderid): +def test_get_job_does_fails_on_too_many_retries(hsspiderid: str) -> None: # Prepare client = hsclient_with_retries(max_retries=2, max_retry_time=1) job_metadata = { diff --git a/tests/hubstorage/test_serialization.py b/tests/hubstorage/test_serialization.py index a4f434d8..84014d12 100644 --- a/tests/hubstorage/test_serialization.py +++ b/tests/hubstorage/test_serialization.py @@ -5,17 +5,17 @@ from scrapinghub.hubstorage.serialization import jsondefault -def test_jsondefault_timezones(): +def test_jsondefault_timezones() -> None: class TestTZ(tzinfo): - def utcoffset(self, dt): + def utcoffset(self, dt: datetime | None) -> timedelta: return timedelta(minutes=-399) # base tz unaware datetime dt = datetime(2017, 5, 4, 3, 2, 1, 123456) dt_ts = jsondefault(dt) # tz aware datetime with test TZ - dt_tz = dt.replace(tzinfo=TestTZ()) + dt_tz = dt.replace(tzinfo=TestTZ()) # type: ignore[abstract] dt_tz_ts = jsondefault(dt_tz) - assert dt_tz_ts == dt_ts + 399 * 60 * 1000 + assert dt_tz_ts == dt_ts + 399 * 60 * 1000 # type: ignore[operator] diff --git a/tests/hubstorage/test_system.py b/tests/hubstorage/test_system.py index bdbc2152..e5e04f63 100644 --- a/tests/hubstorage/test_system.py +++ b/tests/hubstorage/test_system.py @@ -1,11 +1,15 @@ import random from contextlib import closing +from collections.abc import Iterator +from typing import Any import pytest from six.moves import range from scrapinghub import HubstorageClient from scrapinghub.hubstorage.utils import millitime +from scrapinghub.hubstorage.job import Job +from scrapinghub.hubstorage.project import Project from ..conftest import TEST_ENDPOINT, TEST_SPIDER_NAME from ..conftest import TEST_PROJECT_ID, TEST_AUTH @@ -16,31 +20,35 @@ @pytest.fixture -def panelclient(): +def panelclient() -> HubstorageClient: # Panel - no client auth, only project auth using user auth token return HubstorageClient(endpoint=TEST_ENDPOINT) @pytest.fixture -def panelproject(panelclient): +def panelproject(panelclient: HubstorageClient) -> Project: return panelclient.get_project(TEST_PROJECT_ID, auth=TEST_AUTH) @pytest.fixture(autouse=True) -def close_panelclient(panelclient): +def close_panelclient(panelclient: HubstorageClient) -> Iterator[None]: yield panelclient.close() -def test_succeed_with_close_reason(hsproject, panelproject): +def test_succeed_with_close_reason( + hsproject: Project, panelproject: Project, +) -> None: _do_test_success(hsproject, panelproject, 'all-good', 'all-good') -def test_succeed_without_close_reason(hsproject, panelproject): +def test_succeed_without_close_reason( + hsproject: Project, panelproject: Project, +) -> None: _do_test_success(hsproject, panelproject, None, 'no_reason') -def _do_test_success(*args): +def _do_test_success(*args: Any) -> None: """Simple wrapper around _do_test_job with additonal checks""" job = _do_test_job(*args) assert job.items.stats()['totals']['input_values'] == MAGICN @@ -48,7 +56,7 @@ def _do_test_success(*args): assert job.requests.stats()['totals']['input_values'] == MAGICN -def test_scraper_failure(hsproject, panelproject): +def test_scraper_failure(hsproject: Project, panelproject: Project) -> None: job = _do_test_job( hsproject, panelproject, @@ -61,8 +69,10 @@ def test_scraper_failure(hsproject, panelproject): assert stats['totals']['input_values'] == MAGICN * 4 + 1 -def _do_test_job(hsproject, panelproject, - job_close_reason, expected_close_reason): +def _do_test_job( + hsproject: Project, panelproject: Project, + job_close_reason: str | Exception | None, expected_close_reason: str, +) -> Job: pushed = panelproject.jobq.push(TEST_SPIDER_NAME) # check pending state job = panelproject.get_job(pushed['key']) @@ -76,7 +86,10 @@ def _do_test_job(hsproject, panelproject, return job -def _run_runner(hsproject, pushed, close_reason): +def _run_runner( + hsproject: Project, pushed: dict[str, Any], + close_reason: str | Exception | None, +) -> None: client = HubstorageClient(endpoint=TEST_ENDPOINT, auth=TEST_AUTH) with closing(client) as runnerclient: job = start_job(hsproject) @@ -97,7 +110,10 @@ def _run_runner(hsproject, pushed, close_reason): job.jobq.finish(job, close_reason=close_reason or 'no_reason') -def _run_scraper(jobkey, jobauth, close_reason=None): +def _run_scraper( + jobkey: str, jobauth: str | tuple[str, str] | None, + close_reason: str | Exception | None = None, +) -> None: httpmethods = 'GET PUT POST DELETE HEAD OPTIONS TRACE CONNECT'.split() # Scraper - uses job level auth, no global or project auth available client = HubstorageClient(endpoint=TEST_ENDPOINT) diff --git a/tests/hubstorage/test_utils.py b/tests/hubstorage/test_utils.py index 3101cc8f..79ce640e 100644 --- a/tests/hubstorage/test_utils.py +++ b/tests/hubstorage/test_utils.py @@ -5,7 +5,7 @@ from scrapinghub.hubstorage.utils import sizeof_fmt -def test_sizeof_fmt(): +def test_sizeof_fmt() -> None: assert sizeof_fmt(1000) == '1000 B' assert sizeof_fmt(1024) == '1 KiB' assert sizeof_fmt(1024 * 1024) == '1 MiB' diff --git a/tests/hubstorage/testutil.py b/tests/hubstorage/testutil.py index fe0e37d2..358817d8 100644 --- a/tests/hubstorage/testutil.py +++ b/tests/hubstorage/testutil.py @@ -1,9 +1,14 @@ from contextlib import contextmanager +from collections.abc import Callable, Iterable, Iterator +from typing import Any from requests import Timeout @contextmanager -def failing_downloader(downloader, N=5, exc=Timeout, msg='test error'): +def failing_downloader( + downloader: Any, N: int = 5, exc: type[BaseException] = Timeout, + msg: str = 'test error', +) -> Iterator[Any]: # reduce wait times and simulate network timeout orig_iter = downloader._iter_lines orig_wait = downloader.RETRY_INTERVAL @@ -18,15 +23,20 @@ def failing_downloader(downloader, N=5, exc=Timeout, msg='test error'): downloader.RETRY_INTERVAL = orig_wait -def wrap_seq_to_fail(func, exception, N, *exc_args): +def wrap_seq_to_fail( + func: Callable[..., Iterable[Any]], exception: type[BaseException], N: int, + *exc_args: Any, +) -> Callable[..., Iterator[Any]]: """Wrap a function that returns a sequence with failafter""" - def _wrapper(*args, **kwargs): + def _wrapper(*args: Any, **kwargs: Any) -> Iterator[Any]: seq = func(*args, **kwargs) return failafter(seq, exception, N, *exc_args) return _wrapper -def failafter(seq, exception, N, *exc_args): +def failafter( + seq: Iterable[Any], exception: type[BaseException], N: int, *exc_args: Any, +) -> Iterator[Any]: """iterate over seq, and raise exception after N items""" for i, item in enumerate(seq): if i == N: diff --git a/tests/legacy/conftest.py b/tests/legacy/conftest.py index c207066f..dbd5dbf6 100644 --- a/tests/legacy/conftest.py +++ b/tests/legacy/conftest.py @@ -7,25 +7,25 @@ @pytest.fixture -def connection(): +def connection() -> Connection: return Connection(apikey='testkey', url='http://test-url') @pytest.fixture -def proxy_mixin(): +def proxy_mixin() -> RequestProxyMixin: return RequestProxyMixin() @pytest.fixture -def project(connection): +def project(connection: Connection) -> Project: return Project(connection, 12345) @pytest.fixture -def jobset(project): +def jobset(project: Project) -> JobSet: return JobSet(project, param='value') @pytest.fixture -def job(project): +def job(project: Project) -> Job: return Job(project, '1/2/3', {'field': 'data'}) diff --git a/tests/legacy/test_connection.py b/tests/legacy/test_connection.py index b5ec68fe..35e567b0 100644 --- a/tests/legacy/test_connection.py +++ b/tests/legacy/test_connection.py @@ -10,85 +10,87 @@ from scrapinghub import __version__ -def test_connection_class_attrs(): +def test_connection_class_attrs() -> None: assert Connection.DEFAULT_ENDPOINT == 'https://app.zyte.com/api/' assert isinstance(Connection.API_METHODS, dict) -def test_connection_init_fail_wo_apikey(monkeypatch): +def test_connection_init_fail_wo_apikey( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv('SH_APIKEY', raising=False) with pytest.raises(RuntimeError): Connection() @mock.patch.dict(os.environ, {'SH_APIKEY': 'testkey'}) -def test_connection_init_use_key_from_env(): +def test_connection_init_use_key_from_env() -> None: conn = Connection() assert conn.apikey == 'testkey' -def test_connection_init_assert_apikey_not_url(): +def test_connection_init_assert_apikey_not_url() -> None: with pytest.raises(AssertionError): Connection(password='testpass', apikey='http://some-url') -def test_connection_init_with_default_url(): +def test_connection_init_with_default_url() -> None: conn = Connection(apikey='testkey') assert conn.url == Connection.DEFAULT_ENDPOINT -def test_connection_init_with_default_timeout(): +def test_connection_init_with_default_timeout() -> None: conn = Connection(apikey='testkey') assert conn._connection_timeout is None -def test_connection_init_with_custom_timeout(): +def test_connection_init_with_custom_timeout() -> None: conn = Connection(apikey='testkey', connection_timeout=60) assert conn._connection_timeout == 60 -def test_connection_init_ok(connection): +def test_connection_init_ok(connection: Connection) -> None: assert connection.apikey == 'testkey' assert connection.url == 'http://test-url' assert connection._session -def test_connection_repr(connection): +def test_connection_repr(connection: Connection) -> None: assert repr(connection) == "Connection('testkey')" -def test_connection_auth(connection): +def test_connection_auth(connection: Connection) -> None: assert connection.auth == ('testkey', '') -def test_connection_create_session(connection): +def test_connection_create_session(connection: Connection) -> None: session = connection._session assert isinstance(session, requests.Session) assert session.auth == ('testkey', '') assert (session.headers.get('User-Agent') == 'python-scrapinghub/{}'.format(__version__)) assert session.stream - assert not session.prefetch + assert not session.prefetch # type: ignore[attr-defined] -def test_connection_build_url_unknown_method(connection): +def test_connection_build_url_unknown_method(connection: Connection) -> None: with pytest.raises(APIError): connection._build_url('unknown_method', 'json') -def test_connection_build_url_ok(connection): +def test_connection_build_url_ok(connection: Connection) -> None: assert (connection._build_url('addversion', 'json') == 'http://test-url/scrapyd/addversion.json') -def test_connection_request_wrong_format(connection): +def test_connection_request_wrong_format(connection: Connection) -> None: with pytest.raises(APIError): connection._request('http://some-url', 'data', {}, - 'wrongformat', None, None) + 'wrongformat', None, None) # type: ignore[arg-type] -def test_connection_get_wo_params(connection): - connection._request = mock.Mock() +def test_connection_get_wo_params(connection: Connection) -> None: + connection._request = mock.Mock() # type: ignore[method-assign] connection._request.return_value = 'expected' assert connection._get('addversion', 'json', headers={'Header': 'value'}, raw=True) == 'expected' @@ -98,8 +100,8 @@ def test_connection_get_wo_params(connection): {'Header': 'value'}, 'json', True), {})] -def test_connection_get_with_params(connection): - connection._request = mock.Mock() +def test_connection_get_with_params(connection: Connection) -> None: + connection._request = mock.Mock() # type: ignore[method-assign] connection._request.return_value = 'expected' assert connection._get( 'addversion', 'json', headers={'Header': 'value'}, @@ -110,8 +112,8 @@ def test_connection_get_with_params(connection): {'Header': 'value'}, 'json', True), {})] -def test_connection_post(connection): - connection._request = mock.Mock() +def test_connection_post(connection: Connection) -> None: + connection._request = mock.Mock() # type: ignore[method-assign] connection._request.return_value = 'expected' assert connection._post( 'addversion', 'json', headers={'Header': 'value'}, @@ -124,12 +126,14 @@ def test_connection_post(connection): @pytest.mark.parametrize("timeout", [None, 0.1]) -def test_connection_request_handle_get(connection, timeout): +def test_connection_request_handle_get( + connection: Connection, timeout: float | None, +) -> None: if timeout: connection._connection_timeout = timeout connection._session = mock.Mock() connection._session.get.return_value = 'get_response' - connection._decode_response = mock.Mock() + connection._decode_response = mock.Mock() # type: ignore[method-assign] connection._decode_response.return_value = 'expected' assert connection._request( 'http://some-url', None, {'HeaderA': 'value'}, @@ -144,12 +148,14 @@ def test_connection_request_handle_get(connection, timeout): @pytest.mark.parametrize("timeout", [None, 0.1]) -def test_connection_request_handle_post(connection, timeout): +def test_connection_request_handle_post( + connection: Connection, timeout: float | None, +) -> None: if timeout: connection._connection_timeout = timeout connection._session = mock.Mock() connection._session.post.return_value = 'post_response' - connection._decode_response = mock.Mock() + connection._decode_response = mock.Mock() # type: ignore[method-assign] connection._decode_response.return_value = 'expected' assert connection._request( 'http://some-url', 'data', {'HeaderA': 'value'}, @@ -165,7 +171,7 @@ def test_connection_request_handle_post(connection, timeout): (('post_response', 'json', True),)] -def test_connection_decode_response_raw(connection): +def test_connection_decode_response_raw(connection: Connection) -> None: response = mock.Mock() response.status_code = 200 response.raw = 'expected' @@ -173,7 +179,7 @@ def test_connection_decode_response_raw(connection): response, 'json', raw=True) == 'expected' -def test_connection_decode_response_json_ok(connection): +def test_connection_decode_response_json_ok(connection: Connection) -> None: response = mock.Mock() response.status_code = 200 response.text = '{"status":"ok","data":"some-data"}' @@ -181,7 +187,7 @@ def test_connection_decode_response_json_ok(connection): response, 'json', raw=False) == {"status": "ok", "data": "some-data"} -def test_connection_decode_response_json_error(connection): +def test_connection_decode_response_json_error(connection: Connection) -> None: response = mock.Mock() response.status_code = 400 for status in ['error', 'badrequest']: @@ -191,7 +197,9 @@ def test_connection_decode_response_json_error(connection): assert exc.value.args == ("error",) -def test_connection_decode_response_json_unknown(connection): +def test_connection_decode_response_json_unknown( + connection: Connection, +) -> None: response = mock.Mock() response.status_code = 400 response.text = json.dumps({"status": "unexpected", "message": "error"}) @@ -200,7 +208,7 @@ def test_connection_decode_response_json_unknown(connection): assert exc.value.args == ("Unknown response status: unexpected",) -def test_connection_decode_response_jl(connection): +def test_connection_decode_response_jl(connection: Connection) -> None: jl_data = [{'row1': 'data1'}, {'row2': 'data2'}] response = mock.Mock() response.status_code = 200 @@ -209,21 +217,21 @@ def test_connection_decode_response_jl(connection): response, 'jl', raw=False)) == jl_data -def test_connection_getitem(connection): +def test_connection_getitem(connection: Connection) -> None: project = connection['key'] assert isinstance(project, Project) assert project.id == 'key' -def test_connection_project_ids(connection): - connection._get = mock.Mock() +def test_connection_project_ids(connection: Connection) -> None: + connection._get = mock.Mock() # type: ignore[method-assign] connection._get.return_value = {'projects': [1, 2, 3]} assert connection.project_ids() == [1, 2, 3] assert connection._get.called assert connection._get.call_args_list == [(('listprojects', 'json'), {})] -def test_connection_project_names(connection): - connection._get = mock.Mock() +def test_connection_project_names(connection: Connection) -> None: + connection._get = mock.Mock() # type: ignore[method-assign] connection._get.return_value = {'projects': [1, 2, 3]} assert connection.project_names() == [1, 2, 3] diff --git a/tests/legacy/test_job.py b/tests/legacy/test_job.py index 9472683f..aadba97c 100644 --- a/tests/legacy/test_job.py +++ b/tests/legacy/test_job.py @@ -1,32 +1,35 @@ + +from collections.abc import Iterator +from io import StringIO + import mock import pytest -import requests from scrapinghub import Job from scrapinghub import Project -def test_job_attributes(): +def test_job_attributes() -> None: assert Job.MAX_RETRIES == 180 assert Job.RETRY_INTERVAL == 60 -def test_job_init(job): +def test_job_init(job: Job) -> None: assert isinstance(job.project, Project) assert job._id == '1/2/3' assert job.info == {'field': 'data'} -def test_job_id(job): +def test_job_id(job: Job) -> None: assert job.id == job._id == '1/2/3' -def test_job_repr(job): +def test_job_repr(job: Job) -> None: assert repr(job) == "Job(Project(Connection('testkey'), 12345), 1/2/3)" -def test_job_items_max_retries(job): - job._get = mock.Mock() +def test_job_items_max_retries(job: Job) -> None: + job._get = mock.Mock() # type: ignore[method-assign] job._get.side_effect = ValueError job.MAX_RETRIES = 10 job.RETRY_INTERVAL = 0 @@ -35,18 +38,18 @@ def test_job_items_max_retries(job): assert job._get.call_count == 10 -def test_job_items_base_void(job): - job._get = mock.Mock() +def test_job_items_base_void(job: Job) -> None: + job._get = mock.Mock() # type: ignore[method-assign] job._get.return_value = iter([]) assert list(job.items()) == [] assert job._get.call_args_list == [ (('items', 'jl'), {'params': {'offset': 0}})] -def test_job_items_base_with_items(job): - job._get = mock.Mock() +def test_job_items_base_with_items(job: Job) -> None: + job._get = mock.Mock() # type: ignore[method-assign] job._get.return_value = ['itemA', 'itemB'] - assert list(job.items(offset=50, count=10, meta={'meta': 'data'})) == [ + assert list(job.items(offset=50, count=10, meta={'meta': 'data'})) == [ # type: ignore[arg-type] 'itemA', 'itemB'] assert job._get.call_count == 1 assert job._get.call_args_list == [ @@ -54,20 +57,20 @@ def test_job_items_base_with_items(job): 'offset': 50, 'count': 10, 'meta': {'meta': 'data'}}})] -def test_job_items_base_with_retry(job): - job._get = mock.Mock() +def test_job_items_base_with_retry(job: Job) -> None: + job._get = mock.Mock() # type: ignore[method-assign] job.RETRY_INTERVAL = 0 - def fake_first_get(): + def fake_first_get() -> Iterator[str]: yield 'itemA1' yield 'itemA2' raise ValueError() - def fake_second_get(): + def fake_second_get() -> Iterator[str]: yield 'itemB1' job._get.side_effect = [fake_first_get(), fake_second_get()] - items = job.items(offset=50, count=10, meta={'meta': 'data'}) + items = job.items(offset=50, count=10, meta={'meta': 'data'}) # type: ignore[arg-type] assert next(items) == 'itemA1' assert job._get.call_count == 1 assert job._get.mock_calls == [ @@ -85,8 +88,8 @@ def fake_second_get(): next(items) -def test_job_update(job): - job._post = mock.Mock() +def test_job_update(job: Job) -> None: + job._post = mock.Mock() # type: ignore[method-assign] job._post.return_value = {'count': 100} assert job.update(field='newvalue', paramB='valueB') == 100 assert job._post.call_args_list == [ @@ -94,29 +97,29 @@ def test_job_update(job): {"field": "newvalue", "paramB": "valueB"}), {})] -def test_job_stop_ok(job): - job._post = mock.Mock() +def test_job_stop_ok(job: Job) -> None: + job._post = mock.Mock() # type: ignore[method-assign] job._post.return_value = {'status': 'ok'} assert job.stop() assert job._post.call_args_list == [(('jobs_stop', 'json'), {})] -def test_job_stop_error(job): - job._post = mock.Mock() +def test_job_stop_error(job: Job) -> None: + job._post = mock.Mock() # type: ignore[method-assign] job._post.return_value = {'status': 'error'} assert not job.stop() assert job._post.call_args_list == [(('jobs_stop', 'json'), {})] -def test_job_delete(job): - job._post = mock.Mock() +def test_job_delete(job: Job) -> None: + job._post = mock.Mock() # type: ignore[method-assign] job._post.return_value = {'count': 1} assert job.delete() == 1 assert job._post.call_args_list == [(('jobs_delete', 'json'), {})] -def test_job_add_report(job): - job._post = mock.Mock() +def test_job_add_report(job: Job) -> None: + job._post = mock.Mock() # type: ignore[method-assign] job.add_report('testkey', 'testcontent', content_type='custom/type') args = job._post.call_args_list[0] assert args[0] == ('reports_add', 'json', { @@ -125,21 +128,21 @@ def test_job_add_report(job): content = args[1].get('files').get('content') assert len(content) == 2 assert content[0] == 'report' - assert isinstance(content[1], requests.compat.StringIO) + assert isinstance(content[1], StringIO) assert content[1].read() == 'testcontent' -def test_job_log(job): - job._get = mock.Mock() +def test_job_log(job: Job) -> None: + job._get = mock.Mock() # type: ignore[method-assign] job._get.return_value = ['jl logs'] assert job.log(param='value') == ['jl logs'] assert job._get.call_args_list == [(('log', 'jl', {'param': 'value'}), {})] -def test_job_request_proxy(job): +def test_job_request_proxy(job: Job) -> None: assert job._request_proxy == job.project -def test_job_add_params(job): +def test_job_add_params(job: Job) -> None: assert job._add_params({'param': 'value'}) == { 'param': 'value', 'job': '1/2/3'} diff --git a/tests/legacy/test_jobset.py b/tests/legacy/test_jobset.py index c96da1e7..3d5cbae1 100644 --- a/tests/legacy/test_jobset.py +++ b/tests/legacy/test_jobset.py @@ -4,39 +4,41 @@ from scrapinghub import APIError from scrapinghub import Job from scrapinghub import Project +from scrapinghub.legacy import JobSet -def test_jobset_init(jobset): +def test_jobset_init(jobset: JobSet) -> None: assert isinstance(jobset.project, Project) assert jobset.params == {'param': 'value'} assert jobset._jobs is None -def test_jobset_repr(jobset): +def test_jobset_repr(jobset: JobSet) -> None: s = repr(jobset) assert s == "JobSet(Project(Connection('testkey'), 12345), param=value)" -def test_jobset_iter(jobset): - jobset._jobs = [{'id': '1/2/3', 'job_data': 'info'}] +def test_jobset_iter(jobset: JobSet) -> None: + jobs_data = [{'id': '1/2/3', 'job_data': 'info'}] + jobset._jobs = jobs_data jobs = list(iter(jobset)) assert jobs assert len(jobs) == 1 assert isinstance(jobs[0], Job) assert jobs[0].project == jobset.project assert jobs[0]._id == '1/2/3' - assert jobs[0].info == jobset._jobs[0] + assert jobs[0].info == jobs_data[0] -def test_jobset_count(jobset): - jobset._get = mock.Mock() +def test_jobset_count(jobset: JobSet) -> None: + jobset._get = mock.Mock() # type: ignore[method-assign] jobset._get.return_value = {'total': 100} assert jobset.count() == 100 assert jobset._get.call_args_list == [(('jobs_count', 'json'), {})] -def test_jobset_update(jobset): - jobset._post = mock.Mock() +def test_jobset_update(jobset: JobSet) -> None: + jobset._post = mock.Mock() # type: ignore[method-assign] jobset._post.return_value = {'count': 100} assert jobset.update(param='newvalue', paramB='valueB') == 100 assert jobset._post.call_args_list == [ @@ -45,59 +47,59 @@ def test_jobset_update(jobset): @mock.patch('scrapinghub.legacy.Job') -def test_jobset_stop(job_mock, jobset): +def test_jobset_stop(job_mock: mock.MagicMock, jobset: JobSet) -> None: jobset._jobs = [{'id': '1/2/3'}] jobset.stop() assert job_mock.return_value.stop.called @mock.patch('scrapinghub.legacy.Job') -def test_jobset_delete(job_mock, jobset): +def test_jobset_delete(job_mock: mock.MagicMock, jobset: JobSet) -> None: jobset._jobs = [{'id': '1/2/3'}] jobset.delete() assert job_mock.return_value.delete.called -def test_jobset_load_jobs_already_loaded(jobset): +def test_jobset_load_jobs_already_loaded(jobset: JobSet) -> None: jobset._jobs = [{'id': '1/2/3'}] jobset._load_jobs() -def test_jobset_load_jobs_stop_iteration(jobset): - jobset._get = mock.Mock() +def test_jobset_load_jobs_stop_iteration(jobset: JobSet) -> None: + jobset._get = mock.Mock() # type: ignore[method-assign] jobset._get.return_value = iter([]) with pytest.raises(APIError) as exc: jobset._load_jobs() assert 'does not contain status' in exc.value.args[0] -def test_jobset_load_jobs_bad_status(jobset): +def test_jobset_load_jobs_bad_status(jobset: JobSet) -> None: job = {'status': 'bad-status'} - jobset._get = mock.Mock() + jobset._get = mock.Mock() # type: ignore[method-assign] jobset._get.return_value = iter([job]) with pytest.raises(APIError) as exc: jobset._load_jobs() assert 'Unknown response status' in exc.value.args[0] -def test_jobset_load_jobs_base(jobset): +def test_jobset_load_jobs_base(jobset: JobSet) -> None: jobs_iter = iter([{'status': 'ok', 'id': '1/2/3'}, {'status': 'ok', 'id': '1/2/4'}]) - jobset._get = mock.Mock() + jobset._get = mock.Mock() # type: ignore[method-assign] jobset._get.return_value = jobs_iter jobset._load_jobs() assert jobset._jobs == jobs_iter -def test_jobset_request_proxy(jobset): +def test_jobset_request_proxy(jobset: JobSet) -> None: assert jobset._request_proxy == jobset.project -def test_jobset_add_params_void(jobset): +def test_jobset_add_params_void(jobset: JobSet) -> None: assert jobset._add_params({}) == {'param': 'value'} -def test_jobset_add_params_base(jobset): +def test_jobset_add_params_base(jobset: JobSet) -> None: params = {'paramX': 'valueX', 'param': 'newvalue'} assert jobset._add_params(params) == { 'param': 'newvalue', 'paramX': 'valueX'} diff --git a/tests/legacy/test_project.py b/tests/legacy/test_project.py index 985b5048..095405fd 100644 --- a/tests/legacy/test_project.py +++ b/tests/legacy/test_project.py @@ -2,61 +2,62 @@ from scrapinghub import Connection from scrapinghub import Job, JobSet +from scrapinghub.legacy import Project -def test_project_init(project): +def test_project_init(project: Project) -> None: assert isinstance(project.connection, Connection) assert project.id == 12345 -def test_project_repr(project): +def test_project_repr(project: Project) -> None: assert repr(project) == "Project(Connection('testkey'), 12345)" -def test_project_name(project): +def test_project_name(project: Project) -> None: assert project.name == project.id -def test_project_schedule(project): - project._post = mock.Mock() +def test_project_schedule(project: Project) -> None: + project._post = mock.Mock() # type: ignore[method-assign] project._post.return_value = {'jobid': '1/2/3'} assert project.schedule('testspider', param='value') == '1/2/3' assert project._post.call_args_list == [ (('schedule', 'json', {'spider': 'testspider', 'param': 'value'}), {})] -def test_project_jobs(project): +def test_project_jobs(project: Project) -> None: jobset = project.jobs(job=123, count=1) assert isinstance(jobset, JobSet) -def test_project_job(project): +def test_project_job(project: Project) -> None: test_job = Job(project, '1/2/3', {}) - project.jobs = mock.Mock() + project.jobs = mock.Mock() # type: ignore[method-assign] project.jobs.return_value = iter([test_job]) assert project.job('1/2/3') == test_job assert project.jobs.call_args_list == [ ((), {'job': '1/2/3', 'count': 1})] -def test_project_spiders(project): - project._get = mock.Mock() +def test_project_spiders(project: Project) -> None: + project._get = mock.Mock() # type: ignore[method-assign] project._get.return_value = {'spiders': ['spiderA']} assert project.spiders(param='value') == ['spiderA'] assert project._get.call_args_list == [ (('spiders', 'json', {'param': 'value'}), {})] -def test_project_request_proxy(project): +def test_project_request_proxy(project: Project) -> None: assert project._request_proxy == project.connection -def test_project_add_params(project): +def test_project_add_params(project: Project) -> None: assert project._add_params({}) == {'project': project.id} -def test_project_as_project_slybot_wo_output_copy(project): - project._get = mock.Mock() +def test_project_as_project_slybot_wo_output_copy(project: Project) -> None: + project._get = mock.Mock() # type: ignore[method-assign] project._get.return_value = 'project-data' assert project.autoscraping_project_slybot( ('testspider',)) == 'project-data' @@ -65,8 +66,10 @@ def test_project_as_project_slybot_wo_output_copy(project): {'raw': True})] -def test_project_as_project_spider_props_with_start_urls(project): - project._post = mock.Mock() +def test_project_as_project_spider_props_with_start_urls( + project: Project, +) -> None: + project._post = mock.Mock() # type: ignore[method-assign] project._post.return_value = {'property': 'value'} assert project.autoscraping_spider_properties( 'testspider', start_urls=['start-url']) == {'property': 'value'} @@ -75,8 +78,10 @@ def test_project_as_project_spider_props_with_start_urls(project): {'spider': 'testspider', 'start_url': ['start-url']}), {})] -def test_project_as_project_spider_props_wo_start_urls(project): - project._get = mock.Mock() +def test_project_as_project_spider_props_wo_start_urls( + project: Project, +) -> None: + project._get = mock.Mock() # type: ignore[method-assign] project._get.return_value = {'property': 'value'} assert project.autoscraping_spider_properties( 'testspider') == {'property': 'value'} diff --git a/tests/legacy/test_proxymixin.py b/tests/legacy/test_proxymixin.py index 9f126027..e64b65b2 100644 --- a/tests/legacy/test_proxymixin.py +++ b/tests/legacy/test_proxymixin.py @@ -1,26 +1,31 @@ +from typing import Any + import mock +from scrapinghub.legacy import RequestProxyMixin + -def test_requestproxymixin_add_params(proxy_mixin): +def test_requestproxymixin_add_params(proxy_mixin: RequestProxyMixin) -> None: params = {'param': 'val'} assert proxy_mixin._add_params(params) == params -def test_requestproxymixin_get(proxy_mixin): - proxy_mixin._request_proxy = mock.Mock() +def test_requestproxymixin_get(proxy_mixin: RequestProxyMixin) -> None: + proxy_mixin._request_proxy = mock.Mock() # type: ignore[misc] proxy_mixin._request_proxy._get.return_value = 'expected' args = ('method', 'json') - kwargs = {'params': {}, 'headers': {}, 'raw': True} + kwargs: dict[str, Any] = {'params': {}, 'headers': {}, 'raw': True} assert proxy_mixin._get(*args, **kwargs) == 'expected' assert proxy_mixin._request_proxy._get.call_args_list == [ (('method', 'json', {}, {}, True), {})] -def test_requestproxymixin_post(proxy_mixin): - proxy_mixin._request_proxy = mock.Mock() +def test_requestproxymixin_post(proxy_mixin: RequestProxyMixin) -> None: + proxy_mixin._request_proxy = mock.Mock() # type: ignore[misc] proxy_mixin._request_proxy._post.return_value = 'expected' args = ('method', 'json') - kwargs = {'params': {}, 'headers': {}, 'raw': True, 'files': {}} + kwargs: dict[str, Any] = {'params': {}, 'headers': {}, 'raw': True, + 'files': {}} assert proxy_mixin._post(*args, **kwargs) == 'expected' assert proxy_mixin._request_proxy._post.call_args_list == [ (('method', 'json', {}, {}, True, {}), {})] diff --git a/tests/test_jobq_client.py b/tests/test_jobq_client.py index f5c5290b..b60a2e01 100644 --- a/tests/test_jobq_client.py +++ b/tests/test_jobq_client.py @@ -1,19 +1,27 @@ +import pytest + from scrapinghub import HubstorageClient -def test_hubstorage_jobq_defaults_to_storage_endpoint(monkeypatch): +def test_hubstorage_jobq_defaults_to_storage_endpoint( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv('SHUB_JOBQ', raising=False) client = HubstorageClient(auth='apikey', endpoint='https://storage.example/') assert client.jobq.url.startswith('https://storage.example/') -def test_hubstorage_jobq_endpoint_uses_env_var(monkeypatch): +def test_hubstorage_jobq_endpoint_uses_env_var( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.setenv('SHUB_JOBQ', 'https://jobq-internal.zyte.com/') client = HubstorageClient(auth='apikey', endpoint='https://storage.example/') assert client.jobq.url.startswith('https://jobq-internal.zyte.com/') -def test_hubstorage_jobq_endpoint_argument(monkeypatch): +def test_hubstorage_jobq_endpoint_argument( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv('SHUB_JOBQ', raising=False) client = HubstorageClient( auth='apikey', @@ -23,14 +31,16 @@ def test_hubstorage_jobq_endpoint_argument(monkeypatch): assert client.jobq.url.startswith('https://jobq.example/') -def test_hubstorage_connection_timeout_positional_compatibility(monkeypatch): +def test_hubstorage_connection_timeout_positional_compatibility( + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv('SHUB_JOBQ', raising=False) client = HubstorageClient('apikey', 'https://storage.example/', 12) assert client.connection_timeout == 12 assert client.jobq.url.startswith('https://storage.example/') -def test_project_and_job_jobq_use_configured_endpoint(): +def test_project_and_job_jobq_use_configured_endpoint() -> None: client = HubstorageClient( auth='apikey', endpoint='https://storage.example/', diff --git a/tox.ini b/tox.ini index 6f0dbb36..476b8f36 100644 --- a/tox.ini +++ b/tox.ini @@ -15,3 +15,15 @@ deps = msgpack: -r tests/requirements-msgpack.txt commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests} + +[testenv:mypy] +basepython = python3 +deps = + mypy==2.3.1 + pytest==9.1.1 + responses==0.26.3 + types-mock==5.2.0.20260518 + types-requests==2.33.0.20260906 + types-six==1.17.0.20260724 +commands = + mypy {posargs: scrapinghub tests} From 3e4849e7320867f1756954f2625c61d14b62b58f Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sat, 26 Sep 2026 00:27:43 +0200 Subject: [PATCH 2/2] Improve coverage --- tests/hubstorage/test_batchuploader.py | 8 ++++++++ tests/hubstorage/test_collections.py | 20 ++++++++++++++++++++ tests/legacy/test_project.py | 6 ++++++ 3 files changed, 34 insertions(+) diff --git a/tests/hubstorage/test_batchuploader.py b/tests/hubstorage/test_batchuploader.py index 7ec971eb..7aeb0865 100644 --- a/tests/hubstorage/test_batchuploader.py +++ b/tests/hubstorage/test_batchuploader.py @@ -3,6 +3,7 @@ """ import time from typing import Any +import mock import pytest from six.moves import range from collections import defaultdict @@ -107,3 +108,10 @@ def test_writer_interval( groups[doc['_ts']] += 1 assert len(groups) == 2 + + +def test_tryupload_non_retryable_failure(hsclient: HubstorageClient) -> None: + batch_uploader = hsclient.batchuploader + with mock.patch.object(batch_uploader, '_upload', + side_effect=RuntimeError): + assert batch_uploader._tryupload({'url': 'url', 'offset': 0}) is None diff --git a/tests/hubstorage/test_collections.py b/tests/hubstorage/test_collections.py index b860b8e1..c6c68a3c 100644 --- a/tests/hubstorage/test_collections.py +++ b/tests/hubstorage/test_collections.py @@ -6,7 +6,9 @@ from collections.abc import Callable from typing import Any +import mock import pytest +from requests import HTTPError, Response from scrapinghub import HubstorageClient from scrapinghub.hubstorage.collectionsrt import Collection from scrapinghub.hubstorage.project import Project @@ -181,3 +183,21 @@ def test_truncate(hscollection: Collection) -> None: hscollection.truncate() assert len(list(hscollection.iter_values(prefix='my_key'))) == 0 + + +@pytest.mark.parametrize('status_code,expected_exception', [ + (400, ValueError), + (500, HTTPError), +]) +def test_count_http_error( + hscollection: Collection, status_code: int, + expected_exception: type[Exception], +) -> None: + response = Response() + response.status_code = status_code + response._content = b'error' + collections = hscollection._collections + with mock.patch.object(collections, 'apirequest', + side_effect=HTTPError(response=response)): + with pytest.raises(expected_exception): + hscollection.count() diff --git a/tests/legacy/test_project.py b/tests/legacy/test_project.py index 095405fd..70b0a9ba 100644 --- a/tests/legacy/test_project.py +++ b/tests/legacy/test_project.py @@ -40,6 +40,12 @@ def test_project_job(project: Project) -> None: ((), {'job': '1/2/3', 'count': 1})] +def test_project_job_not_found(project: Project) -> None: + project.jobs = mock.Mock() # type: ignore[method-assign] + project.jobs.return_value = iter([]) + assert project.job('1/2/3') is None + + def test_project_spiders(project: Project) -> None: project._get = mock.Mock() # type: ignore[method-assign] project._get.return_value = {'spiders': ['spiderA']}