From 03d13dded5d2f7cec3122c5295f8f4300284cb7e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 28 Sep 2026 13:00:58 +0200 Subject: [PATCH] Make msgpack a required dependency --- .github/workflows/main.yml | 25 +++++-------------------- README.rst | 5 ----- docs/intro/install.rst | 8 -------- scrapinghub/hubstorage/client.py | 6 +----- scrapinghub/hubstorage/serialization.py | 9 +-------- setup.py | 11 ++--------- tests/client/conftest.py | 3 +-- tests/conftest.py | 1 - tests/hubstorage/conftest.py | 3 +-- tests/requirements-msgpack.txt | 2 -- tox.ini | 4 ++-- 11 files changed, 13 insertions(+), 64 deletions(-) delete mode 100644 tests/requirements-msgpack.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5bd464b0..c2440c3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,28 +15,16 @@ jobs: include: - python-version: "3.10" env: - TOXENV: "msgpack" - - python-version: "3.10" - env: - TOXENV: "json" + TOXENV: "py" - python-version: "3.11" env: - TOXENV: "msgpack" - - python-version: "3.11" - env: - TOXENV: "json" - - python-version: "3.12" - env: - TOXENV: "msgpack" + TOXENV: "py" - python-version: "3.12" env: - TOXENV: "json" - - python-version: "3.13" - env: - TOXENV: "msgpack" + TOXENV: "py" - python-version: "3.13" env: - TOXENV: "json" + TOXENV: "py" # Keep in sync with pyproject.toml's # tool.sphinx-scrapy.python-version. - python-version: "3.14" @@ -46,10 +34,7 @@ jobs: # https://github.com/kevin1024/vcrpy/issues/885 # - python-version: "pypy3.11-v7.3.20" # env: - # TOXENV: "msgpack" - # - python-version: "pypy3.11-v7.3.20" - # env: - # TOXENV: "json" + # TOXENV: "py" steps: - uses: actions/checkout@v2 diff --git a/README.rst b/README.rst index 92c4f12c..a40c1e2d 100644 --- a/README.rst +++ b/README.rst @@ -25,11 +25,6 @@ The quick way:: pip install scrapinghub -You can also install the library with MessagePack support, it provides better -response time and improved bandwidth usage:: - - pip install scrapinghub[msgpack] - Documentation ------------- diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 6dcd1b95..6bc2f49b 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -13,11 +13,3 @@ Installing scrapinghub The quick way:: pip install scrapinghub - -It is recommended to install the library with `MessagePack`_ support, -it provides better response time and improved bandwidth usage:: - - pip install scrapinghub[msgpack] - - -.. _MessagePack: https://en.wikipedia.org/wiki/MessagePack diff --git a/scrapinghub/hubstorage/client.py b/scrapinghub/hubstorage/client.py index 72d2a556..6b92eaf2 100644 --- a/scrapinghub/hubstorage/client.py +++ b/scrapinghub/hubstorage/client.py @@ -12,7 +12,6 @@ from .jobq import JobQ from .batchuploader import BatchUploader from .resourcetype import ResourceType -from .serialization import MSGPACK_AVAILABLE __all__ = ["HubstorageClient"] @@ -108,10 +107,7 @@ def __init__(self, auth=None, endpoint=None, connection_timeout=None, self.projects = Projects(self, None) self.root = ResourceType(self, None) self._batchuploader = None - self.use_msgpack = MSGPACK_AVAILABLE and use_msgpack - if use_msgpack != self.use_msgpack: - logger.warning('Messagepack is not available, please ensure that ' - 'msgpack library is properly installed.') + self.use_msgpack = use_msgpack def request(self, is_idempotent=False, **kwargs): """ diff --git a/scrapinghub/hubstorage/serialization.py b/scrapinghub/hubstorage/serialization.py index 70567805..125bacd0 100644 --- a/scrapinghub/hubstorage/serialization.py +++ b/scrapinghub/hubstorage/serialization.py @@ -2,19 +2,12 @@ from json import dumps, loads import six +from msgpack import Unpacker EPOCH = datetime(1970, 1, 1, 0, 0) ADAYINSECONDS = 24 * 3600 -try: - from msgpack import Unpacker - - MSGPACK_AVAILABLE = True -except ImportError: - MSGPACK_AVAILABLE = False - - def jlencode(iterable): if isinstance(iterable, (dict, six.string_types)): iterable = [iterable] diff --git a/setup.py b/setup.py index 315ea184..55e308d6 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -import sys from os.path import dirname, join try: @@ -10,11 +9,6 @@ with open(join(dirname(__file__), 'scrapinghub/VERSION'), 'rb') as f: version = f.read().decode('ascii').strip() -is_pypy = '__pypy__' in sys.builtin_module_names -mpack_required = ['msgpack>=1.0.0'] -if is_pypy: - mpack_required.append('msgpack-pypy>=0.0.2') - setup( name='scrapinghub', version=version, @@ -26,9 +20,8 @@ platforms=['Any'], packages=['scrapinghub', 'scrapinghub.client', 'scrapinghub.hubstorage'], package_data={'scrapinghub': ['VERSION']}, - install_requires=['python-dotenv>=1.0.0', 'requests>=1.0', - 'retrying>=1.3.3', 'six>=1.10.0'], - extras_require={'msgpack': mpack_required}, + install_requires=['msgpack>=1.0.0', 'python-dotenv>=1.0.0', + 'requests>=1.0', 'retrying>=1.3.3', 'six>=1.10.0'], python_requires='>=3.10', classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/tests/client/conftest.py b/tests/client/conftest.py index 8d253c0d..9e7e06bb 100644 --- a/tests/client/conftest.py +++ b/tests/client/conftest.py @@ -6,7 +6,6 @@ from scrapinghub import ScrapinghubClient from scrapinghub.client.exceptions import NotFound -from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE from ..conftest import request_accept_header_matcher from ..conftest import VCRGzipSerializer @@ -107,7 +106,7 @@ def setup_session(client, project, collection, request): def json_and_msgpack(client, monkeypatch, request): if request.param == 'json': monkeypatch.setattr(client._hsclient, 'use_msgpack', False) - elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"): + elif request.config.getoption("--disable-msgpack"): pytest.skip("messagepack-based tests are disabled") return request.param diff --git a/tests/conftest.py b/tests/conftest.py index 59ad94f2..da0ef54f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,6 @@ import sys import zlib -from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE from scrapinghub import HubstorageClient from scrapinghub.legacy import Connection diff --git a/tests/hubstorage/conftest.py b/tests/hubstorage/conftest.py index 6ede3546..8c3e4979 100644 --- a/tests/hubstorage/conftest.py +++ b/tests/hubstorage/conftest.py @@ -8,7 +8,6 @@ from scrapinghub import HubstorageClient from scrapinghub.hubstorage.utils import urlpathjoin -from scrapinghub.hubstorage.serialization import MSGPACK_AVAILABLE from ..conftest import request_accept_header_matcher from ..conftest import VCRGzipSerializer @@ -89,7 +88,7 @@ def setup_session(hsclient, hsproject, hscollection, request): def json_and_msgpack(hsclient, monkeypatch, request): if request.param == 'json': monkeypatch.setattr(hsclient, 'use_msgpack', False) - elif not MSGPACK_AVAILABLE or request.config.getoption("--disable-msgpack"): + elif request.config.getoption("--disable-msgpack"): pytest.skip("messagepack-based tests are disabled") return request.param diff --git a/tests/requirements-msgpack.txt b/tests/requirements-msgpack.txt deleted file mode 100644 index 3a9b0fab..00000000 --- a/tests/requirements-msgpack.txt +++ /dev/null @@ -1,2 +0,0 @@ -msgpack>=1.0.0; implementation_name != 'pypy' -msgpack-pypy>=0.0.2; implementation_name == 'pypy' diff --git a/tox.ini b/tox.ini index 6f0dbb36..71035a96 100644 --- a/tox.ini +++ b/tox.ini @@ -4,14 +4,14 @@ # and then run "tox" from this directory. [tox] +min_version = 4.25.0 requires = sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.7.1 # pypy3 excluded due to https://github.com/kevin1024/vcrpy/issues/885 -envlist = py{310,311,312}-{json,msgpack},docs +envlist = py3{10-13},docs [testenv] deps = -r tests/requirements.txt - msgpack: -r tests/requirements-msgpack.txt commands = py.test --cov=scrapinghub --cov-report=xml {posargs: scrapinghub tests}