From a74ccafc50650e2149bcfe7ac65f0af0e6920111 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 1 Oct 2026 14:45:36 -0400 Subject: [PATCH 1/2] Bump required Python version to >=3.12 Generated-by: OpenAI Codex --- .github/workflows/build.yml | 2 +- .github/workflows/create-branch.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/update_ci.yml | 2 +- CHANGES/+python-3.12.feature | 1 + pyproject.toml | 3 +-- 7 files changed, 7 insertions(+), 7 deletions(-) create mode 100644 CHANGES/+python-3.12.feature diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c28d5d11..c0f6f63f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,7 @@ jobs: path: "pulp-openapi-generator" - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.12" - name: "Install uv" uses: "astral-sh/setup-uv@v7" with: diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index bc141977..4ad455f6 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -38,7 +38,7 @@ jobs: - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.12" - name: "Install uv" uses: "astral-sh/setup-uv@v7" with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ee9d46da..00fabd5d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.12" - name: "Install uv" uses: "astral-sh/setup-uv@v7" with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 514e2029..584532ff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.12" - name: "Install uv" uses: "astral-sh/setup-uv@v7" with: diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 6a867d7b..d08c5134 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -34,7 +34,7 @@ jobs: - uses: "actions/setup-python@v6" with: - python-version: "3.11" + python-version: "3.12" - name: "Install uv" uses: "astral-sh/setup-uv@v7" with: diff --git a/CHANGES/+python-3.12.feature b/CHANGES/+python-3.12.feature new file mode 100644 index 00000000..e7a70b19 --- /dev/null +++ b/CHANGES/+python-3.12.feature @@ -0,0 +1 @@ +Updated minimum required python version to >=3.12. diff --git a/pyproject.toml b/pyproject.toml index ae166c20..53a6a464 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,10 @@ classifiers=[ "Framework :: Django", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] -requires-python = ">=3.11" +requires-python = ">=3.12" dependencies = [ "pulpcore>=3.105.0,<3.130", "pkginfo>=1.12.0,<1.13.0", From f598a5fef175bb39236d72f1da0ba9bf6e464264 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Thu, 1 Oct 2026 14:45:47 -0400 Subject: [PATCH 2/2] Bump bandersnatch to 8.0 Generated-by: OpenAI Codex --- CHANGES/+bandersnatch-8.feature | 1 + pulp_python/app/tasks/sync.py | 66 ++++++++----------- .../functional/api/test_download_content.py | 2 +- pyproject.toml | 2 +- 4 files changed, 32 insertions(+), 39 deletions(-) create mode 100644 CHANGES/+bandersnatch-8.feature diff --git a/CHANGES/+bandersnatch-8.feature b/CHANGES/+bandersnatch-8.feature new file mode 100644 index 00000000..6e0bfa83 --- /dev/null +++ b/CHANGES/+bandersnatch-8.feature @@ -0,0 +1 @@ +Upgraded Bandersnatch to 8.0. Full-index syncs now list packages from either the JSON or HTML representation of the Simple API. diff --git a/pulp_python/app/tasks/sync.py b/pulp_python/app/tasks/sync.py index dd5e3826..72b89d4a 100644 --- a/pulp_python/app/tasks/sync.py +++ b/pulp_python/app/tasks/sync.py @@ -1,15 +1,14 @@ import asyncio +import json import logging from functools import partial -from urllib.parse import urljoin, urlparse +from urllib.parse import urlparse -from aiohttp import ClientError, ClientResponseError from bandersnatch.configuration import BandersnatchConfig from bandersnatch.master import Master from bandersnatch.mirror import Mirror -from lxml.etree import LxmlError from packaging.requirements import Requirement -from pypi_simple import IndexPage +from pypi_simple import ACCEPT_ANY, IndexPage, UnsupportedContentTypeError from pulpcore.plugin.download import HttpDownloader from pulpcore.plugin.exceptions import SyncError @@ -169,41 +168,34 @@ def __init__(self, serial, master, workers, deferred_download, python_stage, pro async def determine_packages_to_sync(self): """ - Calling this means that includes wasn't specified, - so try to get all of the packages from Mirror (hopefully PyPi) + Called when includes wasn't specified. List all projects from the remote's simple index. """ - number_xmlrpc_attempts = 3 - for attempt in range(number_xmlrpc_attempts): - logger.info("Attempt {} to get package list from {}".format(attempt, self.master.url)) - try: - if not self.synced_serial: - logger.info("Syncing all packages.") - # First get the current serial, then start to sync. - all_packages = await self.master.all_packages() - self.packages_to_sync.update(all_packages) - self.target_serial = max( - [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] - ) - else: - logger.info("Syncing based on changelog.") - changed_packages = await self.master.changed_packages(self.synced_serial) - self.packages_to_sync.update(changed_packages) - self.target_serial = max( - [self.synced_serial] + [int(v) for v in self.packages_to_sync.values()] - ) - break - except (ClientError, ClientResponseError, LxmlError): - # Retry if XMLRPC endpoint failed, server might not support it. - continue + logger.info("Syncing all packages from %s", self.master.url) + url = f"{self.remote.url.rstrip('/')}/simple/" + downloader = self.remote.get_downloader(url=url) + result = await downloader.run( + extra_data={"request_kwargs": {"headers": {"Accept": ACCEPT_ANY}}} + ) + + content_type = result.headers.get("Content-Type", "text/html").partition(";")[0].lower() + with open(result.path, "rb") as f: + content = f.read() + + if content_type == "application/vnd.pypi.simple.v1+json": + index = IndexPage.from_json_data(json.loads(content)) + elif content_type in {"application/vnd.pypi.simple.v1+html", "text/html"}: + index = IndexPage.from_html(content) + else: + raise UnsupportedContentTypeError(url, content_type) + + self.packages_to_sync = {project: 0 for project in index.projects} + last_serial = index.last_serial + if last_serial is None: + last_serial = result.headers.get(PYPI_LAST_SERIAL) + if last_serial is None: + self.target_serial = self.synced_serial else: - logger.info("Failed to get package list using XMLRPC, trying parse simple page.") - url = urljoin(self.remote.url, "simple/") - downloader = self.remote.get_downloader(url=url) - result = await downloader.run() - with open(result.path) as f: - index = IndexPage.from_html(f.read()) - self.packages_to_sync.update({p: 0 for p in index.projects}) - self.target_serial = result.headers.get(PYPI_LAST_SERIAL, 0) + self.target_serial = max(self.synced_serial, int(last_serial)) self._filter_packages() if self.target_serial: diff --git a/pulp_python/tests/functional/api/test_download_content.py b/pulp_python/tests/functional/api/test_download_content.py index c9820f11..8c6356f6 100644 --- a/pulp_python/tests/functional/api/test_download_content.py +++ b/pulp_python/tests/functional/api/test_download_content.py @@ -48,7 +48,7 @@ def test_full_fixtures_to_pulp_sync( ): """ This test checks that Pulp can fully sync another Python Package repository that is not - PyPI. This reads the repository's simple page if XMLRPC isn't supported. + PyPI. This lists projects from the remote's Simple API index. """ # Repository we are syncing from is the fixtures (default url) remote = python_remote_factory(includes=[], prereleases=True) diff --git a/pyproject.toml b/pyproject.toml index 53a6a464..6fecb976 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ requires-python = ">=3.12" dependencies = [ "pulpcore>=3.105.0,<3.130", "pkginfo>=1.12.0,<1.13.0", - "bandersnatch>=6.6.0,<6.7", + "bandersnatch>=8.0.0,<8.1", "pypi-simple>=1.8.0,<2.0", "pypi-attestations==0.0.28", # API is not stable ]