From a22710beec2b0e2c4902b6e12e5db584df61eb17 Mon Sep 17 00:00:00 2001 From: Dick Tump Date: Wed, 7 Oct 2026 09:19:27 +0200 Subject: [PATCH] Update pins + fix rate limiting error --- README.md | 2 +- pins/awscli.env | 10 ++-- pins/glab.env | 10 ++-- pins/opencode.env | 2 +- pins/openspec.env | 2 +- pins/pnpm.env | 2 +- pins/uv.env | 10 ++-- tests/test_update_pins.py | 104 ++++++++++++++++++++++++++++++++++++++ update_pins.py | 96 +++++++++++++++++++++++++++++++++-- 9 files changed, 215 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 5f176860..778aacda 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ uv run update_pins.py --block-major-bumps # stay within each tool's current maj uv run update_pins.py --pin uv=0.12.3 # pin one tool to a specific version ``` -For each tool it selects the newest stable version at least 7 days old, downloads the `amd64` and `arm64` artifacts, computes their sha256s, and rewrites `pins/.env` — the soak window gives a release time to be vetted (and a bad one pulled) before it enters the image. The script prints a report — each `old → new` bump with its age, a `⬆ MAJOR` marker on major-version jumps, `held` lines for versions still inside the soak window, and `⚠` reminders for the manual pins — then review the diff, build to test, and commit. By default a major-version bump is taken once it has soaked; `--block-major-bumps` keeps a run within each tool's current major. Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid GitHub's unauthenticated rate limit. +For each tool it selects the newest stable version at least 7 days old, downloads the `amd64` and `arm64` artifacts, computes their sha256s, and rewrites `pins/.env` — the soak window gives a release time to be vetted (and a bad one pulled) before it enters the image. The script prints a report — each `old → new` bump with its age, a `⬆ MAJOR` marker on major-version jumps, `held` lines for versions still inside the soak window, and `⚠` reminders for the manual pins — then review the diff, build to test, and commit. By default a major-version bump is taken once it has soaked; `--block-major-bumps` keeps a run within each tool's current major. Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid GitHub's unauthenticated rate limit. If you do hit it, the script waits out the advertised reset window and retries (bounded by `UPDATE_PINS_RATE_LIMIT_MAX_WAIT`, default 600s; set it to `0` to fail fast instead of waiting). `nodejs` (from NodeSource's signed apt repo) and the `ubuntu` base-image digest are pinned manually: the script reports base-digest drift but does not rewrite it, since moving the base OS is a deliberate, separately-reviewed change. diff --git a/pins/awscli.env b/pins/awscli.env index 2b4251ce..94ef1710 100644 --- a/pins/awscli.env +++ b/pins/awscli.env @@ -1,6 +1,6 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin awscli=). -AWSCLI_VERSION=2.35.5 -AWSCLI_URL_X86_64=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.35.5.zip -AWSCLI_SHA256_X86_64=54b7006cbaf125eca01f72f93010b15c2f819c82e8bc8ea6834ce853f87dc9e7 -AWSCLI_URL_AARCH64=https://awscli.amazonaws.com/awscli-exe-linux-aarch64-2.35.5.zip -AWSCLI_SHA256_AARCH64=3993f00fbf80e023842859b42cc031ca6677b8ba14e9483111fe87eaa2b2cf86 +AWSCLI_VERSION=2.37.6 +AWSCLI_URL_X86_64=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.37.6.zip +AWSCLI_SHA256_X86_64=cd40c7d1f41b3a4964e77a65377e480d71fe6ebc96bbbb64eb2239d69af6fbb2 +AWSCLI_URL_AARCH64=https://awscli.amazonaws.com/awscli-exe-linux-aarch64-2.37.6.zip +AWSCLI_SHA256_AARCH64=06b572cfd4b0397145a8173fb3456a6e6f0bea1bae50e9162a56be80a1e19618 diff --git a/pins/glab.env b/pins/glab.env index b5016318..c2af7225 100644 --- a/pins/glab.env +++ b/pins/glab.env @@ -1,6 +1,6 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin glab=). -GLAB_VERSION=1.102.0 -GLAB_DEB_URL_AMD64=https://gitlab.com/gitlab-org/cli/-/releases/v1.102.0/downloads/glab_1.102.0_linux_amd64.deb -GLAB_DEB_SHA256_AMD64=1d77a287ed43d063696613dfe21c4e1045d0ead4bebd68dab7e3d6373086d3b0 -GLAB_DEB_URL_ARM64=https://gitlab.com/gitlab-org/cli/-/releases/v1.102.0/downloads/glab_1.102.0_linux_arm64.deb -GLAB_DEB_SHA256_ARM64=5b2424fd335cc601ce6e01e5f0a2972649d0ee3bb0c6bacc22479d1e75798f57 +GLAB_VERSION=1.120.0 +GLAB_DEB_URL_AMD64=https://gitlab.com/gitlab-org/cli/-/releases/v1.120.0/downloads/glab_1.120.0_linux_amd64.deb +GLAB_DEB_SHA256_AMD64=dbe363a81c44094b7fee4970eb96e3db7003c115c34d2d42c245864f1f936ef9 +GLAB_DEB_URL_ARM64=https://gitlab.com/gitlab-org/cli/-/releases/v1.120.0/downloads/glab_1.120.0_linux_arm64.deb +GLAB_DEB_SHA256_ARM64=aa5e3eb58e7de7c001a884699974889f13b8dbbf887182b5d7fe892253ab5287 diff --git a/pins/opencode.env b/pins/opencode.env index 2bf8c13d..fd740e92 100644 --- a/pins/opencode.env +++ b/pins/opencode.env @@ -1,2 +1,2 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin opencode=). -OPENCODE_VERSION=1.18.16 +OPENCODE_VERSION=1.18.33 diff --git a/pins/openspec.env b/pins/openspec.env index 357144ed..8d795e10 100644 --- a/pins/openspec.env +++ b/pins/openspec.env @@ -1,2 +1,2 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin openspec=). -OPENSPEC_VERSION=1.4.1 +OPENSPEC_VERSION=1.13.2 diff --git a/pins/pnpm.env b/pins/pnpm.env index 6ab7c7d0..2259f571 100644 --- a/pins/pnpm.env +++ b/pins/pnpm.env @@ -1,2 +1,2 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin pnpm=). -PNPM_VERSION=11.7.0 +PNPM_VERSION=12.8.2 diff --git a/pins/uv.env b/pins/uv.env index 19a0c843..71b6ada5 100644 --- a/pins/uv.env +++ b/pins/uv.env @@ -1,6 +1,6 @@ # Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin uv=). -UV_VERSION=0.11.21 -UV_URL_X86_64=https://github.com/astral-sh/uv/releases/download/0.11.21/uv-x86_64-unknown-linux-gnu.tar.gz -UV_SHA256_X86_64=8c88519b0ef0af9801fcdee419bbb12116bd9e6b18e162ae093c932d8b264050 -UV_URL_AARCH64=https://github.com/astral-sh/uv/releases/download/0.11.21/uv-aarch64-unknown-linux-gnu.tar.gz -UV_SHA256_AARCH64=88e800834007cc5efd4675f166eb2a51e7e3ad19876d85fa8805a6fb5c922397 +UV_VERSION=0.12.21 +UV_URL_X86_64=https://github.com/astral-sh/uv/releases/download/0.12.21/uv-x86_64-unknown-linux-gnu.tar.gz +UV_SHA256_X86_64=23f02075b652bb1df64178cfae41b5caf160822e720e2663568f3f5d63bc52c0 +UV_URL_AARCH64=https://github.com/astral-sh/uv/releases/download/0.12.21/uv-aarch64-unknown-linux-gnu.tar.gz +UV_SHA256_AARCH64=030b69227b40af8c1981b7301793dc66e71ed3c796ea8688209dd268bd91ec51 diff --git a/tests/test_update_pins.py b/tests/test_update_pins.py index 2e5e08fc..cb0712fa 100644 --- a/tests/test_update_pins.py +++ b/tests/test_update_pins.py @@ -7,8 +7,11 @@ python3 -m unittest discover -s tests """ import contextlib +import email.message import io +import os import sys +import time import unittest import unittest.mock import urllib.error @@ -197,6 +200,107 @@ def test_non_https_redirect_rejected(self): self._redirect("https://api.github.com/x", "http://cdn.example.com/y") +class TestRateLimitRetry(unittest.TestCase): + """_open() must wait out rate limits (bounded) and retry, and suggest a + token when giving up. _OPENER.open and _sleep are mocked — no network, no + real waiting.""" + + URL = "https://api.github.com/x" + + def _http_error(self, code, headers=None): + msg = email.message.Message() + for k, v in (headers or {}).items(): + msg[k] = v + return urllib.error.HTTPError(self.URL, code, "err", msg, io.BytesIO(b"")) + + def _open(self, side_effects, token=None): + """Run up._open with mocked opener/sleep. Returns + (outcome, value, sleeps) where outcome is 'ok' or 'raise'.""" + sleeps = [] + env = {"GITHUB_TOKEN": token} if token else {} + err_buf = io.StringIO() + with unittest.mock.patch.object( + up._OPENER, "open", side_effect=side_effects + ) as m_open, unittest.mock.patch.object( + up, "_sleep", side_effect=lambda s: sleeps.append(s) + ), unittest.mock.patch.dict(os.environ, env, clear=True), \ + contextlib.redirect_stderr(err_buf): + try: + return "ok", up._open(self.URL), sleeps + except Exception as e: # noqa: BLE001 — the test asserts on it + return "raise", e, sleeps + + def test_core_rate_limit_waits_for_reset_then_retries(self): + """403 with X-RateLimit-Remaining: 0 → sleep until reset, then retry.""" + reset = str(int(time.time()) - 5) # already elapsed → ~1s wait + limited = self._http_error(403, { + "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": reset, + }) + outcome, value, sleeps = self._open([limited, "sentinel"]) + self.assertEqual(outcome, "ok") + self.assertEqual(value, "sentinel") + self.assertEqual(len(sleeps), 1) + self.assertAlmostEqual(sleeps[0], 1.0, delta=0.5) + + def test_retry_after_header_honored(self): + limited = self._http_error(403, {"Retry-After": "30"}) + outcome, _, sleeps = self._open([limited, "sentinel"]) + self.assertEqual(outcome, "ok") + self.assertEqual(len(sleeps), 1) + self.assertAlmostEqual(sleeps[0], 31.0, delta=0.5) + + def test_wait_beyond_cap_raises_with_token_hint(self): + """A reset far beyond the cap must fail fast, not sleep for hours — + and the message must point at GITHUB_TOKEN when none is set.""" + limited = self._http_error(403, {"Retry-After": "99999"}) + outcome, err, sleeps = self._open([limited]) + self.assertEqual(outcome, "raise") + self.assertIsInstance(err, RuntimeError) + self.assertIn("GITHUB_TOKEN", str(err)) + self.assertEqual(sleeps, [], "must not sleep when the wait exceeds the cap") + + def test_token_hint_omitted_when_token_set(self): + limited = self._http_error(403, {"Retry-After": "99999"}) + outcome, err, _ = self._open([limited], token="secret") + self.assertEqual(outcome, "raise") + self.assertNotIn("GITHUB_TOKEN", str(err)) + + def test_plain_403_is_not_retried(self): + """A 403 without rate-limit headers is a real error — raise at once.""" + outcome, err, sleeps = self._open([self._http_error(403)]) + self.assertEqual(outcome, "raise") + self.assertIsInstance(err, urllib.error.HTTPError) + self.assertEqual(sleeps, []) + + def test_500_retried_with_exponential_backoff(self): + outcome, value, sleeps = self._open( + [self._http_error(500), self._http_error(503), "sentinel"] + ) + self.assertEqual(outcome, "ok") + self.assertEqual(value, "sentinel") + self.assertEqual(sleeps, [2.0, 4.0]) + + def test_429_without_retry_after_backs_off(self): + outcome, _, sleeps = self._open([self._http_error(429), "sentinel"]) + self.assertEqual(outcome, "ok") + self.assertEqual(sleeps, [2.0]) + + def test_rate_limit_persisting_past_max_attempts_raises(self): + limited = self._http_error(403, {"Retry-After": "1"}) + outcome, err, sleeps = self._open([limited] * up.MAX_ATTEMPTS) + self.assertEqual(outcome, "raise") + self.assertIsInstance(err, RuntimeError) + self.assertIn(f"after {up.MAX_ATTEMPTS} attempts", str(err)) + self.assertEqual(len(sleeps), up.MAX_ATTEMPTS - 1) + + def test_network_error_retried_then_raises(self): + err = urllib.error.URLError("connection reset") + outcome, raised, sleeps = self._open([err] * up.MAX_ATTEMPTS) + self.assertEqual(outcome, "raise") + self.assertIs(raised, err) + self.assertEqual(sleeps, [2.0, 4.0, 8.0, 16.0]) + + class TestVersionVar(unittest.TestCase): """version_var() must derive the correct env-var name for each npm tool.""" diff --git a/update_pins.py b/update_pins.py index 25235f96..7d0f9c78 100755 --- a/update_pins.py +++ b/update_pins.py @@ -28,8 +28,13 @@ python3 update_pins.py --audit soak-gate check against live npm registry Honors GITHUB_TOKEN / GH_TOKEN (raises the GitHub API rate limit) when set. -Stdlib only — no third-party packages (a supply-chain tool keeps its own trusted -base minimal; `dependencies = []` above makes that visible and enforced). +Rate-limited responses (GitHub core limit, secondary limits, 429s) are retried +after waiting out the advertised reset window, bounded by +UPDATE_PINS_RATE_LIMIT_MAX_WAIT seconds (default 600; set it to 0 to fail fast +instead of waiting). Transient 5xx/network errors get bounded exponential +backoff. Stdlib only — no third-party packages (a supply-chain tool keeps its +own trusted base minimal; `dependencies = []` above makes that visible and +enforced). """ from __future__ import annotations @@ -41,6 +46,7 @@ import shutil import sys import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -77,7 +83,7 @@ ] -# ---- HTTP (https-only, bounded redirects) --------------------------------- +# ---- HTTP (https-only, bounded redirects, bounded rate-limit waits) ------- class _HTTPSOnlyRedirect(urllib.request.HTTPRedirectHandler): """Reject redirects to non-https targets, bound the chain, and drop the Authorization header when a redirect crosses to a different host — the @@ -106,6 +112,62 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): _OPENER = urllib.request.build_opener(_HTTPSOnlyRedirect) +# Retry policy for transient HTTP failures. GitHub's unauthenticated core limit +# is 60 requests/hour per IP — shared CI runners and repeated local runs burn +# through that fast — so instead of aborting on a rate limit we wait out the +# window the server advertises (Retry-After / X-RateLimit-Reset) and retry, +# bounded by RATE_LIMIT_MAX_WAIT so a run can never hang for hours. Set +# UPDATE_PINS_RATE_LIMIT_MAX_WAIT=0 to fail fast instead of waiting. +RATE_LIMIT_MAX_WAIT = float(os.environ.get("UPDATE_PINS_RATE_LIMIT_MAX_WAIT", "600")) +MAX_ATTEMPTS = 5 +BACKOFF_BASE = 2.0 # seconds; exponential backoff for non-rate-limit errors + + +def _sleep(seconds: float) -> None: + """Indirection so the unit tests can patch out real waiting.""" + time.sleep(seconds) + + +def _rate_limit_wait(e: urllib.error.HTTPError) -> float | None: + """Seconds to wait before retrying this error, or None if it is not a + recognisable rate limit: GitHub's core limit answers 403 with + X-RateLimit-Remaining: 0 + X-RateLimit-Reset (epoch seconds); secondary + limits and 429s carry Retry-After.""" + if e.code not in (403, 429): + return None + hdrs = e.headers + if hdrs is None: + return None + retry_after = hdrs.get("Retry-After") + if retry_after: + try: + return float(retry_after) + 1.0 + except ValueError: + pass + if hdrs.get("X-RateLimit-Remaining") == "0": + reset = hdrs.get("X-RateLimit-Reset") + if reset: + try: + return max(0.0, float(reset) - time.time()) + 1.0 + except ValueError: + pass + return None + + +def _rate_limit_give_up(url: str, wait: float) -> RuntimeError: + hint = "" + if not (os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")): + hint = " — set GITHUB_TOKEN (or GH_TOKEN) to raise the GitHub API limit" + if wait > RATE_LIMIT_MAX_WAIT: + return RuntimeError( + f"rate limit on {url}: reset in {wait:.0f}s exceeds the " + f"{RATE_LIMIT_MAX_WAIT:.0f}s wait cap (raise it with " + f"UPDATE_PINS_RATE_LIMIT_MAX_WAIT, or set that to 0 to fail fast){hint}" + ) + return RuntimeError( + f"rate limit on {url} still active after {MAX_ATTEMPTS} attempts{hint}" + ) + def _open(url: str, headers: dict | None = None): if not url.lower().startswith("https://"): @@ -113,7 +175,33 @@ def _open(url: str, headers: dict | None = None): hdrs = {"User-Agent": USER_AGENT} if headers: hdrs.update(headers) - return _OPENER.open(urllib.request.Request(url, headers=hdrs), timeout=30) + for attempt in range(1, MAX_ATTEMPTS + 1): + try: + return _OPENER.open(urllib.request.Request(url, headers=hdrs), timeout=30) + except urllib.error.HTTPError as e: + rl_wait = _rate_limit_wait(e) + if rl_wait is not None: + if rl_wait > RATE_LIMIT_MAX_WAIT or attempt == MAX_ATTEMPTS: + raise _rate_limit_give_up(url, rl_wait) from e + print(f" … {url}: HTTP {e.code} rate limited, waiting {rl_wait:.0f}s " + f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr) + _sleep(rl_wait) + continue + if (500 <= e.code < 600 or e.code == 429) and attempt < MAX_ATTEMPTS: + wait = BACKOFF_BASE * 2 ** (attempt - 1) # transient server error + print(f" … {url}: HTTP {e.code}, retrying in {wait:.0f}s " + f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr) + _sleep(wait) + continue + raise + except urllib.error.URLError as e: + if attempt == MAX_ATTEMPTS: + raise + wait = BACKOFF_BASE * 2 ** (attempt - 1) + print(f" … {url}: {e.reason}, retrying in {wait:.0f}s " + f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr) + _sleep(wait) + raise AssertionError("unreachable") def http_bytes(url: str, headers: dict | None = None) -> bytes: