Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ uv run update_pins.py --block-major-bumps # stay within each tool's current maj
uv run update_pins.py --pin uv=0.12.3 # pin one tool to a specific version
```

For each tool it selects the newest stable version at least 7 days old, downloads the `amd64` and `arm64` artifacts, computes their sha256s, and rewrites `pins/<tool>.env` — the soak window gives a release time to be vetted (and a bad one pulled) before it enters the image. The script prints a report — each `old → new` bump with its age, a `⬆ MAJOR` marker on major-version jumps, `held` lines for versions still inside the soak window, and `⚠` reminders for the manual pins — then review the diff, build to test, and commit. By default a major-version bump is taken once it has soaked; `--block-major-bumps` keeps a run within each tool's current major. Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid GitHub's unauthenticated rate limit.
For each tool it selects the newest stable version at least 7 days old, downloads the `amd64` and `arm64` artifacts, computes their sha256s, and rewrites `pins/<tool>.env` — the soak window gives a release time to be vetted (and a bad one pulled) before it enters the image. The script prints a report — each `old → new` bump with its age, a `⬆ MAJOR` marker on major-version jumps, `held` lines for versions still inside the soak window, and `⚠` reminders for the manual pins — then review the diff, build to test, and commit. By default a major-version bump is taken once it has soaked; `--block-major-bumps` keeps a run within each tool's current major. Set `GITHUB_TOKEN` (or `GH_TOKEN`) to avoid GitHub's unauthenticated rate limit. If you do hit it, the script waits out the advertised reset window and retries (bounded by `UPDATE_PINS_RATE_LIMIT_MAX_WAIT`, default 600s; set it to `0` to fail fast instead of waiting).

`nodejs` (from NodeSource's signed apt repo) and the `ubuntu` base-image digest are pinned manually: the script reports base-digest drift but does not rewrite it, since moving the base OS is a deliberate, separately-reviewed change.

Expand Down
10 changes: 5 additions & 5 deletions pins/awscli.env
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin awscli=<version>).
AWSCLI_VERSION=2.35.5
AWSCLI_URL_X86_64=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.35.5.zip
AWSCLI_SHA256_X86_64=54b7006cbaf125eca01f72f93010b15c2f819c82e8bc8ea6834ce853f87dc9e7
AWSCLI_URL_AARCH64=https://awscli.amazonaws.com/awscli-exe-linux-aarch64-2.35.5.zip
AWSCLI_SHA256_AARCH64=3993f00fbf80e023842859b42cc031ca6677b8ba14e9483111fe87eaa2b2cf86
AWSCLI_VERSION=2.37.6
AWSCLI_URL_X86_64=https://awscli.amazonaws.com/awscli-exe-linux-x86_64-2.37.6.zip
AWSCLI_SHA256_X86_64=cd40c7d1f41b3a4964e77a65377e480d71fe6ebc96bbbb64eb2239d69af6fbb2
AWSCLI_URL_AARCH64=https://awscli.amazonaws.com/awscli-exe-linux-aarch64-2.37.6.zip
AWSCLI_SHA256_AARCH64=06b572cfd4b0397145a8173fb3456a6e6f0bea1bae50e9162a56be80a1e19618
10 changes: 5 additions & 5 deletions pins/glab.env
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin glab=<version>).
GLAB_VERSION=1.102.0
GLAB_DEB_URL_AMD64=https://gitlab.com/gitlab-org/cli/-/releases/v1.102.0/downloads/glab_1.102.0_linux_amd64.deb
GLAB_DEB_SHA256_AMD64=1d77a287ed43d063696613dfe21c4e1045d0ead4bebd68dab7e3d6373086d3b0
GLAB_DEB_URL_ARM64=https://gitlab.com/gitlab-org/cli/-/releases/v1.102.0/downloads/glab_1.102.0_linux_arm64.deb
GLAB_DEB_SHA256_ARM64=5b2424fd335cc601ce6e01e5f0a2972649d0ee3bb0c6bacc22479d1e75798f57
GLAB_VERSION=1.120.0
GLAB_DEB_URL_AMD64=https://gitlab.com/gitlab-org/cli/-/releases/v1.120.0/downloads/glab_1.120.0_linux_amd64.deb
GLAB_DEB_SHA256_AMD64=dbe363a81c44094b7fee4970eb96e3db7003c115c34d2d42c245864f1f936ef9
GLAB_DEB_URL_ARM64=https://gitlab.com/gitlab-org/cli/-/releases/v1.120.0/downloads/glab_1.120.0_linux_arm64.deb
GLAB_DEB_SHA256_ARM64=aa5e3eb58e7de7c001a884699974889f13b8dbbf887182b5d7fe892253ab5287
2 changes: 1 addition & 1 deletion pins/opencode.env
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin opencode=<version>).
OPENCODE_VERSION=1.18.16
OPENCODE_VERSION=1.18.33
2 changes: 1 addition & 1 deletion pins/openspec.env
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin openspec=<version>).
OPENSPEC_VERSION=1.4.1
OPENSPEC_VERSION=1.13.2
2 changes: 1 addition & 1 deletion pins/pnpm.env
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin pnpm=<version>).
PNPM_VERSION=11.7.0
PNPM_VERSION=12.8.2
10 changes: 5 additions & 5 deletions pins/uv.env
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Generated by update_pins.py — do not edit by hand (use: update_pins.py --pin uv=<version>).
UV_VERSION=0.11.21
UV_URL_X86_64=https://github.com/astral-sh/uv/releases/download/0.11.21/uv-x86_64-unknown-linux-gnu.tar.gz
UV_SHA256_X86_64=8c88519b0ef0af9801fcdee419bbb12116bd9e6b18e162ae093c932d8b264050
UV_URL_AARCH64=https://github.com/astral-sh/uv/releases/download/0.11.21/uv-aarch64-unknown-linux-gnu.tar.gz
UV_SHA256_AARCH64=88e800834007cc5efd4675f166eb2a51e7e3ad19876d85fa8805a6fb5c922397
UV_VERSION=0.12.21
UV_URL_X86_64=https://github.com/astral-sh/uv/releases/download/0.12.21/uv-x86_64-unknown-linux-gnu.tar.gz
UV_SHA256_X86_64=23f02075b652bb1df64178cfae41b5caf160822e720e2663568f3f5d63bc52c0
UV_URL_AARCH64=https://github.com/astral-sh/uv/releases/download/0.12.21/uv-aarch64-unknown-linux-gnu.tar.gz
UV_SHA256_AARCH64=030b69227b40af8c1981b7301793dc66e71ed3c796ea8688209dd268bd91ec51
104 changes: 104 additions & 0 deletions tests/test_update_pins.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,11 @@
python3 -m unittest discover -s tests
"""
import contextlib
import email.message
import io
import os
import sys
import time
import unittest
import unittest.mock
import urllib.error
Expand Down Expand Up @@ -197,6 +200,107 @@ def test_non_https_redirect_rejected(self):
self._redirect("https://api.github.com/x", "http://cdn.example.com/y")


class TestRateLimitRetry(unittest.TestCase):
"""_open() must wait out rate limits (bounded) and retry, and suggest a
token when giving up. _OPENER.open and _sleep are mocked — no network, no
real waiting."""

URL = "https://api.github.com/x"

def _http_error(self, code, headers=None):
msg = email.message.Message()
for k, v in (headers or {}).items():
msg[k] = v
return urllib.error.HTTPError(self.URL, code, "err", msg, io.BytesIO(b""))

def _open(self, side_effects, token=None):
"""Run up._open with mocked opener/sleep. Returns
(outcome, value, sleeps) where outcome is 'ok' or 'raise'."""
sleeps = []
env = {"GITHUB_TOKEN": token} if token else {}
err_buf = io.StringIO()
with unittest.mock.patch.object(
up._OPENER, "open", side_effect=side_effects
) as m_open, unittest.mock.patch.object(
up, "_sleep", side_effect=lambda s: sleeps.append(s)
), unittest.mock.patch.dict(os.environ, env, clear=True), \
contextlib.redirect_stderr(err_buf):
try:
return "ok", up._open(self.URL), sleeps
except Exception as e: # noqa: BLE001 — the test asserts on it
return "raise", e, sleeps

def test_core_rate_limit_waits_for_reset_then_retries(self):
"""403 with X-RateLimit-Remaining: 0 → sleep until reset, then retry."""
reset = str(int(time.time()) - 5) # already elapsed → ~1s wait
limited = self._http_error(403, {
"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": reset,
})
outcome, value, sleeps = self._open([limited, "sentinel"])
self.assertEqual(outcome, "ok")
self.assertEqual(value, "sentinel")
self.assertEqual(len(sleeps), 1)
self.assertAlmostEqual(sleeps[0], 1.0, delta=0.5)

def test_retry_after_header_honored(self):
limited = self._http_error(403, {"Retry-After": "30"})
outcome, _, sleeps = self._open([limited, "sentinel"])
self.assertEqual(outcome, "ok")
self.assertEqual(len(sleeps), 1)
self.assertAlmostEqual(sleeps[0], 31.0, delta=0.5)

def test_wait_beyond_cap_raises_with_token_hint(self):
"""A reset far beyond the cap must fail fast, not sleep for hours —
and the message must point at GITHUB_TOKEN when none is set."""
limited = self._http_error(403, {"Retry-After": "99999"})
outcome, err, sleeps = self._open([limited])
self.assertEqual(outcome, "raise")
self.assertIsInstance(err, RuntimeError)
self.assertIn("GITHUB_TOKEN", str(err))
self.assertEqual(sleeps, [], "must not sleep when the wait exceeds the cap")

def test_token_hint_omitted_when_token_set(self):
limited = self._http_error(403, {"Retry-After": "99999"})
outcome, err, _ = self._open([limited], token="secret")
self.assertEqual(outcome, "raise")
self.assertNotIn("GITHUB_TOKEN", str(err))

def test_plain_403_is_not_retried(self):
"""A 403 without rate-limit headers is a real error — raise at once."""
outcome, err, sleeps = self._open([self._http_error(403)])
self.assertEqual(outcome, "raise")
self.assertIsInstance(err, urllib.error.HTTPError)
self.assertEqual(sleeps, [])

def test_500_retried_with_exponential_backoff(self):
outcome, value, sleeps = self._open(
[self._http_error(500), self._http_error(503), "sentinel"]
)
self.assertEqual(outcome, "ok")
self.assertEqual(value, "sentinel")
self.assertEqual(sleeps, [2.0, 4.0])

def test_429_without_retry_after_backs_off(self):
outcome, _, sleeps = self._open([self._http_error(429), "sentinel"])
self.assertEqual(outcome, "ok")
self.assertEqual(sleeps, [2.0])

def test_rate_limit_persisting_past_max_attempts_raises(self):
limited = self._http_error(403, {"Retry-After": "1"})
outcome, err, sleeps = self._open([limited] * up.MAX_ATTEMPTS)
self.assertEqual(outcome, "raise")
self.assertIsInstance(err, RuntimeError)
self.assertIn(f"after {up.MAX_ATTEMPTS} attempts", str(err))
self.assertEqual(len(sleeps), up.MAX_ATTEMPTS - 1)

def test_network_error_retried_then_raises(self):
err = urllib.error.URLError("connection reset")
outcome, raised, sleeps = self._open([err] * up.MAX_ATTEMPTS)
self.assertEqual(outcome, "raise")
self.assertIs(raised, err)
self.assertEqual(sleeps, [2.0, 4.0, 8.0, 16.0])


class TestVersionVar(unittest.TestCase):
"""version_var() must derive the correct env-var name for each npm tool."""

Expand Down
96 changes: 92 additions & 4 deletions update_pins.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,13 @@
python3 update_pins.py --audit soak-gate check against live npm registry

Honors GITHUB_TOKEN / GH_TOKEN (raises the GitHub API rate limit) when set.
Stdlib only — no third-party packages (a supply-chain tool keeps its own trusted
base minimal; `dependencies = []` above makes that visible and enforced).
Rate-limited responses (GitHub core limit, secondary limits, 429s) are retried
after waiting out the advertised reset window, bounded by
UPDATE_PINS_RATE_LIMIT_MAX_WAIT seconds (default 600; set it to 0 to fail fast
instead of waiting). Transient 5xx/network errors get bounded exponential
backoff. Stdlib only — no third-party packages (a supply-chain tool keeps its
own trusted base minimal; `dependencies = []` above makes that visible and
enforced).
"""
from __future__ import annotations

Expand All @@ -41,6 +46,7 @@
import shutil
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request
Expand Down Expand Up @@ -77,7 +83,7 @@
]


# ---- HTTP (https-only, bounded redirects) ---------------------------------
# ---- HTTP (https-only, bounded redirects, bounded rate-limit waits) -------
class _HTTPSOnlyRedirect(urllib.request.HTTPRedirectHandler):
"""Reject redirects to non-https targets, bound the chain, and drop the
Authorization header when a redirect crosses to a different host — the
Expand Down Expand Up @@ -106,14 +112,96 @@ def redirect_request(self, req, fp, code, msg, headers, newurl):

_OPENER = urllib.request.build_opener(_HTTPSOnlyRedirect)

# Retry policy for transient HTTP failures. GitHub's unauthenticated core limit
# is 60 requests/hour per IP — shared CI runners and repeated local runs burn
# through that fast — so instead of aborting on a rate limit we wait out the
# window the server advertises (Retry-After / X-RateLimit-Reset) and retry,
# bounded by RATE_LIMIT_MAX_WAIT so a run can never hang for hours. Set
# UPDATE_PINS_RATE_LIMIT_MAX_WAIT=0 to fail fast instead of waiting.
RATE_LIMIT_MAX_WAIT = float(os.environ.get("UPDATE_PINS_RATE_LIMIT_MAX_WAIT", "600"))
MAX_ATTEMPTS = 5
BACKOFF_BASE = 2.0 # seconds; exponential backoff for non-rate-limit errors


def _sleep(seconds: float) -> None:
"""Indirection so the unit tests can patch out real waiting."""
time.sleep(seconds)


def _rate_limit_wait(e: urllib.error.HTTPError) -> float | None:
"""Seconds to wait before retrying this error, or None if it is not a
recognisable rate limit: GitHub's core limit answers 403 with
X-RateLimit-Remaining: 0 + X-RateLimit-Reset (epoch seconds); secondary
limits and 429s carry Retry-After."""
if e.code not in (403, 429):
return None
hdrs = e.headers
if hdrs is None:
return None
retry_after = hdrs.get("Retry-After")
if retry_after:
try:
return float(retry_after) + 1.0
except ValueError:
pass
if hdrs.get("X-RateLimit-Remaining") == "0":
reset = hdrs.get("X-RateLimit-Reset")
if reset:
try:
return max(0.0, float(reset) - time.time()) + 1.0
except ValueError:
pass
return None


def _rate_limit_give_up(url: str, wait: float) -> RuntimeError:
hint = ""
if not (os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN")):
hint = " — set GITHUB_TOKEN (or GH_TOKEN) to raise the GitHub API limit"
if wait > RATE_LIMIT_MAX_WAIT:
return RuntimeError(
f"rate limit on {url}: reset in {wait:.0f}s exceeds the "
f"{RATE_LIMIT_MAX_WAIT:.0f}s wait cap (raise it with "
f"UPDATE_PINS_RATE_LIMIT_MAX_WAIT, or set that to 0 to fail fast){hint}"
)
return RuntimeError(
f"rate limit on {url} still active after {MAX_ATTEMPTS} attempts{hint}"
)


def _open(url: str, headers: dict | None = None):
if not url.lower().startswith("https://"):
raise ValueError(f"refusing non-https URL: {url}")
hdrs = {"User-Agent": USER_AGENT}
if headers:
hdrs.update(headers)
return _OPENER.open(urllib.request.Request(url, headers=hdrs), timeout=30)
for attempt in range(1, MAX_ATTEMPTS + 1):
try:
return _OPENER.open(urllib.request.Request(url, headers=hdrs), timeout=30)
except urllib.error.HTTPError as e:
rl_wait = _rate_limit_wait(e)
if rl_wait is not None:
if rl_wait > RATE_LIMIT_MAX_WAIT or attempt == MAX_ATTEMPTS:
raise _rate_limit_give_up(url, rl_wait) from e
print(f" … {url}: HTTP {e.code} rate limited, waiting {rl_wait:.0f}s "
f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr)
_sleep(rl_wait)
continue
if (500 <= e.code < 600 or e.code == 429) and attempt < MAX_ATTEMPTS:
wait = BACKOFF_BASE * 2 ** (attempt - 1) # transient server error
print(f" … {url}: HTTP {e.code}, retrying in {wait:.0f}s "
f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr)
_sleep(wait)
continue
raise
except urllib.error.URLError as e:
if attempt == MAX_ATTEMPTS:
raise
wait = BACKOFF_BASE * 2 ** (attempt - 1)
print(f" … {url}: {e.reason}, retrying in {wait:.0f}s "
f"(attempt {attempt}/{MAX_ATTEMPTS})", file=sys.stderr)
_sleep(wait)
raise AssertionError("unreachable")


def http_bytes(url: str, headers: dict | None = None) -> bytes:
Expand Down
Loading