From 997c5862a454f6fb2b148d2dfc78bd71ae364d3c Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 16:20:04 -0400 Subject: [PATCH 1/5] Shard release-mode CI tests across three runners Warm-cache release-mode CI still spends 20-23 minutes compiling roughly 240 test binaries. Reuse the existing test partitioner so each runner compiles and executes one slice, retaining the ci-release profile and aggregate merge gate. Validate profile propagation, failed-shard reporting, matrix completeness, and Cargo test selection with a real two-package fixture. All 268 Python harness tests pass (one skipped); actionlint has no new findings. --- .github/workflows/ci.yml | 24 ++++--- scripts/ci-test-shard.py | 9 +-- scripts/tests/test_ci_test_shard.py | 99 ++++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bebde5fc..032b749d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -412,16 +412,19 @@ jobs: python3 scripts/ci-test-shard.py "$TEST_SHARD" 2 test-release: - # Not in the merge queue: each PR's head already ran this, and so did - # every PR ahead of it, so a merge group would re-spend ~30 min (23 of it - # compiling) on the same release-mode suite while the queue waits. It - # still runs on every PR and on main after each merge; `ci-ok` counts a - # skipped job as passing. + # Each PR and main push runs the complete release-mode suite. The merge + # queue already skips this job because each constituent PR ran it. if: github.event.pull_request.draft != true && github.event_name != 'merge_group' + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] runs-on: ubuntu-latest - # Every tests/ target is its own optimized link (~240 test binaries). - # The manifest-less VEX suites share two multi-module binaries - # (tests/e2e_vex_lockfile/, tests/e2e_vex_build/) to keep the count down. + # Warm-cache runs still spend 20-23 minutes compiling ~240 optimized + # test binaries, followed by 5-6 minutes executing them. Split both + # compilation and execution with the same partitioner as `test`: + # shard 1 owns the unit tests/doctests and fewer integration targets. + # `ci-ok` waits for every shard and fails if any shard fails. timeout-minutes: 40 steps: - name: Checkout @@ -449,7 +452,10 @@ jobs: # semantics this job exists to validate; ~23m of LTO relinking gone. # Default features for the same reason as the `test` job; the # feature-gated suites' compile rot is e2e-build's. - run: cargo test --workspace --profile ci-release + env: + TEST_SHARD: ${{ matrix.shard }} + TEST_SHARD_COUNT: ${{ strategy.job-total }} + run: python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT" --locked --profile ci-release coverage: if: github.event.pull_request.draft != true diff --git a/scripts/ci-test-shard.py b/scripts/ci-test-shard.py index 6aed76b4d..05336ce9b 100644 --- a/scripts/ci-test-shard.py +++ b/scripts/ci-test-shard.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 -"""Run one shard of ci.yml's `test` job: `cargo test --workspace` split over -`COUNT` runners so the macOS / Windows legs stop being the merge queue's -critical path (one leg spent ~10 min linking ~240 test binaries and ~15 min -running them). +"""Run one shard of ci.yml's `test` or `test-release` job: +`cargo test --workspace` split over `COUNT` runners. Each runner compiles +and executes only its assigned integration targets, shortening both the +debug and release-mode jobs without changing their compilation profiles. Shard 1 runs the unit tests (`--lib --bins`, ~4 min of the run on Windows) and the doctests; the integration-test targets (`cargo metadata`, kind @@ -15,6 +15,7 @@ any of them failed. python3 scripts/ci-test-shard.py 1 2 + python3 scripts/ci-test-shard.py 1 3 --locked --profile ci-release """ import json diff --git a/scripts/tests/test_ci_test_shard.py b/scripts/tests/test_ci_test_shard.py index c35eaa61d..bb2d446e0 100644 --- a/scripts/tests/test_ci_test_shard.py +++ b/scripts/tests/test_ci_test_shard.py @@ -1,11 +1,15 @@ -"""ci-test-shard.py: the `test` job's shards together run exactly the old -single `cargo test --workspace` selection.""" +"""ci-test-shard.py: the debug/release shards preserve the workspace tests.""" import importlib.util import json +import os +import re +import shutil import subprocess +import tempfile import unittest from pathlib import Path +from unittest.mock import patch ROOT = Path(__file__).parents[2] spec = importlib.util.spec_from_file_location("ci_test_shard", ROOT / "scripts" / "ci-test-shard.py") @@ -49,6 +53,24 @@ def test_every_run_is_workspace_wide_and_keeps_going(self): self.assertEqual(args[:4], ["cargo", "test", "--workspace", "--no-fail-fast"]) self.assertIn("--locked", args) + def test_release_profile_is_kept_for_integration_unit_and_doc_tests(self): + for k in range(1, 4): + for args in shard.invocations(k, 3, self.NAMES, ["--locked", "--profile", "ci-release"]): + self.assertIn("--locked", args) + self.assertEqual(args[args.index("--profile") + 1], "ci-release") + + def test_failed_unit_or_integration_run_still_runs_docs_and_fails_the_shard(self): + metadata = {"workspace_members": ["a"], "packages": [ + {"id": "a", "targets": [{"name": "integration", "kind": ["test"]}]}]} + with patch.object(shard.subprocess, "run", side_effect=[ + subprocess.CompletedProcess([], 0, stdout=json.dumps(metadata)), + subprocess.CompletedProcess([], 1), + subprocess.CompletedProcess([], 0), + ]) as run: + self.assertEqual(shard.main(["1", "3", "--locked", "--profile", "ci-release"]), 1) + self.assertEqual(run.call_count, 3) + self.assertIn("--doc", run.call_args_list[-1].args[0]) + def test_negative_bad_shard(self): with self.assertRaises(ValueError): shard.invocations(3, 2, self.NAMES) @@ -76,5 +98,78 @@ def test_the_checkout_has_integration_targets(self): self.assertGreater(len(names), 100) +class ReleaseWorkflow(unittest.TestCase): + def test_all_release_shards_are_required_and_use_the_matrix_size(self): + # Read the configured matrix, rather than assuming the workflow kept + # the same shard count as this test. A missing shard silently loses + # tests; an unguarded aggregate can turn a failed matrix green. + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + release = workflow.split("\n test-release:\n")[1].split("\n coverage:\n")[0] + count = json.loads(re.search(r"^ shard: (\[.*\])$", release, re.M)[1]) + self.assertEqual(count, list(range(1, len(count) + 1))) + self.assertGreater(len(count), 1) + self.assertIn("TEST_SHARD: ${{ matrix.shard }}", release) + self.assertIn("TEST_SHARD_COUNT: ${{ strategy.job-total }}", release) + self.assertIn('python3 scripts/ci-test-shard.py "$TEST_SHARD" "$TEST_SHARD_COUNT" ' + '--locked --profile ci-release', release) + self.assertIn("fail-fast: false", release) + self.assertIn("github.event_name != 'merge_group'", release) + verdict = workflow.split("\n ci-ok:\n")[1] + needs = re.search(r"^ needs: \[(.*)\]$", verdict, re.M)[1].split(", ") + self.assertIn("test-release", needs) + self.assertIn("if: always()", verdict) + self.assertIn('if v["result"] not in ("success", "skipped")', verdict) + + +@unittest.skipUnless(shutil.which("cargo"), "cargo not available") +class CargoSelection(unittest.TestCase): + def test_three_release_shards_run_the_same_tests_as_cargo_workspace(self): + # Exercise Cargo, including duplicate target names across packages, + # binary/library units, ignored tests and doctests. This catches + # selector interactions that argument-list assertions cannot prove. + with tempfile.TemporaryDirectory(prefix="ci-release-shards-") as directory: + root = Path(directory) + files = { + "Cargo.toml": '[workspace]\nmembers=["one","two"]\nresolver="2"\n' + '[profile.ci-release]\ninherits="release"\nlto=false\n', + "one/Cargo.toml": '[package]\nname="one"\nversion="0.1.0"\nedition="2021"\n', + "one/src/lib.rs": '/// ```\n/// assert_eq!(one::answer(), 42);\n/// ```\n' + 'pub fn answer() -> u8 { 42 }\n' + '#[test] fn release_semantics() {\n' + ' assert!(!cfg!(debug_assertions));\n' + ' let n = std::hint::black_box(u8::MAX);\n' + ' assert_eq!(n + 1, 0);\n}\n', + "one/src/main.rs": 'fn main() {}\n#[test] fn binary_unit() {}\n', + "one/tests/shared.rs": '#[test] fn first_shared() {}\n' + '#[test] #[ignore] fn ignored_case() {}\n', + "one/tests/tail.rs": '#[test] fn tail_case() {}\n', + "two/Cargo.toml": '[package]\nname="two"\nversion="0.1.0"\nedition="2021"\n' + '[lib]\ntest=false\ndoctest=false\n', + "two/src/lib.rs": 'pub fn value() -> u8 { 1 }\n', + "two/tests/shared.rs": '#[test] fn second_shared() {}\n', + } + for name, content in files.items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + env = dict(os.environ, CARGO_TARGET_DIR=str(root / "target")) + + def run(args): + result = subprocess.run(args, cwd=root, env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + return result.stdout + + metadata = json.loads(run(["cargo", "metadata", "--offline", "--no-deps", "--format-version", "1"])) + run(["cargo", "generate-lockfile", "--offline"]) + extra = ["--offline", "--locked", "--profile", "ci-release"] + baseline = run(["cargo", "test", "--workspace", *extra]) + actual = "\n".join(run(args) for k in range(1, 4) + for args in shard.invocations(k, 3, shard.integration_targets(metadata), extra)) + pattern = re.compile(r"^test (.+) \.\.\. (ok|ignored)$", re.M) + expected = pattern.findall(baseline) + self.assertGreaterEqual(len(expected), 7) + self.assertCountEqual(pattern.findall(actual), expected) + + if __name__ == "__main__": unittest.main() From 398b5d122e53258c13b30ed14a8bf89fa8e05413 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 16:38:12 -0400 Subject: [PATCH 2/5] Fall back to Apache when Maven Central downloads fail PR CI lost a Gradle vendor leg after Maven Central returned six 404s for the pinned Maven tarball. Keep Central as the fast path, fall back to the original Apache archive tarball, and verify either download with the existing SHA-512 before extraction. Exercise both workflow steps with primary success, fallback success, corrupt fallback bytes and two unavailable origins, covering Linux and Windows launcher selection. All 16 cases and 264 harness tests pass (one skipped); actionlint has no new findings. --- .github/workflows/ci.yml | 10 +++++++--- .github/workflows/gradle-compatibility.yml | 6 +++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 032b749d4..95015280b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1508,8 +1508,9 @@ jobs: # archive.apache.org throttles bulk downloads to 1.5-5.5 minutes per # leg. Its sha512 still comes from the Apache archive (Central has # none for 3.6.3/3.8.9), so the bytes are checked against a digest - # from a second origin. --ssl-revoke-best-effort: see the `test` - # job's vexctl step. + # from a second origin. If the CDN fails, use the slower archive + # tarball rather than losing a whole CI run to a download outage. + # --ssl-revoke-best-effort: see the `test` job's vexctl step. shell: bash env: MAVEN_VERSION: ${{ matrix.maven || '3.9.16' }} @@ -1518,7 +1519,10 @@ jobs: file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" sha_url="https://archive.apache.org/dist/maven/maven-${major}/${MAVEN_VERSION}/binaries/${file}.sha512" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz" + if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then + echo "::warning::Maven Central download failed; falling back to the Apache archive." + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "${sha_url%.sha512}" -o "$RUNNER_TEMP/maven.tgz" + fi curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' # Python accepts native Windows paths for both archive and destination. diff --git a/.github/workflows/gradle-compatibility.yml b/.github/workflows/gradle-compatibility.yml index 70081afb1..fc3641f84 100644 --- a/.github/workflows/gradle-compatibility.yml +++ b/.github/workflows/gradle-compatibility.yml @@ -258,10 +258,14 @@ jobs: # the body is checked against a digest right after. A no-op elsewhere. # Tarball from Maven Central's CDN, digest from the Apache archive, # which throttles the tarball itself to minutes (ci.yml's copy). + # Keep the archive tarball as a fallback for CDN failures. file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" sha_url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${file}.sha512" - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz" + if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then + echo "::warning::Maven Central download failed; falling back to the Apache archive." + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "${sha_url%.sha512}" -o "$RUNNER_TEMP/maven.tgz" + fi curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' python -m tarfile -e "$RUNNER_TEMP/maven.tgz" "$RUNNER_TEMP" From e903d7f64dd370f9260b8dd0014763fc133d4740 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 16:59:12 -0400 Subject: [PATCH 3/5] Pin Maven download checksums before using either mirror --- .github/workflows/ci.yml | 22 ++-- .github/workflows/gradle-compatibility.yml | 19 +-- scripts/maven-sha512.json | 9 ++ scripts/tests/test_ci_maven_download.py | 130 +++++++++++++++++++++ 4 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 scripts/maven-sha512.json create mode 100644 scripts/tests/test_ci_maven_download.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95015280b..0f80d9ce4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1506,10 +1506,9 @@ jobs: # The exact release a leg names rather than the runner's Maven. The # tarball comes from Maven Central's CDN (well under a second); # archive.apache.org throttles bulk downloads to 1.5-5.5 minutes per - # leg. Its sha512 still comes from the Apache archive (Central has - # none for 3.6.3/3.8.9), so the bytes are checked against a digest - # from a second origin. If the CDN fails, use the slower archive - # tarball rather than losing a whole CI run to a download outage. + # leg. If the CDN fails, use the slower archive tarball. Both + # sources must match the committed SHA512 in scripts/maven-sha512.json; + # an origin cannot replace both the archive and its expected digest. # --ssl-revoke-best-effort: see the `test` job's vexctl step. shell: bash env: @@ -1518,13 +1517,20 @@ jobs: major="${MAVEN_VERSION%%.*}" file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" - sha_url="https://archive.apache.org/dist/maven/maven-${major}/${MAVEN_VERSION}/binaries/${file}.sha512" + archive_url="https://archive.apache.org/dist/maven/maven-${major}/${MAVEN_VERSION}/binaries/${file}" if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then echo "::warning::Maven Central download failed; falling back to the Apache archive." - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "${sha_url%.sha512}" -o "$RUNNER_TEMP/maven.tgz" + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$archive_url" -o "$RUNNER_TEMP/maven.tgz" fi - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" - python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' + python - "$MAVEN_VERSION" "$RUNNER_TEMP/maven.tgz" <<'PY' + import hashlib, json, sys + from pathlib import Path + # Pins come from Apache's release checksums, cross-checked against + # the Maven Central tarballs. Add a pin when adding a matrix version. + expected = json.loads(Path("scripts/maven-sha512.json").read_text())[sys.argv[1]] + if hashlib.sha512(Path(sys.argv[2]).read_bytes()).hexdigest() != expected: + raise SystemExit("Maven archive SHA512 mismatch") + PY # Python accepts native Windows paths for both archive and destination. python -m tarfile -e "$RUNNER_TEMP/maven.tgz" "$RUNNER_TEMP" launcher="$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin/mvn" diff --git a/.github/workflows/gradle-compatibility.yml b/.github/workflows/gradle-compatibility.yml index fc3641f84..e83199e9f 100644 --- a/.github/workflows/gradle-compatibility.yml +++ b/.github/workflows/gradle-compatibility.yml @@ -44,6 +44,7 @@ on: paths: - '.github/workflows/gradle-compatibility.yml' - 'scripts/ci-e2e-bundle.py' + - 'scripts/maven-sha512.json' - 'Cargo.lock' - 'Cargo.toml' - 'crates/*/Cargo.toml' @@ -256,18 +257,22 @@ jobs: # TLS handshake with CRYPT_E_REVOCATION_OFFLINE whenever the CA's # revocation server is unreachable. A revoked certificate still fails; # the body is checked against a digest right after. A no-op elsewhere. - # Tarball from Maven Central's CDN, digest from the Apache archive, - # which throttles the tarball itself to minutes (ci.yml's copy). - # Keep the archive tarball as a fallback for CDN failures. + # Use Maven Central's fast CDN, falling back to the slower Apache + # archive. Both must match the committed digest (ci.yml's copy). file="apache-maven-${MAVEN_VERSION}-bin.tar.gz" url="https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/${MAVEN_VERSION}/${file}" - sha_url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${file}.sha512" + archive_url="https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/${file}" if ! curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$url" -o "$RUNNER_TEMP/maven.tgz"; then echo "::warning::Maven Central download failed; falling back to the Apache archive." - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "${sha_url%.sha512}" -o "$RUNNER_TEMP/maven.tgz" + curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$archive_url" -o "$RUNNER_TEMP/maven.tgz" fi - curl -fsSL --retry 5 --retry-all-errors --ssl-revoke-best-effort "$sha_url" -o "$RUNNER_TEMP/maven.sha512" - python -c 'import hashlib, pathlib, os; p=pathlib.Path(os.environ["RUNNER_TEMP"]); assert hashlib.sha512((p/"maven.tgz").read_bytes()).hexdigest() == (p/"maven.sha512").read_text().split()[0]' + python - "$MAVEN_VERSION" "$RUNNER_TEMP/maven.tgz" <<'PY' + import hashlib, json, sys + from pathlib import Path + expected = json.loads(Path("scripts/maven-sha512.json").read_text())[sys.argv[1]] + if hashlib.sha512(Path(sys.argv[2]).read_bytes()).hexdigest() != expected: + raise SystemExit("Maven archive SHA512 mismatch") + PY python -m tarfile -e "$RUNNER_TEMP/maven.tgz" "$RUNNER_TEMP" launcher="$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin/mvn" if [ "$RUNNER_OS" = Windows ]; then launcher="${launcher}.cmd"; fi diff --git a/scripts/maven-sha512.json b/scripts/maven-sha512.json new file mode 100644 index 000000000..066ab5605 --- /dev/null +++ b/scripts/maven-sha512.json @@ -0,0 +1,9 @@ +{ + "3.6.3": "c35a1803a6e70a126e80b2b3ae33eed961f83ed74d18fcd16909b2d44d7dada3203f1ffe726c17ef8dcca2dcaa9fca676987befeadc9b9f759967a8cb77181c0", + "3.8.9": "4a490b7f331a0e7869b61da24600241e445339f2801ed94e32f835b63ed78597ad05ef8c1cce2501b4c2c3dcde30030eb395cd5756be739c20ac687ad6f82f0e", + "3.9.2": "900bdeeeae550d2d2b3920fe0e00e41b0069f32c019d566465015bdd1b3866395cbe016e22d95d25d51d3a5e614af2c83ec9b282d73309f644859bbad08b63db", + "3.9.3": "400fc5b6d000c158d5ee7937543faa06b6bda8408caa2444a9c947c21472fde0f0b64ac452b8cec8855d528c0335522ed5b6c8f77085811c7e29e1bedbb5daa2", + "3.9.4": "deaa39e16b2cf20f8cd7d232a1306344f04020e1f0fb28d35492606f647a60fe729cc40d3cba33e093a17aed41bd161fe1240556d0f1b80e773abd408686217e", + "3.9.16": "831a8591fe20c8243b1dbe7d71e3244f31d1665b0804b2e825e38cbbe5ce0cafb8338851f90780735568773e0a6cd07bbec107cda0b896b008b861075358b6f6", + "4.0.0-rc-6": "3fba58e1c345a5aa1dbacfa7aceaf7b1a0fa9626e368eec4814fa7a7ebf0fe74f0e41481faef77f95d8738f9c1365f918c8b8c94d7c28656f067db61a8af7f2e" +} diff --git a/scripts/tests/test_ci_maven_download.py b/scripts/tests/test_ci_maven_download.py new file mode 100644 index 000000000..033f8269f --- /dev/null +++ b/scripts/tests/test_ci_maven_download.py @@ -0,0 +1,130 @@ +"""Exercise the workflow installers against a fake CDN and Apache archive. + +The archive bytes must match the committed pin before extraction, even if +the download origin serves a matching checksum for substituted bytes. +""" + +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tarfile +import tempfile +import textwrap +import unittest + +ROOT = Path(__file__).parents[2] +WORKFLOWS = ("ci.yml", "gradle-compatibility.yml") +VERSION = "3.9.16" +spec = importlib.util.spec_from_file_location("ci_maven_rows", Path(__file__).with_name("test_ci_vlt_rows.py")) +rows = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rows) + + +def installer(workflow): + jobs = rows.jobs((ROOT / ".github/workflows" / workflow).read_text()) + step = next(text for job in jobs.values() for name, text in rows.steps(job) + if name.startswith("Install Maven")) + match = re.search(r"(?m)^ run: \|\n((?: .*\n|\n)+)", step + "\n") + return textwrap.dedent(match[1]) + + +def archive_bytes(contents): + data = io.BytesIO() + with tarfile.open(fileobj=data, mode="w:gz") as archive: + for name in ("mvn", "mvn.cmd"): + info = tarfile.TarInfo(f"apache-maven-{VERSION}/bin/{name}") + info.size = len(contents) + info.mode = 0o755 + archive.addfile(info, io.BytesIO(contents)) + return data.getvalue() + + +CURL = r''' +import hashlib, os, pathlib, sys +root = pathlib.Path(os.environ["FIXTURE_ROOT"]) +mode = os.environ["FIXTURE_MODE"] +url = next(a for a in sys.argv if a.startswith("https://")) +dest = pathlib.Path(sys.argv[sys.argv.index("-o") + 1]) +with (root / "requests").open("a") as log: + log.write(url + "\n") +if mode == "both-fail" or ("repo.maven.apache.org" in url and mode in ("fallback", "corrupt-fallback")): + dest.write_bytes(b"partial download") + sys.exit(22) +payload = (root / ("tampered.tgz" if mode.startswith("corrupt-") else "fixture.tgz")).read_bytes() +if url.endswith(".sha512"): + # An origin can replace both its tarball and its online digest. + payload = hashlib.sha512(payload).hexdigest().encode() +dest.write_bytes(payload) +''' + + +class MavenDownload(unittest.TestCase): + def test_every_matrix_version_has_a_sha512_pin(self): + pins = json.loads((ROOT / "scripts/maven-sha512.json").read_text()) + for workflow in WORKFLOWS: + text = (ROOT / ".github/workflows" / workflow).read_text() + versions = re.findall(r"(?:maven|MAVEN_VERSION): '([^']+)'", text) + self.assertTrue(versions) + self.assertFalse(set(versions) - pins.keys(), f"unpinned Maven version in {workflow}") + for version, digest in pins.items(): + self.assertRegex(digest, r"^[0-9a-f]{128}$", version) + + def test_workflow_downloads_fail_closed_before_extraction(self): + for workflow in WORKFLOWS: + script = installer(workflow) + for runner_os in ("Linux", "Windows"): + for mode in ("primary", "fallback", "corrupt-primary", "corrupt-fallback", "both-fail", "unpinned"): + with self.subTest(workflow=workflow, runner_os=runner_os, mode=mode): + self.check_installer(script, workflow, runner_os, mode) + + def check_installer(self, script, workflow, runner_os, mode): + with tempfile.TemporaryDirectory(prefix="maven-download-") as directory: + work = Path(directory) + trusted = archive_bytes(b"trusted launcher\n") + (work / "fixture.tgz").write_bytes(trusted) + (work / "tampered.tgz").write_bytes(archive_bytes(b"substituted launcher\n")) + (work / "scripts").mkdir() + pins = {} if mode == "unpinned" else {VERSION: hashlib.sha512(trusted).hexdigest()} + (work / "scripts/maven-sha512.json").write_text(json.dumps(pins)) + (work / "curl").write_text("#!" + sys.executable + "\n" + CURL) + (work / "curl").chmod(0o755) + (work / "python").symlink_to(sys.executable) + github_env = work / "github-env" + github_env.touch() + env = dict(os.environ, PATH=str(work) + os.pathsep + os.environ["PATH"], + FIXTURE_ROOT=str(work), FIXTURE_MODE=mode, RUNNER_TEMP=str(work), + RUNNER_OS=runner_os, MAVEN_VERSION=VERSION, GITHUB_ENV=str(github_env), + PYTHONOPTIMIZE="1") # Verification must not rely on assert. + result = subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=work, env=env, capture_output=True, text=True, timeout=30) + requests = (work / "requests").read_text().splitlines() + self.assertTrue(requests[0].startswith("https://repo.maven.apache.org/")) + self.assertFalse(any(url.endswith(".sha512") for url in requests), requests) + fallback = mode in ("fallback", "corrupt-fallback", "both-fail") + self.assertEqual(len(requests), 2 if fallback else 1, requests) + if fallback: + self.assertEqual(requests[1], f"https://archive.apache.org/dist/maven/maven-3/{VERSION}/binaries/apache-maven-{VERSION}-bin.tar.gz") + if mode in ("primary", "fallback"): + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + launcher = work / f"apache-maven-{VERSION}/bin" / ("mvn.cmd" if runner_os == "Windows" else "mvn") + self.assertEqual(launcher.read_bytes(), b"trusted launcher\n") + self.assertIn(f"SOCKET_PATCH_MAVEN_E2E_MVN={launcher}\n", github_env.read_text()) + if workflow == "gradle-compatibility.yml": + self.assertIn(f"SOCKET_PATCH_MAVEN_E2E_VERSION={VERSION}\n", github_env.read_text()) + self.assertIn("SOCKET_PATCH_MAVEN_E2E_REQUIRED=1\n", github_env.read_text()) + else: + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse((work / f"apache-maven-{VERSION}").exists()) + self.assertFalse(github_env.read_text()) + if mode.startswith("corrupt-"): + self.assertIn("SHA512 mismatch", result.stderr) + + +if __name__ == "__main__": + unittest.main() From e78ad365003460e82016324eba3e49299d4cc535 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 17:26:47 -0400 Subject: [PATCH 4/5] Avoid the release-job insertion conflict with CI scheduling --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f80d9ce4..106f591a6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -415,11 +415,11 @@ jobs: # Each PR and main push runs the complete release-mode suite. The merge # queue already skips this job because each constituent PR ran it. if: github.event.pull_request.draft != true && github.event_name != 'merge_group' + runs-on: ubuntu-latest strategy: fail-fast: false matrix: shard: [1, 2, 3] - runs-on: ubuntu-latest # Warm-cache runs still spend 20-23 minutes compiling ~240 optimized # test binaries, followed by 5-6 minutes executing them. Split both # compilation and execution with the same partitioner as `test`: From 5c780cf8a7ac2cedab6140e608b0dc2c3e77c56b Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 17:54:55 -0400 Subject: [PATCH 5/5] Keep Maven pin path filter separate from queued Gradle installer --- .github/workflows/gradle-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gradle-compatibility.yml b/.github/workflows/gradle-compatibility.yml index e83199e9f..847ba7521 100644 --- a/.github/workflows/gradle-compatibility.yml +++ b/.github/workflows/gradle-compatibility.yml @@ -42,9 +42,9 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] paths: + - 'scripts/maven-sha512.json' - '.github/workflows/gradle-compatibility.yml' - 'scripts/ci-e2e-bundle.py' - - 'scripts/maven-sha512.json' - 'Cargo.lock' - 'Cargo.toml' - 'crates/*/Cargo.toml'