From 27ca53bb7a931e82de5fd0f89286b20565d13022 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Thu, 8 Oct 2026 17:16:02 -0400 Subject: [PATCH 1/2] Compress shared E2E artifacts as a single Zstandard stream --- .github/actions/upload-artifact/action.yml | 6 ++ .github/workflows/ci.yml | 30 ++++++- scripts/tests/test_ci_e2e_archive.py | 100 +++++++++++++++++++++ 3 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 scripts/tests/test_ci_e2e_archive.py diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index 1a6b0db85..96d80ecea 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -16,6 +16,9 @@ inputs: include-hidden-files: description: Include hidden files in the artifact default: 'false' + compression-level: + description: ZIP compression level (0 for files that are already compressed) + default: '6' outputs: artifact-id: description: ID of the uploaded artifact @@ -47,6 +50,7 @@ runs: if-no-files-found: ${{ inputs.if-no-files-found }} retention-days: ${{ inputs.retention-days }} include-hidden-files: ${{ inputs.include-hidden-files }} + compression-level: ${{ inputs.compression-level }} overwrite: true - name: Wait before retrying @@ -65,6 +69,7 @@ runs: if-no-files-found: ${{ inputs.if-no-files-found }} retention-days: ${{ inputs.retention-days }} include-hidden-files: ${{ inputs.include-hidden-files }} + compression-level: ${{ inputs.compression-level }} overwrite: true - name: Wait before the final attempt @@ -84,4 +89,5 @@ runs: if-no-files-found: ${{ inputs.if-no-files-found }} retention-days: ${{ inputs.retention-days }} include-hidden-files: ${{ inputs.include-hidden-files }} + compression-level: ${{ inputs.compression-level }} overwrite: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bebde5fc..b499fd983 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -892,10 +892,21 @@ jobs: BUNDLE_OS: ${{ matrix.os }} run: python3 scripts/ci-e2e-bundle.py --os "$BUNDLE_OS" --cargo-json target-build.json --dest target/e2e-bin + - name: Compress the e2e binaries + # Compress the bundle as one stream so identical Rust code across + # test binaries shares a dictionary. Zstd is on every runner image. + # The 128 MiB window cuts the Linux artifact from ~200 MB to ~94 MB; + # ZIP compresses each binary separately and repeats the common code. + shell: bash + run: | + set -euo pipefail + tar -C target/e2e-bin -cf - . | zstd -q -f -10 --long=27 -o target/e2e-bin.tar.zst + - uses: ./.github/actions/upload-artifact with: name: e2e-bin-${{ matrix.os }} - path: target/e2e-bin/ + path: target/e2e-bin.tar.zst + compression-level: 0 if-no-files-found: error retention-days: 3 @@ -1292,7 +1303,14 @@ jobs: with: pattern: e2e-bin-${{ matrix.os }}* merge-multiple: true - path: target/e2e-bin + path: target/e2e-archive + + - name: Unpack the e2e binaries + shell: bash + run: | + set -euo pipefail + mkdir -p target/e2e-bin + zstd -q -d -c target/e2e-archive/e2e-bin.tar.zst | tar -xf - -C target/e2e-bin - name: Stage the CLI where the test binaries expect it # `CARGO_BIN_EXE_socket-patch` was baked in at compile time as @@ -2096,7 +2114,13 @@ jobs: with: pattern: e2e-bin-${{ matrix.os }}* merge-multiple: true - path: target/e2e-bin + path: target/e2e-archive + - name: Unpack the e2e binaries + shell: bash + run: | + set -euo pipefail + mkdir -p target/e2e-bin + zstd -q -d -c target/e2e-archive/e2e-bin.tar.zst | tar -xf - -C target/e2e-bin - name: Install Rust run: rustup show - name: Install the cargo under test diff --git a/scripts/tests/test_ci_e2e_archive.py b/scripts/tests/test_ci_e2e_archive.py new file mode 100644 index 000000000..1cae12028 --- /dev/null +++ b/scripts/tests/test_ci_e2e_archive.py @@ -0,0 +1,100 @@ +"""Run CI's real archive commands and check binary/permission round trips.""" + +import importlib.util +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import tempfile +import textwrap +import unittest +import zipfile + +ROOT = Path(__file__).parents[2] +spec = importlib.util.spec_from_file_location("archive_rows", Path(__file__).with_name("test_ci_vlt_rows.py")) +rows = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rows) +JOBS = rows.jobs((ROOT / ".github/workflows/ci.yml").read_text()) + + +def commands(name): + scripts = [] + for job in JOBS.values(): + for step_name, step in rows.steps(job): + if step_name == name: + match = re.search(r"(?m)^ run: \|\n((?: .*\n|\n)+)", step + "\n") + scripts.append(textwrap.dedent(match[1])) + return scripts + + +def run(script, work): + return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script], + cwd=work, capture_output=True, text=True, timeout=30) + + +@unittest.skipUnless(shutil.which("zstd"), "zstd is required for the CI archive round trip") +class E2eArchive(unittest.TestCase): + def test_all_consumers_restore_every_binary_and_its_permissions(self): + pack = commands("Compress the e2e binaries") + unpack = commands("Unpack the e2e binaries") + self.assertTrue(pack) + self.assertTrue(unpack) + for compressor in pack: + for consumer in unpack: + with self.subTest(consumer=consumer), tempfile.TemporaryDirectory(prefix="e2e-archive-") as directory: + work = Path(directory) + bundle = work / "target/e2e-bin" + bundle.mkdir(parents=True) + # Both native Unix and Windows names must survive unchanged. + files = { + "socket-patch": (b"\x7fELF\x00cli fixture\xff", 0o755), + "socket-patch.exe": (b"MZ\x00windows fixture\xff", 0o755), + "e2e_redirect_gradle_build": (bytes(range(256)) * 4096, 0o555), + "e2e_vendor_jvm_build.exe": (b"MZ\x00test fixture\x80", 0o755), + } + for name, (payload, mode) in files.items(): + path = bundle / name + path.write_bytes(payload) + path.chmod(mode) + result = run(compressor, work) + self.assertEqual(result.returncode, 0, result.stderr) + archive = work / "target/e2e-bin.tar.zst" + # upload-artifact stores the compressed tar in a ZIP, then + # download-artifact extracts it into target/e2e-archive. + transport = work / "artifact.zip" + with zipfile.ZipFile(transport, "w", compression=zipfile.ZIP_STORED) as uploaded: + uploaded.write(archive, arcname=archive.name) + shutil.rmtree(bundle) + archive.unlink() + with zipfile.ZipFile(transport) as downloaded: + downloaded.extractall(work / "target/e2e-archive") + result = run(consumer, work) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual({path.name for path in bundle.iterdir()}, set(files)) + for name, (payload, mode) in files.items(): + path = bundle / name + self.assertEqual(path.read_bytes(), payload, name) + if os.name != "nt": + self.assertEqual(stat.S_IMODE(path.stat().st_mode), mode, name) + + def test_missing_bundle_fails_even_when_compressor_accepts_empty_input(self): + for compressor in commands("Compress the e2e binaries"): + with tempfile.TemporaryDirectory(prefix="e2e-no-bundle-") as directory: + work = Path(directory) + (work / "target").mkdir() + self.assertNotEqual(run(compressor, work).returncode, 0) + + def test_corrupt_archive_fails_the_consumer_step(self): + for consumer in commands("Unpack the e2e binaries"): + with tempfile.TemporaryDirectory(prefix="e2e-bad-archive-") as directory: + work = Path(directory) + archive_dir = work / "target/e2e-archive" + archive_dir.mkdir(parents=True) + (archive_dir / "e2e-bin.tar.zst").write_bytes(b"not a zstd archive") + self.assertNotEqual(run(consumer, work).returncode, 0) + + +if __name__ == "__main__": + unittest.main() From 77870a6c6bfc9514aeadac21c29d1c6ff34f85de Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 20:53:03 +0000 Subject: [PATCH 2/2] Make gem global-gemfile refusal test hermetic gem_hosted_global_gemfile_setting_is_refused only reached the redirect stage because the scan found the host's globally installed gems via `gem env`: the refused lock contributes no packages, so without an installed package no batch call fires and the refusal never runs. On Windows runners `gem env` sometimes outlives the 10s probe budget. The scan then reports scannedPackages: 0 and the test fails. This evicted two merge-queue entries on 2026-10-08 (#1147 and one at 17:31 UTC). Lay the gem down in the project with materialize_installed_gem, as the other tests in this file do, so the test no longer depends on the host's Ruby install. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01Xr7gMxM5ugBStCpk6kJ3V4 --- .../tests/e2e_redirect_gem_stale_install.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs b/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs index 7add4b6cd..597266ee2 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_gem_stale_install.rs @@ -976,6 +976,12 @@ async fn gem_hosted_global_gemfile_setting_is_refused() { let proj = tmp.path().join("proj"); std::fs::create_dir_all(&proj).unwrap(); write_manifest_pair(&proj); + // The refused lock contributes no packages, so the scan only reaches + // the redirect (and its refusal) through an installed copy of the gem. + // Lay one down in the project rather than leaning on whatever `gem env` + // reports for the host: on Windows that probe can outlive its 10s + // budget, the scan then finds nothing, and the refusal never runs. + materialize_installed_gem(&proj, "3.3.0", UPSTREAM_LIB); std::fs::write( proj.join("Gemfile.next"), format!("source \"https://rubygems.org\"\ngem \"{DEP}\", \"{DEP_VERSION}\"\n"),