diff --git a/.github/actions/ci-fixtures/action.yml b/.github/actions/ci-fixtures/action.yml index acdc075ff..5e2efcda3 100644 --- a/.github/actions/ci-fixtures/action.yml +++ b/.github/actions/ci-fixtures/action.yml @@ -12,8 +12,10 @@ runs: support=$(python3 -I -B -c 'import sys; sys.path.insert(0, "scripts"); from test_oracle import SUPPORT; print(" ".join("scripts/" + name for name in SUPPORT))') identity="$(cat scripts/gen-fixtures.sh scripts/install-oracle-ffmpeg.sh scripts/oracle-ffmpeg.env $helpers $support | sha256sum | cut -c1-32)" composed="$(cat docs/design/composed-opus-aac-v2.json scripts/composed_audio.py scripts/composed_audio_fixtures.py scripts/install-oracle-ffmpeg.sh $support | sha256sum | cut -c1-32)" + comparator="$(python3 -I -c 'import json; print(json.load(open("docs/design/composed-opus-aac-v2.json"))["opus_compare"]["sha256"])')" echo "key=linux-x64-fixtures-$identity" >> "$GITHUB_OUTPUT" echo "composed=linux-x64-composed-$composed" >> "$GITHUB_OUTPUT" + echo "comparator=comparator-archive-$comparator" >> "$GITHUB_OUTPUT" - name: Restore the fixtures id: fixtures uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 @@ -26,6 +28,12 @@ runs: with: path: ${{ runner.temp }}/coreav-composed-fixtures key: ${{ steps.key.outputs.composed }} + - name: Cache the pinned comparator archive + if: steps.composed.outputs.cache-hit != 'true' + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-comparator-archive + key: ${{ steps.key.outputs.comparator }} - name: Cache pinned black-box binaries if: steps.fixtures.outputs.cache-hit != 'true' || steps.composed.outputs.cache-hit != 'true' id: ffmpeg-cache @@ -48,14 +56,45 @@ runs: python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" export PATH="$RUNNER_TEMP/coreav-oracle-bin:$PATH" scripts/gen-fixtures.sh "$RUNNER_TEMP/coreav-fixtures" + - name: Fetch the pinned comparator archive on a cache miss + if: steps.composed.outputs.cache-hit != 'true' + shell: bash + run: | + set -euo pipefail + read -r url digest < <(python3 -I -c 'import json; pin = json.load(open("docs/design/composed-opus-aac-v2.json"))["opus_compare"]; print(pin["url"], pin["sha256"])') + archive="$RUNNER_TEMP/coreav-comparator-archive/opus.tar.gz" + if [ -f "$archive" ] && ! echo "$digest $archive" | sha256sum --check --status -; then + echo "::warning::the cached comparator archive has another digest; fetching it again" + rm -f "$archive" + fi + if [ ! -f "$archive" ]; then + download="$RUNNER_TEMP/comparator-archive.download" + status=0 + curl --fail --silent --show-error --location --proto =https --proto-redir =https --connect-timeout 15 \ + --max-time 120 --max-filesize 10485760 --retry 3 --retry-all-errors --retry-delay 5 \ + --output "$download" "$url" || status=$? + if [ "$status" -ne 0 ] || ! echo "$digest $download" | sha256sum --check --status -; then + echo "::warning::comparator archive fetch failed (curl exit $status); the generator downloads it itself" + rm -f "$download" + else + mkdir -p "${archive%/*}" + mv "$download" "$archive" + fi + fi + if [ -f "$archive" ]; then + mkdir -p "$RUNNER_TEMP/coreav-comparator-curl" + cp .github/actions/ci-fixtures/comparator-curl.sh "$RUNNER_TEMP/coreav-comparator-curl/curl" + chmod +x "$RUNNER_TEMP/coreav-comparator-curl/curl" + fi - name: Generate the registered composed-audio fixtures if: steps.composed.outputs.cache-hit != 'true' shell: bash run: | url=$(python3 -I -c 'import json; print(json.load(open("docs/design/composed-opus-aac-v2.json"))["opus_compare"]["url"])') + export COREAV_COMPARATOR_URL="$url" COREAV_COMPARATOR_ARCHIVE="$RUNNER_TEMP/coreav-comparator-archive/opus.tar.gz" rm -rf "$RUNNER_TEMP/coreav-composed-fixtures" for attempt in 1 2 3; do - if python3 -I scripts/composed_audio_fixtures.py \ + if env PATH="$RUNNER_TEMP/coreav-comparator-curl:$PATH" python3 -I scripts/composed_audio_fixtures.py \ --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" --out "$RUNNER_TEMP/coreav-composed-fixtures"; then exit 0 fi diff --git a/.github/actions/ci-fixtures/comparator-curl.sh b/.github/actions/ci-fixtures/comparator-curl.sh new file mode 100755 index 000000000..bdf3ccab2 --- /dev/null +++ b/.github/actions/ci-fixtures/comparator-curl.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Stands in for curl while scripts/composed_audio_fixtures.py builds the +# comparator: the request for the registered comparator archive is answered +# from the Actions cache, and the generator's own digest check still judges +# the bytes. Every other request is refused, so a changed address fails at +# once instead of reaching the network unnoticed. +set -euo pipefail +target= +while [ "$#" -gt 1 ]; do + case "$1" in + -o|--output) target="$2"; shift ;; + esac + shift +done +if [ "$#" -ne 1 ] || [ "$1" != "$COREAV_COMPARATOR_URL" ] || [ -z "$target" ]; then + echo 'comparator stand-in for curl: only the registered comparator archive is served' >&2 + exit 2 +fi +cp "$COREAV_COMPARATOR_ARCHIVE" "$target" diff --git a/.github/workflows/audio-conformance.yml b/.github/workflows/audio-conformance.yml index 1988f89ec..a88cae6e7 100644 --- a/.github/workflows/audio-conformance.yml +++ b/.github/workflows/audio-conformance.yml @@ -505,7 +505,8 @@ jobs: OPUS_VECTORS_URL: https://www.ietf.org/proceedings/98/slides/materials-98-codec-opus-newvectors-00.tar.gz OPUS_VECTORS_SHA256: 6b26a22f9ba87b2b836906a9bb7afec5f8e54d49553b1200382520ee6fedfa55 # opus_compare from libopus 1.6.1. Built and run as a black box: its - # source is compiled, never read, and deleted. + # source is compiled, never read, and deleted. The archive is cached by + # this digest, under the key the ci-fixtures action uses for the same file. LIBOPUS_URL: https://downloads.xiph.org/releases/opus/opus-1.6.1.tar.gz LIBOPUS_SHA256: 6ffcb593207be92584df15b32466ed64bbec99109f007c82205f0194572411a1 steps: @@ -534,16 +535,29 @@ jobs: echo "$OPUS_VECTORS_SHA256 vectors.tar.gz" | sha256sum -c - tar xzf vectors.tar.gz rm vectors.tar.gz + - name: Cache the pinned comparator archive + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-comparator-archive + key: comparator-archive-${{ env.LIBOPUS_SHA256 }} - name: Build opus_compare run: | set -euo pipefail + archive="$RUNNER_TEMP/coreav-comparator-archive/opus.tar.gz" + if [ -f "$archive" ] && ! echo "$LIBOPUS_SHA256 $archive" | sha256sum --check --status -; then + rm -f "$archive" + fi + if [ ! -f "$archive" ]; then + curl -sSfL --retry 3 --retry-all-errors --retry-delay 5 -o "$RUNNER_TEMP/libopus.tar.gz" "$LIBOPUS_URL" + echo "$LIBOPUS_SHA256 $RUNNER_TEMP/libopus.tar.gz" | sha256sum -c - + mkdir -p "${archive%/*}" + mv "$RUNNER_TEMP/libopus.tar.gz" "$archive" + fi mkdir -p "$RUNNER_TEMP/opus-compare" cd "$RUNNER_TEMP/opus-compare" - curl -sSfL --retry 3 --retry-all-errors --retry-delay 5 -o libopus.tar.gz "$LIBOPUS_URL" - echo "$LIBOPUS_SHA256 libopus.tar.gz" | sha256sum -c - - tar xzf libopus.tar.gz opus-1.6.1/src/opus_compare.c + tar xzf "$archive" opus-1.6.1/src/opus_compare.c cc -O2 -o opus_compare opus-1.6.1/src/opus_compare.c -lm - rm -rf libopus.tar.gz opus-1.6.1 + rm -rf opus-1.6.1 - name: The generated tables are what scripts/opus_tables.py makes from the RFC's code run: python3 -I scripts/opus_tables.py --check --cache "$RUNNER_TEMP/pinned-sources" - name: Read every packet and decode the twelve vectors (durations against the reference outputs, range state of every packet, opus_compare above 90) diff --git a/docs/agents/ci.md b/docs/agents/ci.md index ccbdb9956..03fc4aeaa 100644 --- a/docs/agents/ci.md +++ b/docs/agents/ci.md @@ -46,3 +46,4 @@ Workflow required checks, path filters, DCO, pinned actions, merge-group reuse, | Design notes change in the same pull request as the design | test_videotoolbox_owner_docs_do_not_describe_it_as_unimplemented rejects former preimplementation wording for the implemented VideoToolbox owner and requires the plan's explicit CoreAV-to-FFmpeg payload ratio and inclusive bounds (deny.yml). Other design consistency is not enforced: judgment. Rust types cannot establish the meaning of external prose | | A change of an oracle platform's pin or installer starts every workflow that runs that installer | `scripts/workflow_check.py` (`deny.yml`) checks the platform-specific input lists, including the macOS verifier and its frozen Linux imports. `test_macos_installers_track_only_their_own_required_inputs` checks each macOS input and nested shell calls. A job-level filter (`changed-paths`) is review judgment | | Fixture cache identities and fixture-consuming workflow triggers include every shared qualification support input, not only the directly called script | The existing `test_oracle.SUPPORT` owner supplies both cache keys; `test_both_fixture_cache_keys_bind_every_owner_support_file` executes both key blocks, changes every support file and rejects missing inputs. `test_all_fixture_support_changes_start_both_wasm_events` and `test_release_and_avi_triggers_cover_qualification_and_fixture_inputs` check the WASM, release and AVI filters against that owner and the generators they run. Workflow path filters cannot consume a Python owner at parse time, so checks prevent drift | +| The pinned comparator archive comes from an Actions cache keyed by its registered digest on every branch, only digest-verified bytes enter that cache, and the generator's own digest check still judges what it is served | `test_comparator_archive_is_cached_by_its_registered_digest_on_every_branch` checks both cache steps' key and path against the registration and that neither save is limited to main; `test_fetch_keeps_only_a_digest_verified_archive_and_installs_the_stand_in_beside_it`, `test_stand_in_serves_only_the_generators_request_for_the_registered_archive` and `test_audio_conformance_builds_the_comparator_from_the_cached_archive` execute the shell blocks and the stand-in with a stand-in curl (`deny.yml`). A YAML type cannot govern what a cache step saves | diff --git a/docs/design/ci-checks.md b/docs/design/ci-checks.md index bfa323563..5b51a0fb7 100644 --- a/docs/design/ci-checks.md +++ b/docs/design/ci-checks.md @@ -423,6 +423,16 @@ key is one entry per job id, toolchain and `Cargo.lock`. script itself, so a new helper joins the key without an edit here) and `scripts/install-oracle-ffmpeg.sh`. A pull request that changes one of them generates the fixtures on every push. +- **The comparator archive is cached by its digest on every branch** + (`comparator-archive-`, the digest the composed-audio + registration pins). `ci-changes` restores it before generating the + composed-audio fixtures and answers the generator's download of the + registered address from it; the `opus` job of `audio-conformance.yml` + restores the same entry before compiling `opus_compare`. Only an archive + whose digest matched enters the entry, and the generator checks the + digest again on what it is served. A pull request that changes the + composed sources no longer downloads the archive from its publisher on + every push. - **The Matroska and transport stream oracles restore `isobmff-fixtures`' cache** (`shared-key: isobmff-fixtures`, `save-if: false`). That job builds every crate, profile and feature the other two build, so their diff --git a/docs/design/oracle-invocation-dispositions.json b/docs/design/oracle-invocation-dispositions.json index 834bcf50c..64da2c0ad 100644 --- a/docs/design/oracle-invocation-dispositions.json +++ b/docs/design/oracle-invocation-dispositions.json @@ -1,9 +1,9 @@ { "dependencies": { - ".github/actions/ci-fixtures/action.yml": "5d9e14cc90f259012d7b0c2be7a48f4458073734f56f1742c7c48dd308990d97", + ".github/actions/ci-fixtures/action.yml": "af2f2f64163519fb4e3282610f4c180a1d327e2e95ed8be0f06bb10378a27192", ".github/scripts/wasm-memory-failure.cjs": "75a688444375c7ec306718cb494d2235c9e418daa4211f88833961823a772614", ".github/workflows/aac-encode.yml": "922a43d86c9c29f5637f31611182f959011d974b727a3b5c16e4aa70f8d00105", - ".github/workflows/audio-conformance.yml": "4c093ef6fa807dc536a0e10006e0c0370460d5d253e8bf41dc58f3ee37eb0cbe", + ".github/workflows/audio-conformance.yml": "ad5c141b44a3984b801aeca6d17a6af68ec04e3bc368d8b4f53fb1ee1e012d67", ".github/workflows/ci.yml": "5f98113f86b38699e1efac1e2b8414918ff78e8c0a2c51d55dc195847f192b68", ".github/workflows/codec-coverage.yml": "141a7b5d078c18d85728b2d85f0a3cc7e4ba7753f02823a3eb0c4cfcf655e34a", ".github/workflows/compat-0.1.0.yml": "b979061433995d3a504edddac660216fefbdef556080967f7facf76c89eb1a80", @@ -1623,6 +1623,7 @@ "scripts/tests/test_av1_measurement.py": "e188bed6a77662db3bd86b285dff0be267fdcd87788e029ae1a0d99b3ba073f6", "scripts/tests/test_bar_sources.py": "7cf60c7f31475d1188bfe33fdf87671c166a505ed1b42bdc9c1c9467de1b5936", "scripts/tests/test_benchlib_oracle.py": "d19a0eddaaa405a2b4037a99604af90a6c1288b76e2b5ef3c6cf9517c0f843a1", + "scripts/tests/test_ci_fixtures.py": "121603c788ef77f07e226555d17bef86f6f1da751f0c6d62138119a27930eb82", "scripts/tests/test_codec_coverage_gcp.py": "7dc9311de22ac97a63c39af25b1890e7eeaa673c279c64ed9e7534de12d001ef", "scripts/tests/test_compat_sources.py": "932eee2a3ea0a2a98615d7acce880b745095845458cea6c89a6ae7285c867796", "scripts/tests/test_composed_audio.py": "923ab3ba8adc5d33055a743ebf922d65b37467d17250d583feec285938542cd0", @@ -3104,16 +3105,16 @@ "tools/stage3-report/report.py", "tools/stage3-report/run.py" ], - "sha256": "5d9e14cc90f259012d7b0c2be7a48f4458073734f56f1742c7c48dd308990d97", + "sha256": "af2f2f64163519fb4e3282610f4c180a1d327e2e95ed8be0f06bb10378a27192", "sites": [ { "class": "reviewed-reference", - "evidence": ".github/actions/ci-fixtures/action.yml:58 (shell, occurrence 0). .github/actions/ci-fixtures/action.yml:55-69 retries the same fixture entry point at most three times, removing its output once before the loop at .github/actions/ci-fixtures/action.yml:56 and after each failed generator attempt at .github/actions/ci-fixtures/action.yml:62, including the final failure; .github/actions/ci-fixtures/action.yml:58-60 returns immediately on generator success, preserving its output. .github/actions/ci-fixtures/action.yml:58-59 supplies only the requested ffmpeg selector. scripts/composed_audio_fixtures.py:78-89 parses that selector, calls test_oracle.require(args.ffmpeg) at scripts/composed_audio_fixtures.py:86, and passes pair.ffmpeg unchanged to generate at scripts/composed_audio_fixtures.py:89. scripts/composed_audio_fixtures.py:53-75 uses that ffmpeg as argv[0] in every generation call; scripts/composed_audio.py:326-334 passes the same command through list(map(str, cmd)) into subprocess.Popen. scripts/test_oracle.py:18-21 defines the immutable pair; scripts/test_oracle.py:73-108 resolves both executable paths, verifies the complete selection with the platform pin owner and returns that pair. scripts/oracle_pin.py:108-121 verifies both pinned version banners; scripts/oracle_pin_macos.py:48-78 also checks binary digests, architecture and signing before accepting the pair. The failure diagnostic at .github/actions/ci-fixtures/action.yml:64-65 is a 0-1023 byte curl probe to the registration URL read at .github/actions/ci-fixtures/action.yml:55; docs/design/composed-opus-aac-v2.json:60-64 identifies an Opus source archive, not an FFmpeg executable. scripts/composed_audio_fixtures.py:16-40 downloads that digest-pinned archive and compiles the separate comparator; these curl/tar/cc commands are not reference FFmpeg executions.", + "evidence": ".github/actions/ci-fixtures/action.yml:97 (shell, occurrence 0). .github/actions/ci-fixtures/action.yml:96-108 retries the same fixture entry point at most three times, removing its output once before the loop at .github/actions/ci-fixtures/action.yml:95 and after each failed generator attempt at .github/actions/ci-fixtures/action.yml:101, including the final failure; .github/actions/ci-fixtures/action.yml:97-100 returns immediately on generator success, preserving its output. .github/actions/ci-fixtures/action.yml:97-98 supplies only the requested ffmpeg selector. The env prefix at .github/actions/ci-fixtures/action.yml:97 puts $RUNNER_TEMP/coreav-comparator-curl first on the generator's PATH; .github/actions/ci-fixtures/action.yml:84-88 fills that directory with .github/actions/ci-fixtures/comparator-curl.sh as curl only after .github/actions/ci-fixtures/action.yml:64-81 holds an archive whose SHA-256 equals the registration's, and that stand-in copies the cached archive for the registered URL, refuses any other request, and replaces curl only, never the ffmpeg selector or the ffprobe beside it. scripts/composed_audio_fixtures.py:78-89 parses that selector, calls test_oracle.require(args.ffmpeg) at scripts/composed_audio_fixtures.py:86, and passes pair.ffmpeg unchanged to generate at scripts/composed_audio_fixtures.py:89. scripts/composed_audio_fixtures.py:53-75 uses that ffmpeg as argv[0] in every generation call; scripts/composed_audio.py:326-334 passes the same command through list(map(str, cmd)) into subprocess.Popen. scripts/test_oracle.py:18-21 defines the immutable pair; scripts/test_oracle.py:73-108 resolves both executable paths, verifies the complete selection with the platform pin owner and returns that pair. scripts/oracle_pin.py:108-121 verifies both pinned version banners; scripts/oracle_pin_macos.py:48-78 also checks binary digests, architecture and signing before accepting the pair. The failure diagnostic at .github/actions/ci-fixtures/action.yml:103-105 is a 0-1023 byte curl probe to the registration URL read at .github/actions/ci-fixtures/action.yml:93, run outside the stand-in's PATH; docs/design/composed-opus-aac-v2.json:60-64 identifies an Opus source archive, not an FFmpeg executable. scripts/composed_audio_fixtures.py:16-40 downloads that digest-pinned archive, rejects it at scripts/composed_audio_fixtures.py:27-28 unless its digest is the registered one whatever served it, and compiles the separate comparator; these curl/tar/cc commands are not reference FFmpeg executions.", "site": [ - 58, + 97, "shell", "", - "python3 -I scripts/composed_audio_fixtures.py \\\n --ffmpeg \"$RUNNER_TEMP/coreav-oracle-bin/ffmpeg\" --out \"$RUNNER_TEMP/coreav-composed-fixtures\"" + "env PATH=\"$RUNNER_TEMP/coreav-comparator-curl:$PATH\" python3 -I scripts/composed_audio_fixtures.py \\\n --ffmpeg \"$RUNNER_TEMP/coreav-oracle-bin/ffmpeg\" --out \"$RUNNER_TEMP/coreav-composed-fixtures\"" ], "verdict": "qualified-via-owner" } @@ -4207,7 +4208,7 @@ "tools/stage3-report/report.py", "tools/stage3-report/run.py" ], - "sha256": "4c093ef6fa807dc536a0e10006e0c0370460d5d253e8bf41dc58f3ee37eb0cbe", + "sha256": "ad5c141b44a3984b801aeca6d17a6af68ec04e3bc368d8b4f53fb1ee1e012d67", "sites": [ { "class": "reviewed-reference", @@ -163284,6 +163285,9 @@ "collections": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "common": "27fa8792c0ae851d5df80b5f16f355b0e88e9d5d7af51edf2c3dc5d020319921", "compat_check": "d511b6e719efa9a1018e44c2fdf0a203d46b8692a53a831542e0a0e51dcedbf2", + "composed_audio": "d7238c7dd06b879baed8c1dc280034b4700094935b3c7d9b1b7d0a5ab325b5e7", + "composed_audio_fixtures": "9bc5d7a5747e9eb5019e740c1688c09aa32ba65e34163d75ef1e03c0619b5e2a", + "composed_audio_source": "3fe5f9b15505fc58ed65232f64280aed1ecabf9caeb320b5f30817e7c7e9b950", "concurrent": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "config": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "contextlib": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", @@ -163300,6 +163304,7 @@ "dims_ff": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "download": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "evaluate": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "expected_encoder": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "extra_engines": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "extra_hostile": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "extract_check": "61b924249ee1662f38170ad1a82ac57670b2347b140dbe33fe8c47b894dc477f", @@ -163368,6 +163373,7 @@ "shutil": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "sig": "7be6eea02d417940e5cf6220ea9f0670810650f0e0dac01ed8192298ab6acfee", "signal": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "source_reference": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", "speed": "24ea60aaa67dd8446935234ff3a7ac487c87bb5a45b7bb1d6e721aef51a71137", "split_check": "1864d4582c074700131633fbe966227536fd7e734a2361873c6004538d49979f", "ssl": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", @@ -163700,6 +163706,9 @@ "docs/spikes/spike-a/h7-sidx/evidence/time/time-us-west-20261015T044716Z/status.json", "scripts/clip_check.py", "scripts/compat_check.py", + "scripts/composed_audio.py", + "scripts/composed_audio_fixtures.py", + "scripts/composed_audio_source.py", "scripts/extract_check.py", "scripts/hls_check.py", "scripts/oracle-audio-args.txt", @@ -163775,13 +163784,13 @@ "tools/stage3-report/report.py", "tools/stage3-report/run.py" ], - "sha256": "e64b1a75d207f2e83a598c6611e2b310ebbfe941717e4fe324caf3438484b18a", + "sha256": "121603c788ef77f07e226555d17bef86f6f1da751f0c6d62138119a27930eb82", "sites": [ { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:22 (process, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:20-23 invokes only Ruby with Psych to parse YAML as JSON. Its workflow/action callers at scripts/tests/test_ci_fixtures.py:29-30 and scripts/tests/test_ci_fixtures.py:140 read configuration; no YAML run block is executed.", + "evidence": "scripts/tests/test_ci_fixtures.py:65 (process, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", "site": [ - 22, + 65, "process", ".read_yaml", "'ruby'" @@ -163790,9 +163799,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:29 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:20-23 invokes only Ruby with Psych to parse YAML as JSON. Its workflow/action callers at scripts/tests/test_ci_fixtures.py:29-30 and scripts/tests/test_ci_fixtures.py:140 read configuration; no YAML run block is executed.", + "evidence": "scripts/tests/test_ci_fixtures.py:72 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", "site": [ - 29, + 72, "delegated", "", "read_yaml" @@ -163801,9 +163810,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:30 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:20-23 invokes only Ruby with Psych to parse YAML as JSON. Its workflow/action callers at scripts/tests/test_ci_fixtures.py:29-30 and scripts/tests/test_ci_fixtures.py:140 read configuration; no YAML run block is executed.", + "evidence": "scripts/tests/test_ci_fixtures.py:73 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", "site": [ - 30, + 73, "delegated", "", "read_yaml" @@ -163812,9 +163821,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:114 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:157 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 114, + 157, "delegated", "", "compute" @@ -163823,9 +163832,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:115 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:158 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 115, + 158, "delegated", "", "compute" @@ -163834,9 +163843,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:120 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:164 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 120, + 164, "delegated", "", "compute" @@ -163845,9 +163854,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:129 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:173 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 129, + 173, "delegated", "", "compute" @@ -163856,9 +163865,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:135 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:180 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 135, + 180, "delegated", "", "compute" @@ -163867,9 +163876,9 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:137 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:85-105 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text. scripts/tests/test_ci_fixtures.py:107-112 runs only key['run'] with bash and reads the resulting key; scripts/tests/test_ci_fixtures.py:114-137 calls compute before and after byte mutations. .github/actions/ci-fixtures/action.yml:10-16 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the new retry/probe block is selected.", + "evidence": "scripts/tests/test_ci_fixtures.py:184 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 137, + 184, "delegated", "", "compute" @@ -163878,9 +163887,174 @@ }, { "class": "reviewed-exclusion", - "evidence": "scripts/tests/test_ci_fixtures.py:140 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:20-23 invokes only Ruby with Psych to parse YAML as JSON. Its workflow/action callers at scripts/tests/test_ci_fixtures.py:29-30 and scripts/tests/test_ci_fixtures.py:140 read configuration; no YAML run block is executed.", + "evidence": "scripts/tests/test_ci_fixtures.py:186 (delegated, occurrence 0). scripts/tests/test_ci_fixtures.py:128-149 selects only the action step whose id is key and constructs temporary cache inputs, including placeholder composed-generator source text and a placeholder registration digest. scripts/tests/test_ci_fixtures.py:150-155 runs only key['run'] with bash and reads the resulting keys. .github/actions/ci-fixtures/action.yml:10-18 consists of grep/sed/sort, a Python import of SUPPORT, cat/hash/cut, a Python JSON read of the registration digest, and echo. scripts/test_oracle.py:28-32 initializes metadata; its require function at scripts/test_oracle.py:73-108 and CLI at scripts/test_oracle.py:137-138 are not called by the SUPPORT import. The imported pin owners scripts/oracle_pin.py:27-47 and scripts/oracle_pin_macos.py:3-15 initialize definitions and metadata, not reference processes. Helper, installer, pin and composed source bytes are hashed, not executed. Neither fixture-generation step nor the retry/probe block is selected.", "site": [ - 140, + 186, + "delegated", + "", + "compute" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:220 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", + "site": [ + 220, + "delegated", + "", + "read_yaml" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:249 (delegated, occurrence 0). run_block at scripts/tests/test_ci_fixtures.py:58-60 runs one workflow run block with bash in a stand-in checkout whose PATH starts with the stand-in curl that stand_in_curl at scripts/tests/test_ci_fixtures.py:44-55 writes (it copies a prepared body to the --output file and exits as told). The blocks run are the action's fetch step at .github/actions/ci-fixtures/action.yml:63-88 (a Python JSON read of the registration, sha256sum, curl, mkdir, mv, cp and chmod) and the Build opus_compare step at .github/workflows/audio-conformance.yml:543-560 (sha256sum, curl, tar and cc on the test's own crafted archive from comparator_archive at scripts/tests/test_ci_fixtures.py:30-41). Neither block names ffmpeg or ffprobe; the generator step is never selected.", + "site": [ + 249, + "delegated", + "", + "run_block" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:257 (delegated, occurrence 0). self.fetch_step at scripts/tests/test_ci_fixtures.py:232-252 prepares the stand-in checkout, the placeholder registration and the stand-in curl, then delegates to run_block for the action's fetch step; see the run_block evidence.", + "site": [ + 257, + "delegated", + "", + "self.fetch_step" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:266 (delegated, occurrence 0). self.fetch_step at scripts/tests/test_ci_fixtures.py:232-252 prepares the stand-in checkout, the placeholder registration and the stand-in curl, then delegates to run_block for the action's fetch step; see the run_block evidence.", + "site": [ + 266, + "delegated", + "", + "self.fetch_step" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:271 (delegated, occurrence 0). self.fetch_step at scripts/tests/test_ci_fixtures.py:232-252 prepares the stand-in checkout, the placeholder registration and the stand-in curl, then delegates to run_block for the action's fetch step; see the run_block evidence.", + "site": [ + 271, + "delegated", + "", + "self.fetch_step" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:277 (delegated, occurrence 0). self.fetch_step at scripts/tests/test_ci_fixtures.py:232-252 prepares the stand-in checkout, the placeholder registration and the stand-in curl, then delegates to run_block for the action's fetch step; see the run_block evidence.", + "site": [ + 277, + "delegated", + "", + "self.fetch_step" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:301 (process, occurrence 0). The executable is .github/actions/ci-fixtures/comparator-curl.sh, a bash script that parses -o/--output, compares its last argument with COREAV_COMPARATOR_URL and either copies COREAV_COMPARATOR_ARCHIVE to the target or exits 2. It is run with the generator's curl arguments or deliberately wrong ones; its only child command is cp, which copies the prepared archive and invokes neither FFmpeg nor ffprobe.", + "site": [ + 301, + "process", + ".SharedFixtures.test_stand_in_serves_only_the_generators_request_for_the_registered_archive.bounded", + "str(STAND_IN)" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:307 (delegated, occurrence 0). fixtures.build_comparator runs under patch.object of fixtures.composed.registration and fixtures.composed.bounded at scripts/tests/test_ci_fixtures.py:283-319; the replacement bounded at scripts/tests/test_ci_fixtures.py:298-302 runs only .github/actions/ci-fixtures/comparator-curl.sh for the curl command and raises for the tar command, so no curl, tar, cc, ffmpeg or ffprobe process starts. scripts/composed_audio_fixtures.py:16-40 contains no reference FFmpeg execution; its digest check at scripts/composed_audio_fixtures.py:27-28 is what the test exercises.", + "site": [ + 307, + "delegated", + "", + "fixtures.build_comparator" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:312 (delegated, occurrence 0). fixtures.build_comparator runs under patch.object of fixtures.composed.registration and fixtures.composed.bounded at scripts/tests/test_ci_fixtures.py:283-319; the replacement bounded at scripts/tests/test_ci_fixtures.py:298-302 runs only .github/actions/ci-fixtures/comparator-curl.sh for the curl command and raises for the tar command, so no curl, tar, cc, ffmpeg or ffprobe process starts. scripts/composed_audio_fixtures.py:16-40 contains no reference FFmpeg execution; its digest check at scripts/composed_audio_fixtures.py:27-28 is what the test exercises.", + "site": [ + 312, + "delegated", + "", + "fixtures.build_comparator" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:316 (process, occurrence 0). The executable is .github/actions/ci-fixtures/comparator-curl.sh, a bash script that parses -o/--output, compares its last argument with COREAV_COMPARATOR_URL and either copies COREAV_COMPARATOR_ARCHIVE to the target or exits 2. It is run with the generator's curl arguments or deliberately wrong ones; its only child command is cp, which copies the prepared archive and invokes neither FFmpeg nor ffprobe.", + "site": [ + 316, + "process", + ".SharedFixtures.test_stand_in_serves_only_the_generators_request_for_the_registered_archive", + "str(STAND_IN)" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:322 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", + "site": [ + 322, + "delegated", + "", + "read_yaml" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:339 (delegated, occurrence 0). run_block at scripts/tests/test_ci_fixtures.py:58-60 runs one workflow run block with bash in a stand-in checkout whose PATH starts with the stand-in curl that stand_in_curl at scripts/tests/test_ci_fixtures.py:44-55 writes (it copies a prepared body to the --output file and exits as told). The blocks run are the action's fetch step at .github/actions/ci-fixtures/action.yml:63-88 (a Python JSON read of the registration, sha256sum, curl, mkdir, mv, cp and chmod) and the Build opus_compare step at .github/workflows/audio-conformance.yml:543-560 (sha256sum, curl, tar and cc on the test's own crafted archive from comparator_archive at scripts/tests/test_ci_fixtures.py:30-41). Neither block names ffmpeg or ffprobe; the generator step is never selected.", + "site": [ + 339, + "delegated", + "", + "run_block" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:345 (process, occurrence 0). The executable is the program that the delegated Build opus_compare block compiled with cc from the test's crafted source (comparator_archive at scripts/tests/test_ci_fixtures.py:30-41, a main returning 0), run to show the block built it from the cached archive. It is not an FFmpeg executable.", + "site": [ + 345, + "process", + ".SharedFixtures.test_audio_conformance_builds_the_comparator_from_the_cached_archive", + "root / 'temp/opus-compare/opus_compare'" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:351 (delegated, occurrence 0). run_block at scripts/tests/test_ci_fixtures.py:58-60 runs one workflow run block with bash in a stand-in checkout whose PATH starts with the stand-in curl that stand_in_curl at scripts/tests/test_ci_fixtures.py:44-55 writes (it copies a prepared body to the --output file and exits as told). The blocks run are the action's fetch step at .github/actions/ci-fixtures/action.yml:63-88 (a Python JSON read of the registration, sha256sum, curl, mkdir, mv, cp and chmod) and the Build opus_compare step at .github/workflows/audio-conformance.yml:543-560 (sha256sum, curl, tar and cc on the test's own crafted archive from comparator_archive at scripts/tests/test_ci_fixtures.py:30-41). Neither block names ffmpeg or ffprobe; the generator step is never selected.", + "site": [ + 351, + "delegated", + "", + "run_block" + ], + "verdict": "excluded" + }, + { + "class": "reviewed-exclusion", + "evidence": "scripts/tests/test_ci_fixtures.py:358 (delegated, occurrence 0). read_yaml at scripts/tests/test_ci_fixtures.py:63-66 invokes only Ruby with Psych to parse YAML as JSON; its callers read workflow and action configuration, and no run block is executed by it.", + "site": [ + 358, "delegated", "", "read_yaml" @@ -263721,6 +263895,7 @@ ".github/actions/changed-paths/tests/test_changed_paths.py": "source", ".github/actions/changed-paths/tests/test_package_filters.py": "source", ".github/actions/ci-fixtures/action.yml": "source", + ".github/actions/ci-fixtures/comparator-curl.sh": "source", ".github/actions/merge-group-result/action.yml": "source", ".github/labels.yml": "source", ".github/pull_request_template.md": "data", diff --git a/scripts/tests/test_ci_fixtures.py b/scripts/tests/test_ci_fixtures.py index c5aa6fee1..1218004ce 100644 --- a/scripts/tests/test_ci_fixtures.py +++ b/scripts/tests/test_ci_fixtures.py @@ -1,20 +1,63 @@ +import gzip +import hashlib +import io import itertools import json import os import pathlib +import shutil import subprocess import sys +import tarfile import tempfile import unittest +from unittest.mock import patch from test_main_ci_plan import evaluate ROOT = pathlib.Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / 'scripts')) +import composed_audio_fixtures as fixtures import test_oracle ACTION = './.github/actions/ci-fixtures' +STAND_IN = ROOT / '.github/actions/ci-fixtures/comparator-curl.sh' +REGISTRATION = ROOT / 'docs/design/composed-opus-aac-v2.json' +ARCHIVE_URL = 'https://example.invalid/opus.tar.gz' + + +def comparator_archive(): + """A tar.gz holding only the comparator source the workflows extract.""" + source = b'int main(void) { return 0; }\n' + tar = io.BytesIO() + with tarfile.open(fileobj=tar, mode='w') as archive: + member = tarfile.TarInfo('opus-1.6.1/src/opus_compare.c') + member.size = len(source) + archive.addfile(member, io.BytesIO(source)) + compressed = io.BytesIO() + with gzip.GzipFile(fileobj=compressed, mode='wb', mtime=0) as stream: + stream.write(tar.getvalue()) + return compressed.getvalue() + + +def stand_in_curl(directory, body, exit_code=0): + """Put a curl on PATH that writes body to the --output file, records every call and exits as told.""" + (directory / 'bin').mkdir(exist_ok=True) + (directory / 'body').write_bytes(body) + script = directory / 'bin/curl' + script.write_text('#!/usr/bin/env bash\n' + f'printf "%s\\n" "$*" >> "{directory}/calls"\n' + 'while [ "$#" -gt 1 ]; do case "$1" in -o|--output) target="$2"; shift ;; esac; shift; done\n' + f'cp "{directory}/body" "$target"\n' + f'exit {exit_code}\n') + script.chmod(0o755) + return directory / 'calls' + + +def run_block(script, cwd, env): + return subprocess.run(['bash', '-c', script], cwd=cwd, env=os.environ | env, + capture_output=True, text=True) def read_yaml(path): @@ -95,7 +138,7 @@ def test_key_changes_with_the_pin_generator_and_discovered_helpers(self): for name, content in contents.items(): (scripts / name).write_text(content) composed_inputs = { - 'docs/design/composed-opus-aac-v2.json': '{}\n', + 'docs/design/composed-opus-aac-v2.json': '{"opus_compare": {"sha256": "0f"}}\n', 'scripts/composed_audio.py': 'composed source\n', 'scripts/composed_audio_fixtures.py': 'fixture source\n', } @@ -115,6 +158,7 @@ def compute(): self.assertEqual(original, compute()) original_keys = dict(line.split('=', 1) for line in original.splitlines()) self.assertIn('composed', original_keys) + self.assertEqual(original_keys['comparator'], 'comparator-archive-0f') for name, content in contents.items(): (scripts / name).write_text(content + '\n') changed = compute() @@ -129,13 +173,187 @@ def compute(): changed = dict(line.split('=', 1) for line in compute().splitlines()) self.assertEqual(original_keys['key'], changed['key'], name) self.assertNotEqual(original_keys['composed'], changed['composed'], name) + self.assertEqual(original_keys['comparator'], changed['comparator'], name) path.write_text(content) + registration = root / 'docs/design/composed-opus-aac-v2.json' + registration.write_text('{"opus_compare": {"sha256": "1e"}}\n') + self.assertEqual(dict(line.split('=', 1) for line in compute().splitlines())['comparator'], + 'comparator-archive-1e') (scripts / 'extra.py').write_text('extra\n') (scripts / 'gen-fixtures.sh').write_text(contents['gen-fixtures.sh'] + '$SCRIPTS/extra.py\n') added = compute() (scripts / 'extra.py').write_text('changed\n') self.assertNotEqual(added, compute()) + def test_comparator_archive_is_cached_by_its_registered_digest_on_every_branch(self): + steps = self.action['runs']['steps'] + composed, = [s for s in steps if s.get('id') == 'composed'] + cache, = [s for s in steps if s.get('name') == 'Cache the pinned comparator archive'] + fetch, = [s for s in steps if s.get('name') == 'Fetch the pinned comparator archive on a cache miss'] + generate, = [s for s in steps if s.get('name') == 'Generate the registered composed-audio fixtures'] + self.assertTrue(cache['uses'].startswith('actions/cache@'), 'restore and save in one step') + self.assertEqual(cache['with']['key'], '${{ steps.key.outputs.comparator }}') + self.assertEqual(cache['with']['path'], '${{ runner.temp }}/coreav-comparator-archive') + self.assertNotIn('restore-keys', cache['with']) + for ref, hit in itertools.product(('refs/heads/main', 'refs/pull/1/merge', + 'refs/heads/gh-readonly-queue/main/group'), ('true', 'false')): + context = {'github': {'ref': ref}, 'steps': {'composed': {'outputs': {'cache-hit': hit}}}} + self.assertEqual(bool(evaluate(cache['if'], context)), hit != 'true', ref) + self.assertEqual(bool(evaluate(fetch['if'], context)), hit != 'true', ref) + self.assertLess(steps.index(composed), steps.index(cache)) + self.assertLess(steps.index(cache), steps.index(fetch)) + self.assertLess(steps.index(fetch), steps.index(generate)) + archive = '"$RUNNER_TEMP/coreav-comparator-archive/opus.tar.gz"' + self.assertIn(f'archive={archive}', fetch['run']) + self.assertIn(f'COREAV_COMPARATOR_ARCHIVE={archive}', generate['run']) + self.assertIn('env PATH="$RUNNER_TEMP/coreav-comparator-curl:$PATH" python3 -I scripts/composed_audio_fixtures.py', + generate['run']) + for name in ('COREAV_COMPARATOR_URL', 'COREAV_COMPARATOR_ARCHIVE'): + self.assertIn(f'"${name}"', STAND_IN.read_text(), name) + pin = json.loads(REGISTRATION.read_text())['opus_compare'] + key, = [s for s in steps if s.get('id') == 'key'] + with tempfile.NamedTemporaryFile('r') as output: + subprocess.run(['bash', '-e', '-c', key['run']], cwd=ROOT, check=True, + env=os.environ | {'GITHUB_OUTPUT': output.name}) + keys = dict(line.split('=', 1) for line in output.read().splitlines()) + self.assertEqual(keys['comparator'], 'comparator-archive-' + pin['sha256']) + opus = read_yaml(ROOT / '.github/workflows/audio-conformance.yml')['jobs']['opus'] + self.assertEqual(opus['env']['LIBOPUS_URL'], pin['url']) + self.assertEqual(opus['env']['LIBOPUS_SHA256'], pin['sha256']) + shared, = [s for s in opus['steps'] if s.get('name') == 'Cache the pinned comparator archive'] + self.assertEqual(shared['uses'], cache['uses']) + self.assertEqual(shared['with']['path'], cache['with']['path']) + self.assertNotIn('if', shared) + self.assertEqual(shared['with']['key'].replace('${{ env.LIBOPUS_SHA256 }}', pin['sha256']), keys['comparator']) + build, = [s for s in opus['steps'] if s.get('name') == 'Build opus_compare'] + self.assertIn(f'archive={archive}', build['run']) + self.assertLess(opus['steps'].index(shared), opus['steps'].index(build)) + + def fetch_step(self, directory, cached, body, exit_code=0): + """Run the fetch step in a checkout stand-in and return (calls, archive, stand-in curl) paths.""" + step, = [s for s in self.action['runs']['steps'] + if s.get('name') == 'Fetch the pinned comparator archive on a cache miss'] + root = pathlib.Path(directory) + shutil.rmtree(root, ignore_errors=True) + (root / 'docs/design').mkdir(parents=True) + (root / '.github/actions/ci-fixtures').mkdir(parents=True) + shutil.copy(STAND_IN, root / '.github/actions/ci-fixtures/comparator-curl.sh') + archive = root / 'temp/coreav-comparator-archive/opus.tar.gz' + (root / 'temp').mkdir() + if cached is not None: + archive.parent.mkdir() + archive.write_bytes(cached) + (root / 'docs/design/composed-opus-aac-v2.json').write_text(json.dumps( + {'opus_compare': {'url': ARCHIVE_URL, 'sha256': hashlib.sha256(comparator_archive()).hexdigest()}})) + calls = stand_in_curl(root, body, exit_code) + result = run_block(step['run'], root, {'RUNNER_TEMP': str(root / 'temp'), + 'PATH': f"{root / 'bin'}:{os.environ['PATH']}"}) + self.assertEqual(result.returncode, 0, result.stderr) + return result, calls, archive, root / 'temp/coreav-comparator-curl/curl' + + def test_fetch_keeps_only_a_digest_verified_archive_and_installs_the_stand_in_beside_it(self): + registered = comparator_archive() + with tempfile.TemporaryDirectory() as directory: + result, calls, archive, stand_in = self.fetch_step(directory, None, registered) + self.assertEqual(archive.read_bytes(), registered) + self.assertEqual(len(calls.read_text().splitlines()), 1) + self.assertIn('--retry 3 --retry-all-errors', calls.read_text()) + self.assertIn(ARCHIVE_URL, calls.read_text()) + self.assertEqual(stand_in.read_text(), STAND_IN.read_text()) + self.assertTrue(os.access(stand_in, os.X_OK)) + self.assertNotIn('::warning::', result.stdout) + + result, calls, archive, stand_in = self.fetch_step(directory, registered, b'never fetched') + self.assertEqual(archive.read_bytes(), registered) + self.assertFalse(calls.exists()) + self.assertTrue(stand_in.exists()) + + result, calls, archive, stand_in = self.fetch_step(directory, b'another archive', registered) + self.assertEqual(archive.read_bytes(), registered) + self.assertEqual(len(calls.read_text().splitlines()), 1) + self.assertIn('::warning::the cached comparator archive has another digest', result.stdout) + + for body, exit_code in ((b'another archive', 0), (registered, 7)): + result, calls, archive, stand_in = self.fetch_step(directory, None, body, exit_code) + self.assertFalse(archive.parent.exists(), 'nothing unverified enters the cached directory') + self.assertFalse(stand_in.parent.exists(), 'the generator downloads on its own') + self.assertFalse(list(archive.parents[1].glob('*.download'))) + self.assertIn('::warning::comparator archive fetch failed', result.stdout) + + def test_stand_in_serves_only_the_generators_request_for_the_registered_archive(self): + registered = comparator_archive() + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + cached = root / 'opus.tar.gz' + cached.write_bytes(registered) + env = {'COREAV_COMPARATOR_URL': ARCHIVE_URL, 'COREAV_COMPARATOR_ARCHIVE': str(cached), + 'PATH': os.environ['PATH']} + pin = {'url': ARCHIVE_URL, 'sha256': hashlib.sha256(registered).hexdigest(), + 'source': 'opus-1.6.1/src/opus_compare.c'} + served = [] + + class Extracting(Exception): + pass + + def bounded(cmd, environment, **kwargs): + if cmd[0] != 'curl': + raise Extracting(cmd) + served.append(subprocess.run([str(STAND_IN), *map(str, cmd[1:])], env=env, + capture_output=True, text=True)) + + with patch.object(fixtures.composed, 'registration', return_value=({'opus_compare': pin}, {})), \ + patch.object(fixtures.composed, 'bounded', bounded): + with self.assertRaises(Extracting) as stopped: + fixtures.build_comparator(root / 'tools', env) + self.assertEqual(stopped.exception.args[0][0], 'tar', 'the digest check passed on the served bytes') + self.assertEqual([run.returncode for run in served], [0]) + cached.write_bytes(b'another archive') + with self.assertRaisesRegex(fixtures.composed.Invalid, 'digest differs'): + fixtures.build_comparator(root / 'tools-again', env) + request = ['--fail', '--silent', '--location', '-o', str(root / 'out.tar.gz'), ARCHIVE_URL] + for refused in (request[:-1] + ['https://example.invalid/other.tar.gz'], + request[:3] + [ARCHIVE_URL], request[:-1] + ['--output', ARCHIVE_URL]): + result = subprocess.run([str(STAND_IN), *refused], env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 2, refused) + self.assertIn('only the registered comparator archive is served', result.stderr) + self.assertFalse((root / 'out.tar.gz').exists(), refused) + + def test_audio_conformance_builds_the_comparator_from_the_cached_archive(self): + build, = [s for s in read_yaml(ROOT / '.github/workflows/audio-conformance.yml')['jobs']['opus']['steps'] + if s.get('name') == 'Build opus_compare'] + registered = comparator_archive() + digest = hashlib.sha256(registered).hexdigest() + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + archive = root / 'temp/coreav-comparator-archive/opus.tar.gz' + for cached, body, exit_code, fetched in ((registered, b'never fetched', 1, 0), + (None, registered, 0, 1), + (b'another archive', registered, 0, 1)): + shutil.rmtree(root / 'temp', ignore_errors=True) + (root / 'temp').mkdir() + if cached is not None: + archive.parent.mkdir() + archive.write_bytes(cached) + calls = stand_in_curl(root, body, exit_code) + calls.unlink(missing_ok=True) + result = run_block(build['run'], root, { + 'RUNNER_TEMP': str(root / 'temp'), 'LIBOPUS_URL': ARCHIVE_URL, 'LIBOPUS_SHA256': digest, + 'PATH': f"{root / 'bin'}:{os.environ['PATH']}"}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(archive.read_bytes(), registered) + self.assertEqual(len(calls.read_text().splitlines()) if calls.exists() else 0, fetched) + self.assertEqual(subprocess.run([root / 'temp/opus-compare/opus_compare']).returncode, 0) + self.assertFalse((root / 'temp/opus-compare/opus-1.6.1').exists()) + calls = stand_in_curl(root, b'another archive') + calls.unlink(missing_ok=True) + shutil.rmtree(root / 'temp') + (root / 'temp').mkdir() + result = run_block(build['run'], root, { + 'RUNNER_TEMP': str(root / 'temp'), 'LIBOPUS_URL': ARCHIVE_URL, 'LIBOPUS_SHA256': digest, + 'PATH': f"{root / 'bin'}:{os.environ['PATH']}"}) + self.assertNotEqual(result.returncode, 0, 'a fetched archive with another digest fails the job') + self.assertFalse(archive.parent.exists(), 'and never enters the cached directory') + def test_all_fixture_support_changes_start_both_wasm_events(self): workflow = read_yaml(ROOT / '.github/workflows/wasm.yml') events = workflow.get('on', workflow.get('true'))