diff --git a/.github/actions/ci-fixtures/action.yml b/.github/actions/ci-fixtures/action.yml index 720696c04..c98c84d60 100644 --- a/.github/actions/ci-fixtures/action.yml +++ b/.github/actions/ci-fixtures/action.yml @@ -1,5 +1,12 @@ name: Shared CI fixtures description: Publish one cached or generated fixture set for this run +inputs: + fresh: + description: Exercise fixture generation even on a cache hit + default: 'false' + composed: + description: Include the registered composed-audio fixture set + default: 'false' runs: using: composite steps: @@ -19,26 +26,27 @@ runs: path: ${{ runner.temp }}/coreav-fixtures key: ${{ steps.key.outputs.key }} - name: Restore the registered composed-audio fixtures + if: inputs.composed == 'true' id: composed uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ${{ runner.temp }}/coreav-composed-fixtures key: linux-x64-composed-${{ hashFiles('docs/design/composed-opus-aac-v2.json', 'scripts/composed_audio.py', 'scripts/composed_audio_fixtures.py', 'scripts/oracle_audio.py', 'scripts/oracle-audio-args.txt', 'scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh') }} - name: Cache pinned black-box binaries - if: steps.fixtures.outputs.cache-hit != 'true' || steps.composed.outputs.cache-hit != 'true' + if: inputs.fresh == 'true' || steps.fixtures.outputs.cache-hit != 'true' || (inputs.composed == 'true' && steps.composed.outputs.cache-hit != 'true') id: ffmpeg-cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ${{ runner.temp }}/coreav-oracle-bin key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} - name: Install pinned FFmpeg, ffprobe and MediaInfo - if: (steps.fixtures.outputs.cache-hit != 'true' || steps.composed.outputs.cache-hit != 'true') && steps.ffmpeg-cache.outputs.cache-hit != 'true' + if: (inputs.fresh == 'true' || steps.fixtures.outputs.cache-hit != 'true' || (inputs.composed == 'true' && steps.composed.outputs.cache-hit != 'true')) && steps.ffmpeg-cache.outputs.cache-hit != 'true' shell: bash run: | scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" - name: Generate all required fixtures - if: steps.fixtures.outputs.cache-hit != 'true' + if: inputs.fresh == 'true' || steps.fixtures.outputs.cache-hit != 'true' shell: bash env: COREAV_REQUIRE_ALL_FIXTURES: '1' @@ -47,12 +55,13 @@ runs: export PATH="$RUNNER_TEMP/coreav-oracle-bin:$PATH" scripts/gen-fixtures.sh "$RUNNER_TEMP/coreav-fixtures" - name: Generate the registered composed-audio fixtures - if: steps.composed.outputs.cache-hit != 'true' + if: inputs.composed == 'true' && steps.composed.outputs.cache-hit != 'true' shell: bash run: | python3 -I scripts/composed_audio_fixtures.py \ --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" --out "$RUNNER_TEMP/coreav-composed-fixtures" - name: Publish the registered composed-audio fixtures + if: inputs.composed == 'true' uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: composed-audio-fixture-set @@ -60,7 +69,7 @@ runs: if-no-files-found: error retention-days: 1 - name: Save the registered composed-audio fixtures - if: steps.composed.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + if: inputs.composed == 'true' && steps.composed.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ${{ runner.temp }}/coreav-composed-fixtures diff --git a/.github/actions/merge-group-result/action.yml b/.github/actions/merge-group-result/action.yml index 04637b0e1..d1f8e0983 100644 --- a/.github/actions/merge-group-result/action.yml +++ b/.github/actions/merge-group-result/action.yml @@ -1,13 +1,19 @@ name: Merge-group result -description: Reuse this workflow's successful merge-group run for the same tree on main. +description: Reuse successful checks for an identical tested tree and base. inputs: workflow: - description: Workflow file whose successful merge-group result may be reused. + description: Workflow file whose successful result may be reused. required: true outputs: reuse: - description: Whether this workflow already passed on a merge group with this tree. + description: Whether this workflow has eligible successful same-tree evidence. value: ${{ steps.result.outputs.reuse }} + run: + description: Covering push run for a scheduled full suite. + value: ${{ steps.result.outputs.run }} + attempt: + description: Covering push attempt for a scheduled full suite. + value: ${{ steps.result.outputs.attempt }} runs: using: composite steps: @@ -17,3 +23,11 @@ runs: GH_TOKEN: ${{ github.token }} WORKFLOW: ${{ inputs.workflow }} run: python3 -I scripts/merge_group_result.py --workflow "$WORKFLOW" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + if: steps.result.outputs.record_name != '' + continue-on-error: true + with: + name: ${{ steps.result.outputs.record_name }} + path: ${{ steps.result.outputs.record_path }} + if-no-files-found: error + retention-days: 2 diff --git a/.github/workflows/ci-full-failure.yml b/.github/workflows/ci-full-failure.yml index 9e286cc50..0fde27686 100644 --- a/.github/workflows/ci-full-failure.yml +++ b/.github/workflows/ci-full-failure.yml @@ -4,7 +4,7 @@ name: ci-full-failure on: workflow_run: - workflows: [hevc-conformance] + workflows: [ci-full, vp9-corpus, vp9-conformance, hevc-conformance] types: [completed] permissions: diff --git a/.github/workflows/ci-full.yml b/.github/workflows/ci-full.yml index 7f19ebb44..f8b025940 100644 --- a/.github/workflows/ci-full.yml +++ b/.github/workflows/ci-full.yml @@ -1,16 +1,392 @@ name: ci-full - # workflow-check: not required - on: + push: + branches: [main] + paths: + - 'crates/**' + - 'test-support/**' + - 'scripts/**' + - 'tools/**' + - 'Cargo.toml' + - 'Cargo.lock' + - 'rust-toolchain.toml' + - 'clippy.toml' + - 'docs/schema/**' + - 'docs/known-divergences*' + - 'docs/design/composed-opus-aac-v2.json' + - '.github/workflows/ci*.yml' + - '.github/actions/**' + schedule: + - cron: '30 3 * * *' + - cron: '30 4 * * 0' workflow_dispatch: +permissions: + contents: read +jobs: + full-coverage: + name: ci-full-coverage + if: github.event_name == 'schedule' && github.event.schedule == '30 3 * * *' + concurrency: + group: ${{ github.workflow }}-full-coverage-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + reuse: ${{ steps.result.outputs.reuse == 'true' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + continue-on-error: true + with: + persist-credentials: false + - id: result + uses: ./.github/actions/merge-group-result + continue-on-error: true + with: + workflow: ci-full.yml + - name: Covered by run ${{ steps.result.outputs.run }} attempt ${{ steps.result.outputs.attempt }} + if: steps.result.outputs.reuse == 'true' + run: 'true' -permissions: {} + full-fixtures: + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' + concurrency: + group: ${{ github.workflow }}-full-fixtures-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Publish the full fixture sets + uses: ./.github/actions/ci-fixtures + with: + fresh: 'true' + composed: 'true' -jobs: - ci-full-registration: + h264-full: + needs: full-fixtures + if: ${{ !cancelled() && needs.full-fixtures.result == 'success' }} + concurrency: + group: ${{ github.workflow }}-h264-full-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Restore the conformance zips + id: conformance-cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/h264-conformance + key: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }} + restore-keys: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }}-partial- + - name: Verify or download the conformance zips + id: conformance-acquisition + run: python3 -I scripts/h264_conformance.py --download-only --zips ~/.cache/h264-conformance + - name: Save the conformance zips before decoding + if: >- + ${{ !cancelled() && steps.conformance-cache.outputs.cache-hit != 'true' && + (steps.conformance-acquisition.outcome == 'success' || steps.conformance-acquisition.outcome == 'failure') }} + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/h264-conformance + key: ${{ steps.conformance-cache.outputs.cache-primary-key }}${{ steps.conformance-acquisition.outcome == 'failure' && format('-partial-{0}-{1}-{2}', github.job, github.run_id, github.run_attempt) || '' }} + - name: Decode every in-scope stream + run: | + python3 -I scripts/h264_conformance.py \ + --zips ~/.cache/h264-conformance \ + --work "$RUNNER_TEMP/h264" \ + --threads 1,2,8 \ + --report "$RUNNER_TEMP/conformance.json" + # The same suite in safe mode, a build without the + # feature `fast`, which compiles no `unsafe` and runs the baseline + # instance of the hot paths. The step above ran the default build, + # and the x86-64-v3 instance where the runner's CPU has its features. + - name: Decode every in-scope stream in safe mode + if: ${{ !cancelled() && steps.conformance-acquisition.outcome == 'success' }} + run: | + python3 -I scripts/h264_conformance.py \ + --zips ~/.cache/h264-conformance \ + --work "$RUNNER_TEMP/h264" \ + --threads 1,2,8 \ + --mode safe \ + --report "$RUNNER_TEMP/conformance-safe.json" + - name: Lint and test safe mode + if: ${{ !cancelled() }} + run: | + cargo clippy -p coreav-h264 --no-default-features --all-targets -- -D warnings + cargo test -p coreav-h264 --no-default-features --release + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: h264-full-results + path: | + ${{ runner.temp }}/conformance.json + ${{ runner.temp }}/conformance-safe.json + if-no-files-found: ignore + + conformance-arm64: + needs: full-fixtures + if: ${{ !cancelled() && needs.full-fixtures.result == 'success' }} + concurrency: + group: ${{ github.workflow }}-conformance-arm64-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04-arm + timeout-minutes: 120 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Restore the conformance zips + id: conformance-cache + uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/h264-conformance + key: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }} + restore-keys: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }}-partial- + - name: Verify or download the conformance zips + id: conformance-acquisition + run: python3 -I scripts/h264_conformance.py --download-only --zips ~/.cache/h264-conformance + - name: Save the conformance zips before decoding + if: >- + ${{ !cancelled() && steps.conformance-cache.outputs.cache-hit != 'true' && + (steps.conformance-acquisition.outcome == 'success' || steps.conformance-acquisition.outcome == 'failure') }} + uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ~/.cache/h264-conformance + key: ${{ steps.conformance-cache.outputs.cache-primary-key }}${{ steps.conformance-acquisition.outcome == 'failure' && format('-partial-{0}-{1}-{2}', github.job, github.run_id, github.run_attempt) || '' }} + - name: Decode every in-scope stream + run: | + python3 -I scripts/h264_conformance.py \ + --zips ~/.cache/h264-conformance \ + --work "$RUNNER_TEMP/h264" \ + --threads 1,2,8 \ + --report "$RUNNER_TEMP/conformance-arm64.json" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: always() + with: + name: h264-conformance-arm64 + path: ${{ runner.temp }}/conformance-arm64.json + if-no-files-found: ignore + + # The CI benchmark guard (docs/design/benchmarks.md): + # instruction counts of the coreav command on generated fixtures. + # - Pushes to main that change what the binary or the fixtures are built + # from, and manual runs: against bench/guard/baseline.json. + # - Ready pull requests that change the crates or the lock file: the pull + # request's merge commit against its base commit, both built and counted + # in this job, so a regression shows before it merges. Drafts skip it. + # A manual run with `update` keeps the counts as an artifact for a pull + # request that replaces the baseline. ci-changes' `bench-guard` is the + # filter, one list for both events: a pull request that changes the + # fixture generator, the pinned FFmpeg installer or the web-upload policy + # measures too, which is the safe direction. + transcode-compat-shard: + needs: full-fixtures + if: ${{ !cancelled() && needs.full-fixtures.result == 'success' }} + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + concurrency: + group: ${{ github.workflow }}-transcode-compat-shard-${{ strategy.job-index }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + shared-key: isobmff-fixtures + save-if: false + - name: Cache pinned black-box binaries + id: ffmpeg-cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-oracle-bin + key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} + # MediaInfo is a gating oracle for extracted M4A files only; elsewhere it stays informational. + - name: Install pinned FFmpeg, ffprobe and MediaInfo + if: steps.ffmpeg-cache.outputs.cache-hit != 'true' + run: | + scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" + scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" + - name: Use pinned binaries + run: | + python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" + echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: isobmff-fixture-set + path: ${{ runner.temp }}/coreav-fixtures + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: composed-audio-fixture-set + path: ${{ runner.temp }}/coreav-fixtures + - name: Use the registered comparator + run: chmod +x "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" + # The same build as isobmff-decode's. + - name: Install libdav1d and libvpx + run: sudo apt-get update -q && sudo apt-get install -y --no-install-recommends libdav1d-dev libvpx-dev + - name: Build coreav with the FFmpeg fallback, dav1d and libvpx, and the decode oracle + run: | + cargo build --locked --release -p coreav-cli -p coreav-decode-oracle \ + --features coreav-cli/ffmpeg-fallback,coreav-cli/dav1d,coreav-cli/libvpx,coreav-cli/input-capture,coreav-decode-oracle/dav1d,coreav-decode-oracle/libvpx \ + --target-dir target/fallback + - name: Check actual-stdin capture transport + run: cargo test --locked --release -p coreav-transcode --features input-capture --lib --target-dir target/fallback + - name: Cache the x264 program + id: x264-cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-x264 + key: linux-x64-x264-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-x264.sh') }} + - name: Build x264 + if: steps.x264-cache.outputs.cache-hit != 'true' + run: | + sudo apt-get install -y --no-install-recommends nasm + scripts/install-x264.sh "$RUNNER_TEMP/coreav-x264" + # docs/design/transcode.md, bar B9: the coreav ffmpeg command lines that + # encode with libx264, on the fixtures, each arm against FFmpeg running + # the same command line. The ffmpeg arm names no x264 program, so the + # pipeline falls back to the pinned FFmpeg's libx264. + - name: Prove composed output identity and semantic mutations + run: | + python3 -I scripts/composed_audio_selftest.py \ + --coreav target/fallback/release/coreav --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" \ + --ffprobe "$RUNNER_TEMP/coreav-oracle-bin/ffprobe" \ + --opus-compare "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" --fixtures "$RUNNER_TEMP/coreav-fixtures" + - name: Run the registered composed compatibility check + env: + PYTHONDONTWRITEBYTECODE: '1' + SHARD: ${{ matrix.shard }} + run: | + python3 -I scripts/transcode_check.py --compat --coreav target/fallback/release/coreav \ + --compat-check composed-opus-aac-v2 --opus-compare "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" \ + --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" --ffprobe "$RUNNER_TEMP/coreav-oracle-bin/ffprobe" \ + --preflight "$RUNNER_TEMP/coreav-fixtures/basic.mp4" --fixtures "$RUNNER_TEMP/coreav-fixtures" \ + --arm "x264=COREAV_X264=$RUNNER_TEMP/coreav-x264/x264" --arm ffmpeg=COREAV_X264=none \ + --jobs 4 --shard "$SHARD/3" --out "$RUNNER_TEMP/coreav-transcode-compat" + - name: Keep the shard's evidence for transcode-compat + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: transcode-compat-shard-${{ matrix.shard }} + path: ${{ runner.temp }}/coreav-transcode-compat/evidence + retention-days: 1 + + # B9's bars over the three shards' results, judged as one run judges + # them (scripts/transcode_check.py --merge). + transcode-compat: + needs: [full-fixtures, transcode-compat-shard] + if: ${{ !cancelled() && needs.full-fixtures.result == 'success' && needs.transcode-compat-shard.result == 'success' }} + concurrency: + group: ${{ github.workflow }}-transcode-compat-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: Cache the pinned oracle + id: ffmpeg-cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-oracle-bin + key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} + - name: Install the pinned oracle + if: steps.ffmpeg-cache.outputs.cache-hit != 'true' + run: | + scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" + scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" + - name: Use pinned binaries + run: | + python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" + echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: transcode-compat-shard-* + path: ${{ runner.temp }}/transcode-compat-shards + - name: Judge the registered compatibility check over every shard + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + python3 -I scripts/transcode_check.py --compat-check composed-opus-aac-v2 --out "$RUNNER_TEMP/coreav-transcode-compat" --merge \ + "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-1" "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-2" "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-3" + - name: Check the registered version and pinned oracle + run: | + python3 -I - <<'PY' + import hashlib + import json + import os + import pathlib + import subprocess + root = pathlib.Path(os.environ['RUNNER_TEMP']) + run = json.loads((root / 'coreav-transcode-compat/run.json').read_text()) + registration = pathlib.Path('docs/design/composed-opus-aac-v2.json').read_bytes() + expected = {'version': 'composed-opus-aac-v2', 'registration_sha256': hashlib.sha256(registration).hexdigest()} + if run.get('compat_check') != expected: + raise SystemExit('the compatibility evidence has the wrong registered version') + tools = run['tools'] if isinstance(run['tools'], list) else [run['tools']] + for name in ('ffmpeg', 'ffprobe'): + binary = root / 'coreav-oracle-bin' / name + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + if not tools or any(t[name]['sha256'] != digest for t in tools): + raise SystemExit(f'B9 did not use the pinned {name}') + version = subprocess.check_output([str(root / 'coreav-oracle-bin/ffmpeg'), '-version']).decode().splitlines()[:1] + if run['ffmpeg_version'] != version: + raise SystemExit('B9 did not record the pinned FFmpeg version') + PY + - name: Preserve the registered compatibility evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: transcode-compat-evidence + path: ${{ runner.temp }}/coreav-transcode-compat/evidence + if-no-files-found: ignore + # docs/design/mkv.md: the Matroska and WebM fixtures through coreav-mkv and + # the coreav binary, against the pinned ffprobe on every change (100% after + # the documented divergences of docs/known-divergences-mkv.md), and against + # MediaInfo as the informational second oracle; and every + # fixture remuxed to MP4 (docs/design/mkv-remux.md). + # It follows ci-changes' `run`. + ci-full-report: + needs: [full-coverage, full-fixtures, h264-full, conformance-arm64, transcode-compat-shard, transcode-compat] + if: always() && needs.full-coverage.outputs.reuse != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 1 + timeout-minutes: 5 steps: - - name: Register manual full-suite execution - run: ':' + - name: Require every full-suite job to succeed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + set -euo pipefail + jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' <<< "$RESULTS" >> "$GITHUB_STEP_SUMMARY" + jq -e 'del(."full-coverage") | length == 5 and all(.[]; .result == "success")' <<< "$RESULTS" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54c7e9c1c..a1be85bf6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,10 +75,9 @@ jobs: outputs: run: ${{ steps.result.outputs.reuse != 'true' && steps.filter.outputs.run == 'true' }} h264-conformance: ${{ steps.result.outputs.reuse != 'true' && steps.h264-conformance.outputs.run == 'true' }} - h264-main: ${{ steps.h264-conformance.outputs.run }} - main-cache: ${{ steps.result.outputs.reuse == 'true' && steps.filter.outputs.run == 'true' }} + main-cache: ${{ github.event_name == 'push' && steps.result.outputs.reuse == 'true' && steps.filter.outputs.run == 'true' }} oracle-installer: ${{ steps.result.outputs.reuse != 'true' && steps.oracle-installer.outputs.run == 'true' }} - bench-guard: ${{ steps.bench-guard.outputs.run }} + bench-guard: ${{ !(github.event_name == 'merge_group' && steps.result.outputs.reuse == 'true') && steps.bench-guard.outputs.run == 'true' }} fuzz-plan: ${{ steps.fuzz-plan.outputs.plan }} # Whether `check-macos` and `check-windows` run: always on a push or # a manual run, and on a pull request or a merge group when it changes @@ -147,6 +146,8 @@ jobs: crates/coreav-h264/** crates/coreav-kernels-h264/** scripts/h264_conformance.py + scripts/h264_ci.py + scripts/tests/test_core_ci_tiers.py scripts/h264_modes_identity.py scripts/h264_agreement.py scripts/oracle-ffmpeg.env @@ -203,8 +204,11 @@ jobs: - name: Publish the shared fixtures before fan-out if: >- steps.filter.outputs.run == 'true' && + !(github.event_name == 'merge_group' && steps.result.outputs.reuse == 'true') && (github.event_name != 'pull_request' || !github.event.pull_request.draft) uses: ./.github/actions/ci-fixtures + with: + fresh: ${{ steps.result.outputs.reuse != 'true' }} check: needs: ci-changes @@ -686,12 +690,10 @@ jobs: run: | python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" - # Generated on every run: every run must exercise generation, and - # required cases must never silently disappear. - - name: Generate all required fixtures - env: - COREAV_REQUIRE_ALL_FIXTURES: '1' - run: scripts/gen-fixtures.sh "$RUNNER_TEMP/coreav-fixtures" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: isobmff-fixture-set + path: ${{ runner.temp }}/coreav-fixtures - name: Check oracle audio input threading env: COREAV_FIXTURES: ${{ runner.temp }}/coreav-fixtures @@ -762,43 +764,6 @@ jobs: --media "$RUNNER_TEMP/coreav-fixtures" \ --kd-overlay "$RUNNER_TEMP/coreav-isobmff-oracle/known-divergences.json" \ --report "$RUNNER_TEMP/coreav-compat.json" - # docs/design/compat-cli.md (B1, B2 and B3a on the fixtures, B6): - # every ffprobe command of the coverage set through `coreav ffprobe` - # on every fixture, against the pinned ffprobe, and every refusal; a - # failure the committed baseline does not list fails the run. - - name: Run the command-coverage set through coreav ffprobe - env: - PYTHONDONTWRITEBYTECODE: '1' - run: | - COREAV_CURRENT_CI_ORACLE=1 python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_current_ci.py - python3 -I tools/compat-corpus/current_ci.py commands --coreav target/release/coreav \ - --media "$RUNNER_TEMP/coreav-fixtures" --baseline tools/compat-corpus/baseline-isobmff.json \ - --report "$RUNNER_TEMP/coreav-compat-commands.json" - - name: List frames through coreav ffprobe - run: | - cargo build --locked --release -p coreav-cli --features native-h264 --target-dir target/native - python3 -I tools/compat-corpus/frames.py --coreav target/native/release/coreav \ - --media "$RUNNER_TEMP/coreav-fixtures" --native-h264 \ - --baseline tools/compat-corpus/baseline-frames.json --report "$RUNNER_TEMP/coreav-frames.json" - # R1 of the 0.1.0 round (docs/measurements/compat-0.1.0/README.md): the - # 0.1.0 ffprobe commands on these fixtures, with no baseline. The round - # reads its verdict from the job summary; a miss or a void run does not - # fail the job, so that a change of the oracle's output (FFmpeg 9 prints - # keys 7.0.2 did not) is reported here rather than blocking unrelated - # pull requests. A run that measured nothing fails it. - - name: R1 of the 0.1.0 round - env: - PYTHONDONTWRITEBYTECODE: '1' - run: | - cargo build --locked --profile dist -p coreav-cli - sha256sum target/dist/coreav > "$RUNNER_TEMP/coreav-r1.sha256" - code=0 - python3 -I tools/compat-corpus/run.py commands --set 0.1.0 --coreav target/dist/coreav \ - --media "$RUNNER_TEMP/coreav-fixtures" --report "$RUNNER_TEMP/coreav-r1.json" > "$RUNNER_TEMP/coreav-r1.txt" || code=$? - case "$code" in 0) verdict=met ;; 1) verdict=missed ;; 2) verdict=void ;; *) verdict="not measured (exit $code)" ;; esac - [ -s "$RUNNER_TEMP/coreav-r1.json" ] || { verdict='not measured (no report)'; code=3; } - { echo "### R1 of the 0.1.0 round, ISO base media fixtures: $verdict"; echo '```'; cat "$RUNNER_TEMP/coreav-r1.sha256"; tail -n 15 "$RUNNER_TEMP/coreav-r1.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" - [ "$code" -le 2 ] - name: Validate native library parity and output schemas env: COREAV_FIXTURES: ${{ runner.temp }}/coreav-fixtures @@ -915,16 +880,105 @@ jobs: ${{ runner.temp }}/coreav-isobmff-oracle/known-divergences.json ${{ runner.temp }}/coreav-isobmff-oracle/compare.log ${{ runner.temp }}/coreav-compat.json - ${{ runner.temp }}/coreav-compat-commands.json - ${{ runner.temp }}/coreav-frames.json - ${{ runner.temp }}/coreav-r1.json - ${{ runner.temp }}/coreav-r1.sha256 ${{ runner.temp }}/coreav-extract.json ${{ runner.temp }}/coreav-extract-cli.json ${{ runner.temp }}/coreav-text.json if-no-files-found: ignore + isobmff-command-compat: + # Draft PRs skip it (workflow_dispatch still runs it on demand). + needs: ci-changes + if: >- + needs.ci-changes.outputs.run == 'true' && + (github.event_name != 'pull_request' || !github.event.pull_request.draft) + concurrency: + group: ${{ github.workflow }}-isobmff-command-compat-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + with: + toolchain: stable + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + with: + save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Cache pinned black-box binaries + id: ffmpeg-cache + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: ${{ runner.temp }}/coreav-oracle-bin + key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} + # MediaInfo is a gating oracle for extracted M4A files only; elsewhere it stays informational. + - name: Install pinned FFmpeg, ffprobe and MediaInfo + if: steps.ffmpeg-cache.outputs.cache-hit != 'true' + run: | + scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" + scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" + - name: Use pinned binaries + run: | + python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" + echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: isobmff-fixture-set + path: ${{ runner.temp }}/coreav-fixtures + - name: Build the coreav binary + run: cargo build --locked --release -p coreav-cli + # docs/design/compat-cli.md (B1, B2 and B3a on the fixtures, B6): + # every ffprobe command of the coverage set through `coreav ffprobe` + # on every fixture, against the pinned ffprobe, and every refusal; a + # failure the committed baseline does not list fails the run. + - name: Run the command-coverage set through coreav ffprobe + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + COREAV_CURRENT_CI_ORACLE=1 python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_current_ci.py + python3 -I tools/compat-corpus/current_ci.py commands --coreav target/release/coreav \ + --media "$RUNNER_TEMP/coreav-fixtures" --baseline tools/compat-corpus/baseline-isobmff.json \ + --report "$RUNNER_TEMP/coreav-compat-commands.json" + - name: List frames through coreav ffprobe + run: | + cargo build --locked --release -p coreav-cli --features native-h264 --target-dir target/native + python3 -I tools/compat-corpus/frames.py --coreav target/native/release/coreav \ + --media "$RUNNER_TEMP/coreav-fixtures" --native-h264 \ + --baseline tools/compat-corpus/baseline-frames.json --report "$RUNNER_TEMP/coreav-frames.json" + # R1 of the 0.1.0 round (docs/measurements/compat-0.1.0/README.md): the + # 0.1.0 ffprobe commands on these fixtures, with no baseline. The round + # reads its verdict from the job summary; a miss or a void run does not + # fail the job, so that a change of the oracle's output (FFmpeg 9 prints + # keys 7.0.2 did not) is reported here rather than blocking unrelated + # pull requests. A run that measured nothing fails it. + - name: R1 of the 0.1.0 round + env: + PYTHONDONTWRITEBYTECODE: '1' + run: | + cargo build --locked --profile dist -p coreav-cli + sha256sum target/dist/coreav > "$RUNNER_TEMP/coreav-r1.sha256" + code=0 + python3 -I tools/compat-corpus/run.py commands --set 0.1.0 --coreav target/dist/coreav \ + --media "$RUNNER_TEMP/coreav-fixtures" --report "$RUNNER_TEMP/coreav-r1.json" > "$RUNNER_TEMP/coreav-r1.txt" || code=$? + case "$code" in 0) verdict=met ;; 1) verdict=missed ;; 2) verdict=void ;; *) verdict="not measured (exit $code)" ;; esac + [ -s "$RUNNER_TEMP/coreav-r1.json" ] || { verdict='not measured (no report)'; code=3; } + { echo "### R1 of the 0.1.0 round, ISO base media fixtures: $verdict"; echo '```'; cat "$RUNNER_TEMP/coreav-r1.sha256"; tail -n 15 "$RUNNER_TEMP/coreav-r1.txt"; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + [ "$code" -le 2 ] + - name: Keep command compatibility results + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: isobmff-command-results + path: | + ${{ runner.temp }}/coreav-compat-commands.json + ${{ runner.temp }}/coreav-frames.json + ${{ runner.temp }}/coreav-r1.json + ${{ runner.temp }}/coreav-r1.sha256 + ${{ runner.temp }}/coreav-r1.txt + if-no-files-found: ignore + # The writers' oracles of the ISO base media fixtures, beside # isobmff-fixtures (docs/design/ci-checks.md, "The fixture jobs"). isobmff-writers: @@ -1361,185 +1415,6 @@ jobs: # every fixture three times, one line scaling it to 1920 columns. A third # of the fixtures a job; `transcode-compat` judges the bars once over all three # (docs/design/ci-checks.md, "The fixture jobs"). - transcode-compat-shard: - needs: ci-changes - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3] - if: >- - needs.ci-changes.outputs.run == 'true' && - (github.event_name != 'pull_request' || !github.event.pull_request.draft) - concurrency: - group: ${{ github.workflow }}-transcode-compat-shard-${{ strategy.job-index }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - shared-key: isobmff-fixtures - save-if: false - - name: Cache pinned black-box binaries - id: ffmpeg-cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: ${{ runner.temp }}/coreav-oracle-bin - key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} - # MediaInfo is a gating oracle for extracted M4A files only; elsewhere it stays informational. - - name: Install pinned FFmpeg, ffprobe and MediaInfo - if: steps.ffmpeg-cache.outputs.cache-hit != 'true' - run: | - scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" - scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" - - name: Use pinned binaries - run: | - python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" - echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: isobmff-fixture-set - path: ${{ runner.temp }}/coreav-fixtures - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: composed-audio-fixture-set - path: ${{ runner.temp }}/coreav-fixtures - - name: Use the registered comparator - run: chmod +x "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" - # The same build as isobmff-decode's. - - name: Install libdav1d and libvpx - run: sudo apt-get update -q && sudo apt-get install -y --no-install-recommends libdav1d-dev libvpx-dev - - name: Build coreav with the FFmpeg fallback, dav1d and libvpx, and the decode oracle - run: | - cargo build --locked --release -p coreav-cli -p coreav-decode-oracle \ - --features coreav-cli/ffmpeg-fallback,coreav-cli/dav1d,coreav-cli/libvpx,coreav-cli/input-capture,coreav-decode-oracle/dav1d,coreav-decode-oracle/libvpx \ - --target-dir target/fallback - - name: Check actual-stdin capture transport - run: cargo test --locked --release -p coreav-transcode --features input-capture --lib --target-dir target/fallback - - name: Cache the x264 program - id: x264-cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: ${{ runner.temp }}/coreav-x264 - key: linux-x64-x264-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-x264.sh') }} - - name: Build x264 - if: steps.x264-cache.outputs.cache-hit != 'true' - run: | - sudo apt-get install -y --no-install-recommends nasm - scripts/install-x264.sh "$RUNNER_TEMP/coreav-x264" - # docs/design/transcode.md, bar B9: the coreav ffmpeg command lines that - # encode with libx264, on the fixtures, each arm against FFmpeg running - # the same command line. The ffmpeg arm names no x264 program, so the - # pipeline falls back to the pinned FFmpeg's libx264. - - name: Prove composed output identity and semantic mutations - run: | - python3 -I scripts/composed_audio_selftest.py \ - --coreav target/fallback/release/coreav --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" \ - --ffprobe "$RUNNER_TEMP/coreav-oracle-bin/ffprobe" \ - --opus-compare "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" --fixtures "$RUNNER_TEMP/coreav-fixtures" - - name: Run the registered composed compatibility check - env: - PYTHONDONTWRITEBYTECODE: '1' - SHARD: ${{ matrix.shard }} - run: | - python3 -I scripts/transcode_check.py --compat --coreav target/fallback/release/coreav \ - --compat-check composed-opus-aac-v2 --opus-compare "$RUNNER_TEMP/coreav-fixtures/tools/opus_compare" \ - --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" --ffprobe "$RUNNER_TEMP/coreav-oracle-bin/ffprobe" \ - --preflight "$RUNNER_TEMP/coreav-fixtures/basic.mp4" --fixtures "$RUNNER_TEMP/coreav-fixtures" \ - --arm "x264=COREAV_X264=$RUNNER_TEMP/coreav-x264/x264" --arm ffmpeg=COREAV_X264=none \ - --jobs 4 --shard "$SHARD/3" --out "$RUNNER_TEMP/coreav-transcode-compat" - - name: Keep the shard's evidence for transcode-compat - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: transcode-compat-shard-${{ matrix.shard }} - path: ${{ runner.temp }}/coreav-transcode-compat/evidence - retention-days: 1 - - # B9's bars over the three shards' results, judged as one run judges - # them (scripts/transcode_check.py --merge). - transcode-compat: - needs: [ci-changes, transcode-compat-shard] - if: >- - needs.ci-changes.outputs.run == 'true' && - (github.event_name != 'pull_request' || !github.event.pull_request.draft) - concurrency: - group: ${{ github.workflow }}-transcode-compat-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - name: Cache the pinned oracle - id: ffmpeg-cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: ${{ runner.temp }}/coreav-oracle-bin - key: linux-x64-oracle-mediainfo-26.10-${{ hashFiles('scripts/oracle-ffmpeg.env', 'scripts/install-oracle-ffmpeg.sh', 'scripts/install-oracle-mediainfo.sh') }} - - name: Install the pinned oracle - if: steps.ffmpeg-cache.outputs.cache-hit != 'true' - run: | - scripts/install-oracle-ffmpeg.sh "$RUNNER_TEMP/coreav-oracle-bin" - scripts/install-oracle-mediainfo.sh "$RUNNER_TEMP/coreav-oracle-bin" - - name: Use pinned binaries - run: | - python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" - echo "$RUNNER_TEMP/coreav-oracle-bin" >> "$GITHUB_PATH" - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - pattern: transcode-compat-shard-* - path: ${{ runner.temp }}/transcode-compat-shards - - name: Judge the registered compatibility check over every shard - env: - PYTHONDONTWRITEBYTECODE: '1' - run: | - python3 -I scripts/transcode_check.py --compat-check composed-opus-aac-v2 --out "$RUNNER_TEMP/coreav-transcode-compat" --merge \ - "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-1" "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-2" "$RUNNER_TEMP/transcode-compat-shards/transcode-compat-shard-3" - - name: Check the registered version and pinned oracle - run: | - python3 -I - <<'PY' - import hashlib - import json - import os - import pathlib - import subprocess - root = pathlib.Path(os.environ['RUNNER_TEMP']) - run = json.loads((root / 'coreav-transcode-compat/run.json').read_text()) - registration = pathlib.Path('docs/design/composed-opus-aac-v2.json').read_bytes() - expected = {'version': 'composed-opus-aac-v2', 'registration_sha256': hashlib.sha256(registration).hexdigest()} - if run.get('compat_check') != expected: - raise SystemExit('the compatibility evidence has the wrong registered version') - tools = run['tools'] if isinstance(run['tools'], list) else [run['tools']] - for name in ('ffmpeg', 'ffprobe'): - binary = root / 'coreav-oracle-bin' / name - digest = hashlib.sha256(binary.read_bytes()).hexdigest() - if not tools or any(t[name]['sha256'] != digest for t in tools): - raise SystemExit(f'B9 did not use the pinned {name}') - version = subprocess.check_output([str(root / 'coreav-oracle-bin/ffmpeg'), '-version']).decode().splitlines()[:1] - if run['ffmpeg_version'] != version: - raise SystemExit('B9 did not record the pinned FFmpeg version') - PY - - name: Preserve the registered compatibility evidence - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: transcode-compat-evidence - path: ${{ runner.temp }}/coreav-transcode-compat/evidence - if-no-files-found: ignore - # docs/design/mkv.md: the Matroska and WebM fixtures through coreav-mkv and - # the coreav binary, against the pinned ffprobe on every change (100% after - # the documented divergences of docs/known-divergences-mkv.md), and against - # MediaInfo as the informational second oracle; and every - # fixture remuxed to MP4 (docs/design/mkv-remux.md). - # It follows ci-changes' `run`. mkv-fixtures: # Draft PRs skip it (workflow_dispatch still runs it on demand). needs: ci-changes @@ -1941,11 +1816,11 @@ jobs: uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: path: ~/.cache/h264-conformance - key: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }} - restore-keys: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }}-partial- + key: h264-smoke-v1-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json', 'scripts/h264_ci.py') }} + restore-keys: h264-smoke-v1-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json', 'scripts/h264_ci.py') }}-partial- - name: Verify or download the conformance zips id: conformance-acquisition - run: python3 -I scripts/h264_conformance.py --download-only --zips ~/.cache/h264-conformance + run: python3 -I scripts/h264_ci.py --only BA1_Sony_D CABACI3_Sony_B cavlc_mot_frm0_full_B HCHP1_HHI_B HPCV_BRCM_A alphaconformanceG CVPCMNL1_SVA_C MR1_BT_A --download-only --zips ~/.cache/h264-conformance - name: Save the conformance zips before decoding if: >- ${{ !cancelled() && steps.conformance-cache.outputs.cache-hit != 'true' && @@ -1956,7 +1831,8 @@ jobs: key: ${{ steps.conformance-cache.outputs.cache-primary-key }}${{ steps.conformance-acquisition.outcome == 'failure' && format('-partial-{0}-{1}-{2}', github.job, github.run_id, github.run_attempt) || '' }} - name: Decode every in-scope stream run: | - python3 -I scripts/h264_conformance.py \ + python3 -I scripts/h264_ci.py \ + --only BA1_Sony_D CABACI3_Sony_B cavlc_mot_frm0_full_B HCHP1_HHI_B HPCV_BRCM_A alphaconformanceG CVPCMNL1_SVA_C MR1_BT_A \ --zips ~/.cache/h264-conformance \ --work "$RUNNER_TEMP/h264" \ --threads 1,2,8 \ @@ -1968,7 +1844,8 @@ jobs: - name: Decode every in-scope stream in safe mode if: ${{ !cancelled() && steps.conformance-acquisition.outcome == 'success' }} run: | - python3 -I scripts/h264_conformance.py \ + python3 -I scripts/h264_ci.py \ + --only BA1_Sony_D CABACI3_Sony_B cavlc_mot_frm0_full_B HCHP1_HHI_B HPCV_BRCM_A alphaconformanceG CVPCMNL1_SVA_C MR1_BT_A \ --zips ~/.cache/h264-conformance \ --work "$RUNNER_TEMP/h264" \ --threads 1,2,8 \ @@ -2232,71 +2109,6 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - conformance-arm64: - # The whole suite on arm64, on a push to main. - needs: ci-changes - if: >- - needs.ci-changes.outputs.h264-main == 'true' && - github.event_name == 'push' - concurrency: - group: ${{ github.workflow }}-conformance-arm64-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04-arm - timeout-minutes: 120 - steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - persist-credentials: false - - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable - with: - toolchain: stable - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 - with: - save-if: ${{ github.ref == 'refs/heads/main' }} - - name: Restore the conformance zips - id: conformance-cache - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: ~/.cache/h264-conformance - key: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }} - restore-keys: h264-conformance-v2-${{ hashFiles('crates/coreav-h264/tests/data/conformance-streams.json') }}-partial- - - name: Verify or download the conformance zips - id: conformance-acquisition - run: python3 -I scripts/h264_conformance.py --download-only --zips ~/.cache/h264-conformance - - name: Save the conformance zips before decoding - if: >- - ${{ !cancelled() && steps.conformance-cache.outputs.cache-hit != 'true' && - (steps.conformance-acquisition.outcome == 'success' || steps.conformance-acquisition.outcome == 'failure') }} - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: ~/.cache/h264-conformance - key: ${{ steps.conformance-cache.outputs.cache-primary-key }}${{ steps.conformance-acquisition.outcome == 'failure' && format('-partial-{0}-{1}-{2}', github.job, github.run_id, github.run_attempt) || '' }} - - name: Decode every in-scope stream - run: | - python3 -I scripts/h264_conformance.py \ - --zips ~/.cache/h264-conformance \ - --work "$RUNNER_TEMP/h264" \ - --threads 1,2,8 \ - --report "$RUNNER_TEMP/conformance-arm64.json" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: h264-conformance-arm64 - path: ${{ runner.temp }}/conformance-arm64.json - if-no-files-found: ignore - - # The CI benchmark guard (docs/design/benchmarks.md): - # instruction counts of the coreav command on generated fixtures. - # - Pushes to main that change what the binary or the fixtures are built - # from, and manual runs: against bench/guard/baseline.json. - # - Ready pull requests that change the crates or the lock file: the pull - # request's merge commit against its base commit, both built and counted - # in this job, so a regression shows before it merges. Drafts skip it. - # A manual run with `update` keeps the counts as an artifact for a pull - # request that replaces the baseline. ci-changes' `bench-guard` is the - # filter, one list for both events: a pull request that changes the - # fixture generator, the pinned FFmpeg installer or the web-upload policy - # measures too, which is the safe direction. bench-guard: needs: ci-changes if: >- @@ -2585,7 +2397,7 @@ jobs: # (docs/design/ci-checks.md). It waits for every job of the workflow, # and its summary names each job's result. It has no concurrency group: # it has to report on a cancelled run too. - needs: [ci-changes, check, check-macos, check-windows, msrv, fuzz, isobmff-fixtures, isobmff-writers, isobmff-decode, transcode-compat-shard, transcode-compat, mkv-fixtures, ts-fixtures, conformance, vector-width, modes-identity, training-identity, kernels-arm64, conformance-arm64, bench-guard, main-cache, svt-av1] + needs: [ci-changes, check, check-macos, check-windows, msrv, fuzz, isobmff-fixtures, isobmff-command-compat, isobmff-writers, isobmff-decode, mkv-fixtures, ts-fixtures, conformance, vector-width, modes-identity, training-identity, kernels-arm64, bench-guard, main-cache, svt-av1] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 diff --git a/.github/workflows/deny.yml b/.github/workflows/deny.yml index 627c060bb..34f0ee22d 100644 --- a/.github/workflows/deny.yml +++ b/.github/workflows/deny.yml @@ -201,7 +201,7 @@ jobs: # (crates/*/fuzz), so a new one is checked without editing this file; # finding none fails. cargo-deny is the version the action above runs. - name: Install cargo-deny - if: steps.result.outputs.reuse != 'true' + if: steps.result.outputs.reuse != 'true' || github.event_name == 'merge_group' env: VERSION: 0.20.2 SHA256: 9f12ed4c49936e09b48bf862b595cde2fe64fcbd9d74dfacac6131ca824c8d5f @@ -213,6 +213,23 @@ jobs: echo "$SHA256 $RUNNER_TEMP/deny.tar.gz" | sha256sum -c - tar -xzf "$RUNNER_TEMP/deny.tar.gz" -C "$RUNNER_TEMP" "$name/cargo-deny" echo "$RUNNER_TEMP/$name" >> "$GITHUB_PATH" + - name: Refresh advisory checks on a reused merge group + id: fresh-advisories + if: github.event_name == 'merge_group' && steps.result.outputs.reuse == 'true' + run: | + set -euo pipefail + python3 scripts/fuzz_plan.py --projects > "$RUNNER_TEMP/fuzz-projects.txt" + status=0 + cargo deny --all-features check advisories || status=1 + while read -r project; do + cargo deny --all-features --manifest-path "$project/Cargo.toml" check advisories || status=1 + done < "$RUNNER_TEMP/fuzz-projects.txt" + exit "$status" + - name: Require fresh advisories on a reused merge group + if: always() && github.event_name == 'merge_group' && steps.result.outputs.reuse == 'true' + env: + ADVISORIES: ${{ steps.fresh-advisories.outcome }} + run: test "$ADVISORIES" = success - name: Check the licenses of every fuzz project if: steps.result.outputs.reuse != 'true' run: | diff --git a/.github/workflows/h264-encoder.yml b/.github/workflows/h264-encoder.yml index 8ab55ff37..be123406e 100644 --- a/.github/workflows/h264-encoder.yml +++ b/.github/workflows/h264-encoder.yml @@ -45,8 +45,6 @@ on: - .github/workflows/h264-encoder.yml - Cargo.toml - Cargo.lock - merge_group: - types: [checks_requested] workflow_dispatch: permissions: diff --git a/.github/workflows/hevc-conformance.yml b/.github/workflows/hevc-conformance.yml index 011f63500..8ce6980b2 100644 --- a/.github/workflows/hevc-conformance.yml +++ b/.github/workflows/hevc-conformance.yml @@ -11,6 +11,7 @@ on: paths: ["crates/coreav-hevc/**", "crates/coreav-kernels-hevc/**", "bench/hevc-conformance/**", "scripts/hevc_conformance.py", "scripts/hevc_intra.py", "scripts/hevc_decode.py", "scripts/hevc_fixture.py", "scripts/hevc_synthetic.py", "scripts/hevc_smoke.py", "scripts/test_oracle.py", "scripts/hevc_oracle_input.py", "scripts/tests/test_hevc*.py", "docs/measurements/hevc-native/roster.json", "docs/measurements/transcode/round-2/corpus-selection.json", "docs/spikes/thumbnail-bench/evidence/speed-dev-2.jsonl", "docs/spikes/thumbnail-bench/evidence/speed-2-hevc.jsonl", "docs/measurements/transcode/speed-selection.json", "tools/compat-corpus/**", "tools/stage3-report/**", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "scripts/oracle_audio.py", "scripts/oracle-audio-args.txt", "Cargo.toml", "Cargo.lock", ".github/workflows/hevc-conformance.yml"] schedule: - cron: "45 3 * * *" + - cron: "45 4 * * 0" workflow_dispatch: inputs: scalar: @@ -26,7 +27,34 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + full-coverage: + name: hevc-conformance-coverage + if: github.event_name == 'schedule' && github.event.schedule == '45 3 * * *' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + reuse: ${{ steps.result.outputs.reuse == 'true' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + continue-on-error: true + with: + persist-credentials: false + - id: result + uses: ./.github/actions/merge-group-result + continue-on-error: true + with: + workflow: hevc-conformance.yml + - name: Covered by run ${{ steps.result.outputs.run }} attempt ${{ steps.result.outputs.attempt }} + if: steps.result.outputs.reuse == 'true' + run: 'true' + headers: + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' runs-on: ${{ matrix.os }} timeout-minutes: 15 strategy: @@ -56,6 +84,9 @@ jobs: run: python3 -m unittest discover -s scripts/tests -p 'test_hevc*.py' hevc-smoke: + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' runs-on: ubuntu-24.04 timeout-minutes: 30 steps: @@ -89,7 +120,10 @@ jobs: retention-days: 30 hevc-vectors: - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' && + ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-24.04 timeout-minutes: 60 steps: @@ -143,7 +177,10 @@ jobs: retention-days: 30 profile-census: - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' && + ((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-24.04 timeout-minutes: 240 strategy: diff --git a/.github/workflows/npm.yml b/.github/workflows/npm.yml index a32dfc923..bbe0ad843 100644 --- a/.github/workflows/npm.yml +++ b/.github/workflows/npm.yml @@ -46,6 +46,12 @@ on: # Every binary once a week (Monday 04:30 UTC), on main. - cron: '30 4 * * 1' workflow_dispatch: + inputs: + parity_comparison: + description: Compare complete npm tests with one and four parity workers + type: boolean + required: false + default: false permissions: contents: read @@ -114,7 +120,7 @@ jobs: needs.npm-changes.outputs.run == 'true' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) runs-on: ubuntu-24.04 - timeout-minutes: 40 + timeout-minutes: ${{ github.event_name == 'workflow_dispatch' && inputs.parity_comparison && 90 || 40 }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -192,7 +198,22 @@ jobs: env: COREAV_BIN: ${{ github.workspace }}/packages/cli/npm/linux-x64/bin/coreav COREAV_FIXTURE_DIRS: ${{ runner.temp }}/coreav-fixtures,${{ runner.temp }}/coreav-mkv-fixtures - run: npm test + PARITY_COMPARISON: ${{ github.event_name == 'workflow_dispatch' && inputs.parity_comparison }} + COREAV_PARITY_COMPARISON_REPORT: ${{ runner.temp }}/parity-comparison.json + run: | + if [ "$PARITY_COMPARISON" = true ]; then + node test/parity-comparison.mjs + else + npm test + fi + - name: Retain CI harness comparison identities and timings + if: always() && github.event_name == 'workflow_dispatch' && inputs.parity_comparison + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: npm-parity-comparison + path: ${{ runner.temp }}/parity-comparison.json + if-no-files-found: error + retention-days: 3 - name: Cache the x264 program id: x264-cache uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 diff --git a/.github/workflows/vp9-conformance.yml b/.github/workflows/vp9-conformance.yml index f6361e914..0ac93ac70 100644 --- a/.github/workflows/vp9-conformance.yml +++ b/.github/workflows/vp9-conformance.yml @@ -18,7 +18,8 @@ on: types: [opened, synchronize, reopened, ready_for_review] paths: ["crates/coreav-vp9/**", "crates/coreav-kernels-vp9/**", "crates/coreav-backend-vp9/**", "crates/coreav-decode/**", "crates/coreav-frame/**", "tools/decode-oracle/**", "scripts/fuzz_run.sh", "scripts/vp9_pixel*", "scripts/tests/test_vp9_pixel*", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "docs/measurements/vp9-pixel-round/**", "Cargo.toml", "Cargo.lock", ".github/workflows/vp9-conformance.yml"] schedule: - - cron: '41 6 * * 3' + - cron: '41 17 * * *' + - cron: '41 18 * * 0' workflow_dispatch: inputs: mode: @@ -39,8 +40,34 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + full-coverage: + name: vp9-conformance-coverage + if: github.event_name == 'schedule' && github.event.schedule == '41 17 * * *' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + reuse: ${{ steps.result.outputs.reuse == 'true' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + continue-on-error: true + with: + persist-credentials: false + - id: result + uses: ./.github/actions/merge-group-result + continue-on-error: true + with: + workflow: vp9-conformance.yml + - name: Covered by run ${{ steps.result.outputs.run }} attempt ${{ steps.result.outputs.attempt }} + if: steps.result.outputs.reuse == 'true' + run: 'true' + vectors: - if: inputs.mode != 'ceiling' && inputs.mode != 'build-round' + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' && inputs.mode != 'ceiling' && inputs.mode != 'build-round' strategy: fail-fast: false matrix: @@ -141,7 +168,9 @@ jobs: cargo test --locked --release -p coreav-backend-vp9 -p coreav-kernels-vp9 --features coreav-kernels-vp9/safe --test vectors -- --include-ignored --nocapture interpolation-fuzz: - if: inputs.mode != 'ceiling' && inputs.mode != 'build-round' + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' && inputs.mode != 'ceiling' && inputs.mode != 'build-round' strategy: fail-fast: false matrix: @@ -162,10 +191,13 @@ jobs: - name: Compare automatic and scalar interpolation for C3 env: ASAN_OPTIONS: quarantine_size_mb=64 - run: scripts/fuzz_run.sh run "$RUNNER_TEMP/interpolation-plan.txt" 600 "$RUNNER_TEMP/interpolation-fuzz" + FUZZ_SECONDS: ${{ github.event_name == 'pull_request' && '30' || '600' }} + run: scripts/fuzz_run.sh run "$RUNNER_TEMP/interpolation-plan.txt" "$FUZZ_SECONDS" "$RUNNER_TEMP/interpolation-fuzz" adapter-contract: - if: inputs.mode != 'ceiling' && inputs.mode != 'build-round' + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' && inputs.mode != 'ceiling' && inputs.mode != 'build-round' runs-on: ubuntu-24.04 timeout-minutes: 30 steps: diff --git a/.github/workflows/vp9-corpus.yml b/.github/workflows/vp9-corpus.yml index 2f7df2002..5f7d38a4d 100644 --- a/.github/workflows/vp9-corpus.yml +++ b/.github/workflows/vp9-corpus.yml @@ -3,12 +3,13 @@ name: vp9-corpus on: push: branches: [main] - paths: ["crates/coreav-vp9/**", "crates/coreav-backend-vp9/**", "crates/coreav-decode/**", "crates/coreav-frame/**", "crates/coreav-kernels-vp9/**", "crates/coreav-mkv/**", "crates/coreav-isobmff/**", "tools/vp9-corpus-oracle/**", "scripts/vp9_corpus.py", "scripts/tests/test_vp9_corpus*.py", "docs/measurements/vp9-native/**", "tools/compat-corpus/round_files.py", "tools/compat-corpus/release_round.py", "tools/compat-corpus/evidence-public.pem", "tools/stage3-report/r2pool.py", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "Cargo.toml", "Cargo.lock", ".github/workflows/vp9-corpus.yml"] + paths: ["crates/coreav-vp9/**", "crates/coreav-backend-vp9/**", "crates/coreav-decode/**", "crates/coreav-frame/**", "crates/coreav-kernels-vp9/**", "crates/coreav-mkv/**", "crates/coreav-isobmff/**", "tools/vp9-corpus-oracle/**", "scripts/vp9_corpus.py", "scripts/vp9_smoke.py", "scripts/tests/test_vp9_smoke.py", "scripts/tests/test_vp9_corpus*.py", "docs/measurements/vp9-native/**", "tools/compat-corpus/round_files.py", "tools/compat-corpus/release_round.py", "tools/compat-corpus/evidence-public.pem", "tools/stage3-report/r2pool.py", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "Cargo.toml", "Cargo.lock", ".github/workflows/vp9-corpus.yml"] pull_request: types: [opened, synchronize, reopened, ready_for_review] - paths: ["crates/coreav-vp9/**", "crates/coreav-backend-vp9/**", "crates/coreav-decode/**", "crates/coreav-frame/**", "crates/coreav-kernels-vp9/**", "crates/coreav-mkv/**", "crates/coreav-isobmff/**", "tools/vp9-corpus-oracle/**", "scripts/vp9_corpus.py", "scripts/tests/test_vp9_corpus*.py", "docs/measurements/vp9-native/**", "tools/compat-corpus/round_files.py", "tools/compat-corpus/release_round.py", "tools/compat-corpus/evidence-public.pem", "tools/stage3-report/r2pool.py", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "Cargo.toml", "Cargo.lock", ".github/workflows/vp9-corpus.yml"] + paths: ["crates/coreav-vp9/**", "crates/coreav-backend-vp9/**", "crates/coreav-decode/**", "crates/coreav-frame/**", "crates/coreav-kernels-vp9/**", "crates/coreav-mkv/**", "crates/coreav-isobmff/**", "tools/vp9-corpus-oracle/**", "scripts/vp9_corpus.py", "scripts/vp9_smoke.py", "scripts/tests/test_vp9_smoke.py", "scripts/tests/test_vp9_corpus*.py", "docs/measurements/vp9-native/**", "tools/compat-corpus/round_files.py", "tools/compat-corpus/release_round.py", "tools/compat-corpus/evidence-public.pem", "tools/stage3-report/r2pool.py", "scripts/oracle-ffmpeg.env", "scripts/install-oracle-ffmpeg.sh", "scripts/oracle_pin.py", "scripts/oracle_from_r2.py", "Cargo.toml", "Cargo.lock", ".github/workflows/vp9-corpus.yml"] schedule: - - cron: '11 6 * * 1' + - cron: '11 17 * * *' + - cron: '11 18 * * 0' workflow_dispatch: permissions: @@ -19,7 +20,34 @@ concurrency: cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + full-coverage: + name: vp9-corpus-coverage + if: github.event_name == 'schedule' && github.event.schedule == '11 17 * * *' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + reuse: ${{ steps.result.outputs.reuse == 'true' }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + continue-on-error: true + with: + persist-credentials: false + - id: result + uses: ./.github/actions/merge-group-result + continue-on-error: true + with: + workflow: vp9-corpus.yml + - name: Covered by run ${{ steps.result.outputs.run }} attempt ${{ steps.result.outputs.attempt }} + if: steps.result.outputs.reuse == 'true' + run: 'true' + contract: + needs: full-coverage + if: >- + !cancelled() && needs.full-coverage.outputs.reuse != 'true' runs-on: ubuntu-24.04 timeout-minutes: 30 steps: @@ -65,6 +93,20 @@ jobs: python3 -I scripts/vp9_corpus.py preflight --file "$RUNNER_TEMP/preflight.webm" \ --native target/release/vp9-corpus-oracle --ffmpeg "$RUNNER_TEMP/coreav-oracle-bin/ffmpeg" \ --ffprobe "$RUNNER_TEMP/coreav-oracle-bin/ffprobe" --private "$RUNNER_TEMP/preflight" + - name: Compare the routine original-file smoke subset + run: | + set -euo pipefail + python3 -I scripts/oracle_pin.py --verify-bin "$RUNNER_TEMP/coreav-oracle-bin" + python3 -I scripts/vp9_smoke.py --native target/release/vp9-corpus-oracle \ + --oracle-bin "$RUNNER_TEMP/coreav-oracle-bin" --out "$RUNNER_TEMP/vp9-smoke" + - name: Retain exact smoke coverage + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: vp9-smoke + path: ${{ runner.temp }}/vp9-smoke/smoke.json + if-no-files-found: error + retention-days: 7 - name: Stage the verified tools run: | set -euo pipefail @@ -81,7 +123,11 @@ jobs: corpus: needs: contract - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + if: >- + !cancelled() && needs.contract.result == 'success' && + github.repository == 'coreav-dev/coreav' && + ((github.event_name == 'push' && github.ref == 'refs/heads/main') || + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-24.04 timeout-minutes: 180 strategy: @@ -154,8 +200,11 @@ jobs: retention-days: 30 corpus-summary: - needs: corpus - if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) + needs: [full-coverage, corpus] + if: >- + needs.full-coverage.outputs.reuse != 'true' && always() && github.repository == 'coreav-dev/coreav' && + ((github.event_name == 'push' && github.ref == 'refs/heads/main') || + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-24.04 timeout-minutes: 10 steps: diff --git a/crates/coreav-cli/tests/ffmpeg_pcm.rs b/crates/coreav-cli/tests/ffmpeg_pcm.rs index bc8a31f49..056a8f78f 100644 --- a/crates/coreav-cli/tests/ffmpeg_pcm.rs +++ b/crates/coreav-cli/tests/ffmpeg_pcm.rs @@ -402,6 +402,27 @@ fn raw_stdin_writes_before_eof_without_a_whole_input_spool() { fn tiny_pcm_stdout_reports_a_closed_reader_during_final_flush() { use std::io::Write; use std::process::Stdio; + const NAME: &str = "tiny_pcm_stdout_reports_a_closed_reader_during_final_flush"; + if std::env::var_os("COREAV_PCM_CLOSED_READER_PROBE").is_none() { + let result = Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + NAME, + "--nocapture", + "--color", + "never", + "--test-threads=1", + ]) + .env("COREAV_PCM_CLOSED_READER_PROBE", "1") + .output() + .unwrap(); + assert!(result.status.success(), "{result:?}"); + assert!( + String::from_utf8_lossy(&result.stdout).contains(&format!("test {NAME} ... ok")), + "{result:?}" + ); + return; + } let mut child = Command::new(env!("CARGO_BIN_EXE_coreav")) .args([ "--strict-safe", diff --git a/docs/agents/ci.md b/docs/agents/ci.md index eca8322fc..9c9504513 100644 --- a/docs/agents/ci.md +++ b/docs/agents/ci.md @@ -5,9 +5,16 @@ Workflow required checks, path filters, DCO, pinned actions, merge-group reuse, | Rule | Enforced by | |---|---| | A credentialed manual workflow that builds a dispatched commit admits only commits on this repository's branches, so pull-request code from forks never runs beside corpus credentials | The HEVC reference workflow's admission step refuses a candidate no branch contains before the build and every credentialed step; `test_candidate_must_be_on_a_branch_before_it_builds_or_sees_credentials` pins that order. Writers already hold the repository's secrets, so this closes the only path that widens access. Workflow step order cannot be encoded in Rust types | +| A daily full run skips only for a successful same-workflow main-push attempt at its exact SHA; weekly full runs never skip for coverage, and a covered daily never masquerades as missing full work | `merge-group-result` owns bounded search and revalidation; `ci_full.covers` owns complete job/proof validation shared with the pinned-claim reporter. `test_full_coverage.py` rejects wrong identities, non-successes, missing legs, reruns and malformed claims, while preserving cancellation and manual evidence. An exact daily-cron gate excludes each weekly trigger; the weekly scheduling test supplies a covering success and still requires every registered full job, and the reporter test preserves the normal failure path. Workflow selection is external to Python types; weekly full work checks drift while main stays unchanged | +| Merge-group reuse proves both the actual PR-tested tree and base at the newest successful run attempt; missing evidence runs the ordinary checks and advisory scans stay fresh | `merge-group-result` owns checkout-bound attempt receipts and metadata lookup. `test_merge_group_result.py` rejects wrong identities, stale or duplicate receipts, incomplete indexes, newer failures/reruns and head races; `test_main_ci_plan.py` keeps reused groups out of fixture production and main-only maintenance, retains main's baseline guard, and executes root/fuzz advisory commands and their skipped/failure witness. Workflow self-tests reject non-required merge-group triggers. Tree equality alone cannot prove changed-path selection, a base comparison or a current advisory database | +| A closed-reader stdout test cannot inherit a pipe reader through another concurrently spawned test process | `tiny_pcm_stdout_reports_a_closed_reader_during_final_flush` runs its unchanged two-byte pipe assertion in a single-test subprocess and requires the exact success witness, so zero selected tests fail. The subprocess owns pipe creation; the probe still requires exit 141 and empty stderr on every platform. A descriptor's local lifetime cannot constrain inheritance by another thread's concurrent spawn | +| Required fixture branches consume one fresh run-bound set, H.264 smoke has exactly eight names and complete results, and full H.264 and transcode stay independent of queue reuse | `scripts/tests/test_core_ci_tiers.py` checks producer freshness on cache hits, artifact consumption without regeneration, moved command predicates, exact smoke membership and reports, full events, thread and safe-mode commands, and full-report failures. Workflow ownership is structural; types cannot govern YAML scheduling | +| Full-suite reporting uses trusted main code and metadata, and final-head verification requires all three newest manual suites with complete matrix and proof-step success | `scripts/tests/test_ci_full.py` covers event and repository refusal, bot-owned issue deduplication, fixed public text, API and pagination failures, latest dispatch and attempt identity, skipped proofs, missing matrix legs, and unchanged PR heads. Live token permissions and first-merge dispatch registration remain operational checks, not test-proven claims | +| VP9 routine original-file smoke and full-tier scheduling retain exact coverage | `scripts/vp9_smoke.py` owns the exact-eight selection, checksums, and completion checks. `scripts/tests/test_vp9_smoke.py` covers membership, original bytes, bounded retries, comparator execution, refusal or zero-frame failures, and closed stage/reason diagnostics without arbitrary exception arguments. `scripts/tests/test_vp9_ci_tiers.py` checks corpus events, PR and full fuzz budgets, proof-step wiring, and metadata-only reporting. Types cannot govern workflow scheduling | | Non-required HEVC full jobs run only on explicit main push, schedule or dispatch events, never pull requests or merge groups | `test_full_work_runs_only_on_explicit_full_events` pins both job gates, the schedule, path inputs and PR-only cancellation. Workflow event selection cannot be encoded in Rust types | | Full-suite failure reporting reads only trusted Actions metadata and final-head evidence names its selected suites | `test_ci_full.py` covers trusted workflow wiring, complete proof steps, identity, pagination, races, deduplication and closed suite selection. Runtime API evidence cannot be established by a declared type | | HEVC final-head full evidence includes complete scalar conformance and every original census shard | `FULL_WORKFLOWS` names all four matrix jobs; the shared `PROOF_STEPS` owner requires complete C1 and the packet comparison in each shard. `test_verifier_requires_every_unique_shard_and_complete_scalar_proof` rejects missing or duplicate shards, skipped shard proof and missing scalar comparison. Runtime Actions identities and completion require boundary validation | +| Manual-only HEVC C3 differential fuzz remains outside push/schedule C1 and C2 coverage without losing its dispatch failure gate | `test_manual_only_jobs_never_run_on_automatic_events` closes the exception to named jobs and tests both crons, all automatic events and scalar/mode inputs. The full-evidence test requires exact C1/C2 membership plus exactly two automatic-mode dispatch C3 jobs; reporter/verifier tests retain covered-skip handling and reject a failed differential workflow. Separate event scopes cannot share one unconditional proof roster | | The macOS crop case executes real media with qualified tools rather than passing with zero selected tests | The explicit `check-macos` step requires the exact crop test success line after oracle and x264 provisioning. `test_macos_crop_runs_real_media_with_qualified_tools` checks its command and rejects zero-test and failed-test logs; Rust types cannot select workflow features or prove execution | | The macOS SAR media case actually runs with its qualified tools rather than disappearing in an oracle-free workspace build | The explicit step in `check-macos` installs pinned x264 after oracle provisioning, selects the native-H.264 media test and requires its success line, so zero tests fail. `test_macos_runs_the_declared_sar_media_case_with_qualified_tools` pins the command, success witness, required-oracle environment and provisioning order; Rust types cannot select Cargo features or prevent a zero-test workflow invocation | | Every test in the input-seek oracle fixture target is excluded on unregistered oracle platforms | The shared `test-support/oracle-dependent.json` roster owns the exact selection; `test_input_seek_oracle_tests_are_excluded_on_unregistered_platforms` derives the target's tests and checks each skip. Rust types cannot select test functions in a platform's Cargo invocation | diff --git a/docs/agents/docs.md b/docs/agents/docs.md index a42cfdff3..dfab14e0d 100644 --- a/docs/agents/docs.md +++ b/docs/agents/docs.md @@ -4,6 +4,8 @@ Documentation-site prose, schema wording, and design-note updates. | Rule | Enforced by | |---|---| +| Final HEVC SIMD guidance requires separate scalar and automatic evidence rather than treating a newest-dispatch verifier as a two-mode check | The owning HEVC tier note defines both modes; the central CI procedure links it and requires independently checked run attempts. `test_manual_c3_failure_blocks_verification_but_scalar_skip_has_no_c3_credit` demonstrates the verifier's scalar limit. Operator verification of both runs remains review judgment | +| A weekly full-run drift claim distinguishes re-executing jobs from re-fetching cached pinned inputs | Not enforced: review judgment. Existing acquisition conditions can skip remote downloads on cache hits, so a schedule alone cannot establish fresh remote-input availability | | The documented SVT verification command includes feature-gated process integration targets | `test_documented_checks_include_feature_gated_process_targets` derives required targets from Cargo metadata and checks the published command. Rust types cannot establish arguments in prose; the required AV1 CI job executes that same explicit test selection | | Native `--force` help describes admitted regular-file overwrite rather than symbolic-link replacement | `FORCE_HELP` in `coreav-cli/src/main.rs` owns the wording; `every_native_force_flag_uses_the_shared_destination_help` builds each deferred command and names the shared attribute to restore for any missing consumer. `docs_site` compares generated snippets with actual help. A boolean flag type cannot constrain help metadata or a runtime destination's filesystem type | | HEVC plan scope distinguishes 8-bit Main from 8- or 10-bit Main10 | Not enforced: review of the plan against the existing profile/depth admission contract. Runtime tests already reject invalid profile/depth combinations; a type cannot establish the meaning of prose | diff --git a/docs/agents/release-and-notices.md b/docs/agents/release-and-notices.md index 75ea500eb..9ad26ebaa 100644 --- a/docs/agents/release-and-notices.md +++ b/docs/agents/release-and-notices.md @@ -4,6 +4,7 @@ Licenses, distribution notices, and package rehearsal. | Rule | Enforced by | |---|---| +| A launcher lifecycle fixture publishes a complete positive child PID before the test probes or signals it | The stand-in writes a sibling file and atomically renames it into place; `stopping the launcher stops the binary` validates the record and requires ESRCH after launcher exit. Atomic publication removes the empty-file-to-PID-0 process-group race; cleanup signals only a validated PID. A numeric conversion alone cannot distinguish a complete record from an empty file | | The installer and Node launcher reject the same missing, wrong-version, non-executable or failed-check platform binary, including a Unix version process that ignores termination | `nativeBinary` owns validation and uses SIGKILL on Unix at its timeout. `native.test.mjs` executes recovery without an environment override and runs a real SIGTERM-ignoring child with the production signal option. External package contents and process signals cannot be enforced by a type | | Dependencies keep to the license policy | `cargo deny check` and the fuzz projects' license check (`deny.yml`); `scripts/svt_av1_licenses.py` checks the native SVT dependency against its recipe identity and independently pinned notice bytes (`svt-av1-native.yml`). Its rewritten-manifest regressions reject changing both the text and its recorded hash | | SPDX operators never become notice file names, and neither a named nor a generic file covers an LLVM exception without its complete text | `renderNotices` checks both normalized exception paragraphs after file validation. The registry-term, generic-file content and toolchain-operator regressions in `scripts/notices/test/legal.test.mjs` preserve alternatives and wrapping while refusing base-only text, missing paragraphs, changed permissions and operator filenames (`wasm.yml`, `release.yml`). A filename, identifier or type cannot prove file contents | diff --git a/docs/design/ci-checks.md b/docs/design/ci-checks.md index bfa323563..f382c4e4e 100644 --- a/docs/design/ci-checks.md +++ b/docs/design/ci-checks.md @@ -4,6 +4,61 @@ How the workflows under `.github/workflows` are shaped so that a branch ruleset can require them. The exact list of names it requires is the table under "Required status checks" below. +## Validation tiers + +[Shorter PR validation](ci-tiers.md) records this branch's implemented fixture and npm changes, routine smoke checks, independent full suites, and exact-final-head verification procedure. Its forecasts remain assumptions, not measured improvements. Required context names are unchanged. Full H.264, arm64 conformance, and the three composed-transcode shards and merger belong to `ci-full.yml`, outside `ci-report`. The integrated HEVC tier keeps headers and eight synthetic pictures routine; complete archive and fixed-file work stays in its independent full workflow. + +### Scheduled full runs already covered by main + +Each full workflow first checks daily scheduled runs through the existing `merge-group-result` lookup. Only the newest main-push attempt of the same workflow at the exact scheduled SHA can cover it, and every registered full job, matrix leg and proof step must have succeeded. Running, failed, cancelled, mismatching, incomplete or unavailable evidence runs the full work. Covered dailies do no media or build work and write one line naming the covering run and attempt. Push filters, push events, daily cron times, manual dispatch and PR-only cancellation are unchanged; the metadata gate starts a runner only for the exact daily cron expression. + +The claim is pinned in the coverage step's name. Trusted failure reporting independently verifies that exact attempt using Actions metadata and the same full-proof predicate, so legitimate covered skips do not become missing-work issues. No artifacts or logs are read with its issue-writing token. Other skipped or failed full jobs retain the existing failure path. Manual final-head verification never accepts push or scheduled coverage in place of its dispatches. + +Coverage has no age limit. While main remains at one SHA, a successful covering push can suppress later daily runs. Each of `ci-full`, `vp9-corpus`, `vp9-conformance` and `hevc-conformance` also has one Sunday weekly schedule that bypasses the coverage gate and always runs the existing full jobs, even at a covered SHA. These weekly runs check external toolchain and runner-image drift and use the same trusted failure reporter as daily runs. Existing pinned-input caches remain in use: a cache hit can avoid remote acquisition, so weekly execution does not guarantee detection of changed or unavailable remote inputs. + +HEVC's kernel-differential campaign is manual-only and runs only with automatic kernels, not with the scalar-forced input. It is separate from the registered C1 vector and four C2 census jobs, so a push or schedule does not need a job it cannot select. A failing campaign still makes its dispatch fail verification through the workflow conclusion; a scalar dispatch supplies no C3 credit. The campaign's two architectures, 600-second target budgets and private findings remain unchanged. + +## Verify a correctness PR's final head + +Prerequisite: all workflow files must be registered on the default branch. Before first use, register the new `ci-full.yml` in a dispatch-only change, then dispatch this branch's full implementation. Registration itself is not full-suite evidence. + +1. Read the current branch and exact head from the pull API. Replace `N` with the PR number. + + ```bash + N=448 + REPO=coreav-dev/coreav + BRANCH="$(gh api "repos/$REPO/pulls/$N" --jq '.head.ref')" + SHA="$(gh api "repos/$REPO/pulls/$N" --jq '.head.sha')" + test "$(gh api "repos/$REPO/pulls/$N" --jq '.head.repo.full_name')" = "$REPO" + ``` + +2. Dispatch all three workflows on that branch. + + ```bash + gh workflow run ci-full.yml --repo "$REPO" --ref "$BRANCH" + gh workflow run vp9-corpus.yml --repo "$REPO" --ref "$BRANCH" + gh workflow run vp9-conformance.yml --repo "$REPO" --ref "$BRANCH" -f mode=conformance + ``` + +3. Find the newest dispatch for each workflow at `$SHA` and `$BRANCH`. Use `gh api --paginate "repos/$REPO/actions/workflows/WORKFLOW/runs?event=workflow_dispatch&head_sha=$SHA&per_page=100"` with each workflow filename. Check the run's branch, SHA, creation time, and latest `run_attempt`. Wait for those exact runs with `gh run watch RUN --repo "$REPO" --exit-status`. Check each exact run with `gh run view RUN --repo "$REPO"`; do not substitute an older success or a similarly named required check. +4. Run the verifier after all three complete, immediately before enqueueing. + + ```bash + test "$(gh api "repos/$REPO/pulls/$N" --jq '.head.sha')" = "$SHA" + GH_TOKEN="$(gh auth token)" python3 -I scripts/ci_full.py verify --pr "$N" + ``` + +Stop on any API or verification failure. Repeat after any head change, newer dispatch, or rerun. The verifier re-lists all selected dispatch indexes before its final PR-head check. Successful jobs carried over from an earlier attempt of the same exact-head manual run remain valid; a new attempt must finish successfully before evidence is accepted. HEVC changes additionally dispatch `hevc-conformance.yml` and verify with `--suite hevc-conformance`; repeat `--suite` to verify it together with the three default suites. Relevant audio manual full checks remain additional owning-area obligations. + +For a final HEVC SIMD head, [HEVC CI tiers](hevc-ci-tiers.md) requires both scalar and automatic dispatches. Start scalar mode first and automatic mode second, retaining both run IDs: + +```bash +gh workflow run hevc-conformance.yml --repo "$REPO" --ref "$BRANCH" -f scalar=true +gh workflow run hevc-conformance.yml --repo "$REPO" --ref "$BRANCH" -f scalar=false +``` + +Independently check each run's latest attempt at `$SHA` and `$BRANCH`, including every registered HEVC C1/C2 job and proof step. In the automatic run, also require both `kernel-differential` matrix jobs and their preparation and differential steps to succeed. Recheck both attempts and the PR head immediately before enqueueing. The verifier checks only the newest HEVC dispatch; it does not prove the two-mode requirement, and a scalar run supplies no C3 evidence. + ## The problem Branch protection matches a required status check by **name**, and the name @@ -315,11 +370,12 @@ The sole historical-extraction exception is `training-identity`: `docs/measureme `isobmff-fixtures` ran every check of the ISO base media fixtures in one job: a median of 34.9 minutes over the successful jobs of the last 60 completed pull request runs, against 15.2 for `fuzz`, 11.6 for `bench-guard` and under 9 for every other job, so it alone set how long a pull request waited. Its checks now run in separate jobs, with the same commands, bars and thresholds: -- `isobmff-fixtures`: fixture generation, the oracle self-checks, the stream and packet oracles against `ffprobe` and their known-divergence audit, `coreav probe` and `coreav ffprobe`, the output schemas, the hostile suite through the `coreav` binary, `coreav check`, audio extraction, subtitles and chapters, and the text known-divergence audit. It generates its own fixtures on every run, as before, so every run exercises generation. Its artifact is `oracle-output`. -- `ci-changes` publishes `isobmff-fixture-set`, the one fixture artifact that the three jobs below check, before starting the fan-out. The local `ci-fixtures` action restores the exact generated-fixture cache or generates once on a miss. Generation is not byte for byte reproducible (a WebM fixture generated twice with the same FFmpeg differs), so independent generation in consumers would check different bytes, and B9's `--merge` refuses shards whose fixtures differ. The artifact, not a mutable cache lookup in each consumer, binds the bytes for the run. Moving this short preparation into the existing gate removes a separate producer's runner queue and its additional scheduling level. A cold miss delays all gate-dependent jobs while generating, rather than just the consumers; the gate has a 15-minute limit for that path. `isobmff-fixtures` still generates independently on every full run. +- `isobmff-fixtures`: oracle self-checks, stream and packet comparison and known-divergence checks, probe commands, schemas, hostile inputs, policy checking, audio and text extraction, and the text known-divergence audit. Its artifact is `oracle-output`. +- `isobmff-command-compat`: command coverage, frame listing, and informational R1, with unchanged comparators, baselines, build variants, and verdict handling. It is covered by `ci-report` and retains `isobmff-command-results`. +- `ci-changes` publishes `isobmff-fixture-set` before the fan-out. The local `ci-fixtures` action generates once on every non-reused run, even on a cache hit. Reused main maintenance retains cache restoration. Consumers download the same run-bound artifact and never independently regenerate it. Generation is not byte for byte reproducible, so the artifact binds the bytes rather than separate cache lookups. The gate has a 15-minute limit. `ci-full.yml` has its own fresh producer, `full-fixtures`, which also publishes the composed-audio set. - `isobmff-writers`: clip, remux, video replacement, split, sanitize, HLS, join and the 0.1.0 `ffmpeg` commands. Their reports are the `writer-output` artifact. - `isobmff-decode`: the build with the FFmpeg fallback, dav1d and libvpx, then decoding, thumbnails and `coreav transcode` against the pinned FFmpeg, and the hostile suite through `coreav thumbnail` and `coreav transcode`. -- `transcode-compat-shard`, three legs, and `transcode-compat`: the explicitly selected `composed-opus-aac-v2` registration. The legacy B9 reader and frozen records keep their original exact semantics. +- In `ci-full.yml`, `transcode-compat-shard`, three legs, and `transcode-compat`: the explicitly selected `composed-opus-aac-v2` registration, with unchanged shard and merger predicates. The legacy B9 reader and frozen records keep their original exact semantics. The CLI transcode test helpers bound each `coreav transcode` call to 90 seconds, each stand-in script writer to 30 seconds, and the hostile-encoder test's single-picture FFmpeg generation to 60 seconds. Input writes and both output reads run concurrently. On a stall, Linux records process and thread states, wait channels, file descriptors and owners of the tracked pipe inodes, including reparented owners, before cleanup. Family threads also report syscall numbers and timer slack. On Linux x86-64, timed futex waits in the root executable and its workers include a 16-byte timespec sample and the matching clock when the syscall agrees before and after the read, so an expired wait can be distinguished from a future deadline. Arguments, environment values, general memory dumps and core dumps are not collected. One owned collector has a two-second caller budget and is never joined; the existing process/file/thread caps and 32 KiB text cap remain. `COREAV_TEST_DIAGNOSTICS` selects the test-only artifact directory; timeout paths write a snapshot on a best-effort basis, and a write failure does not replace the timeout. The transcode fixture step uploads that directory on failure. Other platforms keep the child and pending-I/O report without the Linux snapshot. These are diagnostic limits, not a fix for the original unexplained stall, and no production deadline or CI job timeout changes. @@ -377,23 +433,33 @@ The `main` ruleset can require a merge queue. The queue applies each pull reques - **Every workflow with a required check runs on `merge_group`**: `deny.yml`, `ci.yml`, `python.yml` and `npm.yml`. A required check that never reports holds the queue until its timeout and then ejects the pull request. `workflow_check.py` fails on a workflow in the table without the trigger, whose `types:` do not include `checks_requested` as a whole item, or whose `merge_group:` trigger carries a `branches` or `branches-ignore` filter. Ruby's standard YAML library (Psych), already used for action pins, parses the trigger mapping: quoted, escaped and spaced keys and flow mappings mean the same thing as their plain block forms. `types:` may be a scalar or a flow or block sequence. Duplicate keys, merge keys, aliases, complex keys and explicit tags in the trigger mapping are refused rather than ignored; the error names the form to expand or remove. No YAML loader constructs objects, so YAML 1.1's boolean reading of `on` cannot hide the trigger. A branch filter is refused even when it names `main`: a required check has to report for every group, and the checker would otherwise have to read branch patterns as GitHub does. - **The changes jobs diff the group's base against the group's commit** (`github.event.merge_group.base_sha` to the checked-out `head_sha`), so a group runs the jobs a pull request with the same changes would, the platform jobs included when it touches their list. The base is the tip of `main` or the commit of the group queued ahead; either can be more than the one parent `fetch-depth: 2` brings, so the changed-paths action fetches it (`--depth=1`) before the diff. When it cannot be fetched, everything runs. -- **`bench-guard` counts the group's base**, fetched the same way, as a pull request's run counts the first parent of its merge commit, and compares the group's counts with it. +- **`bench-guard` counts the group's base**, fetched the same way, as a pull request's run counts the first parent of its merge commit, and compares the group's counts with it. A proven reuse requires that exact tested base as well as the tree, so it retains the same comparison rather than substituting the baseline. - **No fuzzing in the queue.** `fuzz-plan` stays empty for a merge group, so `fuzz` is skipped and `ci-report` covers it. The pull request already fuzzed every target its change reaches, on its own merge commit, and `fuzz-nightly.yml` runs every target on `main` each day. A fuzz run is a search, not a property of the commit: a finding in the queue would most likely be an old bug found by chance, and it would eject whichever pull request happened to be queued. The fuzz job, among the longest of a pull request's run, would also lengthen every entry. -- **No pull request text or DCO check in the queue.** A merge group has no pull request. `pr-text.yml` is not required and does not run on it. The DCO step of `deny-checks` runs on `pull_request` only: the queue adds no contributor's commit, only the squash of a pull request whose commits that step passed, as the merge button would. Every other step of `deny-checks` runs, and `deny` reports the result. +- **No pull request text or DCO check in the queue.** A merge group has no pull request. `pr-text.yml` is not required and does not run on it. The DCO step of `deny-checks` runs on `pull_request` only: the queue adds no contributor's commit, only the squash of a pull request whose commits that step passed, as the merge button would. Without proven reuse, every other step of `deny-checks` runs, and `deny` reports the result. - **The label-gated builds are skipped**: a merge group carries no labels, and no report job covers those jobs ("What a report job must not hide"). - **The draft condition** lets every job run on a merge group: only a ready pull request can be queued. - **Concurrency.** Merge groups have run-id groups of their own and cannot cancel one another. Main jobs in the filtered workflows are independent too; an unfiltered deny replacement may cancel its whole predecessor ("Concurrency"). `workflow_check.py` refuses a required workflow with neither a workflow-level group nor job groups: its superseded PR runs would never be cancelled. When the queue drops a group, GitHub deletes its branch and starts a new group; the old run is not cancelled and ends on its own. +### Merge groups after a pull request + +Merge-group reuse keeps `bench-guard` by accepting only the PR run's comparison of the identical tested tree against the identical base; main pushes still compare against the committed baseline. + +The existing `merge-group-result` action also records the actual checkout of each ready, same-repository PR. A small artifact's name binds its tested tree, first-parent base and run attempt. The helper requires the checked-out merge SHA and both parents to match the event. Dependabot-authored or triggered runs do not publish reusable receipts because their secret availability differs. Receipt upload failure leaves ordinary checks running but makes later reuse unavailable. + +A merge group may reuse only the newest successful PR run of the same workflow at the queued PR's current head, created within 24 hours and before the group. The helper validates the group's own tree and base, repository identities, workflow, current open non-draft PR, and one unexpired attempt-specific receipt. Both the tested tree and tested base must match. It rechecks the newest run and PR head after receipt lookup; newer failures, pending runs or reruns block older evidence. The PR API's head-commit tree is not the tested merge tree and is never a substitute. Missing, ambiguous, incomplete, changed or unavailable metadata runs the normal suite. No artifact content or log is downloaded. + +On a hit, the required workflows keep their changes and report jobs; `deny-checks` also retains expression validation and freshly checks advisories for the root and every discovered fuzz manifest. The advisory database is fetched at execution, never restored from a cache or checked offline. An always-run witness requires the fresh step's successful outcome, so a missing or skipped scan fails the job and its strict report. Tree-only licence checks can be reused, but a previous advisory result cannot replace this current scan. The queue does not regenerate fixtures, start main-cache maintenance, or repeat already-proved build, media and head-versus-base instruction checks. Required context names and independent run groups remain unchanged. Full suites never consume this evidence. Non-required workflows do not run on `merge_group`; `workflow_check.py` rejects the trigger even in quoted or shorthand forms. The encoder workflow retains its PR, main and manual checks. + ## Main pushes after the queue A squash commit does not have the merge group's commit SHA, but can have the same Git tree. `.github/actions/merge-group-result` is the single lookup used by each required workflow. On a push to `main` only, it searches up to 20 recent merge-group runs of that same workflow created in the last 24 hours. The newest matching tree must have completed successfully; a newer matching failure or unfinished run blocks reuse of an older success. It accepts a run only if it completed successfully, came from this repository's `gh-readonly-queue/main/` branch, names the expected workflow file, and its head commit's tree equals the checked-out tree. It logs and links the reused run in the job summary. Commit messages, parents and squash SHAs need not match; every tracked input, including the workflows and the lookup itself, must match. The run API retains `head_commit.tree_id` alongside the head commit id, so the comparison needs one metadata request per workflow and does not depend on fetching a deleted queue branch. -Missing evidence, a mismatching tree, an API error, unavailable commit metadata, or the search limit means the normal suite runs. PRs, merge groups, tags, schedules and manual runs never reuse results. The token gets `actions: read` as well as `contents: read`, and only the lookup step sees it. +Missing evidence, a mismatching tree, an API error, unavailable commit metadata, or the search limit means the normal suite runs. PRs, tags, schedules and manual runs never reuse results; merge groups have the stricter PR-receipt path above. The token gets `actions: read` as well as `contents: read`, and only the lookup step sees it. On a proven tree, `python-report` and `npm-report` retain their changes and report jobs, but skip the repeated wheel/package tests. `deny-checks` logs its evidence instead of repeating the licence, advisory and repository checks. Their required names still report conclusions. `ci-report` retains: - `ci-changes` and the report itself, so the evidence and all maintenance outcomes remain visible. -- `bench-guard` against the committed baseline, and `conformance-arm64` when its paths changed. The queue checks neither of these main-only comparisons. +- `bench-guard` against the committed baseline. Arm64 conformance belongs to the independent full tier, not reused-main maintenance. - Fixture preparation inside `ci-changes`, which updates main's generated-fixture cache, and a build-only `main-cache` job using `isobmff-fixtures`' shared Rust-cache key. The latter warms the oracle/CLI dependencies and fallback builds without rerunning tests. When evidence is missing, the ordinary fixture jobs run and warm those caches instead. The other job-specific caches are refreshed by full fallback/manual runs rather than by every merge. A reused main push records no second copy of the queue's oracle results; its summary names the run holding them. Main-only maintenance and fallback checks continue in their original run, even when later pushes arrive or older gates finish late. @@ -558,7 +624,7 @@ None of them adds a required check name, so the table below is unchanged. ## The 0.1.0 compatibility round -`compat-0.1.0.yml` runs R2 and R3 of the 0.1.0 round (`docs/measurements/compat-0.1.0/README.md`) by hand, at a commit given as input, on spent files (`rehearsal`) or on draw 6 (`draw6`, which also needs `open_draw`), split over four runners. The R2 and R3 shards receive the corpus credentials only in their fetch steps. The R4 rehearsal and scan steps combine fetching with tool execution and retain those read-only credentials in their environment. It publishes counts only, in its job summary and the `compat-0.1.0-summary` artifact; the per-file records are sealed to `tools/compat-corpus/evidence-public.pem` before they leave a runner, because an artifact of a public repository is readable by anyone. A pull request that changes the workflow runs the rehearsal, which also runs the R4 host's job on a runner (`scripts/compat_r4_gcp.py --local`) and scans which R4 pairs of the gate's and the rehearsal's files write to the fallback log (`tools/compat-corpus/r4_scan.py`): any such line voids R4, so the scan names, before the round, the commands and files that would void it. It is not required (`# workflow-check: not required`): it is a measurement, not a gate. R1 runs as a step of the three fixture jobs in `ci.yml` (`isobmff-fixtures`, `mkv-fixtures`, `ts-fixtures`), `run.py commands --set 0.1.0` with no baseline; its verdict goes to their job summaries and its report to their artifacts, and a miss or a void run does not fail them, so a change of the oracle's output is reported there without blocking unrelated pull requests; a run that measured nothing fails them. The workflow's report step likewise takes a miss or a void run as a result and fails on a run that measured nothing. +`compat-0.1.0.yml` runs R2 and R3 of the 0.1.0 round (`docs/measurements/compat-0.1.0/README.md`) by hand, at a commit given as input, on spent files (`rehearsal`) or on draw 6 (`draw6`, which also needs `open_draw`), split over four runners. The R2 and R3 shards receive the corpus credentials only in their fetch steps. The R4 rehearsal and scan steps combine fetching with tool execution and retain those read-only credentials in their environment. It publishes counts only, in its job summary and the `compat-0.1.0-summary` artifact; the per-file records are sealed to `tools/compat-corpus/evidence-public.pem` before they leave a runner, because an artifact of a public repository is readable by anyone. A pull request that changes the workflow runs the rehearsal, which also runs the R4 host's job on a runner (`scripts/compat_r4_gcp.py --local`) and scans which R4 pairs of the gate's and the rehearsal's files write to the fallback log (`tools/compat-corpus/r4_scan.py`): any such line voids R4, so the scan names, before the round, the commands and files that would void it. It is not required (`# workflow-check: not required`): it is a measurement, not a gate. R1 runs as a step of the three fixture jobs in `ci.yml` (`isobmff-command-compat`, `mkv-fixtures`, `ts-fixtures`), `run.py commands --set 0.1.0` with no baseline; its verdict goes to their job summaries and its report to their artifacts, and a miss or a void run does not fail them, so a change of the oracle's output is reported there without blocking unrelated pull requests; a run that measured nothing fails them. The workflow's report step likewise takes a miss or a void run as a result and fails on a run that measured nothing. `codec-coverage.yml` runs V1, V2 and V3 of the 0.1.0 codec coverage bars (`docs/measurements/compat-0.1.0/codec-coverage-plan.md`) by hand, at a commit given as input, on the rehearsal's 40 files of corpus v1 (`rehearsal`) or on every `real` file of corpus v1 (`v1`). It builds the plan's run manifest once (the full build's `coreav`, `decode-oracle` linked against the same static dav1d and libvpx, the `ffmpeg` test executable with `release-full`, and x264), runs V1 over 40 runners for `v1` and 4 for `rehearsal`, and runs V2 and V3 in one job on the fixtures the pinned FFmpeg makes. A V1 shard fetches each file when its turn comes and deletes it after its record, because a shard's files do not fit a runner's disk at once, so the corpus store's read-only credentials stay in that step's environment; `decode_breadth.py` passes none of them to the programs it runs on the files. V1's per-file records are sealed to `tools/compat-corpus/evidence-public.pem` before they leave a runner, and a runner refuses to seal to a key whose fingerprint is not the one the workflow names. A shard stops starting files, and stops the sides still running, at a deadline before the job's limit, so that its records are sealed; those files, and those of a shard that reported nothing, run again on one Compute Engine VM (`scripts/codec_coverage_gcp.py rerun`). The report job puts the manifest together and fails when an item differs from the plan's table. A pull request that changes the workflow or the tools runs V2 and V3 on the fixtures and V1's tests on the decode fixtures they make, and reads no corpus file. It is not required (`# workflow-check: not required`): it is a measurement, not a gate. @@ -614,7 +680,7 @@ failed. | Check | Workflow | Note | |---|---|---| | `deny` | `deny.yml` | The licence, advisory, repository and PR DCO checks, reported by the `deny` job, which fails when they failed, were cancelled or were skipped; on main, a successful same-tree merge-group result may replace repeated checks | -| `ci-report` | `ci.yml` | Every job of `ci.yml`: `cargo fmt`, `clippy` and the workspace tests on Linux, `clippy` and the workspace tests on macOS arm64 and Windows x86-64, the MSRV build, on a pull request the fuzz run of the targets it can affect, the ISO base media, Matroska and transport stream oracles, H.264 conformance (bar C1 of `docs/design/h264.md`) and the instruction-count guard; on proven main trees, only the baseline guard, arm64 conformance and shared-cache maintenance are repeated | +| `ci-report` | `ci.yml` | Every job of `ci.yml`: formatting, strict Clippy and workspace tests with existing platform filters, MSRV, affected-target PR fuzzing, ISO base media fixtures including `isobmff-command-compat`, Matroska and transport stream oracles, exact-eight H.264 fast and safe smoke, kernel and identity checks, SVT-AV1 and the instruction-count guard; proven main trees retain the baseline guard and shared-cache maintenance. Full H.264, arm64 conformance and composed-transcode shards and merger run independently in `ci-full.yml`, not under this report | | `python-report` | `python.yml` | The abi3 wheel and its tests, or on main their successful same-tree merge-group result | | `npm-report` | `npm.yml` | The npm packages, launcher and smoke tests, or on main their successful same-tree merge-group result | diff --git a/docs/design/ci-tiers.md b/docs/design/ci-tiers.md new file mode 100644 index 000000000..74a5665a9 --- /dev/null +++ b/docs/design/ci-tiers.md @@ -0,0 +1,148 @@ +# Shorter PR validation + +Status: implemented in this branch. The first commit recorded the plan and changed no executable check; implementation follows in the same pull request. The HEVC tiering prerequisite is integrated. The forecasts below are not measured worker improvements. + +## The measured problem + +The maintainer's direction is a routine PR gate of 20–30 minutes, with full compatibility checks after merge and daily. A correctness PR runs the applicable full checks once at its final head. A regression visible only to those checks can therefore reach main before detection. + +The 2026-11-06 API sample contains 2,325 runs and 12,391 jobs across 41 current and historical workflow records. The four required workflows have up to 60 runs per event. Other workflows have up to 60 recent relevant runs. Equal job names in historical workflows are not pooled with the current workflow. + +Each successful required cohort uses the latest selected run of `ci.yml`, `deny.yml`, `python.yml` and `npm.yml` at one event and head SHA. Its interval starts at the earliest workflow creation and ends at the last required report completion. A source cohort means Linux `check` actually started. It does not mean every platform job ran. Failed, cancelled and unfinished heads are excluded from successful quantiles, not counted as fast completions. + +| Successful cohort | Heads | Median minutes | p90 minutes | +|---|---:|---:|---:| +| Required PR source heads | 32 | 21.27 | 25.78 | +| Required merge-group source heads | 50 | 25.04 | 41.91 | +| HEVC-triggered PR heads, all enumerated PR Actions | 20 of 58 | 43.58 | 79.73 | +| VP9-corpus-triggered PR heads, all enumerated PR Actions | 30 of 59 | 36.65 | 64.85 | +| Audio-triggered PR heads, all enumerated PR Actions | 32 of 59 | 31.66 | 55.27 | + +The heavy-path rows use separate exact-SHA enumeration of PR Actions, rather than requiring membership in the 32-head source sample. They overlap. They exclude external app checks, reviews and later runs, so they are not unconditional merge-readiness measurements. In the HEVC sample, 19 trigger runs were cancelled, nine failed and one was still running. The corresponding successful-only quantiles omit those outcomes. + +`ci-report` finishes last on 15 of the 32 source PR heads, `npm-report` on 14 and `deny` on three. The main serial costs are fixture command checking and npm WASI parity. Queue delay also matters. At head `815ab24b`, Linux `check` queued for 31.83 minutes and executed for 14.93. At head `28b2365a`, `isobmff-decode` queued for 28.55 minutes and executed for 13.42. The worst five merge-group cohorts all used attempt 1. Only one of 50 had any rerun (`deny`, attempt 2). Cross-SHA ejection is not established by this sample. Full H.264 acquisition took 24.70–34.10 minutes in four merge-group legs, despite a warm full-conformance median near four minutes. + +## Changes to the critical paths + +### Keep npm parity complete and use bounded workers + +Before this change, npm parity took 13.78 minutes median on PRs and 16.40 on merge groups. Native package compilation took 3.00 and 3.40 minutes, respectively. WASI compilation took 1.85 and 2.12 minutes. The parity test now uses bounded workers instead of serial comparisons. + +The test enumerates the existing fixture, command, and three path-variant combinations before execution. At most four asynchronous workers run inside the existing job. Each owns an exclusive scratch directory. Native execution, capture, cleanup, WASI execution, and byte comparison remain ordered within each case. Bounded output capture and child-failure behavior remain. Workers retain counters and the earliest failure, not all cases' binary outputs. After active workers drain, the test reports the earliest failing enumeration index deterministically. + +Every existing output, exit-code, stderr, successful-command, and temporary-file assertion remains. Every enumerated case must execute exactly once. `outputs > 50` and the complete successful-command set remain. Native launcher and installation lifecycle tests stay whole. One-worker mode remains available for verification. The worker change adds no runner jobs, artifact transfers, or package build variants. + +### Split fixture command checks without dropping them + +Before the split, `isobmff-fixtures` took 18.60 minutes median and 19.50 p90 on PRs. Its command-coverage, frame-list, and informational R1 steps now run in `isobmff-command-compat`, covered by `ci-report`. Their pre-change PR medians are 1.77, 2.02, and 3.43 minutes. All comparators, baselines, profiles, flags, and R1 verdict handling are unchanged. + +Both jobs consume the same run-bound fixture artifact. On a full run, fixture preparation in `ci-changes` generates the set even on a cache hit before publishing it. Reused main maintenance keeps its existing cache behavior. The full-run generation obligation moves to that producer rather than disappearing. No consumer independently regenerates the shared set. + +The base fixture job retains stream and packet comparison, dependent known-divergence checks, schemas, hostile inputs, policy checking, audio and text extraction, and the text known-divergence audit. The latter takes 3.77 minutes but stays in the base job to balance the two branches. The new job builds the same required command variants and retains its reports under a separate artifact name. + +### Reuse the exact tested tree and base in the queue + +The longest ten projected merge-group cohorts are limited by Linux `check` four times, decode fixtures twice, and macOS checks, npm smoke, deny and bench-guard once each. These are the projected last prerequisites plus retained report admission, not proof of a single cause. Full H.264 and transcode moves alone cannot remove this tail. + +The required workflows record the actual ready-PR checkout tree, first-parent base and attempt through the existing `merge-group-result` action. A merge group reuses only the newest successful same-workflow PR run at its current head when both recorded identities match. This preserves changed-path selection and the head-versus-base instruction comparison. A new base, missing receipt, newer failure or pending run, rerun without a fresh attempt receipt, fork, Dependabot run, incomplete index or API failure runs the normal suite. Main cache maintenance and fixture generation do not start on queue hits; main's baseline guard and all independent full suites retain their existing obligations. Advisory checks remain fresh on a hit for the root and every discovered fuzz manifest; an explicit outcome witness refuses a skipped scan. A failed-job-only rerun without a new changes-job receipt safely misses rather than borrowing the earlier attempt's receipt. + +The non-required H.264 encoder workflow no longer repeats in the queue. It retains PR, main and manual validation. The workflow checker enforces the absence of merge-group triggers on non-required workflows, including quoted and shorthand triggers. + +## Implemented job tiers + +These are this branch's job tiers on the normal, non-reused path. A proved queue hit retains only gates, reports and fresh advisory checks as described above. Durations are pre-change successful PR job execution median/p90 in minutes unless marked otherwise. They exclude runner queues. Matrix rows group related legs without adding their parallel durations. The required names remain in [the required-check table](ci-checks.md#required-status-checks-exact-names). + +| Jobs | Measured execution | Tier | +|---|---|---| +| `ci-changes` | 0.32 / 0.42 | PR and merge group, with fresh shared fixtures on full runs | +| `check` | 14.85 / 16.82 | PR and merge group, unchanged configurations | +| `check-macos`, `check-windows` | 11.13 / 13.08; 10.57 / 11.37 | PR and merge group with the existing platform-path filters | +| `msrv` | 0.92 / 1.00 | PR and merge group | +| `fuzz`, two shards | 6.20–8.33 medians | PR compilation and existing short affected-target runs; no queue campaign | +| `isobmff-fixtures` | 18.60 / 19.50 before split | PR and merge group, base fixture checks | +| `isobmff-command-compat` | modelled near 11–12, not measured | Required PR and merge group, unchanged moved checks | +| `isobmff-writers` | 10.35 / 11.30 | PR and merge group | +| `isobmff-decode` | 13.38 / 15.21 | PR and merge group, including complete AAC-enabled transcode fixture targets | +| `mkv-fixtures`, `ts-fixtures` | 8.18 / 8.69; 7.55 / 8.09 | PR and merge group | +| `conformance` | 3.87 / 4.10 on PRs; merge-group legs reached 37.93 | PR and merge-group smoke; full H.264 moves after merge, daily and dispatch | +| `vector-width`, `modes-identity`, `kernels-arm64` | 0.83–1.03 medians | PR and merge group, including all current Clippy invocations | +| `training-identity` | 6.00 / 6.99 | PR and merge group | +| `bench-guard` | 14.32 / 21.05 | Required PR and merge-group base comparison, unchanged | +| `svt-av1` | 6.99 / 11.73 | PR and merge group, retaining its existing tests and licence coverage | +| `transcode-compat-shard`, three legs; `transcode-compat` | 8.18–9.18 shard medians; 0.27 merger | After merge, daily and final-head dispatch, outside `ci-report` | +| `conformance-arm64` | formerly main-only | Full-suite workflow after merge, daily and dispatch | +| `main-cache` | main-only today | Existing reused-main cache maintenance | +| `ci-report` | 0.05 / 0.07 | Required PR and merge-group report | +| `deny-checks`, `deny` | 13.88 / 14.83 work; report seconds | Unfiltered required checks, unchanged | +| `python-changes`, `python-fixtures`, `linux-x86_64`, `python-report` | fixture median 0.60; wheel median 6.37 | Existing required Python path and same-tree reuse | +| `npm-changes`, `package`, `smoke`, `npm-report` | package 20.38 / 24.90 | Existing required npm path, complete parity with bounded workers | +| Label-selected Python and npm platform builds | opt-in today | Existing opt-in, tag, scheduled and manual behavior | +| `pr-text` | short, edit-triggered | Existing PR text gate, not a merge-group ruleset context | +| HEVC workflow | headers 0.46–0.60; vectors 5.50 / 12.73; census 19.82 / 27.48 before the full inter census | Routine headers and eight synthetic Main10 pictures; complete archive and fixed-file work after merge, daily or final-head dispatch | +| VP9 `vectors`, `adapter-contract` | vectors 11.80–14.01 medians; contract 0.92 | Existing PR comparisons, including the complete 353-vector set | +| VP9 `interpolation-fuzz` | about 11.1 per platform | Compile and run 30 seconds on PRs; retain the 600-second pass after merge, daily and dispatch | +| VP9 `contract`, `corpus`, `corpus-summary` | slow shard 19.60 / 24.51 | Contract and exact-eight original-file smoke on PRs; full corpus on relevant main pushes, daily and dispatch | +| Audio `raw-pcm`, `flac`, `mp3`, `mp3-transforms`, `opus`, `aac` | 0.20–14.85 medians; AAC p90 17.12 | Existing PR checks, retaining their complete lists and qualification controls | +| `h264-encoder-conformance` | 2.18 / 2.47 | Existing fast encoder checks, including strict Clippy | + +Other workflows retain their existing scope and events. In particular, publication and release procedures do not change. The frozen legacy compatibility workflow and judging registration stay byte-identical. A separately registered current-head R3 regression is a follow-up, not part of this implementation. + +### Smoke selection does not change registered lists + +H.264 smoke selects exact names from `crates/coreav-h264/tests/data/conformance-streams.json`: `BA1_Sony_D`, `CABACI3_Sony_B`, `cavlc_mot_frm0_full_B`, `HCHP1_HHI_B`, `HPCV_BRCM_A`, `alphaconformanceG`, `CVPCMNL1_SVA_C`, and `MR1_BT_A`. Acquisition and both fast and safe decoding use the same selection, existing thread counts, and comparisons. Exactly those eight distinct in-scope names and results are required. Unknown, absent, or extra results fail. Full checks still use all 99 admitted streams and unchanged hashes. + +HEVC routine smoke compares the existing four registered Main10 recipes and all eight 64×64 unfiltered pictures through unchanged metadata and pixel validators. It is not full 8-bit, filtered-intra or inter coverage. Complete archive and fixed-file checks retain their full job bodies in the independent workflow. HEVC final-head evidence is explicitly selected; it does not become a multi-hour default obligation for unrelated correctness PRs. + +VP9 retains all 353 vectors in routine `vp9-conformance.yml` validation. Separately, `vp9-corpus.yml` runs `scripts/vp9_smoke.py` over exactly eight existing pinned public vectors as original files, using the corpus comparator and pinned oracle. Hash, membership, positive frame count, and complete exact coverage are required. Full corpus shards run only on relevant main pushes, daily, and dispatch. Interpolation fuzzing compiles and runs for 30 seconds on PRs and 600 seconds in the full tier. Audio retains its complete existing coverage and qualification controls; no reduction is approved. No roster, baseline, tolerance or frozen judging file changes. + +The original-file smoke uses inputs whose container metadata satisfies the unchanged C2 admission checks. The intra-only vector remains in the complete 353-vector suite, but its container probe exposes unknown bit depth and subsampling, so it receives no original-file pixel credit. The smoke selects the existing `vp90-2-07-frame_parallel-1.webm` instead; it never skips an admission refusal or substitutes metadata from the oracle. Failure reports name the stage and a closed reason without exposing arbitrary exception arguments. + +## Full checks and final-head evidence + +`ci-full.yml` owns all 99 admitted H.264 streams in fast and safe modes at thread counts 1, 2, and 8, arm64 conformance, and the unchanged three transcode shards and merger. Its fresh producer prepares the registered fixture sets once and keeps exact shard completeness, byte binding, and merger checks. Full work runs after relevant main pushes, daily without path filtering, and by manual dispatch, independently of required-workflow same-tree reuse. + +`ci-full-failure.yml` observes completed `ci-full`, `vp9-corpus`, `vp9-conformance` and `hevc-conformance` runs. It executes main's reporting code, not code from the observed branch. It accepts only this repository's main-push and scheduled runs and reads complete observed-attempt job and proof-step metadata, never logs or artifacts. Its isolated token needs `actions: read` and `issues: write`, with no new secret. PR, merge-group, dispatch, and fork results cannot create issues. + +The reporter uses fixed workflow, job, and result labels and public run links. It creates or updates bot-owned area issues, deduplicates attempts, and reconciles simultaneous creation without cancelling another report. Complete pagination and successful non-skipped proof steps are required. API failure or incomplete evidence fails reporting rather than producing a false success. The owning area fixes or reverts within one day. There is no automatic revert or regression-issue closure; only duplicate issues are reconciled. + +For a correctness PR, require successful manual evidence at its exact final head before enqueueing. The verifier defaults to **all three general suites**; explicitly include the area-selected HEVC suite when applicable. Every selected suite's registered matrix legs and proof steps must succeed without skipping. It uses the newest dispatch and latest attempt, with complete pagination, and rechecks that the open PR head is unchanged. A later failed or pending dispatch or a new commit invalidates the evidence. A green `ci-report` is not full-suite evidence. + +The operational procedure is in [Verify a correctness PR's final head](ci-checks.md#verify-a-correctness-prs-final-head). + +All four daily full workflows avoid a duplicate only when their newest successful main-push attempt proves the exact scheduled SHA, every registered full job and every proof step. The existing lookup owns search and revalidation, while the full-suite registry owns the shared coverage predicate. A covered daily records the pinned run and attempt in metadata; the trusted reporter rechecks it without reading logs or artifacts. Missing or invalid evidence runs the full work, and manual dispatches never reuse it. + +There is no age cutoff: an unchanged SHA with a green covering push keeps skipping daily runs. Each workflow also has one Sunday weekly trigger that never enters the coverage lookup, so the existing full jobs test external dependency and runner drift even while main stays at one commit. Weekly failures use the same trusted reporter; no new jobs, runners or manual inputs are added. + +## Forecast and its limits + +Replay each of the 32 PR and 50 merge-group successful source cohorts using its own job and step timestamps. Keep observed queues, unchanged jobs and report admission delays. Do not add median stage times together or assume that moving work eliminates hosted-runner queues. + +The model adds 0.4 minutes for fresh common fixture production. The new fixture job receives its moved measured steps plus four minutes for duplicate setup, compilation, transfer and admission. Npm native tests receive a conservative 0.5-minute allowance plus 0.1 minutes of worker overhead. The pool's effective speedup is an assumption, not a measured result. H.264 smoke is assumed to take at most five execution minutes; that assumption must be checked on Actions. No queue improvement is credited to the heavy-tier moves. + +| Source cohort scenario | PR median / p90 minutes | Merge-group median / p90 minutes | +|---|---|---| +| Observed baseline | 21.27 / 25.78 | 25.04 / 41.91 | +| Forecast with assumed effective 1.5× npm parity | 18.63 / 23.40 | 20.47 / 41.83 | +| Forecast with assumed effective 2× npm parity | 17.17 / 23.40 | 19.06 / 39.81 | + +These forecasts describe routine required validation on the matched source cohorts, not every heavy-triggered PR, an unconditional service guarantee, or the final-head manual exception. Heavy-path measurements are reported separately above. Moving full suites prevents their automatic execution on every PR revision, but a numerical forecast for their full final-head manual duration is not justified. + +The queue-reuse extension has separate conditional projections on the same 50 successful source cohorts. It retains each gate's observed queue and each report's admission/execution lag, removes only the work skipped by proven reuse, retains deny's container preparation, expression validation and cleanup, and adds 1.1 minutes per workflow for the bounded metadata lookup. Fresh advisory work uses historical root full-deny, binary installation and fuzz-licence durations as a proxy, plus one extra minute for network work; this is an unmeasured allowance, not a runtime bound. Gates run in parallel. No queue relief from fewer jobs or the encoder-trigger removal is credited. + +| Conditional queue scenario | Projected median / p90 minutes | +|---|---| +| All four workflows prove the tested tree and base | 3.31 / 10.51 | +| All miss, assumed effective 1.5× npm parity | 21.57 / 42.93 | +| All miss, assumed effective 2× npm parity | 20.16 / 40.91 | + +These are counterfactuals, not observed latencies or an overall projection. Seven of 13 cache-identifiable groups had matching prior tested trees and bases, but 37 of 50 lack a prior PR association, including all ten slowest projected cohorts. Branch association and incomplete run coverage add selection uncertainty. This cannot establish a reuse probability, especially in the tail. Mixed hit/miss workflows lie between the conditional paths for each cohort; their aggregate median and p90 cannot be inferred by averaging these quantiles. Measure actual receipt hits, fallbacks, failures, unfinished runs and queue latency after deployment before claiming an overall sub-30-minute p90. + +## Atomic implementation and verification + +This branch updates the required-check table, workflow dependencies and triggers, `workflow_check.py` tier and merge-group rules, scheduling tests, and affected agent rules together. The first commit remains the historical plan-only commit. Enforcement claims below refer to implemented tests, not production timing evidence. + +Preserve the prescribed filtered-workflow per-job groups, matrix suffixes, PR-only cancellation, independent main/merge-group run groups, report behavior and deny's existing whole-run exception. New required jobs use the same forms. A main push may reuse only a successful same-tree result of the same workflow. That reuse covers the reduced gate, never the full tier. Required context names stay `ci-report`, `deny`, `python-report` and `npm-report`. + +Deny, licence checks, DCO, pr-text, workflow checking, unsafe-site checking, and strict Clippy configurations remain. Bench-guard remains a head-versus-base comparison in PRs and merge groups and a baseline comparison on main. The reductions are full H.264 and composed-transcode work on each PR and queue run, VP9's per-PR fuzz duration, and VP9 corpus work on each PR revision. Their original full predicates remain in the full tier. Legacy R3 stays frozen and unchanged. No additional runner capacity is requested; only schedules add a small metadata gate. + +Verification includes complete workflow-checker self-tests, main-plan scheduling and matrix tests, changed-path tests, exact smoke membership and result-completeness tests, shared-fixture binding, reporter trust/idempotence tests, and final-head evidence tests. Test the worker executor on synthetic children for exact case coverage, isolated outputs, deterministic failures, bounded output, launch errors and cleanup. On Actions, compare one-worker and parallel npm runs over identical generated bytes and require equal complete case coverage. Run the resulting ordinary PR gate and observe actual queue outcomes. Keep all heavy verification on Actions. diff --git a/packages/cli/test/cli.test.mjs b/packages/cli/test/cli.test.mjs index 6db095e7c..002832cca 100644 --- a/packages/cli/test/cli.test.mjs +++ b/packages/cli/test/cli.test.mjs @@ -7,11 +7,12 @@ import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; +import { compareCases } from "./parity.mjs"; const here = dirname(fileURLToPath(import.meta.url)); const launcher = join(here, "..", "lib", "wasi.mjs"); @@ -23,6 +24,8 @@ const skip = !bin || dirs.length === 0 ? "needs COREAV_BIN and COREAV_FIXTURE_DI /** Exit status and output as bytes: a clip to `-` writes MP4 to stdout. */ function run(command, args, cwd) { const done = spawnSync(command, args, { cwd, maxBuffer: 64 << 20 }); + if (done.error) throw done.error; + if (done.signal) throw new Error(`child terminated by ${done.signal}`); return { status: done.status, stdout: done.stdout, stderr: done.stderr }; } @@ -66,20 +69,6 @@ test("a file written to stdout survives a reader slower than the writer", { skip assert.ok(Buffer.concat(chunks).equals(expected), "stdout differs from the native command's"); }); -/** A case's output: a file's bytes, or every segment `split` wrote, by name; null if none. */ -function snapshot(output) { - if (!output.includes("%")) return existsSync(output) ? readFileSync(output) : null; - const dir = dirname(output); - const names = existsSync(dir) ? readdirSync(dir).sort() : []; - return names.length === 0 ? null : names.map((n) => [n, readFileSync(join(dir, n))]); -} - -function clear(output) { - if (!output.includes("%")) return rmSync(output, { force: true }); - rmSync(dirname(output), { recursive: true, force: true }); - mkdirSync(dirname(output)); -} - /** * Every case: [label, arguments with IN and OUT (a file) or SEGMENTS (a * `split` pattern) placeholders, writes an output]. @@ -104,53 +93,21 @@ function cases() { return list; } -test("every subcommand equals the native command on every fixture", { skip }, () => { - const scratch = mkdtempSync(join(tmpdir(), "coreav-npx-")); - let compared = 0; - let outputs = 0; - // Labels of the cases that succeeded on some fixture: none may be a - // usage error everywhere. - const succeeded = new Set(); - try { - for (const dir of dirs) { - for (const name of readdirSync(dir).filter((f) => f !== "MANIFEST").sort()) { - for (const [label, template, writes] of cases()) { - // Absolute paths; an input relative to the working directory; - // outputs relative to it. - for (const [input, relative, cwd] of [ - [join(dir, name), false, scratch], - [name, false, dir], - [join(dir, name), true, scratch], - ]) { - const segments = template.includes("SEGMENTS"); - const argument = segments ? join("segments", "%03d.mp4") : "out"; - const target = join(scratch, argument); - const output = relative ? argument : target; - const args = template.map((a) => (a === "IN" ? input : a === "OUT" || a === "SEGMENTS" ? output : a)); - clear(target); - const expected = run(bin, args, cwd); - const bytes = writes ? snapshot(target) : null; - clear(target); - const got = npx(args, cwd); - const where = `${name} ${label} (${input}${relative ? ", relative output" : ""})`; - assert.deepEqual(got, expected, where); - if (expected.status === 0) succeeded.add(label); - if (bytes !== null) { - assert.deepEqual(snapshot(target), bytes, `${where}: output bytes differ`); - outputs += 1; - } else if (writes) { - assert.equal(snapshot(target), null, `${where}: an output the native command did not write`); - } - compared += 1; - } +test("every subcommand equals the native command on every fixture", { skip }, async () => { + const commands = cases(); + const enumeration = []; + for (const dir of dirs) { + for (const name of readdirSync(dir).filter((f) => f !== "MANIFEST").sort()) { + for (const [label, template, writes] of commands) { + for (const [relativeInput, relativeOutput] of [[false, false], [true, false], [false, true]]) { + enumeration.push({ label, template, writes, input: join(dir, name), relativeInput, relativeOutput }); } } } - assert.deepEqual(readdirSync(scratch).filter((f) => f.endsWith(".part")), [], "temporary files left behind"); - } finally { - rmSync(scratch, { recursive: true, force: true }); } - assert.deepEqual(cases().map(([label]) => label).filter((label) => !succeeded.has(label)), []); - assert.ok(outputs > 50, `only ${outputs} outputs compared`); - console.log(`${compared} comparisons, ${outputs} outputs`); + const result = await compareCases(enumeration, { native: [bin], wasi: [process.execPath, launcher] }); + assert.deepEqual(commands.map(([label]) => label).filter((label) => !result.succeeded.includes(label)), []); + assert.ok(result.outputs > 50, `only ${result.outputs} outputs compared`); + console.log(`${result.compared} comparisons, ${result.outputs} outputs`); + if (process.env.COREAV_PARITY_SUMMARY) writeFileSync(process.env.COREAV_PARITY_SUMMARY, JSON.stringify(result)); }); diff --git a/packages/cli/test/native.test.mjs b/packages/cli/test/native.test.mjs index 75a63cede..b2a259e76 100644 --- a/packages/cli/test/native.test.mjs +++ b/packages/cli/test/native.test.mjs @@ -207,22 +207,33 @@ test("the install script changes nothing outside node_modules, and ignores COREA test("stopping the launcher stops the binary", { skip: process.platform === "win32" && "POSIX signals" }, async () => { const scratch = mkdtempSync(join(tmpdir(), "coreav-signal-")); + let launcher; + let pid; try { // A stand-in binary that records its pid and waits. const pidFile = join(scratch, "pid"); const slow = join(scratch, "slow"); - writeFileSync(slow, `#!/bin/sh\necho $$ > ${pidFile}\nexec sleep 30\n`); + writeFileSync(slow, `#!/bin/sh\nprintf "%s\\n" "$$" > "$COREAV_TEST_PID_FILE.tmp"\nmv "$COREAV_TEST_PID_FILE.tmp" "$COREAV_TEST_PID_FILE"\nexec sleep 30\n`); chmodSync(slow, 0o755); const { spawn } = await import("node:child_process"); - const launcher = spawn(process.execPath, [main], { env: { ...process.env, COREAV_BINARY: slow }, stdio: "ignore" }); + launcher = spawn(process.execPath, [main], { + env: { ...process.env, COREAV_BINARY: slow, COREAV_TEST_PID_FILE: pidFile }, stdio: "ignore", + }); for (let i = 0; i < 100 && !existsSync(pidFile); i++) await new Promise((r) => setTimeout(r, 50)); - const pid = Number(readFileSync(pidFile, "utf8")); + const printedPid = readFileSync(pidFile, "utf8"); + assert.match(printedPid, /^[1-9][0-9]*\n$/, "signal fixture: invalid_pid_record"); + pid = Number(printedPid); + assert.ok(Number.isSafeInteger(pid) && pid > 1, "signal fixture: invalid_child_pid"); launcher.kill("SIGTERM"); const [code, signal] = await new Promise((resolve) => launcher.on("exit", (c, s) => resolve([c, s]))); assert.ok(signal === "SIGTERM" || code === 143, `launcher exit ${code} ${signal}`); await new Promise((r) => setTimeout(r, 200)); - assert.throws(() => process.kill(pid, 0), "the binary is still running"); + assert.throws(() => process.kill(pid, 0), { code: "ESRCH" }, `the binary is still running (pid ${pid})`); } finally { + launcher?.kill("SIGKILL"); + if (Number.isSafeInteger(pid) && pid > 1) { + try { process.kill(pid, "SIGKILL"); } catch {} + } rmSync(scratch, { recursive: true, force: true }); } }); diff --git a/packages/cli/test/parity-child.mjs b/packages/cli/test/parity-child.mjs new file mode 100644 index 000000000..247f03922 --- /dev/null +++ b/packages/cli/test/parity-child.mjs @@ -0,0 +1,15 @@ +import { appendFileSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; + +const [arm, input, output] = process.argv.slice(2); +const spec = JSON.parse(readFileSync(input)); +if (spec.delay?.[arm]) await new Promise((resolve) => setTimeout(resolve, spec.delay[arm])); +if (spec.log) appendFileSync(spec.log, `${JSON.stringify({ id: spec.id, arm, output, cwd: process.cwd() })}\n`); +if (output) { + mkdirSync(dirname(output), { recursive: true }); + writeFileSync(output.replace("%03d", "000"), String(spec.id)); + if (spec.part) writeFileSync(join(process.cwd(), "left.part"), "partial"); +} +process.stdout.write(`${spec.id}:${spec.mismatch && arm === "wasi" ? "different" : "same"}`); +process.stderr.write(`stderr:${spec.id}`); +process.exitCode = spec.status?.[arm] ?? spec.status ?? 0; diff --git a/packages/cli/test/parity-comparison.mjs b/packages/cli/test/parity-comparison.mjs new file mode 100644 index 000000000..a66e54ad0 --- /dev/null +++ b/packages/cli/test/parity-comparison.mjs @@ -0,0 +1,80 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageDir = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const repo = resolve(packageDir, "../.."); +const order = [1, 4, 4, 1]; +const scratch = mkdtempSync(join(tmpdir(), "coreav-parity-comparison-")); + +function identities() { + const files = []; + function add(path, label) { + if (statSync(path).isDirectory()) { + for (const name of readdirSync(path).sort()) add(join(path, name), `${label}/${name}`); + } else { + files.push({ name: label, bytes: statSync(path).size, sha256: createHash("sha256").update(readFileSync(path)).digest("hex") }); + } + } + const dirs = (process.env.COREAV_FIXTURE_DIRS ?? "").split(",").filter(Boolean); + assert.ok(process.env.COREAV_BIN && dirs.length, "comparison requires native binary and complete fixtures"); + dirs.forEach((dir, index) => add(dir, `fixtures-${index}`)); + for (const path of ["test", "lib", "bin", "install.js", "package.json", "wasm/coreav.wasm"]) add(join(packageDir, path), `package/${path}`); + add(join(repo, "docs/schema/policy-examples"), "policies"); + add(join(repo, "Cargo.toml"), "Cargo.toml"); + add(process.env.COREAV_BIN, "native"); + add(process.execPath, "node"); + return files; +} + +try { + const before = identities(); + const digest = createHash("sha256").update(JSON.stringify(before)).digest("hex"); + const destination = process.env.COREAV_PARITY_COMPARISON_REPORT ?? join(packageDir, "parity-comparison.json"); + const report = { kind: "CI harness comparison, not FFmpeg speed", order, inputDigest: digest, manifest: before, runs: [] }; + const deadline = performance.now() + 70 * 60 * 1000; + writeFileSync(destination, JSON.stringify(report, null, 2)); + for (const [index, workers] of order.entries()) { + assert.deepEqual(identities(), before, "shared input identities changed before test"); + const summary = join(scratch, `${index}.json`); + const start = performance.now(); + const remaining = Math.floor(deadline - start); + assert.ok(remaining > 0, "comparison exceeded its 70 minute execution budget"); + await new Promise((resolveRun, reject) => { + const child = spawn("npm", ["test"], { + cwd: packageDir, + env: { ...process.env, COREAV_PARITY_WORKERS: String(workers), COREAV_PARITY_SUMMARY: summary }, + stdio: "inherit", + detached: true, + }); + const timer = setTimeout(() => process.kill(-child.pid, "SIGKILL"), remaining); + child.on("error", (error) => { clearTimeout(timer); reject(error); }); + child.on("close", (code, signal) => { + clearTimeout(timer); + if (code === 0 && !signal) resolveRun(); + else reject(new Error(`npm test failed: code=${code}, signal=${signal}`)); + }); + }); + const elapsedSeconds = (performance.now() - start) / 1000; + const counts = JSON.parse(readFileSync(summary, "utf8")); + assert.ok(counts.compared > 0 && counts.outputs > 50 && counts.succeeded.length > 0); + if (report.runs.length) assert.deepEqual(counts, report.runs[0].counts, "serial and parallel complete coverage differs"); + assert.deepEqual(identities(), before, "shared input identities changed after test"); + report.runs.push({ workers, elapsedSeconds, counts, inputDigestBefore: digest, inputDigestAfter: digest }); + writeFileSync(destination, JSON.stringify(report, null, 2)); + } + const serial = report.runs.filter((run) => run.workers === 1).reduce((sum, run) => sum + run.elapsedSeconds, 0); + const parallel = report.runs.filter((run) => run.workers === 4).reduce((sum, run) => sum + run.elapsedSeconds, 0); + report.serialOverParallelCiHarnessRatio = serial / parallel; + writeFileSync(destination, JSON.stringify(report, null, 2)); + console.log(JSON.stringify(report)); + if (process.env.GITHUB_STEP_SUMMARY) { + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `\nCI harness comparison (not FFmpeg speed). Shared input SHA-256: \`${digest}\`. Fixed order: 1, 4, 4, 1 workers.\n\n| Workers | npm test seconds | Comparisons | Outputs |\n|---|---|---|---|\n${report.runs.map((run) => `| ${run.workers} | ${run.elapsedSeconds.toFixed(3)} | ${run.counts.compared} | ${run.counts.outputs} |`).join("\n")}\n\nSerial/parallel CI harness elapsed ratio: ${report.serialOverParallelCiHarnessRatio.toFixed(3)}.\n`); + } +} finally { + rmSync(scratch, { recursive: true, force: true }); +} diff --git a/packages/cli/test/parity.mjs b/packages/cli/test/parity.mjs new file mode 100644 index 000000000..dc79d8936 --- /dev/null +++ b/packages/cli/test/parity.mjs @@ -0,0 +1,105 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, dirname, join } from "node:path"; + +export function workerCount(value = process.env.COREAV_PARITY_WORKERS ?? "4") { + assert.match(String(value), /^[1-4]$/, "COREAV_PARITY_WORKERS must be an integer from 1 to 4"); + return Number(value); +} + +export function run(command, args, cwd, maxBuffer = 64 << 20) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { cwd, stdio: ["ignore", "pipe", "pipe"] }); + const chunks = { stdout: [], stderr: [] }; + const lengths = { stdout: 0, stderr: 0 }; + let failure; + child.on("error", (error) => { failure ??= error; }); + for (const stream of ["stdout", "stderr"]) { + child[stream].on("data", (chunk) => { + if (failure) return; + lengths[stream] += chunk.length; + if (lengths[stream] > maxBuffer) { + failure = new Error(`${stream} exceeds ${maxBuffer} bytes`); + child.kill("SIGKILL"); + chunks.stdout.length = 0; + chunks.stderr.length = 0; + return; + } + chunks[stream].push(chunk); + }); + } + child.on("close", (status, signal) => { + if (failure) return reject(failure); + if (signal) return reject(new Error(`child terminated by ${signal}`)); + resolve({ status, stdout: Buffer.concat(chunks.stdout), stderr: Buffer.concat(chunks.stderr) }); + }); + }); +} + +function snapshot(output) { + if (!output.includes("%")) return existsSync(output) ? readFileSync(output) : null; + const dir = dirname(output); + const names = existsSync(dir) ? readdirSync(dir).sort() : []; + return names.length === 0 ? null : names.map((name) => [name, readFileSync(join(dir, name))]); +} + +function clear(output) { + if (!output.includes("%")) return rmSync(output, { force: true }); + rmSync(dirname(output), { recursive: true, force: true }); + mkdirSync(dirname(output)); +} + +export async function compareCases(cases, { native, wasi, workers = workerCount(), root = tmpdir() }) { + workers = workerCount(workers); + let next = 0; + let completed = 0; + let compared = 0; + let outputs = 0; + let failure; + const succeeded = new Set(); + await Promise.all(Array.from({ length: workers }, async () => { + const scratch = mkdtempSync(join(root, "coreav-npx-")); + try { + while (next < cases.length) { + const index = next++; + const { label, template, writes, input, relativeInput, relativeOutput } = cases[index]; + try { + const argument = template.includes("SEGMENTS") ? join("segments", "%03d.mp4") : "out"; + const target = join(scratch, argument); + const output = relativeOutput ? argument : target; + const cwd = relativeInput ? dirname(input) : scratch; + const source = relativeInput ? basename(input) : input; + const args = template.map((a) => a === "IN" ? source : a === "OUT" || a === "SEGMENTS" ? output : a); + const where = `case ${index}: ${input} ${label}${relativeOutput ? " (relative output)" : ""}`; + clear(target); + const expected = await run(native[0], [...native.slice(1), ...args], cwd); + const bytes = writes ? snapshot(target) : null; + clear(target); + const got = await run(wasi[0], [...wasi.slice(1), ...args], cwd); + assert.deepEqual(got, expected, where); + if (expected.status === 0) succeeded.add(label); + if (bytes !== null) { + assert.deepEqual(snapshot(target), bytes, `${where}: output bytes differ`); + outputs += 1; + } else if (writes) { + assert.equal(snapshot(target), null, `${where}: an output the native command did not write`); + } + assert.deepEqual(readdirSync(scratch).filter((f) => f.endsWith(".part")), [], `${where}: temporary files left behind`); + compared += 1; + } catch (error) { + if (!failure || index < failure.index) failure = { index, error }; + } finally { + completed += 1; + } + } + } finally { + rmSync(scratch, { recursive: true, force: true }); + } + })); + assert.equal(completed, cases.length, "every enumerated case must execute exactly once"); + if (failure) throw new Error(`parity case ${failure.index} failed`, { cause: failure.error }); + assert.equal(compared, cases.length); + return { compared, outputs, succeeded: [...succeeded].sort() }; +} diff --git a/packages/cli/test/parity.test.mjs b/packages/cli/test/parity.test.mjs new file mode 100644 index 000000000..f1c79a590 --- /dev/null +++ b/packages/cli/test/parity.test.mjs @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { compareCases, run, workerCount } from "./parity.mjs"; + +const child = join(dirname(fileURLToPath(import.meta.url)), "parity-child.mjs"); +const native = [process.execPath, child, "native"]; +const wasi = [process.execPath, child, "wasi"]; + +function fixture(t, specs) { + const root = realpathSync(mkdtempSync(join(tmpdir(), "parity-test-"))); + t.after(() => rmSync(root, { recursive: true, force: true })); + const log = join(root, "calls.jsonl"); + const cases = specs.map((spec, id) => { + const input = join(root, `${id}.json`); + writeFileSync(input, JSON.stringify({ ...spec, id, log })); + return { label: `command-${id % 3}`, template: ["IN", id % 2 ? "SEGMENTS" : "OUT"], writes: true, input }; + }); + return { root, log, cases }; +} + +test("serial and four workers cover every path variant once with isolated outputs", async (t) => { + for (const workers of [1, 4]) { + const { root, log, cases } = fixture(t, Array.from({ length: 18 }, () => ({}))); + const enumeration = cases.flatMap((item) => [[false, false], [true, false], [false, true]].map( + ([relativeInput, relativeOutput]) => ({ ...item, relativeInput, relativeOutput }), + )); + const result = await compareCases(enumeration, { native, wasi, root, workers }); + assert.deepEqual(result, { compared: 54, outputs: 54, succeeded: ["command-0", "command-1", "command-2"] }); + const calls = readFileSync(log, "utf8").trim().split("\n").map(JSON.parse); + assert.equal(calls.length, enumeration.length * 2); + for (let id = 0; id < cases.length; id++) { + const own = calls.filter((call) => call.id === id); + const variants = new Map(); + for (const call of own) { + const key = `${call.cwd}|${call.output}`; + if (!variants.has(key)) variants.set(key, []); + variants.get(key).push(call); + } + assert.equal(variants.size, 3); + for (const pair of variants.values()) assert.deepEqual(pair.map((call) => call.arm), ["native", "wasi"]); + assert.equal(own.filter((call) => call.cwd === root).length, 2); + assert.equal(own.filter((call) => !call.output.startsWith("/")).length, 2); + } + const scratches = new Set(calls.filter((call) => call.cwd !== root).map((call) => call.cwd)); + assert.equal(scratches.size, workers); + assert.deepEqual(readdirSync(root).filter((name) => name.startsWith("coreav-npx-")), []); + } +}); + +test("the earliest enumeration failure wins after later children drain and cleanup", async (t) => { + const { root, log, cases } = fixture(t, [ + { mismatch: true, delay: { native: 150 } }, { mismatch: true }, {}, {}, {}, {}, + ]); + await assert.rejects(compareCases(cases, { native, wasi, root, workers: 4 }), (error) => { + assert.equal(error.message, "parity case 0 failed"); + assert.match(error.cause.message, /case 0:/); + return true; + }); + assert.equal(readFileSync(log, "utf8").trim().split("\n").length, 12); + assert.deepEqual(readdirSync(root).filter((name) => name.startsWith("coreav-npx-")), []); +}); + +test("nonzero exit codes compare, while leftover temporary files fail and cleanup", async (t) => { + const { root, cases } = fixture(t, [{ status: 3 }]); + assert.deepEqual(await compareCases(cases, { native, wasi, root }), { compared: 1, outputs: 1, succeeded: [] }); + const partial = fixture(t, [{ part: true }]); + await assert.rejects(compareCases(partial.cases, { native, wasi, root: partial.root }), (error) => { + assert.match(error.cause.message, /temporary files left behind/); + return true; + }); + assert.deepEqual(readdirSync(partial.root).filter((name) => name.startsWith("coreav-npx-")), []); + const different = fixture(t, [{ status: { native: 0, wasi: 3 } }]); + await assert.rejects(compareCases(different.cases, { native, wasi, root: different.root }), /parity case 0 failed/); +}); + +test("capture is isolated, bounded on both pipes, and reports launch and signal errors", async () => { + assert.deepEqual(await run(process.execPath, ["-e", "process.stdout.write('out'); process.stderr.write('err'); process.exitCode=3"]), { + status: 3, stdout: Buffer.from("out"), stderr: Buffer.from("err"), + }); + assert.equal((await run(process.execPath, ["-e", ""])).stderr.length, 0); + for (const stream of ["stdout", "stderr"]) { + assert.equal((await run(process.execPath, ["-e", `process.${stream}.write(Buffer.alloc(1024))`], undefined, 1024))[stream].length, 1024); + await assert.rejects(run(process.execPath, ["-e", `process.${stream}.write(Buffer.alloc(1025))`], undefined, 1024), new RegExp(`${stream} exceeds 1024 bytes`)); + } + await assert.rejects(run("/no-such-coreav-child", []), { code: "ENOENT" }); + await assert.rejects(run(process.execPath, ["-e", "process.kill(process.pid, 'SIGTERM')"]), /SIGTERM/); +}); + +test("launch failure also removes worker scratch and invalid worker counts fail", async (t) => { + const { root, cases } = fixture(t, [{}]); + await assert.rejects(compareCases(cases, { native: ["/no-such-coreav-child"], wasi, root }), (error) => error.cause.code === "ENOENT"); + assert.deepEqual(readdirSync(root).filter((name) => name.startsWith("coreav-npx-")), []); + for (const value of [0, 5, "1.5", "", "04"]) assert.throws(() => workerCount(value)); +}); diff --git a/scripts/ci_full.py b/scripts/ci_full.py index 38058f8d7..d33fa89ef 100644 --- a/scripts/ci_full.py +++ b/scripts/ci_full.py @@ -17,9 +17,32 @@ BOT = 'github-actions[bot]' AREAS = { 'ci': ('CI infrastructure', 'area:ci'), + 'h264': ('H.264 decoding', 'area:h264'), + 'transcode': ('Transcode compatibility', 'area:transcode'), + 'vp9': ('VP9 decoding', 'area:ci'), 'hevc': ('HEVC decoding', 'area:ci'), } FULL_WORKFLOWS = { + 'ci-full': { + 'file': 'ci-full.yml', + 'final_head': 'always', + 'areas': { + 'ci': ('full-fixtures',), + 'h264': ('h264-full', 'conformance-arm64'), + 'transcode': tuple(f'transcode-compat-shard ({i})' for i in (1, 2, 3)) + ('transcode-compat',), + }, + }, + 'vp9-corpus': { + 'file': 'vp9-corpus.yml', + 'final_head': 'always', + 'areas': {'vp9': ('contract',) + tuple(f'corpus ({i})' for i in range(8)) + ('corpus-summary',)}, + }, + 'vp9-conformance': { + 'file': 'vp9-conformance.yml', + 'final_head': 'always', + 'areas': {'vp9': tuple(f'{job} ({host})' for job in ('vectors', 'interpolation-fuzz') + for host in ('ubuntu-24.04', 'macos-15')) + ('adapter-contract',)}, + }, 'hevc-conformance': { 'file': 'hevc-conformance.yml', 'final_head': 'area', @@ -28,6 +51,24 @@ }, } PROOF_STEPS = { + ('ci-full', 'full-fixtures'): ('Publish the full fixture sets',), + ('ci-full', 'h264-full'): ('Decode every in-scope stream', 'Decode every in-scope stream in safe mode', + 'Lint and test safe mode'), + ('ci-full', 'conformance-arm64'): ('Decode every in-scope stream',), + ('ci-full', 'transcode-compat-shard'): ('Prove composed output identity and semantic mutations', + 'Run the registered composed compatibility check'), + ('ci-full', 'transcode-compat'): ('Judge the registered compatibility check over every shard', + 'Check the registered version and pinned oracle'), + ('vp9-corpus', 'contract'): ('Check the comparison contract before corpus acquisition', + 'Preflight the original-file oracle on a pinned public vector', + 'Compare the routine original-file smoke subset'), + ('vp9-corpus', 'corpus'): ('Compare exact clocks and pixels without credentials',), + ('vp9-corpus', 'corpus-summary'): ('Require every registered shard and comparison',), + ('vp9-conformance', 'vectors'): ('Read vector headers and compare decoded frames', + 'Compare native boundary frames and keyframe-only selection', + 'Compare scalar-only vector frames without changing comparators'), + ('vp9-conformance', 'interpolation-fuzz'): ('Compare automatic and scalar interpolation for C3',), + ('vp9-conformance', 'adapter-contract'): ('Check the scratch adapter without building timing artifacts',), ('hevc-conformance', 'hevc-vectors'): ( 'Verify registered synthetic references before native classification', 'Acquire every registered official archive', @@ -164,6 +205,69 @@ def area_failures(name, jobs, conclusion): return failures +def covers(name, scheduled, covering, jobs): + try: + require(name in FULL_WORKFLOWS, 'unknown_full_workflow') + for run in (scheduled, covering): + require(positive(run['id']) and positive(run['run_attempt']) + and positive(run['workflow_id']) and run['name'] == name + and run['path'] == '.github/workflows/' + FULL_WORKFLOWS[name]['file'] + and run['head_branch'] == 'main' + and run['repository']['full_name'] == REPOSITORY + and run['head_repository']['full_name'] == REPOSITORY + and isinstance(run['head_sha'], str) + and re.fullmatch('[0-9a-f]{40}', run['head_sha']), 'wrong_coverage_identity') + require(scheduled['event'] == 'schedule' and covering['event'] == 'push' + and scheduled['workflow_id'] == covering['workflow_id'] + and scheduled['head_sha'] == covering['head_sha'] + and covering['status'] == 'completed' and covering['conclusion'] == 'success' + and dispatch_order(covering)[0] <= dispatch_order(scheduled)[0], 'uncovered_schedule') + require(isinstance(jobs, list) and all(isinstance(job, dict) + and job['run_id'] == covering['id'] and job['run_attempt'] == covering['run_attempt'] + and job['head_sha'] == covering['head_sha'] for job in jobs), 'wrong_coverage_jobs') + return not area_failures(name, jobs, 'success') + except (Failure, KeyError, TypeError, ValueError): + return False + + +def covered_schedule(api, name, run, jobs): + if run.get('event') != 'schedule' or run.get('conclusion') != 'success': + return False + gates = [job for job in jobs if job.get('name') == f'{name}-coverage'] + if len(gates) != 1 or gates[0].get('status') != 'completed' or gates[0].get('conclusion') != 'success': + return False + gate, = gates + others = [job for job in jobs if job is not gate] + if not others or any(job.get('status') != 'completed' or job.get('conclusion') != 'skipped' + for job in others): + return False + claims = [] + steps = gate.get('steps') + if not isinstance(steps, list) or not all(isinstance(step, dict) for step in steps): + return False + for step in steps: + if not isinstance(step.get('name'), str): + return False + match = re.fullmatch(r'Covered by run ([1-9][0-9]{0,18}) attempt ([1-9][0-9]{0,3})', + step.get('name', '')) + if step.get('name', '').startswith('Covered by run '): + if match is None: + return False + claims.append((match, step)) + if len(claims) != 1: + return False + match, step = claims[0] + if step.get('status') != 'completed' or step.get('conclusion') != 'success': + return False + try: + covering, full_jobs = attempt_evidence(api, name, int(match[1]), int(match[2])) + except Failure as error: + if str(error).startswith('api_'): + raise + return False + return covers(name, run, covering, full_jobs) + + def issue_marker(name, area): require(name in FULL_WORKFLOWS and area in AREAS, 'invalid_issue_class') return f'' @@ -234,6 +338,8 @@ def report(event, api): require(run.get('status') == 'completed' and run.get('event') == candidate['event'] and run.get('head_branch') == 'main' and run.get('head_sha') == candidate.get('head_sha'), 'wrong_completed_event') + if covered_schedule(api, name, run, jobs): + return [] results = [] for area, bad in area_failures(name, jobs, run.get('conclusion')).items(): marker = issue_marker(name, area) diff --git a/scripts/h264_ci.py b/scripts/h264_ci.py new file mode 100644 index 000000000..d1805835c --- /dev/null +++ b/scripts/h264_ci.py @@ -0,0 +1,61 @@ +"""Run the registered H.264 smoke selection with exact result membership.""" +import json +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) +import h264_conformance + + +SMOKE = ( + 'BA1_Sony_D', 'CABACI3_Sony_B', 'cavlc_mot_frm0_full_B', 'HCHP1_HHI_B', + 'HPCV_BRCM_A', 'alphaconformanceG', 'CVPCMNL1_SVA_C', 'MR1_BT_A', +) + + +def validate_selection(names, manifest): + admitted = [row['name'] for row in manifest['streams'] if row['in_scope']] + if len(names) != len(SMOKE) or set(names) != set(SMOKE): + raise ValueError('smoke selection must contain exactly the eight approved names') + if any(admitted.count(name) != 1 for name in SMOKE): + raise ValueError('smoke selection is absent or duplicated in the admitted manifest') + + +def validate_report(report, mode): + rows = report['results'] + names = [row['name'] for row in rows] + if len(names) != len(SMOKE) or set(names) != set(SMOKE): + raise ValueError('smoke results must contain exactly the eight approved names') + if (report['streams'] != 8 or report['passed'] != 8 or report['failed'] != 0 + or report['threads'] != '1,2,8' or report['mode'] != mode + or any(row['passed'] is not True or set(row['bars']) != {'C1', 'C5'} + or any(not value.startswith('pass ') for value in row['bars'].values()) + for row in rows)): + raise ValueError('smoke results require eight passing C1/C5 results at threads 1,2,8') + + +def main(): + args = sys.argv[1:] + start = args.index('--only') + 1 + end = next((i for i in range(start, len(args)) if args[i].startswith('--')), len(args)) + manifest = json.loads(pathlib.Path(h264_conformance.MANIFEST).read_text()) + validate_selection(args[start:end], manifest) + if '--manifest' in args: + raise ValueError('smoke uses the registered manifest only') + download = '--download-only' in args + report = None if download else pathlib.Path(args[args.index('--report') + 1]) + if report is not None: + report.unlink(missing_ok=True) + try: + h264_conformance.main() + except SystemExit as error: + if error.code != 0: + raise + if report is not None: + mode = args[args.index('--mode') + 1] if '--mode' in args else 'fast' + validate_report(json.loads(report.read_text()), mode) + print('H.264 smoke: exactly 8 of 8 streams passed') + + +if __name__ == '__main__': + main() diff --git a/scripts/merge_group_result.py b/scripts/merge_group_result.py index d8201b362..54e53b71b 100644 --- a/scripts/merge_group_result.py +++ b/scripts/merge_group_result.py @@ -1,16 +1,202 @@ #!/usr/bin/env python3 -"""Reuse a recent successful merge-group workflow for the same Git tree on main.""" +"""Reuse successful checks only for the same tested tree and event boundary.""" import argparse import datetime +import importlib.util import json import os import pathlib import re import subprocess +import time import urllib.error import urllib.parse import urllib.request +SHA = re.compile('[0-9a-f]{40}') +REQUIRED_WORKFLOWS = ('ci.yml', 'deny.yml', 'python.yml', 'npm.yml') +FULL_WORKFLOW_FILES = ('ci-full.yml', 'vp9-corpus.yml', 'vp9-conformance.yml', 'hevc-conformance.yml') + + +def load_full(): + try: + spec = importlib.util.spec_from_file_location('ci_full_coverage', pathlib.Path(__file__).with_name('ci_full.py')) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + except Exception: + return None + + +def find_push_run(request, env, tree, workflow, full): + name = next((name for name, spec in full.FULL_WORKFLOWS.items() if spec['file'] == workflow), None) + if (name is None or env.get('GITHUB_EVENT_NAME') != 'schedule' + or env.get('GITHUB_REF') != 'refs/heads/main' or env.get('GITHUB_REPOSITORY') != full.REPOSITORY): + return None + root = f'/repos/{full.REPOSITORY}/actions' + current = request(f'{root}/runs/{env["GITHUB_RUN_ID"]}') + if (str(current['id']) != env['GITHUB_RUN_ID'] + or str(current['run_attempt']) != env['GITHUB_RUN_ATTEMPT'] + or current['event'] != 'schedule' or current['head_sha'] != env['GITHUB_SHA'] + or current['head_commit']['id'] != env['GITHUB_SHA'] + or current['head_commit']['tree_id'] != tree): + return None + query = urllib.parse.urlencode({'event': 'push', 'branch': 'main', 'head_sha': env['GITHUB_SHA'], 'per_page': 20}) + path = f'{root}/workflows/{workflow}/runs?{query}' + runs = run_index(request, path) + if not runs: + return None + latest = max(runs, key=run_order) + if latest['status'] != 'completed' or latest['conclusion'] != 'success': + return None + attempt_path = f'{root}/runs/{latest["id"]}/attempts/{latest["run_attempt"]}' + covering = request(attempt_path) + if any(covering[key] != latest[key] for key in + ('id', 'run_attempt', 'name', 'path', 'workflow_id', 'head_sha', 'head_branch', + 'event', 'status', 'conclusion')): + return None + data = request(attempt_path + '/jobs?per_page=100') + jobs = data['jobs'] + if not isinstance(jobs, list) or type(data['total_count']) is not int or data['total_count'] != len(jobs) or len(jobs) > 100: + return None + if not full.covers(name, current, covering, jobs): + return None + fresh = run_index(request, path) + if not fresh or max(fresh, key=run_order) != latest: + return None + return covering['id'], covering['run_attempt'] + + +def full_result(env, request, tree, workflow, full=None): + run = None + try: + full = full if full is not None else load_full() + if full is not None: + run = find_push_run(request, env, tree, workflow, full) + except Exception: + pass + message = (f'Covered by successful push run https://github.com/{full.REPOSITORY}/actions/runs/{run[0]}' + f'/attempts/{run[1]} at {env["GITHUB_SHA"]}; full suite skipped.' if run is not None + else 'No covering main push; run the full work.') + return run, message + + +def eligible_pull(pull, repo): + return (pull['state'] == 'open' and pull['draft'] is False + and pull['base']['ref'] == 'main' + and pull['base']['repo']['full_name'] == repo + and pull['head']['repo']['full_name'] == repo + and pull['user']['login'] != 'dependabot[bot]' + and SHA.fullmatch(pull['head']['sha']) is not None) + + +def receipt_name(tree, base, attempt): + if not SHA.fullmatch(tree) or not SHA.fullmatch(base) or type(attempt) is not int or attempt < 1: + raise ValueError('invalid_receipt_identity') + return f'tested-tree-v1-{tree}-{base}-{attempt}' + + +def pr_receipt(env, event, tree, commit, parents): + if env.get('GITHUB_EVENT_NAME') != 'pull_request': + return None + try: + pull = event['pull_request'] + number = event['number'] + attempt = int(env['GITHUB_RUN_ATTEMPT']) + if (type(number) is not int or number < 1 + or env.get('GITHUB_REF') != f'refs/pull/{number}/merge' + or not eligible_pull(pull, env['GITHUB_REPOSITORY']) + or 'dependabot[bot]' in (env.get('GITHUB_ACTOR'), env.get('GITHUB_TRIGGERING_ACTOR')) + or commit != env.get('GITHUB_SHA') or not SHA.fullmatch(commit) + or len(parents) != 2 or parents != [pull['base']['sha'], pull['head']['sha']]): + return None + return receipt_name(tree, parents[0], attempt) + except (KeyError, TypeError, ValueError): + return None + + +def run_index(request, path): + data = request(path) + runs = data['workflow_runs'] + if (not isinstance(runs, list) or type(data['total_count']) is not int + or data['total_count'] != len(runs) or len(runs) > 20): + raise ValueError('incomplete_run_index') + for run in runs: + if (type(run['id']) is not int or run['id'] < 1 + or type(run['run_attempt']) is not int or run['run_attempt'] < 1): + raise ValueError('invalid_run_identity') + datetime.datetime.fromisoformat(run['created_at'].replace('Z', '+00:00')) + return runs + + +def run_order(run): + return datetime.datetime.fromisoformat(run['created_at'].replace('Z', '+00:00')), run['id'] + + +def find_pr_run(request, env, event, tree, parents, now, workflow): + ref = re.fullmatch(r'refs/heads/gh-readonly-queue/main/pr-([1-9][0-9]*)-[0-9a-f]{40}', + env.get('GITHUB_REF', '')) + if ref is None or not parents or parents[0] != event['merge_group']['base_sha']: + return None + group = event['merge_group'] + if group['base_ref'] != 'refs/heads/main' or group['head_sha'] != env['GITHUB_SHA']: + return None + repo = env['GITHUB_REPOSITORY'] + root = f'/repos/{repo}' + current = request(f'{root}/actions/runs/{env["GITHUB_RUN_ID"]}') + if (current['event'] != 'merge_group' or str(current['id']) != env['GITHUB_RUN_ID'] + or current['head_sha'] != env['GITHUB_SHA'] + or current['head_commit']['id'] != env['GITHUB_SHA'] + or current['head_commit']['tree_id'] != tree + or current['head_branch'] != env['GITHUB_REF'].removeprefix('refs/heads/') + or current['path'] != f'.github/workflows/{workflow}' + or current['head_repository']['full_name'] != repo): + return None + created = datetime.datetime.fromisoformat(current['created_at'].replace('Z', '+00:00')) + pull_path = f'{root}/pulls/{ref[1]}' + pull = request(pull_path) + if not eligible_pull(pull, repo): + return None + head = pull['head']['sha'] + query = urllib.parse.urlencode({'event': 'pull_request', 'head_sha': head, 'per_page': 20}) + path = f'{root}/actions/workflows/{workflow}/runs?{query}' + runs = run_index(request, path) + if not runs: + return None + latest = max(runs, key=run_order) + stamp = datetime.datetime.fromisoformat(latest['created_at'].replace('Z', '+00:00')) + if (latest['event'] != 'pull_request' or latest['head_sha'] != head + or latest['head_branch'] != pull['head']['ref'] + or latest['path'] != f'.github/workflows/{workflow}' + or latest['repository']['full_name'] != repo + or latest['head_repository']['full_name'] != repo + or latest['status'] != 'completed' or latest['conclusion'] != 'success' + or not now - datetime.timedelta(days=1) <= stamp <= created <= now): + return None + name = receipt_name(tree, parents[0], latest['run_attempt']) + query = urllib.parse.urlencode({'name': name, 'per_page': 100}) + data = request(f'{root}/actions/runs/{latest["id"]}/artifacts?{query}') + artifacts = data['artifacts'] + if (not isinstance(artifacts, list) or type(data['total_count']) is not int + or data['total_count'] != len(artifacts) or len(artifacts) != 1): + return None + artifact, = artifacts + if (artifact['name'] != name or artifact['expired'] is not False + or type(artifact['id']) is not int or artifact['id'] < 1 + or artifact['workflow_run']['id'] != latest['id'] + or artifact['workflow_run']['head_sha'] != head + or artifact['workflow_run']['head_branch'] != pull['head']['ref'] + or artifact['workflow_run']['head_repository_id'] != pull['head']['repo']['id'] + or artifact['workflow_run']['repository_id'] != pull['base']['repo']['id']): + return None + fresh = run_index(request, path) + if not fresh or max(fresh, key=run_order) != latest: + return None + fresh_pull = request(pull_path) + if not eligible_pull(fresh_pull, repo) or fresh_pull['head']['sha'] != head: + return None + return latest['id'] + def find_run(request, repo, workflow, tree, now): since = (now - datetime.timedelta(days=1)).isoformat() @@ -36,7 +222,16 @@ def find_run(request, repo, workflow, tree, now): return None -def result(env, request, tree, now, workflow): +def result(env, request, tree, now, workflow, event=None, parents=()): + if env.get('GITHUB_EVENT_NAME') == 'merge_group': + try: + run = find_pr_run(request, env, event, tree, parents, now, workflow) + except (OSError, ValueError, KeyError, TypeError, AttributeError): + return None, 'PR evidence unavailable; run the normal checks.' + if run is None: + return None, 'No successful PR run proves this tested tree and base; run the normal checks.' + url = f"https://github.com/{env['GITHUB_REPOSITORY']}/actions/runs/{run}" + return run, f'Reusing successful pull_request run {url} for tested tree {tree} and base {parents[0]}.' if env.get('GITHUB_EVENT_NAME') != 'push' or env.get('GITHUB_REF') != 'refs/heads/main': return None, 'Not a push to main; run the normal checks.' try: @@ -51,22 +246,57 @@ def result(env, request, tree, now, workflow): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--workflow', choices=('ci.yml', 'deny.yml', 'python.yml', 'npm.yml'), required=True) + parser.add_argument('--workflow', choices=REQUIRED_WORKFLOWS + FULL_WORKFLOW_FILES, required=True) args = parser.parse_args() env = os.environ + deadline = time.monotonic() + 60 + calls = 0 def request(path): + nonlocal calls + calls += 1 + if calls > 20 or time.monotonic() >= deadline: + raise ValueError('api_budget_exhausted') req = urllib.request.Request(env.get('GITHUB_API_URL', 'https://api.github.com') + path, headers={'Authorization': 'Bearer ' + env['GH_TOKEN'], - 'Accept': 'application/vnd.github+json'}) - with urllib.request.urlopen(req, timeout=10) as response: - return json.load(response) + 'Accept': 'application/vnd.github+json', + 'Cache-Control': 'no-cache'}) + with urllib.request.urlopen(req, timeout=min(10, max(1, deadline - time.monotonic()))) as response: + raw = response.read((2 << 20) + 1) + if len(raw) > 2 << 20: + raise ValueError('api_response_too_large') + return json.loads(raw) tree = subprocess.check_output(['git', 'rev-parse', 'HEAD^{tree}'], text=True).strip() - run, message = result(env, request, tree, datetime.datetime.now(datetime.timezone.utc), args.workflow) + commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() + parents = subprocess.check_output(['git', 'rev-list', '--parents', '-n', '1', 'HEAD'], text=True).split()[1:] + event = {} + try: + if env.get('GITHUB_EVENT_PATH'): + with pathlib.Path(env['GITHUB_EVENT_PATH']).open('rb') as source: + raw = source.read((2 << 20) + 1) + if len(raw) <= 2 << 20: + event = json.loads(raw) + except (OSError, ValueError): + pass + if args.workflow in REQUIRED_WORKFLOWS: + run, message = result(env, request, tree, datetime.datetime.now(datetime.timezone.utc), args.workflow, + event, parents) + else: + run, message = full_result(env, request, tree, args.workflow) print(message) with pathlib.Path(env['GITHUB_OUTPUT']).open('a', encoding='utf-8') as output: output.write(f'reuse={"true" if run is not None else "false"}\n') + if isinstance(run, tuple): + output.write(f'run={run[0]}\nattempt={run[1]}\n') + receipt = pr_receipt(env, event, tree, commit, parents) if args.workflow in REQUIRED_WORKFLOWS else None + if receipt is not None: + try: + path = pathlib.Path(env['RUNNER_TEMP']) / 'tested-tree.txt' + path.write_text(f'tree={tree}\nbase={parents[0]}\ncommit={commit}\n') + output.write(f'record_name={receipt}\nrecord_path={path}\n') + except (OSError, KeyError): + print('Tested-tree receipt unavailable; merge groups will run the normal checks.') if env.get('GITHUB_STEP_SUMMARY'): with pathlib.Path(env['GITHUB_STEP_SUMMARY']).open('a', encoding='utf-8') as summary: summary.write(message + '\n') diff --git a/scripts/tests/test_ci_fixtures.py b/scripts/tests/test_ci_fixtures.py index 0dd10691b..57445501c 100644 --- a/scripts/tests/test_ci_fixtures.py +++ b/scripts/tests/test_ci_fixtures.py @@ -27,12 +27,13 @@ class SharedFixtures(unittest.TestCase): @classmethod def setUpClass(cls): cls.workflow = read_yaml(ROOT / '.github/workflows/ci.yml') + cls.full = read_yaml(ROOT / '.github/workflows/ci-full.yml') cls.action = read_yaml(ROOT / '.github/actions/ci-fixtures/action.yml') def test_consumers_start_beside_the_other_checks(self): jobs = self.workflow['jobs'] self.assertNotIn('isobmff-fixture-set', jobs) - for name in ('isobmff-writers', 'isobmff-decode', 'transcode-compat-shard'): + for name in ('isobmff-fixtures', 'isobmff-command-compat', 'isobmff-writers', 'isobmff-decode'): with self.subTest(job=name): self.assertEqual(jobs[name]['needs'], 'ci-changes') downloads = [s for s in jobs[name]['steps'] @@ -43,6 +44,13 @@ def test_consumers_start_beside_the_other_checks(self): report = jobs['ci-report'] self.assertEqual(set(report['needs']), set(jobs) - {'ci-report'}) self.assertEqual(report['if'], 'always()') + shard = self.full['jobs']['transcode-compat-shard'] + self.assertEqual(shard['needs'], 'full-fixtures') + fixture, = [s for s in shard['steps'] if s.get('uses', '').startswith('actions/download-artifact@') + and s.get('with', {}).get('name') == 'isobmff-fixture-set'] + self.assertEqual(fixture['with']['path'], '${{ runner.temp }}/coreav-fixtures') + self.assertFalse(any('gen-fixtures.sh' in s.get('run', '') for s in shard['steps'])) + self.assertEqual(set(self.full['jobs']['ci-full-report']['needs']), set(self.full['jobs']) - {'ci-full-report'}) def test_prepare_runs_for_ready_fixtures_including_reused_main(self): gate = self.workflow['jobs']['ci-changes'] @@ -63,10 +71,11 @@ def test_cache_misses_generate_once_before_publication_and_only_main_saves(self) save, = [s for s in steps if s.get('name') == 'Save the fixtures'] publish, = [s for s in steps if s.get('uses', '').startswith('actions/upload-artifact@') and s.get('with', {}).get('name') == 'isobmff-fixture-set'] - for ref, hit in itertools.product(('refs/heads/main', 'refs/pull/1/merge', - 'refs/heads/gh-readonly-queue/main/group'), ('true', 'false')): - context = {'github': {'ref': ref}, 'steps': {'fixtures': {'outputs': {'cache-hit': hit}}}} - self.assertEqual(bool(evaluate(generate['if'], context)), hit != 'true') + for ref, hit, fresh in itertools.product(('refs/heads/main', 'refs/pull/1/merge', + 'refs/heads/gh-readonly-queue/main/group'), ('true', 'false'), ('true', 'false')): + context = {'github': {'ref': ref}, 'steps': {'fixtures': {'outputs': {'cache-hit': hit}}}, + 'inputs': {'fresh': fresh}} + self.assertEqual(bool(evaluate(generate['if'], context)), fresh == 'true' or hit != 'true') self.assertEqual(bool(evaluate(save['if'], context)), hit != 'true' and ref == 'refs/heads/main') self.assertNotIn('if', publish) self.assertEqual(publish['with']['name'], 'isobmff-fixture-set') diff --git a/scripts/tests/test_ci_full.py b/scripts/tests/test_ci_full.py index c6f441e56..9cc2dbf33 100644 --- a/scripts/tests/test_ci_full.py +++ b/scripts/tests/test_ci_full.py @@ -2,8 +2,8 @@ import importlib.util import io import json -import re from pathlib import Path +import re import sys import tempfile import unittest @@ -31,7 +31,7 @@ def __init__(self): self.changed_head = False self.pull_reads = 0 for index, (name, spec) in enumerate(ci.FULL_WORKFLOWS.items(), 42): - run = {'id': index, 'run_attempt': 1, 'name': name, 'path': '.github/workflows/' + spec['file'], + run = {'id': index, 'run_attempt': 1, 'workflow_id': index + 100, 'name': name, 'path': '.github/workflows/' + spec['file'], 'event': 'push', 'head_branch': 'main', 'head_sha': HEAD, 'status': 'completed', 'conclusion': 'success', 'created_at': STAMP, 'repository': {'full_name': ci.REPOSITORY}, 'head_repository': {'full_name': ci.REPOSITORY}} @@ -114,13 +114,13 @@ def create_issue(self, data): class FailureReporting(unittest.TestCase): def test_real_report_path_opens_an_area_issue_and_deduplicates_attempts(self): api = FakeApi() - api.fail('hevc-vectors') + api.fail('h264-full') self.assertEqual(ci.report(api.event(), api), [1]) issue, = api.issues - self.assertIn('Owning area: HEVC decoding', issue['body']) + self.assertIn('Owning area: H.264 decoding', issue['body']) self.assertIn('within one day', issue['body']) self.assertIn('/actions/runs/42/attempts/1', issue['body']) - self.assertEqual(set(issue['labels']), {'bug', 'area:ci'}) + self.assertEqual(set(issue['labels']), {'bug', 'area:ci', 'area:h264'}) self.assertEqual(ci.report(api.event(), api), [1]) self.assertEqual(len(api.issues), 1) self.assertEqual(api.comments, {}) @@ -133,8 +133,12 @@ def test_real_report_path_opens_an_area_issue_and_deduplicates_attempts(self): self.assertIn('ci-full-run:42:2', api.comments[1][0]['body']) def test_each_reduced_full_counterpart_reports_its_owning_area(self): - for run_id, name, area in [(42, 'hevc-vectors', 'HEVC decoding')] + [ - (42, f'profile-census ({index})', 'HEVC decoding') for index in range(4)]: + for run_id, name, area in [(42, 'h264-full', 'H.264 decoding'), + (42, 'transcode-compat-shard (3)', 'Transcode compatibility'), + (43, 'corpus (7)', 'VP9 decoding'), + (44, 'interpolation-fuzz (macos-15)', 'VP9 decoding'), + (45, 'hevc-vectors', 'HEVC decoding')] + [ + (45, f'profile-census ({index})', 'HEVC decoding') for index in range(4)]: with self.subTest(job=name): api = FakeApi() api.fail(name, run_id) @@ -156,42 +160,42 @@ def test_ineligible_events_never_read_or_write_issues(self): def test_schedule_and_cancelled_or_missing_full_work_are_reportable(self): for result in ('failure', 'cancelled', 'timed_out', 'skipped'): api = FakeApi() - api.fail('hevc-vectors') + api.fail('h264-full') api.runs[42]['event'] = 'schedule' - api.jobs[42][0]['conclusion'] = result + api.jobs[42][1]['conclusion'] = result self.assertEqual(ci.report(api.event(), api), [1]) - api = FakeApi() - api.jobs[42] = [job for job in api.jobs[42] if job['name'] != 'profile-census (0)'] - ci.report(api.event(), api) - self.assertIn('CI infrastructure', api.issues[0]['body']) + for run_id, missing in ((42, 'transcode-compat-shard (2)'), (45, 'profile-census (0)')): + api = FakeApi() + api.jobs[run_id] = [job for job in api.jobs[run_id] if job['name'] != missing] + ci.report(api.event(run_id), api) + self.assertIn('CI infrastructure', api.issues[0]['body']) def test_failed_preparation_does_not_blame_skipped_decoder_or_transcode_work(self): api = FakeApi() - api.jobs[42][0]['conclusion'] = 'skipped' - api.runs[42]['conclusion'] = 'failure' + api.fail('full-fixtures') for job in api.jobs[42][1:]: job['conclusion'] = 'skipped' ci.report(api.event(), api) self.assertEqual(len(api.issues), 1) self.assertIn('CI infrastructure', api.issues[0]['body']) - self.assertNotIn('HEVC decoding', api.issues[0]['body']) + self.assertNotIn('area:h264', api.issues[0]['labels']) def test_skipped_proof_cannot_hide_behind_a_successful_job(self): api = FakeApi() api.jobs[42][1]['steps'][0]['conclusion'] = 'skipped' ci.report(api.event(), api) - self.assertIn('HEVC decoding', api.issues[0]['body']) + self.assertIn('H.264 decoding', api.issues[0]['body']) def test_ambiguous_creation_is_read_back_without_reposting(self): api = FakeApi() - api.fail('hevc-vectors') + api.fail('h264-full') api.ambiguous_create = True self.assertEqual(ci.report(api.event(), api), [1]) self.assertEqual(len([call for call in api.calls if call[0] == 'POST' and call[1].endswith('/issues')]), 1) def test_competing_creations_preserve_both_runs_and_close_only_the_duplicate(self): api = FakeApi() - api.fail('hevc-vectors') + api.fail('h264-full') api.race_create = True ci.report(api.event(), api) self.assertEqual([item['number'] for item in api.issues if item['state'] == 'open'], [1]) @@ -201,7 +205,7 @@ def test_competing_creations_preserve_both_runs_and_close_only_the_duplicate(sel def test_lookup_failure_cannot_create_an_issue_or_become_green(self): api = FakeApi() - api.fail('hevc-vectors') + api.fail('h264-full') api.fail_lookup = True with self.assertRaisesRegex(ci.Failure, 'api_http_403'): ci.report(api.event(), api) @@ -239,6 +243,20 @@ def test_untrusted_names_and_exception_arguments_never_enter_public_issue_text(s class FinalHead(unittest.TestCase): + def test_manual_c3_failure_blocks_verification_but_scalar_skip_has_no_c3_credit(self): + for scalar in (False, True): + api = FakeApi() + api.manual() + for host in ('ubuntu-24.04', 'macos-15'): + api.jobs[45].append(dict(api.jobs[45][0], name=f'kernel-differential ({host})', + conclusion='skipped' if scalar else 'failure', steps=[])) + if scalar: + self.assertEqual(ci.verify(26, api, ['hevc-conformance'])[1], [('hevc-conformance', 45, 1)]) + else: + api.runs[45]['conclusion'] = 'failure' + with self.assertRaisesRegex(ci.Failure, 'final_head_dispatch_not_green'): + ci.verify(26, api, ['hevc-conformance']) + def test_verifier_requires_every_unique_shard_and_complete_scalar_proof(self): for index in range(4): for mutation in ('missing', 'duplicate', 'skipped_proof'): @@ -246,18 +264,18 @@ def test_verifier_requires_every_unique_shard_and_complete_scalar_proof(self): api = FakeApi() api.manual() name = f'profile-census ({index})' - job = next(job for job in api.jobs[42] if job['name'] == name) + job = next(job for job in api.jobs[45] if job['name'] == name) if mutation == 'missing': - api.jobs[42].remove(job) + api.jobs[45].remove(job) elif mutation == 'duplicate': - api.jobs[42].append(copy.deepcopy(job)) + api.jobs[45].append(copy.deepcopy(job)) else: job['steps'][0]['conclusion'] = 'skipped' with self.assertRaisesRegex(ci.Failure, 'final_head_dispatch_not_green'): ci.verify(26, api, ['hevc-conformance']) api = FakeApi() api.manual() - job = next(job for job in api.jobs[42] if job['name'] == 'hevc-vectors') + job = next(job for job in api.jobs[45] if job['name'] == 'hevc-vectors') job['steps'] = [step for step in job['steps'] if step['name'] != 'Check complete scalar conformance output'] with self.assertRaisesRegex(ci.Failure, 'final_head_dispatch_not_green'): @@ -274,22 +292,23 @@ def test_cli_accepts_repeatable_explicit_suite_selection(self): verify.assert_called_once_with(26, api.return_value, ['hevc-conformance', 'hevc-conformance']) def test_suite_selection_is_explicit_repeatable_and_closed(self): - for selection in (None, [], ['unknown'], ['hevc-conformance', 'unknown']): + for selection in ([], ['unknown'], ['hevc-conformance', 'unknown']): api = FakeApi() with self.assertRaisesRegex(ci.Failure, 'invalid_suite_selection'): ci.verify(26, api, selection) self.assertEqual(api.calls, []) self.assertEqual(ci.selected_suites(['hevc-conformance', 'hevc-conformance']), ['hevc-conformance']) + self.assertEqual(ci.selected_suites(), ['ci-full', 'vp9-corpus', 'vp9-conformance']) with mock.patch.dict(ci.FULL_WORKFLOWS['hevc-conformance'], final_head='always'): - self.assertEqual(ci.selected_suites(), ['hevc-conformance']) + self.assertEqual(ci.selected_suites(), list(ci.FULL_WORKFLOWS)) def test_verifier_requires_every_full_workflow_at_the_exact_unchanged_head(self): api = FakeApi() api.manual() - head, evidence = ci.verify(26, api, ['hevc-conformance']) + head, evidence = ci.verify(448, api) self.assertEqual(head, HEAD) - self.assertEqual({row[0] for row in evidence}, set(ci.FULL_WORKFLOWS)) + self.assertEqual({row[0] for row in evidence}, set(ci.selected_suites())) self.assertTrue(all(call[0] in ('GET', 'PAGES') for call in api.calls)) def test_newest_failure_or_pending_dispatch_blocks_an_older_success(self): @@ -300,9 +319,9 @@ def test_newest_failure_or_pending_dispatch_blocks_an_older_success(self): newer.update(id=99, created_at='2026-11-06T02:00:00Z', status=status, conclusion=conclusion) api.runs[99] = newer api.jobs[99] = [dict(job, run_id=99) for job in api.jobs[42]] - api.dispatches['hevc-conformance.yml'].append(newer) + api.dispatches['ci-full.yml'].append(newer) with self.assertRaisesRegex(ci.Failure, 'final_head_dispatch_not_green'): - ci.verify(26, api, ['hevc-conformance']) + ci.verify(448, api) def test_wrong_head_branch_fork_or_attempt_cannot_supply_dispatch_evidence(self): for field, value in [('head_sha', 'b' * 40), ('head_branch', 'other'), @@ -311,33 +330,33 @@ def test_wrong_head_branch_fork_or_attempt_cannot_supply_dispatch_evidence(self) api.manual() api.runs[42][field] = value with self.assertRaises(ci.Failure): - ci.verify(26, api, ['hevc-conformance']) + ci.verify(448, api) def test_missing_matrix_leg_skipped_proof_or_duplicate_job_is_not_a_pass(self): for mutation in ('missing', 'skipped', 'duplicate'): api = FakeApi() api.manual() if mutation == 'missing': - api.jobs[42].pop() + api.jobs[43].pop() elif mutation == 'skipped': - api.jobs[42][1]['steps'][0]['conclusion'] = 'skipped' + api.jobs[43][1]['steps'][0]['conclusion'] = 'skipped' else: - api.jobs[42].append(copy.deepcopy(api.jobs[42][1])) + api.jobs[43].append(copy.deepcopy(api.jobs[43][1])) with self.assertRaisesRegex(ci.Failure, 'final_head_dispatch_not_green'): - ci.verify(26, api, ['hevc-conformance']) + ci.verify(448, api) def test_missing_dispatch_invalid_timestamp_and_head_change_fail_closed(self): for mutation in ('missing', 'timestamp', 'head-change'): api = FakeApi() api.manual() if mutation == 'missing': - api.dispatches['hevc-conformance.yml'] = [] + api.dispatches['ci-full.yml'] = [] elif mutation == 'timestamp': - api.dispatches['hevc-conformance.yml'][0]['created_at'] = 'not-a-time' + api.dispatches['ci-full.yml'][0]['created_at'] = 'not-a-time' else: api.changed_head = True with self.assertRaises(ci.Failure): - ci.verify(26, api, ['hevc-conformance']) + ci.verify(448, api) def test_dispatch_or_rerun_started_during_verification_invalidates_old_evidence(self): for mutation in ('rerun', 'new-dispatch'): @@ -348,21 +367,21 @@ def racing_pages(path, key=None): result = pages(path, key) if '/runs/42/attempts/1/jobs' in path: if mutation == 'rerun': - api.dispatches['hevc-conformance.yml'][0].update(run_attempt=2, status='in_progress', conclusion=None) + api.dispatches['ci-full.yml'][0].update(run_attempt=2, status='in_progress', conclusion=None) else: newer = dict(api.runs[42], id=99, created_at='2026-11-06T02:00:00Z', status='in_progress', conclusion=None) - api.dispatches['hevc-conformance.yml'].append(newer) + api.dispatches['ci-full.yml'].append(newer) return result with mock.patch.object(api, 'pages', side_effect=racing_pages): with self.assertRaises(ci.Failure): - ci.verify(26, api, ['hevc-conformance']) + ci.verify(448, api) class Pagination(unittest.TestCase): def test_hevc_registry_matches_existing_full_proof_steps_and_trusted_wiring(self): - self.assertEqual(set(ci.AREAS), {'ci', 'hevc'}) - self.assertEqual(set(ci.FULL_WORKFLOWS), {'hevc-conformance'}) + self.assertEqual(set(ci.AREAS), {'ci', 'h264', 'transcode', 'vp9', 'hevc'}) + self.assertEqual(set(ci.FULL_WORKFLOWS), {'ci-full', 'vp9-corpus', 'vp9-conformance', 'hevc-conformance'}) self.assertEqual(ci.FULL_WORKFLOWS['hevc-conformance']['final_head'], 'area') self.assertEqual(ci.FULL_WORKFLOWS['hevc-conformance']['areas']['hevc'], ('hevc-vectors', 'profile-census (0)', 'profile-census (1)', @@ -370,12 +389,14 @@ def test_hevc_registry_matches_existing_full_proof_steps_and_trusted_wiring(self self.assertIn('Check complete scalar conformance output', ci.PROOF_STEPS['hevc-conformance', 'hevc-vectors']) source = (ROOT / '.github/workflows/hevc-conformance.yml').read_text() - for (_, job), names in ci.PROOF_STEPS.items(): + for (workflow, job), names in ci.PROOF_STEPS.items(): + if workflow != 'hevc-conformance': + continue body = re.split(r'\n \S', source.split(' ' + job + ':\n', 1)[1], maxsplit=1)[0] for name in names: self.assertIn('- name: ' + name + '\n', body) reporter = (ROOT / '.github/workflows/ci-full-failure.yml').read_text() - self.assertIn('workflows: [hevc-conformance]', reporter) + self.assertIn('workflows: [' + ', '.join(ci.FULL_WORKFLOWS) + ']', reporter) self.assertIn('ref: ${{ github.workflow_sha }}', reporter) self.assertIn('persist-credentials: false', reporter) for permission in ('contents: read', 'actions: read', 'issues: write'): @@ -398,7 +419,7 @@ def test_duplicate_json_fields_and_boolean_ids_are_refused(self): with self.assertRaisesRegex(ci.Failure, 'duplicate_api_field'): json.loads('{"head_sha":"a","head_sha":"b"}', object_pairs_hook=ci.object_unique) with self.assertRaises(ci.Failure): - ci.attempt_evidence(FakeApi(), 'hevc-conformance', True, 1) + ci.attempt_evidence(FakeApi(), 'ci-full', True, 1) if __name__ == '__main__': diff --git a/scripts/tests/test_core_ci_tiers.py b/scripts/tests/test_core_ci_tiers.py new file mode 100644 index 000000000..5a4077df1 --- /dev/null +++ b/scripts/tests/test_core_ci_tiers.py @@ -0,0 +1,189 @@ +"""Small contracts for the core smoke and independent full tier.""" +import copy +import json +import os +import pathlib +import shlex +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +sys.path.insert(0, str(pathlib.Path(__file__).resolve().parents[1])) +import h264_ci +import test_main_ci_plan + + +ROOT = pathlib.Path(__file__).resolve().parents[2] + + +def load(path): + return json.loads(subprocess.check_output([ + 'ruby', '-rjson', '-rpsych', '-e', + 'd = Psych.load_file(ARGV[0]); d["on"] ||= d.delete(true); puts JSON.generate(d)', + str(ROOT / path), + ], text=True)) + + +class Smoke(unittest.TestCase): + def report(self, mode='fast'): + return {'streams': 8, 'passed': 8, 'failed': 0, 'threads': '1,2,8', 'mode': mode, + 'results': [{'name': name, 'passed': True, + 'bars': {'C1': 'pass exact pixels', 'C5': 'pass keyframes'}} + for name in h264_ci.SMOKE]} + + def test_registered_selection_and_all_99_remain_admitted(self): + manifest = json.loads((ROOT / h264_ci.h264_conformance.MANIFEST).read_text()) + self.assertEqual(sum(row['in_scope'] for row in manifest['streams']), 99) + h264_ci.validate_selection(h264_ci.SMOKE, manifest) + for names in (h264_ci.SMOKE[:-1], (*h264_ci.SMOKE, 'unknown'), + (*h264_ci.SMOKE[:-1], 'unknown'), (*h264_ci.SMOKE[:-1], h264_ci.SMOKE[0])): + with self.assertRaises(ValueError): + h264_ci.validate_selection(names, manifest) + manifest['streams'] = [row for row in manifest['streams'] if row['name'] != h264_ci.SMOKE[0]] + with self.assertRaises(ValueError): + h264_ci.validate_selection(h264_ci.SMOKE, manifest) + + def test_missing_extra_unknown_duplicate_failed_or_wrong_mode_results_fail(self): + h264_ci.validate_report(self.report(), 'fast') + mutations = [ + lambda r: r['results'].pop(), + lambda r: r['results'].append(copy.deepcopy(r['results'][0])), + lambda r: r['results'][0].update(name='unknown'), + lambda r: r['results'][0].update(name=h264_ci.SMOKE[1]), + lambda r: r.update(streams=7), lambda r: r.update(passed=7), + lambda r: r.update(failed=1), lambda r: r.update(threads='1'), + lambda r: r.update(mode='safe'), + lambda r: r['results'][0].update(passed=False), + lambda r: r['results'][0]['bars'].pop('C5'), + lambda r: r['results'][0]['bars'].update(C1='FAIL pixels'), + ] + for mutate in mutations: + report = self.report() + mutate(report) + with self.assertRaises(ValueError): + h264_ci.validate_report(report, 'fast') + + def test_entrypoint_preserves_arguments_and_requires_new_exact_report(self): + with tempfile.TemporaryDirectory() as directory: + report = pathlib.Path(directory) / 'report.json' + args = ['h264_ci.py', '--only', *h264_ci.SMOKE, '--zips', directory, + '--work', directory, '--threads', '1,2,8', '--report', str(report)] + for mode in ('fast', 'safe'): + argv = args + (['--mode', 'safe'] if mode == 'safe' else []) + def run(): + self.assertEqual(sys.argv, argv) + self.assertFalse(report.exists()) + report.write_text(json.dumps(self.report(mode))) + raise SystemExit(0) + with mock.patch.object(sys, 'argv', argv), mock.patch.object( + h264_ci.h264_conformance, 'main', side_effect=run): + report.write_text('stale') + h264_ci.main() + with mock.patch.object(sys, 'argv', args), mock.patch.object(h264_ci.h264_conformance, 'main'): + with self.assertRaises(FileNotFoundError): + h264_ci.main() + + def test_isolated_entrypoint_refuses_unknown_names_before_acquisition(self): + done = subprocess.run([sys.executable, '-I', str(ROOT / 'scripts/h264_ci.py'), + '--only', 'unknown', '--download-only', '--zips', '/nonexistent'], + capture_output=True, text=True, cwd=ROOT) + self.assertNotEqual(done.returncode, 0) + self.assertIn('exactly the eight approved names', done.stderr) + + +class Tiers(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.ci = load('.github/workflows/ci.yml') + cls.full = load('.github/workflows/ci-full.yml') + cls.action = load('.github/actions/ci-fixtures/action.yml') + + def test_full_events_jobs_and_no_queue_reuse(self): + self.assertEqual(self.full['name'], 'ci-full') + self.assertEqual(set(self.full['on']), {'push', 'schedule', 'workflow_dispatch'}) + self.assertEqual(self.full['on']['push']['branches'], ['main']) + self.assertEqual(set(self.full['jobs']), {'full-coverage', 'full-fixtures', 'h264-full', 'conformance-arm64', + 'transcode-compat-shard', 'transcode-compat', 'ci-full-report'}) + self.assertNotIn('concurrency', self.full) + self.assertIn('merge-group-result', json.dumps(self.full['jobs']['full-coverage'])) + self.assertEqual(self.full['jobs']['full-coverage']['if'], + "github.event_name == 'schedule' && github.event.schedule == '30 3 * * *'") + for name in ('h264-full', 'conformance-arm64', 'transcode-compat-shard', 'transcode-compat'): + self.assertNotIn(name, self.ci['jobs']) + for event in ('schedule', 'workflow_dispatch', 'push'): + plans = test_main_ci_plan.MainPlan() + plans.workflows = {'ci-full': self.full} + self.assertEqual(sum(plans.plan(event, True)['ci-full'].values()), 9 if event == 'schedule' else 8) + self.assertEqual(plans.plan('schedule', True, full_covered=True)['ci-full'], {'full-coverage': 1}) + + def test_full_report_rejects_every_non_success(self): + job = self.full['jobs']['ci-full-report'] + self.assertEqual(job['if'], "always() && needs.full-coverage.outputs.reuse != 'true'") + self.assertEqual(set(job['needs']), set(self.full['jobs']) - {'ci-full-report'}) + self.assertNotIn('concurrency', job) + for result in ('success', 'failure', 'cancelled', 'skipped', 'unknown'): + needs = {name: {'result': 'success'} for name in job['needs']} + needs['h264-full']['result'] = result + with tempfile.TemporaryDirectory() as directory: + done = subprocess.run(['bash', '-c', job['steps'][0]['run']], capture_output=True, + env={**os.environ, 'RESULTS': json.dumps(needs), + 'GITHUB_STEP_SUMMARY': str(pathlib.Path(directory) / 'summary')}) + self.assertEqual(done.returncode == 0, result == 'success') + + def test_smoke_selects_identical_eight_for_acquisition_fast_and_safe(self): + steps = self.ci['jobs']['conformance']['steps'] + commands = [shlex.split(s['run'].replace('\\\n', '')) for s in steps + if 'scripts/h264_ci.py' in s.get('run', '')] + self.assertEqual(len(commands), 3) + for command in commands: + start = command.index('--only') + 1 + end = next(i for i in range(start, len(command)) if command[i].startswith('--')) + self.assertEqual(command[start:end], list(h264_ci.SMOKE)) + full = self.full['jobs']['h264-full']['steps'] + commands = [s['run'] for s in full if 'scripts/h264_conformance.py' in s.get('run', '')] + self.assertEqual(len(commands), 3) + self.assertTrue(all('--only' not in c for c in commands)) + self.assertEqual(sum('--threads 1,2,8' in c for c in commands), 2) + self.assertIn('cargo clippy -p coreav-h264 --no-default-features --all-targets -- -D warnings', json.dumps(full)) + + def test_shared_fixtures_are_produced_once_and_consumed_without_generation(self): + for name in ('isobmff-fixtures', 'isobmff-command-compat'): + job = self.ci['jobs'][name] + self.assertIn(name, self.ci['jobs']['ci-report']['needs']) + downloads = [s for s in job['steps'] if s.get('uses', '').startswith('actions/download-artifact@')] + self.assertEqual([s['with']['name'] for s in downloads], ['isobmff-fixture-set']) + self.assertNotIn('scripts/gen-fixtures.sh', json.dumps(job)) + steps = self.action['runs']['steps'] + generate, = [s for s in steps if s.get('name') == 'Generate all required fixtures'] + for fresh, hit, expected in (('true', 'true', True), ('false', 'true', False), ('false', 'false', True)): + context = {'steps': {'fixtures': {'outputs': {'cache-hit': hit}}}} + expression = generate['if'].replace('inputs.fresh', repr(fresh)) + self.assertEqual(test_main_ci_plan.evaluate(expression, context), expected) + for step in steps: + if 'composed' in step.get('name', ''): + self.assertIn("inputs.composed == 'true'", step['if']) + publisher, = [s for s in self.ci['jobs']['ci-changes']['steps'] if s.get('uses') == './.github/actions/ci-fixtures'] + self.assertEqual(publisher['with'], {'fresh': "${{ steps.result.outputs.reuse != 'true' }}"}) + publisher = self.full['jobs']['full-fixtures']['steps'][-1] + self.assertEqual(publisher['with'], {'fresh': 'true', 'composed': 'true'}) + + def test_command_branch_keeps_builds_comparators_and_informational_r1(self): + base = self.ci['jobs']['isobmff-fixtures']['steps'] + commands = self.ci['jobs']['isobmff-command-compat']['steps'] + self.assertIn('Audit the text known divergences', [s.get('name') for s in base]) + stages = {s.get('name'): s for s in commands} + self.assertEqual(stages['Build the coreav binary']['run'], + 'cargo build --locked --release -p coreav-cli') + self.assertIn('--baseline tools/compat-corpus/baseline-isobmff.json', + stages['Run the command-coverage set through coreav ffprobe']['run']) + self.assertIn('--baseline tools/compat-corpus/baseline-frames.json', + stages['List frames through coreav ffprobe']['run']) + self.assertIn('--features native-h264 --target-dir target/native', + stages['List frames through coreav ffprobe']['run']) + self.assertIn('[ "$code" -le 2 ]', stages['R1 of the 0.1.0 round']['run']) + self.assertIn("[ -s \"$RUNNER_TEMP/coreav-r1.json\" ]", stages['R1 of the 0.1.0 round']['run']) + for name in ('Cache pinned black-box binaries', 'Install pinned FFmpeg, ffprobe and MediaInfo', + 'Use pinned binaries'): + self.assertEqual(stages[name], next(s for s in base if s.get('name') == name)) diff --git a/scripts/tests/test_full_coverage.py b/scripts/tests/test_full_coverage.py new file mode 100644 index 000000000..27c7118d0 --- /dev/null +++ b/scripts/tests/test_full_coverage.py @@ -0,0 +1,327 @@ +import copy +import json +from pathlib import Path +import subprocess +import sys +import unittest +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / 'scripts/tests')) +import ci_full as ci +import merge_group_result as reuse +from test_ci_full import FakeApi, HEAD +from test_main_ci_plan import evaluate + +TREE = 'd' * 40 +DISPATCH_ONLY = {('vp9-conformance', 'build-round'), ('vp9-conformance', 'pixel-ceiling'), + ('hevc-conformance', 'kernel-differential')} + + +def document(name): + return json.loads(subprocess.check_output(['ruby', '-rjson', '-rpsych', '-e', + 'd=Psych.load_file(ARGV[0]); d["on"] ||= d.delete(true); puts JSON.generate(d)', + str(ROOT / '.github/workflows' / (name + '.yml'))], text=True)) + + +class Coverage(unittest.TestCase): + def setUp(self): + self.api = FakeApi() + self.name = 'ci-full' + self.push = self.api.runs[42] + self.scheduled = dict(self.push, id=900, event='schedule', created_at='2026-11-09T01:00:00Z', + head_commit={'id': HEAD, 'tree_id': TREE}) + self.api.runs[900] = self.scheduled + self.api.jobs[900] = [dict(job, run_id=900, conclusion='skipped', steps=[]) + for job in self.api.jobs[42]] + self.gate = {'name': 'ci-full-coverage', 'run_id': 900, 'run_attempt': 1, 'head_sha': HEAD, + 'status': 'completed', 'conclusion': 'success', + 'steps': [{'name': 'Covered by run 42 attempt 1', 'status': 'completed', + 'conclusion': 'success'}]} + self.api.jobs[900].append(self.gate) + self.index = [self.push] + self.fresh_index = None + self.index_reads = 0 + self.paths = [] + self.env = {'GITHUB_EVENT_NAME': 'schedule', 'GITHUB_REF': 'refs/heads/main', + 'GITHUB_REPOSITORY': ci.REPOSITORY, 'GITHUB_SHA': HEAD, + 'GITHUB_RUN_ID': '900', 'GITHUB_RUN_ATTEMPT': '1'} + + def request(self, path): + self.paths.append(path) + if '/workflows/' in path: + self.index_reads += 1 + runs = self.fresh_index if self.index_reads > 1 and self.fresh_index is not None else self.index + return {'workflow_runs': copy.deepcopy(runs), 'total_count': len(runs)} + if path.endswith('/runs/900'): + return copy.deepcopy(self.scheduled) + if '/jobs?' in path: + jobs = self.api.jobs[42] + return {'jobs': copy.deepcopy(jobs), 'total_count': len(jobs)} + return copy.deepcopy(self.push) + + def lookup(self): + return reuse.full_result(self.env, self.request, TREE, 'ci-full.yml', ci) + + def test_covered_push_has_no_age_bound_and_names_its_attempt(self): + self.assertEqual(self.lookup()[0], (42, 1)) + self.assertIn('/actions/runs/42/attempts/1', self.lookup()[1]) + self.assertIn('full suite skipped', self.lookup()[1]) + self.assertNotIn('\n', self.lookup()[1]) + self.assertTrue(all('created=' not in path for path in self.paths)) + self.assertEqual(ci.report(self.api.event(900), self.api), []) + self.assertFalse(any(method in ('POST', 'PATCH', 'DELETE') for method, *_ in self.api.calls)) + + def test_every_ineligible_push_falls_back(self): + changes = [('event', 'pull_request'), ('event', 'merge_group'), ('event', 'workflow_dispatch'), + ('head_sha', 'b' * 40), ('head_branch', 'topic'), ('path', '.github/workflows/other.yml'), + ('name', 'other'), ('workflow_id', 999), ('head_repository', {'full_name': 'fork/repo'}), + ('repository', {'full_name': 'fork/repo'}), ('status', 'in_progress'), + ('status', 'queued'), ('conclusion', 'failure'), ('conclusion', 'cancelled'), + ('conclusion', 'timed_out'), ('created_at', '2026-11-10T01:00:00Z')] + for key, value in changes: + with self.subTest(key=key, value=value): + self.setUp() + self.push[key] = value + self.assertIsNone(self.lookup()[0]) + + def test_missing_newer_failed_rerun_and_truncated_indexes_do_not_cover(self): + self.index = [] + self.assertIsNone(self.lookup()[0]) + self.setUp() + self.index.append(dict(self.push, id=43, created_at='2026-11-08T01:00:00Z', conclusion='failure')) + self.assertIsNone(self.lookup()[0]) + self.setUp() + self.fresh_index = [dict(self.push, run_attempt=2, status='in_progress', conclusion=None)] + self.assertIsNone(self.lookup()[0]) + self.setUp() + self.index *= 21 + self.assertIsNone(self.lookup()[0]) + self.setUp() + original = self.request + def truncated(path): + data = original(path) + if 'total_count' in data: + data['total_count'] += 1 + return data + with mock.patch.object(self, 'request', side_effect=truncated): + self.assertIsNone(self.lookup()[0]) + + def test_every_full_job_and_proof_is_required_but_measurement_jobs_are_not(self): + original = copy.deepcopy(self.api.jobs[42]) + for mutation in ('missing', 'duplicate', 'skipped', 'wrong_attempt', 'skipped_proof'): + with self.subTest(mutation=mutation): + jobs = copy.deepcopy(original) + if mutation == 'missing': jobs = [job for job in jobs if job['name'] != 'transcode-compat-shard (2)'] + elif mutation == 'duplicate': jobs.append(copy.deepcopy(jobs[0])) + elif mutation == 'skipped': jobs[0]['conclusion'] = 'skipped' + elif mutation == 'wrong_attempt': jobs[0]['run_attempt'] = 2 + else: jobs[0]['steps'][0]['conclusion'] = 'skipped' + self.api.jobs[42] = jobs + self.assertIsNone(self.lookup()[0]) + self.api.jobs[42] = original + [dict(original[0], name='measurement-only', conclusion='skipped', steps=[])] + self.assertEqual(self.lookup()[0], (42, 1)) + + def test_claim_must_be_unique_successful_exact_and_all_other_jobs_skipped(self): + for mutation in ('missing', 'duplicate', 'skipped', 'failed', 'extra_text', 'wrong_type', 'ran_full', + 'failed_schedule', 'failed_gate', 'duplicate_gate'): + with self.subTest(mutation=mutation): + self.setUp() + steps = self.gate['steps'] + if mutation == 'missing': steps.clear() + elif mutation == 'duplicate': steps.append(copy.deepcopy(steps[0])) + elif mutation == 'skipped': steps[0]['conclusion'] = 'skipped' + elif mutation == 'failed': steps[0]['conclusion'] = 'failure' + elif mutation == 'extra_text': steps[0]['name'] += ' SECRET' + elif mutation == 'wrong_type': steps[0]['name'] = None + elif mutation == 'ran_full': self.api.jobs[900][0]['conclusion'] = 'success' + elif mutation == 'failed_schedule': self.scheduled['conclusion'] = 'failure' + elif mutation == 'failed_gate': self.gate['conclusion'] = 'failure' + else: self.api.jobs[900].append(copy.deepcopy(self.gate)) + self.assertFalse(ci.covered_schedule(self.api, self.name, self.scheduled, self.api.jobs[900])) + self.assertTrue(ci.report(self.api.event(900), self.api)) + self.assertNotIn('SECRET', json.dumps(self.api.issues)) + + def test_claim_rechecks_the_pinned_attempt_not_a_later_rerun(self): + self.assertTrue(ci.covered_schedule(self.api, self.name, self.scheduled, self.api.jobs[900])) + self.assertIn(('GET', '/repos/' + ci.REPOSITORY + '/actions/runs/42/attempts/1', None), self.api.calls) + self.scheduled['event'] = 'push' + self.assertFalse(ci.covered_schedule(self.api, self.name, self.scheduled, self.api.jobs[900])) + + def test_uncovered_weekly_work_uses_the_existing_failure_reporter(self): + self.gate.update(conclusion='skipped', steps=[]) + self.api.jobs[900] = [dict(job, run_id=900) for job in self.api.jobs[42]] + [self.gate] + self.assertEqual(ci.report(self.api.event(900), self.api), []) + self.scheduled['conclusion'] = 'failure' + self.api.jobs[900][0]['conclusion'] = 'failure' + self.assertTrue(ci.report(self.api.event(900), self.api)) + + def test_api_failure_does_not_write_an_issue_or_skip_full_work(self): + with mock.patch.object(self, 'request', side_effect=OSError('SECRET')): + result, message = self.lookup() + self.assertIsNone(result) + self.assertNotIn('SECRET', message) + with mock.patch.object(self.api, 'request', side_effect=ci.Failure('api_transport_error')): + with self.assertRaises(ci.Failure): + ci.covered_schedule(self.api, self.name, self.scheduled, self.api.jobs[900]) + self.assertEqual(self.api.issues, []) + + def test_full_push_pr_merge_group_and_dispatch_never_reuse(self): + for event in ('push', 'pull_request', 'merge_group', 'workflow_dispatch'): + with self.subTest(event=event), mock.patch.object(self, 'request') as request: + self.env['GITHUB_EVENT_NAME'] = event + self.assertIsNone(self.lookup()[0]) + request.assert_not_called() + + def test_default_and_explicit_manual_suites_are_independent_of_push_coverage(self): + self.assertEqual(ci.selected_suites(), ['ci-full', 'vp9-corpus', 'vp9-conformance']) + self.assertEqual(ci.selected_suites(['hevc-conformance']), ['hevc-conformance']) + for suites in ([], ['unknown']): + with self.assertRaises(ci.Failure): ci.selected_suites(suites) + self.api.manual() + self.assertEqual(ci.verify(26, self.api, ['hevc-conformance'])[1], [('hevc-conformance', 45, 1)]) + self.assertFalse(any('event=push' in call[1] or 'event=schedule' in call[1] for call in self.api.calls)) + + def test_manual_only_c3_skips_preserve_hevc_push_coverage_and_reporting(self): + push = self.api.runs[45] + for host in ('ubuntu-24.04', 'macos-15'): + self.api.jobs[45].append(dict(self.api.jobs[45][0], name=f'kernel-differential ({host})', + conclusion='skipped', steps=[])) + scheduled = dict(push, id=900, event='schedule', created_at='2026-11-09T01:00:00Z') + self.assertTrue(ci.covers('hevc-conformance', scheduled, push, self.api.jobs[45])) + self.api.runs[900] = scheduled + self.api.jobs[900] = [dict(job, run_id=900, conclusion='skipped', steps=[]) + for job in self.api.jobs[45]] + self.gate['name'] = 'hevc-conformance-coverage' + self.gate['steps'][0]['name'] = 'Covered by run 45 attempt 1' + self.api.jobs[900].append(self.gate) + self.assertEqual(ci.report(self.api.event(900), self.api), []) + + +class Wiring(unittest.TestCase): + def test_manual_only_jobs_never_run_on_automatic_events(self): + for name, job_id in DISPATCH_ONLY: + workflow = document(name) + job = workflow['jobs'][job_id] + self.assertFalse(job.get('needs')) + events = [(event, '') for event in ('push', 'pull_request', 'merge_group', 'workflow_dispatch')] + events += [('schedule', item['cron']) for item in workflow['on']['schedule']] + for event, cron in events: + for mode in ('conformance', 'ceiling', 'build-round'): + for scalar in (False, True): + with self.subTest(workflow=name, job=job_id, event=event, cron=cron, + mode=mode, scalar=scalar): + context = {'github': {'event_name': event, 'event': {'schedule': cron}, + 'ref': 'refs/heads/main', 'repository': ci.REPOSITORY}, + 'inputs': {'mode': mode, 'scalar': scalar}} + selected = event == 'workflow_dispatch' and ( + not scalar if name == 'hevc-conformance' else + mode == ('ceiling' if job_id == 'pixel-ceiling' else 'build-round')) + self.assertEqual(bool(evaluate(job['if'], context)), selected) + + def test_gates_are_schedule_only_fail_open_and_share_the_registry(self): + self.assertEqual(set(reuse.FULL_WORKFLOW_FILES), {spec['file'] for spec in ci.FULL_WORKFLOWS.values()}) + for name, spec in ci.FULL_WORKFLOWS.items(): + with self.subTest(workflow=name): + workflow = document(name) + self.assertEqual(workflow['name'], name) + gate = workflow['jobs']['full-coverage'] + self.assertEqual(gate['name'], name + '-coverage') + daily, weekly = workflow['on']['schedule'] + self.assertEqual(daily['cron'].split()[2:], ['*', '*', '*']) + self.assertEqual(weekly['cron'].split()[2:], ['*', '*', '0']) + self.assertEqual(gate['if'], "github.event_name == 'schedule' && github.event.schedule == '" + daily['cron'] + "'") + self.assertEqual(gate['permissions'], {'contents': 'read', 'actions': 'read'}) + self.assertEqual(gate['outputs']['reuse'], "${{ steps.result.outputs.reuse == 'true' }}") + self.assertTrue(gate['steps'][0]['continue-on-error']) + action = gate['steps'][1] + self.assertEqual(action['uses'], './.github/actions/merge-group-result') + self.assertEqual(action['with']['workflow'], spec['file']) + self.assertTrue(action['continue-on-error']) + self.assertEqual(gate['steps'][2]['name'], 'Covered by run ${{ steps.result.outputs.run }} attempt ${{ steps.result.outputs.attempt }}') + self.assertEqual(gate['steps'][2]['run'], 'true') + + def test_weekly_schedule_cannot_take_the_covered_skip(self): + for name, spec in ci.FULL_WORKFLOWS.items(): + workflow = document(name) + daily, weekly = workflow['on']['schedule'] + for cron, covered in ((daily['cron'], True), (weekly['cron'], False), ('unknown', False)): + with self.subTest(workflow=name, cron=cron): + context = {'github': {'event_name': 'schedule', 'event': {'schedule': cron}, + 'ref': 'refs/heads/main', 'repository': ci.REPOSITORY}, + 'inputs': {'mode': 'conformance'}, 'needs': {}, + 'steps': {'result': {'outputs': {'reuse': 'true'}}}} + selected = set() + for job_id, job in workflow['jobs'].items(): + condition = job.get('if', 'true') + needs = job.get('needs', []) + needs = [needs] if isinstance(needs, str) else needs + parents = all(context['needs'][parent]['result'] == 'success' for parent in needs) + explicit = any(function in condition for function in ('always()', 'cancelled()')) + run = (parents or explicit) and bool(evaluate(condition, context)) + outputs = {key: str(evaluate(value, context)).lower() + for key, value in job.get('outputs', {}).items()} if run else {} + context['needs'][job_id] = {'result': 'success' if run else 'skipped', 'outputs': outputs} + if run: + selected.add(job_id) + self.assertEqual('full-coverage' in selected, covered) + full_jobs = {job.split(' (', 1)[0] for area in spec['areas'].values() for job in area} + if covered: + self.assertEqual(selected, {'full-coverage'}) + else: + self.assertTrue(full_jobs <= selected) + + def test_all_scheduled_work_is_gated_and_descendants_require_success(self): + for name in ci.FULL_WORKFLOWS: + jobs = document(name)['jobs'] + for job_id, job in jobs.items(): + if job_id == 'full-coverage': + continue + if (name, job_id) in DISPATCH_ONLY: + continue + condition = job.get('if', '') + needs = job.get('needs', []) + needs = [needs] if isinstance(needs, str) else needs + with self.subTest(workflow=name, job=job_id): + if name == 'hevc-conformance' and job_id in ('headers', 'hevc-smoke'): + self.assertEqual(needs, ['full-coverage']) + self.assertEqual(condition, "!cancelled() && needs.full-coverage.outputs.reuse != 'true'") + if 'always()' in condition: + self.assertIn('full-coverage', needs) + self.assertIn("needs.full-coverage.outputs.reuse != 'true'", condition) + elif needs == ['full-coverage']: + self.assertIn('!cancelled()', condition) + self.assertIn("needs.full-coverage.outputs.reuse != 'true'", condition) + for covered, cancelled in ((True, False), (False, False), (False, True)): + context = {'cancelled': cancelled, 'github': {'event_name': 'schedule'}, + 'inputs': {'mode': 'conformance'}, + 'needs': {'full-coverage': {'outputs': {'reuse': str(covered).lower()}}}} + self.assertEqual(bool(evaluate(condition, context)), not covered and not cancelled) + else: + self.assertTrue(needs) + self.assertNotIn('full-coverage', needs) + self.assertIn('!cancelled()', condition) + for parent in needs: + self.assertIn(f"needs.{parent}.result == 'success'", condition) + + def test_skipped_gate_outputs_do_not_skip_normal_event_roots(self): + for name in ci.FULL_WORKFLOWS: + workflow = document(name) + for job_id, job in workflow['jobs'].items(): + if job.get('needs') != 'full-coverage': + continue + for event in ('push', 'pull_request', 'workflow_dispatch'): + if event not in workflow['on']: + continue + context = {'github': {'event_name': event, 'ref': 'refs/heads/main'}, + 'inputs': {'mode': 'conformance'}, + 'needs': {'full-coverage': {'result': 'skipped', 'outputs': {'reuse': ''}}}} + with self.subTest(workflow=name, job=job_id, event=event): + expected = not (name == 'hevc-conformance' and event == 'pull_request' + and job_id in ('hevc-vectors', 'profile-census')) + self.assertEqual(bool(evaluate(job['if'], context)), expected) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_h264_conformance_acquisition.py b/scripts/tests/test_h264_conformance_acquisition.py index 4e85c2da0..be7477bd4 100644 --- a/scripts/tests/test_h264_conformance_acquisition.py +++ b/scripts/tests/test_h264_conformance_acquisition.py @@ -5,6 +5,7 @@ import io import json import pathlib +import re import subprocess import sys import tempfile @@ -150,13 +151,17 @@ def test_download_only_verifies_cached_bytes_without_cargo_or_extraction(self): class Workflow(unittest.TestCase): def test_both_jobs_restore_acquire_and_save_before_decoding(self): root = pathlib.Path(__file__).resolve().parents[2] - workflow = (root / '.github/workflows/ci.yml').read_text() - for job in ('conformance', 'conformance-arm64'): - body = workflow.split(f'\n {job}:\n', 1)[1].split('\n #', 1)[0] + for filename, job in (('ci.yml', 'conformance'), ('ci-full.yml', 'h264-full'), + ('ci-full.yml', 'conformance-arm64')): + workflow = (root / '.github/workflows' / filename).read_text() + body = re.split(r'\n (?=\S)', workflow.split(f'\n {job}:\n', 1)[1], maxsplit=1)[0] self.assertLess(body.index('actions/cache/restore@'), body.index('--download-only')) self.assertLess(body.index('--download-only'), body.index('actions/cache/save@')) self.assertLess(body.index('actions/cache/save@'), body.index('name: Decode every in-scope stream')) - self.assertIn("hashFiles('crates/coreav-h264/tests/data/conformance-streams.json')", body) + inputs = "'crates/coreav-h264/tests/data/conformance-streams.json'" + if job == 'conformance': + inputs += ", 'scripts/h264_ci.py'" + self.assertIn(f'hashFiles({inputs})', body) self.assertIn("steps.conformance-acquisition.outcome == 'failure'", body) self.assertIn("format('-partial-{0}-{1}-{2}'", body) self.assertIn("!cancelled()", body) diff --git a/scripts/tests/test_hevc_workflow.py b/scripts/tests/test_hevc_workflow.py index e29827b0c..72ee93988 100644 --- a/scripts/tests/test_hevc_workflow.py +++ b/scripts/tests/test_hevc_workflow.py @@ -55,9 +55,12 @@ def test_full_work_runs_only_on_explicit_full_events(self): self.assertIn(' workflow_dispatch:', workflow) self.assertNotIn('merge_group:', workflow) self.assertIn("cancel-in-progress: ${{ github.event_name == 'pull_request' }}", workflow) - gate = "if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + gate = "((github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')" for job in ('hevc-vectors', 'profile-census'): - self.assertEqual(workflow.split(' ' + job + ':\n', 1)[1].splitlines()[0].strip(), gate) + body = workflow.split(' ' + job + ':\n', 1)[1].split(' runs-on:', 1)[0] + self.assertIn('needs: full-coverage', body) + self.assertIn("!cancelled() && needs.full-coverage.outputs.reuse != 'true' &&", body) + self.assertIn(gate, body) for line in workflow.splitlines(): if line.strip().startswith('paths:'): paths = json.loads(line.split(':', 1)[1]) diff --git a/scripts/tests/test_main_ci_plan.py b/scripts/tests/test_main_ci_plan.py index f69a14f5e..46fd6a6e6 100644 --- a/scripts/tests/test_main_ci_plan.py +++ b/scripts/tests/test_main_ci_plan.py @@ -21,10 +21,11 @@ def value(match): return repr(item) expression = expression.strip().removeprefix('${{').removesuffix('}}').strip() - expression = re.sub(r'\b(?:github|steps|needs|strategy)\.[\w.*-]+', value, expression) + expression = re.sub(r'\b(?:github|steps|needs|strategy|inputs)\.[\w.*-]+', value, expression) expression = re.sub(r'!(?!=)', ' not ', expression.replace('&&', ' and ').replace('||', ' or ')) return eval(expression, {'__builtins__': {}}, - {'always': lambda: True, 'contains': lambda values, entry: entry in values, + {'always': lambda: True, 'cancelled': lambda: context.get('cancelled', False), + 'contains': lambda values, entry: entry in values, 'startsWith': lambda value, prefix: value.startswith(prefix), 'true': True, 'false': False}) @@ -33,7 +34,7 @@ class MainPlan(unittest.TestCase): @classmethod def setUpClass(cls): cls.workflows = {} - for name in ('ci', 'deny', 'python', 'npm'): + for name in ('ci', 'ci-full', 'deny', 'python', 'npm'): text = subprocess.check_output(['ruby', '-rjson', '-rpsych', '-e', 'puts JSON.generate(Psych.load_file(ARGV[0]))', str(ROOT / '.github/workflows' / (name + '.yml'))], text=True) @@ -46,6 +47,84 @@ def concurrency(self, setting, workflow, run_id, event='push', ref='refs/heads/m key = ''.join(str(evaluate(piece, context)) if piece.startswith('${{') else piece for piece in pieces) return key, bool(evaluate(setting['cancel-in-progress'], context)) + def test_reused_groups_skip_fixtures_cache_and_the_already_proved_base_guard(self): + gate = self.workflows['ci']['jobs']['ci-changes'] + publish, = [s for s in gate['steps'] if s.get('name') == 'Publish the shared fixtures before fan-out'] + for event in ('pull_request', 'merge_group', 'push'): + for reused in (False, True): + context = {'github': {'event_name': event, 'event': {'pull_request': {'draft': False}}}, + 'steps': {'result': {'outputs': {'reuse': str(reused).lower()}}, + 'filter': {'outputs': {'run': 'true'}}, + 'bench-guard': {'outputs': {'run': 'true'}}}} + with self.subTest(event=event, reused=reused): + self.assertEqual(bool(evaluate(publish['if'], context)), + not (event == 'merge_group' and reused)) + self.assertEqual(bool(evaluate(gate['outputs']['main-cache'], context)), + event == 'push' and reused) + self.assertEqual(bool(evaluate(gate['outputs']['bench-guard'], context)), + not (event == 'merge_group' and reused)) + self.assertEqual(bool(evaluate(gate['outputs']['run'], context)), not reused) + + def test_every_required_workflow_records_or_checks_reuse_before_work(self): + for workflow, job in [('ci', 'ci-changes'), ('deny', 'deny-checks'), + ('python', 'python-changes'), ('npm', 'npm-changes')]: + steps = self.workflows[workflow]['jobs'][job]['steps'] + self.assertTrue(steps[0]['uses'].startswith('actions/checkout@')) + self.assertEqual(steps[1]['uses'], './.github/actions/merge-group-result') + self.assertEqual(steps[1]['with'], {'workflow': workflow + '.yml'}) + text = subprocess.check_output(['ruby', '-rjson', '-rpsych', '-e', + 'puts JSON.generate(Psych.load_file(ARGV[0]))', + str(ROOT / '.github/actions/merge-group-result/action.yml')], text=True) + action = json.loads(text) + upload, = [s for s in action['runs']['steps'] if s.get('uses', '').startswith('actions/upload-artifact@')] + self.assertEqual(upload['if'], "steps.result.outputs.record_name != ''") + self.assertTrue(upload['continue-on-error']) + self.assertEqual(upload['with']['name'], '${{ steps.result.outputs.record_name }}') + + def test_reused_groups_require_fresh_root_and_fuzz_advisories(self): + steps = self.workflows['deny']['jobs']['deny-checks']['steps'] + fresh, = [s for s in steps if s.get('id') == 'fresh-advisories'] + witness, = [s for s in steps if s.get('name') == 'Require fresh advisories on a reused merge group'] + install, = [s for s in steps if s.get('name') == 'Install cargo-deny'] + condition = "github.event_name == 'merge_group' && steps.result.outputs.reuse == 'true'" + self.assertEqual(fresh['if'], condition) + self.assertEqual(witness['if'], 'always() && ' + condition) + self.assertEqual(install['if'], "steps.result.outputs.reuse != 'true' || github.event_name == 'merge_group'") + self.assertNotIn('continue-on-error', fresh) + self.assertNotIn('continue-on-error', witness) + self.assertEqual(witness['env'], {'ADVISORIES': '${{ steps.fresh-advisories.outcome }}'}) + self.assertIn('python3 scripts/fuzz_plan.py --projects', fresh['run']) + self.assertNotIn('--offline', fresh['run']) + self.assertNotIn('--disable-fetch', fresh['run']) + self.assertFalse(any('advisory' in str(s.get('with', {}).get('path', '')).lower() + for s in steps if 'cache@' in s.get('uses', ''))) + for outcome in ('success', '', 'skipped', 'cancelled', 'failure'): + done = subprocess.run(['bash', '-c', witness['run']], env={'ADVISORIES': outcome}, + capture_output=True) + self.assertEqual(done.returncode == 0, outcome == 'success') + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + (root / 'scripts').mkdir() + (root / 'scripts/fuzz_plan.py').write_text("print('crates/alpha/fuzz\\ncrates/beta/fuzz')\n") + (root / 'cargo').write_text('#!/usr/bin/env python3\nimport json, os, sys\n' + 'with open(os.environ["CALLS"], "a") as output: output.write(json.dumps(sys.argv[1:]) + "\\n")\n' + 'fail = os.environ.get("FAIL", "")\n' + 'sys.exit(int(bool(fail) and (fail == "root" and "--manifest-path" not in sys.argv ' + 'or fail in " ".join(sys.argv))))\n') + (root / 'cargo').chmod(0o755) + for failure in ('', 'root', 'beta'): + calls = root / 'calls.jsonl' + calls.unlink(missing_ok=True) + done = subprocess.run(['bash', '-c', fresh['run']], cwd=root, capture_output=True, + env={'PATH': str(root) + ':/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin', + 'RUNNER_TEMP': directory, 'CALLS': str(calls), 'FAIL': failure}) + self.assertEqual(done.returncode == 0, failure == '', done.stderr) + self.assertEqual([json.loads(line) for line in calls.read_text().splitlines()], [ + ['deny', '--all-features', 'check', 'advisories'], + ['deny', '--all-features', '--manifest-path', 'crates/alpha/fuzz/Cargo.toml', 'check', 'advisories'], + ['deny', '--all-features', '--manifest-path', 'crates/beta/fuzz/Cargo.toml', 'check', 'advisories'], + ]) + def test_aac_source_build_runs_the_whole_transcode_target(self): steps = self.workflows['ci']['jobs']['isobmff-decode']['steps'] step, = [s for s in steps if s.get('name') == 'coreav transcode on the fixtures, with both encoders'] @@ -217,10 +296,10 @@ def test_main_work_survives_every_three_push_gate_ordering(self): for evidence in itertools.product((False, True), repeat=3): plans = [self.plan('push', evidence[run], code=run != 1, h264=run != 1) for run in range(3)] - for name in ('ci', 'python', 'npm'): + for name in ('ci', 'ci-full', 'python', 'npm'): workflow = self.workflows[name] self.assertNotIn('concurrency', workflow) - gate = name + '-changes' + gate = 'full-fixtures' if name == 'ci-full' else name + '-changes' selected = [[(job, index) for job, count in plan[name].items() if not job.endswith('-report') for index in range(count)] for plan in plans] expected = {(run, job, index) for run in range(3) for job, index in selected[run]} @@ -250,20 +329,28 @@ def test_main_work_survives_every_three_push_gate_ordering(self): retained = completed | set(running.values()) | set(pending.values()) if not expected <= retained: failures.append((name, evidence, order, blocked, expected - retained)) - self.assertEqual(checked, 4320) + self.assertEqual(checked, 5760) self.assertEqual(len(failures), 0, f'{len(failures)} lost-work orderings; first: {failures[:1]}') - def plan(self, event, reused, code=True, h264=True, oracle_installer=False): + def plan(self, event, reused, code=True, h264=True, oracle_installer=False, full_covered=False): plans = {} for name, workflow in self.workflows.items(): + if name == 'ci-full' and (event not in ('push', 'schedule', 'workflow_dispatch') + or (event == 'push' and not code)): + plans[name] = {} + continue github = {'event_name': event, 'ref': 'refs/heads/main', 'run_id': '17', 'workflow': name, 'repository': 'owner/repo', 'event': {'pull_request': {'draft': False, 'labels': [], 'head': {'repo': {'full_name': 'owner/repo'}}}}} + if event == 'schedule': + github['event']['schedule'] = workflow['on']['schedule'][0]['cron'] steps = {key: {'outputs': {'run': 'true' if selected else 'false'}} for key, selected in (('filter', code), ('platforms', code), ('h264-conformance', h264), ('bench-guard', code), ('oracle-installer', oracle_installer))} steps['result'] = {'outputs': {'reuse': 'true' if reused else 'false'}} + if name == 'ci-full': + steps['result']['outputs']['reuse'] = 'true' if full_covered else 'false' steps['fuzz-plan'] = {'outputs': {'plan': ''}} context = {'github': github, 'steps': steps, 'needs': {}} jobs = workflow['jobs'] @@ -273,7 +360,8 @@ def plan(self, event, reused, code=True, h264=True, oracle_installer=False): needs = [needs] if isinstance(needs, str) else needs condition = fields.get('if', 'true') dependencies = all(context['needs'][dependency]['result'] == 'success' for dependency in needs) - run = (dependencies or 'always()' in condition) and evaluate(condition, context) + explicit_status = any(function in condition for function in ('always()', 'cancelled()')) + run = (dependencies or explicit_status) and evaluate(condition, context) result = {'result': 'success' if run else 'skipped', 'outputs': {}} if run: for key, expression in fields.get('outputs', {}).items(): @@ -289,11 +377,24 @@ def plan(self, event, reused, code=True, h264=True, oracle_installer=False): def test_reused_main_keeps_only_main_maintenance_and_required_reports(self): plan = self.plan('push', True) self.assertEqual(set(plan['ci']), {'ci-changes', 'ci-report', - 'main-cache', 'conformance-arm64', 'bench-guard'}) + 'main-cache', 'bench-guard'}) + self.assertEqual(plan['ci-full'], { + 'full-fixtures': 1, 'h264-full': 1, 'conformance-arm64': 1, + 'transcode-compat-shard': 3, 'transcode-compat': 1, 'ci-full-report': 1, + }) self.assertEqual(set(plan['python']), {'python-changes', 'python-report'}) self.assertEqual(set(plan['npm']), {'npm-changes', 'npm-report'}) self.assertEqual(set(plan['deny']), {'deny', 'deny-checks'}) - self.assertEqual(sum(sum(jobs.values()) for jobs in plan.values()), 11) + self.assertEqual(sum(sum(jobs.values()) for jobs in plan.values()), 18) + + def test_proven_groups_keep_only_gate_and_report_jobs(self): + plan = self.plan('merge_group', True) + self.assertEqual(set(plan['ci']), {'ci-changes', 'ci-report'}) + self.assertEqual(set(plan['deny']), {'deny-checks', 'deny'}) + self.assertEqual(set(plan['python']), {'python-changes', 'python-report'}) + self.assertEqual(set(plan['npm']), {'npm-changes', 'npm-report'}) + self.assertEqual(plan['ci-full'], {}) + self.assertEqual(sum(sum(jobs.values()) for jobs in plan.values()), 8) def test_installer_only_proof_is_reused_without_starting_the_h264_group(self): for event, reused in (('pull_request', False), ('merge_group', False), @@ -313,22 +414,24 @@ def test_unproven_main_and_merge_groups_do_not_skip_the_suite(self): for plan in (main, group): self.assertIn('check', plan['ci']) self.assertIn('isobmff-writers', plan['ci']) - self.assertIn('transcode-compat-shard', plan['ci']) + self.assertIn('isobmff-command-compat', plan['ci']) + self.assertNotIn('transcode-compat-shard', plan['ci']) self.assertIn('svt-av1', plan['ci']) self.assertIn('linux-x86_64', plan['python']) self.assertIn('package', plan['npm']) self.assertNotIn('main-cache', plan['ci']) - self.assertEqual(sum(sum(jobs.values()) for jobs in main.values()), 35) + self.assertEqual(sum(sum(jobs.values()) for jobs in main.values()), 39) + self.assertEqual(group['ci-full'], {}) self.assertNotIn('conformance-arm64', group['ci']) def test_documentation_only_main_keeps_the_eight_lightweight_jobs(self): plan = self.plan('push', True, code=False, h264=False) self.assertEqual(sum(sum(jobs.values()) for jobs in plan.values()), 8) - def test_representative_non_h264_merge_drops_from_28_to_10_jobs(self): + def test_representative_non_h264_main_retains_independent_full_work(self): counts = [sum(sum(jobs.values()) for jobs in self.plan('push', reused, h264=False).values()) for reused in (False, True)] - self.assertEqual(counts, [29, 10]) + self.assertEqual(counts, [34, 18]) def test_concurrency_never_cancels_or_joins_merge_groups(self): for name, workflow in self.workflows.items(): diff --git a/scripts/tests/test_merge_group_result.py b/scripts/tests/test_merge_group_result.py index 7d0235bae..08b514de4 100644 --- a/scripts/tests/test_merge_group_result.py +++ b/scripts/tests/test_merge_group_result.py @@ -1,12 +1,16 @@ """A main push can reuse only a recent successful queue run of the same workflow and tree.""" import copy +import contextlib import datetime import importlib.util +import io +import json import os import pathlib import subprocess import tempfile import unittest +from unittest import mock import urllib.error SPEC = importlib.util.spec_from_file_location( @@ -104,3 +108,211 @@ def test_cli_reports_the_no_reuse_output_without_action_permissions(self): self.assertEqual(output.read_text(), 'reuse=false\n') self.assertIn('Not a push to main', command.stdout) self.assertEqual(summary.read_text(), command.stdout) + + +class PullRequestEvidence(unittest.TestCase): + def setUp(self): + self.base = 'c' * 40 + self.merge = 'd' * 40 + self.repo = {'full_name': 'owner/repo', 'id': 7} + self.pull = {'state': 'open', 'draft': False, 'user': {'login': 'owner'}, + 'base': {'ref': 'main', 'sha': self.base, 'repo': self.repo}, + 'head': {'ref': 'topic', 'sha': SHA, 'repo': self.repo}} + self.event = {'number': 17, 'pull_request': self.pull, + 'merge_group': {'base_ref': 'refs/heads/main', 'base_sha': self.base, + 'head_sha': self.merge}} + self.env = dict(ENV, GITHUB_EVENT_NAME='merge_group', GITHUB_RUN_ID='100', + GITHUB_RUN_ATTEMPT='1', GITHUB_SHA=self.merge, + GITHUB_REF=f'refs/heads/gh-readonly-queue/main/pr-{self.event["number"]}-' + SHA) + self.run = dict(RUN, event='pull_request', head_branch='topic', run_attempt=1, + repository=self.repo) + self.current = dict(RUN, id=100, event='merge_group', head_sha=self.merge, + head_commit={'id': self.merge, 'tree_id': TREE}, + head_branch=self.env['GITHUB_REF'].removeprefix('refs/heads/'), + created_at='2026-11-01T02:30:00Z') + self.artifact = {'id': 55, 'name': reuse.receipt_name(TREE, self.base, 1), 'expired': False, + 'workflow_run': {'id': 42, 'head_sha': SHA, 'head_branch': 'topic', + 'repository_id': 7, 'head_repository_id': 7}} + self.runs = [self.run] + self.artifacts = [self.artifact] + self.fresh_runs = None + self.fresh_pull = None + self.calls = [] + self.indices = self.pulls = 0 + + def request(self, path): + self.calls.append(path) + if path.endswith('/actions/runs/100'): + return copy.deepcopy(self.current) + if '/pulls/' in path: + self.pulls += 1 + return copy.deepcopy(self.fresh_pull if self.pulls > 1 and self.fresh_pull else self.pull) + if '/actions/workflows/' in path: + self.indices += 1 + runs = self.fresh_runs if self.indices > 1 and self.fresh_runs is not None else self.runs + return {'total_count': len(runs), 'workflow_runs': copy.deepcopy(runs)} + if '/artifacts?' in path: + return {'total_count': len(self.artifacts), 'artifacts': copy.deepcopy(self.artifacts)} + self.fail(path) + + def lookup(self, **kwargs): + options = dict(event=self.event, parents=[self.base]) + options.update(kwargs) + return reuse.result(self.env, self.request, TREE, NOW, 'ci.yml', **options) + + def test_actual_tested_tree_and_base_receipt_proves_a_successful_pr_run(self): + run, message = self.lookup() + self.assertEqual(run, 42) + self.assertIn('successful pull_request', message) + self.assertIn('base ' + self.base, message) + self.assertEqual(len(self.calls), 6) + self.assertIn('tested-tree-v1-' + TREE + '-' + self.base + '-1', self.calls[3]) + self.assertNotEqual(self.run['head_commit']['id'], self.merge) + + def test_tip_tree_alone_is_never_sufficient(self): + self.artifacts = [] + self.assertIsNone(self.lookup()[0]) + + def test_tree_base_attempt_and_artifact_ownership_are_exact(self): + for change in ({'name': reuse.receipt_name('f' * 40, self.base, 1)}, + {'name': reuse.receipt_name(TREE, 'f' * 40, 1)}, + {'name': reuse.receipt_name(TREE, self.base, 2)}, + {'expired': True}, {'expired': None}, {'id': 0}, + *({'workflow_run': dict(self.artifact['workflow_run'], **{key: value})} + for key, value in [('id', 43), ('head_sha', 'f' * 40), + ('head_branch', 'other'), ('repository_id', 8), + ('head_repository_id', 8)])): + with self.subTest(change=change): + self.artifacts = [dict(self.artifact, **change)] + self.assertIsNone(self.lookup()[0]) + self.artifacts = [self.artifact, self.artifact] + self.assertIsNone(self.lookup()[0]) + self.assertIsNone(self.lookup(parents=['f' * 40])[0]) + + def test_ineligible_pulls_and_runs_fail_closed(self): + original_pull = copy.deepcopy(self.pull) + original_run = copy.deepcopy(self.run) + for pull in (dict(original_pull, state='closed'), dict(original_pull, draft=True), + dict(original_pull, user={'login': 'dependabot[bot]'}), + dict(original_pull, base=dict(original_pull['base'], ref='release')), + dict(original_pull, head=dict(original_pull['head'], repo={'full_name': 'fork/repo'}))): + with self.subTest(pull=pull): + self.pull = pull + self.assertIsNone(self.lookup()[0]) + self.pull = original_pull + for key, value in [('event', 'push'), ('head_sha', 'f' * 40), ('head_branch', 'other'), + ('path', '.github/workflows/other.yml'), ('repository', {'full_name': 'fork/repo'}), + ('head_repository', {'full_name': 'fork/repo'}), ('status', 'in_progress'), + ('conclusion', 'failure'), ('conclusion', 'cancelled'), + ('created_at', '2026-10-30T02:00:00Z'), + ('created_at', '2026-11-01T02:45:00Z')]: + with self.subTest(key=key, value=value): + self.runs = [dict(original_run, **{key: value})] + self.assertIsNone(self.lookup()[0]) + + def test_newer_pending_failure_and_rerun_block_old_success(self): + for status, conclusion in [('in_progress', None), ('completed', 'failure')]: + self.runs = [self.run, dict(self.run, id=43, status=status, conclusion=conclusion, + created_at='2026-11-01T02:01:00Z')] + self.assertIsNone(self.lookup()[0]) + self.runs = [dict(self.run, run_attempt=2)] + self.assertIsNone(self.lookup()[0]) + + def test_revalidation_rejects_new_dispatches_reruns_and_head_pushes(self): + for change in ({'id': 43, 'created_at': '2026-11-01T02:01:00Z'}, + {'run_attempt': 2}, {'status': 'in_progress', 'conclusion': None}): + with self.subTest(change=change): + self.setUp() + self.fresh_runs = [dict(self.run, **change)] + self.assertIsNone(self.lookup()[0]) + self.setUp() + self.fresh_pull = dict(self.pull, head=dict(self.pull['head'], sha='f' * 40)) + self.assertIsNone(self.lookup()[0]) + + def test_missing_malformed_and_incomplete_metadata_runs_normal_checks(self): + for value in ({}, {'workflow_runs': [], 'total_count': 1}, + {'workflow_runs': [self.run] * 21, 'total_count': 21}): + with self.subTest(value=value): + with mock.patch.object(self, 'request', return_value=value): + self.assertIsNone(self.lookup()[0]) + with mock.patch.object(self, 'request', side_effect=ValueError('SECRET')): + run, message = self.lookup() + self.assertIsNone(run) + self.assertNotIn('SECRET', message) + + def test_recording_binds_the_checkout_merge_and_base_not_the_tip(self): + env = dict(self.env, GITHUB_EVENT_NAME='pull_request', GITHUB_REF='refs/pull/17/merge') + parents = [self.base, SHA] + name = reuse.pr_receipt(env, self.event, TREE, self.merge, parents) + self.assertEqual(name, reuse.receipt_name(TREE, self.base, 1)) + for change in ({'GITHUB_SHA': SHA}, {'GITHUB_REF': 'refs/heads/topic'}, + {'GITHUB_EVENT_NAME': 'push'}, {'GITHUB_ACTOR': 'dependabot[bot]'}, + {'GITHUB_TRIGGERING_ACTOR': 'dependabot[bot]'}, {'GITHUB_RUN_ATTEMPT': '0'}): + self.assertIsNone(reuse.pr_receipt(dict(env, **change), self.event, TREE, self.merge, parents)) + self.assertIsNone(reuse.pr_receipt(env, self.event, TREE, self.merge, [self.base])) + self.assertIsNone(reuse.pr_receipt(env, self.event, TREE, self.merge, [])) + for key, value in [('draft', True), ('state', 'closed'), ('user', {'login': 'dependabot[bot]'})]: + event = dict(self.event, pull_request=dict(self.pull, **{key: value})) + self.assertIsNone(reuse.pr_receipt(env, event, TREE, self.merge, parents)) + + def test_cli_records_the_actual_checkout_receipt_without_api_requests(self): + with tempfile.TemporaryDirectory() as directory: + root = pathlib.Path(directory) + event = root / 'event.json' + event.write_text(json.dumps(self.event)) + env = dict(self.env, GITHUB_EVENT_NAME='pull_request', GITHUB_REF='refs/pull/17/merge', + GITHUB_EVENT_PATH=str(event), GITHUB_OUTPUT=str(root / 'output'), RUNNER_TEMP=directory) + git = [TREE + '\n', self.merge + '\n', self.merge + ' ' + self.base + ' ' + SHA + '\n'] + with mock.patch.dict(os.environ, env, clear=True), \ + mock.patch.object(reuse.subprocess, 'check_output', side_effect=git), \ + mock.patch('sys.argv', ['merge_group_result.py', '--workflow', 'ci.yml']), \ + mock.patch.object(reuse.urllib.request, 'urlopen') as network, \ + contextlib.redirect_stdout(io.StringIO()): + reuse.main() + network.assert_not_called() + self.assertEqual((root / 'output').read_text(), + 'reuse=false\nrecord_name=' + reuse.receipt_name(TREE, self.base, 1) + + '\nrecord_path=' + str(root / 'tested-tree.txt') + '\n') + self.assertEqual((root / 'tested-tree.txt').read_text(), + f'tree={TREE}\nbase={self.base}\ncommit={self.merge}\n') + + def test_group_metadata_must_bind_the_checked_out_tree(self): + for key, value in [('head_sha', SHA), ('event', 'push'), + ('head_branch', f'gh-readonly-queue/release/pr-{self.event["number"]}-' + SHA), + ('head_commit', {'id': self.merge, 'tree_id': 'f' * 40})]: + with self.subTest(key=key): + self.setUp() + self.current[key] = value + self.assertIsNone(self.lookup()[0]) + + def test_required_cli_paths_never_load_the_full_suite_module(self): + for event, ref in [('push', 'refs/heads/main'), ('merge_group', 'refs/heads/other')]: + with self.subTest(event=event), tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + env = dict(self.env, GITHUB_EVENT_NAME=event, GITHUB_REF=ref, + GITHUB_OUTPUT=str(output), GH_TOKEN='example-not-a-key') + git = [TREE + '\n', self.merge + '\n', self.merge + ' ' + self.base + '\n'] + with mock.patch.dict(os.environ, env, clear=True), \ + mock.patch.object(reuse.subprocess, 'check_output', side_effect=git), \ + mock.patch('sys.argv', ['merge_group_result.py', '--workflow', 'ci.yml']), \ + mock.patch.object(reuse, 'load_full', side_effect=ImportError('broken')) as loader, \ + mock.patch.object(reuse.urllib.request, 'urlopen', + return_value=io.BytesIO(b'{"workflow_runs":[]}')), \ + contextlib.redirect_stdout(io.StringIO()): + reuse.main() + loader.assert_not_called() + self.assertEqual(output.read_text(), 'reuse=false\n') + + def test_full_cli_publishes_only_the_covered_run_attempt(self): + with tempfile.TemporaryDirectory() as directory: + output = pathlib.Path(directory) / 'output' + env = dict(self.env, GITHUB_EVENT_NAME='schedule', GITHUB_REF='refs/heads/main', + GITHUB_OUTPUT=str(output)) + git = [TREE + '\n', self.merge + '\n', self.merge + ' ' + self.base + '\n'] + with mock.patch.dict(os.environ, env, clear=True), \ + mock.patch.object(reuse.subprocess, 'check_output', side_effect=git), \ + mock.patch('sys.argv', ['merge_group_result.py', '--workflow', 'ci-full.yml']), \ + mock.patch.object(reuse, 'full_result', return_value=((42, 1), 'Covered by push')), \ + contextlib.redirect_stdout(io.StringIO()): + reuse.main() + self.assertEqual(output.read_text(), 'reuse=true\nrun=42\nattempt=1\n') diff --git a/scripts/tests/test_vp9_ci_tiers.py b/scripts/tests/test_vp9_ci_tiers.py new file mode 100644 index 000000000..61ebad37d --- /dev/null +++ b/scripts/tests/test_vp9_ci_tiers.py @@ -0,0 +1,121 @@ +import json +import itertools +from pathlib import Path +import subprocess +import sys +import unittest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +sys.path.insert(0, str(ROOT / 'scripts/tests')) +import ci_full +import test_main_ci_plan + + +def load(name): + return json.loads(subprocess.check_output([ + 'ruby', '-rjson', '-rpsych', '-e', + 'd = Psych.load_file(ARGV[0]); d["on"] ||= d.delete(true); puts JSON.generate(d)', + str(ROOT / '.github/workflows' / name), + ], text=True)) + + +class Tiers(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.corpus = load('vp9-corpus.yml') + cls.conformance = load('vp9-conformance.yml') + + def test_corpus_smoke_runs_on_every_relevant_revision_but_full_work_is_not_per_push(self): + for event in ('push', 'pull_request'): + paths = self.corpus['on'][event]['paths'] + self.assertIn('scripts/vp9_smoke.py', paths) + self.assertIn('scripts/tests/test_vp9_smoke.py', paths) + self.assertIn('schedule', self.corpus['on']) + smoke = self.corpus['jobs']['contract'] + self.assertEqual(smoke['if'], "!cancelled() && needs.full-coverage.outputs.reuse != 'true'") + command, = [step['run'] for step in smoke['steps'] if step.get('name') == 'Compare the routine original-file smoke subset'] + self.assertIn('scripts/vp9_smoke.py --native target/release/vp9-corpus-oracle', command) + self.assertNotIn('vp9_corpus_smoke.py', command) + self.assertEqual(self.corpus['jobs']['corpus']['strategy']['matrix']['shard'], list(range(8))) + for event, branch, expected in [('pull_request', 'feature', False), ('merge_group', 'main', False), + ('push', 'feature', False), ('push', 'main', True), + ('schedule', 'main', True), ('workflow_dispatch', 'feature', True)]: + context = {'github': {'repository': ci_full.REPOSITORY, 'event_name': event, 'ref': 'refs/heads/' + branch}, + 'needs': {'contract': {'result': 'success'}}} + self.assertEqual(test_main_ci_plan.evaluate(self.corpus['jobs']['corpus']['if'], context), expected) + + def test_interpolation_smoke_retains_full_daily_main_and_manual_budgets(self): + self.assertIn('schedule', self.conformance['on']) + step, = [step for step in self.conformance['jobs']['interpolation-fuzz']['steps'] + if step.get('name') == 'Compare automatic and scalar interpolation for C3'] + self.assertIn('"$FUZZ_SECONDS"', step['run']) + for event, seconds in [('pull_request', '30'), ('push', '600'), ('schedule', '600'), ('workflow_dispatch', '600')]: + expression = step['env']['FUZZ_SECONDS'].removeprefix('${{').removesuffix('}}').strip() + self.assertEqual(test_main_ci_plan.evaluate(expression, {'github': {'event_name': event}}), seconds) + vectors = self.conformance['jobs']['vectors'] + self.assertEqual(vectors['strategy']['matrix']['os'], ['ubuntu-24.04', 'macos-15']) + listing, = [s['run'] for s in vectors['steps'] if s.get('name') == 'List the vectors and their checksums'] + self.assertIn('for v in m["vectors"]:', listing) + self.assertIn('crates/coreav-vp9/tests/data/vectors.json', listing) + self.assertEqual(sum(s.get('run', '').count('--include-ignored') for s in vectors['steps']), 4) + + def test_full_evidence_names_are_wired_to_real_nonempty_workflow_steps(self): + for workflow, spec in ci_full.FULL_WORKFLOWS.items(): + document = load(spec['file']) + events = [('push', ''), ('workflow_dispatch', '')] + events += [('schedule', item['cron']) for item in document['on']['schedule']] + for (event, cron), scalar in itertools.product(events, (False, True)): + context = {'github': {'event_name': event, 'ref': 'refs/heads/main', + 'repository': ci_full.REPOSITORY, 'event': {'schedule': cron}}, + 'inputs': {'mode': 'conformance', 'scalar': scalar}, + 'needs': {key: {'result': 'success', 'outputs': {'reuse': 'false'}} + for key in document['jobs']}} + expanded = set() + for job_id, job in document['jobs'].items(): + if job_id in ('ci-full-report', 'full-coverage'): + continue + if workflow == 'hevc-conformance' and job_id in ('headers', 'hevc-smoke'): + continue + condition = job.get('if', 'true') + if not test_main_ci_plan.evaluate(condition, context): + continue + matrix = job.get('strategy', {}).get('matrix') + if matrix: + expanded.update(job_id + ' (' + ', '.join(map(str, values)) + ')' + for values in itertools.product(*matrix.values())) + else: + expanded.add(job_id) + expected = {name for group in spec['areas'].values() for name in group} + if workflow == 'hevc-conformance' and event == 'workflow_dispatch' and not scalar: + expected |= {'kernel-differential (ubuntu-24.04)', 'kernel-differential (macos-15)'} + with self.subTest(workflow=workflow, event=event, cron=cron, scalar=scalar): + self.assertEqual(expanded, expected) + self.assertNotIn((workflow, 'kernel-differential'), ci_full.PROOF_STEPS) + self.assertFalse(any(name.startswith('kernel-differential') + for group in spec['areas'].values() for name in group)) + for expected in spec['areas'].values(): + for name in expected: + job_id = name.split(' (', 1)[0] + job = document['jobs'][job_id] + for proof in ci_full.PROOF_STEPS[workflow, job_id]: + step, = [step for step in job['steps'] if step.get('name') == proof] + self.assertTrue(step.get('run') or step.get('uses')) + self.assertNotIn('continue-on-error', step) + + def test_reporter_uses_trusted_code_and_only_metadata_without_cancelling_older_reports(self): + reporter = load('ci-full-failure.yml') + self.assertEqual(set(reporter['on']), {'workflow_run'}) + self.assertEqual(set(reporter['on']['workflow_run']['workflows']), set(ci_full.FULL_WORKFLOWS)) + self.assertNotIn('concurrency', reporter) + job = reporter['jobs']['full-failure-issues'] + self.assertNotIn('concurrency', job) + self.assertEqual(job['permissions'], {'contents': 'read', 'actions': 'read', 'issues': 'write'}) + checkout, = [step for step in job['steps'] if step.get('uses', '').startswith('actions/checkout@')] + self.assertEqual(checkout['with']['ref'], '${{ github.workflow_sha }}') + self.assertNotIn('download-artifact', json.dumps(job)) + self.assertNotIn('workflow_run.head_sha', json.dumps(job['steps'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/tests/test_vp9_smoke.py b/scripts/tests/test_vp9_smoke.py new file mode 100644 index 000000000..f1a173524 --- /dev/null +++ b/scripts/tests/test_vp9_smoke.py @@ -0,0 +1,130 @@ +import hashlib +import contextlib +import io +import json +from pathlib import Path +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest import mock +import urllib.error + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import vp9_smoke + + +class Smoke(unittest.TestCase): + def test_selection_is_eight_distinct_existing_pinned_vectors(self): + source, rows = vp9_smoke.registration() + self.assertEqual(len(rows), 8) + self.assertEqual(len({row['name'] for row in rows}), 8) + self.assertEqual(tuple(row['name'] for row in rows), vp9_smoke.VECTORS) + self.assertTrue(source.startswith('https://')) + for row in rows: + self.assertRegex(row['sha256'], '^[0-9a-f]{64}$') + + def test_unknown_duplicate_or_missing_registered_names_fail(self): + manifest = vp9_smoke.vp9_corpus.read_json(vp9_smoke.ROOT / 'crates/coreav-vp9/tests/data/vectors.json') + for rows in ([row for row in manifest['vectors'] if row['name'] != vp9_smoke.VECTORS[0]], manifest['vectors'] + [next( + row for row in manifest['vectors'] if row['name'] == vp9_smoke.VECTORS[0])]): + with mock.patch.object(vp9_smoke.vp9_corpus, 'read_json', return_value=dict(manifest, vectors=rows)): + with self.assertRaises((StopIteration, ValueError)): + vp9_smoke.registration() + with mock.patch.object(vp9_smoke, 'VECTORS', (*vp9_smoke.VECTORS[:-1], 'unknown.webm')): + with self.assertRaises(StopIteration): + vp9_smoke.registration() + + def fetch(self, data, expected=None, side_effect=None): + row = {'name': 'sample.webm', 'sha256': hashlib.sha256(data if expected is None else expected).hexdigest()} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / row['name'] + response = mock.MagicMock() + response.__enter__.return_value = io.BytesIO(data) + with mock.patch.object(vp9_smoke.urllib.request, 'urlopen', return_value=response, + side_effect=side_effect) as request, mock.patch.object(vp9_smoke.time, 'sleep'): + vp9_smoke.fetch('https://example.invalid/', row, path) + self.assertEqual(path.read_bytes(), data) + return request.call_count + + def test_fetch_verifies_original_bytes_before_publishing_a_file(self): + self.assertEqual(self.fetch(b'original'), 1) + with self.assertRaises(ValueError): + self.fetch(b'changed', expected=b'original') + with mock.patch.object(vp9_smoke, 'LIMIT', 3): + with self.assertRaises(ValueError): + self.fetch(b'oversize') + + def test_fetch_retries_transient_failures_only_within_the_cap(self): + response = mock.MagicMock() + response.__enter__.return_value = io.BytesIO(b'original') + self.assertEqual(self.fetch(b'original', side_effect=[ConnectionResetError(), TimeoutError(), response]), 3) + with self.assertRaises(TimeoutError): + self.fetch(b'original', side_effect=TimeoutError()) + with self.assertRaises(urllib.error.HTTPError): + self.fetch(b'original', side_effect=urllib.error.HTTPError('https://example.invalid', 404, 'missing', {}, None)) + + def exercise(self, comparison): + with tempfile.TemporaryDirectory() as directory: + args = SimpleNamespace(native=Path('native'), oracle_bin=Path('oracle'), out=Path(directory) / 'out') + _, rows = vp9_smoke.registration() + seen = [] + def fetch(source, row, path): + path.write_bytes(b'original') + def compare(entry, tools, b5, private): + seen.append(Path(entry['path']).name) + self.assertEqual(entry['sha256'], rows[len(seen) - 1]['sha256']) + self.assertEqual(Path(entry['path']).read_bytes(), b'original') + self.assertEqual(b5, set()) + self.assertEqual((tools.native, tools.ffmpeg, tools.ffprobe), ('native', 'ffmpeg', 'ffprobe')) + if isinstance(comparison, Exception): + raise comparison + return comparison + with mock.patch.object(vp9_smoke.sys, 'platform', 'linux'), \ + mock.patch.object(vp9_smoke.test_oracle, 'require', return_value=SimpleNamespace(ffmpeg='ffmpeg', ffprobe='ffprobe')), \ + mock.patch.object(vp9_smoke, 'fetch', side_effect=fetch), \ + mock.patch.object(vp9_smoke.vp9_corpus, 'compare', side_effect=compare): + if isinstance(comparison, Exception): + with self.assertRaises(type(comparison)): + vp9_smoke.run(args) + elif comparison.get('status') == 'agree' and type(comparison.get('frames')) is int and comparison['frames'] > 0: + self.assertEqual(vp9_smoke.run(args), 0) + else: + with self.assertRaises(ValueError): + vp9_smoke.run(args) + self.assertEqual(list(args.out.glob('*.webm')), []) + return seen, json.loads((args.out / 'smoke.json').read_text()) + + def test_actual_entrypoint_runs_every_original_through_the_existing_comparator(self): + seen, report = self.exercise({'status': 'agree', 'frames': 7}) + self.assertEqual(tuple(seen), vp9_smoke.VECTORS) + self.assertTrue(report['complete']) + self.assertEqual(len(report['vectors']), 8) + self.assertTrue(all(row['frames'] == 7 for row in report['vectors'])) + + def test_no_pixel_credit_for_refusal_zero_frames_or_false_agreement(self): + for result in ({'status': 'outside_profile0', 'frames': 7}, {'status': 'failed', 'frames': 7}, + {'status': 'agree', 'frames': 0}, {'status': 'agree', 'frames': True}): + seen, report = self.exercise(result) + self.assertEqual(len(seen), 1) + self.assertFalse(report['complete']) + self.assertEqual(report['vectors'], []) + self.assertEqual(report['failed_stage'], 'comparison') + + def test_diagnostics_name_the_stage_and_closed_reason_without_arbitrary_arguments(self): + with contextlib.redirect_stderr(io.StringIO()) as stderr: + _, report = self.exercise({'status': 'outside_profile0', 'frames': 7}) + self.assertEqual(report['failure_reason'], 'missing_pixel_coverage') + self.assertEqual(report['failed_vector'], vp9_smoke.VECTORS[0]) + self.assertIn('stage=comparison reason=missing_pixel_coverage', stderr.getvalue()) + self.assertEqual(vp9_smoke.failure_reason(ValueError('unknown native eligibility')), + 'native_eligibility_unknown') + for error in (ValueError('CREDENTIAL_MARKER_untrusted'), RuntimeError('CREDENTIAL_MARKER_untrusted')): + with contextlib.redirect_stderr(io.StringIO()) as stderr: + _, report = self.exercise(error) + self.assertFalse(report['complete']) + self.assertNotIn('CREDENTIAL_MARKER', json.dumps(report) + stderr.getvalue()) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/vp9_smoke.py b/scripts/vp9_smoke.py new file mode 100644 index 000000000..8857fd364 --- /dev/null +++ b/scripts/vp9_smoke.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run original-file VP9 comparisons on eight existing pinned public vectors.""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time +from types import SimpleNamespace +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'scripts')) +import test_oracle +import vp9_corpus + +VECTORS = ( + 'vp90-2-00-quantizer-10.webm', + 'vp90-2-00-quantizer-40.webm', + 'vp90-2-00-quantizer-63.webm', + 'vp90-2-02-size-320x180-05-resize-12-21-encode_speed-5.webm', + 'vp90-2-10-show-existing-frame.webm', + 'vp90-2-14-resize-10frames-fp-tiles-1-2-4-8.webm', + 'vp90-2-07-frame_parallel-1.webm', + 'vp90-2-17-show-existing-frame.webm', +) +LIMIT = 16 << 20 +FAILURE_REASONS = { + 'unknown native eligibility': 'native_eligibility_unknown', + 'invalid native subsampling': 'native_subsampling_invalid', + 'independent eligibility observations disagree': 'eligibility_disagreement', + 'frame count, order, timestamp, size or MD5 differs': 'frame_comparison_mismatch', + 'public vector checksum mismatch': 'input_checksum_mismatch', + 'public vector exceeds smoke limit': 'input_size_limit', + 'smoke did not compare pixels': 'missing_pixel_coverage', + 'probe failed': 'oracle_probe_failed', + 'inspect failed': 'native_inspection_failed', + 'native failed': 'native_decode_failed', + 'oracle failed': 'oracle_decode_failed', + 'oracle reported an error': 'oracle_diagnostic_failure', +} + + +def failure_reason(error): + if type(error) is ValueError and len(error.args) == 1 and type(error.args[0]) is str: + return FAILURE_REASONS.get(error.args[0], 'invalid_evidence') + if isinstance(error, (TimeoutError, subprocess.TimeoutExpired)): + return 'timeout' + if isinstance(error, urllib.error.HTTPError): + return 'http_failure' + if isinstance(error, (urllib.error.URLError, ConnectionError)): + return 'transport_failure' + if isinstance(error, test_oracle.QualificationError): + return 'oracle_qualification_failed' + return 'unexpected_failure' + + +def registration(): + manifest = vp9_corpus.read_json(ROOT / 'crates/coreav-vp9/tests/data/vectors.json') + rows = manifest['vectors'] + selected = [next(row for row in rows if row['name'] == name) for name in VECTORS] + vp9_corpus.require(len(set(VECTORS)) == 8 and all(sum(row['name'] == name for row in rows) == 1 + for name in VECTORS), 'smoke membership changed') + return manifest['source'], selected + + +def fetch(source, row, destination): + for attempt in range(3): + try: + with urllib.request.urlopen(source + row['name'], timeout=30) as response: + data = response.read(LIMIT + 1) + vp9_corpus.require(len(data) <= LIMIT, 'public vector exceeds smoke limit') + vp9_corpus.require(hashlib.sha256(data).hexdigest() == row['sha256'], 'public vector checksum mismatch') + destination.write_bytes(data) + return + except urllib.error.HTTPError as error: + transient = error.code in (408, 429) or 500 <= error.code <= 599 + error.close() + if not transient or attempt == 2: + raise + except (urllib.error.URLError, TimeoutError, ConnectionError): + if attempt == 2: + raise + time.sleep(2 ** attempt) + + +def run(args): + vp9_corpus.require(sys.platform == 'linux', 'smoke comparisons require Linux') + args.out.mkdir(mode=0o700, parents=True, exist_ok=False) + report = {'complete': False, 'vectors': []} + stage = 'registration' + current = None + try: + source, rows = registration() + stage = 'oracle_qualification' + oracle = test_oracle.require(args.oracle_bin / 'ffmpeg', args.oracle_bin / 'ffprobe') + tools = SimpleNamespace(native=str(args.native), ffmpeg=str(oracle.ffmpeg), ffprobe=str(oracle.ffprobe)) + for row in rows: + current = row['name'] + stage = 'acquisition' + path = args.out / row['name'] + fetch(source, row, path) + try: + stage = 'comparison' + private = args.out / 'private' + private.mkdir(mode=0o700, exist_ok=True) + result = vp9_corpus.compare({'path': str(path), 'sha256': row['sha256']}, tools, set(), private) + vp9_corpus.require(result.get('status') == 'agree' and type(result.get('frames')) is int + and result['frames'] > 0, 'smoke did not compare pixels') + report['vectors'].append({'name': row['name'], 'sha256': row['sha256'], 'frames': result['frames']}) + finally: + path.unlink(missing_ok=True) + vp9_corpus.require(tuple(row['name'] for row in report['vectors']) == VECTORS, 'incomplete smoke') + report['complete'] = True + return 0 + except Exception as error: + reason = failure_reason(error) + report.update(failed_stage=stage, failure_reason=reason) + if current: + report['failed_vector'] = current + print(f'vp9-smoke: stage={stage} reason={reason}', file=sys.stderr) + raise + finally: + if not report['complete']: + report['failed_stage'] = stage + vp9_corpus.write_json(args.out / 'smoke.json', report) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--native', required=True, type=Path) + parser.add_argument('--oracle-bin', required=True, type=Path) + parser.add_argument('--out', required=True, type=Path) + args = parser.parse_args() + try: + return run(args) + except Exception: + print('vp9-smoke: preparation_or_comparison_failed', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/workflow_check.py b/scripts/workflow_check.py index bc13f19eb..25b5b2b6d 100644 --- a/scripts/workflow_check.py +++ b/scripts/workflow_check.py @@ -106,7 +106,7 @@ MATRIX = '-${{ strategy.job-index }}' # The workflows that cancel job by job, so that removing every group from # one fails rather than leaving nothing to check. -JOB_GROUPS = {'ci.yml', 'python.yml', 'npm.yml'} +JOB_GROUPS = {'ci-full.yml', 'ci.yml', 'python.yml', 'npm.yml'} # A step's ref: a full lowercase commit sha, or a Docker image by digest. PINNED = re.compile(r'\A[^@\s]+@[0-9a-f]{40}\Z|\Adocker://[^@\s]+@sha256:[0-9a-f]{64}\Z') WORKFLOW_SUFFIXES = ('.yml', '.yaml') @@ -345,6 +345,8 @@ def trigger_filters(text): if isinstance(data, str): return data out = [] + if not isinstance(data['on'], dict): + return out for event, fields in data['on'].items(): for key in ('paths', 'paths-ignore'): if not isinstance(fields, dict) or key not in fields: @@ -522,7 +524,6 @@ def workflow_files(root): root = stream.children.first&.root top = root ? mapping.(root) : {} events = top["on"] -abort "use an event mapping under on:" if events && !events.is_a?(Psych::Nodes::Mapping) setting = top["concurrency"] abort "use a mapping for workflow concurrency" if setting && !setting.is_a?(Psych::Nodes::Mapping) puts JSON.generate({ "on" => events ? decode.(events) : {}, "concurrency" => setting ? decode.(setting) : nil }) @@ -548,6 +549,7 @@ def read_yaml(text, program): return json.loads(run.stdout) +@functools.lru_cache(maxsize=512) def yaml_entries(text): """[{'line': n, 'ref': str} or {'line': n, 'error': str}] for the `uses` keys of a file and [{'line': n, 'run': str}] for its `run` keys with a @@ -597,6 +599,17 @@ def check(root): names = {} # check name -> [(workflow, job id)] for path in workflows: lines = path.read_text(encoding='utf-8').splitlines() + if path.name in unlisted: + data = workflow_yaml('\n'.join(lines)) + if isinstance(data, str): + problems.append(f'{path.name}: {data}') + else: + events = data['on'] + if isinstance(events, str): + events = [events] + if isinstance(events, (list, dict)) and 'merge_group' in events: + problems.append(f'{path.name}: remove `merge_group:` from a workflow marked not required; ' + 'keep its PR, main and full-tier triggers') jobs[path.name] = found = jobs_of(lines) if not found: problems.append(f'{path.name}: no job found; the parser expects two-space job ids') diff --git a/scripts/workflow_check_selftest.py b/scripts/workflow_check_selftest.py index 7f4f72f27..7ca0628d2 100755 --- a/scripts/workflow_check_selftest.py +++ b/scripts/workflow_check_selftest.py @@ -133,6 +133,22 @@ def call_local(self, ref): def test_the_repository_passes(self): self.assertEqual(check.check(self.root), []) + def test_nonrequired_workflows_do_not_repeat_work_in_the_merge_queue(self): + for events in (' merge_group:\n types: [checks_requested]\n', + ' "merge_group" : {types: [checks_requested]}\n'): + with self.subTest(events=events): + self.write('.github/workflows/probe.yml', '# workflow-check: not required\non:\n' + events + + 'jobs:\n probe:\n runs-on: ubuntu-24.04\n steps:\n - run: echo ok\n') + self.assertEqual(check.check(self.root), + ['probe.yml: remove `merge_group:` from a workflow marked not required; ' + 'keep its PR, main and full-tier triggers']) + for events in ('merge_group', '[push, merge_group]'): + self.write('.github/workflows/probe.yml', '# workflow-check: not required\non: ' + events + + '\njobs:\n probe:\n runs-on: ubuntu-24.04\n steps:\n - run: echo ok\n') + self.assertEqual(check.check(self.root), + ['probe.yml: remove `merge_group:` from a workflow marked not required; ' + 'keep its PR, main and full-tier triggers']) + def test_a_comment_is_not_a_step(self): self.labels.write_text(self.labels.read_text() + '# Example: {uses: actions/checkout@v4}\n') self.assertEqual(check.check(self.root), []) @@ -198,8 +214,13 @@ def test_a_workflow_without_any_job_group_is_reported(self): self.assertTrue(any(p.startswith('ci.yml: give `check` exactly') for p in found), found) def test_matrix_legs_sharing_a_group_are_reported(self): - found = self.ci('-transcode-compat-shard-${{ strategy.job-index }}-', '-transcode-compat-shard-') - self.assertTrue(any(p.startswith('ci.yml: give `transcode-compat-shard` exactly') for p in found), found) + path = self.root / '.github/workflows/ci-full.yml' + text = path.read_text() + original = '-transcode-compat-shard-${{ strategy.job-index }}-' + self.assertIn(original, text) + path.write_text(text.replace(original, '-transcode-compat-shard-')) + found = check.check(self.root) + self.assertTrue(any(p.startswith('ci-full.yml: give `transcode-compat-shard` exactly') for p in found), found) def test_a_report_job_with_a_group_is_reported(self): found = self.ci(' ci-report:\n', ' ci-report:\n concurrency:\n group: x\n')