From 03db0664dd1861c3f6040d17b00d7595cb6cfe7c Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:32:22 +0900 Subject: [PATCH 1/6] docs(shapes): register the PNS-free fixture method Signed-off-by: Gunwoo Kim --- docs/design/github-shapes-200-pns-free.md | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 docs/design/github-shapes-200-pns-free.md diff --git a/docs/design/github-shapes-200-pns-free.md b/docs/design/github-shapes-200-pns-free.md new file mode 100644 index 000000000..9793ff448 --- /dev/null +++ b/docs/design/github-shapes-200-pns-free.md @@ -0,0 +1,55 @@ +# GitHub Shapes 200 PNS-free registration + +This registers `github-shapes-200-pns-free`, a separate measurement requested by the maintainer. It uses the [original registration](github-shapes-200.md), with only the fixture change and generation gate below. This document supplies a method, not results. No judged command or judged fixture generation runs before this registration and its tools merge. + +## Purpose and unchanged scope + +The original registration refuses exact credit for AAC-decoded output whose input contains perceptual noise substitution, or PNS. A separately registered fixture disables that encoder tool without changing CoreAV's AAC decoder or the output predicates. This isolates the fixture's effect on this generated representative. It does not establish decoder conformance or compatibility on other inputs. + +The frozen roster, its digest, all 200 shapes and 1,470 repository-shape pairs stay unchanged. Instantiation, value and filter representatives, unresolved construction, denominators, outcome order, weights, exact decoded-output and copied-packet predicates, probe witnesses and replay rules are inherited unchanged. `compatible_aac` and `compatible_avc1` remain separate misses for the exact headline. The original registration and its retained results are not relabeled or replaced. + +## Fixture recipe + +Only command 0 of `tools/compat-corpus/github_shapes_fixtures.py` changes for the new registration. Insert `-aac_pns 0` immediately after `-c:a aac`: + +```text +-f lavfi -i testsrc2=size=64x48:rate=10:duration=2 -f lavfi -i sine=frequency=440:sample_rate=48000:duration=2 -c:v libx264 -pix_fmt yuv420p -c:a aac -aac_pns 0 source.mp4 +``` + +All other settings retain their defaults. The Matroska copy, concat list, PNG sequence and raw BGR24 remain derived from that source by the unchanged commands. The WAV, MP3, raw S16, PNG, JPEG and GIF recipes do not change. The original recipe's command list remains available unchanged under `github-shapes-200`. + +## Generation gate and retained receipts + +Before any judged command, scan the AAC in both `source.mp4` and `source.mkv` with the existing opt-in `pns_diagnostic` built by: + +```sh +cargo build --locked --release -p coreav-aac --features diagnostics --example pns_diagnostic +``` + +Require a complete `coreav.aac-pns-diagnostic/1` scan under the existing `github_shapes_aac.noise_count` rules. Packet count must equal the frame-list length, every channel must be present, and the number of PNS bands must be zero. Independently require that packet count equals the pinned ffprobe's audio packet count. A container adapter may copy AAC into a diagnostic MP4 only if the complete ordered packet payloads and AAC configuration remain identical to the bound input. It must not decode or re-encode audio. + +Any missing or failed scan, PNS band, incomplete channel or packet-count mismatch is `fixture_generation`. It voids the acquisition before any shape runs, rather than counting as an output miss. The bundle retains scan metadata, packet-count evidence, any adapter receipts, input digests, the diagnostic binary digest and its build command. The manifest binds those receipts and the selected registration. Replay independently repeats the scan on the bound fixtures and refuses missing, changed or nonzero-PNS evidence. + +The per-command `exact_without_pns` guard stays unchanged. If it fails, that command remains `different`; passing the generation gate does not grant exact output credit. + +## Paired acquisition + +One main-only workflow dispatch selects both recipes at one clean merged main SHA. It acquires each recipe once in separate evidence directories using the same build and pinned programs. The original recipe is a pairing control, not a retry of an earlier acquisition. The workflow also accepts either registration explicitly; every bundle and summary names its selected registration so evidence cannot be judged under the other recipe. + +The host remains GitHub-hosted `ubuntu-24.04`, Linux x86-64. The build remains `cargo build --release --locked -p coreav-cli --features release-full`. FFmpeg 9.0.2, ffprobe and x264 retain the original pins and qualification. Build identities, dependencies, timeouts, fallback restrictions, program-start rules, accounting and failure handling stay unchanged. Failures remain misses. A replacement acquisition is allowed only for a recorded infrastructure or harness fault, with the failed evidence retained. + +The method manifest additionally binds this design note. Tool-file digests change with implementation, as in the original registration's later method versions. Historical bundles remain replayable at their own measured commits, not under substituted method digests. + +## Controls before merge + +PR CI uses only independent controls, never the judged fixture generator or a command from the frozen roster. Encode a separate pink-noise control with the pinned encoder twice. Default settings must produce PNS bands; `-aac_pns 0` must produce none. Exercise the MP4 and Matroska gate paths. Mutation controls must reject one PNS band, an incomplete scan, missing channels, packet-count mismatch, changed adapter packets or configuration, missing receipts and a substituted registration. Each generation-gate failure must occur before a judged command can start. + +## Order and evidence custody + +1. Check the premise by counting PNS on the retained generated source from the completed original acquisition. This diagnostic judges no command. Stop if no PNS is found. +2. Commit this plan first, then the tools, workflow selection, generation gate, replay checks and controls in the same pull request. +3. Pass CI, independent review, exact-head similarity and the independent PNS controls before merge. +4. Dispatch the paired acquisition from main at one merged SHA, then replay each bound bundle. +5. Retain all acquisition evidence and summaries in restricted storage. No acquisition evidence, summaries or per-shape outcomes are uploaded as public workflow artifacts or printed in public logs. Credentials exist only in the workflow's retention step and never reach media child processes. + +This change adds no public results page, README text, site text or documentation-index entry. Results are not announced and are not used in outward copy. Any separately retained comparison must name both fixtures and commits. A difference between them is a fixture effect, not a CoreAV improvement, a replacement for the registered coverage number or a claim about other inputs. Outcomes may fall as well as rise because the no-PNS PCM bar is stricter. Lossy encoded output still needs the unchanged exact predicate; removing PNS does not establish bit-identical AAC decode. From 6c82dfa4e16a7d9a28295fd4031c52975b37d04a Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:50:28 +0900 Subject: [PATCH 2/6] feat(shapes): gate the paired PNS-free acquisition Signed-off-by: Gunwoo Kim --- .github/workflows/github-shapes.yml | 70 +++- docs/design/github-shapes-200-pns-free.md | 2 +- tools/compat-corpus/github_shapes.py | 56 ++- tools/compat-corpus/github_shapes_fixtures.py | 19 +- tools/compat-corpus/github_shapes_pns_gate.py | 114 ++++++ tools/compat-corpus/github_shapes_store.py | 132 +++++++ .../compat-corpus/tests/test_github_shapes.py | 5 +- .../tests/test_github_shapes_pns_free.py | 339 ++++++++++++++++++ .../tests/test_github_shapes_store.py | 113 ++++++ 9 files changed, 821 insertions(+), 29 deletions(-) create mode 100644 tools/compat-corpus/github_shapes_pns_gate.py create mode 100644 tools/compat-corpus/github_shapes_store.py create mode 100644 tools/compat-corpus/tests/test_github_shapes_pns_free.py create mode 100644 tools/compat-corpus/tests/test_github_shapes_store.py diff --git a/.github/workflows/github-shapes.yml b/.github/workflows/github-shapes.yml index d973add8b..8c38a6375 100644 --- a/.github/workflows/github-shapes.yml +++ b/.github/workflows/github-shapes.yml @@ -7,10 +7,11 @@ on: paths: - .github/workflows/github-shapes.yml - docs/design/github-shapes-200.md + - docs/design/github-shapes-200-pns-free.md - tools/compat-corpus/github_shapes*.py - tools/compat-corpus/decoded_output.py - tools/compat-corpus/encode_breadth.py - - tools/compat-corpus/tests/test_github_shapes.py + - tools/compat-corpus/tests/test_github_shapes*.py - scripts/install-oracle-ffmpeg.sh - scripts/install-x264.sh - scripts/oracle* @@ -27,6 +28,11 @@ on: description: Full merged main SHA containing the registration and tools required: true type: string + registration: + description: Acquire both registered recipes at the same candidate + type: choice + options: [paired] + default: paired permissions: contents: read @@ -73,7 +79,21 @@ jobs: scripts/oracle-ffmpeg-macos.env "$RUNNER_TEMP/oracle-owner/scripts/" git checkout --detach "$COMMIT" test -f docs/design/github-shapes-200.md + test -f docs/design/github-shapes-200-pns-free.md test -f tools/compat-corpus/github_shapes.py + - name: Require private evidence retention + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: | + python3 - <<'PY' + import os + names = ('R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'R2_BUCKET') + if not all(os.environ.get(name) for name in names): + raise SystemExit('private_retention_configuration') + PY - name: Install build dependencies and pinned programs run: | set -euo pipefail @@ -83,8 +103,10 @@ jobs: scripts/install-x264.sh "$RUNNER_TEMP/oracle" python3 -m venv "$RUNNER_TEMP/aac-python" "$RUNNER_TEMP/aac-python/bin/pip" install --quiet --only-binary=:all: --require-hashes -r scripts/aac-pns-requirements.txt + "$RUNNER_TEMP/aac-python/bin/pip" install --quiet 'boto3==1.35.36' echo "$RUNNER_TEMP/aac-python/bin" >> "$GITHUB_PATH" - name: Build the full feature set + id: build run: | set -euo pipefail cargo build --release --locked -p coreav-cli --features release-full @@ -97,9 +119,12 @@ jobs: record = {'commit': os.environ['COMMIT'], 'command': ['cargo', 'build', '--release', '--locked', '-p', 'coreav-cli', '--features', 'release-full'], 'coreav_sha256': sha('target/release/coreav'), 'lock_sha256': sha('Cargo.lock')} - Path(os.environ['RUNNER_TEMP'], 'shape-build.json').write_text(json.dumps(record)) + root = Path(os.environ['RUNNER_TEMP'], 'shape-evidence') + root.mkdir() + (root / 'build.json').write_text(json.dumps(record)) PY - name: Acquire and replay all registered shapes + id: acquire run: | set -euo pipefail ORACLE_PATHS="$(python3 -I -B "$RUNNER_TEMP/oracle-owner/scripts/test_oracle.py" --resolve \ @@ -109,23 +134,29 @@ jobs: echo "stage=oracle_qualification: invalid retained oracle pair" >&2 exit 1 fi - python3 tools/compat-corpus/github_shapes.py run --commit "$COMMIT" \ - --build-record "$RUNNER_TEMP/shape-build.json" \ - --coreav "$PWD/target/release/coreav" --ffmpeg "$FFMPEG" \ - --pns-diagnostic "$PWD/target/release/examples/pns_diagnostic" \ - --ffprobe "$FFPROBE" --x264 "$RUNNER_TEMP/oracle/x264" \ - --bundle "$RUNNER_TEMP/shape-evidence" --scratch "$RUNNER_TEMP/shape-replay" \ - --summary "$RUNNER_TEMP/shape-summary.json" - - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - if: always() - with: - name: github-shapes-evidence - path: | - ${{ runner.temp }}/shape-evidence - ${{ runner.temp }}/shape-summary.json - ${{ runner.temp }}/shape-build.json - if-no-files-found: error - retention-days: 30 + for REGISTRATION in github-shapes-200-pns-free github-shapes-200; do + ROOT="$RUNNER_TEMP/shape-evidence/$REGISTRATION" + mkdir "$ROOT" + python3 tools/compat-corpus/github_shapes.py run --commit "$COMMIT" \ + --registration "$REGISTRATION" \ + --build-record "$RUNNER_TEMP/shape-evidence/build.json" \ + --coreav "$PWD/target/release/coreav" --ffmpeg "$FFMPEG" \ + --pns-diagnostic "$PWD/target/release/examples/pns_diagnostic" \ + --ffprobe "$FFPROBE" --x264 "$RUNNER_TEMP/oracle/x264" \ + --bundle "$ROOT/bundle" --scratch "$RUNNER_TEMP/shape-replay/$REGISTRATION" \ + --summary "$ROOT/summary.json" + done + printf '%s\n' '{"complete":true}' > "$RUNNER_TEMP/shape-evidence/completion.json" + - name: Keep acquisition evidence privately + if: always() && steps.acquire.outcome != 'skipped' + env: + R2_ENDPOINT: ${{ secrets.COREAV_CORPUS_R2_ENDPOINT }} + R2_ACCESS_KEY_ID: ${{ secrets.COREAV_CORPUS_R2_ACCESS_KEY_ID }} + R2_SECRET_ACCESS_KEY: ${{ secrets.COREAV_CORPUS_R2_SECRET_ACCESS_KEY }} + R2_BUCKET: ${{ vars.COREAV_CORPUS_R2_BUCKET }} + run: | + python3 tools/compat-corpus/github_shapes_store.py --root "$RUNNER_TEMP/shape-evidence" \ + --prefix "evidence/github-shapes-200-paired/$COMMIT/$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT" aac-controls: if: github.event_name == 'pull_request' runs-on: ubuntu-24.04 @@ -154,5 +185,6 @@ jobs: COREAV_SHAPES_AAC_DIAGNOSTIC: ${{ github.workspace }}/target/release/examples/pns_diagnostic run: | python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_aac.py + python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_pns_free.py python3 -I -m unittest discover -s scripts/tests -p test_aac_pns.py python3 -I -m unittest discover -s scripts/tests -p test_aac_pcm_eligibility.py diff --git a/docs/design/github-shapes-200-pns-free.md b/docs/design/github-shapes-200-pns-free.md index 9793ff448..6acc50972 100644 --- a/docs/design/github-shapes-200-pns-free.md +++ b/docs/design/github-shapes-200-pns-free.md @@ -47,7 +47,7 @@ PR CI uses only independent controls, never the judged fixture generator or a co ## Order and evidence custody 1. Check the premise by counting PNS on the retained generated source from the completed original acquisition. This diagnostic judges no command. Stop if no PNS is found. -2. Commit this plan first, then the tools, workflow selection, generation gate, replay checks and controls in the same pull request. +2. Commit this plan first, then the tools, workflow selection, generation gate, replay checks and controls in the same change. 3. Pass CI, independent review, exact-head similarity and the independent PNS controls before merge. 4. Dispatch the paired acquisition from main at one merged SHA, then replay each bound bundle. 5. Retain all acquisition evidence and summaries in restricted storage. No acquisition evidence, summaries or per-shape outcomes are uploaded as public workflow artifacts or printed in public logs. Credentials exist only in the workflow's retention step and never reach media child processes. diff --git a/tools/compat-corpus/github_shapes.py b/tools/compat-corpus/github_shapes.py index 1d3d29f68..f55a9ee50 100644 --- a/tools/compat-corpus/github_shapes.py +++ b/tools/compat-corpus/github_shapes.py @@ -22,6 +22,7 @@ import github_shapes_inputs as inputs import github_shapes_set as roster import github_shapes_aac as aac +import github_shapes_pns_gate as pns_gate import oracle_pin from oracle_audio import INPUT_ARGS as AUDIO_DECODE_INPUT @@ -32,7 +33,8 @@ EXACT_CODECS = ('h264', 'vp9', 'aac', 'opus') COMPATIBLE = 'compatible_avc1' MAPPING = re.compile(r'^\s*Stream #\d+:\d+(?:\[[^]]*\])?(?: \([^)]*\))? -> #(\d+):(\d+) \((.*)\)\s*$', re.M) -METHOD = ('docs/design/github-shapes-200.md', 'scripts/oracle-ffmpeg.env', +METHOD = ('docs/design/github-shapes-200.md', 'docs/design/github-shapes-200-pns-free.md', + 'scripts/oracle-ffmpeg.env', '.github/workflows/github-shapes.yml', 'Cargo.lock', 'docs/design/aac-pcm-eligibility.md', 'docs/design/aac-pns-eligibility.md', 'docs/compat/audio-decoder-eligibility.json', 'scripts/aac-pns-requirements.txt', @@ -501,6 +503,8 @@ def replay(opts): if opts.summary.resolve().is_relative_to(bundle) or opts.scratch.resolve().is_relative_to(bundle): raise EvidenceError('replay_destinations') manifest = json.loads((bundle / 'manifest.json').read_text()) + if manifest.get('registration') != opts.registration or opts.registration not in fixtures.REGISTRATIONS: + raise EvidenceError('registration') if manifest['method'] != method() or manifest['inventory'] != inventory(bundle): raise EvidenceError('evidence_identity') if manifest['programs'] != verify_programs(opts) or manifest['features'] != FEATURES or manifest['build_command'] != BUILD: @@ -517,11 +521,45 @@ def replay(opts): '$FFPROBE': str(Path(opts.ffprobe).resolve()), '$X264': str(Path(opts.x264).resolve()), '$AAC_DIAGNOSTIC': str(Path(opts.pns_diagnostic).resolve())} opts.scratch.mkdir(parents=True, exist_ok=False) + if json.loads((bundle / 'fixtures/commands.json').read_text()) != fixtures.recipe(opts.registration): + raise EvidenceError('fixture_recipe') + if opts.registration == fixtures.PNS_FREE: + expected = manifest.get('pns_gate') + if not isinstance(expected, dict) or set(expected) != {'source.mp4', 'source.mkv'}: + raise EvidenceError('fixture_generation:scan_incomplete') + try: + for name in expected: + retained = bundle / 'pns-gate' / name + metadata = retained / 'diagnostic/metadata.json' + if aac.noise_count(json.loads(metadata.read_text())) or pns_gate.sha(metadata) != expected[name]['metadata_sha256']: + raise EvidenceError('fixture_generation:scan_incomplete') + if json.loads((retained / 'gate.json').read_text()) != expected[name]: + raise EvidenceError('fixture_generation:scan_incomplete') + scanned = retained / 'carriage.mp4' if name.endswith('.mkv') else bundle / 'fixtures' / name + if pns_gate.sha(scanned) != expected[name]['scanned_sha256']: + raise EvidenceError('fixture_generation:packet_binding') + for receipt in expected[name]['receipts'].values(): + for stream in ('stdout', 'stderr'): + if pns_gate.sha(retained / receipt[stream]) != receipt[stream + '_sha256']: + raise EvidenceError('fixture_generation:scan_incomplete') + actual = pns_gate.scan_fixture(bundle / 'fixtures' / name, + opts.scratch / 'pns-gate' / name, programs, environment(), execute) + if actual != expected[name]: + raise EvidenceError('fixture_generation:packet_binding') + except pns_gate.FixtureError as error: + raise EvidenceError('fixture_generation:' + str(error)) from None + except EvidenceError: + raise + except (ValueError, OSError, KeyError, TypeError): + raise EvidenceError('fixture_generation:scan_incomplete') from None + elif 'pns_gate' in manifest or (bundle / 'pns-gate').exists(): + raise EvidenceError('registration') outcomes = [{'rank': row['rank'], 'outcome': judge(row, record, bundle, programs, environment(), opts.scratch)} for row, record in zip(rows, records)] summary = summarize(rows, outcomes) summary['results'] = outcomes summary['commit'] = manifest['commit'] + summary['registration'] = opts.registration write(opts.summary, summary) if manifest['inventory'] != inventory(bundle): raise EvidenceError('evidence_changed_during_replay') @@ -544,7 +582,15 @@ def run(opts): bundle = opts.bundle.resolve() bundle.mkdir(parents=True, exist_ok=False) env = environment() - fixtures.generate(bundle / 'fixtures', programs['$FFMPEG'], env, execute) + try: + fixtures.generate(bundle / 'fixtures', programs['$FFMPEG'], env, execute, opts.registration) + scans = {name: pns_gate.scan_fixture(bundle / 'fixtures' / name, bundle / 'pns-gate' / name, + programs, env, execute) for name in ('source.mp4', 'source.mkv') + if opts.registration == fixtures.PNS_FREE} + except pns_gate.FixtureError as error: + raise EvidenceError('fixture_generation:' + str(error)) from None + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError): + raise EvidenceError('fixture_generation:scan_failed') from None fixture_identity = inventory(bundle) initial_method = method() records = [] @@ -562,8 +608,11 @@ def run(opts): raise EvidenceError('acquisition_identity_changed') if any(digest(bundle / path) != identity for path, identity in fixture_identity.items()): raise EvidenceError('fixture_changed') - manifest = {'commit': opts.commit, 'method': initial_method, 'programs': identities, 'build': build, + manifest = {'commit': opts.commit, 'registration': opts.registration, + 'method': initial_method, 'programs': identities, 'build': build, 'features': FEATURES, 'build_command': BUILD, 'inventory': inventory(bundle)} + if opts.registration == fixtures.PNS_FREE: + manifest['pns_gate'] = scans write(bundle / 'manifest.json', manifest) return replay(opts) @@ -571,6 +620,7 @@ def run(opts): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('mode', choices=('run', 'replay')) + parser.add_argument('--registration', choices=fixtures.REGISTRATIONS, required=True) for name in ('coreav', 'ffmpeg', 'ffprobe', 'x264', 'pns-diagnostic'): parser.add_argument('--' + name, required=True) for name in ('bundle', 'scratch', 'summary'): diff --git a/tools/compat-corpus/github_shapes_fixtures.py b/tools/compat-corpus/github_shapes_fixtures.py index d68a63a29..4a21a7526 100644 --- a/tools/compat-corpus/github_shapes_fixtures.py +++ b/tools/compat-corpus/github_shapes_fixtures.py @@ -4,13 +4,19 @@ from oracle_audio import INPUT_ARGS as AUDIO_DECODE_INPUT +ORIGINAL = 'github-shapes-200' +PNS_FREE = 'github-shapes-200-pns-free' +REGISTRATIONS = (ORIGINAL, PNS_FREE) -def generate(folder, ffmpeg, env, execute): - folder.mkdir() + +def recipe(registration): + if registration not in REGISTRATIONS: + raise ValueError('fixture_registration') video = ['-f', 'lavfi', '-i', 'testsrc2=size=64x48:rate=10:duration=2'] audio = ['-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=2'] - commands = [ - [*video, *audio, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-c:a', 'aac', 'source.mp4'], + pns = ['-aac_pns', '0'] if registration == PNS_FREE else [] + return [ + [*video, *audio, '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-c:a', 'aac', *pns, 'source.mp4'], ['-i', 'source.mp4', '-c', 'copy', 'source.mkv'], [*audio, '-c:a', 'pcm_s16le', 'source.wav'], [*audio, '-c:a', 'libmp3lame', 'source.mp3'], @@ -21,6 +27,11 @@ def generate(folder, ffmpeg, env, execute): [*AUDIO_DECODE_INPUT, '-i', 'source.wav', '-f', 's16le', 'source.s16'], [*AUDIO_DECODE_INPUT, '-i', 'source.mp4', '-pix_fmt', 'bgr24', '-f', 'rawvideo', 'source.bgr'], ] + + +def generate(folder, ffmpeg, env, execute, registration): + commands = recipe(registration) + folder.mkdir() for index, args in enumerate(commands): receipt = execute([ffmpeg, '-nostdin', '-v', 'error', '-y', *args], folder, env, 120, f'generate-{index}') diff --git a/tools/compat-corpus/github_shapes_pns_gate.py b/tools/compat-corpus/github_shapes_pns_gate.py new file mode 100644 index 000000000..345e930f1 --- /dev/null +++ b/tools/compat-corpus/github_shapes_pns_gate.py @@ -0,0 +1,114 @@ +"""Prove complete PNS-free AAC fixtures without changing output predicates.""" +import hashlib +import json +from pathlib import Path +import re +import subprocess + +from github_shapes_aac import noise_count + + +class FixtureError(ValueError): + pass + + +def sha(path): + with path.open('rb') as source: + return hashlib.file_digest(source, 'sha256').hexdigest() + + +def scan_fixture(source, folder, programs, env, execute): + source, folder = Path(source).resolve(), Path(folder).resolve() + folder.mkdir(parents=True, exist_ok=False) + receipts = {} + + def capture(program, args, tag): + receipt = execute([programs[program], *args], folder, env, 120, tag) + if receipt['code'] or receipt['timed_out']: + raise FixtureError('carriage' if tag == 'carriage' else 'scan_failed') + normalized = dict(receipt) + normalized['args'] = [program, *['$FIXTURE' if arg == str(source) else arg.replace(str(folder), '$SCAN') + for arg in receipt['args'][1:]]] + for stream in ('stdout', 'stderr'): + normalized[stream + '_sha256'] = sha(folder / receipt[stream]) + receipts[tag] = normalized + return folder / receipt['stdout'] + + def probe(path, tag): + output = capture('$FFPROBE', ['-v', 'error', '-select_streams', 'a', '-count_packets', + '-show_streams', '-show_packets', '-show_data_hash', 'sha256', '-of', 'json', str(path)], tag) + document = json.loads(output.read_text()) + streams, packets = document['streams'], document['packets'] + if len(streams) != 1 or not isinstance(packets, list) or not 0 < len(packets) <= 16384: + raise FixtureError('configuration') + stream = streams[0] + if stream['codec_type'] != 'audio' or stream['codec_name'] != 'aac' or stream['profile'] != 'LC': + raise FixtureError('configuration') + if type(stream['channels']) is not int or stream['channels'] not in (1, 2): + raise FixtureError('configuration') + if not str(stream['sample_rate']).isdecimal() or not 0 < int(stream['sample_rate']) <= 192000: + raise FixtureError('configuration') + if not str(stream['nb_read_packets']).isdecimal() or int(stream['nb_read_packets']) != len(packets): + raise FixtureError('packet_count') + hashes = [packet['data_hash'] for packet in packets] + config = stream['extradata_hash'] + if any(not isinstance(value, str) or not re.fullmatch(r'SHA256:[0-9a-f]{64}', value) + for value in [config, *hashes]): + raise FixtureError('packet_binding') + if any(packet['stream_index'] != stream['index'] or not str(packet['size']).isdecimal() + or not 0 < int(packet['size']) <= 16 << 20 for packet in packets): + raise FixtureError('packet_binding') + return {'rate': int(stream['sample_rate']), 'channels': stream['channels'], + 'configuration': config, 'packets': [(p['data_hash'], int(p['size'])) for p in packets]} + + try: + original = probe(source, 'fixture-probe') + scanned = source + if source.suffix == '.mkv': + scanned = folder / 'carriage.mp4' + capture('$FFMPEG', ['-nostdin', '-v', 'error', '-i', str(source), '-map', '0:a:0', + '-c:a', 'copy', '-f', 'mp4', str(scanned)], 'carriage') + carried = probe(scanned, 'carriage-probe') + if carried['configuration'] != original['configuration'] or any( + carried[key] != original[key] for key in ('rate', 'channels')): + raise FixtureError('configuration') + if carried['packets'] != original['packets']: + raise FixtureError('packet_binding') + diagnostic = folder / 'diagnostic' + diagnostic.mkdir() + capture('$AAC_DIAGNOSTIC', [str(scanned), str(diagnostic)], 'diagnostic') + metadata_path = diagnostic / 'metadata.json' + metadata = json.loads(metadata_path.read_text()) + try: + bands = noise_count(metadata) + except (ValueError, TypeError, KeyError): + raise FixtureError('scan_incomplete') from None + if bands: + raise FixtureError('pns_present') + if metadata['packets'] != len(original['packets']): + raise FixtureError('packet_count') + if any(type(metadata[key]) is not int for key in ('rate', 'channels')) or ( + metadata['rate'] != original['rate'] or metadata['channels'] != original['channels']): + raise FixtureError('configuration') + end = 0 + with scanned.open('rb') as media: + length = scanned.stat().st_size + for frame, (packet_hash, packet_size) in zip(metadata['frames'], original['packets']): + offset, size = frame['offset'], frame['size'] + if type(offset) is not int or type(size) is not int or not ( + end <= offset and size == packet_size and offset + size <= length): + raise FixtureError('packet_binding') + media.seek(offset) + if 'SHA256:' + hashlib.sha256(media.read(size)).hexdigest() != packet_hash: + raise FixtureError('packet_binding') + end = offset + size + result = {'fixture_sha256': sha(source), 'scanned_sha256': sha(scanned), + 'metadata_sha256': sha(metadata_path), 'packets': metadata['packets'], + 'packet_hashes_sha256': hashlib.sha256(json.dumps(original['packets']).encode()).hexdigest(), + 'extradata_hash': original['configuration'], 'pns_bands': bands, 'receipts': receipts} + (folder / 'gate.json').write_text(json.dumps(result, sort_keys=True, indent=2) + '\n') + return result + except FixtureError: + raise + except (ValueError, OSError, KeyError, TypeError, IndexError, subprocess.SubprocessError): + raise FixtureError('scan_failed') from None diff --git a/tools/compat-corpus/github_shapes_store.py b/tools/compat-corpus/github_shapes_store.py new file mode 100644 index 000000000..ad5a66311 --- /dev/null +++ b/tools/compat-corpus/github_shapes_store.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Retain a bounded evidence tree and its verified receipt.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import sys +import tempfile +import zipfile + +LIMIT = 512 * 1024 * 1024 +REGISTRATIONS = ('github-shapes-200', 'github-shapes-200-pns-free') +ENVIRONMENT = ('R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'R2_BUCKET') + + +def store(root, prefix, client, bucket): + root = Path(root) + if not stat.S_ISDIR(root.lstat().st_mode) or not prefix or prefix.startswith('/') or any( + part in ('', '.', '..') for part in prefix.split('/')): + raise ValueError('invalid input') + files = [] + for directory, dirs, names in os.walk(root, followlinks=False): + for name in dirs: + if not stat.S_ISDIR((Path(directory) / name).lstat().st_mode): + raise ValueError('invalid directory') + for name in names: + path = Path(directory) / name + if not stat.S_ISREG(path.lstat().st_mode): + raise ValueError('invalid file') + files.append(path) + if not files: + raise ValueError('empty evidence') + inventory = {} + total = 0 + with tempfile.TemporaryFile() as archive: + with zipfile.ZipFile(archive, 'w', compression=zipfile.ZIP_DEFLATED, allowZip64=True) as output: + for path in sorted(files): + name = path.relative_to(root).as_posix() + flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) + with os.fdopen(os.open(path, flags), 'rb') as source: + info = os.fstat(source.fileno()) + if not stat.S_ISREG(info.st_mode): + raise ValueError('invalid file') + total += info.st_size + if total > LIMIT: + raise ValueError('evidence limit') + checksum = hashlib.sha256() + size = 0 + with output.open(name, 'w', force_zip64=True) as target: + while chunk := source.read(1024 * 1024): + size += len(chunk) + if size > info.st_size: + raise ValueError('file changed') + checksum.update(chunk) + target.write(chunk) + if size != info.st_size: + raise ValueError('file changed') + inventory[name] = {'sha256': checksum.hexdigest(), 'size': size} + size = archive.tell() + if size > LIMIT: + raise ValueError('archive limit') + archive.seek(0) + checksum = hashlib.sha256() + while chunk := archive.read(1024 * 1024): + checksum.update(chunk) + archive_hash = checksum.hexdigest() + archive.seek(0) + client.put_object(Bucket=bucket, Key=f'{prefix}/evidence.zip', Body=archive) + response = client.get_object(Bucket=bucket, Key=f'{prefix}/evidence.zip')['Body'] + try: + checksum = hashlib.sha256() + received = 0 + while chunk := response.read(1024 * 1024): + received += len(chunk) + if received > size: + raise ValueError('archive verification failed') + checksum.update(chunk) + finally: + response.close() + if received != size or checksum.hexdigest() != archive_hash: + raise ValueError('archive verification failed') + summaries = {name: inventory[name] for name in inventory if name.endswith('/summary.json')} + complete = (inventory.get('completion.json') == { + 'sha256': hashlib.sha256(b'{"complete":true}\n').hexdigest(), 'size': len(b'{"complete":true}\n')} + and 'build.json' in inventory and all(f'{name}/summary.json' in summaries for name in REGISTRATIONS)) + receipt = {'schema': 'coreav.github-shapes-retention/1', 'archive': {'sha256': archive_hash, 'size': size}, + 'files': inventory, 'summaries': summaries, 'complete': complete} + body = json.dumps(receipt, sort_keys=True, separators=(',', ':')).encode() + client.put_object(Bucket=bucket, Key=f'{prefix}/retention.json', Body=body) + response = client.get_object(Bucket=bucket, Key=f'{prefix}/retention.json')['Body'] + try: + if response.read(len(body) + 1) != body: + raise ValueError('receipt verification failed') + finally: + response.close() + return complete + + +class Parser(argparse.ArgumentParser): + def error(self, message): + raise ValueError('invalid arguments') + + +def main(argv=None): + try: + parser = Parser(add_help=False) + parser.add_argument('--root', required=True) + parser.add_argument('--prefix', required=True) + args = parser.parse_args(argv) + env = {name: os.environ.get(name) for name in ENVIRONMENT} + if not all(env.values()): + raise ValueError('missing credentials') + import boto3 + from botocore.config import Config + client = boto3.client('s3', endpoint_url=env['R2_ENDPOINT'], + aws_access_key_id=env['R2_ACCESS_KEY_ID'], + aws_secret_access_key=env['R2_SECRET_ACCESS_KEY'], region_name='auto', + config=Config(connect_timeout=10, read_timeout=60, + retries={'total_max_attempts': 3, 'mode': 'standard'})) + complete = store(args.root, args.prefix, client, env['R2_BUCKET']) + print('retention_complete' if complete else 'retention_incomplete') + return 0 if complete else 1 + except Exception: + print('retention_failed', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/compat-corpus/tests/test_github_shapes.py b/tools/compat-corpus/tests/test_github_shapes.py index 34b6d7d6e..ac5a603a7 100644 --- a/tools/compat-corpus/tests/test_github_shapes.py +++ b/tools/compat-corpus/tests/test_github_shapes.py @@ -111,6 +111,7 @@ def setUp(self): fixture = self.bundle / 'fixtures' fixture.mkdir() (fixture / 'source.mp4').write_bytes(b'synthetic control, not media') + run.write(fixture / 'commands.json', run.fixtures.recipe(run.fixtures.ORIGINAL)) self.fake = self.root / 'fake' self.fake.write_text('#!' + sys.executable + '\nimport sys\n' + 'print(\'{"format":{"control_sentinel":"control result"}}\' if "json" in sys.argv ' @@ -189,7 +190,7 @@ def test_reference_failure_keeps_the_row(self): self.assertEqual(self.judge(), 'reference_failed') def test_replay_checks_inventory_and_recomputes_instead_of_trusting_labels(self): - opts = types.SimpleNamespace(bundle=self.bundle, scratch=self.root / 'scratch', + opts = types.SimpleNamespace(bundle=self.bundle, scratch=self.root / 'scratch', registration=run.fixtures.ORIGINAL, summary=self.root / 'summary.json', **{ k: str(self.fake) for k in ('coreav', 'ffmpeg', 'ffprobe', 'x264', 'pns_diagnostic')}) identity = {'coreav': {'sha256': run.digest(self.fake)}} @@ -202,7 +203,7 @@ def test_replay_checks_inventory_and_recomputes_instead_of_trusting_labels(self) entry['shape']['options'].append(['-wrapper-unresolved', '']) records.append({'rank': entry['rank'], 'uninstantiable': 'lost_construction'}) run.write(self.bundle / 'records.json', records) - manifest = {'method': {}, 'programs': identity, 'features': run.FEATURES, + manifest = {'registration': run.fixtures.ORIGINAL, 'method': {}, 'programs': identity, 'features': run.FEATURES, 'build_command': run.BUILD, 'build': build, 'commit': 'a' * 40, 'inventory': run.inventory(self.bundle)} run.write(self.bundle / 'manifest.json', manifest) diff --git a/tools/compat-corpus/tests/test_github_shapes_pns_free.py b/tools/compat-corpus/tests/test_github_shapes_pns_free.py new file mode 100644 index 000000000..a62741992 --- /dev/null +++ b/tools/compat-corpus/tests/test_github_shapes_pns_free.py @@ -0,0 +1,339 @@ +"""Independent controls for fixture registration and PNS generation admission.""" +import copy +import hashlib +import json +import os +from pathlib import Path +import sys +import subprocess +import tempfile +import types +import unittest +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import github_shapes as run +import github_shapes_fixtures as fixtures +import github_shapes_pns_gate as gate + + +class Recipes(unittest.TestCase): + def test_original_recipe_retains_its_registered_bytes(self): + encoded = (json.dumps(fixtures.recipe(fixtures.ORIGINAL), indent=2) + '\n').encode() + self.assertEqual(hashlib.sha256(encoded).hexdigest(), + 'a7cb02bfd0775ff560b5cb4ad52b69e9c3ca6261ef1dd2f075ef7254c461b8c8') + + def test_only_pns_encoder_switch_differs(self): + original = fixtures.recipe(fixtures.ORIGINAL) + changed = fixtures.recipe(fixtures.PNS_FREE) + position = changed[0].index('-aac_pns') + self.assertEqual(changed[0][position - 2:position + 2], ['-c:a', 'aac', '-aac_pns', '0']) + del changed[0][position:position + 2] + self.assertEqual(changed, original) + + def test_unknown_registration_creates_nothing(self): + with tempfile.TemporaryDirectory() as temporary: + target = Path(temporary) / 'fixtures' + execute = mock.Mock() + with self.assertRaisesRegex(ValueError, 'fixture_registration'): + fixtures.generate(target, '/unused', {}, execute, 'unknown') + self.assertFalse(target.exists()) + execute.assert_not_called() + + +class FakeScan: + def __init__(self, fault=None): + self.fault = fault + self.calls = [] + + def __call__(self, argv, cwd, env, seconds, tag, stdin=None): + argv = list(map(str, argv)) + self.calls.append(argv) + output = b'' + code = 0 + if tag == 'carriage': + content = Path(argv[argv.index('-i') + 1]).read_bytes() + Path(argv[-1]).write_bytes(b'wrong' if self.fault == 'carriage_payload' else content) + code = int(self.fault == 'carriage_failed') + elif tag.endswith('probe'): + packet = Path(argv[-1]).read_bytes() + doc = {'streams': [{'index': 0, 'codec_type': 'audio', 'codec_name': 'aac', 'profile': 'LC', + 'sample_rate': '48000', 'channels': 1, 'nb_read_packets': '1', + 'extradata_hash': 'SHA256:' + 'a' * 64}], 'packets': [{'stream_index': 0, + 'size': str(len(packet)), 'data_hash': 'SHA256:' + hashlib.sha256(packet).hexdigest()}]} + if self.fault == 'packet_count': + doc['streams'][0]['nb_read_packets'] = '2' + if self.fault == 'extra_stream': + doc['streams'].append(copy.deepcopy(doc['streams'][0])) + if self.fault == 'carriage_configuration' and tag == 'carriage-probe': + doc['streams'][0]['extradata_hash'] = 'SHA256:' + 'b' * 64 + output = json.dumps(doc).encode() + else: + metadata = {'schema': 'coreav.aac-pns-diagnostic/1', 'packets': 1, 'rate': 48000, + 'channels': 1, 'frames': [{'offset': 0, 'size': 4, 'channels': [{'noise': []}]}]} + if self.fault == 'pns_present': + metadata['frames'][0]['channels'][0]['noise'] = [{}] + if self.fault == 'scan_incomplete': + metadata['packets'] = 2 + if self.fault == 'missing_channel': + metadata['frames'][0]['channels'] = [] + if self.fault == 'invalid_noise': + metadata['frames'][0]['channels'][0]['noise'] = None + if self.fault == 'diagnostic_packet': + metadata['frames'][0]['offset'] = 1 + if self.fault == 'diagnostic_count': + metadata['packets'] = 2 + metadata['frames'].append(copy.deepcopy(metadata['frames'][0])) + if self.fault == 'diagnostic_rate': + metadata['rate'] = 44100 + (Path(argv[-1]) / 'metadata.json').write_text(json.dumps(metadata)) + code = int(self.fault == 'scan_failed') + (cwd / (tag + '.stdout')).write_bytes(output) + (cwd / (tag + '.stderr')).write_bytes(b'') + return {'args': argv, 'code': code, 'timed_out': False, 'stdin': stdin, + 'stdout': tag + '.stdout', 'stderr': tag + '.stderr'} + + +class Controls(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.programs = {key: '/' + key[1:].lower() for key in ('$FFMPEG', '$FFPROBE', '$AAC_DIAGNOSTIC')} + + +class GateControls(Controls): + def test_both_containers_have_complete_packet_bound_receipts(self): + for name in ('source.mp4', 'source.mkv'): + source = self.root / name + source.write_bytes(b'data') + execute = FakeScan() + result = gate.scan_fixture(source, self.root / ('scan-' + name), self.programs, {}, execute) + self.assertEqual((result['packets'], result['pns_bands']), (1, 0)) + self.assertEqual(result['fixture_sha256'], gate.sha(source)) + self.assertNotIn(str(self.root), json.dumps(result)) + if name.endswith('mkv'): + self.assertIn(['-c:a', 'copy'], [call[i:i + 2] for call in execute.calls + for i in range(len(call) - 1)]) + + def test_all_generation_faults_stop_before_any_shape(self): + faults = {'pns_present': 'pns_present', 'scan_incomplete': 'scan_incomplete', + 'missing_channel': 'scan_incomplete', 'invalid_noise': 'scan_incomplete', + 'scan_failed': 'scan_failed', 'packet_count': 'packet_count', + 'extra_stream': 'configuration', 'carriage_failed': 'carriage', + 'carriage_payload': 'packet_binding', 'carriage_configuration': 'configuration', + 'diagnostic_packet': 'packet_binding', 'diagnostic_count': 'packet_count', + 'diagnostic_rate': 'configuration'} + binary = self.root / 'binary' + binary.write_bytes(b'fake executable') + identity = {'coreav': {'sha256': run.digest(binary)}} + build = self.root / 'build.json' + run.write(build, {'commit': 'a' * 40, 'command': run.BUILD, 'coreav_sha256': run.digest(binary), + 'lock_sha256': run.digest(run.ROOT / 'Cargo.lock')}) + + def generate(folder, ffmpeg, env, execute, registration): + folder.mkdir() + for name in ('source.mp4', 'source.mkv'): + (folder / name).write_bytes(b'data') + + for fault, reason in faults.items(): + with self.subTest(fault=fault): + opts = types.SimpleNamespace(commit='a' * 40, registration=fixtures.PNS_FREE, + build_record=build, bundle=self.root / fault, **{name: str(binary) + for name in ('coreav', 'ffmpeg', 'ffprobe', 'x264', 'pns_diagnostic')}) + with mock.patch.object(run, 'merged_source'), \ + mock.patch.object(run.platform, 'system', return_value='Linux'), \ + mock.patch.object(run.platform, 'machine', return_value='x86_64'), \ + mock.patch.dict(os.environ, {'GITHUB_ACTIONS': 'true'}), \ + mock.patch.object(run, 'verify_programs', return_value=identity), \ + mock.patch.object(fixtures, 'generate', side_effect=generate), \ + mock.patch.object(run, 'execute', side_effect=FakeScan(fault)), \ + mock.patch.object(run, 'acquire_one') as acquire, \ + self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:' + reason): + run.run(opts) + acquire.assert_not_called() + self.assertFalse((opts.bundle / 'calls').exists()) + + +class ReplayRegistration(Controls): + def prepare(self, registration): + bundle = self.root / 'bundle' + folder = bundle / 'fixtures' + folder.mkdir(parents=True) + for name in ('source.mp4', 'source.mkv'): + (folder / name).write_bytes(b'data') + run.write(folder / 'commands.json', fixtures.recipe(registration)) + self.rows = [{'rank': i, 'repository_count': 1271 if i == 200 else 1, + 'shape': {'tool': 'ffmpeg'}} for i in range(1, 201)] + run.write(bundle / 'records.json', [{'rank': i} for i in range(1, 201)]) + binary = self.root / 'fake-binary' + binary.write_bytes(b'fake') + self.identity = {'coreav': {'sha256': run.digest(binary)}} + manifest = {'registration': registration, 'commit': 'a' * 40, 'features': run.FEATURES, + 'build_command': run.BUILD, 'method': {}, 'programs': self.identity, + 'build': {'commit': 'a' * 40, 'command': run.BUILD, 'coreav_sha256': run.digest(binary), + 'lock_sha256': run.digest(run.ROOT / 'Cargo.lock')}} + if registration == fixtures.PNS_FREE: + manifest['pns_gate'] = {name: gate.scan_fixture(folder / name, bundle / 'pns-gate' / name, + self.programs, {}, FakeScan()) for name in ('source.mp4', 'source.mkv')} + manifest['inventory'] = run.inventory(bundle) + run.write(bundle / 'manifest.json', manifest) + return types.SimpleNamespace(bundle=bundle, registration=registration, + scratch=self.root / 'scratch', summary=self.root / 'summary.json', + **{name: str(binary) for name in ('coreav', 'ffmpeg', 'ffprobe', 'x264', 'pns_diagnostic')}), manifest + + def replay(self, opts): + with mock.patch.object(run, 'method', return_value={}), \ + mock.patch.object(run, 'verify_programs', return_value=self.identity), \ + mock.patch.object(run.roster, 'load', return_value=self.rows), \ + mock.patch.object(run, 'judge', return_value='different'), \ + mock.patch.object(run, 'execute', side_effect=FakeScan()): + return run.replay(opts) + + def test_replay_repeats_scan_and_names_the_registration(self): + opts, _ = self.prepare(fixtures.PNS_FREE) + self.assertEqual(self.replay(opts), 0) + self.assertEqual(json.loads(opts.summary.read_text())['registration'], fixtures.PNS_FREE) + self.assertTrue((opts.scratch / 'pns-gate/source.mkv/diagnostic/metadata.json').is_file()) + + def test_recipe_substitution_is_refused_even_with_rebound_inventory(self): + opts, manifest = self.prepare(fixtures.PNS_FREE) + run.write(opts.bundle / 'fixtures/commands.json', fixtures.recipe(fixtures.ORIGINAL)) + manifest['inventory'] = run.inventory(opts.bundle) + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'fixture_recipe'): + self.replay(opts) + + def test_missing_and_substituted_registration_are_refused(self): + opts, manifest = self.prepare(fixtures.ORIGINAL) + for value in (None, fixtures.PNS_FREE, 'unknown'): + manifest['registration'] = value + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'registration'): + self.replay(opts) + + def test_missing_gate_and_nonzero_retained_scan_are_refused(self): + opts, manifest = self.prepare(fixtures.PNS_FREE) + stored = copy.deepcopy(manifest['pns_gate']) + del manifest['pns_gate'] + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:scan_incomplete'): + self.replay(opts) + opts.scratch = self.root / 'scratch-again' + manifest['pns_gate'] = stored + path = opts.bundle / 'pns-gate/source.mp4/diagnostic/metadata.json' + metadata = json.loads(path.read_text()) + metadata['frames'][0]['channels'][0]['noise'] = [{}] + run.write(path, metadata) + manifest['inventory'] = run.inventory(opts.bundle) + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:scan_incomplete'): + self.replay(opts) + + def test_original_cannot_carry_a_pns_free_gate(self): + opts, manifest = self.prepare(fixtures.ORIGINAL) + manifest['pns_gate'] = {} + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'registration'): + self.replay(opts) + + + def test_changed_retained_carriage_is_refused_even_after_inventory_update(self): + opts, manifest = self.prepare(fixtures.PNS_FREE) + (opts.bundle / 'pns-gate/source.mkv/carriage.mp4').write_bytes(b'changed') + manifest['inventory'] = run.inventory(opts.bundle) + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:packet_binding'): + self.replay(opts) + + def test_valid_but_substituted_metadata_cannot_replace_an_independent_scan(self): + opts, manifest = self.prepare(fixtures.PNS_FREE) + path = opts.bundle / 'pns-gate/source.mp4/diagnostic/metadata.json' + metadata = json.loads(path.read_text()) + metadata['rate'] = 44100 + run.write(path, metadata) + manifest['pns_gate']['source.mp4']['metadata_sha256'] = gate.sha(path) + run.write(path.parents[1] / 'gate.json', manifest['pns_gate']['source.mp4']) + manifest['inventory'] = run.inventory(opts.bundle) + run.write(opts.bundle / 'manifest.json', manifest) + with self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:packet_binding'): + self.replay(opts) + + +class WorkflowPrivacy(unittest.TestCase): + def test_paired_run_stops_on_pns_free_failure_and_keeps_results_private(self): + workflow = (run.ROOT / '.github/workflows/github-shapes.yml').read_text() + measure = workflow.split(' measure:\n', 1)[1].split(' aac-controls:\n', 1)[0] + self.assertNotIn('upload-artifact', measure) + self.assertIn("if: github.event_name == 'workflow_dispatch'", measure) + self.assertLess(measure.index('Require private evidence retention'), measure.index('Acquire and replay')) + self.assertIn("if: always() && steps.acquire.outcome != 'skipped'", measure) + self.assertIn('test_github_shapes_pns_free.py', workflow.split(' aac-controls:', 1)[1]) + step = measure.split(' - name: Acquire and replay all registered shapes\n', 1)[1] + block = step.split(' run: |\n', 1)[1].split(' - name:', 1)[0] + script = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binary = root / 'bin' + binary.mkdir() + fake = binary / 'python3' + fake.write_text('#!' + sys.executable + '\n' + + 'import os,sys\n' + 'if "--resolve" in sys.argv:\n' + ' print("/oracle/ffmpeg\\n/oracle/ffprobe"); sys.exit(0)\n' + 'registration=sys.argv[sys.argv.index("--registration")+1]\n' + 'with open(os.environ["CALLS"],"a") as out: out.write(registration+"\\n")\n' + 'sys.exit(2 if os.environ.get("FAIL_REG") == registration else 0)\n') + fake.chmod(0o755) + for failure in (fixtures.PNS_FREE, ''): + runner = root / ('failure' if failure else 'success') + (runner / 'shape-evidence').mkdir(parents=True) + calls = runner / 'calls' + result = subprocess.run(['/bin/bash', '-c', script], cwd=root, capture_output=True, + timeout=10, env={'PATH': str(binary) + ':/bin:/usr/bin', 'RUNNER_TEMP': str(runner), + 'COMMIT': 'a' * 40, 'CALLS': str(calls), 'FAIL_REG': failure}) + expected = [fixtures.PNS_FREE] if failure else [fixtures.PNS_FREE, fixtures.ORIGINAL] + self.assertEqual(calls.read_text().splitlines(), expected, result.stderr) + self.assertEqual(result.returncode, 2 if failure else 0, result.stderr) + self.assertEqual((runner / 'shape-evidence/completion.json').exists(), not failure) + for key in ('R2_ENDPOINT', 'R2_ACCESS_KEY_ID', 'R2_SECRET_ACCESS_KEY', 'R2_BUCKET'): + self.assertNotIn(key, run.environment()) + + +@unittest.skipUnless(os.environ.get('COREAV_SHAPES_AAC_DIAGNOSTIC') and os.environ.get('COREAV_SHAPES_ORACLE'), + 'pinned PNS controls run in dedicated Linux CI') +class PinnedPinkNoise(unittest.TestCase): + def test_default_has_pns_and_disabled_has_none_in_both_containers(self): + programs = {'$FFMPEG': str(Path(os.environ['COREAV_SHAPES_ORACLE'], 'ffmpeg')), + '$FFPROBE': str(Path(os.environ['COREAV_SHAPES_ORACLE'], 'ffprobe')), + '$AAC_DIAGNOSTIC': os.environ['COREAV_SHAPES_AAC_DIAGNOSTIC']} + import test_oracle + test_oracle.require(programs['$FFMPEG'], programs['$FFPROBE']) + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for disabled in (False, True): + folder = root / ('off' if disabled else 'default') + folder.mkdir() + args = [programs['$FFMPEG'], '-nostdin', '-v', 'error', '-f', 'lavfi', '-i', + 'anoisesrc=c=pink:a=0.25:r=48000:d=2:seed=1', '-c:a', 'aac', + *(['-aac_pns', '0'] if disabled else []), 'source.mp4'] + self.assertEqual(run.execute(args, folder, run.environment(), 120, 'encode')['code'], 0) + self.assertEqual(run.execute([programs['$FFMPEG'], '-nostdin', '-v', 'error', + '-i', 'source.mp4', '-c', 'copy', 'source.mkv'], folder, run.environment(), + 120, 'copy')['code'], 0) + for name in ('source.mp4', 'source.mkv'): + target = folder / ('scan-' + name) + if disabled: + receipt = gate.scan_fixture(folder / name, target, programs, run.environment(), run.execute) + self.assertGreater(receipt['packets'], 0) + self.assertEqual(receipt['pns_bands'], 0) + repeated = gate.scan_fixture(folder / name, folder / ('repeat-' + name), + programs, run.environment(), run.execute) + self.assertEqual(repeated, receipt) + else: + with self.assertRaisesRegex(gate.FixtureError, 'pns_present'): + gate.scan_fixture(folder / name, target, programs, run.environment(), run.execute) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/compat-corpus/tests/test_github_shapes_store.py b/tools/compat-corpus/tests/test_github_shapes_store.py new file mode 100644 index 000000000..b41f71d03 --- /dev/null +++ b/tools/compat-corpus/tests/test_github_shapes_store.py @@ -0,0 +1,113 @@ +import contextlib +import hashlib +import io +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock +import zipfile + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import github_shapes_store as store + + +class Client: + def __init__(self): + self.objects = {} + self.corrupt = False + + def put_object(self, Bucket, Key, Body): + self.objects[Key] = Body.read() if hasattr(Body, 'read') else Body + + def get_object(self, Bucket, Key): + data = self.objects[Key] + return {'Body': io.BytesIO(data + b'x' if self.corrupt else data)} + + +class StoreTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.client = Client() + + def write(self, name, data): + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + + def test_round_trip_and_receipt(self): + self.write('completion.json', b'{"complete":true}\n') + self.write('build.json', b'build') + for name in store.REGISTRATIONS: + self.write(f'{name}/summary.json', name.encode()) + self.write(f'{name}/evidence/data', b'evidence') + self.assertTrue(store.store(self.root, 'run/one', self.client, 'bucket')) + archive = self.client.objects['run/one/evidence.zip'] + receipt = json.loads(self.client.objects['run/one/retention.json']) + self.assertEqual(receipt['archive'], {'size': len(archive), 'sha256': hashlib.sha256(archive).hexdigest()}) + self.assertTrue(receipt['complete']) + with zipfile.ZipFile(io.BytesIO(archive)) as zipped: + self.assertEqual(set(zipped.namelist()), set(receipt['files'])) + for name, info in receipt['files'].items(): + data = zipped.read(name) + self.assertEqual(info, {'size': len(data), 'sha256': hashlib.sha256(data).hexdigest()}) + self.assertEqual(receipt['summaries']['github-shapes-200/summary.json'], + receipt['files']['github-shapes-200/summary.json']) + + def test_incomplete_still_retained(self): + self.write('build.json', b'build') + self.write('github-shapes-200/evidence/partial', b'partial') + self.assertFalse(store.store(self.root, 'run', self.client, 'bucket')) + self.assertFalse(json.loads(self.client.objects['run/retention.json'])['complete']) + with zipfile.ZipFile(io.BytesIO(self.client.objects['run/evidence.zip'])) as archive: + self.assertEqual(archive.read('github-shapes-200/evidence/partial'), b'partial') + + def test_summaries_alone_cannot_claim_completed_replay(self): + self.write('build.json', b'build') + for name in store.REGISTRATIONS: + self.write(f'{name}/summary.json', b'summary') + self.assertFalse(store.store(self.root, 'run', self.client, 'bucket')) + self.write('completion.json', b'{"complete":false}\n') + self.assertFalse(store.store(self.root, 'other-run', self.client, 'bucket')) + + def test_corruption_prevents_receipt(self): + self.write('build.json', b'build') + self.client.corrupt = True + with self.assertRaises(ValueError): + store.store(self.root, 'run', self.client, 'bucket') + self.assertNotIn('run/retention.json', self.client.objects) + + def test_empty_and_symlinks_rejected(self): + with self.assertRaises(ValueError): + store.store(self.root, 'run', self.client, 'bucket') + self.write('file', b'data') + (self.root / 'link').symlink_to(self.root / 'file') + with self.assertRaises(ValueError): + store.store(self.root, 'run', self.client, 'bucket') + self.assertFalse(self.client.objects) + + def test_limits_and_prefix(self): + self.write('file', b'data') + with mock.patch.object(store, 'LIMIT', 1), self.assertRaises(ValueError): + store.store(self.root, 'run', self.client, 'bucket') + with self.assertRaises(ValueError): + store.store(self.root, '../run', self.client, 'bucket') + + def test_missing_credentials_and_exception_are_redacted(self): + stderr = io.StringIO() + with mock.patch.dict(os.environ, {}, clear=True), contextlib.redirect_stderr(stderr): + self.assertEqual(store.main(['--root', 'private-path', '--prefix', 'private-key']), 1) + self.assertEqual(stderr.getvalue(), 'retention_failed\n') + with mock.patch.object(store.Parser, 'parse_args', side_effect=ValueError('private secret')): + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertEqual(store.main([]), 1) + self.assertEqual(stderr.getvalue(), 'retention_failed\n') + + +if __name__ == '__main__': + unittest.main() From 3418e1e779ef8de9a6c8ad0ffdfddc2e1e0a75fd Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 12:54:44 +0900 Subject: [PATCH 3/6] fix(shapes): confine retained scan receipt paths Signed-off-by: Gunwoo Kim --- tools/compat-corpus/github_shapes.py | 11 +++++++++-- .../tests/test_github_shapes_pns_free.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tools/compat-corpus/github_shapes.py b/tools/compat-corpus/github_shapes.py index f55a9ee50..bda51f265 100644 --- a/tools/compat-corpus/github_shapes.py +++ b/tools/compat-corpus/github_shapes.py @@ -538,9 +538,16 @@ def replay(opts): scanned = retained / 'carriage.mp4' if name.endswith('.mkv') else bundle / 'fixtures' / name if pns_gate.sha(scanned) != expected[name]['scanned_sha256']: raise EvidenceError('fixture_generation:packet_binding') - for receipt in expected[name]['receipts'].values(): + receipts = expected[name]['receipts'] + tags = {'fixture-probe', 'diagnostic'} + if name.endswith('.mkv'): + tags.update(('carriage', 'carriage-probe')) + if set(receipts) != tags: + raise EvidenceError('fixture_generation:scan_incomplete') + for tag, receipt in receipts.items(): for stream in ('stdout', 'stderr'): - if pns_gate.sha(retained / receipt[stream]) != receipt[stream + '_sha256']: + if receipt[stream] != tag + '.' + stream or ( + pns_gate.sha(retained / receipt[stream]) != receipt[stream + '_sha256']): raise EvidenceError('fixture_generation:scan_incomplete') actual = pns_gate.scan_fixture(bundle / 'fixtures' / name, opts.scratch / 'pns-gate' / name, programs, environment(), execute) diff --git a/tools/compat-corpus/tests/test_github_shapes_pns_free.py b/tools/compat-corpus/tests/test_github_shapes_pns_free.py index a62741992..335cddc1a 100644 --- a/tools/compat-corpus/tests/test_github_shapes_pns_free.py +++ b/tools/compat-corpus/tests/test_github_shapes_pns_free.py @@ -260,6 +260,21 @@ def test_valid_but_substituted_metadata_cannot_replace_an_independent_scan(self) self.replay(opts) + def test_retained_receipt_paths_cannot_escape_the_gate_directory(self): + opts, manifest = self.prepare(fixtures.PNS_FREE) + receipt = manifest['pns_gate']['source.mp4'] + receipt['receipts']['fixture-probe']['stdout'] = '../../../outside' + run.write(opts.bundle / 'pns-gate/source.mp4/gate.json', receipt) + manifest['inventory'] = run.inventory(opts.bundle) + run.write(opts.bundle / 'manifest.json', manifest) + with mock.patch.object(gate, 'sha', wraps=gate.sha) as checksum, \ + self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:scan_incomplete'): + self.replay(opts) + self.assertTrue(all(any(call.args[0].resolve().is_relative_to(folder) + for folder in (opts.bundle, opts.scratch)) + for call in checksum.call_args_list)) + + class WorkflowPrivacy(unittest.TestCase): def test_paired_run_stops_on_pns_free_failure_and_keeps_results_private(self): workflow = (run.ROOT / '.github/workflows/github-shapes.yml').read_text() From f6e217cbab879fa00c340500eab901d2d8ac2962 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 14:09:07 +0900 Subject: [PATCH 4/6] ci(shapes): report every new control by name Signed-off-by: Gunwoo Kim --- .github/workflows/github-shapes.yml | 3 ++- tools/compat-corpus/tests/test_github_shapes_pns_free.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/github-shapes.yml b/.github/workflows/github-shapes.yml index 8c38a6375..574cf2677 100644 --- a/.github/workflows/github-shapes.yml +++ b/.github/workflows/github-shapes.yml @@ -185,6 +185,7 @@ jobs: COREAV_SHAPES_AAC_DIAGNOSTIC: ${{ github.workspace }}/target/release/examples/pns_diagnostic run: | python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_aac.py - python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_pns_free.py + python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_pns_free.py -v + python3 -I -m unittest discover -s tools/compat-corpus/tests -p test_github_shapes_store.py -v python3 -I -m unittest discover -s scripts/tests -p test_aac_pns.py python3 -I -m unittest discover -s scripts/tests -p test_aac_pcm_eligibility.py diff --git a/tools/compat-corpus/tests/test_github_shapes_pns_free.py b/tools/compat-corpus/tests/test_github_shapes_pns_free.py index 335cddc1a..ea53b9e64 100644 --- a/tools/compat-corpus/tests/test_github_shapes_pns_free.py +++ b/tools/compat-corpus/tests/test_github_shapes_pns_free.py @@ -283,7 +283,8 @@ def test_paired_run_stops_on_pns_free_failure_and_keeps_results_private(self): self.assertIn("if: github.event_name == 'workflow_dispatch'", measure) self.assertLess(measure.index('Require private evidence retention'), measure.index('Acquire and replay')) self.assertIn("if: always() && steps.acquire.outcome != 'skipped'", measure) - self.assertIn('test_github_shapes_pns_free.py', workflow.split(' aac-controls:', 1)[1]) + for name in ('test_github_shapes_pns_free.py', 'test_github_shapes_store.py'): + self.assertIn(name + ' -v', workflow.split(' aac-controls:', 1)[1]) step = measure.split(' - name: Acquire and replay all registered shapes\n', 1)[1] block = step.split(' run: |\n', 1)[1].split(' - name:', 1)[0] script = '\n'.join(line[10:] for line in block.splitlines()) From f8dcebdc472a9a76f292d36012e9b13cd9ad766f Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 14:10:11 +0900 Subject: [PATCH 5/6] fix(shapes): reject unknown paired selections Signed-off-by: Gunwoo Kim --- .github/workflows/github-shapes.yml | 3 ++- .../tests/test_github_shapes_pns_free.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/github-shapes.yml b/.github/workflows/github-shapes.yml index 574cf2677..f97048ea5 100644 --- a/.github/workflows/github-shapes.yml +++ b/.github/workflows/github-shapes.yml @@ -63,6 +63,7 @@ jobs: timeout-minutes: 240 env: COMMIT: ${{ inputs.commit }} + REGISTRATION: ${{ inputs.registration }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -72,6 +73,7 @@ jobs: run: | set -euo pipefail [ "$GITHUB_REF" = refs/heads/main ] || { echo '::error::main_dispatch_required'; exit 1; } + [ "$REGISTRATION" = paired ] || { echo '::error::fixture_registration'; exit 1; } [[ "$COMMIT" =~ ^[0-9a-f]{40}$ ]] || { echo '::error::candidate_sha'; exit 1; } git merge-base --is-ancestor "$COMMIT" origin/main || { echo '::error::candidate_not_merged'; exit 1; } mkdir -p "$RUNNER_TEMP/oracle-owner/scripts" @@ -106,7 +108,6 @@ jobs: "$RUNNER_TEMP/aac-python/bin/pip" install --quiet 'boto3==1.35.36' echo "$RUNNER_TEMP/aac-python/bin" >> "$GITHUB_PATH" - name: Build the full feature set - id: build run: | set -euo pipefail cargo build --release --locked -p coreav-cli --features release-full diff --git a/tools/compat-corpus/tests/test_github_shapes_pns_free.py b/tools/compat-corpus/tests/test_github_shapes_pns_free.py index ea53b9e64..d6bf99244 100644 --- a/tools/compat-corpus/tests/test_github_shapes_pns_free.py +++ b/tools/compat-corpus/tests/test_github_shapes_pns_free.py @@ -276,10 +276,24 @@ def test_retained_receipt_paths_cannot_escape_the_gate_directory(self): class WorkflowPrivacy(unittest.TestCase): + def test_invalid_pair_selection_stops_before_checkout_or_media(self): + workflow = (run.ROOT / '.github/workflows/github-shapes.yml').read_text() + step = workflow.split(' - name: Require a merged main candidate\n', 1)[1] + block = step.split(' run: |\n', 1)[1].split(' - name:', 1)[0] + script = '\n'.join(line[10:] for line in block.splitlines()) + with tempfile.TemporaryDirectory() as temporary: + result = subprocess.run(['/bin/bash', '-c', script], cwd=temporary, capture_output=True, + timeout=10, env={'PATH': '/bin:/usr/bin', 'GITHUB_REF': 'refs/heads/main', + 'COMMIT': 'a' * 40, 'REGISTRATION': 'unknown'}) + self.assertEqual(result.returncode, 1) + self.assertIn(b'fixture_registration', result.stdout) + self.assertEqual(list(Path(temporary).iterdir()), []) + def test_paired_run_stops_on_pns_free_failure_and_keeps_results_private(self): workflow = (run.ROOT / '.github/workflows/github-shapes.yml').read_text() measure = workflow.split(' measure:\n', 1)[1].split(' aac-controls:\n', 1)[0] self.assertNotIn('upload-artifact', measure) + self.assertIn('[ "$REGISTRATION" = paired ]', measure) self.assertIn("if: github.event_name == 'workflow_dispatch'", measure) self.assertLess(measure.index('Require private evidence retention'), measure.index('Acquire and replay')) self.assertIn("if: always() && steps.acquire.outcome != 'skipped'", measure) From 6706d7f3934eba9b0da1487f0e586754889fbf58 Mon Sep 17 00:00:00 2001 From: Gunwoo Kim Date: Sun, 8 Nov 2026 14:26:27 +0900 Subject: [PATCH 6/6] fix(shapes): distinguish fixture generation failures Signed-off-by: Gunwoo Kim --- docs/agents/compat-corpus.md | 2 ++ docs/design/github-shapes-200-pns-free.md | 2 +- tools/compat-corpus/github_shapes.py | 3 +++ .../tests/test_github_shapes_pns_free.py | 13 +++++++++++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/agents/compat-corpus.md b/docs/agents/compat-corpus.md index 32eb89e99..67715c20f 100644 --- a/docs/agents/compat-corpus.md +++ b/docs/agents/compat-corpus.md @@ -4,6 +4,8 @@ Corpus exclusions, compatibility and codec-coverage workflows with their candida | Rule | Enforced by | |---|---| +| A paired top-200 PNS acquisition distinguishes fixture encoding failures from diagnostic failures before any judged command starts | Separate generation and scan handlers emit closed failure labels in `github_shapes.run`; `test_all_generation_faults_stop_before_any_shape` injects both failures and proves neither can reach acquisition. An external encoder exit cannot be established by a type | +| The PNS registration describes the workflow's paired-only dispatch separately from the CLI's per-registration selection | The workflow's admission guard owns paired-only dispatch; `test_invalid_pair_selection_stops_before_checkout_or_media` executes its rejection and `test_paired_run_stops_on_pns_free_failure_and_keeps_results_private` checks the matching design statement. External workflow selection and English prose cannot share a declared type | | Both exact and avc1-compatible top-200 copy comparisons preserve every packet's keyframe flag | `packet_rows` and `avc1_carriage` include the `K` flag in their compared rows. The keyframe-only regressions in `test_github_shapes.py` use complete packet records and assert the exact-path documents differ only in the selected flag. They reject both mismatch directions for first and later video and audio packets, while other flag bits stay outside this predicate. External probe records require runtime comparison, not a declared stream type | | HEVC correctness shards retain the complete original roster and cannot count a partial, reordered or substituted part as complete | `corpus_shard` owns the deterministic assignment. `test_fixed_corpus_shards_cover_each_original_identity_once` proves the 54-file union without duplicates; `test_shard_completion_requires_its_exact_members_and_every_eof_arm` rejects missing, duplicate, reordered and changed identities and missing EOF arms. The workflow test requires all four non-fail-fast matrix entries and distinct artifact names | | Metadata-only HEVC acquisition does not apply the complete-decoder acceptance predicate | The existing `packet_native` mode selects that predicate only for C2. `test_online_census_releases_each_owned_media_file_and_keeps_receipts` preserves metadata-only cleanup and false full-coverage flags on its two-file fixture; the complete-roster/EOF tests retain the 54-file C2 denominator. The command boundary owns runtime mode selection | diff --git a/docs/design/github-shapes-200-pns-free.md b/docs/design/github-shapes-200-pns-free.md index 6acc50972..b45912620 100644 --- a/docs/design/github-shapes-200-pns-free.md +++ b/docs/design/github-shapes-200-pns-free.md @@ -34,7 +34,7 @@ The per-command `exact_without_pns` guard stays unchanged. If it fails, that com ## Paired acquisition -One main-only workflow dispatch selects both recipes at one clean merged main SHA. It acquires each recipe once in separate evidence directories using the same build and pinned programs. The original recipe is a pairing control, not a retry of an earlier acquisition. The workflow also accepts either registration explicitly; every bundle and summary names its selected registration so evidence cannot be judged under the other recipe. +One main-only workflow dispatch selects both recipes at one clean merged main SHA. It acquires each recipe once in separate evidence directories using the same build and pinned programs. The original recipe is a pairing control, not a retry of an earlier acquisition. The workflow accepts only the paired selection. The CLI selects each registration explicitly, and every bundle and summary names its selected registration so evidence cannot be judged under the other recipe. The host remains GitHub-hosted `ubuntu-24.04`, Linux x86-64. The build remains `cargo build --release --locked -p coreav-cli --features release-full`. FFmpeg 9.0.2, ffprobe and x264 retain the original pins and qualification. Build identities, dependencies, timeouts, fallback restrictions, program-start rules, accounting and failure handling stay unchanged. Failures remain misses. A replacement acquisition is allowed only for a recorded infrastructure or harness fault, with the failed evidence retained. diff --git a/tools/compat-corpus/github_shapes.py b/tools/compat-corpus/github_shapes.py index bda51f265..f2ce33b1c 100644 --- a/tools/compat-corpus/github_shapes.py +++ b/tools/compat-corpus/github_shapes.py @@ -591,6 +591,9 @@ def run(opts): env = environment() try: fixtures.generate(bundle / 'fixtures', programs['$FFMPEG'], env, execute, opts.registration) + except (ValueError, OSError, KeyError, TypeError, subprocess.SubprocessError): + raise EvidenceError('fixture_generation:generate_failed') from None + try: scans = {name: pns_gate.scan_fixture(bundle / 'fixtures' / name, bundle / 'pns-gate' / name, programs, env, execute) for name in ('source.mp4', 'source.mkv') if opts.registration == fixtures.PNS_FREE} diff --git a/tools/compat-corpus/tests/test_github_shapes_pns_free.py b/tools/compat-corpus/tests/test_github_shapes_pns_free.py index d6bf99244..918098837 100644 --- a/tools/compat-corpus/tests/test_github_shapes_pns_free.py +++ b/tools/compat-corpus/tests/test_github_shapes_pns_free.py @@ -117,7 +117,8 @@ def test_both_containers_have_complete_packet_bound_receipts(self): for i in range(len(call) - 1)]) def test_all_generation_faults_stop_before_any_shape(self): - faults = {'pns_present': 'pns_present', 'scan_incomplete': 'scan_incomplete', + faults = {'generate_failed': 'generate_failed', 'original_generate_failed': 'generate_failed', + 'pns_present': 'pns_present', 'scan_incomplete': 'scan_incomplete', 'missing_channel': 'scan_incomplete', 'invalid_noise': 'scan_incomplete', 'scan_failed': 'scan_failed', 'packet_count': 'packet_count', 'extra_stream': 'configuration', 'carriage_failed': 'carriage', @@ -132,6 +133,8 @@ def test_all_generation_faults_stop_before_any_shape(self): 'lock_sha256': run.digest(run.ROOT / 'Cargo.lock')}) def generate(folder, ffmpeg, env, execute, registration): + if fault.endswith('generate_failed'): + raise ValueError('fixture_generation') folder.mkdir() for name in ('source.mp4', 'source.mkv'): (folder / name).write_bytes(b'data') @@ -141,17 +144,21 @@ def generate(folder, ffmpeg, env, execute, registration): opts = types.SimpleNamespace(commit='a' * 40, registration=fixtures.PNS_FREE, build_record=build, bundle=self.root / fault, **{name: str(binary) for name in ('coreav', 'ffmpeg', 'ffprobe', 'x264', 'pns_diagnostic')}) + if fault == 'original_generate_failed': + opts.registration = fixtures.ORIGINAL with mock.patch.object(run, 'merged_source'), \ mock.patch.object(run.platform, 'system', return_value='Linux'), \ mock.patch.object(run.platform, 'machine', return_value='x86_64'), \ mock.patch.dict(os.environ, {'GITHUB_ACTIONS': 'true'}), \ mock.patch.object(run, 'verify_programs', return_value=identity), \ mock.patch.object(fixtures, 'generate', side_effect=generate), \ - mock.patch.object(run, 'execute', side_effect=FakeScan(fault)), \ + mock.patch.object(run, 'execute', side_effect=FakeScan(fault)) as scanner, \ mock.patch.object(run, 'acquire_one') as acquire, \ self.assertRaisesRegex(run.EvidenceError, 'fixture_generation:' + reason): run.run(opts) acquire.assert_not_called() + if fault.endswith('generate_failed'): + scanner.assert_not_called() self.assertFalse((opts.bundle / 'calls').exists()) @@ -291,6 +298,8 @@ def test_invalid_pair_selection_stops_before_checkout_or_media(self): def test_paired_run_stops_on_pns_free_failure_and_keeps_results_private(self): workflow = (run.ROOT / '.github/workflows/github-shapes.yml').read_text() + note = (run.ROOT / 'docs/design/github-shapes-200-pns-free.md').read_text() + self.assertIn('The workflow accepts only the paired selection.', note) measure = workflow.split(' measure:\n', 1)[1].split(' aac-controls:\n', 1)[0] self.assertNotIn('upload-artifact', measure) self.assertIn('[ "$REGISTRATION" = paired ]', measure)