diff --git a/integrations/isolation/acq-kits/README.md b/integrations/isolation/acq-kits/README.md index ba96a979..d3d783e7 100644 --- a/integrations/isolation/acq-kits/README.md +++ b/integrations/isolation/acq-kits/README.md @@ -96,3 +96,46 @@ make test-kits # Live, per-backend end-to-end check (needs a backend CLI + a sandbox-capable host): /scripts/verify ``` + +### What a `scripts/verify` exit code means + +The shared verdict contract lives in `verify-report.sh` in this directory, and +**adoption is staged**: only the kits listed as converted below implement it +today. A verdict is a function of a coverage record, not a failure counter, so a +converted run that exercised nothing cannot report the same result as a run that +passed. + +**Converted kits** — `git-ssh-sign`, `usai-provider`. For these, and only these, +the exit codes and reporters below apply. + +| Exit | Meaning | +|---|---| +| `0` | every check that ran passed, and at least one check ran | +| `1` | at least one check FAILED | +| `3` | nothing failed, but the run is **not** a pass: a check could not be performed, or no check ran at all | + +`3` is distinct from `1` so a caller can tell *"we looked and found problems"* +from *"we could not look"*. Treat any non-zero exit as "do not ship". + +The five reporters: `ok` / `bad` are the ran-and-passed / ran-and-failed cases; +`unver` is **could not check** (missing tool, absent credential, no network) and +degrades the verdict to `3`; `skip` is *not applicable in this configuration* +(e.g. a backend-specific check on the other backend) and does **not** degrade +it; `warn` is advisory. + +**Unconverted kits** — `agentic-coding-playbook`, `oci-engine`, `opencode`, +`openchamber`, `paseo`, `pi-coding-agent`, `zscaler-ca-certificate`. These still +use the older two-state verdict, so nothing above applies to them. A non-zero +exit means a check failed *or* a precondition could not be met — the two are not +distinguished — and a `0` does **not** establish that every check ran: these +scripts report an unmet precondition with a bare message that increments no +counter, then exit `0` on the strength of the checks that did run. Treat a `0` +here as "nothing that ran failed", never as full coverage. + +The authoritative classification is `scripts/tests/test_verify_contract.py`. It +executes the verdict contract for converted kits, asserts each unconverted kit +still matches its description, and compares both lists above against its own, so +the split cannot rot in either direction: converting a kit without moving it +fails, a converted kit reverting fails, and a README edit alone fails. The +unconverted list must reach empty, at which point this carve-out goes away and +the contract above applies to every kit. diff --git a/integrations/isolation/acq-kits/git-ssh-sign/scripts/verify b/integrations/isolation/acq-kits/git-ssh-sign/scripts/verify index 9529bd7d..bf866392 100755 --- a/integrations/isolation/acq-kits/git-ssh-sign/scripts/verify +++ b/integrations/isolation/acq-kits/git-ssh-sign/scripts/verify @@ -31,11 +31,11 @@ WORK="$(mktemp -d "${TMPDIR:-/tmp}/git-ssh-sign-verify.XXXXXX")" CREATE_LOG="$WORK/sbx-create.log" KEEP="${KEEP:-0}" -pass=0 -fail=0 -ok() { printf ' \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); } -bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); } -info() { printf '\n\033[1m%s\033[0m\n' "$1"; } +# Shared verdict contract (ok / bad / unver / skip / warn + verify_verdict). +# Sourced rather than redefined here so a future fix to the contract cannot +# leave this kit's copy behind. +. "$KIT_DIR/../verify-report.sh" +verify_reset cleanup() { if [ "$KEEP" = "1" ]; then @@ -51,16 +51,16 @@ in_sbx() { sbx exec "$SBX_NAME" -- sh -c "$1" /dev/null || true; } info "0. Preconditions" if ! command -v sbx >/dev/null 2>&1; then - echo " sbx not on PATH. Install sbx >= 0.34.0 and retry." >&2 - exit 1 + unver "sbx not on PATH — install sbx >= 0.34.0 and retry" + verify_verdict "Summary" + exit $? fi echo " sbx version: $(sbx version 2>/dev/null | grep -oE 'v?[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n1)" [ -f "$KIT_DIR/spec.yaml" ] && ok "kit spec.yaml present" || { bad "spec.yaml missing"; exit 1; } if ssh-add -L >/dev/null 2>&1; then ok "host SSH agent has a key loaded (signing can succeed)" else - echo " NOTE: no key in the host SSH agent. The kit installs fine, but signing" >&2 - echo " will fail until you run: ssh-add ~/.ssh/id_ed25519" >&2 + unver "host SSH agent has no key loaded — signing cannot be exercised (run: ssh-add ~/.ssh/id_ed25519)" fi info "1. sbx kit validate" @@ -102,12 +102,10 @@ if [ "$(in_sbx 'ssh-add -L >/dev/null 2>&1 && echo yes')" = "yes" ]; then ') [ "$out" = "SIGNED" ] && ok "commit was signed and verified" || bad "commit not signed (got: ${out:-none})" else - echo " SKIP: no key in the forwarded SSH agent inside the sandbox." >&2 - echo " Load one on the host (ssh-add) and re-run for the full check." >&2 + # This is the kit's whole purpose. Reporting it as anything other than + # UNVERIFIED would let the script claim the signing path works when it was + # never exercised. + unver "end-to-end signed commit — no key in the forwarded SSH agent (load one with ssh-add, then re-run)" fi -info "Summary" -printf ' %d passed, %d failed\n' "$pass" "$fail" -[ "$fail" -eq 0 ] && printf ' \033[32mAll checks passed.\033[0m\n' || printf ' \033[31mSome checks failed — see notes above.\033[0m\n' - -[ "$fail" -eq 0 ] +verify_verdict "Summary" diff --git a/integrations/isolation/acq-kits/usai-provider/scripts/verify b/integrations/isolation/acq-kits/usai-provider/scripts/verify index b6771750..26da1986 100755 --- a/integrations/isolation/acq-kits/usai-provider/scripts/verify +++ b/integrations/isolation/acq-kits/usai-provider/scripts/verify @@ -39,11 +39,11 @@ CREATE_LOG="$WORK/sbx-create.log" KEEP="${KEEP:-0}" USAI_MODELS_URL="https://api.gsa.usai.gov/api/v1/models" -pass=0 -fail=0 -ok() { printf ' \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); } -bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); } -info() { printf '\n\033[1m%s\033[0m\n' "$1"; } +# Shared verdict contract (ok / bad / unver / skip / warn + verify_verdict). +# Sourced rather than redefined here so a future fix to the contract cannot +# leave this kit's copy behind. +. "$KIT_DIR/../verify-report.sh" +verify_reset cleanup() { if [ "$KEEP" = "1" ]; then @@ -59,16 +59,16 @@ in_sbx() { sbx exec "$SBX_NAME" -- sh -c "$1" /dev/null || true; } info "0. Preconditions" if ! command -v sbx >/dev/null 2>&1; then - echo " sbx not on PATH. Install sbx >= 0.34.0 and retry." >&2 - exit 1 + unver "sbx not on PATH — install sbx >= 0.34.0 and retry" + verify_verdict "Summary" + exit $? fi echo " sbx version: $(sbx version 2>/dev/null | grep -oE 'v?[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n1)" [ -f "$KIT_DIR/spec.yaml" ] && ok "kit spec.yaml present" || { bad "spec.yaml missing"; exit 1; } if sbx secret ls 2>/dev/null | grep -q 'api.gsa.usai.gov'; then ok "USAi custom secret is set" else - echo " NOTE: no USAi custom secret found. Set it with:" >&2 - echo " sbx secret set-custom -g --host api.gsa.usai.gov --env USAI_API_KEY" >&2 + unver "USAi custom secret is not set — the credential path cannot be exercised (sbx secret set-custom -g --host api.gsa.usai.gov --env USAI_API_KEY)" fi info "1. sbx kit validate" @@ -121,14 +121,12 @@ echo " HTTP status: ${status:-}" if [ "$status" = "200" ]; then ok "USAi models API returned 200" elif [ -z "$status" ]; then - echo " NOTE: could not probe (no curl, or no network); skipping." >&2 + # in_sbx swallows the exit status, so an empty result means either "no curl in + # the guest" or "sbx exec failed" — both are "could not look", never a pass. + unver "USAi models API — could not probe (no curl in the guest, or no network)" else bad "USAi API returned $status (key missing/expired, or egress blocked)" echo " Check: sbx policy log $SBX_NAME (api.gsa.usai.gov), and the USAi secret." >&2 fi -info "Summary" -printf ' %d passed, %d failed\n' "$pass" "$fail" -[ "$fail" -eq 0 ] && printf ' \033[32mAll checks passed.\033[0m\n' || printf ' \033[31mSome checks failed — see notes above.\033[0m\n' - -[ "$fail" -eq 0 ] +verify_verdict "Summary" diff --git a/integrations/isolation/acq-kits/verify-report.sh b/integrations/isolation/acq-kits/verify-report.sh new file mode 100644 index 00000000..cc69b452 --- /dev/null +++ b/integrations/isolation/acq-kits/verify-report.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# +# verify-report.sh — the shared verdict contract for acq-kit `scripts/verify`. +# +# A verdict is a function of a COVERAGE RECORD, not a failure counter: a run +# that checked nothing, and a run where every check passed, must not produce +# the same result. This library supplies that record and the five states a +# verify script needs to express it. +# +# STATES +# +# ok "" a check ran and passed +# bad "" a check ran and failed +# unver "" a check COULD NOT RUN — missing tool, absent +# credential, no network. Not a pass. Counted, +# and it degrades the verdict. +# skip "" deliberately not applicable in this +# configuration (e.g. a backend-specific check on +# the other backend). Counted and reported, but +# does NOT degrade the verdict. +# warn "" advisory only; a documented fallback was taken. +# Counted and reported; does not degrade. +# +# `unver` degrades the verdict because a check could not run. `skip` records a +# check that does not apply to the current configuration and does not degrade it. +# +# EXIT CODES (verify_verdict) +# +# 0 every check that ran passed, and at least one check ran +# 1 at least one check FAILED +# 3 nothing failed, but the run is not a clean pass: +# - at least one check could not be performed (unverified > 0), or +# - no check ran at all (pass == 0) +# +# 3 is separate from 1 so a caller can tell "we looked and found problems" from +# "we could not look". A caller that only cares about red/green can test for +# non-zero and will correctly refuse to treat a degraded run as success. +# +# USAGE +# +# KIT_DIR="$(cd "$(dirname "$0")/.." && pwd)" +# . "$KIT_DIR/../verify-report.sh" +# verify_reset # initialize the counters +# info "1. Something" +# ok "it worked" +# unver "API probe — no network" +# verify_verdict "Offline gate" # prints the summary, returns 0/1/3 +# +# `verify_verdict` takes an optional label so a script with more than one gate +# (openchamber, paseo and pi-coding-agent each have three) can name each one. +# +# POSIX sh compatible on purpose: these scripts run on contributor machines with +# whatever /bin/sh is, and the arithmetic and parameter expansion here stay +# within that subset. + +# Counters. Deliberately not `local` — they are the script-wide coverage record. +verify_reset() { + pass=0 + fail=0 + unverified=0 + skipped=0 + warned=0 +} + +ok() { + printf ' \033[32mPASS\033[0m %s\n' "$1" + pass=$((pass + 1)) +} + +bad() { + printf ' \033[31mFAIL\033[0m %s\n' "$1" + fail=$((fail + 1)) +} + +# Could not check. The message SHOULD say what was not checked AND why, because +# the reader's next question is always "why not". +unver() { + printf ' \033[35mUNVERIFIED\033[0m %s\n' "$1" + unverified=$((unverified + 1)) +} + +# Not applicable in this configuration. Does not degrade the verdict. +skip() { + printf ' \033[33mSKIP\033[0m %s\n' "$1" + skipped=$((skipped + 1)) +} + +# Advisory note; a documented fallback was taken. Does not degrade the verdict. +warn() { + printf ' \033[33mWARN\033[0m %s\n' "$1" + warned=$((warned + 1)) +} + +info() { + printf '\n\033[1m%s\033[0m\n' "$1" +} + +# Print the coverage record and return the verdict. +# +# $1 — optional gate label (default "Summary"). +verify_verdict() { + _vr_label="${1:-Summary}" + + info "$_vr_label" + printf ' %d passed, %d failed, %d unverified, %d skipped, %d warned\n' \ + "$pass" "$fail" "$unverified" "$skipped" "$warned" + + if [ "$fail" -gt 0 ]; then + printf ' \033[31m%s FAILED — see above.\033[0m\n' "$_vr_label" + return 1 + fi + + # Nothing ran. Arithmetic over zero must never produce a pass: pass=0 and + # fail=0 is a report of nothing proven, not a report of success. + if [ "$pass" -eq 0 ]; then + printf ' \033[35m%s UNVERIFIED — no check ran, so nothing was proven.\033[0m\n' \ + "$_vr_label" + return 3 + fi + + if [ "$unverified" -gt 0 ]; then + printf ' \033[35m%s UNVERIFIED — %d check(s) could not be performed; this is not a pass.\033[0m\n' \ + "$_vr_label" "$unverified" + return 3 + fi + + printf ' \033[32m%s passed.\033[0m\n' "$_vr_label" + return 0 +} diff --git a/scripts/tests/test_verify_contract.py b/scripts/tests/test_verify_contract.py new file mode 100644 index 00000000..80782678 --- /dev/null +++ b/scripts/tests/test_verify_contract.py @@ -0,0 +1,295 @@ +"""Contract tests for the acq-kit `scripts/verify` verdict discipline. + +`integrations/isolation/acq-kits/verify-report.sh` gives converted kits a +five-state verdict (`ok`/`bad`/`unver`/`skip`/`warn`) whose exit code is a +function of a coverage record, not a failure counter — this distinguishes "we +checked and it passed" from "nothing ran" and from "we could not check this +part." These tests keep that contract in force as kits adopt it. + +CI does not run the kit verify scripts because they need a live sandbox. These +tests execute the shared reporting library in an isolated shell and statically +check that converted scripts use it. + +MIGRATION STATE: the conversion is deliberately staged. `UNCONVERTED` lists the +kits still on the old pattern. Each test asserts the contract for converted kits +AND asserts that `UNCONVERTED` still describes reality, so the allowlist cannot +rot in either direction — a kit that gets converted without being removed from +the list fails, and a listed kit that silently reverts fails too. +""" + +from __future__ import annotations + +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +ACQ_KITS = ROOT / "integrations/isolation/acq-kits" +# Lives at the acq-kits root beside validate-kits.py, NOT in a lib/ subdir: +# .gitignore carries the stock Python build-artifact block, whose `lib/` rule +# would silently exclude it from the repo. Two kits sourcing a file git never +# tracked would fail at run time with "No such file or directory". +LIB = ACQ_KITS / "verify-report.sh" + +# Kits whose verify script has NOT yet been migrated to the shared contract. +# Shrink this as they are converted; it must reach empty. +# +# This is an explicit DENY list, not a default. A kit absent from both this set +# and CONVERTED is a hard error (see test_every_kit_is_classified) — the +# alternative, treating an unclassified kit as converted by subtraction, lets a +# new kit silently inherit a contract it does not implement. +UNCONVERTED = { + "agentic-coding-playbook", + "oci-engine", + "opencode", + "openchamber", + "paseo", + "pi-coding-agent", + "zscaler-ca-certificate", +} + +# Kits that HAVE adopted the shared contract. Listing them explicitly, rather +# than deriving them by subtraction, is what makes an unclassified kit visible. +CONVERTED = { + "git-ssh-sign", + "usai-provider", +} + +# The old two-state verdict. Its absence is the thing being enforced. +LEGACY_VERDICT = re.compile(r'\[\s*"\$fail"\s+-eq\s+0\s*\]') +SHARED_REPORTER_SOURCE = re.compile(r'^\s*\.\s+"\$KIT_DIR/\.\./verify-report\.sh"\s*(?:#.*)?$', re.MULTILINE) + + +def _verify_scripts() -> dict[str, Path]: + found = {p.parent.parent.name: p for p in sorted(ACQ_KITS.glob("*/scripts/verify"))} + # Guard the guard: if the glob stops matching, every test below would pass + # vacuously over an empty set. That is the exact "no evidence read as a pass" + # defect these tests exist to prevent. + assert found, f"no scripts/verify found under {ACQ_KITS} — this suite cannot pass vacuously" + return found + + +def _converted() -> dict[str, Path]: + return {k: p for k, p in _verify_scripts().items() if k in CONVERTED} + + +def test_shared_library_exists_and_defines_the_five_states(): + assert LIB.is_file(), f"missing shared verdict library at {LIB}" + text = LIB.read_text(encoding="utf-8") + for fn in ("ok()", "bad()", "unver()", "skip()", "warn()", "verify_verdict()", "verify_reset()"): + assert fn in text, f"{LIB.name} does not define {fn}" + + +@pytest.mark.parametrize( + ("reporters", "expected"), + [ + pytest.param([], 3, id="zero-activity"), + pytest.param(["ok"], 0, id="pass"), + pytest.param(["bad"], 1, id="failure"), + pytest.param(["ok", "unver"], 3, id="pass-and-unverified"), + pytest.param(["skip"], 3, id="skip-only"), + pytest.param(["ok", "skip", "warn"], 0, id="pass-skip-and-warn"), + ], +) +def test_library_verdict_contract(reporters: list[str], expected: int): + commands = [f". {shlex.quote(str(LIB))}", "verify_reset"] + commands.extend(f'{reporter} "contract fixture"' for reporter in reporters) + commands.append('verify_verdict "Contract fixture"') + + result = subprocess.run( + ["sh", "-c", "\n".join(commands)], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == expected, result.stdout + result.stderr + + +def test_every_kit_is_classified(): + """A kit in neither set is a hard error, not a default. + + The failure message names the real problem directly: a kit appeared that + nobody classified, rather than failing a different test with a misleading + cause (e.g. "does not source lib/verify-report.sh"). + """ + scripts = _verify_scripts() + classified = CONVERTED | UNCONVERTED + + unclassified = scripts.keys() - classified + assert not unclassified, ( + f"verify script(s) in no classification set: {sorted(unclassified)}. " + "Add each to CONVERTED (if it sources verify-report.sh) or to " + "UNCONVERTED (if it is still on the old two-state verdict). A kit is " + "never converted by default." + ) + + phantom = classified - scripts.keys() + assert not phantom, f"classification sets name kits with no verify script: {sorted(phantom)} — remove them" + + overlap = CONVERTED & UNCONVERTED + assert not overlap, f"kits in both sets: {sorted(overlap)}" + + +def test_unconverted_allowlist_is_accurate(): + """The allowlist must describe reality, in both directions.""" + scripts = _verify_scripts() + + unknown = UNCONVERTED - scripts.keys() + assert not unknown, f"UNCONVERTED names kits with no verify script: {sorted(unknown)} — remove them from the list" + + for kit in sorted(UNCONVERTED): + text = scripts[kit].read_text(encoding="utf-8") + assert "verify-report.sh" not in text, ( + f"{kit} now sources the shared library but is still listed as " + "UNCONVERTED — remove it from the allowlist so it is covered by the " + "contract tests" + ) + + assert _converted(), "no kit has adopted the shared contract — these tests would pass vacuously over an empty set" + + +@pytest.mark.parametrize("kit", sorted(_converted())) +def test_converted_kit_sources_the_shared_library(kit: str): + text = _converted()[kit].read_text(encoding="utf-8") + reset = re.search(r"^\s*verify_reset\s*(?:#.*)?$", text, re.MULTILINE) + assert SHARED_REPORTER_SOURCE.search(text), f'{kit}/scripts/verify does not source "$KIT_DIR/../verify-report.sh"' + assert reset, f"{kit}/scripts/verify sources the library but never calls verify_reset, so its counters are unset" + + +def test_shared_library_source_must_be_the_command_operand(): + text = ". other.sh # verify-report.sh\n" + assert not SHARED_REPORTER_SOURCE.search(text) + + +@pytest.mark.parametrize("kit", sorted(_converted())) +def test_missing_sbx_is_unverified(kit: str, tmp_path: Path): + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + for command in ("dirname", "mktemp", "rm"): + executable = shutil.which(command) + assert executable, f"test prerequisite missing: {command}" + (bin_dir / command).symlink_to(executable) + + bash = shutil.which("bash") + assert bash, "test prerequisite missing: bash" + result = subprocess.run( + [bash, str(_converted()[kit])], + env={"PATH": str(bin_dir)}, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 3, result.stdout + result.stderr + assert "UNVERIFIED" in result.stdout + assert "sbx not on PATH" in result.stdout + + +@pytest.mark.parametrize("kit", sorted(_converted())) +def test_converted_kit_has_no_legacy_two_state_verdict(kit: str): + text = _converted()[kit].read_text(encoding="utf-8") + hits = LEGACY_VERDICT.findall(text) + assert not hits, ( + f'{kit}/scripts/verify still gates on [ "$fail" -eq 0 ] ({len(hits)} ' + "occurrence(s)). That verdict reports success when zero checks ran and " + "when checks were skipped; use verify_verdict instead" + ) + + +@pytest.mark.parametrize("kit", sorted(_converted())) +def test_converted_kit_does_not_shadow_the_shared_reporters(kit: str): + """Shared reporter functions must remain the active implementations. + + Local `ok()`/`skip()` definitions bypass the shared coverage counters and + invalidate the verdict contract. + """ + text = _converted()[kit].read_text(encoding="utf-8") + for fn in ("ok", "bad", "unver", "skip", "warn", "info"): + shadow = re.compile(rf"^\s*{fn}\s*\(\)\s*\{{", re.MULTILINE) + assert not shadow.search(text), f"{kit}/scripts/verify redefines {fn}() locally, shadowing the shared contract" + + +@pytest.mark.parametrize("kit", sorted(_converted())) +def test_converted_kit_reports_every_gate(kit: str): + """Each `info "Summary"`-style gate must end in a verify_verdict call. + + openchamber, paseo and pi-coding-agent each have three gates; a conversion + that replaced only the first would leave the others reporting nothing. + """ + text = _converted()[kit].read_text(encoding="utf-8") + verdicts = re.findall(r"^\s*verify_verdict(?:\s|$)", text, re.MULTILINE) + assert verdicts, f"{kit}/scripts/verify never calls verify_verdict" + + # A gate that prints a summary header but reaches no verdict is a gate whose + # result is discarded. + headers = len(re.findall(r'info\s+"Summary"', text)) + assert headers == 0, ( + f'{kit}/scripts/verify still prints its own info "Summary" header ' + f"({headers}); verify_verdict prints the summary itself, so a hand-rolled " + "header means a gate bypassing the contract" + ) + + +def test_no_silent_skip_remains_in_converted_kits(): + """Skip and unverified outcomes must update the coverage record. + + Bare `echo " SKIP..."` and `NOTE:` messages bypass the shared counters; + converted kits use `skip` or `unver` instead. + """ + offenders: list[str] = [] + for kit, path in sorted(_converted().items()): + for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + if re.match(r'\s*echo\s+"\s*(SKIP|NOTE)', line): + offenders.append(f"{kit}/scripts/verify:{lineno}: {line.strip()}") + assert not offenders, ( + "converted kits still report skips via bare echo, which increments no counter:\n " + "\n ".join(offenders) + ) + + +def _readme_classification(label: str, *, required: bool) -> set[str]: + """Kits the acq-kits README lists for one side of the staged migration. + + Scans the whole paragraph after the heading, not just its first sentence, so + a later claim in the same paragraph cannot contradict the list unnoticed. + + `required=False` treats a missing section as the empty set. That is the + documented end state for the unconverted side: when migration completes the + README drops that carve-out, and an absent section then means "no kits", + not "the README is malformed". + """ + text = (ACQ_KITS / "README.md").read_text(encoding="utf-8") + match = re.search(rf"\*\*{label} kits\*\* — (.*?)(?:\n\n|\Z)", text, re.S) + if match is None: + assert not required, f"acq-kits/README.md has no '**{label} kits** — ...' paragraph" + return set() + # Kit names are the backticked tokens that name a real verify script; other + # backticked tokens in the paragraph (exit codes, file names) are prose. + scripts = _verify_scripts() + return {token for token in re.findall(r"`([^`]+)`", match.group(1)) if token in scripts} + + +@pytest.mark.parametrize( + ("label", "expected", "required"), + [("Converted", CONVERTED, True), ("Unconverted", UNCONVERTED, False)], +) +def test_readme_classification_matches_this_suite(label: str, expected: set[str], required: bool): + """The README's per-kit split must match the authoritative classification. + + The README documents exit codes `0`/`1`/`3` as the verdict contract, and + that contract only holds for converted kits. A README naming the wrong kits + tells a caller to trust a `0` from a kit whose `0` does not distinguish + "every check passed" from "a check could not run". Deriving both lists from + the README paragraph fails a README-only edit and a suite-only edit alike, + so the two cannot drift as kits migrate. + """ + documented = _readme_classification(label, required=required) + assert documented == expected, ( + f"acq-kits/README.md lists {label.lower()} kits {sorted(documented)} but this suite " + f"classifies {sorted(expected)}. Update the README section when a kit migrates: " + f"README-only={sorted(documented - expected)}, suite-only={sorted(expected - documented)}" + )