fix(acq-kits): give verify scripts a "could not check" verdict state

integrations/isolation/acq-kits/README.md

@@ -96,3 +96,46 @@ make test-kits
 # Live, per-backend end-to-end check (needs a backend CLI + a sandbox-capable host):
 <kit>/scripts/verify
 ```
+
+### What a `scripts/verify` exit code means
+
+The shared verdict contract lives in `verify-report.sh` in this directory, and
+**adoption is staged**: only the kits listed as converted below implement it
+today. A verdict is a function of a coverage record, not a failure counter, so a
+converted run that exercised nothing cannot report the same result as a run that
+passed.
+
+**Converted kits** — `git-ssh-sign`, `usai-provider`. For these, and only these,
+the exit codes and reporters below apply.
+
+| Exit | Meaning |
+|---|---|
+| `0` | every check that ran passed, and at least one check ran |
+| `1` | at least one check FAILED |
+| `3` | nothing failed, but the run is **not** a pass: a check could not be performed, or no check ran at all |
+
+`3` is distinct from `1` so a caller can tell *"we looked and found problems"*
+from *"we could not look"*. Treat any non-zero exit as "do not ship".
+
+The five reporters: `ok` / `bad` are the ran-and-passed / ran-and-failed cases;
+`unver` is **could not check** (missing tool, absent credential, no network) and
+degrades the verdict to `3`; `skip` is *not applicable in this configuration*
+(e.g. a backend-specific check on the other backend) and does **not** degrade
+it; `warn` is advisory.
+
+**Unconverted kits** — `agentic-coding-playbook`, `oci-engine`, `opencode`,
+`openchamber`, `paseo`, `pi-coding-agent`, `zscaler-ca-certificate`. These still
+use the older two-state verdict, so nothing above applies to them. A non-zero
+exit means a check failed *or* a precondition could not be met — the two are not
+distinguished — and a `0` does **not** establish that every check ran: these
+scripts report an unmet precondition with a bare message that increments no
+counter, then exit `0` on the strength of the checks that did run. Treat a `0`
+here as "nothing that ran failed", never as full coverage.
+
+The authoritative classification is `scripts/tests/test_verify_contract.py`. It
+executes the verdict contract for converted kits, asserts each unconverted kit
+still matches its description, and compares both lists above against its own, so
+the split cannot rot in either direction: converting a kit without moving it
+fails, a converted kit reverting fails, and a README edit alone fails. The
+unconverted list must reach empty, at which point this carve-out goes away and
+the contract above applies to every kit.

integrations/isolation/acq-kits/git-ssh-sign/scripts/verify

@@ -31,11 +31,11 @@ WORK="$(mktemp -d "${TMPDIR:-/tmp}/git-ssh-sign-verify.XXXXXX")"
 CREATE_LOG="$WORK/sbx-create.log"
 KEEP="${KEEP:-0}"
 
-pass=0
-fail=0
-ok()   { printf '  \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); }
-bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
-info() { printf '\n\033[1m%s\033[0m\n' "$1"; }
+# Shared verdict contract (ok / bad / unver / skip / warn + verify_verdict).
+# Sourced rather than redefined here so a future fix to the contract cannot
+# leave this kit's copy behind.
+. "$KIT_DIR/../verify-report.sh"
+verify_reset
 
 cleanup() {
   if [ "$KEEP" = "1" ]; then
@@ -51,16 +51,16 @@ in_sbx() { sbx exec "$SBX_NAME" -- sh -c "$1" </dev/null 2>/dev/null || true; }
 
 info "0. Preconditions"
 if ! command -v sbx >/dev/null 2>&1; then
-  echo "  sbx not on PATH. Install sbx >= 0.34.0 and retry." >&2
-  exit 1
+  unver "sbx not on PATH — install sbx >= 0.34.0 and retry"
+  verify_verdict "Summary"
+  exit $?
 fi
 echo "  sbx version: $(sbx version 2>/dev/null | grep -oE 'v?[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n1)"
 [ -f "$KIT_DIR/spec.yaml" ] && ok "kit spec.yaml present" || { bad "spec.yaml missing"; exit 1; }
 if ssh-add -L >/dev/null 2>&1; then
   ok "host SSH agent has a key loaded (signing can succeed)"
 else
-  echo "  NOTE: no key in the host SSH agent. The kit installs fine, but signing" >&2
-  echo "        will fail until you run: ssh-add ~/.ssh/id_ed25519" >&2
+  unver "host SSH agent has no key loaded — signing cannot be exercised (run: ssh-add ~/.ssh/id_ed25519)"
 fi
 
 info "1. sbx kit validate"
@@ -102,12 +102,10 @@ if [ "$(in_sbx 'ssh-add -L >/dev/null 2>&1 && echo yes')" = "yes" ]; then
   ')
   [ "$out" = "SIGNED" ] && ok "commit was signed and verified" || bad "commit not signed (got: ${out:-none})"
 else
-  echo "  SKIP: no key in the forwarded SSH agent inside the sandbox." >&2
-  echo "        Load one on the host (ssh-add) and re-run for the full check." >&2
+  # This is the kit's whole purpose. Reporting it as anything other than
+  # UNVERIFIED would let the script claim the signing path works when it was
+  # never exercised.
+  unver "end-to-end signed commit — no key in the forwarded SSH agent (load one with ssh-add, then re-run)"
 fi
 
-info "Summary"
-printf '  %d passed, %d failed\n' "$pass" "$fail"
-[ "$fail" -eq 0 ] && printf '  \033[32mAll checks passed.\033[0m\n' || printf '  \033[31mSome checks failed — see notes above.\033[0m\n'
-
-[ "$fail" -eq 0 ]
+verify_verdict "Summary"

integrations/isolation/acq-kits/usai-provider/scripts/verify

@@ -39,11 +39,11 @@ CREATE_LOG="$WORK/sbx-create.log"
 KEEP="${KEEP:-0}"
 USAI_MODELS_URL="https://api.gsa.usai.gov/api/v1/models"
 
-pass=0
-fail=0
-ok()   { printf '  \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); }
-bad()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
-info() { printf '\n\033[1m%s\033[0m\n' "$1"; }
+# Shared verdict contract (ok / bad / unver / skip / warn + verify_verdict).
+# Sourced rather than redefined here so a future fix to the contract cannot
+# leave this kit's copy behind.
+. "$KIT_DIR/../verify-report.sh"
+verify_reset
 
 cleanup() {
   if [ "$KEEP" = "1" ]; then
@@ -59,16 +59,16 @@ in_sbx() { sbx exec "$SBX_NAME" -- sh -c "$1" </dev/null 2>/dev/null || true; }
 
 info "0. Preconditions"
 if ! command -v sbx >/dev/null 2>&1; then
-  echo "  sbx not on PATH. Install sbx >= 0.34.0 and retry." >&2
-  exit 1
+  unver "sbx not on PATH — install sbx >= 0.34.0 and retry"
+  verify_verdict "Summary"
+  exit $?
 fi
 echo "  sbx version: $(sbx version 2>/dev/null | grep -oE 'v?[0-9]+\.[0-9]+(\.[0-9]+)?' | head -n1)"
 [ -f "$KIT_DIR/spec.yaml" ] && ok "kit spec.yaml present" || { bad "spec.yaml missing"; exit 1; }
 if sbx secret ls 2>/dev/null | grep -q 'api.gsa.usai.gov'; then
   ok "USAi custom secret is set"
 else
-  echo "  NOTE: no USAi custom secret found. Set it with:" >&2
-  echo "        sbx secret set-custom -g --host api.gsa.usai.gov --env USAI_API_KEY" >&2
+  unver "USAi custom secret is not set — the credential path cannot be exercised (sbx secret set-custom -g --host api.gsa.usai.gov --env USAI_API_KEY)"
 fi
 
 info "1. sbx kit validate"
@@ -121,14 +121,12 @@ echo "  HTTP status: ${status:-<none>}"
 if [ "$status" = "200" ]; then
   ok "USAi models API returned 200"
 elif [ -z "$status" ]; then
-  echo "  NOTE: could not probe (no curl, or no network); skipping." >&2
+  # in_sbx swallows the exit status, so an empty result means either "no curl in
+  # the guest" or "sbx exec failed" — both are "could not look", never a pass.
+  unver "USAi models API — could not probe (no curl in the guest, or no network)"
 else
   bad "USAi API returned $status (key missing/expired, or egress blocked)"
   echo "  Check: sbx policy log $SBX_NAME (api.gsa.usai.gov), and the USAi secret." >&2
 fi
 
-info "Summary"
-printf '  %d passed, %d failed\n' "$pass" "$fail"
-[ "$fail" -eq 0 ] && printf '  \033[32mAll checks passed.\033[0m\n' || printf '  \033[31mSome checks failed — see notes above.\033[0m\n'
-
-[ "$fail" -eq 0 ]
+verify_verdict "Summary"

integrations/isolation/acq-kits/verify-report.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+#
+# verify-report.sh — the shared verdict contract for acq-kit `scripts/verify`.
+#
+# A verdict is a function of a COVERAGE RECORD, not a failure counter: a run
+# that checked nothing, and a run where every check passed, must not produce
+# the same result. This library supplies that record and the five states a
+# verify script needs to express it.
+#
+# STATES
+#
+#   ok    "<what passed>"        a check ran and passed
+#   bad   "<what failed>"        a check ran and failed
+#   unver "<what, and why not>"  a check COULD NOT RUN — missing tool, absent
+#                                credential, no network. Not a pass. Counted,
+#                                and it degrades the verdict.
+#   skip  "<what, and why>"      deliberately not applicable in this
+#                                configuration (e.g. a backend-specific check on
+#                                the other backend). Counted and reported, but
+#                                does NOT degrade the verdict.
+#   warn  "<note>"               advisory only; a documented fallback was taken.
+#                                Counted and reported; does not degrade.
+#
+# `unver` degrades the verdict because a check could not run. `skip` records a
+# check that does not apply to the current configuration and does not degrade it.
+#
+# EXIT CODES (verify_verdict)
+#
+#   0  every check that ran passed, and at least one check ran
+#   1  at least one check FAILED
+#   3  nothing failed, but the run is not a clean pass:
+#        - at least one check could not be performed (unverified > 0), or
+#        - no check ran at all (pass == 0)
+#
+# 3 is separate from 1 so a caller can tell "we looked and found problems" from
+# "we could not look". A caller that only cares about red/green can test for
+# non-zero and will correctly refuse to treat a degraded run as success.
+#
+# USAGE
+#
+#   KIT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+#   . "$KIT_DIR/../verify-report.sh"
+#   verify_reset                      # initialize the counters
+#   info "1. Something"
+#   ok "it worked"
+#   unver "API probe — no network"
+#   verify_verdict "Offline gate"     # prints the summary, returns 0/1/3
+#
+# `verify_verdict` takes an optional label so a script with more than one gate
+# (openchamber, paseo and pi-coding-agent each have three) can name each one.
+#
+# POSIX sh compatible on purpose: these scripts run on contributor machines with
+# whatever /bin/sh is, and the arithmetic and parameter expansion here stay
+# within that subset.
+
+# Counters. Deliberately not `local` — they are the script-wide coverage record.
+verify_reset() {
+  pass=0
+  fail=0
+  unverified=0
+  skipped=0
+  warned=0
+}
+
+ok() {
+  printf '  \033[32mPASS\033[0m %s\n' "$1"
+  pass=$((pass + 1))
+}
+
+bad() {
+  printf '  \033[31mFAIL\033[0m %s\n' "$1"
+  fail=$((fail + 1))
+}
+
+# Could not check. The message SHOULD say what was not checked AND why, because
+# the reader's next question is always "why not".
+unver() {
+  printf '  \033[35mUNVERIFIED\033[0m %s\n' "$1"
+  unverified=$((unverified + 1))
+}
+
+# Not applicable in this configuration. Does not degrade the verdict.
+skip() {
+  printf '  \033[33mSKIP\033[0m %s\n' "$1"
+  skipped=$((skipped + 1))
+}
+
+# Advisory note; a documented fallback was taken. Does not degrade the verdict.
+warn() {
+  printf '  \033[33mWARN\033[0m %s\n' "$1"
+  warned=$((warned + 1))
+}
+
+info() {
+  printf '\n\033[1m%s\033[0m\n' "$1"
+}
+
+# Print the coverage record and return the verdict.
+#
+# $1 — optional gate label (default "Summary").
+verify_verdict() {
+  _vr_label="${1:-Summary}"
+
+  info "$_vr_label"
+  printf '  %d passed, %d failed, %d unverified, %d skipped, %d warned\n' \
+    "$pass" "$fail" "$unverified" "$skipped" "$warned"
+
+  if [ "$fail" -gt 0 ]; then
+    printf '  \033[31m%s FAILED — see above.\033[0m\n' "$_vr_label"
+    return 1
+  fi
+
+  # Nothing ran. Arithmetic over zero must never produce a pass: pass=0 and
+  # fail=0 is a report of nothing proven, not a report of success.
+  if [ "$pass" -eq 0 ]; then
+    printf '  \033[35m%s UNVERIFIED — no check ran, so nothing was proven.\033[0m\n' \
+      "$_vr_label"
+    return 3
+  fi
+
+  if [ "$unverified" -gt 0 ]; then
+    printf '  \033[35m%s UNVERIFIED — %d check(s) could not be performed; this is not a pass.\033[0m\n' \
+      "$_vr_label" "$unverified"
+    return 3
+  fi
+
+  printf '  \033[32m%s passed.\033[0m\n' "$_vr_label"
+  return 0
+}