From 64065049f39f45cdd589a7b41a869abc2ec3f17b Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Mon, 19 Oct 2026 12:12:55 +0800 Subject: [PATCH 1/2] feat: deliver prepared shared data to Before/After snapshots Signed-off-by: Kent Huang --- .github/workflows/ci.yml | 4 + AGENTS.md | 1 + CODING_GUIDELINES.md | 2 + README.md | 36 +- docs/architecture.md | 72 +- ...ntion-driven-preparation-implementation.md | 562 ++++++++ plugin/skills/behavior-diff/SKILL.md | 39 +- .../references/scenario-and-test-data.md | 219 +++ .../behavior-diff/scripts/prepare_shared.py | 470 +++++++ tests/live-report-contract.sh | 4 - tests/shared-data-test.py | 1186 +++++++++++++++++ 11 files changed, 2577 insertions(+), 18 deletions(-) create mode 100644 plans/2026-10-19-intention-driven-preparation-implementation.md create mode 100644 plugin/skills/behavior-diff/references/scenario-and-test-data.md create mode 100755 plugin/skills/behavior-diff/scripts/prepare_shared.py create mode 100755 tests/shared-data-test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bb0c71..97d9e31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,6 +86,7 @@ jobs: plugin/skills/behavior-diff/scripts/status.py \ plugin/skills/behavior-diff/scripts/source.py \ plugin/skills/behavior-diff/scripts/ci_context.py \ + plugin/skills/behavior-diff/scripts/prepare_shared.py \ plugin/skills/behavior-diff/scripts/reporting/*.py - name: Run hook unit checks @@ -103,6 +104,9 @@ jobs: python3 tests/ci-context-test.py python3 tests/report-ci-context-test.py + - name: Check shared-data preparation boundaries + run: python3 tests/shared-data-test.py + - name: Run decision unit check run: python3 plugin/skills/behavior-diff/scripts/decisions.py --check diff --git a/AGENTS.md b/AGENTS.md index a596ad8..379f941 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,6 +89,7 @@ For the full deterministic suite, run: ```bash bash tests/hooks-test.sh python3 plugin/skills/behavior-diff/scripts/decisions.py --check +python3 tests/shared-data-test.py bash tests/live-report-contract.sh python3 tests/source-provenance-test.py python3 tests/report-source-test.py diff --git a/CODING_GUIDELINES.md b/CODING_GUIDELINES.md index 54eb0c4..eb050de 100644 --- a/CODING_GUIDELINES.md +++ b/CODING_GUIDELINES.md @@ -190,10 +190,12 @@ python3 -m py_compile \ .agents/skills/release-behavior-diff/scripts/bump-version.py \ plugin/skills/behavior-diff/scripts/codex_model.py \ plugin/skills/behavior-diff/scripts/decisions.py \ + plugin/skills/behavior-diff/scripts/prepare_shared.py \ plugin/skills/behavior-diff/scripts/render.py \ plugin/skills/behavior-diff/scripts/reporting/*.py bash tests/hooks-test.sh python3 plugin/skills/behavior-diff/scripts/decisions.py --check +python3 tests/shared-data-test.py bash tests/live-report-contract.sh bash tests/release-workflow-test.sh python3 .agents/skills/release-behavior-diff/scripts/bump-version.py --check diff --git a/README.md b/README.md index 1b2afa4..5a14b29 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,34 @@ In Claude Code, you can also run: ``` Behavior Diff finds the changed instruction file and uses your request as the -comparison task. Once the task is known, it runs the comparison and opens the -report. +comparison task. Once the task is known, and any shared trial data is verified, +it runs the comparison and opens the report. + +### Shared trial data + +Some tasks need files that your project does not contain, such as sample +inputs. Git-based trials start from your last commit, so untracked or +uncommitted files in your checkout never reach them. Behavior Diff does not +generate such data. When you supply or approve the files, the skill can deliver +them to every trial: + +1. A private helper, `prepare_shared.py`, copies your project base, the reviewed + data directory, and the selected Before and After instructions into a new + private stage outside your project. Your checkout is not changed. +2. The helper records a freeze of the stage. It checks the freeze again + immediately before launch, and it stops when a file changed. +3. The skill runs the normal runner from the stage. Both sides receive the same + data, and only the selected instruction differs. + +The helper accepts one instruction file and plain files only. It rejects +symbolic links, special files, submodules, unsafe paths, inputs over +`BEHAVIOR_DIFF_COPY_CAP_KB` (default 200 MiB), and Git attributes that change +archived files. Purpose files, source metadata, and evaluator notes stay out of +the stage. The helper does not scan data for secrets, so review every shared +file first. The freeze detects changes, but it is not a sandbox. Reports for a +staged run stay in the stage under `home/runs/`; keep the stage after the run. +The [skill reference](plugin/skills/behavior-diff/references/scenario-and-test-data.md) +contains the complete commands and limits. ### Trial and extraction models @@ -438,7 +464,8 @@ change. Behavior Diff runs each side in a separate copy of the project. The after side contains only the instruction change that you selected. Other uncommitted files -do not enter the experiment. +do not enter the experiment. Reviewed shared trial data, when used, enters both +sides identically. Reports stay on your machine under: @@ -446,6 +473,9 @@ Reports stay on your machine under: ${BEHAVIOR_DIFF_HOME:-~/.behavior-diff}/runs/ ``` +A run with [shared trial data](#shared-trial-data) keeps its reports in its +private stage under `home/runs/`. + A report can quote code and agent output from your project. Review the report before you share it. diff --git a/docs/architecture.md b/docs/architecture.md index 828770d..38aa7fb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -23,6 +23,14 @@ scripts handle execution, evidence, and reporting. +----------------------------+----------------------------+ v +---------------------------------------------------------+ +| Conditional shared-data delivery | +| Only when the task needs files the project base lacks: | +| prepare_shared.py freezes project, Before, and home; | +| verify passes immediately before launch | +| Otherwise the runner starts from the user's project | ++----------------------------+----------------------------+ + v ++---------------------------------------------------------+ | Trial execution | | behavior-diff.sh builds independent Before/After copies | | run-trial.sh runs the same task/model in each copy | @@ -111,7 +119,11 @@ remain the only source of that guidance. The skill selects the trial host and model, calls the runner, and explains the result. It owns scenario quality and interpretation, not filesystem copying or -report formatting. +report formatting. When the task needs common files that the project base +lacks, the skill reviews files that the user supplied or approved and delivers +them through a frozen stage (see +[Shared-data delivery](#shared-data-delivery)). The skill does not generate the +scenario or the data. Before execution, the skill records distinct concise goals and their provenance (`session`, `commit`, or `diff`; `explicit` or `inferred`) in a reviewed purpose @@ -260,7 +272,58 @@ The configuration records the target path, scenario, Before/After labels, and validated `purpose`. `--purpose-file` is frozen before trials and excluded from trial copies, including when supplied from within the source project. Uncommitted setup files do not enter Git-based trials. Required task state must -already exist in the base project or in a deliberately prepared fixture. +already exist in the base project, in a deliberately prepared fixture, or in a +shared-data stage. + +### Shared-data delivery + +The private helper +[`prepare_shared.py`](../plugin/skills/behavior-diff/scripts/prepare_shared.py) +delivers a reviewed data directory to every trial without changing the user's +checkout or adding runner options. It does not generate data, launch models, +or infer provenance. + +```text +Project base (Git HEAD or folder copy) --+ +Reviewed data directory -----------------+--> stage/project: scratch commit +Selected Before (HEAD, file, or absent) -+ + After in the working tree + minus --exclude paths stage/before: present Before + stage/home: ABSENT marker + stage/freeze.json + | + verify, then behavior-diff.sh from stage/project + with BEHAVIOR_DIFF_HOME=stage/home + and --before-file stage/before when present +``` + +`prepare` resolves Before in the runner's order: an explicit file, the target +in Git HEAD, then the newest hook baseline. Only a store entry can mean +`ABSENT`. It copies the base, the data, and the selected Before into a new +scratch Git repository and commits them. The repository has no remote, and its +hooks and signing are off. The helper then writes After into the working tree. +A present Before, from HEAD or a file, is also frozen as the stage's `before` +file outside the scratch project. The skill passes that file to the runner +with `--before-file`, so the report labels Before with the frozen file rather +than a scratch commit. An absent Before has no such file; it receives a marker +in the stage's `home/baselines/`, and the runner resolves it there. The +existing runner then builds every trial from the scratch commit, unchanged. + +The helper rejects links, special files, Git internals, submodules, unsafe or +colliding paths, data that replaces the instruction or an excluded path, and +inputs over `BEHAVIOR_DIFF_COPY_CAP_KB`. The caller excludes in-project purpose, +source, and evaluator files with `--exclude`. `prepare` also requires +`git archive` of the scratch commit to match the expected files. Therefore, +`.gitattributes` rules such as `export-ignore`, `export-subst`, or filters that +change archived files fail before launch. + +`freeze.json` records the original project and commit, the target, the Before +mode, the scratch commit, exclusions, and the content and executable bit of +every archived and working file. `verify` compares the stage, including the +frozen Before file or absent marker, with this record immediately before +launch. The freeze detects changes. It is not a sandbox, and the stage files +are not read-only. Source provenance still names the original repository and +revisions, never scratch commits. Runs and reports stay under the stage's +`home/runs/`. ### Trial execution sequence @@ -742,6 +805,11 @@ State lives under `${BEHAVIOR_DIFF_HOME:-~/.behavior-diff}/`: - `human-evaluations/`: private source snapshots, frozen scenarios/questions, original reports, quiz projections, and human submissions for manual evaluations. +A shared-data run uses its own stage instead: `project/` (the scratch base), +`before` (a present Before version), `home/` (its `BEHAVIOR_DIFF_HOME`, with +any `ABSENT` marker and its runs), and `freeze.json`. The stage stays after the +run because it holds the reports. + The runner prepares copies without changing the source project. Separate working directories are not a universal security sandbox. Tool restrictions depend on the host, and permitted shell commands can have side effects. diff --git a/plans/2026-10-19-intention-driven-preparation-implementation.md b/plans/2026-10-19-intention-driven-preparation-implementation.md new file mode 100644 index 0000000..0ae3481 --- /dev/null +++ b/plans/2026-10-19-intention-driven-preparation-implementation.md @@ -0,0 +1,562 @@ +# Implement intention-driven scenario and test-data preparation + +Date: 2026-10-19 +Status: PR 1 implemented and verified locally; draft PR delivery in progress. +Next delivery: finish PR 1 draft review, then implement PR 2's intention-driven +preparation. Report assessment/UI work remains deferred until both are complete +and verified. Do not repeat the public-repository survey or retune old cases. + +## 1. User request and product decision + +Bring the scenario and test-data generation method exercised in our experiments +into the installable Behavior Diff skill. + +> We identify the intent behind a change, then use Behavior Diff to check whether +> the agent's actual before-and-after behavior matches that intent. + +The target is **Validate change assumptions**, not reliable detection of every +unknown side effect. Include both behavior intended to change and behavior meant +to survive the edit. A new instruction can fail to achieve its purpose even if +Before and After make the same mistake; do not require a behavioral divergence +before acknowledging an unmet After requirement. + +The user does not want to write a full evaluation dataset or specify the failure +in advance. The skill should derive a natural scenario and suitable data from +available contemporary context. Multiple relevant edge cases may share one +coherent scenario. This is bounded intent coverage, not exhaustive test generation. + +### Already decided + +- Preserve the tested generation recipe below; do not invent a new method while + implementing it. +- Design scenario AND data. A good task with inadequate data may not exercise + the intended capability. +- Use the user's execution model/configuration for trials. Do not silently + substitute a stronger model, and keep preparation/extraction roles distinct. +- Freeze inputs before executing; preserve negative and inconclusive results. +- Keep normal trial counts and host parity. Do not multiply trials by edge count. +- A reproduced gap and an automatically reported gap are different milestones. +- Private notes/raw evidence belong outside tracked files (`memo/` is ignored). + Do not copy private historical skill text or session data into public fixtures. +- Deliver preparation in two independently reviewable PRs, as specified in + section 9. Complete and verify the scenario/test-data workflow before reopening + report design. +- Keep the current automatic report during those PRs. The skill's evidence-based + explanation must disclose coverage and reporting gaps; it must not rewrite + saved report artifacts to make them agree with a raw-evidence audit. + +### Explicitly not established + +- Workflow skills are a promising evaluation focus, not proven easier to test. +- More reasoning or more edge cases is not proven to improve detection. +- The proposed main-goal Before/After classifier and automatic reasoning-level + escalation are **not part of this implementation**. Preservation goals already + belong to the tested recipe; no new classifier is needed to include them. +- A later corrective PR is selection/validation evidence, never an answer key + supplied to historical preparation. + +## 2. Evidence and reading map + +Read these existing artifacts rather than reconstructing the discussion: + +1. [Experimental plan](2026-10-15-validate-change-assumptions.md), especially + **Current generation recipe**, **Select edge cases by intention coverage**, + **Explicit environment preconditions**, and **Evidence limits**. +2. Its experiment-log sections on changelog verification, historical-model + comparison, and **Public workflow replay: EveryInc lfg PR #1702**. +3. [Architecture](../docs/architecture.md), sections 2–3: ownership, snapshot + construction, purpose isolation, and trial/assessment/report distinction. +4. [Public skill inventory](2026-10-13-public-skill-change-stats.md) and + [historical replay workflow](../docs/skill-history-replay/README.md) only when + selecting a new validation case. Existing historical workflow constraints do + not automatically authorize multi-file production support. +5. Repository coding, issue and reviewer guidelines referenced by AGENTS.md. + +Local-only supplemental records, if present: `memo/public-workflow-skill-survey.md` +and `memo/everyinc-lfg-1702-experiment.md`. They are not required for implementing +this plan and will not exist in a fresh public clone. Never feed their historical +outcomes into a supposedly blind preparation session. + +### What the evidence actually supports + +- Changelog experiment: some After configurations read descriptions but did not + perform newly required source verification. The gap was established from raw + actions/artifacts, not the automatic report. Before was not subject to that new + requirement. Another model succeeded on both sides. +- Review-output experiment: one tested model produced identical detailed and + summary outputs in an After trial; a different model did not reproduce it. + Reading a reference was not proof of applying it. +- [EveryInc #1702](https://github.com/EveryInc/compound-engineering-plugin/pull/1702): + isolated preparation generated a bug-fix task with project-state edges. One + Fable 5.1 low After trial ended normally after the child debugging result, + skipping required review continuation. Five other trials hit provider limits; + no complete paired completion-rate comparison exists. Extraction failed. + The later [#1715](https://github.com/EveryInc/compound-engineering-plugin/pull/1715) + describes the same symptom but was not tested as a fix. +- That replay used private multi-file/runtime adapters, not the shipped single- + file CLI. Its driver was edited after launch and its original bytes were not + retained. Do not copy it into the product or treat it as a reference-quality + harness. Source/plugin equality and the normal terminal event supported the + bounded observation, not end-to-end reproducibility of the orchestration. + +These results motivate shipping the preparation discipline. They do not establish +reliable automatic detection or a general model ranking. + +## 3. Current implementation and the actual gap + +Canonical source: `plugin/skills/behavior-diff/SKILL.md` (not the installed copy +in a developer's home directory). One product serves Claude Code and Codex. + +At the time of this handoff: + +| Surface | Current behavior | Required change or constraint | +|---|---|---| +| Skill Ownership + step 1 | Derives and freezes purpose; already includes preservation goals | Retain purpose format/privacy; connect goals to observable capabilities | +| Skill step 2 | Drafts a decision-moment task; asks for a real incident; may start mid-task | Replace with scenario AND data preparation; no mandatory incident question and no pre-completing discovery | +| Skill step 3 | Runs as soon as task is known | Run only after task, data, environment and assessment notes are checked/frozen | +| Skill step 4 | Explains observed flow and unchanged behavior | Check outcome coverage and actual evidence, including unmet After intent without divergence | +| `scripts/behavior-diff.sh` | Git snapshots use `git archive HEAD`; only selected instruction edit enters After | Newly generated untracked data in the user's checkout will NOT enter Git trials | +| `scripts/run-trial.sh` | Host execution and trace normalization | Do not infer model effort from a model name; do not advertise unsupported execution capabilities | +| `scripts/purpose.py` + runner | Purpose is assessment-only, max eight goals | Do not repurpose this schema as a fixture or expected-answer transport | +| `scripts/decisions.py`, `scripts/reporting/evidence.py` | Separate extraction with bounded evidence | No report schema/extractor redesign in this task; declare when raw audit and report disagree | + +The largest implementation trap is a Markdown-only promise to generate data +without arranging for that data to exist identically in every trial. A second +trap is giving the trial a prepared findings list when independent discovery is +what the edit is meant to improve. + +## 4. Required preparation logic + +### A. Derive and freeze intent + +Use the selected instruction diff, relevant unchanged instructions/references, +and available user/PR/commit context. Separate explicit goals from inferred ones. +Retain the existing purpose-file contract and privacy rules. + +For a local edit, do not insist on a PR description that does not exist. Use the +current conversation and diff; do not mine unrelated transcripts or memory. +For historical edits, use an isolated preparation context if the orchestrator +knows later history. Supply only the selected contemporaneous source bundle and +this generic method. Historical selection bias must still be disclosed. + +Do not ask the user to name the failure or supply a full eval. Reuse a real +request if already supplied. Ask only when a material missing requirement cannot +be established from available context and changes what the experiment means. + +### B. Map goals to observable capabilities + +For each chosen outcome, record privately: + +- Source/basis of the intent, including relevant preservation requirements. +- What the agent must discover, decide, execute or produce. +- Necessary environment premises and where the trial can observe them. +- Evidence for support, contradiction, or insufficient evidence. +- Scope not exercised by this experiment. + +A skill's main goal may inform preservation checks. Do not force it into a new +before/after goal classification API or adaptive reasoning policy. + +### C. Generate one natural scenario + +Define the user task, starting state and scope. Do not tell the trial the expected +answer or suspected historical mistake. A normal request may identify the work, +but should not restate new instruction requirements to make both versions obey. + +Start before the capability under test. Starting mid-workflow remains valid only +when earlier work is outside that capability, supplied state is coherent, and +skipped stages are explicitly outside the conclusion. + +### D. Generate shared data + +Derive ordinary inputs plus a bounded set of edges from necessary assumptions. +Rank by relevance to intent, plausibility, assessability and additional coverage, +not by whether the author later reported that failure. + +- Multiple compatible cases can share one dataset; one case need not cover every + goal. Do not impose a quota or add duplicate cases to inflate coverage. +- Separate incompatible states rather than making a contradictory fixture. + Do not silently launch extra scenarios to accommodate every candidate. +- Include coherent underlying facts, source materials and queryable observations. + Do not perform the tested discovery in advance or plant an oracle in the task. +- A legitimate tool response may contain the answer. The objective is preserving + the required investigation, not hiding useful evidence. +- Synthetic tools must allow both correct and incorrect decisions and record + actual submissions. They must not enforce the expected outcome for the agent. +- Reuse relevant existing project data when sufficient. Generating data does not + mean replacing every real project with an unrelated toy task. +- Record cross-case cues, simplifying catalogs, and interactions. Cases combined + in one trial are not independent experiments. + +Antagonist reasoning may propose edges within this preparation step. It is not a +mandatory additional agent or separate paid review pass, and it cannot predict +which edge will become a future incident. + +### E. Preflight, then freeze + +Before paid trials, exercise the tools and premises needed to answer the task: +relevant reads/queries, artifact writes, local tests, and safe action recording. +Do not confuse an intentionally modeled limitation with an accidental harness +failure. A workflow requiring subagents or approval cannot be validated by a +host that lacks that capability without an explicitly narrower conclusion. + +When dates matter, specify a shared assessment time/timezone and coherent tool +responses. Do not patch the clock only after inspecting an unfavorable answer. +When state matters, include permissions, branch/remote status and relevant user +work. Never use real credentials or allow external publishing for fixture actions. + +Freeze task, data, instruction versions/references, tool behavior, private +criteria, execution settings and agreed trial budget. No worker may edit a live +launcher or its dependencies. Capture provenance before execution, not afterward. +Fail preflight rather than spend trials on known missing required capabilities. + +### F. Execute and assess + +Use identical tasks/data/tool access on independent Before/After copies; only the +selected instruction version changes. Keep existing default 3+3 and explicit +fast-mode behavior. More cases can increase tokens within each trial; do not +promise fixed cost simply because the trial count is unchanged. + +After execution, distinguish: + +1. Did the trial reach each selected condition? +2. Did actual behavior satisfy the corresponding intent? +3. Was the observation interpretable, or interrupted/blocked by the environment? +4. Did the automatic report identify the same finding? + +Inspect actions, returned evidence and submitted artifacts, not only narration. +A nonempty provider error is not successful completion. A normal child return is +not completion of its caller. Do not count quota/permission interruption as a +skill defect. Preserve the first results; no retuning/retries to force a finding. + +## 5. Materializing data without inventing a new execution platform + +Preferred implementation: use existing isolated scratch-repository semantics, +not a new generic tool simulator, fixture DSL, or public CLI mode. + +For a comparison needing generated shared files: + +1. Prepare a private scratch project from the selected baseline/project state. + Keep purpose, ground truth, preparation notes and source metadata OUTSIDE it. +2. Materialize the scenario's common data there. This includes any deliberately + selected contextual working state; record deviations from the user's project. + Never edit, stage, stash or commit files in the user's real checkout. +3. Make common data part of the scratch baseline that the runner archives. + Ensure the Before instruction matches the selected real baseline; then apply + only the selected After instruction in the scratch working tree. +4. Invoke the existing runner from that scratch project. Verify that generated + files are present and identical in every trial, and that only the instruction + version differs. Preserve legitimate absent-file and explicit-before semantics. +5. Source provenance must continue to name the original repository/revisions, + never scratch commits. Source metadata does not imply full-PR coverage. + +Do not assume scratch staging solves every environment. Git history, uncommitted +state, dynamic service observations, writable tool permissions, inline skill +loading and multi-file version changes need faithful capabilities. The existing +single-file runner does not automatically provide them. If a required setup is +unsupported, report that before launch; do not quietly omit it or claim full +workflow coverage. General multi-file historical replay is out of scope here. + +Prefer documented orchestration using existing supported mechanics. If reliable +shared-file staging needs code, add the smallest private helper under the canonical +skill scripts, with deterministic boundary tests. Do not add new CLI flags or +public schemas by implication; explain any unavoidable contract extension before +implementation. A recipe that never delivers its generated data is not acceptable. + +PR 1 delivers this shared-data path for an already selected task and dataset, +including its canonical skill entry point. PR 2 adds the full intent-driven +generation workflow that consumes it. Do not merge an unused helper or describe +the new generator as shipped after PR 1 alone. + +## 6. File-level implementation checklist + +This checklist describes the cumulative preparation deliverable. Section 9 +assigns work to PRs; each PR includes its own documentation and verification. + +1. **`plugin/skills/behavior-diff/SKILL.md`** + - Update Ownership and replace task-only step 2 with the preparation sequence. + - Replace the mandatory incident question with available-context derivation. + - Preserve step 1 purpose/provenance boundaries and model selection defaults. + - Change step 3's readiness condition from task-known to preparation-frozen. + - Extend step 4's evidence-based summary to selected outcomes and coverage. + - Keep the main skill readable. Put the detailed recipe in one canonical + reference such as `references/scenario-and-test-data.md`, explicitly read + before preparation. Do not maintain two divergent copies of the recipe. +2. **Preparation support, only if needed for actual staging** + - Reuse current snapshot construction and Spacedock fixture conventions where + applicable; inspect `references/spacedock-duo.md` before touching that lane. + - No special-case EveryInc adapter in the product. No unconditional model-call + additions. No generic mock toolkit disguised as a skill wording change. +3. **`docs/architecture.md`** + - Update skill responsibilities, the ASCII flowchart and scenario construction + explanation to include shared data preparation/preflight and private criteria. + - Keep script/model/report ownership accurate. +4. **`README.md`** + - Describe intent-driven task/data preparation and bounded coverage, not a + guarantee of finding side effects or validating every skill goal. +5. **Relevant deterministic tests** + - Inspect `tests/purpose-handoff-test.py`, `tests/source-provenance-test.py`, + `tests/hooks-test.sh` and `tests/live-report-contract.sh` for affected contracts. + - Extend behavior/boundary tests only where implementation changes contracts. + Do not assert exact skill wording or add tests that merely search source text. + - Do not repair wording-only tests by repinning new prose. + +This planning request does not authorize a plugin version bump, release, commit, +push, or opening PRs. Implementation proceeds in the sequence below when started. +Keep host manifests aligned if a later release changes them. + +## 7. Acceptance criteria + +| ID | Observable acceptance | +|---|---| +| AC1 | A local modification with no incident report produces a purpose, natural task and deliberate data from available intent without demanding a user-authored eval | +| AC2 | Private preparation records connect each selected case to a capability, premises, evidence criteria and omissions; multiple edges are allowed but not exhaustive | +| AC3 | Both variants actually receive the same generated data and tool observations; generated files do not disappear at git archive or leak into the user's checkout | +| AC4 | Discovery remains executable work; task/project inputs contain no private assessment oracle or later-fix answer | +| AC5 | Environmental requirements are exercised before execution; unsupported capabilities produce an explicit blocked/narrow-scope result, not a fake skill finding | +| AC6 | Dates and permissions are explicit when relevant; supplied facts and expected outcomes are internally consistent | +| AC7 | Existing trial counts, purpose/source isolation and model-role defaults remain intact on Claude Code and Codex; execution model is not silently upgraded | +| AC8 | The result distinguishes reached/missed/unknown conditions, unmet After intent, infrastructure failures and automatic-report coverage | +| AC9 | A frozen run cannot be modified by preparation workers; negative results do not trigger hidden fixture edits or reruns | +| AC10 | A smoke run proves the modified skill prepares usable inputs and the runner consumes them; Markdown edits alone are not completion | + +Do not require reproducing the EveryInc historical defect as a release gate. +Generation and execution are stochastic, and that replay was incomplete. Validate +preparation quality and observable contracts rather than guaranteeing a failure. + +AC8 is satisfied in the preparation milestone by the canonical skill's +evidence-based result explanation and private assessment notes, using the current +report as a separate artifact. It does not require a new UI, persisted coverage +schema, or extractor contract. Any automatic-report limitation remains explicit. + +## 8. Verification and handoff checkpoints + +### Deterministic checks + +Follow AGENTS.md's current verification commands. Markdown-only work requires +`git diff --check`; if scripts change, run repository formatting checks and the +relevant deterministic suites. CI must not invoke a model CLI or live runner. + +For any staging helper, test consumer-visible invariants: shared files arrive in +both variants; original checkout remains unchanged; assessment data stays out; +selected Before/After content is preserved; unsupported/path-escape inputs are +rejected. Use synthetic fixtures only. + +### Smoke proof + +- First exercise staging with disposable synthetic files and inspect actual + constructed trial inputs. Use existing deterministic harness conventions. +- Then run the modified canonical skill on a synthetic instruction edit with + clear intent, an ordinary case and compatible boundary conditions. Check that + the skill itself generated/prepared the inputs; do not handwrite a perfect + scenario and call that proof of the new generator. +- Inspect generation and execution separately. Record preparation, trial and + extraction models/settings, source access, omitted cases and actual artifacts. +- Verify the core mechanics on both supported hosts; state any unavailable live + host gate instead of claiming parity from one host's output. +- Set concrete time/cost bounds before paid verification. The user has given + standing approval for requested Behavior Diff experiments in this repository; + do not repeatedly request the same cost authorization. Sensitive-data sharing, + new external effects and substantial scope changes still require approval. +- Do not assume previous provider limits have reset; check availability before + launching. Never replace a requested model to evade a quota. If blocked, finish + deterministic work and report the missing live verification gate. + +Before proposing a PR, obtain an independent read-only review under +REVIEWER_GUIDELINES.md. Review safety, snapshot fidelity and intent leakage as +well as prose clarity. Update docs in the same change; remove obsolete task-only +instructions rather than retaining conflicting alternatives. + +### Suggested skills for the next session + +- `behavior-diff`: required for live comparisons and model/provenance boundaries. +- `design-an-interface`: if an unavoidable staging API/helper contract is needed; + compare small designs before adding a new public interface. +- `review` or `review-by-subagent`: independent review when requested/required. +- `diagnose`: investigate actual runtime failures without labeling them as skill + findings. No need to repeat successful setup or reproduce a reported quota error. + +### Final developer delivery + +List changed canonical files, what now generates/materializes the data, executed +checks and smoke evidence, remaining limits, and whether the automatic report +actually expressed the raw finding. Do not claim the main-goal escalation arm, +full multi-file replay, or a new automated verdict system was delivered. + +## 9. Ordered PR plan + +### Delivery boundaries and status + +The user deferred report UI design on 2026-10-19. The earlier layout preview is +exploratory, not an accepted design or an implementation dependency. Revisit it +only after observing results from the completed preparation workflow. + +| Stage | Deliverable | Dependency | Status | +|---|---|---|---| +| PR 1 | Deliver prepared shared data to real Before/After snapshots safely | None | Implemented; checks and independent review passed; draft PR delivery in progress; live verification pending | +| PR 2 | Generate and preflight scenario/data from intent, then execute and explain evidence | PR 1 | Planned; depends on PR 1 delivery | +| Preparation acceptance gate | Verify the integrated workflow against AC1–AC10 | PR 2 | Pending; required before declaring preparation complete | +| Report design checkpoint | Decide what actual results need the report to communicate | Preparation acceptance gate | Deferred; requires user design review | +| Later report PR(s) | Implement only the agreed assessment/presentation changes | Accepted report design | Not scoped or authorized by this plan | + +PR numbers here are sequence labels, not GitHub PR identifiers. Do not split out +tests, docs, or smoke proof into a later PR to make an incomplete feature mergeable. +The acceptance gate is verification of PR 2, not a third implementation PR. + +### PR 1 — Deliver shared scenario data safely + +**User-visible result:** for an already selected task and dataset, the canonical +skill can arrange an isolated comparison where all trials receive the intended +common files without changing the user's checkout. + +**Scope** + +- Implement the scratch-project orchestration in section 5 using existing + supported mechanics. Add a private helper only if necessary for reliable + staging; agree any unavoidable interface extension before implementing it. +- Wire the path into the canonical skill for tasks that need prepared shared + files. Put delivery instructions in the canonical scenario/test-data reference + that PR 2 will extend, not a second competing recipe. +- Preserve real Before/After instruction selection, legitimate absent-file and + explicit-before semantics, and original source provenance. Exclude purpose, + expected answers and preparation notes from the trial snapshots. +- Freeze the delivered common inputs before execution. Fail before launch when + required project state cannot be represented faithfully by the existing runner. +- Update the relevant snapshot/delivery documentation and boundary tests in this + PR. Do not advertise autonomous intent-driven generation yet. + +**Expected files:** canonical skill and preparation reference; a private script +only if needed; relevant snapshot, purpose-isolation and source-provenance tests; +the affected architecture section and README usage explanation. + +**Verification and merge gate** + +- Exercise the shipped snapshot path with disposable synthetic data and the + repository's deterministic host stubs. Inspect actual constructed trial inputs, + not only helper output. +- Establish common-file equality, correct instruction differences, exclusion of + private assessment material, original-checkout preservation and correct source + provenance. Cover applicable absent-file, explicit-before, path-escape and + unsupported-setup boundaries. +- Exercise mechanics for both Claude Code and Codex. A stubbed run proves delivery + mechanics, not that a model generates good scenarios or follows the skill. +- Run affected deterministic checks and formatting checks, and obtain an + independent read-only approval under REVIEWER_GUIDELINES.md before opening a PR. + +**Not included:** autonomous scenario generation, report changes, a new fixture +DSL, generic tool simulator, public CLI mode, or multi-file historical replay. + +**Local implementation and verification record** + +- Added private `prepare_shared.py` staging/verification and the canonical + delivery reference. Integrated them into the skill without changing the public + runner interface or implementing the PR 2 generator. +- `python3 tests/shared-data-test.py --runner-smoke -v`: 28 checks passed. + Four successful runner scenarios exercised 16 synthetic trial processes across + Claude Code and Codex, including normal counts, explicit fast mode, absent + Before and explicit Before. Inspected actual trial files and captured host + reads, private-context exclusions, source provenance and original state. +- Boundary regressions cover frozen-input changes, unsafe paths and collisions, + unsupported project state, Git environment redirection and archive transforms. + Unsafe Git overrides and archive changes were reproduced before their fixes; + guarded handoff now rejects them without launching a host. +- Required deterministic suite, Bash/Python formatting and syntax checks passed. + Two independent read-only reviewers approved after material findings were fixed. + CI runs only the helper boundary suite; the runner smoke remains manual. +- No model calls or real Recce-Team replays were run. This establishes PR 1's + delivery mechanics, not automatic scenario quality or detection of invalidated + change assumptions. PR 2 and its acceptance gate remain pending. + +### PR 2 — Prepare scenario and data from intent + +**User-visible result:** a local instruction edit, without an incident report or +user-authored evaluation dataset, can produce a deliberate scenario and shared +data, preflight/freeze them, run the comparison, and explain the observed outcome. + +**Scope** + +- Implement the complete recipe in section 4: intent and preservation goals → + observable capabilities → natural task → coherent ordinary/edge data → + preflight/freeze → execution → evidence-based interpretation. +- Replace the mandatory incident question and task-known readiness condition. + Keep discovery as agent work; record selected cases, premises, evidence criteria + and omissions privately rather than embedding an assessment oracle in inputs. +- Reuse PR 1's delivery path for generated files and existing project data where + sufficient. Make required date, permission and tool premises explicit. +- Preserve trial counts, host parity and model-role defaults. Keep unsupported + capabilities visibly blocked instead of silently narrowing the experiment. +- Update the skill's result explanation to cover condition reach, unmet After + intent, interruptions and raw-evidence versus automatic-report differences. + Do not change the report renderer, extractor or report schema in this PR. +- Update the full workflow/flowchart and product explanation in the same PR. + Add or adjust deterministic tests only for changed behavioral contracts. + +**Expected files:** canonical skill, its single preparation reference, relevant +existing tests, docs/architecture.md and README.md; PR 1 support only if integrating +the real generator exposes a delivery defect. + +**Verification and merge gate** + +- Run affected deterministic checks and the repository's required gates. +- Follow section 8's bounded live smoke procedure on a synthetic instruction edit: + the modified skill itself must generate the scenario/data, not receive a + hand-authored successful preparation. +- Inspect actual snapshots and trial evidence for an ordinary case and compatible + boundary conditions. Check preparation separately from execution and extraction. + An unmet goal or inconclusive trial is valid; reproducing a defect is not a gate. +- Verify both supported hosts, recording unavailable live gates honestly. Do not + call the integrated workflow complete while required verification is blocked. +- Obtain independent read-only approval before opening the PR. Keep private + artifacts outside tracked files; public evidence summaries must be synthetic + or sanitized and must not imply stronger findings than the records support. + +### Acceptance ownership + +| Criterion | Primary delivery and proof | +|---|---| +| AC1 — no incident/eval required | PR 2; skill-generated preparation smoke | +| AC2 — capability/case/premise mapping | PR 2; inspect private preparation records | +| AC3 — identical shared inputs | PR 1 delivery proof; repeat with PR 2 generated data | +| AC4 — discovery and oracle isolation | PR 1 exclusion boundaries; PR 2 scenario/data audit | +| AC5 — environmental preflight | PR 1 delivery limits; PR 2 capability-level preflight | +| AC6 — dates, permissions and coherent facts | PR 2 preflight and fixture audit | +| AC7 — counts, roles, isolation and host parity | Both PRs; deterministic mechanics plus PR 2 live gates | +| AC8 — outcome, reach and report distinctions | PR 2 skill explanation; existing automatic report unchanged | +| AC9 — freeze and preserve first results | PR 1 delivery freeze; PR 2 full preparation/execution discipline | +| AC10 — skill prepares, runner consumes | PR 2 integrated smoke using PR 1's proven delivery path | + +### Deferred report design checkpoint + +After preparation passes the acceptance gate, review the generated scenario, +selected cases, private criteria, trial evidence, current automatic report and +skill explanation together. Identify concrete information gaps before selecting +a layout. Do not assume the earlier assumption-first prototype is the winner. + +Keep the existing target assessment as the starting point: goal-linked criteria +and per-trial `met`, `not_met`, `uncertain`, and `unassessable` outcomes. Investigate +whether actual runs require persisted case links, condition-reach records, +preflight/run-health context or raw-audit/report-disagreement records. These are +possible contract changes, not fields authorized for PRs 1–2. + +If the agreed design needs new assessment data, scope that producer/validator/ +consumer change explicitly before the UI PR. If existing data is sufficient, do +not create an extra schema PR. Preview report hierarchy changes against the saved +real-case evidence required by CODING_GUIDELINES.md; label new-state illustrations +as authored synthetic examples. Keep HTML, Markdown and structured output aligned. +Choose the final number and boundaries of report PRs at that checkpoint. + +### Step-by-step implementation protocol + +1. Start only the earliest unblocked PR. Inspect the current implementation and + relevant experimental recipe before editing; the handoff is not proof that + files have stayed unchanged. +2. Finish that PR's end-to-end behavior, docs, checks, smoke evidence and independent + review. Report what it delivers and which milestone criteria remain. +3. Open/commit/push only when requested; use DCO sign-off for every commit. Do not + merge implicitly. Base dependent work on the accepted predecessor; if stacking + PRs, declare the base and dependency and merge in order. +4. Track the executable work in the designated Linear project with the + `behavior-diff` label. PRs 1–2 are Improvements; a separate evidence-only task + is Verification. Publish the proposed ticket breakdown after user review, + reuse existing tickets where applicable, and keep blocked work out of High. +5. After PR 2, record the AC1–AC10 gate results. Only then reopen report design + with the user; no report redesign is required to finish the preparation work. diff --git a/plugin/skills/behavior-diff/SKILL.md b/plugin/skills/behavior-diff/SKILL.md index 6c93a4f..553cce4 100644 --- a/plugin/skills/behavior-diff/SKILL.md +++ b/plugin/skills/behavior-diff/SKILL.md @@ -13,11 +13,12 @@ no automatic verdict. The user judges the evidence. ## Ownership The skill owns judgment. It finds the change, freezes its purpose, drafts the -decision-moment task, selects the current trial stack and model, and explains -the evidence. +decision-moment task, reviews any shared trial data, selects the current trial +stack and model, and explains the evidence. -The scripts own repeatable mechanics. They build variants, run trials, -normalize traces, grade completeness, extract decisions, and render the report. +The scripts own repeatable mechanics. They stage and verify reviewed shared +data, build variants, run trials, normalize traces, grade completeness, extract +decisions, and render the report. The runner is bundled with this skill: `scripts/behavior-diff.sh` inside this skill's base directory. Pass `--agent` for the stack under test: @@ -201,10 +202,18 @@ the single-role or two-agent path. Create the fixtures with nothing. - When the rule fires mid-task, start the task at the decision point: "You already did X (it is in the tree); decide/report Y." - -3. **Run it as soon as the task is known.** Do not ask the user to confirm - the file, task, cost, or run mode. Do not mention trial counts, cost, or - full versus fast modes during normal execution. + - **Shared data.** If the decision moment needs common files that the + project base does not contain, read + `references/scenario-and-test-data.md` inside this skill's base + directory before running. Git trials start from HEAD, so additional + untracked or uncommitted data in the user's checkout never reaches them. That + reference delivers files the user supplied or approved to every trial. + It does not generate a scenario or data. + +3. **Run it as soon as the task is known and any shared data is verified.** + Do not ask the user to confirm the file, task, cost, or run mode. Do not + mention trial counts, cost, or full versus fast modes during normal + execution. First satisfy the privacy and sensitive-content approval boundary above; automatic execution never overrides it. @@ -243,6 +252,16 @@ the single-role or two-agent path. Create the fixtures with does not look anything up. Never put source details in `--task`, a trial prompt, or a sandbox file. + **Shared data.** When step 2 selected shared data, do not run the commands + above from the user's checkout. Prepare and verify the stage with + `references/scenario-and-test-data.md`, then run the same runner command + from `$ROOT/project` with `$ROOT/home` as `BEHAVIOR_DIFF_HOME`, exactly + as that reference shows. Keep the same agent, model, `--fast`, purpose, + and source choices. Pass `--before-file` only as the stage's frozen + `before` file, and only when it exists; never pass the original Before + path. If preparation or verification fails, stop and report the reason; + do not launch the run. + **Keep the run handle.** Use the host's background-execution facility when available so other work can continue. Retain the exact run directory printed at startup; concurrent comparisons have separate IDs. Share the printed @@ -301,7 +320,9 @@ the single-role or two-agent path. Create the fixtures with ## Boundaries - The runner never modifies the user's repo; reports and traces stay - local under the runs root above. + local under the runs root above. Staged shared-data runs keep them under + the stage's `home/runs/`. +- Never write, stage, stash, or commit shared data in the user's checkout. - The report quotes the user's real code. Never publish it anywhere without the user reviewing it first. - Do not edit the user's instruction file yourself — this skill only diff --git a/plugin/skills/behavior-diff/references/scenario-and-test-data.md b/plugin/skills/behavior-diff/references/scenario-and-test-data.md new file mode 100644 index 0000000..7108ae3 --- /dev/null +++ b/plugin/skills/behavior-diff/references/scenario-and-test-data.md @@ -0,0 +1,219 @@ +# Scenario and test data + +Read this file before you run a comparison whose task needs shared files that +the runner's project base does not contain. It explains how to deliver an +already selected dataset to every Before and After trial. It does not generate +a scenario or data. Step 2 of `SKILL.md` still drafts the task, and the user +supplies or approves the files. + +## When to deliver shared data + +The runner builds each trial from the project base. In a Git project, the base +is `git archive HEAD`. Outside Git, the base is a copy of the folder. Apart from +the selected instruction, untracked and uncommitted Git files do not enter trials. + +Use shared-data delivery only when all of these conditions are true: + +- The task needs common files that are not in the project base. +- Every Before and After trial must receive the same files. +- The comparison changes exactly one instruction file. + +If the task needs no extra files, use the normal step 3 command instead. + +Do not use this path for: + +- Spacedock workflow comparisons. They use the fixtures in + [spacedock-duo.md](spacedock-duo.md). +- State that the scratch baseline cannot hold. See [Limits](#limits). + +If a required setup is not supported, tell the user before launch. Do not run a +narrower experiment silently. + +Never write, stage, stash, or commit files in the user's checkout to make data +reach the trials. + +## Prepare the inputs + +1. **Shared data.** Put the trial inputs in a data directory outside the + project. The helper copies this tree into the same relative paths in the + trial project. Include only files that every trial may read. Never include + the purpose, source metadata, expected answers, grading criteria, or + evaluator notes. Review every file before preparation. The helper rejects + unsafe file types and paths. It does not scan contents for secrets or + private text. + Shared paths replace matching base files. Record the intended deviations + from the original project, including any replaced base files, in private + preparation notes outside the trial inputs before staging. +2. **Private context.** Keep the purpose file, the source file, and evaluator + notes in a private directory with mode `0700`, outside the project, the data + directory, and the stage. Give each file mode `0600`. If the project already + contains such a file, pass its project-relative path with `--exclude`. This + applies to a tracked file in a Git project and to any file in a non-Git + folder. You cannot exclude the selected instruction file. +3. **Stage path.** Choose a stage path that does not exist yet. It must be + outside the project and the data directory. + +Set these values. Use absolute paths: + +```bash +SKILL_DIR= +PROJECT= +TARGET= +DATA= +PRIVATE= +STAGE_PARENT=$(mktemp -d "${TMPDIR:-/tmp}/behavior-diff-stage.XXXXXX") +ROOT=$STAGE_PARENT/stage +``` + +Then prepare the stage: + +```bash +python3 "$SKILL_DIR/scripts/prepare_shared.py" prepare \ + --project "$PROJECT" --file "$TARGET" --data "$DATA" --output "$ROOT" +``` + +Optional arguments: + +- `--before-file ` supplies the Before content: a user-supplied original, + user-given before/after files, or one entry that you selected from the hook + baseline store. The helper always reads this file as content, even if its + name ends in `-ABSENT`. If the file is inside the project, the helper excludes + it from the stage automatically. +- `--exclude ` keeps one project path out of the stage. Repeat + the option for each path. + +Without `--before-file`, the helper selects Before in the same order as the +runner: + +1. The target in Git HEAD, if HEAD contains it. +2. Otherwise, the newest hook baseline under + `${BEHAVIOR_DIFF_HOME:-~/.behavior-diff}/baselines/`. Run `prepare` in the + user's normal environment. Do not set `BEHAVIOR_DIFF_HOME` to the stage. + Only a store entry ending in `-ABSENT` means that Before has no file. + +An untracked target is not treated as absent. If no baseline exists, the +helper stops. Ask the user for the original content and pass it with +`--before-file`, as step 1 of `SKILL.md` describes. The helper also refuses +identical Before and After content. + +On success, `prepare` prints the stage path and does these things: + +- It creates `$ROOT/` with mode `0700`. The stage contains `project/`, `home/`, + a private `freeze.json` with mode `0600`, and `before` when Before exists. +- It commits the common base, the shared data, and the selected Before version + in a new scratch Git repository in `$ROOT/project/`. The repository has no + remote, and its hooks and commit signing are off. +- It puts the After instruction in the scratch working tree. +- For a present Before, from Git HEAD or a file, it also freezes the Before + content as `$ROOT/before`, outside the scratch project. The runner receives + this file with `--before-file`. +- For an absent Before, it writes no `$ROOT/before`. It writes an `-ABSENT` + marker under `$ROOT/home/baselines/` instead. The existing runner reads this + marker through `BEHAVIOR_DIFF_HOME`. +- It confirms that `git archive` of the scratch commit reproduces the expected + base exactly. If `.gitattributes` rules such as `export-ignore`, + `export-subst`, or a filter change the archived files, `prepare` fails. +- It records the scratch commit and the content and executable bit of every + file in `freeze.json`. + +The helper never changes the original project or the data directory. It +copies the data, so later changes to the data directory do not affect the +stage. + +If `prepare` exits with a nonzero status, the helper removes the incomplete +stage. Stop. Correct the inputs, then prepare a new stage path. Do not edit +files in a stage by hand. + +## Verify, then run + +Run `verify` immediately before launch: + +```bash +python3 "$SKILL_DIR/scripts/prepare_shared.py" verify "$ROOT" +``` + +`verify` compares the scratch commit, the archived files, the working files, +and the frozen Before file or absent marker with `freeze.json`. Its read-only +Git checks use the launch environment, including ordinary global/system +configuration. Archive transformations such as `tar.umask` removing executable +bits must fail verification, not silently change trial inputs. If `verify` +exits with a nonzero status, do not launch. Prepare a new stage. + +Run verification and launch in the same shell environment. The helper rejects +Git repository, index, object-store, discovery and injected-config overrides +(such as `GIT_DIR`, `GIT_INDEX_FILE`, and `GIT_CONFIG_COUNT`) because the existing +runner would inherit them. Remove those overrides before preparing or launching; +do not bypass the failure. Ordinary user Git hooks/configuration remain existing +runner behavior, not a safety guarantee provided by this helper. + +Run the existing runner from the scratch project, with `$ROOT/home` as Behavior +Diff home. Use the same `--agent`, `--model`, and `--fast` choices as step 3. +The last line adds `--before-file` only when `$ROOT/before` exists. It expands +to nothing for an absent Before, in Bash and zsh: + +```bash +before_args=() +if [ -f "$ROOT/before" ]; then + before_args=(--before-file "$ROOT/before") +fi +python3 "$SKILL_DIR/scripts/prepare_shared.py" verify "$ROOT" && +(cd "$ROOT/project" && + BEHAVIOR_DIFF_HOME="$ROOT/home" \ + "$SKILL_DIR/scripts/behavior-diff.sh" --agent \ + --file "$TARGET" --task "" \ + --purpose-file "$PRIVATE/purpose.json" \ + ${before_args[@]+"${before_args[@]}"}) +``` + +Pass only `$ROOT/before` as `--before-file`, and only when it exists. Never +pass the original Before path or a hook-store entry to a staged run. For a +historical comparison with verified provenance, also append +`--source-file "$PRIVATE/source.json"`. + +Source provenance must name the original repository and the original Before +and After commits. Verify it in the original checkout as step 1 describes. +Never read provenance from `$ROOT/project` or derive it from a scratch commit. +In the report, the Before label shows `$ROOT/before` or states that the file +did not exist, and the After label shows the stage path. These labels are +local names, not original revisions. + +## Freeze discipline + +- After `verify` passes, treat the stage, task, purpose, and source as frozen. + Stop any preparation worker before launch. No agent may edit + `$ROOT/project/`, `$ROOT/before`, `$ROOT/home/baselines/`, `freeze.json`, or + the runner and its scripts. +- To change the data, prepare a new stage and run a new comparison. Never + prepare a new stage or edit data to turn a negative or inconclusive result + into a finding. Keep and report the first results. +- The freeze detects changes before launch. It is not a sandbox. Trial agents + can read and run what their host permits, and the stage files are not + read-only. Separate directories do not protect against a hostile process. + +## Reports and status + +Runs from a stage land under `$ROOT/home/runs//`. Keep `$ROOT` after +the run. Deleting it deletes the reports and the freeze record. Use the run +directory that the runner prints with `status.py`. To list stage runs with +`--list`, set `BEHAVIOR_DIFF_HOME="$ROOT/home"`. + +## Limits + +- Exactly one instruction file differs between Before and After. All other + initial files are the same on both sides. +- In a Git project, the base is HEAD. Other uncommitted work does not enter the + stage. Trials do not receive the original Git history, branches, or remotes. +- Total input bytes are limited by `BEHAVIOR_DIFF_COPY_CAP_KB` (default + `204800`, which is 200 MiB). Every trial copies the stage, so keep it small. +- The helper rejects symbolic links, special files, Git internals, submodules, + unsafe or absolute paths, case-colliding paths, and file/directory + collisions. It also rejects data that would replace the selected instruction + or an excluded path. +- Changing the selected instruction's executable bit is unsupported; shared data + files retain their executable bits. +- The stage keeps file content and the executable bit. It does not keep other + permissions, ownership, timestamps, or empty directories. +- Live services, credentials, writable external systems, and multi-file + instruction versions are not delivered. +- The helper does not detect secrets. The caller reviews every shared file. +- No path provides a filesystem sandbox. See [Freeze discipline](#freeze-discipline). diff --git a/plugin/skills/behavior-diff/scripts/prepare_shared.py b/plugin/skills/behavior-diff/scripts/prepare_shared.py new file mode 100755 index 0000000..1604448 --- /dev/null +++ b/plugin/skills/behavior-diff/scripts/prepare_shared.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""Private skill helper: freeze shared files into an isolated runner project. + +This does not generate scenarios, launch models, or infer source provenance. +The caller reviews trial-visible files and names private paths to exclude. +""" + +import argparse +import hashlib +import io +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import subprocess +import sys +import tarfile +import unicodedata + + +class PreparationError(ValueError): + pass + + +def check_git_environment(): + # The existing runner inherits this environment. Do not verify one world + # and then let Git redirect the runner's repository, index, or attributes. + redirects = { + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_INDEX_FILE", + "GIT_OBJECT_DIRECTORY", + "GIT_ALTERNATE_OBJECT_DIRECTORIES", + "GIT_CONFIG", + "GIT_CONFIG_COUNT", + "GIT_CONFIG_PARAMETERS", + "GIT_NAMESPACE", + "GIT_SHALLOW", + "GIT_REPLACE_REF_BASE", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_ATTR_SOURCE", + } + if any( + key in redirects or key.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_")) + for key in os.environ + ): + raise PreparationError( + "unset Git repository, index, and config overrides before preparation and launch" + ) + + +def git(project, *args, check=True, launch=False): + # Isolate construction; read-only handoff checks must match the runner. + env = None + if not launch: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith("GIT_") + } + env.update(GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull) + result = subprocess.run( + ["git", "--literal-pathspecs", "-C", str(project), *args], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if check and result.returncode: + raise PreparationError( + "Git could not construct or inspect the isolated snapshot" + ) + return result + + +def relative(value): + path = PurePosixPath(value) + if ( + not value + or value == "." + or path.is_absolute() + or str(path) != value + or any(part in (".", "..") or part.lower() == ".git" for part in path.parts) + or "\\" in value + or any(ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise PreparationError("expected a safe project-relative path") + return value + + +def within(path, root): + key, root_key = path_key(str(path)), path_key(str(root)) + return key == root_key or key.startswith(root_key.rstrip("/") + "/") + + +def path_key(name): + return unicodedata.normalize("NFC", name).casefold() + + +def excluded(name, exclusions): + key = path_key(name) + return any( + key == path_key(item) or key.startswith(path_key(item) + "/") + for item in exclusions + ) + + +def regular_file(path): + if not stat.S_ISREG(path.lstat().st_mode): + raise PreparationError( + "inputs must be regular files, not links or special files" + ) + return path.read_bytes(), bool(path.stat().st_mode & 0o111) + + +def safe_file(root, name): + path = root + for part in PurePosixPath(name).parts: + path = path / part + if path.is_symlink(): + raise PreparationError("input paths must not traverse symlinks") + return regular_file(path) + + +def tree(root, exclusions=(), skip_git=False): + files = {} + + def visit(folder): + for path in sorted(folder.iterdir()): + name = path.relative_to(root).as_posix() + if (skip_git and name == ".git") or excluded(name, exclusions): + continue + relative(name) + mode = path.lstat().st_mode + if stat.S_ISDIR(mode): + visit(path) + elif stat.S_ISREG(mode): + files[name] = regular_file(path) + else: + raise PreparationError( + "shared projects cannot contain links or special files" + ) + + visit(root) + return files + + +def archive(project, commit, launch=False): + entries = git(project, "ls-tree", "-r", commit, launch=launch).stdout.splitlines() + if any(entry.startswith(b"160000 ") for entry in entries): + raise PreparationError("submodules require an unsupported project setup") + raw = git(project, "archive", "--format=tar", commit, launch=launch).stdout + files = {} + with tarfile.open(fileobj=io.BytesIO(raw), mode="r:") as bundle: + for entry in bundle: + name = relative(entry.name.rstrip("/")) + if entry.isdir(): + continue + if not entry.isfile(): + raise PreparationError( + "Git snapshots cannot contain links or special files" + ) + if name in files: + raise PreparationError("duplicate archive path") + with bundle.extractfile(entry) as stream: + files[name] = stream.read(), bool(entry.mode & 0o111) + return files + + +def inventory(files): + return { + name: {"sha256": hashlib.sha256(data).hexdigest(), "executable": executable} + for name, (data, executable) in sorted(files.items()) + } + + +def encode(path): + return str(path).replace("%", "%25").replace("/", "%2F") + + +def baseline(project, target, commit, before_file, home): + if before_file is not None: + return "file", regular_file(before_file) + if commit is not None: + found = git(project, "cat-file", "-e", f"{commit}:{target}", check=False) + if found.returncode == 0: + entry = git(project, "ls-tree", commit, "--", target).stdout + if not entry.startswith((b"100644 ", b"100755 ")): + raise PreparationError("Before instruction must be a regular file") + return "head", ( + git(project, "show", f"{commit}:{target}").stdout, + entry.startswith(b"100755 "), + ) + store = home / "baselines" / encode(project / target) + if store.is_symlink(): + raise PreparationError("baseline store must not be a symlink") + candidates = ( + [entry for entry in store.iterdir() if not entry.name.startswith(".")] + if store.is_dir() + else [] + ) + if not candidates: + raise PreparationError( + "no Before source; supply --before-file or capture a hook baseline" + ) + selected = max( + candidates, key=lambda entry: (entry.lstat().st_mtime_ns, entry.name) + ) + content = regular_file(selected) + return ("absent", None) if selected.name.endswith("-ABSENT") else ("file", content) + + +def validate_files(files): + aliases = {} + file_keys = {path_key(name) for name in files} + for name in files: + relative(name) + path = PurePosixPath(name) + for part in (path, *path.parents): + text = str(part) + key = path_key(text) + if key in aliases and aliases[key] != text: + raise PreparationError( + "case-colliding paths are not portable trial inputs" + ) + aliases[key] = text + if any(path_key(str(parent)) in file_keys for parent in path.parents): + raise PreparationError("file and directory paths collide") + + +def write_files(root, files): + for name, (data, executable) in sorted(files.items()): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + # Also catches filesystem-specific aliases, e.g. Unicode normalization. + with path.open("xb") as stream: + stream.write(data) + path.chmod(0o755 if executable else 0o644) + + +def prepare(args): + check_git_environment() + project = Path(args.project).resolve(strict=True) + data_path = Path(args.data) + if data_path.is_symlink(): + raise PreparationError("shared-data directory must not be a symlink") + data_path = data_path.resolve(strict=True) + output_arg = Path(args.output).absolute() + if output_arg.exists() or output_arg.is_symlink(): + raise PreparationError("output must be a new private directory") + output = output_arg.parent.resolve(strict=True) / output_arg.name + if not project.is_dir() or not data_path.is_dir(): + raise PreparationError("project and shared-data inputs must be directories") + if within(data_path, project) or within(project, data_path): + raise PreparationError("shared data must be outside the original project") + if any( + within(output, root) or within(root, output) for root in (project, data_path) + ): + raise PreparationError("output must be outside the project and shared data") + target = relative(args.file) + exclusions = {relative(name) for name in args.exclude} + if excluded(target, exclusions): + raise PreparationError("the selected instruction cannot be excluded") + after = safe_file(project, target) + home = Path( + os.environ.get("BEHAVIOR_DIFF_HOME", str(Path.home() / ".behavior-diff")) + ).resolve() + before_file = None + if args.before_file: + supplied = Path(args.before_file).absolute() + before_file = supplied.parent.resolve(strict=True) / supplied.name + if before_file == project / target: + raise PreparationError("Before and After must be separate files") + if within(before_file, project): + exclusions.add(before_file.relative_to(project).as_posix()) + head = git(project, "rev-parse", "--verify", "HEAD", check=False) + commit = head.stdout.decode().strip() if head.returncode == 0 else None + if commit is not None: + top = git(project, "rev-parse", "--show-toplevel").stdout.decode().strip() + if Path(top).resolve() != project: + raise PreparationError("--project must name the repository root") + base = archive(project, commit) + base = { + name: content + for name, content in base.items() + if not excluded(name, exclusions) + } + else: + copy_exclusions = set(exclusions) + if within(home, project): + if home == project: + raise PreparationError( + "project cannot be the Behavior Diff state directory" + ) + copy_exclusions.add(home.relative_to(project).as_posix()) + base = tree(project, copy_exclusions, skip_git=True) + mode, before = baseline(project, target, commit, before_file, home) + if before is not None and before[0] == after[0]: + raise PreparationError( + "Before and After instructions match; nothing to compare" + ) + if before is not None and before[1] != after[1]: + raise PreparationError("instruction executable-mode changes are not supported") + shared = tree(data_path) + if not shared: + raise PreparationError("shared-data directory contains no files") + for name in shared: + if ( + excluded(name, (target,)) + or excluded(target, (name,)) + or excluded(name, exclusions) + or any(excluded(item, (name,)) for item in exclusions) + ): + raise PreparationError( + "shared data collides with instructions or excluded context" + ) + base.update(shared) + if before is None: + base.pop(target, None) + else: + base[target] = before + working = {**base, target: after} + validate_files(working) + cap = int(os.environ.get("BEHAVIOR_DIFF_COPY_CAP_KB", "204800")) * 1024 + if ( + cap <= 0 + or sum(len(content[0]) for content in working.values()) + + (len(before[0]) if before else 0) + > cap + ): + raise PreparationError("prepared project exceeds BEHAVIOR_DIFF_COPY_CAP_KB") + output.mkdir(mode=0o700) + try: + scratch = output / "project" + scratch.mkdir(mode=0o700) + (output / "home").mkdir(mode=0o700) + write_files(scratch, base) + git(scratch, "init", "-q", "--template=") + for key, value in ( + ("user.name", "Behavior Diff"), + ("user.email", "behavior-diff@example.invalid"), + ("core.hooksPath", os.devnull), + ("commit.gpgSign", "false"), + ("core.autocrlf", "false"), + ("core.attributesFile", os.devnull), + ): + git(scratch, "config", key, value) + git(scratch, "add", "--force", "--all") + git( + scratch, + "commit", + "--quiet", + "--allow-empty", + "--signoff", + "-m", + "Shared scenario baseline", + ) + scratch_commit = git(scratch, "rev-parse", "HEAD").stdout.decode().strip() + expected_archive = inventory(base) + if inventory(archive(scratch, scratch_commit)) != expected_archive: + raise PreparationError( + "Git attributes alter prepared files; this setup cannot be delivered faithfully" + ) + target_path = scratch / target + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_bytes(after[0]) + target_path.chmod(0o755 if after[1] else 0o644) + marker = None + if mode == "absent": + marker = "home/baselines/" + encode(scratch / target) + "/frozen-ABSENT" + marker_path = output / marker + marker_path.parent.mkdir(parents=True, mode=0o700) + marker_path.touch(mode=0o600) + else: + before_path = output / "before" + before_path.write_bytes(before[0]) + before_path.chmod(0o700 if before[1] else 0o600) + frozen = { + "original_project": str(project), + "original_commit": commit, + "target": target, + "before_mode": mode, + "scratch_commit": scratch_commit, + "exclusions": sorted(exclusions), + "archive": expected_archive, + "working": inventory(working), + "absent_marker": marker, + "before": inventory({"before": before}) if before is not None else None, + } + record = output / "freeze.json" + with record.open("x") as stream: + record.chmod(0o600) + json.dump(frozen, stream, indent=2, sort_keys=True) + stream.write("\n") + verify(output) + except BaseException: + shutil.rmtree(output) + raise + return output + + +def verify(root): + check_git_environment() + root = Path(root).resolve(strict=True) + record = root / "freeze.json" + regular_file(record) + frozen = json.loads(record.read_text()) + scratch = root / "project" + if scratch.is_symlink() or (scratch / ".git").is_symlink(): + raise PreparationError("frozen project must not be redirected") + top = ( + git(scratch, "rev-parse", "--show-toplevel", launch=True) + .stdout.decode() + .strip() + ) + if Path(top).resolve() != scratch: + raise PreparationError("frozen project must not be redirected") + current = git(scratch, "rev-parse", "HEAD", launch=True).stdout.decode().strip() + if current != frozen["scratch_commit"]: + raise PreparationError("prepared baseline changed after freeze") + if inventory(tree(scratch, skip_git=True)) != frozen["working"]: + raise PreparationError("prepared files changed after freeze") + if inventory(archive(scratch, current, launch=True)) != frozen["archive"]: + raise PreparationError("launch Git configuration or prepared archive changed") + if frozen["before_mode"] == "absent": + if (root / "before").exists() or (root / "before").is_symlink(): + raise PreparationError("absent Before must not have a frozen content file") + marker = root / frozen["absent_marker"] + if marker.parent.is_symlink() or marker.is_symlink() or not marker.is_file(): + raise PreparationError("frozen absent-file baseline is unavailable") + if sorted(path.name for path in marker.parent.iterdir()) != [marker.name]: + raise PreparationError("frozen absent-file baseline changed") + elif inventory({"before": regular_file(root / "before")}) != frozen["before"]: + raise PreparationError("frozen Before instruction changed") + return root + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + stage = commands.add_parser( + "prepare", help="freeze reviewed shared files; never launch trials" + ) + for name in ("project", "file", "data", "output"): + stage.add_argument("--" + name, required=True) + stage.add_argument("--before-file") + stage.add_argument("--exclude", action="append", default=[]) + check = commands.add_parser( + "verify", help="reject changed inputs before using the existing runner" + ) + check.add_argument("root") + args = parser.parse_args() + try: + result = prepare(args) if args.command == "prepare" else verify(args.root) + except (OSError, ValueError, KeyError, TypeError, tarfile.TarError) as exc: + message = ( + str(exc) + if isinstance(exc, PreparationError) + else "invalid or unreadable preparation inputs" + ) + print("prepare-shared: " + message, file=sys.stderr) + return 2 + print(result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/live-report-contract.sh b/tests/live-report-contract.sh index 7121fa5..41560d7 100755 --- a/tests/live-report-contract.sh +++ b/tests/live-report-contract.sh @@ -168,13 +168,9 @@ require_output 'claude, codex, pi, or omp' "$codex_manifest" \ 'Codex manifest does not name all trial stacks' require_output 'Pi and OMP are trial stacks, not plugin hosts' "$readme" \ 'README does not separate trial stacks from plugin hosts' -require_output 'Run it as soon as the task is known.' "$headless_skill" \ - 'headless skill does not start the default run immediately' require_output "Only add \`--fast\` when the user explicitly requested it" \ "$headless_skill" \ 'headless skill does not reserve fast mode for explicit requests' -require_output 'Do not mention trial counts, cost' "$headless_skill" \ - 'headless skill still exposes run counts or cost to the user' require_output 'full versus fast modes' "$headless_skill" \ 'headless skill still exposes implementation modes to the user' reject_output 'Confirm before running' "$headless_skill" \ diff --git a/tests/shared-data-test.py b/tests/shared-data-test.py new file mode 100755 index 0000000..fc8f147 --- /dev/null +++ b/tests/shared-data-test.py @@ -0,0 +1,1186 @@ +#!/usr/bin/env python3 +"""Synthetic shared-data staging boundaries; no models, credentials, or network. + +Default checks exercise only the private prepare/verify helper and never launch +the runner. Pass --runner-smoke for a manual smoke that runs the shipped runner +and trial adapters with local fake claude/codex executables first on PATH. +""" + +import argparse +import io +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import unittest + +REPO = Path(__file__).resolve().parents[1] +SCRIPTS = REPO / "plugin/skills/behavior-diff/scripts" +HELPER = SCRIPTS / "prepare_shared.py" +RUNNER = SCRIPTS / "behavior-diff.sh" + +TARGET = "CLAUDE.md" +BEFORE = b"# Synthetic rules\n\nAnswer from the records file.\n" +AFTER = b"# Synthetic rules\n\nAnswer from the records file and cite each row.\n" +EXPLICIT = b"# Synthetic rules\n\nExplicit synthetic Before version.\n" +PURPOSE = { + "goals": [ + { + "text": "Cite each synthetic record row.", + "source": "session", + "basis": "explicit", + "reference": "Synthetic request", + } + ] +} +SOURCE = { + "forge": "github", + "repository_url": "https://github.com/example-org/synthetic-skills", + "before_commit": "a" * 40, + "after_commit": "b" * 40, + "pull_requests": [ + { + "number": 7, + "title": "Synthetic rule edit", + "url": "https://github.com/example-org/synthetic-skills/pull/7", + "applies_to": "after", + } + ], +} +COMMITTED = { + TARGET: BEFORE, + ".gitignore": b"data/\n*.local\n", + "docs/guide.md": b"Committed synthetic guide.\n", + "docs/status.md": b"Committed synthetic status.\n", + "run.sh": (b"#!/bin/sh\necho synthetic\n", True), + "notes/purpose.json": json.dumps(PURPOSE).encode(), +} +WORKING = { + TARGET: AFTER, + "docs/guide.md": b"Uncommitted synthetic guide.\n", + "scratch.txt": b"Untracked synthetic note.\n", + "cache.local": b"Ignored synthetic cache.\n", +} +SHARED = { + "data/records.csv": b"id,amount\n1,10\n2,-5\n", + "data/tool.sh": (b"#!/bin/sh\ncat data/records.csv\n", True), + "docs/status.md": b"Shared synthetic status.\n", +} +PRIVATE = ["notes/purpose.json"] +TRIAL_MODEL = "synthetic-trial-model" +EXTRACT_MODEL = "synthetic-extract-model" + + +def normalized(files): + return { + name: content if isinstance(content, tuple) else (content, False) + for name, content in files.items() + } + + +# Expected scratch baseline for the standard Git fixture: HEAD content only, +# reviewed shared files on top, private context removed. +BASELINE = normalized( + { + TARGET: BEFORE, + ".gitignore": COMMITTED[".gitignore"], + "docs/guide.md": COMMITTED["docs/guide.md"], + "docs/status.md": SHARED["docs/status.md"], + "run.sh": COMMITTED["run.sh"], + "data/records.csv": SHARED["data/records.csv"], + "data/tool.sh": SHARED["data/tool.sh"], + } +) + + +def write_files(root, files): + for name, (data, executable) in normalized(files).items(): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + path.chmod(0o755 if executable else 0o644) + + +def encoded(path): + # Hook store key: percent-encode '%' first, then '/'. + return str(path).replace("%", "%25").replace("/", "%2F") + + +def tree_state(root, skip_git=False): + """Every entry under root, including hidden files, by kind and content.""" + state = {} + for current, dirs, files in os.walk(root): + base = Path(current) + if skip_git and base == root and ".git" in dirs: + dirs.remove(".git") + for name in sorted(dirs + files): + path = base / name + key = path.relative_to(root).as_posix() + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + state[key] = ("link", os.readlink(path)) + elif stat.S_ISDIR(mode): + state[key] = ("dir", stat.S_IMODE(mode)) + elif stat.S_ISREG(mode): + state[key] = ("file", path.read_bytes(), stat.S_IMODE(mode)) + else: + state[key] = ("special", stat.S_IFMT(mode)) + return state + + +def trace_events(trace): + results, finals = [], [] + for line in trace.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + event = json.loads(line) + if event.get("type") == "user": + for item in event["message"]["content"]: + if item.get("type") == "tool_result": + results.append(item["content"]) + elif event.get("type") == "result": + finals.append(event.get("result") or "") + return results, finals + + +class Fixtures: + def setUp(self): + temp = tempfile.TemporaryDirectory(prefix="shared-data-test-") + self.addCleanup(temp.cleanup) + self.tmp = Path(temp.name).resolve() + self.home = self.tmp / "home" + self.home.mkdir() + (self.tmp / "tmp").mkdir() + gitconfig = self.tmp / "gitconfig" + gitconfig.write_text( + "[user]\n\tname = Synthetic Tester\n\temail = tester@example.invalid\n" + "[init]\n\tdefaultBranch = main\n[commit]\n\tgpgSign = false\n" + "[core]\n\tautocrlf = false\n", + encoding="utf-8", + ) + self.store_home = self.tmp / "behavior-home" + self.env = { + "PATH": os.environ["PATH"], + "HOME": str(self.home), + "LANG": "C", + "TMPDIR": str(self.tmp / "tmp"), + "GIT_CONFIG_GLOBAL": str(gitconfig), + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_DATE": "2026-01-01T00:00:00Z", + "GIT_COMMITTER_DATE": "2026-01-01T00:00:00Z", + "BEHAVIOR_DIFF_HOME": str(self.store_home), + } + self.stages = 0 + + def git(self, cwd, *args): + return subprocess.run( + ["git", "-C", str(cwd), *args], + env=self.env, + check=True, + capture_output=True, + ).stdout + + def git_project(self, name="project", committed=COMMITTED, working=WORKING): + project = self.tmp / name + project.mkdir() + self.git(project, "init", "-q") + write_files(project, committed) + self.git(project, "add", "-A") + self.git(project, "commit", "-qm", "Synthetic fixture") + write_files(project, working) + return project + + def folder(self, name, files): + root = self.tmp / name + root.mkdir() + write_files(root, files) + return root + + def store_entry(self, target, name, data, mtime, key=None): + store = self.store_home / "baselines" / (key or encoded(target)) + store.mkdir(parents=True, exist_ok=True) + entry = store / name + entry.write_bytes(data) + os.utime(entry, (mtime, mtime)) + return entry + + def prepare( + self, project, data, output=None, file=TARGET, before=None, exclude=(), env=None + ): + if output is None: + self.stages += 1 + output = self.tmp / f"stage-{self.stages}" + argv = [ + sys.executable, + str(HELPER), + "prepare", + "--project", + str(project), + "--file", + file, + "--data", + str(data), + "--output", + str(output), + ] + if before is not None: + argv += ["--before-file", str(before)] + for name in exclude: + argv += ["--exclude", name] + result = subprocess.run( + argv, env=env or self.env, capture_output=True, text=True + ) + return result, output + + def verify(self, root): + return subprocess.run( + [sys.executable, str(HELPER), "verify", str(root)], + env=self.env, + capture_output=True, + text=True, + ) + + def staged(self, project, data, **options): + result, output = self.prepare(project, data, **options) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(Path(result.stdout.strip()).resolve(), output.resolve()) + return output + + def archive_files(self, repo, rev="HEAD"): + raw = self.git(repo, "archive", "--format=tar", rev) + files = {} + with tarfile.open(fileobj=io.BytesIO(raw), mode="r:") as bundle: + for entry in bundle: + if entry.isdir(): + continue + self.assertTrue(entry.isfile(), entry.name) + with bundle.extractfile(entry) as stream: + files[entry.name] = (stream.read(), bool(entry.mode & 0o111)) + return files + + def working_files(self, root): + files = {} + for name, entry in tree_state(root, skip_git=True).items(): + if entry[0] == "dir": + continue + self.assertEqual(entry[0], "file", name) + files[name] = (entry[1], bool(entry[2] & 0o111)) + return files + + def original_state(self, *roots): + states = [] + for root in roots: + facts = [] + if (root / ".git").is_dir(): + for args in ( + ("rev-parse", "--verify", "-q", "HEAD"), + ("for-each-ref",), + ("ls-files", "-s"), + ("stash", "list"), + ("config", "--local", "--list"), + ("status", "--porcelain=v1", "-uall", "--ignored"), + ): + facts.append( + subprocess.run( + ["git", "-C", str(root), *args], + env=self.env, + capture_output=True, + ).stdout + ) + states.append((tree_state(root, skip_git=True), facts)) + return states + + def assert_rejected(self, result, output): + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertTrue(result.stderr.strip()) + self.assertFalse(output.exists() or output.is_symlink(), output) + + def standard(self): + return self.git_project(), self.folder("data", SHARED) + + +class SharedDataBoundaryTests(Fixtures, unittest.TestCase): + def test_git_head_baseline_freezes_shared_files_and_only_after_differs(self): + project, data = self.standard() + original = self.original_state(project, data) + stage = self.staged(project, data, exclude=PRIVATE) + scratch = stage / "project" + + self.assertEqual(stat.S_IMODE(stage.stat().st_mode), 0o700) + self.assertEqual(stat.S_IMODE((stage / "freeze.json").stat().st_mode), 0o600) + self.assertTrue((stage / "home").is_dir()) + self.assertEqual((stage / "before").read_bytes(), BEFORE) + self.assertEqual(self.archive_files(scratch), BASELINE) + self.assertEqual( + self.working_files(scratch), {**BASELINE, TARGET: (AFTER, False)} + ) + status = self.git(scratch, "status", "--porcelain=v1", "-uall").decode() + self.assertEqual(status.splitlines(), [f" M {TARGET}"]) + self.assertEqual(self.git(scratch, "remote").strip(), b"") + self.assertIn(b"Signed-off-by:", self.git(scratch, "log", "-1", "--format=%B")) + self.assertEqual(self.original_state(project, data), original) + self.assertEqual(self.verify(stage).returncode, 0) + + def test_frozen_stage_no_longer_depends_on_original_inputs(self): + project, data = self.standard() + stage = self.staged(project, data, exclude=PRIVATE) + frozen = tree_state(stage / "project", skip_git=True) + (data / "data/records.csv").write_bytes(b"id,amount\n9,99\n") + (project / TARGET).write_bytes(b"# Later unrelated edit\n") + self.store_entry(project / TARGET, "20260105-000000-1", b"later", 5000) + self.assertEqual(self.verify(stage).returncode, 0) + self.assertEqual(tree_state(stage / "project", skip_git=True), frozen) + + def test_git_redirect_environment_is_rejected_before_handoff(self): + project, data = self.standard() + stage = self.staged(project, data, exclude=PRIVATE) + original = self.original_state(project, data) + for key, value in ( + ("GIT_INDEX_FILE", str(project / ".git/index")), + ("GIT_DIR", str(project / ".git")), + ("GIT_CONFIG_COUNT", "1"), + ): + with self.subTest(key=key): + env = dict(self.env, **{key: value}) + result, output = self.prepare(project, data, exclude=PRIVATE, env=env) + self.assert_rejected(result, output) + checked = subprocess.run( + [sys.executable, str(HELPER), "verify", str(stage)], + env=env, + capture_output=True, + text=True, + ) + self.assertEqual(checked.returncode, 2) + self.assertEqual(self.original_state(project, data), original) + + def test_global_git_configuration_cannot_run_staging_hooks(self): + project, data = self.standard() + marker = self.tmp / "hook-ran" + hooks = self.tmp / "hooks" + hooks.mkdir() + for name in ("pre-commit", "commit-msg", "post-commit"): + hook = hooks / name + hook.write_text(f"#!/bin/sh\ntouch '{marker}'\nexit 1\n") + hook.chmod(0o755) + attributes = self.tmp / "attributes" + attributes.write_text("*.csv export-ignore\n") + hostile_home = self.tmp / "hostile-home" + hostile_home.mkdir() + hostile = hostile_home / ".gitconfig" + hostile.write_text( + f"[core]\n\thooksPath = {hooks}\n\tattributesFile = {attributes}\n" + f"[commit]\n\tgpgSign = true\n[gpg]\n\tprogram = {shutil.which('false')}\n" + ) + original = self.original_state(project, data) + env = dict( + self.env, + HOME=str(hostile_home), + GIT_CONFIG_GLOBAL=str(hostile), + ) + stage = self.staged(project, data, exclude=PRIVATE, env=env) + self.assertEqual(self.archive_files(stage / "project"), BASELINE) + self.assertNotIn( + b"gpgsig", self.git(stage / "project", "cat-file", "commit", "HEAD") + ) + self.assertFalse(marker.exists()) + self.assertEqual(self.original_state(project, data), original) + + def test_launch_git_config_cannot_change_frozen_archive(self): + project, data = self.standard() + original = self.original_state(project, data) + stage = self.staged(project, data, exclude=PRIVATE) + self.assertEqual(self.archive_files(stage / "project"), BASELINE) + with Path(self.env["GIT_CONFIG_GLOBAL"]).open("a") as stream: + stream.write("\n[tar]\n\tumask = 0111\n") + self.assertFalse(self.archive_files(stage / "project")["data/tool.sh"][1]) + result = self.verify(stage) + self.assertEqual(result.returncode, 2, result.stderr) + result, output = self.prepare(project, data, exclude=PRIVATE) + self.assert_rejected(result, output) + self.assertEqual(self.original_state(project, data), original) + + def test_attribute_induced_archive_changes_are_rejected(self): + cases = { + "project export-ignore": ( + {".gitattributes": b"data/** export-ignore\n"}, + {"data/records.csv": b"id\n1\n"}, + ), + "shared export-ignore": ( + {}, + { + "data/.gitattributes": b"records.csv export-ignore\n", + "data/records.csv": b"1\n", + }, + ), + "shared export-subst": ( + {}, + { + "data/.gitattributes": b"stamp.txt export-subst\n", + "data/stamp.txt": b"$Format:%H$\n", + }, + ), + } + for label, (extra, shared) in cases.items(): + with self.subTest(label): + name = label.replace(" ", "-") + project = self.git_project(name, {**COMMITTED, **extra}) + data = self.folder(name + "-data", shared) + original = self.original_state(project, data) + result, output = self.prepare(project, data) + self.assert_rejected(result, output) + self.assertEqual(self.original_state(project, data), original) + + def test_explicit_before_wins_and_is_read_literally(self): + project, data = self.standard() + self.store_entry(project / TARGET, "20260101-000000-ABSENT", b"", 1000) + literal = self.tmp / "CLAUDE.md-ABSENT" + literal.write_bytes(EXPLICIT) + stage = self.staged(project, data, before=literal, exclude=PRIVATE) + self.assertEqual( + self.archive_files(stage / "project")[TARGET], (EXPLICIT, False) + ) + self.assertEqual((stage / "before").read_bytes(), EXPLICIT) + self.assertEqual((stage / "project" / TARGET).read_bytes(), AFTER) + + def test_before_file_inside_project_is_excluded_from_shared_baseline(self): + project = self.git_project( + committed={**COMMITTED, "drafts/old-CLAUDE.md": EXPLICIT} + ) + data = self.folder("data", SHARED) + stage = self.staged( + project, data, before=project / "drafts/old-CLAUDE.md", exclude=PRIVATE + ) + archived = self.archive_files(stage / "project") + self.assertNotIn("drafts/old-CLAUDE.md", archived) + self.assertEqual(archived[TARGET], (EXPLICIT, False)) + self.assertNotIn("drafts/old-CLAUDE.md", self.working_files(stage / "project")) + + def test_hook_baseline_uses_newest_visible_entry_under_percent_encoded_key(self): + project = self.folder( + "pct%dir", {TARGET: AFTER, "notes.md": b"Synthetic notes.\n"} + ) + data = self.folder("data", SHARED) + target = project / TARGET + self.store_entry(target, "20260101-000000-1", b"older\n", 1000) + self.store_entry(target, "20260102-000000-2", BEFORE, 2000) + self.store_entry(target, ".session-synthetic", b"hidden marker\n", 3000) + naive = str(target).replace("/", "%2F") + self.assertNotEqual(naive, encoded(target)) + self.store_entry(target, "20260109-000000-3", b"decoy\n", 9000, key=naive) + + stage = self.staged(project, data) + self.assertEqual(self.archive_files(stage / "project")[TARGET], (BEFORE, False)) + self.assertEqual((stage / "before").read_bytes(), BEFORE) + + self.store_entry(target, "20260103-000000-ABSENT", b"", 4000) + stage = self.staged(project, data) + scratch = stage / "project" + self.assertNotIn(TARGET, self.archive_files(scratch)) + self.assertEqual((scratch / TARGET).read_bytes(), AFTER) + status = self.git(scratch, "status", "--porcelain=v1", "-uall").decode() + self.assertEqual(status.splitlines(), [f"?? {TARGET}"]) + self.assertFalse((stage / "before").exists()) + marker_dir = stage / "home/baselines" / encoded(scratch.resolve() / TARGET) + entries = [entry.name for entry in marker_dir.iterdir()] + self.assertEqual(len(entries), 1) + self.assertTrue(entries[0].endswith("-ABSENT")) + self.assertEqual(self.verify(stage).returncode, 0) + + def test_untracked_target_without_recorded_before_is_rejected(self): + committed = {name: data for name, data in COMMITTED.items() if name != TARGET} + data = self.folder("data", SHARED) + for label, project in ( + ("git", self.git_project("git", committed)), + ("folder", self.folder("folder", {TARGET: AFTER})), + ): + with self.subTest(label): + result, output = self.prepare(project, data) + self.assert_rejected(result, output) + + def test_identical_before_and_after_are_rejected(self): + data = self.folder("data", SHARED) + unchanged = self.git_project("unchanged", working={}) + result, output = self.prepare(unchanged, data) + self.assert_rejected(result, output) + project = self.git_project() + same = self.tmp / "same.md" + same.write_bytes(AFTER) + result, output = self.prepare(project, data, before=same) + self.assert_rejected(result, output) + + def test_instruction_executable_mode_change_is_rejected(self): + data = self.folder("data", SHARED) + project = self.git_project() + (project / TARGET).chmod(0o755) + result, output = self.prepare(project, data) + self.assert_rejected(result, output) + project = self.git_project("explicit") + before = self.tmp / "before-exec.md" + before.write_bytes(EXPLICIT) + before.chmod(0o755) + result, output = self.prepare(project, data, before=before) + self.assert_rejected(result, output) + + def test_unborn_repository_copies_folder_without_git_or_nested_state(self): + project = self.tmp / "unborn" + project.mkdir() + self.git(project, "init", "-q") + write_files( + project, + { + TARGET: AFTER, + "notes.md": b"Untracked notes.\n", + "bin/check": (b"#!/bin/sh\n", True), + }, + ) + nested_home = project / ".bd-home" + self.env["BEHAVIOR_DIFF_HOME"] = str(nested_home) + self.store_home = nested_home + self.store_entry(project / TARGET, "20260101-000000-1", BEFORE, 1000) + data = self.folder("data", SHARED) + original = self.original_state(project, data) + stage = self.staged(project, data) + expected = normalized( + { + TARGET: BEFORE, + "notes.md": b"Untracked notes.\n", + "bin/check": (b"#!/bin/sh\n", True), + **SHARED, + } + ) + self.assertEqual(self.archive_files(stage / "project"), expected) + self.assertEqual(self.original_state(project, data), original) + + def test_copy_cap_rejects_oversized_inputs(self): + big = b"x" * 4096 + small = {TARGET: AFTER} + env = dict(self.env, BEHAVIOR_DIFF_COPY_CAP_KB="2") + cases = ( + ( + "git", + self.git_project("git", {**COMMITTED, "big.txt": big}), + SHARED, + None, + ), + ( + "folder", + self.folder("folder", {**small, "big.txt": big}), + SHARED, + BEFORE, + ), + ("shared", self.folder("small", small), {"data/big.txt": big}, BEFORE), + ) + for label, project, shared, before in cases: + with self.subTest(label): + data = self.folder(label + "-data", shared) + before_file = None + if before is not None: + before_file = self.tmp / f"{label}-before.md" + before_file.write_bytes(before) + result, output = self.prepare( + project, data, before=before_file, env=env + ) + self.assert_rejected(result, output) + + def test_unsafe_target_paths_are_rejected(self): + project, data = self.standard() + (project / "link.md").symlink_to(TARGET) + (project / "linkdir").symlink_to("docs") + original = self.original_state(project, data) + for target in ( + "", + ".", + "./CLAUDE.md", + "../project/CLAUDE.md", + str(project / TARGET), + "CLAUDE.md/", + "missing.md", + "docs", + "link.md", + "linkdir/guide.md", + ".git/config", + ): + with self.subTest(target=target): + result, output = self.prepare(project, data, file=target) + self.assert_rejected(result, output) + self.assertEqual(self.original_state(project, data), original) + + def test_unsafe_shared_data_is_rejected(self): + project, _ = self.standard() + outside = self.tmp / "outside.txt" + outside.write_bytes(b"outside\n") + cases = { + "symlink": lambda root: (root / "link.csv").symlink_to(outside), + "fifo": lambda root: os.mkfifo(root / "pipe"), + "git internals": lambda root: write_files( + root, {".git/config": b"[core]\n"} + ), + "selected instruction": lambda root: write_files( + root, {TARGET: b"shadow\n"} + ), + "instruction case alias": lambda root: write_files( + root, {"claude.md": b"x\n"} + ), + "excluded path": lambda root: write_files( + root, {"notes/purpose.json": b"{}\n"} + ), + "excluded ancestor": lambda root: write_files(root, {"notes": b"file\n"}), + "directory collision": lambda root: write_files(root, {"docs": b"file\n"}), + "file ancestor collision": lambda root: write_files( + root, {"run.sh/x": b"x\n"} + ), + "case collision": lambda root: write_files(root, {"Docs/guide.md": b"x\n"}), + "empty": lambda root: None, + } + for label, build in cases.items(): + with self.subTest(label): + data = self.tmp / ("data-" + label.replace(" ", "-")) + data.mkdir() + if label != "empty": + write_files(data, {"data/records.csv": b"id\n1\n"}) + build(data) + result, output = self.prepare(project, data, exclude=PRIVATE) + self.assert_rejected(result, output) + real = self.folder("real-data", SHARED) + (self.tmp / "data-link").symlink_to(real) + result, output = self.prepare(project, self.tmp / "data-link", exclude=PRIVATE) + self.assert_rejected(result, output) + + def test_shared_data_cannot_replace_nested_instruction_ancestry(self): + target = "skills/demo/SKILL.md" + project = self.git_project( + "nested", {target: BEFORE, "skills/demo/ref.md": b"ref\n"}, {target: AFTER} + ) + for label, shared in ( + ("ancestor file", {"skills": b"file\n"}), + ("instruction as directory", {target + "/x": b"x\n"}), + ): + with self.subTest(label): + data = self.folder("data-" + label.replace(" ", "-"), shared) + result, output = self.prepare(project, data, file=target) + self.assert_rejected(result, output) + data = self.folder("data-ok", {"skills/demo/fixture.json": b"{}\n"}) + stage = self.staged(project, data, file=target) + archived = self.archive_files(stage / "project") + self.assertEqual(archived[target], (BEFORE, False)) + self.assertEqual(archived["skills/demo/fixture.json"], (b"{}\n", False)) + + def test_unsafe_project_inputs_are_rejected(self): + data = self.folder("data", SHARED) + before = self.tmp / "before.md" + before.write_bytes(BEFORE) + linked = self.git_project("linked", {**COMMITTED, "alias.md": b""}) + (linked / "alias.md").unlink() + (linked / "alias.md").symlink_to(TARGET) + self.git(linked, "add", "alias.md") + self.git(linked, "commit", "-qm", "Synthetic symlink") + submodule = self.git_project("submodule") + head = self.git(submodule, "rev-parse", "HEAD").decode().strip() + self.git( + submodule, + "update-index", + "--add", + "--cacheinfo", + f"160000,{head},vendor/lib", + ) + self.git(submodule, "commit", "-qm", "Synthetic gitlink") + folder_link = self.folder("folder-link", {TARGET: AFTER}) + (folder_link / "alias.md").symlink_to(TARGET) + folder_fifo = self.folder("folder-fifo", {TARGET: AFTER}) + os.mkfifo(folder_fifo / "pipe") + nested_git = self.folder( + "nested-git", {TARGET: AFTER, "vendor/.git/HEAD": b"ref\n"} + ) + cases = ( + ("tracked symlink", linked, None, TARGET), + ("submodule", submodule, None, TARGET), + ("folder symlink", folder_link, before, TARGET), + ("folder fifo", folder_fifo, before, TARGET), + ("nested git", nested_git, before, TARGET), + ("repository subdirectory", self.git_project(), None, "docs/guide.md"), + ) + for label, project, before_file, target in cases: + with self.subTest(label): + if label == "repository subdirectory": + project, target = project / "docs", "guide.md" + (project / target).write_bytes(b"changed\n") + result, output = self.prepare( + project, data, file=target, before=before_file + ) + self.assert_rejected(result, output) + + def test_output_location_must_be_new_and_outside_inputs(self): + project, data = self.standard() + existing = self.tmp / "existing" + existing.mkdir() + (existing / "keep.txt").write_bytes(b"keep\n") + dangling = self.tmp / "dangling" + dangling.symlink_to(self.tmp / "nowhere") + original = self.original_state(project, data) + for label, output in ( + ("inside project", project / "stage"), + ("inside data", data / "stage"), + ("missing parent", self.tmp / "missing" / "stage"), + ): + with self.subTest(label): + result, _ = self.prepare(project, data, output=output, exclude=PRIVATE) + self.assert_rejected(result, output) + for label, output in (("existing", existing), ("dangling symlink", dangling)): + with self.subTest(label): + entries = sorted(self.tmp.iterdir()) + result, _ = self.prepare(project, data, output=output, exclude=PRIVATE) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(sorted(self.tmp.iterdir()), entries) + self.assertEqual([path.name for path in existing.iterdir()], ["keep.txt"]) + self.assertEqual((existing / "keep.txt").read_bytes(), b"keep\n") + self.assertTrue(dangling.is_symlink()) + self.assertEqual(self.original_state(project, data), original) + + inside = project / "shared" + write_files(inside, SHARED) + result, output = self.prepare(project, inside) + self.assert_rejected(result, output) + result, output = self.prepare(project, self.tmp) + self.assert_rejected(result, output) + + def test_output_containment_rejects_case_aliases(self): + project, data = self.standard() + original = self.original_state(project, data) + for root in (project, data): + alias = root.with_name(root.name.swapcase()) + if not alias.exists() or not alias.samefile(root): + self.skipTest("requires a case-insensitive filesystem") + result, output = self.prepare( + project, data, output=alias / "stage", exclude=PRIVATE + ) + self.assert_rejected(result, output) + self.assertEqual(self.original_state(project, data), original) + + def test_exclusions_remove_private_context_but_never_the_instruction(self): + project, data = self.standard() + stage = self.staged(project, data, exclude=["notes"]) + self.assertEqual(self.archive_files(stage / "project"), BASELINE) + for exclude in ( + [TARGET], + ["../notes"], + ["/notes"], + ["notes/../CLAUDE.md"], + ["."], + ): + with self.subTest(exclude=exclude): + result, output = self.prepare(project, data, exclude=exclude) + self.assert_rejected(result, output) + alias = self.folder("alias-data", {"Notes/answer.md": b"Synthetic answer.\n"}) + result, output = self.prepare(project, alias, exclude=["notes"]) + self.assert_rejected(result, output) + + def test_verify_rejects_changes_after_freeze(self): + def edit(name, data): + return lambda stage: (stage / "project" / name).write_bytes(data) + + def commit(stage): + self.git(stage / "project", "commit", "-q", "--allow-empty", "-m", "later") + + def attributes(stage): + info = stage / "project/.git/info" + info.mkdir(parents=True, exist_ok=True) + (info / "attributes").write_text("data/** export-ignore\n") + + tampers = { + "shared content": edit("data/records.csv", b"id\n9\n"), + "shared mode": lambda stage: (stage / "project/data/records.csv").chmod( + 0o755 + ), + "shared removed": lambda stage: (stage / "project/data/tool.sh").unlink(), + "instruction": edit(TARGET, b"# Different After\n"), + "extra file": edit("extra.txt", b"extra\n"), + "new commit": commit, + "archive attributes": attributes, + "worktree redirect": lambda stage: self.git( + stage / "project", "config", "core.worktree", str(self.tmp) + ), + "before copy": lambda stage: (stage / "before").write_bytes(b"tampered\n"), + "before copy removed": lambda stage: (stage / "before").unlink(), + "freeze record removed": lambda stage: (stage / "freeze.json").unlink(), + } + project, data = self.standard() + for label, tamper in tampers.items(): + with self.subTest(label): + stage = self.staged(project, data, exclude=PRIVATE) + self.assertEqual(self.verify(stage).returncode, 0) + tamper(stage) + self.assertNotEqual(self.verify(stage).returncode, 0) + self.assertNotEqual(self.verify(self.tmp / "no-stage").returncode, 0) + + def test_verify_rejects_changed_absent_before(self): + committed = {name: data for name, data in COMMITTED.items() if name != TARGET} + project = self.git_project(committed=committed) + data = self.folder("data", SHARED) + self.store_entry(project / TARGET, "20260101-000000-ABSENT", b"", 1000) + + def marker_dir(stage): + return ( + stage + / "home/baselines" + / encoded((stage / "project").resolve() / TARGET) + ) + + tampers = { + "marker removed": lambda stage: next(marker_dir(stage).iterdir()).unlink(), + "newer baseline": lambda stage: ( + marker_dir(stage) / "20990101-000000-1" + ).write_bytes(BEFORE), + "before copy added": lambda stage: (stage / "before").write_bytes(BEFORE), + "instruction committed": lambda stage: ( + self.git(stage / "project", "add", "-f", TARGET), + self.git(stage / "project", "commit", "-qm", "later"), + ), + } + for label, tamper in tampers.items(): + with self.subTest(label): + stage = self.staged(project, data, exclude=PRIVATE) + self.assertNotIn(TARGET, self.archive_files(stage / "project")) + self.assertEqual(self.verify(stage).returncode, 0) + tamper(stage) + self.assertNotEqual(self.verify(stage).returncode, 0) + + +FAKE_HOST = r""" +import json, os, shlex, sys +from pathlib import Path + +host, args = sys.argv[1], sys.argv[2:] + + +def option(name): + return args[args.index(name) + 1] if name in args else None + + +def record(kind, model): + entry = {"host": host, "kind": kind, "model": model, "cwd": os.getcwd(), + "trial": os.environ.get("BEHAVIOR_DIFF_TRIAL")} + with open(os.environ["FAKE_HOST_LOG"], "a", encoding="utf-8") as log: + log.write(json.dumps(entry) + "\n") + + +def project_files(): + # Read what is on disk: the runner's trial commit omits gitignored files. + files = [] + for current, dirs, names in os.walk("."): + if current == ".": + dirs[:] = [name for name in dirs if name != ".git"] + for name in names: + path = Path(current, name) + name = path.as_posix().removeprefix("./") + files.append((name, path.read_text(encoding="utf-8"))) + return sorted(files) + + +def emit(event): + sys.stdout.write(json.dumps(event) + "\n") + + +def claude_trial(): + record("trial", option("--model")) + emit({"type": "system", "subtype": "init", "session_id": "synthetic", + "model": option("--model")}) + files = project_files() + for number, (name, text) in enumerate(files, 1): + call = f"toolu_synthetic_{number}" + emit({"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": call, "name": "Read", + "input": {"file_path": name}}]}}) + emit({"type": "user", "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": call, "content": text}]}}) + answer = f"Synthetic answer after reading {len(files)} files." + emit({"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "text", "text": answer}]}}) + emit({"type": "result", "subtype": "success", "is_error": False, + "result": answer}) + + +def codex_trial(): + record("trial", option("-m")) + emit({"type": "thread.started", "thread_id": "synthetic"}) + emit({"type": "turn.started"}) + files = project_files() + for number, (name, text) in enumerate(files, 1): + command = "/bin/bash -lc " + shlex.quote("cat " + shlex.quote(name)) + emit({"type": "item.completed", "item": { + "id": f"item_{number}", "type": "command_execution", + "command": command, "aggregated_output": text, "exit_code": 0, + "status": "completed"}}) + emit({"type": "item.completed", "item": { + "id": "item_final", "type": "agent_message", + "text": f"Synthetic answer after reading {len(files)} files."}}) + emit({"type": "turn.completed", "usage": { + "input_tokens": 0, "cached_input_tokens": 0, "output_tokens": 0}}) + + +def main(): + if host == "claude": + if "--output-format" in args: + claude_trial() + return 0 + record("extract", option("--model")) + sys.stdout.write("Synthetic extraction reply; intentionally not JSON.\n") + return 0 + if args[:1] == ["exec"] and "--json" in args: + codex_trial() + return 0 + if args[:1] == ["exec"] and "-o" in args: + sys.stdin.read() + record("extract", option("-m")) + Path(option("-o")).write_text('{"synthetic": "malformed', encoding="utf-8") + return 0 + sys.stderr.write("fake codex: unexpected invocation\n") + return 9 + + +raise SystemExit(main()) +""" + + +class RunnerSmokeTests(Fixtures, unittest.TestCase): + """Manual: the shipped runner and adapters consume a prepared stage.""" + + enabled = False + + @classmethod + def setUpClass(cls): + if not cls.enabled: + raise unittest.SkipTest("manual smoke; pass --runner-smoke") + + def setUp(self): + super().setUp() + fakebin = self.tmp / "fakebin" + fakebin.mkdir() + host = self.tmp / "fake_host.py" + host.write_text(FAKE_HOST, encoding="utf-8") + for name in ("claude", "codex"): + launcher = fakebin / name + launcher.write_text( + f'#!/bin/sh\nexec "{sys.executable}" "{host}" {name} "$@"\n' + ) + launcher.chmod(0o755) + opener = fakebin / "open" # the runner opens the report on success + opener.write_text("#!/bin/sh\nexit 0\n") + opener.chmod(0o755) + self.log = self.tmp / "fake-host.jsonl" + self.env["PATH"] = f"{fakebin}{os.pathsep}{self.env['PATH']}" + self.env["FAKE_HOST_LOG"] = str(self.log) + context = self.tmp / "context" + context.mkdir() + self.source = context / "source.json" + self.source.write_text(json.dumps(SOURCE), encoding="utf-8") + + def run_stage(self, stage, agent, purpose, fast): + self.assertEqual(self.verify(stage).returncode, 0) + argv = [ + "bash", + str(RUNNER), + "--file", + TARGET, + "--task", + "Read every project file, then summarize the records using the rules.", + "--agent", + agent, + "--model", + TRIAL_MODEL, + "--extract-model", + EXTRACT_MODEL, + "--purpose-file", + str(purpose), + "--source-file", + str(self.source), + ] + if (stage / "before").exists(): + argv += ["--before-file", str(stage / "before")] + if fast: + argv.append("--fast") + env = dict(self.env, BEHAVIOR_DIFF_HOME=str(stage / "home")) + result = subprocess.run( + argv, + cwd=stage / "project", + env=env, + capture_output=True, + text=True, + timeout=900, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.verify(stage).returncode, 0) + + def check_run(self, stage, agent, trials, before, purpose): + runs = sorted((stage / "home/runs").iterdir()) + self.assertEqual(len(runs), 1) + run = runs[0] + report = (run / "report.html").read_text(encoding="utf-8") + config_text = (run / "config.json").read_text(encoding="utf-8") + config = json.loads(config_text) + self.assertEqual(config["source"], SOURCE) + expected_purpose = subprocess.run( + [sys.executable, str(SCRIPTS / "purpose.py"), str(purpose)], + env=self.env, + check=True, + capture_output=True, + text=True, + ).stdout + self.assertEqual(config["purpose"], json.loads(expected_purpose)) + scratch = self.git(stage / "project", "rev-parse", "HEAD").decode().strip() + for text in (config_text, report): + self.assertNotIn(scratch, text) + self.assertNotIn(f"commit {scratch[:7]}", text) + if before is not None: + self.assertEqual(config["before_label"], str(stage / "before")) + + common = { + name: content + for name, content in self.archive_files(stage / "project").items() + if name != TARGET + } + self.assertEqual( + common["data/records.csv"], normalized(SHARED)["data/records.csv"] + ) + self.assertEqual(common["data/tool.sh"], normalized(SHARED)["data/tool.sh"]) + for private in PRIVATE: + self.assertNotIn(private, common) + for variant in ("before", "after"): + for number in range(1, trials + 1): + with self.subTest(trial=f"{variant}-{number}"): + trial = run / f"{variant}-{number}" + # The runner's trial commit uses `git add -A`, so gitignored + # shared files arrive on disk but not in that commit. + snapshot = self.working_files(trial / "project") + committed = self.archive_files(trial / "project") + self.assertLessEqual(committed.items(), snapshot.items()) + self.assertEqual(committed.get(TARGET), snapshot.get(TARGET)) + self.assertEqual( + { + name: content + for name, content in snapshot.items() + if name != TARGET + }, + common, + ) + expected = AFTER if variant == "after" else before + if expected is None: + self.assertNotIn(TARGET, snapshot) + else: + self.assertEqual(snapshot[TARGET][0], expected) + results, finals = trace_events(trial / "trace.jsonl") + self.assertEqual( + sorted(results), + sorted(data.decode() for data, _ in snapshot.values()), + ) + self.assertTrue(any(finals)) + grades = (run / "grades.tsv").read_text().splitlines() + self.assertEqual(len(grades), 2 * trials) + self.assertTrue(all(line.split("\t")[1] == "REVIEW" for line in grades)) + + calls = [json.loads(line) for line in self.log.read_text().splitlines()] + launched = [call for call in calls if call["kind"] == "trial"] + extracted = [call for call in calls if call["kind"] == "extract"] + self.assertEqual(len(launched), 2 * trials) + self.assertTrue(extracted) + self.assertTrue(all(call["host"] == agent for call in calls)) + for call in launched: + self.assertEqual(call["model"], TRIAL_MODEL) + self.assertEqual(call["trial"], "1") + self.assertTrue(Path(call["cwd"]).resolve().is_relative_to(run)) + self.assertTrue(all(call["model"] == EXTRACT_MODEL for call in extracted)) + + def smoke(self, agent, trials, before_file=None, absent=False): + committed = COMMITTED + if absent: + committed = { + name: data for name, data in COMMITTED.items() if name != TARGET + } + project = self.git_project(committed=committed) + data = self.folder("data", SHARED) + if absent: + self.store_entry(project / TARGET, "20260101-000000-ABSENT", b"", 1000) + original = self.original_state(project, data, self.store_home) + stage = self.staged(project, data, before=before_file, exclude=PRIVATE) + purpose = project / "notes/purpose.json" + self.run_stage(stage, agent, purpose, fast=trials == 1) + before = ( + None if absent else (before_file.read_bytes() if before_file else BEFORE) + ) + self.check_run(stage, agent, trials, before, purpose) + self.assertEqual(self.original_state(project, data, self.store_home), original) + + def test_unsafe_git_environment_blocks_runner_handoff(self): + project, data = self.standard() + stage = self.staged(project, data, exclude=PRIVATE) + original = self.original_state(project, data) + masked = self.tmp / "masked-gitconfig" + masked.write_text( + Path(self.env["GIT_CONFIG_GLOBAL"]).read_text() + + "\n[tar]\n\tumask = 0111\n" + ) + overrides = ( + {"GIT_INDEX_FILE": str(project / ".git/index")}, + { + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "core.worktree", + "GIT_CONFIG_VALUE_0": str(project), + }, + {"GIT_CONFIG_GLOBAL": str(masked)}, + ) + for override in overrides: + with self.subTest(override=next(iter(override))): + env = dict(self.env, **override, BEHAVIOR_DIFF_HOME=str(stage / "home")) + result = subprocess.run( + [ + "bash", + "-c", + '"$1" "$2" verify "$3" && cd "$3/project" && ' + 'bash "$4" --file CLAUDE.md --before-file "$3/before" ' + '--task "Read the shared data." --agent claude ' + "--model synthetic-trial-model --fast", + "staged-smoke", + sys.executable, + str(HELPER), + str(stage), + str(RUNNER), + ], + env=env, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 2) + self.assertFalse(self.log.exists(), "unsafe setup launched a host") + self.assertFalse((stage / "home/runs").exists()) + self.assertEqual(self.original_state(project, data), original) + + def test_claude_three_by_three_head_baseline(self): + self.smoke("claude", 3) + + def test_codex_three_by_three_head_baseline(self): + self.smoke("codex", 3) + + def test_claude_fast_absent_before(self): + self.smoke("claude", 1, absent=True) + + def test_codex_fast_explicit_before(self): + literal = self.tmp / "CLAUDE.md-ABSENT" + literal.write_bytes(EXPLICIT) + self.smoke("codex", 1, before_file=literal) + + +def main(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--runner-smoke", action="store_true") + options, remaining = parser.parse_known_args() + RunnerSmokeTests.enabled = options.runner_smoke + unittest.main(argv=[sys.argv[0], *remaining]) + + +if __name__ == "__main__": + main() From 2e49baa409d18cec1c16c6dde22e656297f077dc Mon Sep 17 00:00:00 2001 From: Kent Huang Date: Mon, 19 Oct 2026 14:58:42 +0800 Subject: [PATCH 2/2] fix: isolate nested comparison storage per trial Signed-off-by: Kent Huang --- docs/architecture.md | 6 ++ .../behavior-diff/scripts/prepare_shared.py | 2 +- .../skills/behavior-diff/scripts/run-trial.sh | 4 ++ tests/shared-data-test.py | 59 ++++++++++++++++++- 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 38aa7fb..38f8792 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -239,6 +239,12 @@ instruction versions without asking a model to rewrite either version. the shared `task.md`, and the selected model. The runner starts all trials as background processes and waits before analysis. +`run-trial.sh` gives every host a trial-local `BEHAVIOR_DIFF_HOME` at +`/behavior-diff-home`, outside `project/`. Nested comparisons and hook +baselines cannot populate the orchestrator's run directory or another trial's +store through the inherited variable. This applies to Claude Code, Codex, Pi, +and OMP; it isolates storage defaults, not filesystem access. + Each side contains `N` independent copies, not one shared directory: ```text diff --git a/plugin/skills/behavior-diff/scripts/prepare_shared.py b/plugin/skills/behavior-diff/scripts/prepare_shared.py index 1604448..b6966be 100755 --- a/plugin/skills/behavior-diff/scripts/prepare_shared.py +++ b/plugin/skills/behavior-diff/scripts/prepare_shared.py @@ -265,7 +265,7 @@ def prepare(args): raise PreparationError("the selected instruction cannot be excluded") after = safe_file(project, target) home = Path( - os.environ.get("BEHAVIOR_DIFF_HOME", str(Path.home() / ".behavior-diff")) + os.environ.get("BEHAVIOR_DIFF_HOME") or str(Path.home() / ".behavior-diff") ).resolve() before_file = None if args.before_file: diff --git a/plugin/skills/behavior-diff/scripts/run-trial.sh b/plugin/skills/behavior-diff/scripts/run-trial.sh index d1feaf4..1a85e28 100755 --- a/plugin/skills/behavior-diff/scripts/run-trial.sh +++ b/plugin/skills/behavior-diff/scripts/run-trial.sh @@ -76,6 +76,10 @@ cd "$dir" # trial — otherwise a user-scope install would nudge itself recursively. export BEHAVIOR_DIFF_TRIAL=1 +# A skill may start another comparison. Keep its runs and baselines out of the +# orchestrator's home and other trials, including when both share the same HOME. +export BEHAVIOR_DIFF_HOME="$trace_dir/behavior-diff-home" + normalize_pi_omp_json() { # $1 = raw Pi or OMP JSONL local raw=$1 jq -c ' diff --git a/tests/shared-data-test.py b/tests/shared-data-test.py index fc8f147..60aabd3 100755 --- a/tests/shared-data-test.py +++ b/tests/shared-data-test.py @@ -460,6 +460,19 @@ def test_before_file_inside_project_is_excluded_from_shared_baseline(self): self.assertEqual(archived[TARGET], (EXPLICIT, False)) self.assertNotIn("drafts/old-CLAUDE.md", self.working_files(stage / "project")) + def test_empty_home_override_uses_default_hook_baseline(self): + project = self.folder("project", {TARGET: AFTER}) + data = self.folder("data", SHARED) + self.store_home = self.home / ".behavior-diff" + self.store_entry(project / TARGET, "20260101-000000", BEFORE, 1000) + self.env["BEHAVIOR_DIFF_HOME"] = "" + original = self.original_state(project, data, self.store_home) + stage = self.staged(project, data) + self.assertEqual((stage / "before").read_bytes(), BEFORE) + self.assertEqual(self.archive_files(stage / "project")[TARGET][0], BEFORE) + self.assertEqual((stage / "project" / TARGET).read_bytes(), AFTER) + self.assertEqual(self.original_state(project, data, self.store_home), original) + def test_hook_baseline_uses_newest_visible_entry_under_percent_encoded_key(self): project = self.folder( "pct%dir", {TARGET: AFTER, "notes.md": b"Synthetic notes.\n"} @@ -851,7 +864,7 @@ def marker_dir(stage): FAKE_HOST = r""" -import json, os, shlex, sys +import json, os, shlex, subprocess, sys, tempfile from pathlib import Path host, args = sys.argv[1], sys.argv[2:] @@ -885,7 +898,30 @@ def emit(event): sys.stdout.write(json.dumps(event) + "\n") +def nested_comparison(): + runner = os.environ.get("FAKE_NESTED_RUNNER") + if not runner: + return + # Exercise real nested runner allocation, not just echo its environment. + with tempfile.TemporaryDirectory(dir=Path.cwd().parent) as temporary: + root = Path(temporary) + project = root / "project" + project.mkdir() + (project / "CLAUDE.md").write_text("Synthetic new rule.\n") + before = root / "before.md" + before.write_text("Synthetic old rule.\n") + env = dict(os.environ) + del env["FAKE_NESTED_RUNNER"] + env["FAKE_HOST_LOG"] = str(root / "nested-host.jsonl") + subprocess.run( + ["bash", runner, "--file", "CLAUDE.md", "--before-file", str(before), + "--task", "Read the rules.", "--agent", host, "--model", "synthetic", + "--extract-model", "synthetic", "--fast"], + cwd=project, env=env, check=True, capture_output=True, timeout=120) + + def claude_trial(): + nested_comparison() record("trial", option("--model")) emit({"type": "system", "subtype": "init", "session_id": "synthetic", "model": option("--model")}) @@ -905,6 +941,7 @@ def claude_trial(): def codex_trial(): + nested_comparison() record("trial", option("-m")) emit({"type": "thread.started", "thread_id": "synthetic"}) emit({"type": "turn.started"}) @@ -1114,6 +1151,26 @@ def smoke(self, agent, trials, before_file=None, absent=False): self.check_run(stage, agent, trials, before, purpose) self.assertEqual(self.original_state(project, data, self.store_home), original) + def nested_smoke(self, agent): + self.env["FAKE_NESTED_RUNNER"] = str(RUNNER) + self.smoke(agent, 1) + stage = self.tmp / "stage-1" + (outer,) = (stage / "home/runs").iterdir() + for side in ("before", "after"): + trial = outer / f"{side}-1" + (nested,) = (trial / "behavior-diff-home/runs").iterdir() + self.assertTrue((nested / "report.html").is_file()) + self.assertEqual( + (nested / "grades.tsv").read_text().splitlines(), + ["before-1\tREVIEW\t-", "after-1\tREVIEW\t-"], + ) + + def test_claude_nested_comparisons_preserve_outer_evidence(self): + self.nested_smoke("claude") + + def test_codex_nested_comparisons_preserve_outer_evidence(self): + self.nested_smoke("codex") + def test_unsafe_git_environment_blocks_runner_handoff(self): project, data = self.standard() stage = self.staged(project, data, exclude=PRIVATE)