From 8b0749e71bcba51adfd2ad3a4ab8497b6810a449 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 07:52:07 +0800 Subject: [PATCH 001/155] Check Windows core and store writes on x86 and ARM runners --- .github/workflows/gate.yml | 19 +++ spec/81-engineering.md | 4 +- spec/cards/assurance.md | 2 +- spec/spec.lock.json | 223 ++++++++++++++------------- spec/status.md | 6 +- tools/ci/tests/integration/stages.rs | 15 ++ 6 files changed, 158 insertions(+), 111 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 05b532426..afe8cec25 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -47,6 +47,25 @@ permissions: contents: read jobs: + windows: + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + strategy: + fail-fast: false + matrix: + include: + - runner: windows-2025 + target: x86_64-pc-windows-msvc + - runner: windows-11-arm + target: aarch64-pc-windows-msvc + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - run: rustup target add ${{ matrix.target }} + - run: cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration an_open_directory_handle_syncs_its_entries + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-policy --test integration an_audit_append_syncs_its_directories + - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_ledger_append_syncs_its_directory + dispatch: if: ${{ vars.FLAREDISPATCH_ENDPOINT != '' && github.event.pull_request.head.repo.full_name == github.repository }} runs-on: ubuntu-latest diff --git a/spec/81-engineering.md b/spec/81-engineering.md index fdb9e2221..bd4e72a44 100644 --- a/spec/81-engineering.md +++ b/spec/81-engineering.md @@ -137,7 +137,7 @@ Target directories, the engine-linked invocation, linked query functions, build - `container-image` — The repository's `Dockerfile` builds one profile, `contextful-full` unless `PROFILE` names another, as a static `linux/amd64` binary, and ships it in a shell-free runtime image as a non-root user over a declared store volume. - `targets` — All three profiles cross-compile to `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`; edge and full also build for `aarch64-apple-darwin` and `x86_64-apple-darwin`. *A-assurance* -- `windows-target` — Edge and full also build for `x86_64-pc-windows-msvc`. +- `windows-target` — Edge and full also build for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`. - `wasi-probe` — No profile ships a `wasm32-wasip2` release; a scheduled-tier ledger entry builds the edge profile for it and holds the compressed artifact to {{assurance.gate.edge-budget}}. *A-assurance* - `release-artifact` — Each profile ships a release archive with a SHA-256 checksum and an SBOM, a package-manager formula and an independently tagged container image; the bare formula name and the install script resolve to the full profile. @@ -155,6 +155,8 @@ Stage order, secrets of record, the crate-graph, row-token, egress and dependenc *A-assurance* - `remote-check` — The pull-request workflow dispatches every stage the gate subcommand defines to a remote runner, a split stage one part at a time, each as one status check labelled with its name. *A-assurance* +- `windows-native` — The pull-request workflow checks core, filesystem, policy and store-write packages and exercises directory sync on native x86 and ARM Windows runners. + *because native runners provide the target C toolchain and execute directory flushes that cross-compilation cannot verify* - `workspace-parts` — Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups. *A-assurance* - `remote-predecessors` — Each remote check runs its stage together with every predecessor whose output that stage reads, so no check reads another check's sandbox. diff --git a/spec/cards/assurance.md b/spec/cards/assurance.md index cedd57e6b..90f06a602 100644 --- a/spec/cards/assurance.md +++ b/spec/cards/assurance.md @@ -16,7 +16,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/a | `automate` | Typed subcommands over compiled binaries, and the boundary shell keeps. | 5 | | `test` | Assertion construction, guard validation, suite placement, test-first and acceptance tests, and the connector test kit. | 13 | | `build` | Target directories, the engine-linked invocation, linked query functions, build targets, release artifacts and the dependency allowlist. | 15 | -| `gate` | Stage order, secrets of record, the crate-graph, row-token, egress and dependency rules, the formal stage, the container's ceilings, disk, footprint budgets and surface checks. | 33 | +| `gate` | Stage order, secrets of record, the crate-graph, row-token, egress and dependency rules, the formal stage, the container's ceilings, disk, footprint budgets and surface checks. | 34 | | `evaluate` | The read-path quality harness: case format, deterministic metrics and floors, judged dimensions, the reader and the judge. | 29 | | `baseline` | Holding a run against committed baselines and absolute floors, intervals, golden custody and case generation. | 23 | | `measure` | The target ledger: each tracked target, the clause it serves, how it is measured and its record per commit. | 16 | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 9761ac0ef..51e5d857d 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -20346,7 +20346,7 @@ "operation": "build", "subject": "windows-target", "kind": "behavior", - "statement": "Edge and full also build for `x86_64-pc-windows-msvc`.", + "statement": "Edge and full also build for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`.", "why": "", "file": "spec/81-engineering.md", "line": 140 @@ -20428,6 +20428,17 @@ "file": "spec/81-engineering.md", "line": 156 }, + { + "id": "assurance.gate.windows-native", + "contract": "assurance", + "operation": "gate", + "subject": "windows-native", + "kind": "behavior", + "statement": "The pull-request workflow checks core, filesystem, policy and store-write packages and exercises directory sync on native x86 and ARM Windows runners.", + "why": "because native runners provide the target C toolchain and execute directory flushes that cross-compilation cannot verify", + "file": "spec/81-engineering.md", + "line": 158 + }, { "id": "assurance.gate.workspace-parts", "contract": "assurance", @@ -20437,7 +20448,7 @@ "statement": "Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 158 + "line": 160 }, { "id": "assurance.gate.remote-predecessors", @@ -20448,7 +20459,7 @@ "statement": "Each remote check runs its stage together with every predecessor whose output that stage reads, so no check reads another check's sandbox.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 160 + "line": 162 }, { "id": "assurance.gate.fork-dispatch", @@ -20459,7 +20470,7 @@ "statement": "The pull-request workflow dispatches only a head commit pushed to the repository itself; a pull request from a fork dispatches no stage and so carries none of the required checks.", "why": "because a dispatch carries the org's signing secret and runs the commit on the org's runner, and an absent required check fails closed", "file": "spec/81-engineering.md", - "line": 162 + "line": 164 }, { "id": "assurance.gate.stage-reports", @@ -20470,7 +20481,7 @@ "statement": "Each stage prints the environment it leaves and its memory limit, peak and event counts, and a failing stage prints its diagnostics before propagating its exit code.", "why": "because memory exhaustion is silent, and a kill then reads as a number in the log", "file": "spec/81-engineering.md", - "line": 164 + "line": 166 }, { "id": "assurance.gate.pins-stage", @@ -20481,7 +20492,7 @@ "statement": "The pins stage resolves every pinned artifact identity a run depends on before any compilation.", "why": "", "file": "spec/81-engineering.md", - "line": 166 + "line": 168 }, { "id": "assurance.gate.evaluate-stage", @@ -20492,7 +20503,7 @@ "statement": "The evaluate stage runs every gate-tier ledger entry and the native case set in the deterministic tier, and reports the floor and baseline verdicts.", "why": "", "file": "spec/81-engineering.md", - "line": 167 + "line": 169 }, { "id": "assurance.gate.schema-stage", @@ -20503,7 +20514,7 @@ "statement": "The schema stage regenerates each derived artifact into a scratch location, compares it byte for byte against the committed copy, and runs `contextful-spec lint`.", "why": "", "file": "spec/81-engineering.md", - "line": 168 + "line": 170 }, { "id": "assurance.gate.secret-ciphertext", @@ -20514,7 +20525,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file other than `.env.example` holding a value without the `encrypted:` prefix, or a tracked `.env.keys`, raises `SecretPlaintext`, naming file and key.", "why": "because a tracked file reaches every clone, and only dotenvx ciphertext is safe there while its private key stays untracked", "file": "spec/81-engineering.md", - "line": 169 + "line": 171 }, { "id": "assurance.gate.secret-scope", @@ -20525,7 +20536,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file whose block — the assignments above it, up to the first blank line — opens with no comment carrying text raises `SecretScopeMissing`, naming file and key.", "why": "because a provider token cannot report its own grants, so the comment above it is the one record of what it reaches", "file": "spec/81-engineering.md", - "line": 171 + "line": 173 }, { "id": "assurance.gate.crate-graph", @@ -20536,7 +20547,7 @@ "statement": "The crate-graph stage holds run-path crates to reaching read-path crates through the three crossing crates alone; another edge raises `CrateGraphViolation`, naming both crates.", "why": "P5", "file": "spec/81-engineering.md", - "line": 173 + "line": 175 }, { "id": "assurance.gate.locked-resolve", @@ -20547,7 +20558,7 @@ "statement": "The crate-graph stage resolves every graph `--locked` against the committed `Cargo.lock`, so a lock behind its manifests, or one rewritten before the stage starts, fails the stage before any rule runs; the stage never rewrites it.", "why": "because a rule held over a graph the lock file does not record passes a build that resolves another", "file": "spec/81-engineering.md", - "line": 175 + "line": 177 }, { "id": "assurance.gate.row-token", @@ -20558,7 +20569,7 @@ "statement": "Every row-returning function of the store crate takes the enforcement token type, whose constructor is private to the enforcement module; a row path outside a registered relation raises `EnforceUnmediatedPath`, naming the function.", "why": "P5", "file": "spec/81-engineering.md", - "line": 177 + "line": 179 }, { "id": "assurance.gate.dependency-deny", @@ -20569,7 +20580,7 @@ "statement": "The crate-graph stage runs pinned cargo-deny `check bans` over each profile's resolved graph, denying every crate a profile-wide dependency refusal names; a hit raises that refusal's error, naming the profile and the path.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 179 + "line": 181 }, { "id": "assurance.gate.deny-profile-graph", @@ -20580,7 +20591,7 @@ "statement": "A profile's graph is the binary resolved with default features off and the profile's feature on, across every target, excluding development dependencies; the stage prints each profile the binary declares no feature for.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 181 + "line": 183 }, { "id": "assurance.gate.deny-list", @@ -20591,7 +20602,7 @@ "statement": "`deny.toml` holds one `[bans]` entry per crate or feature a profile-wide dependency refusal names, its reason that refusal's clause id; an entry missing or extra raises `DenyListDrift`, naming it.", "why": "because a refusal whose crate the list omits passes every profile graph linking it", "file": "spec/81-engineering.md", - "line": 183 + "line": 185 }, { "id": "assurance.gate.unconfigured-egress", @@ -20602,7 +20613,7 @@ "statement": "An outbound call reachable under the default configuration — usage ping, license check, update probe — raises `UnconfiguredEgress` in the crate-graph stage, naming the call site.", "why": "because some deployments run with no external reach, and a call nobody configured moves data outside every grant, zone and audit record", "file": "spec/81-engineering.md", - "line": 185 + "line": 187 }, { "id": "assurance.gate.interpolated-claim", @@ -20613,7 +20624,7 @@ "statement": "A source lint over the runtime crates finding a subject claim formatted into SQL text raises `EnforceInterpolatedSubjectClaim`, naming the file and line.", "why": "P1", "file": "spec/81-engineering.md", - "line": 187 + "line": 189 }, { "id": "assurance.gate.formal-stage", @@ -20624,7 +20635,7 @@ "statement": "The formal stage runs {{assurance.audit-assumptions.check-command}}, {{assurance.differential-test.command}} and {{assurance.model.protocol-check}}; a non-zero exit from any reds the run.", "why": "P7", "file": "spec/81-engineering.md", - "line": 189 + "line": 191 }, { "id": "assurance.gate.container", @@ -20635,7 +20646,7 @@ "statement": "The gate container carries four ceilings — processor count, 12 GiB of memory, 18 GiB of usable disk, and a 30 min per-stage wall clock — and a run dies on any one.", "why": "", "file": "spec/81-engineering.md", - "line": 191 + "line": 193 }, { "id": "assurance.gate.parallel-jobs", @@ -20646,7 +20657,7 @@ "statement": "Parallel build jobs number the memory ceiling divided by 3 GiB.", "why": "", "file": "spec/81-engineering.md", - "line": 192 + "line": 194 }, { "id": "assurance.gate.free-disk", @@ -20657,7 +20668,7 @@ "statement": "A stage starting with less than 2 GiB of free disk raises `BuildDiskPrecondition` and exits 28 before doing work.", "why": "P7", "file": "spec/81-engineering.md", - "line": 193 + "line": 195 }, { "id": "assurance.gate.build-size", @@ -20668,7 +20679,7 @@ "statement": "The budget stage holds total build size to 12 GiB, printing that size, the free space and the largest artifacts; a larger build raises `BuildSizeCeilingExceeded`, naming the head of that list.", "why": "P7", "file": "spec/81-engineering.md", - "line": 195 + "line": 197 }, { "id": "assurance.gate.edge-budget", @@ -20679,7 +20690,7 @@ "statement": "The edge profile holds to 50 MiB compressed and 60 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 197 + "line": 199 }, { "id": "assurance.gate.full-budget", @@ -20690,7 +20701,7 @@ "statement": "The full profile holds to 150 MiB compressed and 180 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 198 + "line": 200 }, { "id": "assurance.gate.control-budget", @@ -20701,7 +20712,7 @@ "statement": "The control profile holds to 60 MiB compressed and 80 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 199 + "line": 201 }, { "id": "assurance.gate.footprint", @@ -20712,7 +20723,7 @@ "statement": "Per change and per profile, the footprint step builds the static-linked Linux target, compresses it, and holds its size to the profile's budget and its dynamic dependencies to the platform C library.", "why": "P7", "file": "spec/81-engineering.md", - "line": 200 + "line": 202 }, { "id": "assurance.gate.budget-stage", @@ -20723,7 +20734,7 @@ "statement": "The budget stage runs {{assurance.gate.footprint}} over every profile, in one part per profile dispatched as its own check, and the evaluate stage builds no profile.", "why": "because one link-time-optimized release build of the full profile fills most of one stage's wall clock", "file": "spec/81-engineering.md", - "line": 202 + "line": 204 }, { "id": "assurance.gate.footprint-exceeded", @@ -20734,7 +20745,7 @@ "statement": "An artifact over its profile's budget, or carrying a dynamic dependency beyond the platform C library, raises `FootprintBudgetExceeded`, naming the profile.", "why": "P7", "file": "spec/81-engineering.md", - "line": 204 + "line": 206 }, { "id": "assurance.gate.typescript-surfaces", @@ -20745,7 +20756,7 @@ "statement": "The TypeScript surfaces run typecheck, unit tests and framework build in one stage, and a surface declaring no script for a check skips that check.", "why": "", "file": "spec/81-engineering.md", - "line": 206 + "line": 208 }, { "id": "assurance.gate.surface-check-failed", @@ -20756,7 +20767,7 @@ "statement": "A surface whose typecheck, unit tests or framework build fails raises `SurfaceCheckFailed`, naming the surface and the script.", "why": "P7", "file": "spec/81-engineering.md", - "line": 207 + "line": 209 }, { "id": "assurance.evaluate.real-read-path", @@ -20767,7 +20778,7 @@ "statement": "An evaluation run ingests, indexes, retrieves, reads, judges and scores through the surfaces a caller uses, and the harness adds no storage or runtime primitive.", "why": "", "file": "spec/81-engineering.md", - "line": 252 + "line": 254 }, { "id": "assurance.evaluate.through-the-store", @@ -20778,7 +20789,7 @@ "statement": "The corpus loads through the real store, and the retriever under test calls {{read.retrieve.ranked-call}} with the options a caller passes.", "why": "because an unordered full-scan retriever cannot go red when ranking breaks", "file": "spec/81-engineering.md", - "line": 253 + "line": 255 }, { "id": "assurance.evaluate.policy-labels", @@ -20789,7 +20800,7 @@ "statement": "An evaluation corpus carries zone and row-policy labels in its table declarations, and a run reads it through the access-control path under one admitted credential.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 255, + "line": 257, "scenarios": [ "WHEN a regression case names a row another agent owns under `owner = subject.agent`, THEN no leg returns it and the hybrid forbidden-row rate reads 0." ] @@ -20803,7 +20814,7 @@ "statement": "A run over a corpus whose tables declare no zone or row policy, with no `local:` zone passed as `--zone`, raises `EvalCorpusUnlabeled` before any row lands.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 257, + "line": 259, "scenarios": [ "WHEN a corpus's tables declare no policy and the run passes no `--zone`, THEN the run raises `EvalCorpusUnlabeled` and lands no row." ] @@ -20817,7 +20828,7 @@ "statement": "A case carries an id, a corpus path relative to its case file, a question, tags, and an expected block of answer, artifacts, edges, entities, must-cite, must-abstain, must-not-retrieve, time anchor and recency bound.", "why": "", "file": "spec/81-engineering.md", - "line": 259 + "line": 261 }, { "id": "assurance.evaluate.row-reference", @@ -20828,7 +20839,7 @@ "statement": "A case names a row as `#`, the key joining the row's declared primary-key values with commas, and the runner keys each returned row the same way.", "why": "", "file": "spec/81-engineering.md", - "line": 260 + "line": 262 }, { "id": "assurance.evaluate.regression-case", @@ -20839,7 +20850,7 @@ "statement": "A case tagged `regression` is a regression case, and its must-not-retrieve set scores {{assurance.evaluate.forbidden-row-rate}}.", "why": "", "file": "spec/81-engineering.md", - "line": 261 + "line": 263 }, { "id": "assurance.evaluate.corpus-layout", @@ -20850,7 +20861,7 @@ "statement": "A corpus is a directory holding `contextful.toml`, its table declarations and policy labels, and one `rows/
.jsonl` per table.", "why": "", "file": "spec/81-engineering.md", - "line": 262 + "line": 264 }, { "id": "assurance.evaluate.run-command", @@ -20861,7 +20872,7 @@ "statement": "`contextful eval run --goldens ` loads the case file, lands and folds each referenced corpus into a scratch store at `--landed-at`, the Unix epoch by default, reads every case, and writes the run report.", "why": "because an undated row landed at the epoch predates every question's window, so a replay scores alike on any day", "file": "spec/81-engineering.md", - "line": 263 + "line": 265 }, { "id": "assurance.evaluate.legs", @@ -20872,7 +20883,7 @@ "statement": "Each case calls {{read.retrieve.ranked-call}} three times at limit k: the lexical leg with the question alone, the vector leg with an embedding and an empty query, the hybrid leg with both.", "why": "", "file": "spec/81-engineering.md", - "line": 265 + "line": 267 }, { "id": "assurance.evaluate.stub-embedder", @@ -20883,7 +20894,7 @@ "statement": "The deterministic tier embeds every row and question with a seeded feature-hashing stub embedder; a corpus row or a case carrying its own embedding keeps it.", "why": "", "file": "spec/81-engineering.md", - "line": 266 + "line": 268 }, { "id": "assurance.evaluate.run-verdict", @@ -20894,7 +20905,7 @@ "statement": "A run holds its report to the floors and, given `--baseline`, to that file; either red verdict exits non-zero, and `--update-baseline` applies {{assurance.baseline.raise-only}} on green alone.", "why": "", "file": "spec/81-engineering.md", - "line": 267 + "line": 269 }, { "id": "assurance.evaluate.converter", @@ -20905,7 +20916,7 @@ "statement": "Each benchmark and application ground truth converts once into the case format through its own converter, and the runner reads nothing else.", "why": "", "file": "spec/81-engineering.md", - "line": 268 + "line": 270 }, { "id": "assurance.evaluate.deterministic-tier", @@ -20916,7 +20927,7 @@ "statement": "Recall@k, R-precision, hit-rate@k, reciprocal rank and nDCG@k run with no model call, reported per leg — lexical, vector, hybrid — and per slice.", "why": "", "file": "spec/81-engineering.md", - "line": 269 + "line": 271 }, { "id": "assurance.evaluate.distinct-top-k", @@ -20927,7 +20938,7 @@ "statement": "Membership in the top k is distinct, and a repeated id counts once.", "why": "", "file": "spec/81-engineering.md", - "line": 270 + "line": 272 }, { "id": "assurance.evaluate.absent-truth", @@ -20938,7 +20949,7 @@ "statement": "A ranked metric over a case with no relevant set is NaN and drops out of every aggregate; an empty ranking against a non-empty relevant set scores zero.", "why": "", "file": "spec/81-engineering.md", - "line": 271 + "line": 273 }, { "id": "assurance.evaluate.precision-floor", @@ -20949,7 +20960,7 @@ "statement": "Precision at rank min(k, R), R being the relevant-set size, holds at or above 60 percent.", "why": "because precision@k caps at R/k, below the floor whenever R is small", "file": "spec/81-engineering.md", - "line": 272 + "line": 274 }, { "id": "assurance.evaluate.forbidden-row-rate", @@ -20960,7 +20971,7 @@ "statement": "A row named in a must-not-retrieve set holds at 0 percent of a regression case's returned rows.", "why": "", "file": "spec/81-engineering.md", - "line": 274 + "line": 276 }, { "id": "assurance.evaluate.duplicate-row-rate", @@ -20971,7 +20982,7 @@ "statement": "A repeated table-and-row-key pair holds at 0 percent of the ranking.", "why": "", "file": "spec/81-engineering.md", - "line": 275 + "line": 277 }, { "id": "assurance.evaluate.in-window-rate", @@ -20982,7 +20993,7 @@ "statement": "A case declaring a recency bound holds its in-window rate at or above 95 percent.", "why": "", "file": "spec/81-engineering.md", - "line": 276 + "line": 278 }, { "id": "assurance.evaluate.abstention", @@ -20993,7 +21004,7 @@ "statement": "A must-abstain case returns zero rows.", "why": "", "file": "spec/81-engineering.md", - "line": 277 + "line": 279 }, { "id": "assurance.evaluate.truth-per-surface", @@ -21004,7 +21015,7 @@ "statement": "Expected artifacts score the artifact retriever and expected edges score the edge retriever, reported apart; edge truth with no edge surface configured counts in an unscored tally.", "why": "", "file": "spec/81-engineering.md", - "line": 278 + "line": 280 }, { "id": "assurance.evaluate.judged-dimensions", @@ -21015,7 +21026,7 @@ "statement": "The reading stage adds four judged dimensions: answer accuracy; abstention, as correct-refusal and hallucination-on-unknown rates; citation faithfulness; and temporal correctness against the bitemporal columns.", "why": "", "file": "spec/81-engineering.md", - "line": 279 + "line": 281 }, { "id": "assurance.evaluate.grounded-reader", @@ -21026,7 +21037,7 @@ "statement": "The harness reader answers from retrieved rows alone, and the deterministic tier substitutes a stub reader.", "why": "", "file": "spec/81-engineering.md", - "line": 280 + "line": 282 }, { "id": "assurance.evaluate.model-endpoint", @@ -21037,7 +21048,7 @@ "statement": "The judge and the reader reach a model through {{connector.infer.model-endpoint}}, and the harness holds no credential.", "why": "", "file": "spec/81-engineering.md", - "line": 281 + "line": 283 }, { "id": "assurance.evaluate.judge", @@ -21048,7 +21059,7 @@ "statement": "The judge is one pinned open-weights instruct model, called once per judged item at temperature zero; a judge swap is a full re-baseline.", "why": "because a majority vote at temperature zero samples one mode and adds cost without reducing variance", "file": "spec/81-engineering.md", - "line": 282 + "line": 284 }, { "id": "assurance.evaluate.systems-metrics", @@ -21059,7 +21070,7 @@ "statement": "Tokens per query, latency and cost report beside the quality figures, bucketed by corpus size in tokens relative to the reader's context window.", "why": "", "file": "spec/81-engineering.md", - "line": 284 + "line": 286 }, { "id": "assurance.evaluate.checkpoint", @@ -21070,7 +21081,7 @@ "statement": "A run appends each case's result as JSONL, and a crash re-runs the in-flight case.", "why": "", "file": "spec/81-engineering.md", - "line": 285 + "line": 287 }, { "id": "assurance.evaluate.run-report", @@ -21081,7 +21092,7 @@ "statement": "A run report carries one field per gated metric, the sample count behind each mean, the run block, the per-slice breakdown, and the tally of cases sampled, dropped and unscored.", "why": "", "file": "spec/81-engineering.md", - "line": 286 + "line": 288 }, { "id": "assurance.baseline.file", @@ -21092,7 +21103,7 @@ "statement": "A baseline file carries a reserved run block `{k, tier, model, samples}` and one entry per gated metric: a bare number at the run's dead band, or an object overriding the band.", "why": "", "file": "spec/81-engineering.md", - "line": 297 + "line": 299 }, { "id": "assurance.baseline.default-dead-band", @@ -21103,7 +21114,7 @@ "statement": "A rate-valued entry with no override gates at a 2 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 298 + "line": 300 }, { "id": "assurance.baseline.rank-quality-dead-band", @@ -21114,7 +21125,7 @@ "statement": "The ranked-quality entry gates at a 3 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 299 + "line": 301 }, { "id": "assurance.baseline.band-units", @@ -21125,7 +21136,7 @@ "statement": "A band carries its metric's own units, and a count entry is pinned at zero with no band.", "why": "", "file": "spec/81-engineering.md", - "line": 300 + "line": 302 }, { "id": "assurance.baseline.metric-path", @@ -21136,7 +21147,7 @@ "statement": "An entry names a metric by the report's field path: `retrieval..`, `edge_retrieval..`, `judge.`, `latency_ms`, `n_cases`, or `slices..` before a bare `` or any of these but `latency_ms`.", "why": "", "file": "spec/81-engineering.md", - "line": 301 + "line": 303 }, { "id": "assurance.baseline.sample-count", @@ -21147,7 +21158,7 @@ "statement": "Every mean-valued path takes a `.n` suffix naming its sample count.", "why": "because blanking one case's truth lifts a NaN-filtered mean without moving the case count", "file": "spec/81-engineering.md", - "line": 302 + "line": 304 }, { "id": "assurance.baseline.unresolvable-path", @@ -21158,7 +21169,7 @@ "statement": "A path resolving to nothing, a mean over fewer than 30 cases, a malformed entry or a non-finite band raises `BaselinePathUnresolved` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 304 + "line": 306 }, { "id": "assurance.baseline.run-stamp-drift", @@ -21169,7 +21180,7 @@ "statement": "A run block differing from the run's own configuration raises `BaselineRunStampMismatch` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 306 + "line": 308 }, { "id": "assurance.baseline.interval", @@ -21180,7 +21191,7 @@ "statement": "Each judged figure reports a 95 percent percentile-bootstrap interval over cases from 1000 resamples.", "why": "", "file": "spec/81-engineering.md", - "line": 308 + "line": 310 }, { "id": "assurance.baseline.raise-only", @@ -21191,7 +21202,7 @@ "statement": "A baseline update runs after every gate and floor passes, moves each improved entry to its measured value, moves none in the worse direction, and adds no path.", "why": "", "file": "spec/81-engineering.md", - "line": 309 + "line": 311 }, { "id": "assurance.baseline.floors-are-absolute", @@ -21202,7 +21213,7 @@ "statement": "An absolute floor gates independently of every committed value.", "why": "", "file": "spec/81-engineering.md", - "line": 310 + "line": 312 }, { "id": "assurance.baseline.floor-coverage", @@ -21213,7 +21224,7 @@ "statement": "A report lacking a floor's figure on any leg of its `retrieval` surface breaches that floor; only a forbidden-row or in-window rate over no case holds.", "why": "", "file": "spec/81-engineering.md", - "line": 311 + "line": 313 }, { "id": "assurance.baseline.offline", @@ -21224,7 +21235,7 @@ "statement": "The gate is a local command comparing the report against in-tree baselines and floors, reporting both verdicts, with or without a trace store reachable.", "why": "", "file": "spec/81-engineering.md", - "line": 312 + "line": 314 }, { "id": "assurance.baseline.trace-store", @@ -21235,7 +21246,7 @@ "statement": "A trace store records run history and curation staging, and decides no verdict.", "why": "", "file": "spec/81-engineering.md", - "line": 313 + "line": 315 }, { "id": "assurance.baseline.golden-custody", @@ -21246,7 +21257,7 @@ "statement": "Version-controlled JSONL under `evals/cases/` is the canonical golden set, one loader is its only ingestion door, and a reviewer's edit reaches the gate as a change to the tree.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 314 + "line": 316 }, { "id": "assurance.baseline.answer-key-in-the-store", @@ -21257,7 +21268,7 @@ "statement": "An artifact path normalizing under the evaluation directory, reached by a manifest lint or a connector, raises `GoldenSetIngested`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 316 + "line": 318 }, { "id": "assurance.baseline.generated-redaction", @@ -21268,7 +21279,7 @@ "statement": "A golden mined from a live query or promoted from a labeled outcome passes {{authority.redact.write-time}} before commit; a case it cannot redact raises `GoldenRedactionFailed`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 318 + "line": 320 }, { "id": "assurance.baseline.promotion-source", @@ -21279,7 +21290,7 @@ "statement": "Only an adjudicated outcome label promotes into a golden set; a self-rated row does not.", "why": "", "file": "spec/81-engineering.md", - "line": 320 + "line": 322 }, { "id": "assurance.baseline.generators", @@ -21290,7 +21301,7 @@ "statement": "Three generators draft candidate cases from the deployment's store — entity-to-fact-to-edge walks, composed edge chains, and questions about absent entities — and a human approves each before commit.", "why": "", "file": "spec/81-engineering.md", - "line": 321 + "line": 323 }, { "id": "assurance.baseline.held-out", @@ -21301,7 +21312,7 @@ "statement": "Retrieval and synthesis are tuned on no gate input.", "why": "", "file": "spec/81-engineering.md", - "line": 322 + "line": 324 }, { "id": "assurance.baseline.rotation", @@ -21312,7 +21323,7 @@ "statement": "The native set rotates on a cadence, a sample of its truth is reviewed on each rotation, and a comparison across generations resolves through the pinned run stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 323 + "line": 325 }, { "id": "assurance.baseline.native-gate", @@ -21323,7 +21334,7 @@ "statement": "A native benchmark generated from a real corpus is the red or green gate; public benchmark sets run beside it as held-out comparison and decide nothing.", "why": "", "file": "spec/81-engineering.md", - "line": 324 + "line": 326 }, { "id": "assurance.baseline.trace-export", @@ -21334,7 +21345,7 @@ "statement": "A run touching a deployed store with a hosted trace endpoint configured raises `TraceExportOutOfPerimeter`; hosted collection serves runs over public or synthetic fixtures alone.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 325 + "line": 327 }, { "id": "assurance.measure.ledger", @@ -21345,7 +21356,7 @@ "statement": "`evals/ledger.toml` holds one entry per tracked target: an id, an owning clause id, a run-report metric path, a tier, a method naming one integration test, case set or probe, and a threshold.", "why": "", "file": "spec/81-engineering.md", - "line": 334 + "line": 336 }, { "id": "assurance.measure.unresolved-entry", @@ -21356,7 +21367,7 @@ "statement": "An entry whose owning clause, method or metric path resolves to nothing raises `MeasureEntryUnresolved` before any measure runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 335, + "line": 337, "scenarios": [ "WHEN an entry names `run.journal.entry-keys`, THEN the run raises `MeasureEntryUnresolved` and no test runs." ] @@ -21370,7 +21381,7 @@ "statement": "An entry naming an issue in place of a method reports open and gates nothing, and `evals/ledger.md` carries every entry's computed status.", "why": "", "file": "spec/81-engineering.md", - "line": 337, + "line": 339, "scenarios": [ "WHEN an entry's method is `{ issue = 43 }`, THEN `evals/ledger.md` lists it open and the evaluate stage ignores it." ] @@ -21384,7 +21395,7 @@ "statement": "A gate-tier entry decides the evaluate stage, a trend-tier entry records on every run and decides nothing, and a scheduled-tier entry runs on the scheduled job alone.", "why": "", "file": "spec/81-engineering.md", - "line": 338 + "line": 340 }, { "id": "assurance.measure.count-first", @@ -21395,7 +21406,7 @@ "statement": "A gate-tier entry measures a count, a ratio within one run or a size under a locked resolve; a wall-clock or resident-memory figure is trend-tier.", "why": "because a shared container moves wall-clock figures past any band narrow enough to catch a regression", "file": "spec/81-engineering.md", - "line": 339 + "line": 341 }, { "id": "assurance.measure.record", @@ -21406,7 +21417,7 @@ "statement": "A measure writes one JSON record carrying its entry id, value, sample count, seed and run stamp; a gate-tier method finishing without one raises `MeasureRecordMissing`.", "why": "P7", "file": "spec/81-engineering.md", - "line": 341 + "line": 343 }, { "id": "assurance.measure.seeded", @@ -21417,7 +21428,7 @@ "statement": "Every generated fixture and randomized schedule derives from the record's seed, and replaying that seed reproduces a count-valued entry's value.", "why": "because a failure that cannot replay cannot be fixed", "file": "spec/81-engineering.md", - "line": 343 + "line": 345 }, { "id": "assurance.measure.seed-mismatch", @@ -21428,7 +21439,7 @@ "statement": "A record whose seed differs from its entry's declared seed raises `MeasureSeedMismatch`, and the entry counts as red.", "why": "because a figure measured under another seed replays nothing the ledger names", "file": "spec/81-engineering.md", - "line": 345 + "line": 347 }, { "id": "assurance.measure.timing-iterations", @@ -21439,7 +21450,7 @@ "statement": "A timed batch reports p50 and p95 over 200 repeats of its operation, each after warm-up.", "why": "", "file": "spec/81-engineering.md", - "line": 347 + "line": 349 }, { "id": "assurance.measure.timing-batches", @@ -21450,7 +21461,7 @@ "statement": "A timed figure is the median of 5 repeats of its timed batch.", "why": "", "file": "spec/81-engineering.md", - "line": 348 + "line": 350 }, { "id": "assurance.measure.trend-band", @@ -21461,7 +21472,7 @@ "statement": "A trend figure more than 25 percent worse than its baseline annotates the run report and fails no stage.", "why": "", "file": "spec/81-engineering.md", - "line": 349, + "line": 351, "scenarios": [ "WHEN a trend p95 moves from 40 ms to 52 ms on a matching runner stamp, THEN the report carries a +30 percent annotation and the stage passes." ] @@ -21475,7 +21486,7 @@ "statement": "The run block carries the runner's processor model, processor count and memory limit, and a trend figure compares only against a baseline with the same stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 350 + "line": 352 }, { "id": "assurance.measure.absolute-threshold", @@ -21486,7 +21497,7 @@ "statement": "A threshold is an absolute figure of this system's own measure, and it tightens only through {{assurance.baseline.raise-only}}.", "why": "", "file": "spec/81-engineering.md", - "line": 351 + "line": 353 }, { "id": "assurance.measure.history", @@ -21497,7 +21508,7 @@ "statement": "A run on the default branch attaches its run report to the measured commit under `refs/notes/measures`, and no verdict reads a note.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 352 + "line": 354 }, { "id": "assurance.measure.scheduled", @@ -21508,7 +21519,7 @@ "statement": "A nightly job runs every tier against the default-branch head and reports one check per run.", "why": "", "file": "spec/81-engineering.md", - "line": 354 + "line": 356 }, { "id": "assurance.measure.measured-basis", @@ -21519,7 +21530,7 @@ "statement": "A bound with a measured basis names a trend-tier or scheduled-tier ledger entry id as its benchmark.", "why": "", "file": "spec/81-engineering.md", - "line": 355 + "line": 357 }, { "id": "assurance.release.version", @@ -21530,7 +21541,7 @@ "statement": "A release tag is `v0..`: `` counts the milestones computing `closed` in the tagged commit's `spec/status.md`, and `` counts the existing tags `v0..*`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 383, + "line": 385, "scenarios": [ "WHEN milestones 0 to 7 and 11 compute `closed` and `v0.9.0` exists, THEN the computed version is `v0.9.1`." ] @@ -21544,7 +21555,7 @@ "statement": "`contextful-ci tag` creates a signed annotated tag on `HEAD` naming the closed milestones once every refusal of this operation clears, and never moves or replaces an existing tag.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 385 + "line": 387 }, { "id": "assurance.release.dirty-tree", @@ -21555,7 +21566,7 @@ "statement": "A working tree or index differing from `HEAD` raises `TagTreeDirty`.", "why": "because the gate then judges files the tagged commit does not hold", "file": "spec/81-engineering.md", - "line": 387 + "line": 389 }, { "id": "assurance.release.off-branch", @@ -21566,7 +21577,7 @@ "statement": "A `HEAD` unreachable from the default branch, `origin/HEAD` unless `--branch` names another, raises `TagOffDefaultBranch`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 389 + "line": 391 }, { "id": "assurance.release.version-regressed", @@ -21577,7 +21588,7 @@ "statement": "A computed version below the highest existing `v..` tag raises `TagVersionRegressed`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 391, + "line": 393, "scenarios": [ "WHEN milestone 11 reopens after `v0.9.0`, THEN the computed `v0.8.0` raises `TagVersionRegressed`." ] @@ -21591,7 +21602,7 @@ "statement": "A `[workspace.package]` version in `HEAD`'s `Cargo.toml` that is absent or differs from the computed version raises `TagWorkspaceVersionMismatch`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 393 + "line": 395 }, { "id": "assurance.release.gate-failed", @@ -21602,7 +21613,7 @@ "statement": "Once the other refusals clear, every gate stage runs against `HEAD` with `HEAD~1` as base; a failing stage raises `TagGateFailed`, naming the stage's refusal, and no tag is created.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 395 + "line": 397 } ], "owns": { diff --git a/spec/status.md b/spec/status.md index beadc3229..81e081da2 100644 --- a/spec/status.md +++ b/spec/status.md @@ -5,7 +5,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 135 | 0 | 135 | +| `assurance` | 2 | 15 | 228 | 59 | 22 | 12 | 135 | 0 | 135 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | | `connector` | 2 | 14 | 321 | 80 | 21 | 16 | 195 | 0 | 195 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | @@ -15,7 +15,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | `store` | 1 | 17 | 235 | 55 | 13 | 20 | 198 | 0 | 194 | | `surface` | 2 | 19 | 86 | 46 | 22 | 5 | 51 | 0 | 50 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1335 | 0 | | +| **total** | 19 | 154 | 1957 | 569 | 188 | 133 | 1335 | 0 | | Decision records: 18. @@ -39,7 +39,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 11 — The derive tier | 7 | 78 | 55 | passing | open | | 12 — The console | 11 | 30 | 0 | absent | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | -| 14 — Assurance | 7 | 130 | 83 | passing | closed | +| 14 — Assurance | 7 | 131 | 83 | passing | closed | Unscheduled operations: 10. diff --git a/tools/ci/tests/integration/stages.rs b/tools/ci/tests/integration/stages.rs index d69aced93..46d1bedeb 100644 --- a/tools/ci/tests/integration/stages.rs +++ b/tools/ci/tests/integration/stages.rs @@ -135,6 +135,21 @@ fn predecessors_adds_every_stage_whose_output_a_selected_stage_reads() { ); } +#[test] +fn native_windows_checks_cover_both_targets_and_directory_sync() { + let workflow = std::fs::read_to_string(crate::repo_root().join(".github/workflows/gate.yml")).unwrap(); + for pair in [ + "runner: windows-2025\n target: x86_64-pc-windows-msvc", + "runner: windows-11-arm\n target: aarch64-pc-windows-msvc", + ] { + assert!(workflow.contains(pair), "the native Windows matrix omits {pair}"); + } + assert!(workflow.contains("cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context")); + for test in ["an_open_directory_handle_syncs_its_entries", "an_audit_append_syncs_its_directories", "a_ledger_append_syncs_its_directory"] { + assert!(workflow.contains(test), "the native Windows matrix omits {test}"); + } +} + /// Each stage prints the environment it leaves and its memory limit, peak and event counts, and a failing stage prints its diagnostics before propagating its exit code. // spec: assurance.gate.stage-reports@e77a2384 #[test] From 149502ec0c329363517ab4ac1780e319ecfc6506 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 08:46:23 +0800 Subject: [PATCH 002/155] Dispatch test-first checks per changed package --- .github/workflows/gate.yml | 28 ++- AGENTS.md | 6 +- spec/81-engineering.md | 2 + spec/cards/assurance.md | 2 +- spec/pins.toml | 10 +- spec/spec.lock.json | 221 ++++++++++++----------- spec/status.md | 15 +- tools/ci/src/main.rs | 94 +++++++--- tools/ci/tests/integration/test_first.rs | 47 ++++- tools/ci/tests/integration/workflow.rs | 13 +- 10 files changed, 280 insertions(+), 158 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index 05b532426..c42d14378 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -1,10 +1,11 @@ # The gate, one FlareDispatch `check` per stage. # -# Each matrix entry dispatches `contextful-ci gate --predecessors --stage ` to the +# The prepare job selects each changed test package against the PR base. Each matrix entry +# dispatches `contextful-ci gate --predecessors --stage ` to the # org's Dispatcher, which clones the PR head into a Rust-equipped sandbox and reports the # verdict as the check-run `flare-dispatch/check:`. `--predecessors` also runs each # stage whose output the stage reads, so the formal check runs pins and toolchain first. -# The workspace, features and budget stages dispatch one part each, `.`, so each part fits +# The test-first, workspace, features and budget stages dispatch one part each, `.`, so each part fits # the sandbox's 30 min wall clock; `contextful-ci stages --parts` prints the list. # A matrix job holds its slot until its dispatched check finishes, limiting concurrent # sandbox builds to two. Each stage or part reports one check-run: @@ -12,7 +13,8 @@ # flare-dispatch/check:pins every pinned identity resolved before any compilation # flare-dispatch/check:toolchain the pinned Lean toolchain and the WebAssembly target provisioned # flare-dispatch/check:schema secrets of record, mirrors: annotations, derived artifacts regenerated, contextful-spec lint -# flare-dispatch/check:test-first a test under tests/ fails against the base source +# flare-dispatch/check:test-first.validate a source change carries a test under tests/ +# flare-dispatch/check:test-first. that package's changed tests fail against the base source # flare-dispatch/check:workspace.compile compile every workspace suite over the union of features, then run the CLI suite # flare-dispatch/check:workspace. run each other package suite in one of four groups # flare-dispatch/check:acceptance the milestone acceptance suite @@ -47,14 +49,32 @@ permissions: contents: read jobs: + prepare: + if: ${{ vars.FLAREDISPATCH_ENDPOINT != '' && github.event.pull_request.head.repo.full_name == github.repository }} + runs-on: ubuntu-latest + outputs: + stages: ${{ steps.matrix.outputs.stages }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + - id: matrix + env: + BASE: ${{ github.event.pull_request.base.sha }} + run: | + stages="$(cargo run --locked -q -p contextful-ci -- stages --parts --base "$BASE" --json)" + printf 'stages=%s\n' "$stages" >> "$GITHUB_OUTPUT" + dispatch: if: ${{ vars.FLAREDISPATCH_ENDPOINT != '' && github.event.pull_request.head.repo.full_name == github.repository }} + needs: prepare runs-on: ubuntu-latest strategy: fail-fast: false max-parallel: 2 matrix: - stage: [pins, toolchain, schema, test-first, workspace.compile, workspace.foundation, workspace.runtime, workspace.read, workspace.ci, acceptance, evaluate, features.packages, features.binary-none, features.binary-all, features.binary-data-plane, features.binary-contextful-edge, crate-graph, connectors, surfaces, formal, budget.control, budget.edge, budget.full] + stage: ${{ fromJson(needs.prepare.outputs.stages) }} steps: - id: dispatch uses: fractalboxdev/flare-dispatch/actions/flare-dispatch-action@a5118d6594c8f3e4040bb90afcb02898a6e7b7bd diff --git a/AGENTS.md b/AGENTS.md index 57aa3ef86..175b17641 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,10 +103,10 @@ The gate measures commits, so commit before running it. [`.github/workflows/gate.yml`](./.github/workflows/gate.yml) dispatches each stage of `contextful-ci gate` to the org's FlareDispatch Dispatcher as a `check` run. Each stage reports as its own check-run on the pull request: -`flare-dispatch/check:` for each of `pins`, `toolchain`, `schema`, `test-first`, +`flare-dispatch/check:` for each of `pins`, `toolchain`, `schema`, `workspace`, `acceptance`, `evaluate`, `features`, `crate-graph`, `connectors`, -`surfaces`, `formal` and `budget`. The features and budget stages dispatch one check per -part, `flare-dispatch/check:features.` and `flare-dispatch/check:budget.`, so +`surfaces`, `formal` and `budget`. The test-first, features and budget stages dispatch one check per +part, `flare-dispatch/check:test-first.validate` and `flare-dispatch/check:test-first.`, `flare-dispatch/check:features.` and `flare-dispatch/check:budget.`, so each fits the sandbox's wall clock; `contextful-ci stages --parts` prints the list, and `--stage .` runs one part. A selected subset runs in that order and refuses a stage whose predecessor's output is absent; `--predecessors` runs those too. A local run and diff --git a/spec/81-engineering.md b/spec/81-engineering.md index fdb9e2221..f7bc6a494 100644 --- a/spec/81-engineering.md +++ b/spec/81-engineering.md @@ -155,6 +155,8 @@ Stage order, secrets of record, the crate-graph, row-token, egress and dependenc *A-assurance* - `remote-check` — The pull-request workflow dispatches every stage the gate subcommand defines to a remote runner, a split stage one part at a time, each as one status check labelled with its name. *A-assurance* +- `test-first-parts` — A remote test-first validation checks for a changed test, and each changed test package checks only its own tests against the base in a separate dispatch. + *because one base build per package fits the sandbox's wall clock more reliably* - `workspace-parts` — Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups. *A-assurance* - `remote-predecessors` — Each remote check runs its stage together with every predecessor whose output that stage reads, so no check reads another check's sandbox. diff --git a/spec/cards/assurance.md b/spec/cards/assurance.md index cedd57e6b..90f06a602 100644 --- a/spec/cards/assurance.md +++ b/spec/cards/assurance.md @@ -16,7 +16,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/a | `automate` | Typed subcommands over compiled binaries, and the boundary shell keeps. | 5 | | `test` | Assertion construction, guard validation, suite placement, test-first and acceptance tests, and the connector test kit. | 13 | | `build` | Target directories, the engine-linked invocation, linked query functions, build targets, release artifacts and the dependency allowlist. | 15 | -| `gate` | Stage order, secrets of record, the crate-graph, row-token, egress and dependency rules, the formal stage, the container's ceilings, disk, footprint budgets and surface checks. | 33 | +| `gate` | Stage order, secrets of record, the crate-graph, row-token, egress and dependency rules, the formal stage, the container's ceilings, disk, footprint budgets and surface checks. | 34 | | `evaluate` | The read-path quality harness: case format, deterministic metrics and floors, judged dimensions, the reader and the judge. | 29 | | `baseline` | Holding a run against committed baselines and absolute floors, intervals, golden custody and case generation. | 23 | | `measure` | The target ledger: each tracked target, the clause it serves, how it is measured and its record per commit. | 16 | diff --git a/spec/pins.toml b/spec/pins.toml index 984f9605c..35f63a983 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -52,13 +52,13 @@ # its live performed count, and a live count below its floor raises SpecCoverageRegression. [floor] -assurance = 135 +assurance = 136 authority = 204 connector = 195 corpus = 38 disclosure = 54 -read = 162 -run = 257 -store = 194 -surface = 50 +read = 163 +run = 259 +store = 198 +surface = 51 topology = 38 diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 9761ac0ef..d51eb351a 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -20428,6 +20428,17 @@ "file": "spec/81-engineering.md", "line": 156 }, + { + "id": "assurance.gate.test-first-parts", + "contract": "assurance", + "operation": "gate", + "subject": "test-first-parts", + "kind": "behavior", + "statement": "A remote test-first validation checks for a changed test, and each changed test package checks only its own tests against the base in a separate dispatch.", + "why": "because one base build per package fits the sandbox's wall clock more reliably", + "file": "spec/81-engineering.md", + "line": 158 + }, { "id": "assurance.gate.workspace-parts", "contract": "assurance", @@ -20437,7 +20448,7 @@ "statement": "Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 158 + "line": 160 }, { "id": "assurance.gate.remote-predecessors", @@ -20448,7 +20459,7 @@ "statement": "Each remote check runs its stage together with every predecessor whose output that stage reads, so no check reads another check's sandbox.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 160 + "line": 162 }, { "id": "assurance.gate.fork-dispatch", @@ -20459,7 +20470,7 @@ "statement": "The pull-request workflow dispatches only a head commit pushed to the repository itself; a pull request from a fork dispatches no stage and so carries none of the required checks.", "why": "because a dispatch carries the org's signing secret and runs the commit on the org's runner, and an absent required check fails closed", "file": "spec/81-engineering.md", - "line": 162 + "line": 164 }, { "id": "assurance.gate.stage-reports", @@ -20470,7 +20481,7 @@ "statement": "Each stage prints the environment it leaves and its memory limit, peak and event counts, and a failing stage prints its diagnostics before propagating its exit code.", "why": "because memory exhaustion is silent, and a kill then reads as a number in the log", "file": "spec/81-engineering.md", - "line": 164 + "line": 166 }, { "id": "assurance.gate.pins-stage", @@ -20481,7 +20492,7 @@ "statement": "The pins stage resolves every pinned artifact identity a run depends on before any compilation.", "why": "", "file": "spec/81-engineering.md", - "line": 166 + "line": 168 }, { "id": "assurance.gate.evaluate-stage", @@ -20492,7 +20503,7 @@ "statement": "The evaluate stage runs every gate-tier ledger entry and the native case set in the deterministic tier, and reports the floor and baseline verdicts.", "why": "", "file": "spec/81-engineering.md", - "line": 167 + "line": 169 }, { "id": "assurance.gate.schema-stage", @@ -20503,7 +20514,7 @@ "statement": "The schema stage regenerates each derived artifact into a scratch location, compares it byte for byte against the committed copy, and runs `contextful-spec lint`.", "why": "", "file": "spec/81-engineering.md", - "line": 168 + "line": 170 }, { "id": "assurance.gate.secret-ciphertext", @@ -20514,7 +20525,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file other than `.env.example` holding a value without the `encrypted:` prefix, or a tracked `.env.keys`, raises `SecretPlaintext`, naming file and key.", "why": "because a tracked file reaches every clone, and only dotenvx ciphertext is safe there while its private key stays untracked", "file": "spec/81-engineering.md", - "line": 169 + "line": 171 }, { "id": "assurance.gate.secret-scope", @@ -20525,7 +20536,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file whose block — the assignments above it, up to the first blank line — opens with no comment carrying text raises `SecretScopeMissing`, naming file and key.", "why": "because a provider token cannot report its own grants, so the comment above it is the one record of what it reaches", "file": "spec/81-engineering.md", - "line": 171 + "line": 173 }, { "id": "assurance.gate.crate-graph", @@ -20536,7 +20547,7 @@ "statement": "The crate-graph stage holds run-path crates to reaching read-path crates through the three crossing crates alone; another edge raises `CrateGraphViolation`, naming both crates.", "why": "P5", "file": "spec/81-engineering.md", - "line": 173 + "line": 175 }, { "id": "assurance.gate.locked-resolve", @@ -20547,7 +20558,7 @@ "statement": "The crate-graph stage resolves every graph `--locked` against the committed `Cargo.lock`, so a lock behind its manifests, or one rewritten before the stage starts, fails the stage before any rule runs; the stage never rewrites it.", "why": "because a rule held over a graph the lock file does not record passes a build that resolves another", "file": "spec/81-engineering.md", - "line": 175 + "line": 177 }, { "id": "assurance.gate.row-token", @@ -20558,7 +20569,7 @@ "statement": "Every row-returning function of the store crate takes the enforcement token type, whose constructor is private to the enforcement module; a row path outside a registered relation raises `EnforceUnmediatedPath`, naming the function.", "why": "P5", "file": "spec/81-engineering.md", - "line": 177 + "line": 179 }, { "id": "assurance.gate.dependency-deny", @@ -20569,7 +20580,7 @@ "statement": "The crate-graph stage runs pinned cargo-deny `check bans` over each profile's resolved graph, denying every crate a profile-wide dependency refusal names; a hit raises that refusal's error, naming the profile and the path.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 179 + "line": 181 }, { "id": "assurance.gate.deny-profile-graph", @@ -20580,7 +20591,7 @@ "statement": "A profile's graph is the binary resolved with default features off and the profile's feature on, across every target, excluding development dependencies; the stage prints each profile the binary declares no feature for.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 181 + "line": 183 }, { "id": "assurance.gate.deny-list", @@ -20591,7 +20602,7 @@ "statement": "`deny.toml` holds one `[bans]` entry per crate or feature a profile-wide dependency refusal names, its reason that refusal's clause id; an entry missing or extra raises `DenyListDrift`, naming it.", "why": "because a refusal whose crate the list omits passes every profile graph linking it", "file": "spec/81-engineering.md", - "line": 183 + "line": 185 }, { "id": "assurance.gate.unconfigured-egress", @@ -20602,7 +20613,7 @@ "statement": "An outbound call reachable under the default configuration — usage ping, license check, update probe — raises `UnconfiguredEgress` in the crate-graph stage, naming the call site.", "why": "because some deployments run with no external reach, and a call nobody configured moves data outside every grant, zone and audit record", "file": "spec/81-engineering.md", - "line": 185 + "line": 187 }, { "id": "assurance.gate.interpolated-claim", @@ -20613,7 +20624,7 @@ "statement": "A source lint over the runtime crates finding a subject claim formatted into SQL text raises `EnforceInterpolatedSubjectClaim`, naming the file and line.", "why": "P1", "file": "spec/81-engineering.md", - "line": 187 + "line": 189 }, { "id": "assurance.gate.formal-stage", @@ -20624,7 +20635,7 @@ "statement": "The formal stage runs {{assurance.audit-assumptions.check-command}}, {{assurance.differential-test.command}} and {{assurance.model.protocol-check}}; a non-zero exit from any reds the run.", "why": "P7", "file": "spec/81-engineering.md", - "line": 189 + "line": 191 }, { "id": "assurance.gate.container", @@ -20635,7 +20646,7 @@ "statement": "The gate container carries four ceilings — processor count, 12 GiB of memory, 18 GiB of usable disk, and a 30 min per-stage wall clock — and a run dies on any one.", "why": "", "file": "spec/81-engineering.md", - "line": 191 + "line": 193 }, { "id": "assurance.gate.parallel-jobs", @@ -20646,7 +20657,7 @@ "statement": "Parallel build jobs number the memory ceiling divided by 3 GiB.", "why": "", "file": "spec/81-engineering.md", - "line": 192 + "line": 194 }, { "id": "assurance.gate.free-disk", @@ -20657,7 +20668,7 @@ "statement": "A stage starting with less than 2 GiB of free disk raises `BuildDiskPrecondition` and exits 28 before doing work.", "why": "P7", "file": "spec/81-engineering.md", - "line": 193 + "line": 195 }, { "id": "assurance.gate.build-size", @@ -20668,7 +20679,7 @@ "statement": "The budget stage holds total build size to 12 GiB, printing that size, the free space and the largest artifacts; a larger build raises `BuildSizeCeilingExceeded`, naming the head of that list.", "why": "P7", "file": "spec/81-engineering.md", - "line": 195 + "line": 197 }, { "id": "assurance.gate.edge-budget", @@ -20679,7 +20690,7 @@ "statement": "The edge profile holds to 50 MiB compressed and 60 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 197 + "line": 199 }, { "id": "assurance.gate.full-budget", @@ -20690,7 +20701,7 @@ "statement": "The full profile holds to 150 MiB compressed and 180 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 198 + "line": 200 }, { "id": "assurance.gate.control-budget", @@ -20701,7 +20712,7 @@ "statement": "The control profile holds to 60 MiB compressed and 80 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 199 + "line": 201 }, { "id": "assurance.gate.footprint", @@ -20712,7 +20723,7 @@ "statement": "Per change and per profile, the footprint step builds the static-linked Linux target, compresses it, and holds its size to the profile's budget and its dynamic dependencies to the platform C library.", "why": "P7", "file": "spec/81-engineering.md", - "line": 200 + "line": 202 }, { "id": "assurance.gate.budget-stage", @@ -20723,7 +20734,7 @@ "statement": "The budget stage runs {{assurance.gate.footprint}} over every profile, in one part per profile dispatched as its own check, and the evaluate stage builds no profile.", "why": "because one link-time-optimized release build of the full profile fills most of one stage's wall clock", "file": "spec/81-engineering.md", - "line": 202 + "line": 204 }, { "id": "assurance.gate.footprint-exceeded", @@ -20734,7 +20745,7 @@ "statement": "An artifact over its profile's budget, or carrying a dynamic dependency beyond the platform C library, raises `FootprintBudgetExceeded`, naming the profile.", "why": "P7", "file": "spec/81-engineering.md", - "line": 204 + "line": 206 }, { "id": "assurance.gate.typescript-surfaces", @@ -20745,7 +20756,7 @@ "statement": "The TypeScript surfaces run typecheck, unit tests and framework build in one stage, and a surface declaring no script for a check skips that check.", "why": "", "file": "spec/81-engineering.md", - "line": 206 + "line": 208 }, { "id": "assurance.gate.surface-check-failed", @@ -20756,7 +20767,7 @@ "statement": "A surface whose typecheck, unit tests or framework build fails raises `SurfaceCheckFailed`, naming the surface and the script.", "why": "P7", "file": "spec/81-engineering.md", - "line": 207 + "line": 209 }, { "id": "assurance.evaluate.real-read-path", @@ -20767,7 +20778,7 @@ "statement": "An evaluation run ingests, indexes, retrieves, reads, judges and scores through the surfaces a caller uses, and the harness adds no storage or runtime primitive.", "why": "", "file": "spec/81-engineering.md", - "line": 252 + "line": 254 }, { "id": "assurance.evaluate.through-the-store", @@ -20778,7 +20789,7 @@ "statement": "The corpus loads through the real store, and the retriever under test calls {{read.retrieve.ranked-call}} with the options a caller passes.", "why": "because an unordered full-scan retriever cannot go red when ranking breaks", "file": "spec/81-engineering.md", - "line": 253 + "line": 255 }, { "id": "assurance.evaluate.policy-labels", @@ -20789,7 +20800,7 @@ "statement": "An evaluation corpus carries zone and row-policy labels in its table declarations, and a run reads it through the access-control path under one admitted credential.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 255, + "line": 257, "scenarios": [ "WHEN a regression case names a row another agent owns under `owner = subject.agent`, THEN no leg returns it and the hybrid forbidden-row rate reads 0." ] @@ -20803,7 +20814,7 @@ "statement": "A run over a corpus whose tables declare no zone or row policy, with no `local:` zone passed as `--zone`, raises `EvalCorpusUnlabeled` before any row lands.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 257, + "line": 259, "scenarios": [ "WHEN a corpus's tables declare no policy and the run passes no `--zone`, THEN the run raises `EvalCorpusUnlabeled` and lands no row." ] @@ -20817,7 +20828,7 @@ "statement": "A case carries an id, a corpus path relative to its case file, a question, tags, and an expected block of answer, artifacts, edges, entities, must-cite, must-abstain, must-not-retrieve, time anchor and recency bound.", "why": "", "file": "spec/81-engineering.md", - "line": 259 + "line": 261 }, { "id": "assurance.evaluate.row-reference", @@ -20828,7 +20839,7 @@ "statement": "A case names a row as `
#`, the key joining the row's declared primary-key values with commas, and the runner keys each returned row the same way.", "why": "", "file": "spec/81-engineering.md", - "line": 260 + "line": 262 }, { "id": "assurance.evaluate.regression-case", @@ -20839,7 +20850,7 @@ "statement": "A case tagged `regression` is a regression case, and its must-not-retrieve set scores {{assurance.evaluate.forbidden-row-rate}}.", "why": "", "file": "spec/81-engineering.md", - "line": 261 + "line": 263 }, { "id": "assurance.evaluate.corpus-layout", @@ -20850,7 +20861,7 @@ "statement": "A corpus is a directory holding `contextful.toml`, its table declarations and policy labels, and one `rows/
.jsonl` per table.", "why": "", "file": "spec/81-engineering.md", - "line": 262 + "line": 264 }, { "id": "assurance.evaluate.run-command", @@ -20861,7 +20872,7 @@ "statement": "`contextful eval run --goldens ` loads the case file, lands and folds each referenced corpus into a scratch store at `--landed-at`, the Unix epoch by default, reads every case, and writes the run report.", "why": "because an undated row landed at the epoch predates every question's window, so a replay scores alike on any day", "file": "spec/81-engineering.md", - "line": 263 + "line": 265 }, { "id": "assurance.evaluate.legs", @@ -20872,7 +20883,7 @@ "statement": "Each case calls {{read.retrieve.ranked-call}} three times at limit k: the lexical leg with the question alone, the vector leg with an embedding and an empty query, the hybrid leg with both.", "why": "", "file": "spec/81-engineering.md", - "line": 265 + "line": 267 }, { "id": "assurance.evaluate.stub-embedder", @@ -20883,7 +20894,7 @@ "statement": "The deterministic tier embeds every row and question with a seeded feature-hashing stub embedder; a corpus row or a case carrying its own embedding keeps it.", "why": "", "file": "spec/81-engineering.md", - "line": 266 + "line": 268 }, { "id": "assurance.evaluate.run-verdict", @@ -20894,7 +20905,7 @@ "statement": "A run holds its report to the floors and, given `--baseline`, to that file; either red verdict exits non-zero, and `--update-baseline` applies {{assurance.baseline.raise-only}} on green alone.", "why": "", "file": "spec/81-engineering.md", - "line": 267 + "line": 269 }, { "id": "assurance.evaluate.converter", @@ -20905,7 +20916,7 @@ "statement": "Each benchmark and application ground truth converts once into the case format through its own converter, and the runner reads nothing else.", "why": "", "file": "spec/81-engineering.md", - "line": 268 + "line": 270 }, { "id": "assurance.evaluate.deterministic-tier", @@ -20916,7 +20927,7 @@ "statement": "Recall@k, R-precision, hit-rate@k, reciprocal rank and nDCG@k run with no model call, reported per leg — lexical, vector, hybrid — and per slice.", "why": "", "file": "spec/81-engineering.md", - "line": 269 + "line": 271 }, { "id": "assurance.evaluate.distinct-top-k", @@ -20927,7 +20938,7 @@ "statement": "Membership in the top k is distinct, and a repeated id counts once.", "why": "", "file": "spec/81-engineering.md", - "line": 270 + "line": 272 }, { "id": "assurance.evaluate.absent-truth", @@ -20938,7 +20949,7 @@ "statement": "A ranked metric over a case with no relevant set is NaN and drops out of every aggregate; an empty ranking against a non-empty relevant set scores zero.", "why": "", "file": "spec/81-engineering.md", - "line": 271 + "line": 273 }, { "id": "assurance.evaluate.precision-floor", @@ -20949,7 +20960,7 @@ "statement": "Precision at rank min(k, R), R being the relevant-set size, holds at or above 60 percent.", "why": "because precision@k caps at R/k, below the floor whenever R is small", "file": "spec/81-engineering.md", - "line": 272 + "line": 274 }, { "id": "assurance.evaluate.forbidden-row-rate", @@ -20960,7 +20971,7 @@ "statement": "A row named in a must-not-retrieve set holds at 0 percent of a regression case's returned rows.", "why": "", "file": "spec/81-engineering.md", - "line": 274 + "line": 276 }, { "id": "assurance.evaluate.duplicate-row-rate", @@ -20971,7 +20982,7 @@ "statement": "A repeated table-and-row-key pair holds at 0 percent of the ranking.", "why": "", "file": "spec/81-engineering.md", - "line": 275 + "line": 277 }, { "id": "assurance.evaluate.in-window-rate", @@ -20982,7 +20993,7 @@ "statement": "A case declaring a recency bound holds its in-window rate at or above 95 percent.", "why": "", "file": "spec/81-engineering.md", - "line": 276 + "line": 278 }, { "id": "assurance.evaluate.abstention", @@ -20993,7 +21004,7 @@ "statement": "A must-abstain case returns zero rows.", "why": "", "file": "spec/81-engineering.md", - "line": 277 + "line": 279 }, { "id": "assurance.evaluate.truth-per-surface", @@ -21004,7 +21015,7 @@ "statement": "Expected artifacts score the artifact retriever and expected edges score the edge retriever, reported apart; edge truth with no edge surface configured counts in an unscored tally.", "why": "", "file": "spec/81-engineering.md", - "line": 278 + "line": 280 }, { "id": "assurance.evaluate.judged-dimensions", @@ -21015,7 +21026,7 @@ "statement": "The reading stage adds four judged dimensions: answer accuracy; abstention, as correct-refusal and hallucination-on-unknown rates; citation faithfulness; and temporal correctness against the bitemporal columns.", "why": "", "file": "spec/81-engineering.md", - "line": 279 + "line": 281 }, { "id": "assurance.evaluate.grounded-reader", @@ -21026,7 +21037,7 @@ "statement": "The harness reader answers from retrieved rows alone, and the deterministic tier substitutes a stub reader.", "why": "", "file": "spec/81-engineering.md", - "line": 280 + "line": 282 }, { "id": "assurance.evaluate.model-endpoint", @@ -21037,7 +21048,7 @@ "statement": "The judge and the reader reach a model through {{connector.infer.model-endpoint}}, and the harness holds no credential.", "why": "", "file": "spec/81-engineering.md", - "line": 281 + "line": 283 }, { "id": "assurance.evaluate.judge", @@ -21048,7 +21059,7 @@ "statement": "The judge is one pinned open-weights instruct model, called once per judged item at temperature zero; a judge swap is a full re-baseline.", "why": "because a majority vote at temperature zero samples one mode and adds cost without reducing variance", "file": "spec/81-engineering.md", - "line": 282 + "line": 284 }, { "id": "assurance.evaluate.systems-metrics", @@ -21059,7 +21070,7 @@ "statement": "Tokens per query, latency and cost report beside the quality figures, bucketed by corpus size in tokens relative to the reader's context window.", "why": "", "file": "spec/81-engineering.md", - "line": 284 + "line": 286 }, { "id": "assurance.evaluate.checkpoint", @@ -21070,7 +21081,7 @@ "statement": "A run appends each case's result as JSONL, and a crash re-runs the in-flight case.", "why": "", "file": "spec/81-engineering.md", - "line": 285 + "line": 287 }, { "id": "assurance.evaluate.run-report", @@ -21081,7 +21092,7 @@ "statement": "A run report carries one field per gated metric, the sample count behind each mean, the run block, the per-slice breakdown, and the tally of cases sampled, dropped and unscored.", "why": "", "file": "spec/81-engineering.md", - "line": 286 + "line": 288 }, { "id": "assurance.baseline.file", @@ -21092,7 +21103,7 @@ "statement": "A baseline file carries a reserved run block `{k, tier, model, samples}` and one entry per gated metric: a bare number at the run's dead band, or an object overriding the band.", "why": "", "file": "spec/81-engineering.md", - "line": 297 + "line": 299 }, { "id": "assurance.baseline.default-dead-band", @@ -21103,7 +21114,7 @@ "statement": "A rate-valued entry with no override gates at a 2 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 298 + "line": 300 }, { "id": "assurance.baseline.rank-quality-dead-band", @@ -21114,7 +21125,7 @@ "statement": "The ranked-quality entry gates at a 3 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 299 + "line": 301 }, { "id": "assurance.baseline.band-units", @@ -21125,7 +21136,7 @@ "statement": "A band carries its metric's own units, and a count entry is pinned at zero with no band.", "why": "", "file": "spec/81-engineering.md", - "line": 300 + "line": 302 }, { "id": "assurance.baseline.metric-path", @@ -21136,7 +21147,7 @@ "statement": "An entry names a metric by the report's field path: `retrieval..`, `edge_retrieval..`, `judge.`, `latency_ms`, `n_cases`, or `slices..` before a bare `` or any of these but `latency_ms`.", "why": "", "file": "spec/81-engineering.md", - "line": 301 + "line": 303 }, { "id": "assurance.baseline.sample-count", @@ -21147,7 +21158,7 @@ "statement": "Every mean-valued path takes a `.n` suffix naming its sample count.", "why": "because blanking one case's truth lifts a NaN-filtered mean without moving the case count", "file": "spec/81-engineering.md", - "line": 302 + "line": 304 }, { "id": "assurance.baseline.unresolvable-path", @@ -21158,7 +21169,7 @@ "statement": "A path resolving to nothing, a mean over fewer than 30 cases, a malformed entry or a non-finite band raises `BaselinePathUnresolved` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 304 + "line": 306 }, { "id": "assurance.baseline.run-stamp-drift", @@ -21169,7 +21180,7 @@ "statement": "A run block differing from the run's own configuration raises `BaselineRunStampMismatch` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 306 + "line": 308 }, { "id": "assurance.baseline.interval", @@ -21180,7 +21191,7 @@ "statement": "Each judged figure reports a 95 percent percentile-bootstrap interval over cases from 1000 resamples.", "why": "", "file": "spec/81-engineering.md", - "line": 308 + "line": 310 }, { "id": "assurance.baseline.raise-only", @@ -21191,7 +21202,7 @@ "statement": "A baseline update runs after every gate and floor passes, moves each improved entry to its measured value, moves none in the worse direction, and adds no path.", "why": "", "file": "spec/81-engineering.md", - "line": 309 + "line": 311 }, { "id": "assurance.baseline.floors-are-absolute", @@ -21202,7 +21213,7 @@ "statement": "An absolute floor gates independently of every committed value.", "why": "", "file": "spec/81-engineering.md", - "line": 310 + "line": 312 }, { "id": "assurance.baseline.floor-coverage", @@ -21213,7 +21224,7 @@ "statement": "A report lacking a floor's figure on any leg of its `retrieval` surface breaches that floor; only a forbidden-row or in-window rate over no case holds.", "why": "", "file": "spec/81-engineering.md", - "line": 311 + "line": 313 }, { "id": "assurance.baseline.offline", @@ -21224,7 +21235,7 @@ "statement": "The gate is a local command comparing the report against in-tree baselines and floors, reporting both verdicts, with or without a trace store reachable.", "why": "", "file": "spec/81-engineering.md", - "line": 312 + "line": 314 }, { "id": "assurance.baseline.trace-store", @@ -21235,7 +21246,7 @@ "statement": "A trace store records run history and curation staging, and decides no verdict.", "why": "", "file": "spec/81-engineering.md", - "line": 313 + "line": 315 }, { "id": "assurance.baseline.golden-custody", @@ -21246,7 +21257,7 @@ "statement": "Version-controlled JSONL under `evals/cases/` is the canonical golden set, one loader is its only ingestion door, and a reviewer's edit reaches the gate as a change to the tree.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 314 + "line": 316 }, { "id": "assurance.baseline.answer-key-in-the-store", @@ -21257,7 +21268,7 @@ "statement": "An artifact path normalizing under the evaluation directory, reached by a manifest lint or a connector, raises `GoldenSetIngested`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 316 + "line": 318 }, { "id": "assurance.baseline.generated-redaction", @@ -21268,7 +21279,7 @@ "statement": "A golden mined from a live query or promoted from a labeled outcome passes {{authority.redact.write-time}} before commit; a case it cannot redact raises `GoldenRedactionFailed`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 318 + "line": 320 }, { "id": "assurance.baseline.promotion-source", @@ -21279,7 +21290,7 @@ "statement": "Only an adjudicated outcome label promotes into a golden set; a self-rated row does not.", "why": "", "file": "spec/81-engineering.md", - "line": 320 + "line": 322 }, { "id": "assurance.baseline.generators", @@ -21290,7 +21301,7 @@ "statement": "Three generators draft candidate cases from the deployment's store — entity-to-fact-to-edge walks, composed edge chains, and questions about absent entities — and a human approves each before commit.", "why": "", "file": "spec/81-engineering.md", - "line": 321 + "line": 323 }, { "id": "assurance.baseline.held-out", @@ -21301,7 +21312,7 @@ "statement": "Retrieval and synthesis are tuned on no gate input.", "why": "", "file": "spec/81-engineering.md", - "line": 322 + "line": 324 }, { "id": "assurance.baseline.rotation", @@ -21312,7 +21323,7 @@ "statement": "The native set rotates on a cadence, a sample of its truth is reviewed on each rotation, and a comparison across generations resolves through the pinned run stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 323 + "line": 325 }, { "id": "assurance.baseline.native-gate", @@ -21323,7 +21334,7 @@ "statement": "A native benchmark generated from a real corpus is the red or green gate; public benchmark sets run beside it as held-out comparison and decide nothing.", "why": "", "file": "spec/81-engineering.md", - "line": 324 + "line": 326 }, { "id": "assurance.baseline.trace-export", @@ -21334,7 +21345,7 @@ "statement": "A run touching a deployed store with a hosted trace endpoint configured raises `TraceExportOutOfPerimeter`; hosted collection serves runs over public or synthetic fixtures alone.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 325 + "line": 327 }, { "id": "assurance.measure.ledger", @@ -21345,7 +21356,7 @@ "statement": "`evals/ledger.toml` holds one entry per tracked target: an id, an owning clause id, a run-report metric path, a tier, a method naming one integration test, case set or probe, and a threshold.", "why": "", "file": "spec/81-engineering.md", - "line": 334 + "line": 336 }, { "id": "assurance.measure.unresolved-entry", @@ -21356,7 +21367,7 @@ "statement": "An entry whose owning clause, method or metric path resolves to nothing raises `MeasureEntryUnresolved` before any measure runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 335, + "line": 337, "scenarios": [ "WHEN an entry names `run.journal.entry-keys`, THEN the run raises `MeasureEntryUnresolved` and no test runs." ] @@ -21370,7 +21381,7 @@ "statement": "An entry naming an issue in place of a method reports open and gates nothing, and `evals/ledger.md` carries every entry's computed status.", "why": "", "file": "spec/81-engineering.md", - "line": 337, + "line": 339, "scenarios": [ "WHEN an entry's method is `{ issue = 43 }`, THEN `evals/ledger.md` lists it open and the evaluate stage ignores it." ] @@ -21384,7 +21395,7 @@ "statement": "A gate-tier entry decides the evaluate stage, a trend-tier entry records on every run and decides nothing, and a scheduled-tier entry runs on the scheduled job alone.", "why": "", "file": "spec/81-engineering.md", - "line": 338 + "line": 340 }, { "id": "assurance.measure.count-first", @@ -21395,7 +21406,7 @@ "statement": "A gate-tier entry measures a count, a ratio within one run or a size under a locked resolve; a wall-clock or resident-memory figure is trend-tier.", "why": "because a shared container moves wall-clock figures past any band narrow enough to catch a regression", "file": "spec/81-engineering.md", - "line": 339 + "line": 341 }, { "id": "assurance.measure.record", @@ -21406,7 +21417,7 @@ "statement": "A measure writes one JSON record carrying its entry id, value, sample count, seed and run stamp; a gate-tier method finishing without one raises `MeasureRecordMissing`.", "why": "P7", "file": "spec/81-engineering.md", - "line": 341 + "line": 343 }, { "id": "assurance.measure.seeded", @@ -21417,7 +21428,7 @@ "statement": "Every generated fixture and randomized schedule derives from the record's seed, and replaying that seed reproduces a count-valued entry's value.", "why": "because a failure that cannot replay cannot be fixed", "file": "spec/81-engineering.md", - "line": 343 + "line": 345 }, { "id": "assurance.measure.seed-mismatch", @@ -21428,7 +21439,7 @@ "statement": "A record whose seed differs from its entry's declared seed raises `MeasureSeedMismatch`, and the entry counts as red.", "why": "because a figure measured under another seed replays nothing the ledger names", "file": "spec/81-engineering.md", - "line": 345 + "line": 347 }, { "id": "assurance.measure.timing-iterations", @@ -21439,7 +21450,7 @@ "statement": "A timed batch reports p50 and p95 over 200 repeats of its operation, each after warm-up.", "why": "", "file": "spec/81-engineering.md", - "line": 347 + "line": 349 }, { "id": "assurance.measure.timing-batches", @@ -21450,7 +21461,7 @@ "statement": "A timed figure is the median of 5 repeats of its timed batch.", "why": "", "file": "spec/81-engineering.md", - "line": 348 + "line": 350 }, { "id": "assurance.measure.trend-band", @@ -21461,7 +21472,7 @@ "statement": "A trend figure more than 25 percent worse than its baseline annotates the run report and fails no stage.", "why": "", "file": "spec/81-engineering.md", - "line": 349, + "line": 351, "scenarios": [ "WHEN a trend p95 moves from 40 ms to 52 ms on a matching runner stamp, THEN the report carries a +30 percent annotation and the stage passes." ] @@ -21475,7 +21486,7 @@ "statement": "The run block carries the runner's processor model, processor count and memory limit, and a trend figure compares only against a baseline with the same stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 350 + "line": 352 }, { "id": "assurance.measure.absolute-threshold", @@ -21486,7 +21497,7 @@ "statement": "A threshold is an absolute figure of this system's own measure, and it tightens only through {{assurance.baseline.raise-only}}.", "why": "", "file": "spec/81-engineering.md", - "line": 351 + "line": 353 }, { "id": "assurance.measure.history", @@ -21497,7 +21508,7 @@ "statement": "A run on the default branch attaches its run report to the measured commit under `refs/notes/measures`, and no verdict reads a note.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 352 + "line": 354 }, { "id": "assurance.measure.scheduled", @@ -21508,7 +21519,7 @@ "statement": "A nightly job runs every tier against the default-branch head and reports one check per run.", "why": "", "file": "spec/81-engineering.md", - "line": 354 + "line": 356 }, { "id": "assurance.measure.measured-basis", @@ -21519,7 +21530,7 @@ "statement": "A bound with a measured basis names a trend-tier or scheduled-tier ledger entry id as its benchmark.", "why": "", "file": "spec/81-engineering.md", - "line": 355 + "line": 357 }, { "id": "assurance.release.version", @@ -21530,7 +21541,7 @@ "statement": "A release tag is `v0..`: `` counts the milestones computing `closed` in the tagged commit's `spec/status.md`, and `` counts the existing tags `v0..*`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 383, + "line": 385, "scenarios": [ "WHEN milestones 0 to 7 and 11 compute `closed` and `v0.9.0` exists, THEN the computed version is `v0.9.1`." ] @@ -21544,7 +21555,7 @@ "statement": "`contextful-ci tag` creates a signed annotated tag on `HEAD` naming the closed milestones once every refusal of this operation clears, and never moves or replaces an existing tag.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 385 + "line": 387 }, { "id": "assurance.release.dirty-tree", @@ -21555,7 +21566,7 @@ "statement": "A working tree or index differing from `HEAD` raises `TagTreeDirty`.", "why": "because the gate then judges files the tagged commit does not hold", "file": "spec/81-engineering.md", - "line": 387 + "line": 389 }, { "id": "assurance.release.off-branch", @@ -21566,7 +21577,7 @@ "statement": "A `HEAD` unreachable from the default branch, `origin/HEAD` unless `--branch` names another, raises `TagOffDefaultBranch`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 389 + "line": 391 }, { "id": "assurance.release.version-regressed", @@ -21577,7 +21588,7 @@ "statement": "A computed version below the highest existing `v..` tag raises `TagVersionRegressed`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 391, + "line": 393, "scenarios": [ "WHEN milestone 11 reopens after `v0.9.0`, THEN the computed `v0.8.0` raises `TagVersionRegressed`." ] @@ -21591,7 +21602,7 @@ "statement": "A `[workspace.package]` version in `HEAD`'s `Cargo.toml` that is absent or differs from the computed version raises `TagWorkspaceVersionMismatch`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 393 + "line": 395 }, { "id": "assurance.release.gate-failed", @@ -21602,7 +21613,7 @@ "statement": "Once the other refusals clear, every gate stage runs against `HEAD` with `HEAD~1` as base; a failing stage raises `TagGateFailed`, naming the stage's refusal, and no tag is created.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 395 + "line": 397 } ], "owns": { diff --git a/spec/status.md b/spec/status.md index beadc3229..d06a350ed 100644 --- a/spec/status.md +++ b/spec/status.md @@ -5,17 +5,17 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 135 | 0 | 135 | +| `assurance` | 2 | 15 | 228 | 59 | 22 | 12 | 136 | 0 | 136 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | | `connector` | 2 | 14 | 321 | 80 | 21 | 16 | 195 | 0 | 195 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | -| `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 162 | -| `run` | 3 | 26 | 372 | 98 | 36 | 37 | 259 | 0 | 257 | -| `store` | 1 | 17 | 235 | 55 | 13 | 20 | 198 | 0 | 194 | -| `surface` | 2 | 19 | 86 | 46 | 22 | 5 | 51 | 0 | 50 | +| `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | +| `run` | 3 | 26 | 372 | 98 | 36 | 37 | 259 | 0 | 259 | +| `store` | 1 | 17 | 235 | 55 | 13 | 20 | 198 | 0 | 198 | +| `surface` | 2 | 19 | 86 | 46 | 22 | 5 | 51 | 0 | 51 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1335 | 0 | | +| **total** | 19 | 154 | 1957 | 569 | 188 | 133 | 1336 | 0 | | Decision records: 18. @@ -39,7 +39,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 11 — The derive tier | 7 | 78 | 55 | passing | open | | 12 — The console | 11 | 30 | 0 | absent | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | -| 14 — Assurance | 7 | 130 | 83 | passing | closed | +| 14 — Assurance | 7 | 131 | 84 | passing | closed | Unscheduled operations: 10. @@ -137,6 +137,7 @@ Unscheduled operations: 10. | `assurance.gate.stage-sequence` | `tools/ci/tests/integration/stages.rs::the_gate_defines_the_thirteen_stages_in_run_order_and_runs_a_named_subset` | performed | | `assurance.gate.stage-subset` | `tools/ci/tests/integration/stages.rs::a_subset_omitting_a_predecessor_whose_output_is_absent_is_refused_before_any_stage` | performed | | `assurance.gate.surface-check-failed` | `tools/ci/tests/integration/stages.rs::a_failing_surface_check_is_refused_naming_the_surface_and_the_script` | performed | +| `assurance.gate.test-first-parts` | `tools/ci/tests/integration/test_first.rs::each_changed_test_package_has_its_own_dispatch_part` | performed | | `assurance.gate.typescript-surfaces` | `tools/ci/tests/integration/stages.rs::the_surfaces_stage_installs_then_runs_each_declared_check` | performed | | `assurance.gate.workspace-parts` | `tools/ci/tests/integration/target_dirs.rs::remote_workspace_parts_compile_the_union_and_run_each_package_suite` | performed | | `assurance.measure.ledger` | `tools/eval/tests/integration/ledger.rs::an_entry_carries_its_clause_metric_tier_method_and_threshold` | performed | diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 28974d6d7..556f8a883 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -88,6 +88,12 @@ enum Cmd { /// as its parts. #[arg(long)] parts: bool, + /// Include one test-first part per changed test package against this revision. + #[arg(long, requires = "parts")] + base: Option, + /// Print the selected stage names as a JSON array for a workflow matrix. + #[arg(long)] + json: bool, }, /// Hold every key in a git-tracked `.env*` file to ciphertext under a scope comment. Secrets, @@ -220,11 +226,13 @@ fn refuse(code: &'static str, message: String) -> anyhow::Error { fn main() { let cli = Cli::parse(); let result = match cli.cmd { - Cmd::Stages { parts: false } => { - STAGES.iter().for_each(|s| println!("{s}")); - Ok(()) + Cmd::Stages { parts, base, json } => { + let listed = if parts { repo_root().and_then(|root| dispatched(&root, base.as_deref())) } else { Ok(STAGES.iter().map(|s| s.to_string()).collect()) }; + listed.and_then(|stages| { + if json { println!("{}", serde_json::to_string(&stages)?); } else { stages.iter().for_each(|s| println!("{s}")); } + Ok(()) + }) } - Cmd::Stages { parts: true } => repo_root().and_then(|root| dispatched(&root)).map(|all| all.iter().for_each(|s| println!("{s}"))), Cmd::Gate { stages, predecessors, base, base_bound_secs } => gate(&stages, predecessors, &base, Duration::from_secs(base_bound_secs)), Cmd::Secrets => repo_root().and_then(|root| secrets(&root)), Cmd::Mirrors => repo_root().and_then(|root| mirrors(&root)), @@ -333,13 +341,14 @@ fn repo_root() -> Result { /// The stages whose work splits into parts the pull-request workflow dispatches one check /// each, so each part fits one stage's wall clock (`assurance.build.profile-build`, /// `assurance.gate.budget-stage`). -const SPLIT: [&str; 3] = ["workspace", "features", "budget"]; +const SPLIT: [&str; 4] = ["test-first", "workspace", "features", "budget"]; /// The parts of a split `stage`: the features stage's `packages`, every featured package but /// the binary, then `binary-` per run of the binary package; the budget stage's /// `` per profile the binary declares, its name after `contextful-`. -fn parts(root: &Path, stage: &str) -> Result> { +fn parts(root: &Path, stage: &str, base: Option<&str>) -> Result> { Ok(match stage { + "test-first" => match base { Some(base) => test_first_parts(root, base)?, None => vec!["validate".into()] }, "workspace" => ["compile", "foundation", "runtime", "read", "ci"].into_iter().map(str::to_string).collect(), "features" => { let featured = featured_packages(root)?; @@ -353,11 +362,11 @@ fn parts(root: &Path, stage: &str) -> Result> { /// What the pull-request workflow dispatches, in run order: each stage, a split stage as /// `.` per part. -fn dispatched(root: &Path) -> Result> { +fn dispatched(root: &Path, base: Option<&str>) -> Result> { let mut out = Vec::new(); for stage in STAGES { if SPLIT.contains(&stage) { - out.extend(parts(root, stage)?.into_iter().map(|p| format!("{stage}.{p}"))); + out.extend(parts(root, stage, base)?.into_iter().map(|p| format!("{stage}.{p}"))); } else { out.push(stage.to_string()); } @@ -380,10 +389,10 @@ fn gate(named: &[String], predecessors: bool, base: &str, bound: Duration) -> Re stages.push(stage.to_string()); match part { None => whole.push(stage), - Some(part) if SPLIT.contains(&stage) && parts(&root, stage)?.iter().any(|p| p == part) => { + Some(part) if SPLIT.contains(&stage) && parts(&root, stage, Some(base))?.iter().any(|p| p == part) => { narrowed.entry(stage).or_default().push(part.to_string()) } - Some(_) => bail!("no part `{name}`; the parts are {}", dispatched(&root)?.join(", ")), + Some(_) => bail!("no part `{name}`; the parts are {}", dispatched(&root, None)?.join(", ")), } } narrowed.retain(|stage, _| !whole.contains(stage)); @@ -416,7 +425,7 @@ fn run_stage(root: &Path, stage: &str, only: Option<&[String]>, base: &str, boun "test-first" => { provision_lean(root)?; provision_wasm(root)?; - test_first(root, base, bound)? + test_first(root, base, bound, only)? } "workspace" => { provision_lean(root)?; @@ -893,21 +902,60 @@ fn workspace_packages(root: &Path) -> Result> { // ---------------------------------------------------------------- test-first -fn test_first(root: &Path, base: &str, bound: Duration) -> Result<()> { +fn changed_tests(base: &str) -> Result> { let range = format!("{base}...HEAD"); + Ok(git(&["diff", "--name-only", "--diff-filter=ACMR", &range])?.lines().filter(|p| is_test(p)).map(str::to_string).collect()) +} + +fn test_package(root: &Path, test: &str) -> Result { + let package = test.split_once("/tests/").map(|(p, _)| p).context("a test under a package")?; + let manifest = root.join(package).join("Cargo.toml"); + let contents = std::fs::read_to_string(&manifest).with_context(|| format!("reading {}", manifest.display()))?; + let value: toml::Value = contents.parse().with_context(|| format!("parsing {}", manifest.display()))?; + Ok(value["package"]["name"].as_str().context("the changed test package has a name")?.to_string()) +} + +fn test_first_parts(root: &Path, base: &str) -> Result> { + let mut packages = std::collections::BTreeSet::new(); + if !sources_outside_refactors(base)?.is_empty() { + for test in changed_tests(base)? { + packages.insert(test_package(root, &test)?); + } + } + Ok(std::iter::once("validate".to_string()).chain(packages).collect()) +} + +fn test_first(root: &Path, base: &str, bound: Duration, only: Option<&[String]>) -> Result<()> { let sources = sources_outside_refactors(base)?; if sources.is_empty() { eprintln!("test-first: no Rust source under crates/ or tools/ changed outside `Test-First: {REFACTOR_TRAILER}` commits"); return Ok(()); } - let added = git(&["diff", "--name-only", "--diff-filter=ACMR", &range])?; - let tests: Vec<&str> = added.lines().filter(|p| is_test(p)).collect(); + let changed = changed_tests(base)?; + let tests: Vec<&str> = changed.iter().map(String::as_str).collect(); if tests.is_empty() { return Err(refuse( "TestNotFirst", format!("{} source file(s) changed and no test under a package's tests/ did: {}", sources.len(), sources.join(", ")), )); } + let selected: Vec<&str> = match only { + Some(parts) if parts.len() == 1 && parts[0] == "validate" => return Ok(()), + Some(parts) => { + let mut selected = Vec::new(); + for test in tests { + if parts.contains(&test_package(root, test)?) { + selected.push(test); + } + } + selected + } + None => tests, + }; + let mut by_package: std::collections::BTreeMap> = Default::default(); + for test in selected { + by_package.entry(test_package(root, test)?).or_default().push(test); + } let scratch = root.join("target/test-first"); let tree = scratch.join("tree"); @@ -915,16 +963,20 @@ fn test_first(root: &Path, base: &str, bound: Duration) -> Result<()> { let _ = std::fs::remove_dir_all(&scratch); std::fs::create_dir_all(&scratch)?; git(&["worktree", "add", "--detach", "-q", &tree.to_string_lossy(), &merge_base(base)?])?; - let verdict = red_against_base(root, &tree, &scratch.join("target"), &tests, bound); + let verdict = (|| -> Result> { + let mut all_red = Vec::new(); + for (package, tests) in by_package { + let red = red_against_base(root, &tree, &scratch.join("target"), &tests, bound)?; + if red.is_empty() { + return Err(refuse("TestNotFirst", format!("the change's `{package}` tests pass against the base source, so they specify nothing it adds: {}", tests.join(", ")))); + } + all_red.extend(red); + } + Ok(all_red) + })(); let _ = git(&["worktree", "remove", "--force", &tree.to_string_lossy()]); let _ = std::fs::remove_dir_all(&scratch); let red = verdict?; - if red.is_empty() { - return Err(refuse( - "TestNotFirst", - format!("the change's tests pass against the base source, so they specify nothing it adds: {}", tests.join(", ")), - )); - } eprintln!("test-first: red against the base in {}", red.join(", ")); Ok(()) } diff --git a/tools/ci/tests/integration/test_first.rs b/tools/ci/tests/integration/test_first.rs index 45060788a..e158e4d87 100644 --- a/tools/ci/tests/integration/test_first.rs +++ b/tools/ci/tests/integration/test_first.rs @@ -68,6 +68,42 @@ fn a_test_failing_on_the_base_passes() { assert!(o.status.success(), "{}", stderr(&o)); } +// spec: assurance.gate.test-first-parts@8cdcd3c6 +#[test] +fn each_changed_test_package_has_its_own_dispatch_part() { + let r = Repo::init(); + r.write("crates/other/Cargo.toml", &manifest("other", "")); + r.write("crates/other/src/lib.rs", "pub fn value() -> i32 { 1 }\n"); + r.write("crates/other/tests/integration/main.rs", "mod value;\n"); + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn positive() { assert!(other::value() > 0); }\n"); + r.lock(); + r.commit("second package and workspace lock"); + let base = r.head(); + r.write("crates/demo/src/lib.rs", TRIPLE); + r.write("crates/demo/tests/integration/main.rs", "mod double;\nmod triple;\n"); + r.write("crates/demo/tests/integration/triple.rs", "#[test]\nfn triples() { assert_eq!(demo::triple(2), 6); }\n"); + r.write("crates/other/src/lib.rs", "pub fn value() -> i32 { 2 }\n"); + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn positive() { assert!(other::value() >= 1); }\n"); + r.commit("triple, test first"); + + let listed = r.run_ci(&["stages", "--parts", "--base", &base, "--json"]); + assert!(listed.status.success(), "{}", stderr(&listed)); + let stages: Vec = serde_json::from_slice(&listed.stdout).unwrap(); + assert!(stages.contains(&"test-first.validate".to_string()), "{stages:?}"); + assert!(stages.contains(&"test-first.demo".to_string()), "{stages:?}"); + assert!(stages.contains(&"test-first.other".to_string()), "{stages:?}"); + assert!(!stages.contains(&"test-first".to_string()), "{stages:?}"); + + let part = r.gate(&["--stage", "test-first.demo", "--base", &base]); + assert!(part.status.success(), "{}", stderr(&part)); + let part = r.gate(&["--stage", "test-first.other", "--base", &base]); + assert!(!part.status.success() && stderr(&part).contains("TestNotFirst"), "{}", stderr(&part)); + let whole = r.gate(&["--stage", "test-first", "--base", &base]); + assert!(!whole.status.success() && stderr(&whole).contains("other"), "{}", stderr(&whole)); + let validation = r.gate(&["--stage", "test-first.validate", "--base", &base]); + assert!(validation.status.success(), "{}", stderr(&validation)); +} + #[test] fn a_refactor_trailer_exempts_the_range() { let r = Repo::init(); @@ -92,21 +128,22 @@ fn a_refactor_commit_does_not_exempt_the_rest_of_the_range() { } #[test] -fn a_new_package_counts_red_alone_and_leaves_the_base_workspace_loadable() { +fn a_new_package_counts_red_and_leaves_the_base_workspace_loadable() { let r = Repo::init(); let base = r.head(); - // A new package with its own tests, beside a green-on-base test for a source change in `demo`. + // Both changed packages provide tests red against their own base source. r.write("crates/fresh/Cargo.toml", &crate::manifest("fresh", "")); r.write("crates/fresh/src/lib.rs", "pub fn one() -> i32 {\n 1\n}\n"); r.write("crates/fresh/tests/integration/main.rs", "#[test]\nfn one() {\n assert_eq!(fresh::one(), 1);\n}\n"); r.write("crates/demo/src/lib.rs", TRIPLE); - r.write("crates/demo/tests/integration/double.rs", "#[test]\nfn doubles() {\n assert_eq!(demo::double(3), 6);\n}\n"); - r.commit("a new package, and a demo test that specifies nothing new"); + r.write("crates/demo/tests/integration/main.rs", "mod double;\nmod triple;\n"); + r.write("crates/demo/tests/integration/triple.rs", "#[test]\nfn triples() { assert_eq!(demo::triple(2), 6); }\n"); + r.commit("two packages with red tests"); let o = r.gate(&["--stage", "test-first", "--base", &base]); let err = stderr(&o); assert!(o.status.success(), "{err}"); assert!(err.contains("crates/fresh (absent at base)"), "{err}"); - assert!(!err.contains("crates/demo"), "demo's test passes at base and counts green: {err}"); + assert!(err.contains("crates/demo"), "demo's test must count red: {err}"); assert!(!err.contains("failed to load manifest"), "{err}"); } diff --git a/tools/ci/tests/integration/workflow.rs b/tools/ci/tests/integration/workflow.rs index 4826f13e3..3302a28b9 100644 --- a/tools/ci/tests/integration/workflow.rs +++ b/tools/ci/tests/integration/workflow.rs @@ -18,13 +18,12 @@ fn the_workflow_dispatches_every_gate_stage() { } let yml = std::fs::read_to_string(repo_root().join(".github/workflows/gate.yml")).unwrap(); - let matrix = yml - .lines() - .find_map(|l| l.trim().strip_prefix("stage: [")) - .and_then(|l| l.strip_suffix(']')) - .expect("a `stage: [...]` matrix in gate.yml"); - let dispatched: Vec = matrix.split(',').map(|s| s.trim().to_string()).collect(); - assert_eq!(dispatched, stages); + assert!(stages.contains(&"test-first.validate".to_string()), "{stages:?}"); + assert!(yml.contains("stage: ${{ fromJson(needs.prepare.outputs.stages) }}"), "{yml}"); + assert!(yml.contains("stages --parts --base"), "{yml}"); + assert!(yml.contains("ref: ${{ github.event.pull_request.head.sha }}"), "{yml}"); + assert!(yml.contains("BASE: ${{ github.event.pull_request.base.sha }}"), "{yml}"); + assert!(yml.contains("stages=%s\\n' \"$stages\" >> \"$GITHUB_OUTPUT\""), "{yml}"); assert!(yml.contains("\"checkLabel\": \"${{ matrix.stage }}\"")); assert!(yml.contains("\"command\": \"cargo run --locked -q -p contextful-ci -- gate --predecessors --stage ${{ matrix.stage }}")); } From b89351f84b03d94f50d1ffe1687868a8e29ab260 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 17:12:43 +0800 Subject: [PATCH 003/155] Require changed tests for each source package --- spec/81-engineering.md | 2 +- spec/spec.lock.json | 6 +++- spec/status.md | 2 +- tools/ci/src/main.rs | 35 ++++++++++++++++++------ tools/ci/tests/integration/test_first.rs | 30 +++++++++++++++++++- 5 files changed, 63 insertions(+), 12 deletions(-) diff --git a/spec/81-engineering.md b/spec/81-engineering.md index f7bc6a494..d3240f36c 100644 --- a/spec/81-engineering.md +++ b/spec/81-engineering.md @@ -155,7 +155,7 @@ Stage order, secrets of record, the crate-graph, row-token, egress and dependenc *A-assurance* - `remote-check` — The pull-request workflow dispatches every stage the gate subcommand defines to a remote runner, a split stage one part at a time, each as one status check labelled with its name. *A-assurance* -- `test-first-parts` — A remote test-first validation checks for a changed test, and each changed test package checks only its own tests against the base in a separate dispatch. +- `test-first-parts` — Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each changed source or test package checks only its own tests against the base in a separate dispatch. *because one base build per package fits the sandbox's wall clock more reliably* - `workspace-parts` — Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups. *A-assurance* diff --git a/spec/spec.lock.json b/spec/spec.lock.json index d51eb351a..a1d937a2e 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -20434,7 +20434,7 @@ "operation": "gate", "subject": "test-first-parts", "kind": "behavior", - "statement": "A remote test-first validation checks for a changed test, and each changed test package checks only its own tests against the base in a separate dispatch.", + "statement": "Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each changed source or test package checks only its own tests against the base in a separate dispatch.", "why": "because one base build per package fits the sandbox's wall clock more reliably", "file": "spec/81-engineering.md", "line": 158 @@ -22971,6 +22971,10 @@ "assurance.build.wasi-probe", "assurance.gate.edge-budget" ], + [ + "assurance.gate.test-first-parts", + "assurance.test.test-first" + ], [ "assurance.gate.formal-stage", "assurance.audit-assumptions.check-command" diff --git a/spec/status.md b/spec/status.md index d06a350ed..bd85fb791 100644 --- a/spec/status.md +++ b/spec/status.md @@ -137,7 +137,7 @@ Unscheduled operations: 10. | `assurance.gate.stage-sequence` | `tools/ci/tests/integration/stages.rs::the_gate_defines_the_thirteen_stages_in_run_order_and_runs_a_named_subset` | performed | | `assurance.gate.stage-subset` | `tools/ci/tests/integration/stages.rs::a_subset_omitting_a_predecessor_whose_output_is_absent_is_refused_before_any_stage` | performed | | `assurance.gate.surface-check-failed` | `tools/ci/tests/integration/stages.rs::a_failing_surface_check_is_refused_naming_the_surface_and_the_script` | performed | -| `assurance.gate.test-first-parts` | `tools/ci/tests/integration/test_first.rs::each_changed_test_package_has_its_own_dispatch_part` | performed | +| `assurance.gate.test-first-parts` | `tools/ci/tests/integration/test_first.rs::a_source_package_requires_its_own_changed_test_when_another_package_is_red` | performed | | `assurance.gate.typescript-surfaces` | `tools/ci/tests/integration/stages.rs::the_surfaces_stage_installs_then_runs_each_declared_check` | performed | | `assurance.gate.workspace-parts` | `tools/ci/tests/integration/target_dirs.rs::remote_workspace_parts_compile_the_union_and_run_each_package_suite` | performed | | `assurance.measure.ledger` | `tools/eval/tests/integration/ledger.rs::an_entry_carries_its_clause_metric_tier_method_and_threshold` | performed | diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 556f8a883..8f2b1e7cc 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -907,19 +907,26 @@ fn changed_tests(base: &str) -> Result> { Ok(git(&["diff", "--name-only", "--diff-filter=ACMR", &range])?.lines().filter(|p| is_test(p)).map(str::to_string).collect()) } -fn test_package(root: &Path, test: &str) -> Result { - let package = test.split_once("/tests/").map(|(p, _)| p).context("a test under a package")?; - let manifest = root.join(package).join("Cargo.toml"); +fn changed_package(root: &Path, path: &str) -> Result { + let manifest = Path::new(path) + .ancestors() + .skip(1) + .map(|dir| root.join(dir).join("Cargo.toml")) + .find(|manifest| manifest.is_file()) + .with_context(|| format!("`{path}` belongs to no Cargo package"))?; let contents = std::fs::read_to_string(&manifest).with_context(|| format!("reading {}", manifest.display()))?; let value: toml::Value = contents.parse().with_context(|| format!("parsing {}", manifest.display()))?; - Ok(value["package"]["name"].as_str().context("the changed test package has a name")?.to_string()) + Ok(value.get("package").and_then(|p| p.get("name")).and_then(toml::Value::as_str).context("the changed path's package has a name")?.to_string()) } fn test_first_parts(root: &Path, base: &str) -> Result> { let mut packages = std::collections::BTreeSet::new(); - if !sources_outside_refactors(base)?.is_empty() { + for source in sources_outside_refactors(base)? { + packages.insert(changed_package(root, &source)?); + } + if !packages.is_empty() { for test in changed_tests(base)? { - packages.insert(test_package(root, &test)?); + packages.insert(changed_package(root, &test)?); } } Ok(std::iter::once("validate".to_string()).chain(packages).collect()) @@ -939,12 +946,19 @@ fn test_first(root: &Path, base: &str, bound: Duration, only: Option<&[String]>) format!("{} source file(s) changed and no test under a package's tests/ did: {}", sources.len(), sources.join(", ")), )); } + let source_packages: std::collections::BTreeSet = sources.iter().map(|source| changed_package(root, source)).collect::>()?; + let test_packages: std::collections::BTreeSet = tests.iter().map(|test| changed_package(root, test)).collect::>()?; + if only.is_none() || only.is_some_and(|parts| parts == ["validate"]) { + if let Some(missing) = source_packages.difference(&test_packages).next() { + return Err(refuse("TestNotFirst", format!("source package `{missing}` has no changed test under its tests/"))); + } + } let selected: Vec<&str> = match only { Some(parts) if parts.len() == 1 && parts[0] == "validate" => return Ok(()), Some(parts) => { let mut selected = Vec::new(); for test in tests { - if parts.contains(&test_package(root, test)?) { + if parts.contains(&changed_package(root, test)?) { selected.push(test); } } @@ -954,7 +968,12 @@ fn test_first(root: &Path, base: &str, bound: Duration, only: Option<&[String]>) }; let mut by_package: std::collections::BTreeMap> = Default::default(); for test in selected { - by_package.entry(test_package(root, test)?).or_default().push(test); + by_package.entry(changed_package(root, test)?).or_default().push(test); + } + if let Some(parts) = only { + if let Some(missing) = parts.iter().filter(|part| part.as_str() != "validate").find(|part| !by_package.contains_key(part.as_str())) { + return Err(refuse("TestNotFirst", format!("source package `{missing}` has no changed test under its tests/"))); + } } let scratch = root.join("target/test-first"); diff --git a/tools/ci/tests/integration/test_first.rs b/tools/ci/tests/integration/test_first.rs index e158e4d87..26c8ff371 100644 --- a/tools/ci/tests/integration/test_first.rs +++ b/tools/ci/tests/integration/test_first.rs @@ -68,7 +68,6 @@ fn a_test_failing_on_the_base_passes() { assert!(o.status.success(), "{}", stderr(&o)); } -// spec: assurance.gate.test-first-parts@8cdcd3c6 #[test] fn each_changed_test_package_has_its_own_dispatch_part() { let r = Repo::init(); @@ -104,6 +103,35 @@ fn each_changed_test_package_has_its_own_dispatch_part() { assert!(validation.status.success(), "{}", stderr(&validation)); } +// spec: assurance.gate.test-first-parts@0bd45e25 +#[test] +fn a_source_package_requires_its_own_changed_test_when_another_package_is_red() { + let r = Repo::init(); + r.write("crates/other/Cargo.toml", &manifest("other", "")); + r.write("crates/other/src/lib.rs", "pub fn value() -> i32 { 1 }\n"); + r.write("crates/other/tests/integration/main.rs", "mod value;\n"); + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn value() { assert_eq!(other::value(), 1); }\n"); + r.lock(); + r.commit("second package and workspace lock"); + let base = r.head(); + r.write("crates/demo/src/lib.rs", TRIPLE); + r.write("crates/other/src/lib.rs", "pub fn value() -> i32 { 2 }\n"); + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn value() { assert_eq!(other::value(), 2); }\n"); + r.commit("two source packages, one changed test package"); + + let listed = r.run_ci(&["stages", "--parts", "--base", &base, "--json"]); + assert!(listed.status.success(), "{}", stderr(&listed)); + let stages: Vec = serde_json::from_slice(&listed.stdout).unwrap(); + assert!(stages.contains(&"test-first.demo".to_string()), "{stages:?}"); + assert!(stages.contains(&"test-first.other".to_string()), "{stages:?}"); + + let validation = r.gate(&["--stage", "test-first.validate", "--base", &base]); + assert!(!validation.status.success() && stderr(&validation).contains("TestNotFirst"), "{}", stderr(&validation)); + assert!(stderr(&validation).contains("demo"), "{}", stderr(&validation)); + let whole = r.gate(&["--stage", "test-first", "--base", &base]); + assert!(!whole.status.success() && stderr(&whole).contains("demo"), "{}", stderr(&whole)); +} + #[test] fn a_refactor_trailer_exempts_the_range() { let r = Repo::init(); From 38fd058713d3c6a6c354974fcb392f3fc458d59d Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 19:17:39 +0800 Subject: [PATCH 004/155] test(formal): require seeded protocol store differential --- .../contextful-cli/tests/integration/main.rs | 1 + .../integration/protocol_differential.rs | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 crates/contextful-cli/tests/integration/protocol_differential.rs diff --git a/crates/contextful-cli/tests/integration/main.rs b/crates/contextful-cli/tests/integration/main.rs index 691668620..5416c1a1e 100644 --- a/crates/contextful-cli/tests/integration/main.rs +++ b/crates/contextful-cli/tests/integration/main.rs @@ -16,6 +16,7 @@ mod context; #[cfg(feature = "data-plane")] mod derive; mod differential; +mod protocol_differential; #[cfg(feature = "data-plane")] mod document; #[cfg(feature = "data-plane")] diff --git a/crates/contextful-cli/tests/integration/protocol_differential.rs b/crates/contextful-cli/tests/integration/protocol_differential.rs new file mode 100644 index 000000000..31df062d1 --- /dev/null +++ b/crates/contextful-cli/tests/integration/protocol_differential.rs @@ -0,0 +1,28 @@ +//! The executable protocol model and Rust store agree after each generated step. + +use std::path::Path; +use std::process::Command; + +const BIN: &str = env!("CARGO_BIN_EXE_contextful"); + +fn run(seed: &str) -> std::process::Output { + Command::new(BIN) + .args(["formal", "protocol-differential", "--seed", seed, "--cases", "12"]) + .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) + .output() + .unwrap() +} + +#[test] +fn fixed_seed_replays_protocol_cases_against_the_store() { + if Command::new("lake").arg("--version").output().is_err() { + assert!(std::env::var_os("CONTEXTFUL_REQUIRE_LEAN").is_none(), "Lean is required"); + return; + } + let first = run("76004"); + assert!(first.status.success(), "{}", String::from_utf8_lossy(&first.stderr)); + let second = run("76004"); + assert!(second.status.success(), "{}", String::from_utf8_lossy(&second.stderr)); + assert_eq!(first.stdout, second.stdout); + assert!(String::from_utf8_lossy(&first.stdout).contains("stale-fence-differential: 0")); +} From 9be56d830f8ab6359530bdd9bdee512c4e0c6836 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 19:32:22 +0800 Subject: [PATCH 005/155] feat(formal): compare protocol model with fenced store decisions --- .github/workflows/protocol-differential.yml | 29 ++ crates/contextful-cli/Cargo.toml | 1 + crates/contextful-cli/src/formal.rs | 3 + crates/contextful-cli/src/lib.rs | 1 + .../src/protocol_differential.rs | 463 ++++++++++++++++++ .../integration/protocol_differential.rs | 54 +- evals/ledger.md | 4 +- evals/ledger.toml | 2 +- formal/protocol/Main.lean | 19 +- formal/protocol/regressions.jsonl | 3 + spec/pins.toml | 4 +- spec/status.md | 8 +- 12 files changed, 574 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/protocol-differential.yml create mode 100644 crates/contextful-cli/src/protocol_differential.rs create mode 100644 formal/protocol/regressions.jsonl diff --git a/.github/workflows/protocol-differential.yml b/.github/workflows/protocol-differential.yml new file mode 100644 index 000000000..cf331a1f8 --- /dev/null +++ b/.github/workflows/protocol-differential.yml @@ -0,0 +1,29 @@ +# The fixed seed runs in the workspace and evaluate stages; this schedule explores a +# fresh sequence over the same three nodes and four lease generations. +name: protocol differential + +on: + schedule: + - cron: "17 3 * * 0" + workflow_dispatch: + +permissions: + contents: read + +jobs: + random-seed: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - name: Provision the pinned Lean toolchain and run the fixed gate + run: cargo run --locked -q -p contextful-ci -- gate --predecessors --stage formal + - name: Compare a random protocol schedule + run: | + seed=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ') + cargo run --locked -q -p contextful-cli --no-default-features --bin contextful -- formal protocol-differential --seed "$seed" --cases 256 + - if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: protocol-regressions + path: formal/protocol/regressions.jsonl diff --git a/crates/contextful-cli/Cargo.toml b/crates/contextful-cli/Cargo.toml index 37bc5f74c..00ff98c15 100644 --- a/crates/contextful-cli/Cargo.toml +++ b/crates/contextful-cli/Cargo.toml @@ -44,6 +44,7 @@ url = "2" [dev-dependencies] ed25519-dalek = "2" +contextful-eval = { path = "../../tools/eval" } # The loopback S3 server the S3 sync suite pushes and pulls against. contextful-acceptance = { path = "../acceptance" } # Signing the external assertions `contextful token exchange` verifies. diff --git a/crates/contextful-cli/src/formal.rs b/crates/contextful-cli/src/formal.rs index 904732d5b..91dfd1b11 100644 --- a/crates/contextful-cli/src/formal.rs +++ b/crates/contextful-cli/src/formal.rs @@ -104,6 +104,8 @@ pub enum FormalCmd { /// decision functions and the Lean reference model; exits non-zero on the first /// disagreement. Differential(crate::differential::DifferentialArgs), + /// Compare generated lease and fenced write sequences with the executable protocol model. + ProtocolDifferential(crate::protocol_differential::Args), } pub fn run(cmd: FormalCmd) -> Result<()> { @@ -115,6 +117,7 @@ pub fn run(cmd: FormalCmd) -> Result<()> { } FormalCmd::Recheck { root } => recheck(root), FormalCmd::Differential(args) => crate::differential::run(args), + FormalCmd::ProtocolDifferential(args) => crate::protocol_differential::run(args), } } diff --git a/crates/contextful-cli/src/lib.rs b/crates/contextful-cli/src/lib.rs index 963abcec1..307f38d92 100644 --- a/crates/contextful-cli/src/lib.rs +++ b/crates/contextful-cli/src/lib.rs @@ -18,6 +18,7 @@ mod context; #[cfg(feature = "data-plane")] mod derive; mod differential; +mod protocol_differential; #[cfg(feature = "data-plane")] mod eval; #[cfg(feature = "data-plane")] diff --git a/crates/contextful-cli/src/protocol_differential.rs b/crates/contextful-cli/src/protocol_differential.rs new file mode 100644 index 000000000..a6d79b7ae --- /dev/null +++ b/crates/contextful-cli/src/protocol_differential.rs @@ -0,0 +1,463 @@ +//! Seeded operation sequences compare the executable protocol model with the store's +//! lease, pointer and commit-log decisions after every step. + +use anyhow::{bail, Context, Result}; +use clap::Args as ClapArgs; +use contextful_core::store::commit_log::{self, CommitEntry, Kind}; +use contextful_core::store::lease::{BucketLease, BucketPointer}; +use contextful_core::store::object::Condition; +use contextful_core::time::Instant; +use serde::{Deserialize, Serialize}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +const NODES: usize = 3; +const GENERATIONS: u64 = 4; +const STEPS_PER_CASE: usize = 32; +const DEFAULT_SEED: u64 = 76_004; +const REGRESSIONS: &str = "formal/protocol/regressions.jsonl"; + +#[derive(ClapArgs)] +pub struct Args { + /// The generator seed, printed with the result. + #[arg(long, default_value_t = DEFAULT_SEED)] + seed: u64, + /// Generated operation sequences after replaying regressions. + #[arg(long, default_value_t = 64)] + cases: usize, + /// A prebuilt protocol executable; absent, lake builds the package. + #[arg(long)] + reference: Option, + /// The saved minimized sequences. + #[arg(long)] + regressions: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum Target { + Catalog, + Cursor, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(tag = "op", rename_all = "snake_case")] +enum Step { + Acquire { node: usize }, + Renew { node: usize }, + Expire, + Release { node: usize }, + Send { node: usize, target: Target }, + Deliver { node: usize }, + Pause { node: usize }, + Resume { node: usize }, + Crash { node: usize }, +} + +impl Step { + fn line(self) -> String { + match self { + Self::Acquire { node } => format!("acquire {node}"), + Self::Renew { node } => format!("renew {node}"), + Self::Expire => "expire".into(), + Self::Release { node } => format!("release {node}"), + Self::Send { node, target } => format!("send {node} {}", target.name()), + Self::Deliver { node } => format!("deliver {node}"), + Self::Pause { node } => format!("pause {node}"), + Self::Resume { node } => format!("resume {node}"), + Self::Crash { node } => format!("crash {node}"), + } + } +} + +impl Target { + fn name(self) -> &'static str { + match self { + Self::Catalog => "catalog", + Self::Cursor => "cursor", + } + } +} + +#[derive(Clone, Default)] +struct Node { + belief: Option, + paused: bool, + pending: Option<(Target, u64)>, +} + +// mirrors: assurance.model.protocol-model +/// The conditional store keeps an ETag beside each lease generation. The model projects +/// the holder and fence; an unexpected condition failure remains a harness error. +struct Store { + lease: Option, + live: bool, + etag: u64, + catalog_etag: u64, + cursor_etag: u64, + catalog: BucketPointer, + cursor: Vec, + nodes: [Node; NODES], + now: Instant, +} + +impl Store { + fn new() -> Self { + Self { + lease: None, + live: false, + etag: 0, + catalog_etag: 0, + cursor_etag: 0, + catalog: BucketPointer::default(), + cursor: Vec::new(), + nodes: std::array::from_fn(|_| Node::default()), + now: Instant::from_unix_secs(0).expect("epoch is representable"), + } + } + + fn fence(&self) -> u64 { + self.lease.as_ref().map_or(0, |l| l.fence) + } + fn cursor_fence(&self) -> u64 { + self.cursor.iter().map(|e| e.fence).max().unwrap_or(0) + } + fn live(&self) -> bool { + self.live + } + fn etag_text(&self) -> String { + self.etag.to_string() + } + + fn apply(&mut self, step: Step) -> Result<()> { + let id = |n: usize| n.to_string(); + match step { + Step::Acquire { node } if !self.nodes[node].paused => { + // The protocol scheduler does not replace a live grant held by the same node. + if self.live() + && self.lease.as_ref().and_then(|l| l.holder.as_deref()) + == Some(id(node).as_str()) + { + return Ok(()); + } + let etag = self.etag_text(); + match BucketLease::acquire( + self.lease.as_ref().map(|l| (l, etag.as_str())), + &id(node), + self.now, + ) { + Ok((lease, condition)) => { + self.check_condition(condition)?; + self.etag += 1; + self.nodes[node].belief = Some(lease.fence); + self.catalog.fence = self.catalog.fence.max(lease.fence); + self.catalog_etag += 1; + self.cursor.push(CommitEntry { + kind: Kind::Acquire, + table: "protocol".into(), + run_id: None, + cursor: None, + fence: lease.fence, + }); + self.cursor_etag += 1; + self.lease = Some(lease); + self.live = true; + } + Err(contextful_core::store::StoreError::LeaseHeld(_)) if self.live() => {} + Err(err) => bail!("acquire: {err}"), + } + } + Step::Acquire { .. } => {} + Step::Renew { node } if !self.nodes[node].paused => { + if let (Some(lease), Some(belief)) = (&self.lease, self.nodes[node].belief) { + if lease.holder.as_deref() == Some(id(node).as_str()) && lease.fence == belief { + let etag = self.etag_text(); + let (renewed, condition) = + BucketLease::renew(lease, &etag, lease, self.now)?; + self.check_condition(condition)?; + self.etag += 1; + self.lease = Some(renewed); + self.live = true; + } + } + } + Step::Renew { .. } => {} + Step::Expire => { + if let Some(expiry) = self.lease.as_ref().and_then(|l| l.expires_at) { + if self.now <= expiry { + self.now = expiry.plus_secs(31); + } + } + if self.lease.is_some() { + self.live = false; + } + } + Step::Release { node } if !self.nodes[node].paused => { + if let (Some(lease), Some(belief)) = (&self.lease, self.nodes[node].belief) { + if lease.holder.as_deref() == Some(id(node).as_str()) && lease.fence == belief { + let etag = self.etag_text(); + let (released, condition) = BucketLease::release(lease, &etag, &id(node))?; + self.check_condition(condition)?; + self.etag += 1; + self.lease = Some(released); + self.nodes[node].belief = None; + } + } + } + Step::Release { .. } => {} + Step::Send { node, target } => { + let n = &mut self.nodes[node]; + if !n.paused && n.pending.is_none() { + if let Some(fence) = n.belief { + n.pending = Some((target, fence)); + } + } + } + Step::Deliver { node } => { + if let Some((target, fence)) = self.nodes[node].pending.take() { + match target { + Target::Catalog => { + if self.catalog.admit(fence).is_ok() { + self.catalog.fence = fence; + self.catalog_etag += 1; + } + } + Target::Cursor => { + if commit_log::admit(&self.cursor, "protocol", fence).is_ok() { + self.cursor.push(CommitEntry { + kind: Kind::Commit, + table: "protocol".into(), + run_id: Some(format!("node-{node}")), + cursor: None, + fence, + }); + self.cursor_etag += 1; + } + } + } + } + } + Step::Pause { node } => self.nodes[node].paused = true, + Step::Resume { node } => self.nodes[node].paused = false, + Step::Crash { node } => { + self.nodes[node].belief = None; + self.nodes[node].paused = false; + } + } + Ok(()) + } + + fn check_condition(&self, condition: Condition) -> Result<()> { + match condition { + Condition::IfNoneMatch if self.lease.is_none() => Ok(()), + Condition::IfMatch(etag) if self.lease.is_some() && etag == self.etag_text() => Ok(()), + other => bail!("unexpected lease condition {other:?} at ETag {}", self.etag), + } + } + + fn show(&self) -> String { + let lease = match &self.lease { + None => "none".to_string(), + Some(l) => format!( + "holder={} fence={} live={}", + option(l.holder.as_deref()), + l.fence, + self.live() + ), + }; + let views = self + .nodes + .iter() + .enumerate() + .map(|(i, n)| { + let pending = n + .pending + .map(|(t, f)| format!("(some {}@{f})", t.name())) + .unwrap_or_else(|| "none".into()); + format!( + "n{i}(belief={} paused={} pending={pending})", + option(n.belief), + n.paused + ) + }) + .collect::>() + .join(" "); + format!( + "lease[{lease}] catalog={} cursor={} {views} catalog-etag={} cursor-etag={}", + self.catalog.fence, + self.cursor_fence(), + self.catalog_etag, + self.cursor_etag + ) + } +} + +fn option(value: Option) -> String { + value + .map(|v| format!("(some {v})")) + .unwrap_or_else(|| "none".into()) +} + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n as u64) as usize + } +} + +fn generate(rng: &mut Rng) -> Vec { + let mut store = Store::new(); + let mut steps = Vec::with_capacity(STEPS_PER_CASE); + for _ in 0..STEPS_PER_CASE { + let node = rng.below(NODES); + let step = match rng.below(10) { + 0 if store.fence() < GENERATIONS => Step::Acquire { node }, + 1 => Step::Renew { node }, + 2 => Step::Expire, + 3 => Step::Release { node }, + 4 => Step::Send { + node, + target: Target::Catalog, + }, + 5 => Step::Send { + node, + target: Target::Cursor, + }, + 6 => Step::Deliver { node }, + 7 => Step::Pause { node }, + 8 => Step::Resume { node }, + _ => Step::Crash { node }, + }; + store.apply(step).expect("generated store step applies"); + steps.push(step); + } + steps +} + +fn compare(reference: &Path, steps: &[Step]) -> Result> { + let mut child = Command::new(reference) + .arg("run") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("starting {}", reference.display()))?; + { + let stdin = child.stdin.as_mut().context("protocol stdin")?; + for step in steps { + writeln!(stdin, "{}", step.line())?; + } + } + let out = child.wait_with_output()?; + if !out.status.success() { + bail!("protocol model: {}", String::from_utf8_lossy(&out.stderr)); + } + let text = String::from_utf8(out.stdout)?; + let states: Vec<_> = text.lines().collect(); + if states.len() != steps.len() { + bail!( + "protocol model returned {} states for {} steps", + states.len(), + steps.len() + ); + } + let mut store = Store::new(); + for (i, (step, model)) in steps.iter().zip(states).enumerate() { + store.apply(*step)?; + let actual = store.show(); + if model != actual { + return Ok(Some((i, model.into(), actual))); + } + } + Ok(None) +} + +fn minimize(reference: &Path, mut steps: Vec) -> Result> { + let mut i = 0; + while i < steps.len() && steps.len() > 1 { + let mut candidate = steps.clone(); + candidate.remove(i); + if compare(reference, &candidate)?.is_some() { + steps = candidate; + i = 0; + } else { + i += 1; + } + } + Ok(steps) +} + +pub fn run(args: Args) -> Result<()> { + let cwd = std::env::current_dir()?; + let repo = Command::new("git") + .args(["rev-parse", "--show-toplevel"]) + .current_dir(&cwd) + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| PathBuf::from(String::from_utf8_lossy(&o.stdout).trim())) + .unwrap_or(cwd); + let reference = match args.reference { + Some(path) => path, + None => { + let root = repo.join("formal/protocol"); + let out = Command::new("lake") + .arg("build") + .current_dir(&root) + .output() + .context("building protocol model")?; + if !out.status.success() { + bail!( + "building protocol model: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + root.join(".lake/build/bin/protocol") + } + }; + let regressions = args.regressions.unwrap_or_else(|| repo.join(REGRESSIONS)); + let saved = if regressions.exists() { + std::fs::read_to_string(®ressions)? + } else { + String::new() + }; + let mut cases = Vec::new(); + for line in saved.lines().filter(|line| !line.trim().is_empty()) { + cases.push(serde_json::from_str::>(line)?); + } + let replayed = cases.len(); + let mut rng = Rng(args.seed); + for _ in 0..args.cases { + cases.push(generate(&mut rng)); + } + for (case, steps) in cases.into_iter().enumerate() { + if let Some((at, model, store)) = compare(&reference, &steps)? { + let reduced = minimize(&reference, steps)?; + if let Some(parent) = regressions.parent() { + std::fs::create_dir_all(parent)?; + } + let entry = serde_json::to_string(&reduced)?; + if !saved.lines().any(|line| line == entry) { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(®ressions)?; + writeln!(file, "{entry}")?; + } + bail!("ProtocolConformanceDrift: seed={} case={case} step={at}; sequence={entry}; model={model}; store={store}", args.seed); + } + } + println!( + "stale-fence-differential: 0 drift cases; seed={}; generated={}; regressions={replayed}", + args.seed, args.cases + ); + Ok(()) +} diff --git a/crates/contextful-cli/tests/integration/protocol_differential.rs b/crates/contextful-cli/tests/integration/protocol_differential.rs index 31df062d1..b962c57b4 100644 --- a/crates/contextful-cli/tests/integration/protocol_differential.rs +++ b/crates/contextful-cli/tests/integration/protocol_differential.rs @@ -2,12 +2,20 @@ use std::path::Path; use std::process::Command; +use std::os::unix::fs::PermissionsExt; const BIN: &str = env!("CARGO_BIN_EXE_contextful"); fn run(seed: &str) -> std::process::Output { Command::new(BIN) - .args(["formal", "protocol-differential", "--seed", seed, "--cases", "12"]) + .args([ + "formal", + "protocol-differential", + "--seed", + seed, + "--cases", + "12", + ]) .current_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("../..")) .output() .unwrap() @@ -16,13 +24,49 @@ fn run(seed: &str) -> std::process::Output { #[test] fn fixed_seed_replays_protocol_cases_against_the_store() { if Command::new("lake").arg("--version").output().is_err() { - assert!(std::env::var_os("CONTEXTFUL_REQUIRE_LEAN").is_none(), "Lean is required"); + assert!( + std::env::var_os("CONTEXTFUL_REQUIRE_LEAN").is_none(), + "Lean is required" + ); return; } let first = run("76004"); - assert!(first.status.success(), "{}", String::from_utf8_lossy(&first.stderr)); + assert!( + first.status.success(), + "{}", + String::from_utf8_lossy(&first.stderr) + ); let second = run("76004"); - assert!(second.status.success(), "{}", String::from_utf8_lossy(&second.stderr)); + assert!( + second.status.success(), + "{}", + String::from_utf8_lossy(&second.stderr) + ); assert_eq!(first.stdout, second.stdout); - assert!(String::from_utf8_lossy(&first.stdout).contains("stale-fence-differential: 0")); + let report = String::from_utf8_lossy(&first.stdout); + assert!(report.contains("stale-fence-differential: 0")); + let replayed: u64 = report.split("regressions=").nth(1).unwrap().trim().parse().unwrap(); + contextful_eval::record::emit("stale-fence-differential", 0.0, 12 + replayed, 76004); +} + +#[test] +fn a_protocol_difference_prints_the_minimal_sequence_and_both_states() { + let dir = tempfile::tempdir().unwrap(); + let reference = dir.path().join("reference.sh"); + std::fs::write(&reference, "#!/bin/sh\ncat >/dev/null\necho 'lease[none] catalog=999 cursor=0 n0(belief=none paused=false pending=none) n1(belief=none paused=false pending=none) n2(belief=none paused=false pending=none)'\n").unwrap(); + std::fs::set_permissions(&reference, std::fs::Permissions::from_mode(0o755)).unwrap(); + let regressions = dir.path().join("regressions.jsonl"); + std::fs::write(®ressions, "[{\"op\":\"expire\"}]\n").unwrap(); + let out = Command::new(BIN) + .args(["formal", "protocol-differential", "--cases", "0", "--reference"]) + .arg(&reference) + .arg("--regressions") + .arg(®ressions) + .output().unwrap(); + assert!(!out.status.success()); + let error = String::from_utf8_lossy(&out.stderr); + assert!(error.contains("ProtocolConformanceDrift"), "{error}"); + assert!(error.contains("sequence=[{\"op\":\"expire\"}]"), "{error}"); + assert!(error.contains("model=lease[none] catalog=999"), "{error}"); + assert!(error.contains("store=lease[none] catalog=0"), "{error}"); } diff --git a/evals/ledger.md b/evals/ledger.md index c25820117..ce58e3904 100644 --- a/evals/ledger.md +++ b/evals/ledger.md @@ -2,7 +2,7 @@ # Target ledger -65 entries: 46 gated, 6 recorded, 2 scheduled, 11 open. +65 entries: 47 gated, 6 recorded, 2 scheduled, 10 open. | Entry | Clause | Metric | Tier | Method | Target | Status | | --- | --- | --- | --- | --- | --- | --- | @@ -63,7 +63,7 @@ | `read-session-one-engine` | `read.cache.session-pool` | `session.engine_opens` | gate | test `contextful_context::read::pool::statements_under_one_key_share_one_resolved_session_and_one_engine` | `== 1` | gated | | `row-key-dedup` | `read.retrieve.row-key-dedup` | `retrieval.hybrid.duplicate_row_rate` | gate | test `contextful_context::read::retrieve::a_ranked_read_keeps_the_newest_row_per_content_hash` | `== 0` | gated | | `sqlite-journal-conformance` | `run.journal.sqlite-stores` | `journal.sqlite_conformance.failed_cases` | gate | test `contextful_sqlite::stores::the_sqlite_stores_pass_every_conformance_suite_and_commit_updates_atomically` | `== 0` | gated | -| `stale-fence-differential` | `assurance.differential-test.protocol-harness` | `protocol.drift_cases` | gate | issue 76 | `== 0` | open (issue 76) | +| `stale-fence-differential` | `assurance.differential-test.protocol-harness` | `protocol.drift_cases` | gate | test `contextful_cli::protocol_differential::fixed_seed_replays_protocol_cases_against_the_store` | `== 0` | gated | | `stale-fence-never-lands` | `store.lease.stale-fence` | `lease.stale_commits_landed` | gate | test `contextful_context::run_commit::a_commit_under_a_superseded_fence_loses_and_its_run_stays_unreadable` | `== 0` | gated | | `store-write-deny-set` | `topology.package.store-write-engine-free` | `topology.store_write.forbidden_packages` | gate | test `contextful_ci::topology::this_repository_store_write_half_links_no_forbidden_package` | `== 0` | gated | | `store-write-package-count` | `topology.package.store-write-engine-free` | `topology.store_write.unique_packages` | gate | test `contextful_ci::topology::this_repository_store_write_half_links_no_forbidden_package` | `<= 200` | gated | diff --git a/evals/ledger.toml b/evals/ledger.toml index f0ef0b8e2..7898a2b6a 100644 --- a/evals/ledger.toml +++ b/evals/ledger.toml @@ -323,7 +323,7 @@ clause = "assurance.differential-test.protocol-harness" metric = "protocol.drift_cases" kind = "test" tier = "gate" -method = { issue = 76 } +method = { test = "contextful_cli::protocol_differential::fixed_seed_replays_protocol_cases_against_the_store" } target = { op = "==", value = 0 } [entry.fold-single-publisher] diff --git a/formal/protocol/Main.lean b/formal/protocol/Main.lean index 7865491f6..42ddcc2d7 100644 --- a/formal/protocol/Main.lean +++ b/formal/protocol/Main.lean @@ -127,23 +127,32 @@ def parseStep (line : String) : Option Step := | ["crash", n] => (parseNode n).map .crash | _ => none -partial def runLoop (stdin : IO.FS.Stream) (s : State) : IO UInt32 := do +partial def runLoop (stdin : IO.FS.Stream) (s : State) (catalogEtag cursorEtag : Nat) : IO UInt32 := do let line ← stdin.getLine if line.isEmpty then return 0 let text := line.trimAscii.toString - if text.isEmpty then return (← runLoop stdin s) + if text.isEmpty then return (← runLoop stdin s catalogEtag cursorEtag) match parseStep text with | none => IO.eprintln s!"unrecognised step: {text}" return 2 | some st => let s' := step s st - IO.println (showState s') - runLoop stdin s' + let grant := if s'.granted > s.granted then 1 else 0 + let catalogWrite := match landed s st with + | some (.catalog, _) => 1 + | _ => 0 + let cursorWrite := match landed s st with + | some (.cursor, _) => 1 + | _ => 0 + let catalogEtag' := catalogEtag + grant + catalogWrite + let cursorEtag' := cursorEtag + grant + cursorWrite + IO.println s!"{showState s'} catalog-etag={catalogEtag'} cursor-etag={cursorEtag'}" + runLoop stdin s' catalogEtag' cursorEtag' def main (args : List String) : IO UInt32 := do match args with - | ["run"] => runLoop (← IO.getStdin) State.init + | ["run"] => runLoop (← IO.getStdin) State.init 0 0 | [] | ["check"] | ["check", "--unfenced"] => let stepFn := if args.contains "--unfenced" then stepWith false else step match search stepFn with diff --git a/formal/protocol/regressions.jsonl b/formal/protocol/regressions.jsonl new file mode 100644 index 000000000..7cda2681b --- /dev/null +++ b/formal/protocol/regressions.jsonl @@ -0,0 +1,3 @@ +[{"op":"acquire","node":2},{"op":"release","node":2}] +[{"op":"acquire","node":0},{"op":"send","node":0,"target":"catalog"},{"op":"expire"},{"op":"acquire","node":1},{"op":"deliver","node":0}] +[{"op":"acquire","node":0},{"op":"send","node":0,"target":"cursor"},{"op":"expire"},{"op":"acquire","node":1},{"op":"deliver","node":0}] diff --git a/spec/pins.toml b/spec/pins.toml index 984f9605c..29949a513 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -27,6 +27,8 @@ # `contextful-spec state` reads this file and the tags and writes spec/status.md. [pin] +"assurance.differential-test.protocol-harness" = { test = "contextful_cli::protocol_differential::fixed_seed_replays_protocol_cases_against_the_store" } +"assurance.differential-test.protocol-drift" = { test = "contextful_cli::protocol_differential::a_protocol_difference_prints_the_minimal_sequence_and_both_states" } "corpus.targets.cap-unrecorded" = { test = "contextful_spec::targets::a_capped_shape_not_recording_its_exclusions_is_refused" } "corpus.targets.function-profile" = { test = "contextful_spec::targets::a_function_class_shape_hosting_the_full_profile_is_refused" } "corpus.targets.incomplete" = { test = "contextful_spec::targets::a_shape_leaving_a_role_unfilled_is_refused" } @@ -52,7 +54,7 @@ # its live performed count, and a live count below its floor raises SpecCoverageRegression. [floor] -assurance = 135 +assurance = 137 authority = 204 connector = 195 corpus = 38 diff --git a/spec/status.md b/spec/status.md index beadc3229..ad16eac93 100644 --- a/spec/status.md +++ b/spec/status.md @@ -5,7 +5,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 135 | 0 | 135 | +| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 137 | 0 | 137 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | | `connector` | 2 | 14 | 321 | 80 | 21 | 16 | 195 | 0 | 195 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | @@ -15,7 +15,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | `store` | 1 | 17 | 235 | 55 | 13 | 20 | 198 | 0 | 194 | | `surface` | 2 | 19 | 86 | 46 | 22 | 5 | 51 | 0 | 50 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1335 | 0 | | +| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1337 | 0 | | Decision records: 18. @@ -26,7 +26,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | Milestone | Operations | Clauses | Performed | Acceptance | Closed | | --- | --- | --- | --- | --- | --- | | 0 — The test-first gate | 2 | 18 | 6 | passing | open | -| 1 — The authority core | 14 | 236 | 184 | passing | closed | +| 1 — The authority core | 14 | 236 | 186 | passing | closed | | 2 — The store | 9 | 162 | 130 | passing | closed | | 3 — The run path | 11 | 206 | 142 | passing | closed | | 4 — Ingest | 25 | 468 | 290 | passing | open | @@ -93,6 +93,8 @@ Unscheduled operations: 10. | `assurance.differential-test.harness` | `crates/contextful-cli/tests/integration/differential.rs::the_engine_and_the_lean_reference_agree_over_generated_cases` | performed | | `assurance.differential-test.malformed-bytes` | `crates/contextful-cli/tests/integration/differential.rs::malformed_bytes_reach_every_decider_and_each_decides_them_malformed` | performed | | `assurance.differential-test.minimized` | `crates/contextful-cli/tests/integration/differential.rs::a_recorded_disagreement_is_minimal_under_every_shrinking_step` | performed | +| `assurance.differential-test.protocol-drift` | `spec/pins.toml` | performed | +| `assurance.differential-test.protocol-harness` | `spec/pins.toml` | performed | | `assurance.differential-test.reference-model` | `crates/contextful-cli/tests/integration/differential.rs::the_reference_binary_reads_one_case_and_prints_one_decision` | performed | | `assurance.differential-test.run-budget` | `crates/contextful-cli/tests/integration/differential.rs::a_run_past_its_budget_stops` | performed | | `assurance.differential-test.seed` | `crates/contextful-cli/tests/integration/differential.rs::a_recorded_seed_reproduces_the_case_sequence` | performed | From 865b8c1964176755c40c9e281092c424f1d9a100 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 19:37:14 +0800 Subject: [PATCH 006/155] test(formal): require bounded protocol case generation --- .../integration/protocol_differential.rs | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/crates/contextful-cli/tests/integration/protocol_differential.rs b/crates/contextful-cli/tests/integration/protocol_differential.rs index b962c57b4..5d540864e 100644 --- a/crates/contextful-cli/tests/integration/protocol_differential.rs +++ b/crates/contextful-cli/tests/integration/protocol_differential.rs @@ -70,3 +70,35 @@ fn a_protocol_difference_prints_the_minimal_sequence_and_both_states() { assert!(error.contains("model=lease[none] catalog=999"), "{error}"); assert!(error.contains("store=lease[none] catalog=0"), "{error}"); } + +#[test] +fn generated_protocol_cases_cover_the_three_nodes_and_every_step() { + let first = Command::new(BIN) + .args(["formal", "protocol-differential", "--seed", "76004", "--cases", "64", "--dump-cases"]) + .output() + .unwrap(); + assert!(first.status.success(), "{}", String::from_utf8_lossy(&first.stderr)); + let second = Command::new(BIN) + .args(["formal", "protocol-differential", "--seed", "76004", "--cases", "64", "--dump-cases"]) + .output() + .unwrap(); + assert_eq!(first.stdout, second.stdout); + let cases: Vec = first.stdout.split(|b| *b == b'\n').filter(|line| !line.is_empty()) + .map(|line| serde_json::from_slice(line).unwrap()).collect(); + assert_eq!(cases.len(), 64); + let mut nodes = std::collections::BTreeSet::new(); + let mut operations = std::collections::BTreeSet::new(); + for case in cases { + let steps = case.as_array().unwrap(); + assert_eq!(steps.len(), 32); + for step in steps { + if let Some(node) = step["node"].as_u64() { + assert!(node < 3); + nodes.insert(node); + } + operations.insert(step["op"].as_str().unwrap().to_string()); + } + } + assert_eq!(nodes, [0, 1, 2].into_iter().collect()); + assert_eq!(operations, ["acquire", "renew", "expire", "release", "send", "deliver", "pause", "resume", "crash"].into_iter().map(str::to_string).collect()); +} From c76666c7549092846e23f741536a752e4b6675dc Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 4 Oct 2026 19:42:04 +0800 Subject: [PATCH 007/155] feat(formal): generate protocol cases with proptest state machine --- Cargo.lock | 118 ++++++++++++- crates/contextful-cli/Cargo.toml | 2 + .../src/protocol_differential.rs | 162 +++++++++++++----- spec/pins.toml | 3 +- spec/status.md | 7 +- 5 files changed, 246 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c90a8491..fd1d73542 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,6 +586,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "1.3.2" @@ -945,6 +960,8 @@ dependencies = [ "contextful-wasm", "ed25519-dalek", "jsonwebtoken", + "proptest", + "proptest-state-machine", "serde", "serde_json", "sha2 0.10.9", @@ -3259,6 +3276,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.13.2", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "proptest-state-machine" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1675727965d66ff6f335e7d398e477184da08f4bed22f1a7f0dbf2f077f56f2e" +dependencies = [ + "proptest", +] + [[package]] name = "prost" version = "0.10.4" @@ -3315,6 +3360,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.37.5" @@ -3362,10 +3413,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", - "rand_chacha", + "rand_chacha 0.3.1", "rand_core 0.6.4", ] +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.3" @@ -3387,6 +3448,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -3396,12 +3467,30 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "rand_core" version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "rangemap" version = "1.8.0" @@ -3616,6 +3705,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "rusty-s3" version = "0.10.2" @@ -4485,6 +4586,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-bidi" version = "0.3.18" @@ -4612,6 +4719,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" diff --git a/crates/contextful-cli/Cargo.toml b/crates/contextful-cli/Cargo.toml index 00ff98c15..f7037b3ed 100644 --- a/crates/contextful-cli/Cargo.toml +++ b/crates/contextful-cli/Cargo.toml @@ -32,6 +32,8 @@ contextful-sync = { path = "../contextful-sync", optional = true } # The component host (`topology.package.component-host`), linked under `component-host`. contextful-wasm = { path = "../contextful-wasm", default-features = false, optional = true } anyhow = "1" +proptest = "1.10" +proptest-state-machine = "0.8" clap = { version = "4", features = ["derive", "env"] } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/contextful-cli/src/protocol_differential.rs b/crates/contextful-cli/src/protocol_differential.rs index a6d79b7ae..978fa6ffb 100644 --- a/crates/contextful-cli/src/protocol_differential.rs +++ b/crates/contextful-cli/src/protocol_differential.rs @@ -7,6 +7,10 @@ use contextful_core::store::commit_log::{self, CommitEntry, Kind}; use contextful_core::store::lease::{BucketLease, BucketPointer}; use contextful_core::store::object::Condition; use contextful_core::time::Instant; +use proptest::prelude::{prop_oneof, Just, Strategy}; +use proptest::strategy::ValueTree; +use proptest::test_runner::{Config, RngAlgorithm, TestRng, TestRunner}; +use proptest_state_machine::ReferenceStateMachine; use serde::{Deserialize, Serialize}; use std::io::Write; use std::path::{Path, PathBuf}; @@ -26,6 +30,9 @@ pub struct Args { /// Generated operation sequences after replaying regressions. #[arg(long, default_value_t = 64)] cases: usize, + /// Print generated cases without running the model or the store. + #[arg(long, hide = true)] + dump_cases: bool, /// A prebuilt protocol executable; absent, lake builds the package. #[arg(long)] reference: Option, @@ -117,9 +124,6 @@ impl Store { } } - fn fence(&self) -> u64 { - self.lease.as_ref().map_or(0, |l| l.fence) - } fn cursor_fence(&self) -> u64 { self.cursor.iter().map(|e| e.fence).max().unwrap_or(0) } @@ -299,47 +303,117 @@ fn option(value: Option) -> String { .unwrap_or_else(|| "none".into()) } -struct Rng(u64); -impl Rng { - fn next(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); - z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); - z ^ (z >> 31) - } - fn below(&mut self, n: usize) -> usize { - (self.next() % n as u64) as usize - } +#[derive(Clone, Copy, Debug, Default)] +struct GenerationNode { + belief: Option, + paused: bool, + pending: Option<(Target, u64)>, } -fn generate(rng: &mut Rng) -> Vec { - let mut store = Store::new(); - let mut steps = Vec::with_capacity(STEPS_PER_CASE); - for _ in 0..STEPS_PER_CASE { - let node = rng.below(NODES); - let step = match rng.below(10) { - 0 if store.fence() < GENERATIONS => Step::Acquire { node }, - 1 => Step::Renew { node }, - 2 => Step::Expire, - 3 => Step::Release { node }, - 4 => Step::Send { +#[derive(Clone, Debug, Default)] +struct GenerationState { + fence: u64, + holder: Option, + live: bool, + nodes: [GenerationNode; NODES], +} + +struct ProtocolGenerator; + +impl ReferenceStateMachine for ProtocolGenerator { + type State = GenerationState; + type Transition = Step; + + fn init_state() -> proptest::strategy::BoxedStrategy { + Just(GenerationState::default()).boxed() + } + + fn transitions(_state: &Self::State) -> proptest::strategy::BoxedStrategy { + prop_oneof![ + (0..NODES).prop_map(|node| Step::Acquire { node }), + (0..NODES).prop_map(|node| Step::Renew { node }), + Just(Step::Expire), + (0..NODES).prop_map(|node| Step::Release { node }), + (0..NODES).prop_map(|node| Step::Send { node, - target: Target::Catalog, - }, - 5 => Step::Send { + target: Target::Catalog + }), + (0..NODES).prop_map(|node| Step::Send { node, - target: Target::Cursor, - }, - 6 => Step::Deliver { node }, - 7 => Step::Pause { node }, - 8 => Step::Resume { node }, - _ => Step::Crash { node }, - }; - store.apply(step).expect("generated store step applies"); - steps.push(step); + target: Target::Cursor + }), + (0..NODES).prop_map(|node| Step::Deliver { node }), + (0..NODES).prop_map(|node| Step::Pause { node }), + (0..NODES).prop_map(|node| Step::Resume { node }), + (0..NODES).prop_map(|node| Step::Crash { node }), + ] + .boxed() + } + + fn preconditions(state: &Self::State, transition: &Self::Transition) -> bool { + !matches!(transition, Step::Acquire { .. }) || state.fence < GENERATIONS } - steps + + fn apply(mut state: Self::State, transition: &Self::Transition) -> Self::State { + match *transition { + Step::Acquire { node } + if !state.nodes[node].paused && (state.holder.is_none() || !state.live) => + { + state.fence += 1; + state.holder = Some(node); + state.live = true; + state.nodes[node].belief = Some(state.fence); + } + Step::Renew { node } + if !state.nodes[node].paused + && state.holder == Some(node) + && state.nodes[node].belief == Some(state.fence) => + { + state.live = true; + } + Step::Expire if state.fence > 0 => state.live = false, + Step::Release { node } + if !state.nodes[node].paused + && state.holder == Some(node) + && state.nodes[node].belief == Some(state.fence) => + { + state.holder = None; + state.nodes[node].belief = None; + } + Step::Send { node, target } + if !state.nodes[node].paused && state.nodes[node].pending.is_none() => + { + if let Some(fence) = state.nodes[node].belief { + state.nodes[node].pending = Some((target, fence)); + } + } + Step::Deliver { node } => state.nodes[node].pending = None, + Step::Pause { node } => state.nodes[node].paused = true, + Step::Resume { node } => state.nodes[node].paused = false, + Step::Crash { node } => { + state.nodes[node].belief = None; + state.nodes[node].paused = false; + } + _ => {} + } + state + } +} + +fn generator(seed: u64) -> TestRunner { + let bytes = seed.to_le_bytes().repeat(4); + TestRunner::new_with_rng( + Config::default(), + TestRng::from_seed(RngAlgorithm::ChaCha, &bytes), + ) +} + +fn generate(runner: &mut TestRunner) -> Result> { + let strategy = ProtocolGenerator::sequential_strategy(STEPS_PER_CASE); + let tree = strategy + .new_tree(runner) + .map_err(|e| anyhow::anyhow!("generating a protocol case: {e}"))?; + Ok(tree.current().1) } fn compare(reference: &Path, steps: &[Step]) -> Result> { @@ -396,6 +470,13 @@ fn minimize(reference: &Path, mut steps: Vec) -> Result> { } pub fn run(args: Args) -> Result<()> { + let mut runner = generator(args.seed); + if args.dump_cases { + for _ in 0..args.cases { + println!("{}", serde_json::to_string(&generate(&mut runner)?)?); + } + return Ok(()); + } let cwd = std::env::current_dir()?; let repo = Command::new("git") .args(["rev-parse", "--show-toplevel"]) @@ -434,9 +515,8 @@ pub fn run(args: Args) -> Result<()> { cases.push(serde_json::from_str::>(line)?); } let replayed = cases.len(); - let mut rng = Rng(args.seed); for _ in 0..args.cases { - cases.push(generate(&mut rng)); + cases.push(generate(&mut runner)?); } for (case, steps) in cases.into_iter().enumerate() { if let Some((at, model, store)) = compare(&reference, &steps)? { diff --git a/spec/pins.toml b/spec/pins.toml index 29949a513..69a1b1278 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -27,6 +27,7 @@ # `contextful-spec state` reads this file and the tags and writes spec/status.md. [pin] +"assurance.differential-test.protocol-cases" = { test = "contextful_cli::protocol_differential::generated_protocol_cases_cover_the_three_nodes_and_every_step" } "assurance.differential-test.protocol-harness" = { test = "contextful_cli::protocol_differential::fixed_seed_replays_protocol_cases_against_the_store" } "assurance.differential-test.protocol-drift" = { test = "contextful_cli::protocol_differential::a_protocol_difference_prints_the_minimal_sequence_and_both_states" } "corpus.targets.cap-unrecorded" = { test = "contextful_spec::targets::a_capped_shape_not_recording_its_exclusions_is_refused" } @@ -54,7 +55,7 @@ # its live performed count, and a live count below its floor raises SpecCoverageRegression. [floor] -assurance = 137 +assurance = 138 authority = 204 connector = 195 corpus = 38 diff --git a/spec/status.md b/spec/status.md index ad16eac93..00c129ba4 100644 --- a/spec/status.md +++ b/spec/status.md @@ -5,7 +5,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 137 | 0 | 137 | +| `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 138 | 0 | 138 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | | `connector` | 2 | 14 | 321 | 80 | 21 | 16 | 195 | 0 | 195 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | @@ -15,7 +15,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | `store` | 1 | 17 | 235 | 55 | 13 | 20 | 198 | 0 | 194 | | `surface` | 2 | 19 | 86 | 46 | 22 | 5 | 51 | 0 | 50 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1337 | 0 | | +| **total** | 19 | 154 | 1956 | 569 | 188 | 133 | 1338 | 0 | | Decision records: 18. @@ -26,7 +26,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | Milestone | Operations | Clauses | Performed | Acceptance | Closed | | --- | --- | --- | --- | --- | --- | | 0 — The test-first gate | 2 | 18 | 6 | passing | open | -| 1 — The authority core | 14 | 236 | 186 | passing | closed | +| 1 — The authority core | 14 | 236 | 187 | passing | closed | | 2 — The store | 9 | 162 | 130 | passing | closed | | 3 — The run path | 11 | 206 | 142 | passing | closed | | 4 — Ingest | 25 | 468 | 290 | passing | open | @@ -93,6 +93,7 @@ Unscheduled operations: 10. | `assurance.differential-test.harness` | `crates/contextful-cli/tests/integration/differential.rs::the_engine_and_the_lean_reference_agree_over_generated_cases` | performed | | `assurance.differential-test.malformed-bytes` | `crates/contextful-cli/tests/integration/differential.rs::malformed_bytes_reach_every_decider_and_each_decides_them_malformed` | performed | | `assurance.differential-test.minimized` | `crates/contextful-cli/tests/integration/differential.rs::a_recorded_disagreement_is_minimal_under_every_shrinking_step` | performed | +| `assurance.differential-test.protocol-cases` | `spec/pins.toml` | performed | | `assurance.differential-test.protocol-drift` | `spec/pins.toml` | performed | | `assurance.differential-test.protocol-harness` | `spec/pins.toml` | performed | | `assurance.differential-test.reference-model` | `crates/contextful-cli/tests/integration/differential.rs::the_reference_binary_reads_one_case_and_prints_one_decision` | performed | From cce76589c614646b826e9f29ff434c97075c4302 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Mon, 5 Oct 2026 01:42:46 +0800 Subject: [PATCH 008/155] Run Windows lock and create cases on native runners --- .github/workflows/gate.yml | 3 +++ tools/ci/tests/integration/stages.rs | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index afe8cec25..58852663e 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -63,8 +63,11 @@ jobs: - run: rustup target add ${{ matrix.target }} - run: cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration an_open_directory_handle_syncs_its_entries + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration a_renamed_open_file_does_not_identify_its_replacement + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration racing_creates_land_exactly_one - run: cargo test --locked --target ${{ matrix.target }} -p contextful-policy --test integration an_audit_append_syncs_its_directories - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_ledger_append_syncs_its_directory + - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it dispatch: if: ${{ vars.FLAREDISPATCH_ENDPOINT != '' && github.event.pull_request.head.repo.full_name == github.repository }} diff --git a/tools/ci/tests/integration/stages.rs b/tools/ci/tests/integration/stages.rs index 46d1bedeb..820d04b76 100644 --- a/tools/ci/tests/integration/stages.rs +++ b/tools/ci/tests/integration/stages.rs @@ -145,8 +145,15 @@ fn native_windows_checks_cover_both_targets_and_directory_sync() { assert!(workflow.contains(pair), "the native Windows matrix omits {pair}"); } assert!(workflow.contains("cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context")); - for test in ["an_open_directory_handle_syncs_its_entries", "an_audit_append_syncs_its_directories", "a_ledger_append_syncs_its_directory"] { - assert!(workflow.contains(test), "the native Windows matrix omits {test}"); + for command in [ + "-p contextful-fs --test integration an_open_directory_handle_syncs_its_entries", + "-p contextful-fs --test integration a_renamed_open_file_does_not_identify_its_replacement", + "-p contextful-fs --test integration racing_creates_land_exactly_one", + "-p contextful-policy --test integration an_audit_append_syncs_its_directories", + "-p contextful-context --test integration a_ledger_append_syncs_its_directory", + "-p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it", + ] { + assert!(workflow.contains(command), "the native Windows matrix omits {command}"); } } From fd0305a10d49013d5364cbd75c6f8e74369f3e00 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 03:35:02 +0800 Subject: [PATCH 009/155] Split remote CLI differential checks from remaining tests --- .github/workflows/gate.yml | 4 +- AGENTS.md | 3 ++ spec/81-engineering.md | 4 +- spec/adr/A-assurance.md | 6 +-- spec/pins.toml | 2 +- spec/spec.lock.json | 4 +- spec/status.md | 4 +- tools/ci/src/main.rs | 46 +++++++++++++++++------ tools/ci/tests/integration/cli_parts.rs | 31 +++++++++++++++ tools/ci/tests/integration/features.rs | 9 +++-- tools/ci/tests/integration/main.rs | 1 + tools/ci/tests/integration/target_dirs.rs | 12 +++++- 12 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 tools/ci/tests/integration/cli_parts.rs diff --git a/.github/workflows/gate.yml b/.github/workflows/gate.yml index c42d14378..c45fd6948 100644 --- a/.github/workflows/gate.yml +++ b/.github/workflows/gate.yml @@ -15,12 +15,14 @@ # flare-dispatch/check:schema secrets of record, mirrors: annotations, derived artifacts regenerated, contextful-spec lint # flare-dispatch/check:test-first.validate a source change carries a test under tests/ # flare-dispatch/check:test-first. that package's changed tests fail against the base source -# flare-dispatch/check:workspace.compile compile every workspace suite over the union of features, then run the CLI suite +# flare-dispatch/check:workspace.compile compile the feature union, then run CLI tests outside the differential module +# flare-dispatch/check:workspace.cli-formal compile the feature union, then run the CLI differential module # flare-dispatch/check:workspace. run each other package suite in one of four groups # flare-dispatch/check:acceptance the milestone acceptance suite # flare-dispatch/check:evaluate every gate-tier entry of evals/ledger.toml, held to its target # flare-dispatch/check:features.packages each package but the binary declaring a feature, tested with none, with all and with each listed set # flare-dispatch/check:features.binary- the binary tested under one feature set: none, all, or one its manifest lists +# flare-dispatch/check:features.formal- the differential module under that same feature set # flare-dispatch/check:crate-graph the topology dependency rules over `cargo tree --locked` per package, and cargo-deny bans per profile # flare-dispatch/check:connectors the connector and guest-host suites with every feature on # flare-dispatch/check:surfaces each TypeScript surface's typecheck, unit tests and framework build diff --git a/AGENTS.md b/AGENTS.md index ebcfb72db..d03de6fbc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -132,6 +132,9 @@ stage whose predecessor's output is absent; `--predecessors` runs those too. A l the remote check invoke the identical command; `contextful-ci`'s suite fails when the workflow's matrix and the subcommand's part list differ. +The CLI differential module runs in `workspace.cli-formal` and `features.formal-`; +the paired `workspace.compile` and `features.binary-` checks run the remaining CLI tests. + The schema stage also holds every key in a tracked `.env*` file to dotenvx ciphertext under a comment stating what it grants (`contextful-ci secrets`); `.env.keys` stays untracked. A deliberate restatement of an engine rule carries `mirrors: ` at diff --git a/spec/81-engineering.md b/spec/81-engineering.md index d3240f36c..81d202d79 100644 --- a/spec/81-engineering.md +++ b/spec/81-engineering.md @@ -132,7 +132,7 @@ Target directories, the engine-linked invocation, linked query functions, build *because every engine-linked test binary embeds the bundled engine, and full debug information multiplies each past a sandbox's disk* - `release-profile` — Release builds compile with thin link-time optimization, one codegen unit per crate and symbols stripped, and unwind on panic. *because the run keeper survives a panicking job only by unwinding to its guard* -- `profile-build` — The features stage tests each package under every feature set its manifest lists in `feature-runs`, in parts dispatched as one check each: the other packages, and each binary run — none, all and each listed set. +- `profile-build` — The features stage tests each package under every feature set its manifest lists in `feature-runs`: other packages share one check; each binary feature run has separate checks for differential tests and all remaining tests. *because one cold build of the binary under one feature set fills most of a stage's wall clock, and a listed set reaches what none and all miss* - `container-image` — The repository's `Dockerfile` builds one profile, `contextful-full` unless `PROFILE` names another, as a static `linux/amd64` binary, and ships it in a shell-free runtime image as a non-root user over a declared store volume. - `targets` — All three profiles cross-compile to `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`; edge and full also build for `aarch64-apple-darwin` and `x86_64-apple-darwin`. @@ -157,7 +157,7 @@ Stage order, secrets of record, the crate-graph, row-token, egress and dependenc *A-assurance* - `test-first-parts` — Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each changed source or test package checks only its own tests against the base in a separate dispatch. *because one base build per package fits the sandbox's wall clock more reliably* -- `workspace-parts` — Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups. +- `workspace-parts` — Remote workspace checks compile the feature-unified workspace before each CLI partition: differential tests and all remaining tests; each other non-acceptance package suite runs in exactly one of four groups. *A-assurance* - `remote-predecessors` — Each remote check runs its stage together with every predecessor whose output that stage reads, so no check reads another check's sandbox. *A-assurance* diff --git a/spec/adr/A-assurance.md b/spec/adr/A-assurance.md index a39b42323..099d3af05 100644 --- a/spec/adr/A-assurance.md +++ b/spec/adr/A-assurance.md @@ -144,16 +144,16 @@ Revisit: an external consumer needs a compatibility promise on a public surface, Context: the feature-unified workspace suite exceeds the 1800 s sandbox cap on a cold remote build. Criteria: every package suite runs, the feature union compiles, and each dispatched command runs locally too. -Decision: `assurance.gate.workspace-parts` dispatches one compile part over the full union and runs the CLI suite from its compiled artifacts. Four other parts partition the remaining non-acceptance package suites. The unsplit local workspace stage retains its one cargo invocation under `assurance.build.one-engine-build`. +Decision: `assurance.gate.workspace-parts` compiles the full union in each CLI partition: differential tests and all remaining tests. Four other parts partition the remaining non-acceptance package suites. Binary feature checks use the same test partition. The unsplit local workspace stage retains its one cargo invocation under `assurance.build.one-engine-build`. | Option | Lost on | Cost | | --- | --- | --- | -| One union compile with CLI suite and four other suite groups *(chosen)* | — | Remote sandboxes rebuild shared dependencies; five checks replace one. | +| Two union-compiled CLI partitions and four other suite groups *(chosen)* | — | Remote sandboxes rebuild shared dependencies; six workspace checks replace one. | | A lock-keyed shared build cache | Operability | A cache bucket and credential become part of the gate; a cold lock still exceeds the cap. | | One check for the full workspace | Stage wall clock | A cold sandbox times out before it reports a verdict. | | A higher step timeout | Platform limit | Workflow steps admit at most 30 minutes. | -Consequences: the five remote checks cover the same package set as the unsplit local stage. The compile part reuses CLI artifacts before its sandbox exits; other parts compile their dependencies independently. +Consequences: six remote workspace checks cover the same package set as the unsplit local stage. Every binary feature set retains all tests across its two checks. Each CLI partition compiles independently; reference-program builds occupy only the differential partition. ## One decision module compiles native and to WebAssembly diff --git a/spec/pins.toml b/spec/pins.toml index c6d493fdd..1e20d7a67 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -54,7 +54,7 @@ [floor] assurance = 136 authority = 204 -connector = 207 +connector = 218 corpus = 38 disclosure = 54 read = 163 diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 45305884a..f88a77676 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -20912,7 +20912,7 @@ "operation": "build", "subject": "profile-build", "kind": "behavior", - "statement": "The features stage tests each package under every feature set its manifest lists in `feature-runs`, in parts dispatched as one check each: the other packages, and each binary run — none, all and each listed set.", + "statement": "The features stage tests each package under every feature set its manifest lists in `feature-runs`: other packages share one check; each binary feature run has separate checks for differential tests and all remaining tests.", "why": "because one cold build of the binary under one feature set fills most of a stage's wall clock, and a listed set reaches what none and all miss", "file": "spec/81-engineering.md", "line": 135 @@ -21044,7 +21044,7 @@ "operation": "gate", "subject": "workspace-parts", "kind": "behavior", - "statement": "Remote workspace checks compile the feature-unified workspace and run the CLI suite from that build, then run each other non-acceptance package suite in exactly one of four groups.", + "statement": "Remote workspace checks compile the feature-unified workspace before each CLI partition: differential tests and all remaining tests; each other non-acceptance package suite runs in exactly one of four groups.", "why": "A-assurance", "file": "spec/81-engineering.md", "line": 160 diff --git a/spec/status.md b/spec/status.md index 4b8c4dd61..7c618a9f3 100644 --- a/spec/status.md +++ b/spec/status.md @@ -7,7 +7,7 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `assurance` | 2 | 15 | 228 | 59 | 22 | 12 | 136 | 0 | 136 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | -| `connector` | 2 | 14 | 345 | 90 | 21 | 15 | 218 | 0 | 207 | +| `connector` | 2 | 14 | 345 | 90 | 21 | 15 | 218 | 0 | 218 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | @@ -73,7 +73,7 @@ Unscheduled operations: 10. | `assurance.build.licence-field` | `tools/ci/tests/integration/topology.rs::a_workspace_package_without_the_apache_licence_is_refused` | performed | | `assurance.build.linked-query-functions` | `crates/contextful-context/tests/integration/read/extension.rs::columnar_reading_and_statement_serialization_are_statically_linked_and_loaded` | performed | | `assurance.build.one-engine-build` | `tools/ci/tests/integration/target_dirs.rs::the_workspace_stage_runs_one_invocation_and_the_store_suites_link_no_engine_without_read` | performed | -| `assurance.build.profile-build` | `tools/ci/tests/integration/features.rs::the_features_stage_tests_each_listed_run_in_parts_one_per_binary_run` | performed | +| `assurance.build.profile-build` | `tools/ci/tests/integration/features.rs::the_features_stage_tests_each_listed_run_in_two_binary_parts` | performed | | `assurance.build.release-artifact` | `tools/ci/tests/integration/artifact.rs::a_dry_run_release_packages_three_archives_with_checksums_sboms_and_formulae` | performed | | `assurance.build.release-profile` | `tools/ci/tests/integration/image.rs::the_release_profile_optimizes_across_crates_strips_and_unwinds` | performed | | `assurance.build.runtime-extension-load` | `crates/contextful-context/tests/integration/read/extension.rs::an_explicit_load_or_install_refuses` | performed | diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 8f2b1e7cc..ed6224d1b 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -344,16 +344,16 @@ fn repo_root() -> Result { const SPLIT: [&str; 4] = ["test-first", "workspace", "features", "budget"]; /// The parts of a split `stage`: the features stage's `packages`, every featured package but -/// the binary, then `binary-` per run of the binary package; the budget stage's +/// the binary, then `binary-` and `formal-` per binary feature run; the budget stage's /// `` per profile the binary declares, its name after `contextful-`. fn parts(root: &Path, stage: &str, base: Option<&str>) -> Result> { Ok(match stage { "test-first" => match base { Some(base) => test_first_parts(root, base)?, None => vec!["validate".into()] }, - "workspace" => ["compile", "foundation", "runtime", "read", "ci"].into_iter().map(str::to_string).collect(), + "workspace" => ["compile", "cli-formal", "foundation", "runtime", "read", "ci"].into_iter().map(str::to_string).collect(), "features" => { let featured = featured_packages(root)?; let binary = featured.iter().filter(|p| p.name == topology::BINARY).flat_map(Featured::runs); - std::iter::once("packages".to_string()).chain(binary.map(|(label, _)| format!("binary-{label}"))).collect() + std::iter::once("packages".to_string()).chain(binary.flat_map(|(label, _)| [format!("binary-{label}"), format!("formal-{label}")])).collect() } "budget" => topology::declared_profiles(root)?.iter().map(|p| p.strip_prefix("contextful-").unwrap_or(p).to_string()).collect(), _ => Vec::new(), @@ -539,7 +539,9 @@ fn workspace_part(package: &str) -> &'static str { fn workspace(root: &Path, only: Option<&[String]>) -> Result<()> { if let Some(only) = only { - if only.iter().any(|part| part == "compile") { + let ordinary = only.iter().any(|part| part == "compile"); + let formal = only.iter().any(|part| part == "cli-formal"); + if ordinary || formal { let mut args = vec!["test", "--workspace"]; if root.join(ACCEPTANCE_DIR).join("Cargo.toml").exists() { args.extend(["--exclude", ACCEPTANCE_PACKAGE]); @@ -549,15 +551,21 @@ fn workspace(root: &Path, only: Option<&[String]>) -> Result<()> { } let packages: Vec = workspace_packages(root)? .into_iter() - .filter(|package| only.iter().any(|part| part == workspace_part(package))) + .filter(|package| only.iter().any(|part| part == workspace_part(package)) || (formal && package == topology::BINARY)) .collect(); - if !packages.is_empty() { + let other: Vec<&String> = packages.iter().filter(|package| package.as_str() != topology::BINARY).collect(); + if !other.is_empty() { let mut args = vec!["test"]; - for package in &packages { + for package in other { args.extend(["--package", package.as_str()]); } run_staged(root, "workspace", &args)?; } + if packages.iter().any(|package| package == topology::BINARY) { + let mut args = vec!["test", "--package", topology::BINARY]; + cli_partition(&mut args, ordinary, formal); + run_staged(root, "workspace", &args)?; + } return Ok(()); } let mut args = vec!["test", "--workspace"]; @@ -567,12 +575,23 @@ fn workspace(root: &Path, only: Option<&[String]>) -> Result<()> { run_staged(root, "workspace", &args) } +/// The differential module runs separately from the other CLI tests in remote parts; +/// selecting both partitions retains Cargo's complete suite, including doctests. +fn cli_partition(args: &mut Vec<&str>, ordinary: bool, formal: bool) { + if !ordinary { + args.extend(["--test", "integration", "differential::"]); + } else if !formal { + args.extend(["--", "--skip", "differential::"]); + } +} + /// The feature combinations the workspace stage's unified build does not reach /// (`assurance.build.staged-feature-runs`): each workspace package declaring a feature other /// than `default` runs its suite alone, once with no features, once with every feature, and /// once per feature set its manifest lists under `[package.metadata.contextful] /// feature-runs` (`assurance.build.profile-build`). `only` narrows the stage to its named -/// parts: `packages`, every package but the binary, and `binary-`, one run of the binary. +/// parts: `packages`, every package but the binary, and `binary-` plus `formal-` +/// for the binary's ordinary and differential tests under each feature set. /// `cargo test -p` resolves the selected package's features without its dependents', so the /// store adapter's write suites run with the read face off /// (`topology.package.store-write-engine-free`) and the policy package's without `exchange`. @@ -588,15 +607,18 @@ fn features(root: &Path, only: Option<&[String]>) -> Result<()> { for package in &featured { let name = &package.name; let binary = name == topology::BINARY; - let selected = |label: &str| { - only.is_none_or(|o| o.iter().any(|p| if binary { p.strip_prefix("binary-") == Some(label) } else { p == "packages" })) - }; - for (_, combination) in package.runs().into_iter().filter(|(label, _)| selected(label)) { + for (label, combination) in package.runs() { + let ordinary = only.is_none_or(|o| o.iter().any(|p| if binary { p.strip_prefix("binary-") == Some(label.as_str()) } else { p == "packages" })); + let formal = binary && only.is_none_or(|o| o.iter().any(|p| p.strip_prefix("formal-") == Some(label.as_str()))); + if !ordinary && !formal { continue; } let shown = combination.join(" "); eprintln!("features: {name} {shown}"); + let mut partition = Vec::new(); + if binary { cli_partition(&mut partition, ordinary, formal); } let status = Command::new("cargo") .args(["test", "-p", name]) .args(&combination) + .args(&partition) .env("CARGO_TARGET_DIR", &target) .current_dir(root) .status()?; diff --git a/tools/ci/tests/integration/cli_parts.rs b/tools/ci/tests/integration/cli_parts.rs new file mode 100644 index 000000000..9a5b360d6 --- /dev/null +++ b/tools/ci/tests/integration/cli_parts.rs @@ -0,0 +1,31 @@ +use crate::{manifest, stderr, Repo}; + +#[test] +fn remote_cli_parts_run_every_test_once_and_propagate_differential_failures() { + let r = Repo::init(); + r.write("crates/contextful-cli/Cargo.toml", &(manifest("contextful-cli", "") + "\n[features]\nextra = []\n")); + r.write("crates/contextful-cli/src/lib.rs", ""); + r.write("crates/contextful-cli/tests/integration/main.rs", "#[test] fn ordinary() { std::fs::write(\"ordinary.ran\", \"yes\").unwrap(); }\nmod differential { #[test] fn reference() { std::fs::write(\"reference.ran\", \"yes\").unwrap(); } }\n"); + r.lock(); + r.commit("CLI ordinary and differential suites"); + let package = r.root.join("crates/contextful-cli"); + for (ordinary, differential) in [("workspace.compile", "workspace.cli-formal"), ("features.binary-all", "features.formal-all")] { + let out = r.gate(&["--stage", ordinary]); + assert!(out.status.success(), "{ordinary}: {}", stderr(&out)); + assert!(package.join("ordinary.ran").exists(), "{ordinary} missed ordinary tests"); + assert!(!package.join("reference.ran").exists(), "{ordinary} also ran the differential partition"); + std::fs::remove_file(package.join("ordinary.ran")).unwrap(); + let out = r.gate(&["--stage", differential]); + assert!(out.status.success(), "{differential}: {}", stderr(&out)); + assert!(package.join("reference.ran").exists(), "{differential} missed reference tests"); + assert!(!package.join("ordinary.ran").exists(), "{differential} repeated ordinary tests"); + std::fs::remove_file(package.join("reference.ran")).unwrap(); + } + r.write("crates/contextful-cli/tests/integration/main.rs", "mod differential { #[test] fn reference() { panic!(\"reference regression\"); } }\n"); + r.commit("a failing differential case"); + for part in ["workspace.cli-formal", "features.formal-all"] { + let out = r.gate(&["--stage", part]); + assert!(!out.status.success(), "{part} hid a differential failure"); + assert!(String::from_utf8_lossy(&out.stdout).contains("reference regression"), "{part}: {}", stderr(&out)); + } +} diff --git a/tools/ci/tests/integration/features.rs b/tools/ci/tests/integration/features.rs index 94b1f275a..e30429f47 100644 --- a/tools/ci/tests/integration/features.rs +++ b/tools/ci/tests/integration/features.rs @@ -119,10 +119,10 @@ fn bundled(r: &Repo, lib_edge: &str, runs: &str) { r.write("crates/gated/tests/integration/main.rs", "#[test]\nfn builds() {}\n"); } -/// The features stage tests each package under every feature set its manifest lists in `feature-runs`, in parts dispatched as one check each: the other packages, and each binary run — none, all and each listed set. -// spec: assurance.build.profile-build@5854cf3a +/// Each binary feature run partitions differential tests from its remaining suite. +// spec: assurance.build.profile-build@0250142c #[test] -fn the_features_stage_tests_each_listed_run_in_parts_one_per_binary_run() { +fn the_features_stage_tests_each_listed_run_in_two_binary_parts() { let r = Repo::init(); bundled(&r, "pub fn edge() {}", "\"edge\""); r.write( @@ -140,6 +140,9 @@ fn the_features_stage_tests_each_listed_run_in_parts_one_per_binary_run() { assert!(at("evaluate") < at("features.packages") && at("features.packages") < at("features.binary-none"), "{listed}"); assert!(at("features.binary-none") < at("features.binary-all") && at("features.binary-all") < at("features.binary-extra"), "{listed}"); assert!(at("features.binary-extra") < at("crate-graph") && !parts.contains(&"features"), "{listed}"); + for label in ["none", "all", "extra"] { + assert!(at(&format!("features.binary-{label}")) < at(&format!("features.formal-{label}")), "{listed}"); + } // The whole stage runs every part. let o = features(&r); diff --git a/tools/ci/tests/integration/main.rs b/tools/ci/tests/integration/main.rs index 8d8cee8b2..cc5e5f9a2 100644 --- a/tools/ci/tests/integration/main.rs +++ b/tools/ci/tests/integration/main.rs @@ -4,6 +4,7 @@ mod acceptance_surface; mod allowlist; mod artifact; +mod cli_parts; mod dependency_versions; mod deny; mod disk; diff --git a/tools/ci/tests/integration/target_dirs.rs b/tools/ci/tests/integration/target_dirs.rs index 6c5f5e29b..6a7cd8ecc 100644 --- a/tools/ci/tests/integration/target_dirs.rs +++ b/tools/ci/tests/integration/target_dirs.rs @@ -84,8 +84,8 @@ fn the_workspace_stage_runs_one_invocation_and_the_store_suites_link_no_engine_w } } -/// Remote workspace checks compile the feature-unified workspace, run the CLI suite from that build, then run every other non-acceptance package suite in one of four groups. -// spec: assurance.gate.workspace-parts@55b557c2 +/// Both CLI partitions compile the feature union; four groups run every other package. +// spec: assurance.gate.workspace-parts@e828321f #[test] fn remote_workspace_parts_compile_the_union_and_run_each_package_suite() { let r = Repo::init(); @@ -104,6 +104,14 @@ fn remote_workspace_parts_compile_the_union_and_run_each_package_suite() { let calls = std::fs::read_to_string(&log).unwrap(); assert!(calls.contains("test --workspace --exclude contextful-acceptance --no-run"), "{calls}"); assert!(calls.contains("test --package contextful-cli"), "{calls}"); + assert!(calls.contains("--skip differential::"), "{calls}"); + + std::fs::remove_file(&log).unwrap(); + let formal = r.gate_with_cargo(&recording(&log, 0), &["--stage", "workspace.cli-formal"]); + assert!(formal.status.success(), "{}", stderr(&formal)); + let calls = std::fs::read_to_string(&log).unwrap(); + assert!(calls.contains("test --workspace --exclude contextful-acceptance --no-run"), "{calls}"); + assert!(calls.contains("test --package contextful-cli --test integration differential::"), "{calls}"); std::fs::remove_file(&log).unwrap(); let tested = r.gate_with_cargo(&recording(&log, 0), &["--stage", "workspace.foundation"]); From 956e09486f8321812042d0797a45834f22b3b0ea Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 04:46:18 +0800 Subject: [PATCH 010/155] Keep Unix socket tests behind their platform boundary --- .../tests/integration/verify.rs | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/crates/contextful-policy/tests/integration/verify.rs b/crates/contextful-policy/tests/integration/verify.rs index 5cb53ad4d..36642db42 100644 --- a/crates/contextful-policy/tests/integration/verify.rs +++ b/crates/contextful-policy/tests/integration/verify.rs @@ -13,9 +13,12 @@ use contextful_core::issue::{IssuancePolicy, Lifetime, MintContext, MintRequest, use contextful_policy::verify::BEARER_LIFETIME_SECS; use contextful_policy::possession::{sign_proof, verify_proof, NonceCache, ProofRefusal, ProofRequest}; use contextful_policy::verify::{ - checkpoint_uid, effect_boundary, no_holder_proof, verify_inherited_pipe, verify_local, verify_network, Admission, BiscuitFormat, + effect_boundary, no_holder_proof, verify_inherited_pipe, verify_local, verify_network, Admission, BiscuitFormat, CredentialFormat, LocalTransport, }; +#[cfg(unix)] +use contextful_policy::verify::checkpoint_uid; +#[cfg(unix)] use std::os::unix::net::UnixStream; /// Verification yields an admitted-authority value carrying the normalized subject tuple and its grants. Every read surface and row-landing effect takes that value as an argument; nothing downstream re-parses a credential or reads ambient state. @@ -222,6 +225,7 @@ fn refused_proof(r: Result, error: &str) { } } +#[cfg(unix)] fn own_socket(connection: u64) -> (LocalTransport, UnixStream) { let (checkpoint_end, client_end) = UnixStream::pair().unwrap(); (LocalTransport::socket(connection, &checkpoint_end), client_end) @@ -229,6 +233,7 @@ fn own_socket(connection: u64) -> (LocalTransport, UnixStream) { /// A local transport is a stdio pipe the checkpoint inherited from the process that spawned it, or a Unix socket; every other transport is a network transport. // spec: authority.verify.local-transport@1976a641 +#[cfg(unix)] #[test] fn a_local_transport_is_the_inherited_pipe_or_a_unix_socket_reporting_its_peer_uid() { let (socket, _client) = own_socket(1); @@ -245,8 +250,9 @@ fn a_key_bound_credential_admits_locally_only_with_its_holder_proof() { let credential = key_bound(&signer, &holder); let revocation = no_revocation(); let admission = Admission::new(at(DURING), &revocation).expecting(AUD); + #[cfg(unix)] let (socket, _client) = own_socket(1); - for transport in [LocalTransport::InheritedPipe, socket] { + for transport in [LocalTransport::InheritedPipe, #[cfg(unix)] socket] { // No proof: the peer fallback does not stand in for the key. refused_proof(verify_local(&credential, &keys(&signer), &admission, transport, no_holder_proof), "PossessionProofInvalid"); // A thief's proof. @@ -275,12 +281,17 @@ fn a_credential_binding_no_key_admits_through_the_inherited_pipe_or_a_same_uid_s let piped = verify_local(&credential, &keys(&signer), &admission, LocalTransport::InheritedPipe, no_holder_proof::).unwrap(); assert_eq!(piped.authority().confirmation(), None); assert!(verify_inherited_pipe(&credential, &keys(&signer), &admission).is_ok()); - let (socket, _client) = own_socket(3); - let socketed = verify_local(&credential, &keys(&signer), &admission, socket, no_holder_proof::).unwrap(); - assert_eq!(socketed.transport(), socket); + #[cfg(unix)] + { + let (socket, _client) = own_socket(3); + let socketed = verify_local(&credential, &keys(&signer), &admission, socket, no_holder_proof::).unwrap(); + assert_eq!(socketed.transport(), socket); + let late = Admission::new(at("2030-01-01T00:20:00Z"), &revocation).expecting(AUD); + refused(verify_local(&credential, &keys(&signer), &late, socket, no_holder_proof::), "AuthorityExpired"); + } // The fallback admits after every other admission check: an expired credential still refuses. let late = Admission::new(at("2030-01-01T00:20:00Z"), &revocation).expecting(AUD); - refused(verify_local(&credential, &keys(&signer), &late, socket, no_holder_proof::), "AuthorityExpired"); + refused(verify_local(&credential, &keys(&signer), &late, LocalTransport::InheritedPipe, no_holder_proof::), "AuthorityExpired"); } /// A socket peer presenting a credential with no confirmation claim, whose kernel-reported uid differs from the checkpoint process's uid or which the platform cannot report, raises `TransportPeerMismatch` and admits nothing. @@ -291,7 +302,10 @@ fn a_socket_peer_of_another_or_an_unreported_uid_admits_nothing() { let credential = minted(&signer); let revocation = no_revocation(); let admission = Admission::new(at(DURING), &revocation).expecting(AUD); + #[cfg(unix)] let other = checkpoint_uid().wrapping_add(1); + #[cfg(not(unix))] + let other = 0; for peer_uid in [Some(other), None] { let transport = LocalTransport::Socket { connection: 4, peer_uid }; refused(verify_local(&credential, &keys(&signer), &admission, transport, no_holder_proof::), "TransportPeerMismatch"); @@ -378,6 +392,7 @@ fn a_bearer_living_past_3600_s_admits_nothing_over_a_network() { /// A local admission binds to the pipe or socket connection that presented the credential; a request on any other connection is refused as {{authority.verify.peer-mismatch}} until that connection presents the credential and admits itself. // spec: authority.verify.connection-scoped@bb49fb3d +#[cfg(unix)] #[test] fn a_local_admission_answers_only_on_the_connection_that_presented_the_credential() { let signer = issuer(); @@ -398,12 +413,15 @@ fn a_local_admission_answers_only_on_the_connection_that_presented_the_credentia refused(piped.on(&first), "TransportPeerMismatch"); } +#[cfg(unix)] const OFF_TRANSPORT_SEED: u64 = 0x5eed_0057; +#[cfg(unix)] const OFF_TRANSPORT_ROUNDS: u64 = 32; /// A credential binding no key, presented off the local transport that admits it — from a /// socket peer of another or an unreported uid, or on a connection other than the admitted /// one — admits nothing, over a seeded loop. +#[cfg(unix)] #[test] fn a_credential_binding_no_key_admits_nothing_off_its_local_transport_over_a_seeded_loop() { let signer = issuer(); From e726696c2e7bdbff941cf7f49f2030565813d2ec Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 07:05:54 +0800 Subject: [PATCH 011/155] fix(formal): report states from the minimized protocol case --- crates/contextful-cli/src/protocol_differential.rs | 4 +++- .../tests/integration/protocol_differential.rs | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/contextful-cli/src/protocol_differential.rs b/crates/contextful-cli/src/protocol_differential.rs index 978fa6ffb..fd5cc899a 100644 --- a/crates/contextful-cli/src/protocol_differential.rs +++ b/crates/contextful-cli/src/protocol_differential.rs @@ -519,8 +519,10 @@ pub fn run(args: Args) -> Result<()> { cases.push(generate(&mut runner)?); } for (case, steps) in cases.into_iter().enumerate() { - if let Some((at, model, store)) = compare(&reference, &steps)? { + if compare(&reference, &steps)?.is_some() { let reduced = minimize(&reference, steps)?; + let (at, model, store) = compare(&reference, &reduced)? + .context("the minimized protocol sequence no longer reproduces the drift")?; if let Some(parent) = regressions.parent() { std::fs::create_dir_all(parent)?; } diff --git a/crates/contextful-cli/tests/integration/protocol_differential.rs b/crates/contextful-cli/tests/integration/protocol_differential.rs index 5d540864e..0048a0aed 100644 --- a/crates/contextful-cli/tests/integration/protocol_differential.rs +++ b/crates/contextful-cli/tests/integration/protocol_differential.rs @@ -53,10 +53,10 @@ fn fixed_seed_replays_protocol_cases_against_the_store() { fn a_protocol_difference_prints_the_minimal_sequence_and_both_states() { let dir = tempfile::tempdir().unwrap(); let reference = dir.path().join("reference.sh"); - std::fs::write(&reference, "#!/bin/sh\ncat >/dev/null\necho 'lease[none] catalog=999 cursor=0 n0(belief=none paused=false pending=none) n1(belief=none paused=false pending=none) n2(belief=none paused=false pending=none)'\n").unwrap(); + std::fs::write(&reference, "#!/bin/sh\nwhile IFS= read -r step; do printf 'reference: %s\\n' \"$step\"; done\n").unwrap(); std::fs::set_permissions(&reference, std::fs::Permissions::from_mode(0o755)).unwrap(); let regressions = dir.path().join("regressions.jsonl"); - std::fs::write(®ressions, "[{\"op\":\"expire\"}]\n").unwrap(); + std::fs::write(®ressions, "[{\"op\":\"acquire\",\"node\":0},{\"op\":\"expire\"}]\n").unwrap(); let out = Command::new(BIN) .args(["formal", "protocol-differential", "--cases", "0", "--reference"]) .arg(&reference) @@ -67,7 +67,7 @@ fn a_protocol_difference_prints_the_minimal_sequence_and_both_states() { let error = String::from_utf8_lossy(&out.stderr); assert!(error.contains("ProtocolConformanceDrift"), "{error}"); assert!(error.contains("sequence=[{\"op\":\"expire\"}]"), "{error}"); - assert!(error.contains("model=lease[none] catalog=999"), "{error}"); + assert!(error.contains("model=reference: expire"), "{error}"); assert!(error.contains("store=lease[none] catalog=0"), "{error}"); } From e69538ea0607238a865fc2cd03d93f40a362385d Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:33:52 +0800 Subject: [PATCH 012/155] test(console): define hosted page grants and adapter routes --- apps/console/package.json | 8 +++ apps/console/test/console.test.ts | 116 ++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 apps/console/package.json create mode 100644 apps/console/test/console.test.ts diff --git a/apps/console/package.json b/apps/console/package.json new file mode 100644 index 000000000..ac1f158a9 --- /dev/null +++ b/apps/console/package.json @@ -0,0 +1,8 @@ +{ + "name": "@contextful/console", + "private": true, + "type": "module", + "scripts": { + "test": "node --experimental-strip-types --test test/*.test.ts" + } +} diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts new file mode 100644 index 000000000..d60e56640 --- /dev/null +++ b/apps/console/test/console.test.ts @@ -0,0 +1,116 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; +import { createConsole, issueCognitoSession } from "../src/index.ts"; + +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); +const issuer = "https://access.example"; +const queryAudience = "query-app"; +const adminAudience = "admin-app"; + +function jwt(payload: Record): string { + const header = Buffer.from(JSON.stringify({ alg: "RS256", typ: "JWT" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify(payload)).toString("base64url"); + return `${header}.${claims}.${sign("RSA-SHA256", Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}`; +} + +function access(aud: string, extra: Record = {}): HeadersInit { + return { "cf-access-jwt-assertion": jwt({ iss: issuer, aud, sub: "operator-1", exp: Math.floor(Date.now() / 1000) + 300, ...extra }) }; +} + +function fixture(adminCapability = "server-secret") { + const calls: string[] = []; + const console = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + adminCapability, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async ({ question, operator, store }) => { + calls.push(`turn:${operator.subject}:${store}:${question}`); + return { answer: "Three filings arrived.", sources: [{ title: "Filings", url: "https://example.test/filings" }], widgets: [{ component: "table.v1", props: { columns: ["Name"], rows: [["One"]] } }] }; + }, + read: { list: async () => [{ id: "field-notes", label: "Field notes" }] }, + control: { + workflows: async () => { calls.push("workflows"); return { pipelines: [{ id: "filings", steps: ["fetch"], schedule: "every 1h", runs: [] }], annotations: [] }; }, + record: async () => ({ packs: [], learnings: [] }), + edit: async (_body, capability) => { calls.push(`edit:${capability}`); return { version: 2 }; }, + apply: async (_body, capability) => { calls.push(`apply:${capability}`); return { version: 3 }; }, + }, + }); + const request = (path: string, headers?: HeadersInit, init: RequestInit = {}) => console.fetch(new Request(`https://console.example${path}`, { ...init, headers })); + return { request, calls }; +} + +test("anonymous and forged Access assertions dispatch no page or API", async () => { + const { request, calls } = fixture(); + for (const path of ["/query", "/query/api/stores", "/admin", "/admin/api/workflows"]) { + assert.equal((await request(path)).status, 403); + assert.equal((await request(path, { "cf-access-jwt-assertion": "forged" })).status, 403); + } + assert.deepEqual(calls, []); +}); + +test("verified Access audiences grant only their page and API", async () => { + const { request, calls } = fixture(); + assert.equal((await request("/query", access(queryAudience))).status, 200); + assert.equal((await request("/query/api/stores", access(queryAudience))).status, 200); + assert.equal((await request("/admin", access(queryAudience))).status, 403); + assert.equal((await request("/admin/api/workflows", access(queryAudience))).status, 403); + assert.equal((await request("/query", access(adminAudience))).status, 403); + assert.equal((await request("/admin", access(adminAudience))).status, 200); + assert.equal((await request("/admin/api/workflows", access(adminAudience))).status, 200); + assert.deepEqual(calls, ["workflows"]); +}); + +test("expired or wrong-issuer assertions cannot open a page", async () => { + const { request } = fixture(); + assert.equal((await request("/query", access(queryAudience, { exp: 1 }))).status, 403); + assert.equal((await request("/query", access(queryAudience, { iss: "https://other.example" }))).status, 403); +}); + +test("Query keeps one composer and transcript and delegates a sourced answer", async () => { + const { request, calls } = fixture(); + const page = await (await request("/query", access(queryAudience))).text(); + assert.match(page, /id="composer"/); + assert.match(page, /id="transcript"/); + assert.match(page, /id="widgets"/); + const answer = await request("/query/api/ask", access(queryAudience), { method: "POST", body: JSON.stringify({ store: "field-notes", question: "What arrived?" }) }); + assert.equal(answer.status, 200); + assert.equal((await answer.json() as { sources: unknown[] }).sources.length, 1); + assert.deepEqual(calls, ["turn:operator-1:field-notes:What arrived?"]); +}); + +test("Admin exposes workflows and requires a server-held capability for edit and apply", async () => { + const { request, calls } = fixture(); + const workflows = await request("/admin/api/workflows", access(adminAudience)); + assert.equal((await workflows.json() as { pipelines: unknown[] }).pipelines.length, 1); + const edit = await request("/admin/api/edit", access(adminAudience), { method: "POST", body: "{}" }); + assert.equal(edit.status, 200); + const apply = await request("/admin/api/apply", access(adminAudience), { method: "POST", body: "{}" }); + assert.equal(apply.status, 200); + assert.deepEqual(calls, ["workflows", "edit:server-secret", "apply:server-secret"]); + const withoutCapability = fixture(""); + const refused = await withoutCapability.request("/admin/api/apply", access(adminAudience), { method: "POST", body: "{}" }); + assert.equal(refused.status, 403); + assert.match(await refused.text(), /ConsoleAdminGrantMissing/); + assert.deepEqual(withoutCapability.calls, []); +}); + +test("Cognito sessions are signed, bounded, and mapped from groups", async () => { + const session = issueCognitoSession({ subject: "operator-2", groups: ["console-query"], expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + const { request, calls } = (() => { + const calls: string[] = []; + const app = createConsole({ + identity: { kind: "cognito", sessionSecret: "session-secret", queryGroup: "console-query", adminGroup: "console-admin" }, + adminCapability: "server-secret", + stores: [], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + control: { workflows: async () => { calls.push("workflows"); return {}; }, record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + return { request: (path: string, cookie: string) => app.fetch(new Request(`https://console.example${path}`, { headers: { cookie } })), calls }; + })(); + assert.equal((await request("/query", `console_session=${session}`)).status, 200); + assert.equal((await request("/admin/api/workflows", `console_session=${session}`)).status, 403); + assert.equal((await request("/query", `console_session=${session}tampered`)).status, 403); + assert.deepEqual(calls, []); +}); From 516cdf5e27cbbafdc3c97b1aed93d546d0384a17 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:42:05 +0800 Subject: [PATCH 013/155] feat(console): serve gated Query and Admin pages --- apps/console/pnpm-lock.yaml | 9 + apps/console/src/index.ts | 269 ++++++++++++++++++++++++++++++ apps/console/src/server.ts | 33 ++++ apps/console/test/console.test.ts | 90 +++++++++- 4 files changed, 396 insertions(+), 5 deletions(-) create mode 100644 apps/console/pnpm-lock.yaml create mode 100644 apps/console/src/index.ts create mode 100644 apps/console/src/server.ts diff --git a/apps/console/pnpm-lock.yaml b/apps/console/pnpm-lock.yaml new file mode 100644 index 000000000..9b60ae178 --- /dev/null +++ b/apps/console/pnpm-lock.yaml @@ -0,0 +1,9 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: {} diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts new file mode 100644 index 000000000..1a6408c7a --- /dev/null +++ b/apps/console/src/index.ts @@ -0,0 +1,269 @@ +import { createHmac, createPublicKey, randomBytes, timingSafeEqual, verify, type KeyObject } from "node:crypto"; + +export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin"> }; +export type Store = { id: string; label: string }; +export type AccessIdentity = { + kind: "access"; + issuer: string; + queryAudience: string; + adminAudience: string; + publicKey?: KeyObject | string; + keys?: ReadonlyMap; +}; +export type CognitoIdentity = { + kind: "cognito"; + sessionSecret: string; + queryGroup: string; + adminGroup: string; + issuer?: string; + clientId?: string; + authorizeUrl?: string; + tokenUrl?: string; + redirectUri?: string; + publicKey?: KeyObject | string; + fetcher?: typeof fetch; +}; +export type Identity = AccessIdentity | CognitoIdentity; +export type TurnInput = { operator: Operator; store: string; question: string }; +export type TurnResult = { answer: string; sources: unknown[]; widgets: unknown[] }; +export type PackList = { entries: unknown[]; truncated: boolean; declined: number }; +export type ConsoleAdapters = { + identity: Identity; + stores: Store[]; + adminCapability?: string; + turn: (input: TurnInput) => Promise; + read: { list: (operator: Operator) => Promise }; + control: { + workflows: (operator: Operator, store: string | null) => Promise; + record: (operator: Operator, store: string | null) => Promise; + edit: (document: unknown, capability: string, operator: Operator) => Promise; + apply: (document: unknown, capability: string, operator: Operator) => Promise; + listPackFiles?: (operator: Operator, store: string, prefix: string) => Promise; + }; +}; + +type Session = { subject: string; groups: string[]; expiresAt: number }; + +function encoded(value: unknown): string { + return Buffer.from(JSON.stringify(value)).toString("base64url"); +} + +function decode(value: string): unknown { + return JSON.parse(Buffer.from(value, "base64url").toString("utf8")); +} + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function secureEqual(left: Buffer, right: Buffer): boolean { + return left.length === right.length && timingSafeEqual(left, right); +} + +export function issueCognitoSession(session: Session, secret: string): string { + if (!secret || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1000)) { + throw new Error("CognitoSessionInvalid"); + } + const body = encoded(session); + const signature = createHmac("sha256", secret).update(body).digest("base64url"); + return `${body}.${signature}`; +} + +function verifyCognitoSession(cookie: string, identity: CognitoIdentity): Operator | null { + const token = cookie.split("; ").find((part) => part.startsWith("console_session="))?.slice("console_session=".length); + if (!token) return null; + const parts = token.split("."); + if (parts.length !== 2) return null; + const expected = createHmac("sha256", identity.sessionSecret).update(parts[0]).digest(); + let actual: Buffer; + try { actual = Buffer.from(parts[1], "base64url"); } catch { return null; } + if (!secureEqual(expected, actual)) return null; + let session: unknown; + try { session = decode(parts[0]); } catch { return null; } + if (!object(session) || typeof session.subject !== "string" || !session.subject || + !Number.isSafeInteger(session.expiresAt) || (session.expiresAt as number) <= Math.floor(Date.now() / 1000) || + !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string")) return null; + const grants = new Set<"query" | "admin">(); + if (session.groups.includes(identity.queryGroup)) grants.add("query"); + if (session.groups.includes(identity.adminGroup)) grants.add("admin"); + return { subject: session.subject, grants }; +} + +function verifyAccess(assertion: string, identity: AccessIdentity): Operator | null { + const parts = assertion.split("."); + if (parts.length !== 3) return null; + let header: unknown; + let claims: unknown; + try { header = decode(parts[0]); claims = decode(parts[1]); } catch { return null; } + if (!object(header) || header.alg !== "RS256" || !object(claims)) return null; + if (claims.iss !== identity.issuer || typeof claims.sub !== "string" || !claims.sub || + !Number.isSafeInteger(claims.exp) || (claims.exp as number) <= Math.floor(Date.now() / 1000)) return null; + if (typeof claims.nbf === "number" && claims.nbf > Math.floor(Date.now() / 1000)) return null; + const audience = typeof claims.aud === "string" ? [claims.aud] : Array.isArray(claims.aud) ? claims.aud : []; + const grants = new Set<"query" | "admin">(); + if (audience.includes(identity.queryAudience)) grants.add("query"); + if (audience.includes(identity.adminAudience)) grants.add("admin"); + if (grants.size === 0) return null; + let signature: Buffer; + try { signature = Buffer.from(parts[2], "base64url"); } catch { return null; } + const key = identity.keys ? (typeof header.kid === "string" ? identity.keys.get(header.kid) : undefined) : + typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; + if (!key) return null; + if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, signature)) return null; + return { subject: claims.sub, grants }; +} + +function operatorFor(request: Request, identity: Identity): Operator | null { + if (identity.kind === "access") { + const assertion = request.headers.get("cf-access-jwt-assertion"); + return assertion ? verifyAccess(assertion, identity) : null; + } + return verifyCognitoSession(request.headers.get("cookie") ?? "", identity); +} + +function cognitoLogin(identity: CognitoIdentity): Response { + if (!identity.authorizeUrl || !identity.clientId || !identity.redirectUri) return refusal("ConsoleLoginUnconfigured", 503); + const state = randomBytes(24).toString("base64url"); + const location = new URL(identity.authorizeUrl); + location.searchParams.set("response_type", "code"); + location.searchParams.set("client_id", identity.clientId); + location.searchParams.set("redirect_uri", identity.redirectUri); + location.searchParams.set("scope", "openid email profile"); + location.searchParams.set("state", state); + return new Response(null, { status: 302, headers: { + Location: location.toString(), + "Set-Cookie": `console_login_state=${state}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, + "Cache-Control": "no-store", + } }); +} + +async function cognitoCallback(request: Request, identity: CognitoIdentity): Promise { + if (!identity.tokenUrl || !identity.clientId || !identity.redirectUri || !identity.issuer || !identity.publicKey) return refusal("ConsoleLoginUnconfigured", 503); + const url = new URL(request.url); + const code = url.searchParams.get("code"); + const state = url.searchParams.get("state"); + const cookieState = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); + if (!code || !state || !cookieState || !secureEqual(Buffer.from(state), Buffer.from(cookieState))) return refusal("ConsolePageForbidden"); + const exchange = await (identity.fetcher ?? fetch)(identity.tokenUrl, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity.clientId, redirect_uri: identity.redirectUri, code }), + }); + if (!exchange.ok) return refusal("ConsolePageForbidden"); + const tokens: unknown = await exchange.json(); + if (!object(tokens) || typeof tokens.id_token !== "string") return refusal("ConsolePageForbidden"); + const parts = tokens.id_token.split("."); + if (parts.length !== 3) return refusal("ConsolePageForbidden"); + let header: unknown; + let claims: unknown; + try { header = decode(parts[0]); claims = decode(parts[1]); } catch { return refusal("ConsolePageForbidden"); } + if (!object(header) || header.alg !== "RS256" || !object(claims) || claims.iss !== identity.issuer || claims.aud !== identity.clientId || + typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || (claims.exp as number) <= Math.floor(Date.now() / 1000) || + claims.token_use !== "id") return refusal("ConsolePageForbidden"); + const key = typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; + if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); + const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group): group is string => typeof group === "string") : []; + const lifetime = Math.min(claims.exp as number, Math.floor(Date.now() / 1000) + 3600); + const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime }, identity.sessionSecret); + const destination = groups.includes(identity.adminGroup) ? "/admin" : "/query"; + return new Response(null, { status: 302, headers: { + Location: destination, + "Set-Cookie": `console_session=${session}; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age=${lifetime - Math.floor(Date.now() / 1000)}`, + "Cache-Control": "no-store", + } }); +} + +function refusal(identifier: string, status = 403): Response { + return Response.json({ error: { identifier } }, { status, headers: { "Cache-Control": "no-store" } }); +} + +function json(value: unknown): Response { + return Response.json(value, { headers: { "Cache-Control": "no-store" } }); +} + +async function body(request: Request): Promise { + if (Number(request.headers.get("content-length") ?? 0) > 1_048_576) throw new Error("body too large"); + const text = await request.text(); + if (text.length > 1_048_576) throw new Error("body too large"); + return JSON.parse(text); +} + +function page(name: "query" | "admin"): Response { + return new Response(name === "query" ? queryPage : adminPage, { + headers: { + "Content-Type": "text/html; charset=utf-8", + "Cache-Control": "no-store", + "Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'", + "X-Content-Type-Options": "nosniff", + }, + }); +} + +export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Request) => Promise } { + return { + async fetch(request) { + const url = new URL(request.url); + const path = url.pathname; + if (adapters.identity.kind === "cognito") { + if (path === "/auth/login" && request.method === "GET") return cognitoLogin(adapters.identity); + if (path === "/auth/callback" && request.method === "GET") return cognitoCallback(request, adapters.identity); + } + const grant = path === "/query" || path.startsWith("/query/api/") ? "query" : + path === "/admin" || path.startsWith("/admin/api/") ? "admin" : null; + if (!grant) return refusal("ConsoleRouteNotFound", 404); + let operator: Operator | null; + try { operator = operatorFor(request, adapters.identity); } catch { operator = null; } + if (!operator) return refusal("ConsolePageForbidden", 401); + if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); + if (adapters.identity.kind === "cognito" && request.method === "POST" && request.headers.get("origin") !== url.origin) { + return refusal("ConsolePageForbidden"); + } + if (request.method === "GET" && path === `/${grant}`) return page(grant); + if (grant === "query") { + if (request.method === "GET" && path === "/query/api/stores") return json(await adapters.read.list(operator)); + if (request.method === "POST" && path === "/query/api/ask") { + let input: unknown; + try { input = await body(request); } catch { return refusal("ConsoleRequestMalformed", 400); } + if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || + !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); + return json(await adapters.turn({ operator, store: input.store, question: input.question })); + } + } else { + const store = url.searchParams.get("store"); + if (request.method === "GET" && path === "/admin/api/workflows") return json(await adapters.control.workflows(operator, store)); + if (request.method === "GET" && path === "/admin/api/record") return json(await adapters.control.record(operator, store)); + if (request.method === "GET" && path === "/admin/api/packs") { + if (!store || !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); + if (!adapters.control.listPackFiles) return refusal("ConsoleAdapterUnavailable", 503); + return json(await adapters.control.listPackFiles(operator, store, url.searchParams.get("prefix") ?? "")); + } + if (request.method === "POST" && (path === "/admin/api/edit" || path === "/admin/api/apply")) { + if (!adapters.adminCapability) return refusal("ConsoleAdminGrantMissing"); + const origin = request.headers.get("origin"); + if (origin && origin !== url.origin) return refusal("ConsolePageForbidden"); + let document: unknown; + try { document = await body(request); } catch { return refusal("ConsoleRequestMalformed", 400); } + return json(path.endsWith("/edit") ? + await adapters.control.edit(document, adapters.adminCapability, operator) : + await adapters.control.apply(document, adapters.adminCapability, operator)); + } + } + return refusal("ConsoleRouteNotFound", 404); + }, + }; +} + +const sharedStyle = ``; + +const queryPage = `Query · Contextful${sharedStyle}

Ask the store

Answers cite the rows your access permits.

Conversation

Your answer appears here.

Results

`; + +const adminPage = `Admin · Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; diff --git a/apps/console/src/server.ts b/apps/console/src/server.ts new file mode 100644 index 000000000..3cdc0fe77 --- /dev/null +++ b/apps/console/src/server.ts @@ -0,0 +1,33 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { ConsoleAdapters } from "./index.ts"; +import { createConsole } from "./index.ts"; + +async function receive(message: IncomingMessage, origin: string): Promise { + const method = message.method ?? "GET"; + const chunks: Buffer[] = []; + for await (const chunk of message) chunks.push(Buffer.from(chunk)); + const body = method === "GET" || method === "HEAD" ? undefined : Buffer.concat(chunks); + const headers = new Headers(); + for (const [key, value] of Object.entries(message.headers)) { + if (typeof value === "string") headers.set(key, value); + else if (Array.isArray(value)) headers.set(key, value.join(", ")); + } + return new Request(new URL(message.url ?? "/", origin), { method, headers, body }); +} + +async function send(reply: ServerResponse, response: Response): Promise { + reply.statusCode = response.status; + response.headers.forEach((value, key) => reply.setHeader(key, value)); + reply.end(Buffer.from(await response.arrayBuffer())); +} + +export function serveConsole(adapters: ConsoleAdapters, origin: string): Server { + const app = createConsole(adapters); + return createServer(async (message, reply) => { + try { + await send(reply, await app.fetch(await receive(message, origin))); + } catch { + await send(reply, Response.json({ error: { identifier: "ConsoleServerFailure" } }, { status: 500 })); + } + }); +} diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index d60e56640..b92fbb7ca 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -2,6 +2,7 @@ import test from "node:test"; import assert from "node:assert/strict"; import { generateKeyPairSync, sign } from "node:crypto"; import { createConsole, issueCognitoSession } from "../src/index.ts"; +import { serveConsole } from "../src/server.ts"; const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); const issuer = "https://access.example"; @@ -43,8 +44,8 @@ function fixture(adminCapability = "server-secret") { test("anonymous and forged Access assertions dispatch no page or API", async () => { const { request, calls } = fixture(); for (const path of ["/query", "/query/api/stores", "/admin", "/admin/api/workflows"]) { - assert.equal((await request(path)).status, 403); - assert.equal((await request(path, { "cf-access-jwt-assertion": "forged" })).status, 403); + assert.equal((await request(path)).status, 401); + assert.equal((await request(path, { "cf-access-jwt-assertion": "forged" })).status, 401); } assert.deepEqual(calls, []); }); @@ -63,8 +64,8 @@ test("verified Access audiences grant only their page and API", async () => { test("expired or wrong-issuer assertions cannot open a page", async () => { const { request } = fixture(); - assert.equal((await request("/query", access(queryAudience, { exp: 1 }))).status, 403); - assert.equal((await request("/query", access(queryAudience, { iss: "https://other.example" }))).status, 403); + assert.equal((await request("/query", access(queryAudience, { exp: 1 }))).status, 401); + assert.equal((await request("/query", access(queryAudience, { iss: "https://other.example" }))).status, 401); }); test("Query keeps one composer and transcript and delegates a sourced answer", async () => { @@ -111,6 +112,85 @@ test("Cognito sessions are signed, bounded, and mapped from groups", async () => })(); assert.equal((await request("/query", `console_session=${session}`)).status, 200); assert.equal((await request("/admin/api/workflows", `console_session=${session}`)).status, 403); - assert.equal((await request("/query", `console_session=${session}tampered`)).status, 403); + assert.equal((await request("/query", `console_session=${session}tampered`)).status, 401); assert.deepEqual(calls, []); }); + +test("Cognito hosted login exchanges a code before issuing a first-party session", async () => { + const app = createConsole({ + identity: { + kind: "cognito", sessionSecret: "session-secret", queryGroup: "console-query", adminGroup: "console-admin", + issuer: "https://cognito.example/pool", clientId: "console-client", publicKey, + authorizeUrl: "https://cognito.example/oauth2/authorize", tokenUrl: "https://cognito.example/oauth2/token", + redirectUri: "https://console.example/auth/callback", + fetcher: async (_url, init) => { + assert.equal(init?.method, "POST"); + assert.match(String(init?.body), /code=accepted/); + return Response.json({ id_token: jwt({ iss: "https://cognito.example/pool", aud: "console-client", sub: "operator-3", token_use: "id", "cognito:groups": ["console-admin"], exp: Math.floor(Date.now() / 1000) + 300 }) }); + }, + }, + stores: [], turn: async () => ({ answer: "", sources: [], widgets: [] }), read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const login = await app.fetch(new Request("https://console.example/auth/login")); + assert.equal(login.status, 302); + const location = new URL(login.headers.get("location")!); + const state = location.searchParams.get("state"); + assert.ok(state); + const stateCookie = login.headers.get("set-cookie")!.split(";")[0]; + const callback = await app.fetch(new Request(`https://console.example/auth/callback?code=accepted&state=${state}`, { headers: { cookie: stateCookie } })); + assert.equal(callback.status, 302); + assert.equal(callback.headers.get("location"), "/admin"); + const sessionCookie = callback.headers.get("set-cookie")!.split(";")[0]; + assert.equal((await app.fetch(new Request("https://console.example/admin", { headers: { cookie: sessionCookie } }))).status, 200); + assert.equal((await app.fetch(new Request("https://console.example/query", { headers: { cookie: sessionCookie } }))).status, 403); + const badState = await app.fetch(new Request("https://console.example/auth/callback?code=accepted&state=wrong", { headers: { cookie: stateCookie } })); + assert.equal(badState.status, 403); +}); + +test("the Node HTTP listener verifies a page request before serving content", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + assert.equal((await app.fetch(new Request("https://console.example/query"))).status, 401); + const server = serveConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }, "http://127.0.0.1"); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const url = `http://127.0.0.1:${address.port}`; + assert.equal((await fetch(`${url}/query`)).status, 401); + assert.equal((await fetch(`${url}/query`, { headers: access(queryAudience) })).status, 200); + } finally { + server.close(); + } +}); + +test("Admin pack listing reaches only the injected listing adapter", async () => { + const calls: string[] = []; + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + control: { + workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}), + listPackFiles: async (_operator, store, prefix) => { calls.push(`${store}:${prefix}`); return { entries: ["packs/a.toml"], truncated: false, declined: 2 }; }, + }, + }); + const url = "https://console.example/admin/api/packs?store=field-notes&prefix=packs/"; + assert.equal((await app.fetch(new Request(url, { headers: access(queryAudience) }))).status, 403); + const response = await app.fetch(new Request(url, { headers: access(adminAudience) })); + assert.deepEqual(await response.json(), { entries: ["packs/a.toml"], truncated: false, declined: 2 }); + assert.deepEqual(calls, ["field-notes:packs/"]); +}); From 043c6e9287e09bd6e6629fc8dc2bc2eff38b26eb Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:42:28 +0800 Subject: [PATCH 014/155] test(console): specify governed browse discovery --- apps/console/test/browse.test.ts | 66 ++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 apps/console/test/browse.test.ts diff --git a/apps/console/test/browse.test.ts b/apps/console/test/browse.test.ts new file mode 100644 index 000000000..89f203b1e --- /dev/null +++ b/apps/console/test/browse.test.ts @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { BrowseError, createBrowse, humanizeLabel } from "../src/browse.ts"; + +function fixture() { + const calls: Array<{ tool: string; arguments: Record }> = []; + const transport = { + async call(tool: string, args: Record) { + calls.push({ tool, arguments: args }); + if (tool === "context.describe" && !args.table) return { structuredContent: { + tables: [ + { table: "research/vendor_filings", description: "Filed reports", zone_admitted: true }, + { table: "research/private_payroll", description: "Restricted", zone_admitted: false }, + ], + } }; + if (tool === "context.describe") return { structuredContent: { table: args.table, row_count: 3 } }; + if (tool === "context.files") return { structuredContent: { + columns: ["table", "path"], + rows: [ + ["research/vendor_filings", "research/vendor_filings/runs/run-1/report.parquet"], + ["research/private_payroll", "research/private_payroll/runs/run-1/secret.parquet"], + ], + } }; + if (tool === "context.file") return { structuredContent: { columns: ["title"], rows: [["Q1 report"]] } }; + throw new Error(`unexpected tool ${tool}`); + }, + }; + return { browse: createBrowse(transport), calls }; +} + +test("browse advertises admitted tables as humanized chips and governed insights", async () => { + const { browse, calls } = fixture(); + const result = await browse.discover({ asOf: "2030-01-01T00:00:00Z" }); + assert.deepEqual(result.chips, [{ table: "research/vendor_filings", label: "Vendor Filings", description: "Filed reports" }]); + assert.deepEqual(result.insights, [{ table: "research/vendor_filings", label: "Vendor Filings", rows: 3 }]); + assert.deepEqual(result.files, [{ table: "research/vendor_filings", path: "research/vendor_filings/runs/run-1/report.parquet", label: "Report" }]); + assert.equal(calls[0].tool, "context.describe"); + assert.equal(calls[0].arguments.as_of, "2030-01-01T00:00:00Z"); + assert(calls.every((call) => call.tool.startsWith("context."))); + assert(!calls.some((call) => call.arguments.table === "research/private_payroll")); +}); + +test("a gallery preview refuses paths absent from the governed listing", async () => { + const { browse, calls } = fixture(); + await assert.rejects(browse.preview({ path: "research/vendor_filings/runs/run-1/other.parquet" }), + (error: unknown) => error instanceof BrowseError && error.code === "ConsoleGalleryPathUnlisted"); + assert(!calls.some((call) => call.tool === "context.file")); +}); + +test("a gallery preview uses context.file with the listing's snapshot bound", async () => { + const { browse, calls } = fixture(); + const preview = await browse.preview({ path: "research/vendor_filings/runs/run-1/report.parquet", asOf: "2030-01-01T00:00:00Z" }); + assert.deepEqual(preview, { columns: ["title"], rows: [["Q1 report"]] }); + assert.deepEqual(calls.at(-1), { tool: "context.file", arguments: { path: "research/vendor_filings/runs/run-1/report.parquet", as_of: "2030-01-01T00:00:00Z" } }); +}); + +test("in-band read refusals remain errors", async () => { + const browse = createBrowse({ call: async () => ({ isError: true, structuredContent: { error: { identifier: "EnforceUnknownRelation" } } }) }); + await assert.rejects(browse.discover({}), + (error: unknown) => error instanceof BrowseError && error.code === "EnforceUnknownRelation"); +}); + +test("human labels handle separators and acronyms without adding prompts", () => { + assert.equal(humanizeLabel("reports/q3_sec_filings"), "Q3 SEC Filings"); + assert.equal(humanizeLabel("customer-orders"), "Customer Orders"); +}); From 9f30e129957f83f619ab5a45d9622984280ecfc4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:45:20 +0800 Subject: [PATCH 015/155] feat(console): derive browse views from governed reads --- apps/console/src/browse.ts | 91 ++++++++++++++++++++++++++++++++++++++ spec/71-console.md | 13 ++++++ spec/cards/surface.md | 3 +- spec/spec.lock.json | 81 ++++++++++++++++++++++++++------- spec/status.md | 6 +-- spec/terms/surface.toml | 1 + 6 files changed, 176 insertions(+), 19 deletions(-) create mode 100644 apps/console/src/browse.ts diff --git a/apps/console/src/browse.ts b/apps/console/src/browse.ts new file mode 100644 index 000000000..135da2332 --- /dev/null +++ b/apps/console/src/browse.ts @@ -0,0 +1,91 @@ +type ToolResult = { + structuredContent?: unknown; + isError?: boolean; +}; + +type ReadTransport = { + call(tool: string, arguments_: Record): Promise; +}; + +type TableEntry = { table: string; description?: string; zone_admitted?: boolean }; +type FileEntry = { table: string; path: string; label: string }; +type BrowseRequest = { asOf?: string }; + +export class BrowseError extends Error { + readonly code: string; + + constructor(code: string, message = code) { + super(message); + this.name = "BrowseError"; + this.code = code; + } +} + +const abbreviations = new Set(["API", "CSV", "PDF", "SEC", "SQL", "URL"]); + +export function humanizeLabel(identifier: string): string { + const tail = identifier.split("/").at(-1) ?? identifier; + return tail.replace(/\.[^.]+$/, "").replace(/([a-z])([A-Z])/g, "$1 $2") + .split(/[-_\s]+/).filter(Boolean) + .map((part) => { + const upper = part.toUpperCase(); + return abbreviations.has(upper) || /^[Qq]\d+$/.test(part) + ? upper : part[0].toUpperCase() + part.slice(1).toLowerCase(); + }).join(" "); +} + +function object(value: unknown): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + return value as Record; +} + +function bounds(request: BrowseRequest): Record { + return request.asOf ? { as_of: request.asOf } : {}; +} + +export function createBrowse(transport: ReadTransport) { + async function read(tool: string, arguments_: Record): Promise> { + const result = await transport.call(tool, arguments_); + if (result.isError) { + const error = object(object(result.structuredContent).error); + throw new BrowseError(String(error.identifier ?? "ConsoleBrowseReadRefused")); + } + return object(result.structuredContent); + } + + async function listed(request: BrowseRequest): Promise<{ tables: TableEntry[]; files: FileEntry[] }> { + const listing = await read("context.describe", bounds(request)); + if (!Array.isArray(listing.tables)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + const tables = listing.tables.map((item) => object(item)).filter((item) => item.zone_admitted === true && typeof item.table === "string") + .map((item) => ({ table: item.table as string, description: typeof item.description === "string" ? item.description : undefined })); + const names = new Set(tables.map((table) => table.table)); + const fileListing = await read("context.files", bounds(request)); + if (!Array.isArray(fileListing.rows)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + const files = fileListing.rows.filter((row): row is [string, string] => + Array.isArray(row) && typeof row[0] === "string" && typeof row[1] === "string" && names.has(row[0])) + .map(([table, path]) => ({ table, path, label: humanizeLabel(path) })); + return { tables, files }; + } + + return { + async discover(request: BrowseRequest) { + const { tables, files } = await listed(request); + const chips = tables.map((table) => ({ table: table.table, label: humanizeLabel(table.table), description: table.description })); + const descriptions = await Promise.all(tables.map((table) => read("context.describe", { table: table.table, ...bounds(request) }))); + const insights = tables.flatMap((table, index) => { + const count = descriptions[index].row_count; + return typeof count === "number" && Number.isFinite(count) && count >= 0 + ? [{ table: table.table, label: humanizeLabel(table.table), rows: count }] : []; + }); + return { chips, insights, files }; + }, + + async preview(request: BrowseRequest & { path: string }) { + const { files } = await listed(request); + if (!files.some((file) => file.path === request.path)) throw new BrowseError("ConsoleGalleryPathUnlisted", request.path); + const response = await read("context.file", { path: request.path, ...bounds(request) }); + if (!Array.isArray(response.columns) || !Array.isArray(response.rows)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + return { columns: response.columns, rows: response.rows }; + }, + }; +} diff --git a/spec/71-console.md b/spec/71-console.md index b55f49836..14c65cdb1 100644 --- a/spec/71-console.md +++ b/spec/71-console.md @@ -276,6 +276,19 @@ A session's time basis: its vantage, the bound each leg carries, the snapshot ti What a store advertises before a question: discovered chips, humanized labels, the insights panel and the file gallery. +- `discovered-chips` — Query builds pre-question chips from described tables admitted to the reader's zone and humanizes their table labels. + *A-surface* +- `insights-panel` — The insights panel presents row counts returned by the reader's table descriptions. + *because a row count from the read face already reflects the reader's relation* +- `file-gallery` — The file gallery presents only paths returned by the reader's governed file listing and previews them through the read face. + *P5* +- `gallery-path-unlisted` — A gallery preview naming a path absent from the governed file listing under the request's `as_of` raises `ConsoleGalleryPathUnlisted` before preview dispatch. + *because the browser must not turn an arbitrary path into a store read* + +#### Scenarios + +- `surface.browse.gallery-path-unlisted`: WHEN a Query client requests a path absent from the reader's bound file listing, THEN it raises `ConsoleGalleryPathUnlisted` without a preview call. + ## learn The reading loop's memory: recall ahead of planning, the per-turn distillation and the labels its conclusions inherit. diff --git a/spec/cards/surface.md b/spec/cards/surface.md index b367d1ea7..9d062758e 100644 --- a/spec/cards/surface.md +++ b/spec/cards/surface.md @@ -21,7 +21,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `ground` | Where an answer's material comes from: the closed tool set, the two trust layers, the per-reader credential, the web supplement and the sources block. | 8 | | `plan-turn` | The shape of one turn: planner scaffolding, the replanning round, the answerability test, the code path, and the store's prompt overlay. | 6 | | `set-vantage` | A session's time basis: its vantage, the bound each leg carries, the snapshot timeline and the arrivals strip. | 3 | -| `browse` | What a store advertises before a question: discovered chips, humanized labels, the insights panel and the file gallery. | 0 | +| `browse` | What a store advertises before a question: discovered chips, humanized labels, the insights panel and the file gallery. | 4 | | `learn` | The reading loop's memory: recall ahead of planning, the per-turn distillation and the labels its conclusions inherit. | 3 | | `render` | The three channels, the component union, deterministic component choice, view hints and the sanitizer walk. | 3 | | `brief` | The proactive greeting card: its derivation, payload, matching tiers, client-side clock and the absence it earns. | 6 | @@ -75,6 +75,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `ConsolePlannerReachedMemory` | Planner scaffolding naming a memory relation. | `surface.plan-turn.planner-reached-memory` | | `ConsoleVantageUnparseable` | A vantage that is neither a calendar day nor an instant. | `surface.set-vantage.unparseable` | | `ConsoleWebBoundUnparseable` | A web leg whose publication bound does not parse. | `surface.set-vantage.web-bound-unparseable` | +| `ConsoleGalleryPathUnlisted` | A gallery preview path absent from the reader's governed file listing. | `surface.browse.gallery-path-unlisted` | | `ConsoleLearningUnscoped` | A distilled conclusion landing without the reading-session scope. | `surface.learn.unscoped` | | `ConsoleViewNotServerBuilt` | A view from a client or the model rather than inferred from a result. | `surface.render.client-authored-view` | | `ConsoleBriefUnavailable` | A greeting derivation that errored or overran its budget. | `surface.brief.absence-is-earned` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 50c8feba2..4d581f683 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -19585,6 +19585,53 @@ "file": "spec/71-console.md", "line": 273 }, + { + "id": "surface.browse.discovered-chips", + "contract": "surface", + "operation": "browse", + "subject": "discovered-chips", + "kind": "behavior", + "statement": "Query builds pre-question chips from described tables admitted to the reader's zone and humanizes their table labels.", + "why": "A-surface", + "file": "spec/71-console.md", + "line": 279 + }, + { + "id": "surface.browse.insights-panel", + "contract": "surface", + "operation": "browse", + "subject": "insights-panel", + "kind": "behavior", + "statement": "The insights panel presents row counts returned by the reader's table descriptions.", + "why": "because a row count from the read face already reflects the reader's relation", + "file": "spec/71-console.md", + "line": 281 + }, + { + "id": "surface.browse.file-gallery", + "contract": "surface", + "operation": "browse", + "subject": "file-gallery", + "kind": "behavior", + "statement": "The file gallery presents only paths returned by the reader's governed file listing and previews them through the read face.", + "why": "P5", + "file": "spec/71-console.md", + "line": 283 + }, + { + "id": "surface.browse.gallery-path-unlisted", + "contract": "surface", + "operation": "browse", + "subject": "gallery-path-unlisted", + "kind": "refusal", + "statement": "A gallery preview naming a path absent from the governed file listing under the request's `as_of` raises `ConsoleGalleryPathUnlisted` before preview dispatch.", + "why": "because the browser must not turn an arbitrary path into a store read", + "file": "spec/71-console.md", + "line": 285, + "scenarios": [ + "WHEN a Query client requests a path absent from the reader's bound file listing, THEN it raises `ConsoleGalleryPathUnlisted` without a preview call." + ] + }, { "id": "surface.learn.distillation", "contract": "surface", @@ -19594,7 +19641,7 @@ "statement": "After the answer streams, a second pass distils the exchange into at most 3 entries shaped `{subject, key, learning}`, zero included.", "why": "", "file": "spec/71-console.md", - "line": 283 + "line": 296 }, { "id": "surface.learn.unscoped", @@ -19605,7 +19652,7 @@ "statement": "A distilled conclusion landing without the reading-session scope raises `ConsoleLearningUnscoped`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 284 + "line": 297 }, { "id": "surface.learn.subject-resolution", @@ -19616,7 +19663,7 @@ "statement": "Distillation preserves the subject it observes; recall resolves that subject through entity matching.", "why": "A-surface", "file": "spec/71-console.md", - "line": 286 + "line": 299 }, { "id": "surface.render.client-authored-view", @@ -19627,7 +19674,7 @@ "statement": "A view specification arriving from a client, or composed by the model, raises `ConsoleViewNotServerBuilt`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 293 + "line": 306 }, { "id": "surface.render.component-choice", @@ -19638,7 +19685,7 @@ "statement": "One row with one measure draws a metric; a date column with a measure over 3 rows on distinct days draws a line; anything else draws a table offering a bar view.", "why": "", "file": "spec/71-console.md", - "line": 295 + "line": 308 }, { "id": "surface.render.older-transcript", @@ -19649,7 +19696,7 @@ "statement": "A client reading a transcript with an unknown component-union member renders its view as a table.", "why": "A-surface", "file": "spec/71-console.md", - "line": 296 + "line": 309 }, { "id": "surface.brief.absence-is-earned", @@ -19660,7 +19707,7 @@ "statement": "The card needs a session with no turns, at present time, a live conclusion, an arrived row matching an interest, and a derivation inside its budget. An error or timeout raises `ConsoleBriefUnavailable`, and no card renders.", "why": "A-surface", "file": "spec/71-console.md", - "line": 332 + "line": 345 }, { "id": "surface.brief.topic-tier", @@ -19671,7 +19718,7 @@ "statement": "The topic tier needs at least 2 tokens shared between the conclusion's subject and text and the row's label and topics, one of them naming the subject.", "why": "", "file": "spec/71-console.md", - "line": 334 + "line": 347 }, { "id": "surface.brief.card-subjects", @@ -19682,7 +19729,7 @@ "statement": "A card carries at most 3 subjects.", "why": "", "file": "spec/71-console.md", - "line": 335 + "line": 348 }, { "id": "surface.brief.articles-per-subject", @@ -19693,7 +19740,7 @@ "statement": "A subject carries at most 3 entries of matched article.", "why": "", "file": "spec/71-console.md", - "line": 336 + "line": 349 }, { "id": "surface.brief.catchup-window", @@ -19704,7 +19751,7 @@ "statement": "The backend caps the requested window at 7 d.", "why": "", "file": "spec/71-console.md", - "line": 337 + "line": 350 }, { "id": "surface.brief.greeting", @@ -19715,7 +19762,7 @@ "statement": "Query derives a greeting card from arrived rows matching live conclusions in a turnless present-time session.", "why": "A-surface", "file": "spec/71-console.md", - "line": 338 + "line": 351 }, { "id": "surface.publish-answer.share-affordance", @@ -19726,7 +19773,7 @@ "statement": "A surface offering a share control on access-explanation output raises `VisibilityShareAffordance`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 345 + "line": 358 }, { "id": "surface.publish-answer.askerless-audience", @@ -19737,7 +19784,7 @@ "statement": "A scheduled job posting to an audience under a service identity raises `VisibilityAskerlessAudience`, naming the destination; a scheduled post's corpus is narrowed in the reviewed manifest to what the destination reaches.", "why": "A-surface", "file": "spec/71-console.md", - "line": 347 + "line": 360 }, { "id": "surface.publish-answer.one-operator-principal", @@ -19748,7 +19795,7 @@ "statement": "An answer uses one operator's credential; the console offers no room-intersection principal.", "why": "A-surface", "file": "spec/71-console.md", - "line": 349 + "line": 362 }, { "id": "assurance.model.package", @@ -28383,6 +28430,10 @@ "clause": "surface.ground.direct-file-read", "gloss": "A table function reading stored bytes by path on the caller surface." }, + "ConsoleGalleryPathUnlisted": { + "clause": "surface.browse.gallery-path-unlisted", + "gloss": "A gallery preview path absent from the reader's governed file listing." + }, "ConsoleLearningUnscoped": { "clause": "surface.learn.unscoped", "gloss": "A distilled conclusion landing without the reading-session scope." diff --git a/spec/status.md b/spec/status.md index 4991c91f7..9a4f51412 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 51 | 0 | 51 | +| `surface` | 2 | 20 | 110 | 50 | 22 | 0 | 51 | 0 | 51 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2007 | 581 | 188 | 127 | 1362 | 0 | | +| **total** | 19 | 155 | 2011 | 582 | 188 | 127 | 1362 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 53 | 0 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 130 | 83 | passing | closed | diff --git a/spec/terms/surface.toml b/spec/terms/surface.toml index 37b185793..071f27eb3 100644 --- a/spec/terms/surface.toml +++ b/spec/terms/surface.toml @@ -67,6 +67,7 @@ ConsoleTokenExchangeRefused = { clause = "surface.ground.mint-refused", gloss ConsolePlannerReachedMemory = { clause = "surface.plan-turn.planner-reached-memory", gloss = "Planner scaffolding naming a memory relation." } ConsoleVantageUnparseable = { clause = "surface.set-vantage.unparseable", gloss = "A vantage that is neither a calendar day nor an instant." } ConsoleWebBoundUnparseable = { clause = "surface.set-vantage.web-bound-unparseable", gloss = "A web leg whose publication bound does not parse." } +ConsoleGalleryPathUnlisted = { clause = "surface.browse.gallery-path-unlisted", gloss = "A gallery preview path absent from the reader's governed file listing." } ConsoleLearningUnscoped = { clause = "surface.learn.unscoped", gloss = "A distilled conclusion landing without the reading-session scope." } ConsoleViewNotServerBuilt = { clause = "surface.render.client-authored-view", gloss = "A view from a client or the model rather than inferred from a result." } ConsoleBriefUnavailable = { clause = "surface.brief.absence-is-earned", gloss = "A greeting derivation that errored or overran its budget." } From 103650ce6237d0c53b6016b84d9668bd63ca7765 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:46:30 +0800 Subject: [PATCH 016/155] build(console): typecheck and bundle hosted server --- apps/console/dist/server.js | 442 ++++++++++++++++++++++++++++++++++++ apps/console/package.json | 9 +- apps/console/pnpm-lock.yaml | 303 +++++++++++++++++++++++- apps/console/src/entry.ts | 112 +++++++++ apps/console/src/server.ts | 5 +- apps/console/tsconfig.json | 13 ++ 6 files changed, 880 insertions(+), 4 deletions(-) create mode 100644 apps/console/dist/server.js create mode 100644 apps/console/src/entry.ts create mode 100644 apps/console/tsconfig.json diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js new file mode 100644 index 000000000..63c3a70bf --- /dev/null +++ b/apps/console/dist/server.js @@ -0,0 +1,442 @@ +// src/entry.ts +import { createPublicKey as createPublicKey2 } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { isAbsolute, resolve } from "node:path"; + +// ../gateway/src/index.ts +var RegistryError = class extends Error { + identifier; + constructor(identifier, message) { + super(`${identifier}: ${message}`); + this.identifier = identifier; + this.name = identifier; + } +}; +var RESERVED_IDS = /* @__PURE__ */ new Set(["admin", "query"]); +var ID = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/; +function names(id) { + const bindingName = id.replaceAll("-", "_").toUpperCase(); + return { bindingName, credentialName: `${bindingName}_QUERY_TOKEN` }; +} +function malformed(index) { + return { identifier: "StoreEntryMalformed", entry: index }; +} +function decodeEntry(value, index) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw malformed(index); + const raw = value; + if (typeof raw.id !== "string" || !ID.test(raw.id)) throw malformed(index); + if (RESERVED_IDS.has(raw.id)) throw new RegistryError("StoreIdReserved", raw.id); + for (const key of ["credential", "credentialName", "binding", "bindingName"]) { + if (key in raw) throw new RegistryError("StoreNameAuthored", key); + } + if (typeof raw.endpoint !== "string") throw malformed(index); + let endpoint; + try { + endpoint = new URL(raw.endpoint); + } catch { + throw malformed(index); + } + if (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") throw malformed(index); + for (const key of ["label", "auth", "exchangeRoute", "packPrefix"]) { + if (key in raw && typeof raw[key] !== "string") throw malformed(index); + } + const { credentialName, bindingName } = names(raw.id); + return { + id: raw.id, + label: typeof raw.label === "string" ? raw.label : raw.id, + endpoint: endpoint.toString().replace(/\/$/, ""), + auth: typeof raw.auth === "string" ? raw.auth : void 0, + exchangeRoute: typeof raw.exchangeRoute === "string" ? raw.exchangeRoute : void 0, + packPrefix: typeof raw.packPrefix === "string" ? raw.packPrefix : void 0, + credentialName, + bindingName + }; +} +function registryFromEnv(raw) { + let parsed; + try { + parsed = JSON.parse(raw ?? "[]"); + } catch (error) { + throw new RegistryError("StoreRegistryUnreadable", error instanceof Error ? error.message : "invalid JSON"); + } + if (!Array.isArray(parsed)) throw new RegistryError("StoreRegistryUnreadable", "expected an array"); + const entries = []; + const problems = []; + for (const [index, value] of parsed.entries()) { + try { + entries.push(decodeEntry(value, index)); + } catch (error) { + if (error instanceof RegistryError && (error.identifier === "StoreIdReserved" || error.identifier === "StoreNameAuthored")) throw error; + problems.push(malformed(index)); + } + } + return { entries, problems }; +} + +// src/server.ts +import { createServer } from "node:http"; + +// src/index.ts +import { createHmac, createPublicKey, randomBytes, timingSafeEqual, verify } from "node:crypto"; +function encoded(value) { + return Buffer.from(JSON.stringify(value)).toString("base64url"); +} +function decode(value) { + return JSON.parse(Buffer.from(value, "base64url").toString("utf8")); +} +function object(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +function secureEqual(left, right) { + return left.length === right.length && timingSafeEqual(left, right); +} +function issueCognitoSession(session, secret) { + if (!secret || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3)) { + throw new Error("CognitoSessionInvalid"); + } + const body2 = encoded(session); + const signature = createHmac("sha256", secret).update(body2).digest("base64url"); + return `${body2}.${signature}`; +} +function verifyCognitoSession(cookie, identity2) { + const token = cookie.split("; ").find((part) => part.startsWith("console_session="))?.slice("console_session=".length); + if (!token) return null; + const parts = token.split("."); + if (parts.length !== 2) return null; + const expected = createHmac("sha256", identity2.sessionSecret).update(parts[0]).digest(); + let actual; + try { + actual = Buffer.from(parts[1], "base64url"); + } catch { + return null; + } + if (!secureEqual(expected, actual)) return null; + let session; + try { + session = decode(parts[0]); + } catch { + return null; + } + if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string")) return null; + const grants = /* @__PURE__ */ new Set(); + if (session.groups.includes(identity2.queryGroup)) grants.add("query"); + if (session.groups.includes(identity2.adminGroup)) grants.add("admin"); + return { subject: session.subject, grants }; +} +function verifyAccess(assertion, identity2) { + const parts = assertion.split("."); + if (parts.length !== 3) return null; + let header; + let claims; + try { + header = decode(parts[0]); + claims = decode(parts[1]); + } catch { + return null; + } + if (!object(header) || header.alg !== "RS256" || !object(claims)) return null; + if (claims.iss !== identity2.issuer || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3)) return null; + if (typeof claims.nbf === "number" && claims.nbf > Math.floor(Date.now() / 1e3)) return null; + const audience = typeof claims.aud === "string" ? [claims.aud] : Array.isArray(claims.aud) ? claims.aud : []; + const grants = /* @__PURE__ */ new Set(); + if (audience.includes(identity2.queryAudience)) grants.add("query"); + if (audience.includes(identity2.adminAudience)) grants.add("admin"); + if (grants.size === 0) return null; + let signature; + try { + signature = Buffer.from(parts[2], "base64url"); + } catch { + return null; + } + const key = identity2.keys ? typeof header.kid === "string" ? identity2.keys.get(header.kid) : void 0 : typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; + if (!key) return null; + if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, signature)) return null; + return { subject: claims.sub, grants }; +} +function operatorFor(request, identity2) { + if (identity2.kind === "access") { + const assertion = request.headers.get("cf-access-jwt-assertion"); + return assertion ? verifyAccess(assertion, identity2) : null; + } + return verifyCognitoSession(request.headers.get("cookie") ?? "", identity2); +} +function cognitoLogin(identity2) { + if (!identity2.authorizeUrl || !identity2.clientId || !identity2.redirectUri) return refusal("ConsoleLoginUnconfigured", 503); + const state = randomBytes(24).toString("base64url"); + const location = new URL(identity2.authorizeUrl); + location.searchParams.set("response_type", "code"); + location.searchParams.set("client_id", identity2.clientId); + location.searchParams.set("redirect_uri", identity2.redirectUri); + location.searchParams.set("scope", "openid email profile"); + location.searchParams.set("state", state); + return new Response(null, { status: 302, headers: { + Location: location.toString(), + "Set-Cookie": `console_login_state=${state}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, + "Cache-Control": "no-store" + } }); +} +async function cognitoCallback(request, identity2) { + if (!identity2.tokenUrl || !identity2.clientId || !identity2.redirectUri || !identity2.issuer || !identity2.publicKey) return refusal("ConsoleLoginUnconfigured", 503); + const url = new URL(request.url); + const code = url.searchParams.get("code"); + const state = url.searchParams.get("state"); + const cookieState = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); + if (!code || !state || !cookieState || !secureEqual(Buffer.from(state), Buffer.from(cookieState))) return refusal("ConsolePageForbidden"); + const exchange = await (identity2.fetcher ?? fetch)(identity2.tokenUrl, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity2.clientId, redirect_uri: identity2.redirectUri, code }) + }); + if (!exchange.ok) return refusal("ConsolePageForbidden"); + const tokens = await exchange.json(); + if (!object(tokens) || typeof tokens.id_token !== "string") return refusal("ConsolePageForbidden"); + const parts = tokens.id_token.split("."); + if (parts.length !== 3) return refusal("ConsolePageForbidden"); + let header; + let claims; + try { + header = decode(parts[0]); + claims = decode(parts[1]); + } catch { + return refusal("ConsolePageForbidden"); + } + if (!object(header) || header.alg !== "RS256" || !object(claims) || claims.iss !== identity2.issuer || claims.aud !== identity2.clientId || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3) || claims.token_use !== "id") return refusal("ConsolePageForbidden"); + const key = typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; + if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); + const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group) => typeof group === "string") : []; + const lifetime = Math.min(claims.exp, Math.floor(Date.now() / 1e3) + 3600); + const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime }, identity2.sessionSecret); + const destination = groups.includes(identity2.adminGroup) ? "/admin" : "/query"; + return new Response(null, { status: 302, headers: { + Location: destination, + "Set-Cookie": `console_session=${session}; HttpOnly; Secure; SameSite=Strict; Path=/; Max-Age=${lifetime - Math.floor(Date.now() / 1e3)}`, + "Cache-Control": "no-store" + } }); +} +function refusal(identifier, status = 403) { + return Response.json({ error: { identifier } }, { status, headers: { "Cache-Control": "no-store" } }); +} +function json(value) { + return Response.json(value, { headers: { "Cache-Control": "no-store" } }); +} +async function body(request) { + if (Number(request.headers.get("content-length") ?? 0) > 1048576) throw new Error("body too large"); + const text = await request.text(); + if (text.length > 1048576) throw new Error("body too large"); + return JSON.parse(text); +} +function page(name) { + return new Response(name === "query" ? queryPage : adminPage, { + headers: { + "Content-Type": "text/html; charset=utf-8", + "Cache-Control": "no-store", + "Content-Security-Policy": "default-src 'none'; style-src 'unsafe-inline'; script-src 'unsafe-inline'; connect-src 'self'; form-action 'self'; base-uri 'none'; frame-ancestors 'none'", + "X-Content-Type-Options": "nosniff" + } + }); +} +function createConsole(adapters) { + return { + async fetch(request) { + const url = new URL(request.url); + const path = url.pathname; + if (adapters.identity.kind === "cognito") { + if (path === "/auth/login" && request.method === "GET") return cognitoLogin(adapters.identity); + if (path === "/auth/callback" && request.method === "GET") return cognitoCallback(request, adapters.identity); + } + const grant = path === "/query" || path.startsWith("/query/api/") ? "query" : path === "/admin" || path.startsWith("/admin/api/") ? "admin" : null; + if (!grant) return refusal("ConsoleRouteNotFound", 404); + let operator; + try { + operator = operatorFor(request, adapters.identity); + } catch { + operator = null; + } + if (!operator) return refusal("ConsolePageForbidden", 401); + if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); + if (adapters.identity.kind === "cognito" && request.method === "POST" && request.headers.get("origin") !== url.origin) { + return refusal("ConsolePageForbidden"); + } + if (request.method === "GET" && path === `/${grant}`) return page(grant); + if (grant === "query") { + if (request.method === "GET" && path === "/query/api/stores") return json(await adapters.read.list(operator)); + if (request.method === "POST" && path === "/query/api/ask") { + let input; + try { + input = await body(request); + } catch { + return refusal("ConsoleRequestMalformed", 400); + } + if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); + return json(await adapters.turn({ operator, store: input.store, question: input.question })); + } + } else { + const store = url.searchParams.get("store"); + if (request.method === "GET" && path === "/admin/api/workflows") return json(await adapters.control.workflows(operator, store)); + if (request.method === "GET" && path === "/admin/api/record") return json(await adapters.control.record(operator, store)); + if (request.method === "GET" && path === "/admin/api/packs") { + if (!store || !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); + if (!adapters.control.listPackFiles) return refusal("ConsoleAdapterUnavailable", 503); + return json(await adapters.control.listPackFiles(operator, store, url.searchParams.get("prefix") ?? "")); + } + if (request.method === "POST" && (path === "/admin/api/edit" || path === "/admin/api/apply")) { + if (!adapters.adminCapability) return refusal("ConsoleAdminGrantMissing"); + const origin = request.headers.get("origin"); + if (origin && origin !== url.origin) return refusal("ConsolePageForbidden"); + let document; + try { + document = await body(request); + } catch { + return refusal("ConsoleRequestMalformed", 400); + } + return json(path.endsWith("/edit") ? await adapters.control.edit(document, adapters.adminCapability, operator) : await adapters.control.apply(document, adapters.adminCapability, operator)); + } + } + return refusal("ConsoleRouteNotFound", 404); + } + }; +} +var sharedStyle = ``; +var queryPage = `Query \xB7 Contextful${sharedStyle}

Ask the store

Answers cite the rows your access permits.

Conversation

Your answer appears here.

Results

`; +var adminPage = `Admin \xB7 Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; + +// src/server.ts +async function receive(message, origin) { + const method = message.method ?? "GET"; + const chunks = []; + for await (const chunk of message) chunks.push(Buffer.from(chunk)); + const body2 = method === "GET" || method === "HEAD" ? void 0 : Buffer.concat(chunks); + const headers = new Headers(); + for (const [key, value] of Object.entries(message.headers)) { + if (typeof value === "string") headers.set(key, value); + else if (Array.isArray(value)) headers.set(key, value.join(", ")); + } + return new Request(new URL(message.url ?? "/", origin), { method, headers, body: body2 }); +} +async function send(reply, response) { + reply.statusCode = response.status; + response.headers.forEach((value, key) => reply.setHeader(key, value)); + reply.end(Buffer.from(await response.arrayBuffer())); +} +function serveConsole(adapters, origin) { + const app = createConsole(adapters); + return createServer(async (message, reply) => { + try { + await send(reply, await app.fetch(await receive(message, origin))); + } catch (error) { + const unavailable2 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; + await send(reply, Response.json({ error: { identifier: unavailable2 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable2 ? 503 : 500 })); + } + }); +} + +// src/entry.ts +function required(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} +async function keysAt(url) { + const response = await fetch(url); + if (!response.ok) throw new Error(`identity key route returned ${response.status}`); + const document = await response.json(); + if (!document || typeof document !== "object" || !("keys" in document) || !Array.isArray(document.keys)) { + throw new Error("identity key route returned no key set"); + } + const keys = /* @__PURE__ */ new Map(); + for (const value of document.keys) { + if (!value || typeof value !== "object" || value.kty !== "RSA" || value.alg !== "RS256" || typeof value.kid !== "string") continue; + keys.set(value.kid, createPublicKey2({ key: value, format: "jwk" })); + } + if (keys.size === 0) throw new Error("identity key route returned no RS256 key"); + return keys; +} +async function identity() { + if (process.env.CONTEXTFUL_IDENTITY === "cognito") { + const keys = await keysAt(required("CONTEXTFUL_COGNITO_JWKS_URL")); + if (keys.size !== 1) throw new Error("Cognito key route needs one pinned key"); + return { + kind: "cognito", + sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), + queryGroup: required("CONTEXTFUL_COGNITO_QUERY_GROUP"), + adminGroup: required("CONTEXTFUL_COGNITO_ADMIN_GROUP"), + issuer: required("CONTEXTFUL_COGNITO_ISSUER"), + clientId: required("CONTEXTFUL_COGNITO_CLIENT_ID"), + authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), + tokenUrl: required("CONTEXTFUL_COGNITO_TOKEN_URL"), + redirectUri: required("CONTEXTFUL_COGNITO_REDIRECT_URI"), + publicKey: [...keys.values()][0] + }; + } + return { + kind: "access", + issuer: required("CONTEXTFUL_ACCESS_ISSUER"), + queryAudience: required("CONTEXTFUL_QUERY_ACCESS_AUDIENCE"), + adminAudience: required("CONTEXTFUL_ADMIN_ACCESS_AUDIENCE"), + keys: await keysAt(required("CONTEXTFUL_ACCESS_JWKS_URL")) + }; +} +function unavailable() { + throw new Error("ConsoleAdapterUnavailable"); +} +function unavailableAdapters() { + return { + turn: async () => unavailable(), + control: { + workflows: async () => unavailable(), + record: async () => unavailable(), + edit: async () => unavailable(), + apply: async () => unavailable() + } + }; +} +async function main() { + const addressFlag = process.argv.indexOf("--http"); + if (addressFlag < 0 || !process.argv[addressFlag + 1]) throw new Error("--http is required"); + const address = process.argv[addressFlag + 1]; + const separator = address.lastIndexOf(":"); + const host = address.slice(0, separator); + const port = Number(address.slice(separator + 1)); + if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); + const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); + const stores = registry.entries.map(({ id, label }) => ({ id, label })); + let adapters = unavailableAdapters(); + const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; + if (modulePath) { + const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); + const module = await import(pathToFileURL(absolute).href); + if (!module.createAdapters) throw new Error("console adapter module exports no createAdapters"); + adapters = await module.createAdapters({ stores, env: process.env }); + } + const app = serveConsole({ + identity: await identity(), + stores, + adminCapability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, + turn: adapters.turn, + read: adapters.read ?? { list: async () => stores }, + control: adapters.control + }, process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`); + await new Promise((resolveListen, rejectListen) => { + app.once("error", rejectListen); + app.listen(port, host, resolveListen); + }); + const bound = app.address(); + if (!bound || typeof bound === "string") throw new Error("console listener has no address"); + process.stderr.write(`listening on http://${host}:${bound.port} +`); +} +main().catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : "console startup failed"} +`); + process.exitCode = 1; +}); diff --git a/apps/console/package.json b/apps/console/package.json index ac1f158a9..8b68b29ef 100644 --- a/apps/console/package.json +++ b/apps/console/package.json @@ -3,6 +3,13 @@ "private": true, "type": "module", "scripts": { - "test": "node --experimental-strip-types --test test/*.test.ts" + "typecheck": "tsc --noEmit", + "test": "node --experimental-strip-types --test test/*.test.ts", + "build": "esbuild src/entry.ts --bundle --platform=node --format=esm --target=node22 --outfile=dist/server.js" + }, + "devDependencies": { + "@types/node": "22.20.4", + "esbuild": "0.25.12", + "typescript": "6.0.3" } } diff --git a/apps/console/pnpm-lock.yaml b/apps/console/pnpm-lock.yaml index 9b60ae178..63aff606d 100644 --- a/apps/console/pnpm-lock.yaml +++ b/apps/console/pnpm-lock.yaml @@ -6,4 +6,305 @@ settings: importers: - .: {} + .: + devDependencies: + '@types/node': + specifier: 22.20.4 + version: 22.20.4 + esbuild: + specifier: 0.25.12 + version: 0.25.12 + typescript: + specifier: 6.0.3 + version: 6.0.3 + +packages: + + '@esbuild/aix-ppc64@0.25.12': + resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.12': + resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.12': + resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.12': + resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.12': + resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.12': + resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.12': + resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.12': + resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.12': + resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.12': + resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.12': + resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.12': + resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.12': + resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.12': + resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.12': + resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.12': + resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.12': + resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.12': + resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.12': + resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.12': + resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.12': + resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.25.12': + resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.25.12': + resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.12': + resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.12': + resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.12': + resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@types/node@22.20.4': + resolution: {integrity: sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==} + + esbuild@0.25.12: + resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} + engines: {node: '>=18'} + hasBin: true + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + +snapshots: + + '@esbuild/aix-ppc64@0.25.12': + optional: true + + '@esbuild/android-arm64@0.25.12': + optional: true + + '@esbuild/android-arm@0.25.12': + optional: true + + '@esbuild/android-x64@0.25.12': + optional: true + + '@esbuild/darwin-arm64@0.25.12': + optional: true + + '@esbuild/darwin-x64@0.25.12': + optional: true + + '@esbuild/freebsd-arm64@0.25.12': + optional: true + + '@esbuild/freebsd-x64@0.25.12': + optional: true + + '@esbuild/linux-arm64@0.25.12': + optional: true + + '@esbuild/linux-arm@0.25.12': + optional: true + + '@esbuild/linux-ia32@0.25.12': + optional: true + + '@esbuild/linux-loong64@0.25.12': + optional: true + + '@esbuild/linux-mips64el@0.25.12': + optional: true + + '@esbuild/linux-ppc64@0.25.12': + optional: true + + '@esbuild/linux-riscv64@0.25.12': + optional: true + + '@esbuild/linux-s390x@0.25.12': + optional: true + + '@esbuild/linux-x64@0.25.12': + optional: true + + '@esbuild/netbsd-arm64@0.25.12': + optional: true + + '@esbuild/netbsd-x64@0.25.12': + optional: true + + '@esbuild/openbsd-arm64@0.25.12': + optional: true + + '@esbuild/openbsd-x64@0.25.12': + optional: true + + '@esbuild/openharmony-arm64@0.25.12': + optional: true + + '@esbuild/sunos-x64@0.25.12': + optional: true + + '@esbuild/win32-arm64@0.25.12': + optional: true + + '@esbuild/win32-ia32@0.25.12': + optional: true + + '@esbuild/win32-x64@0.25.12': + optional: true + + '@types/node@22.20.4': + dependencies: + undici-types: 6.21.0 + + esbuild@0.25.12: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.12 + '@esbuild/android-arm': 0.25.12 + '@esbuild/android-arm64': 0.25.12 + '@esbuild/android-x64': 0.25.12 + '@esbuild/darwin-arm64': 0.25.12 + '@esbuild/darwin-x64': 0.25.12 + '@esbuild/freebsd-arm64': 0.25.12 + '@esbuild/freebsd-x64': 0.25.12 + '@esbuild/linux-arm': 0.25.12 + '@esbuild/linux-arm64': 0.25.12 + '@esbuild/linux-ia32': 0.25.12 + '@esbuild/linux-loong64': 0.25.12 + '@esbuild/linux-mips64el': 0.25.12 + '@esbuild/linux-ppc64': 0.25.12 + '@esbuild/linux-riscv64': 0.25.12 + '@esbuild/linux-s390x': 0.25.12 + '@esbuild/linux-x64': 0.25.12 + '@esbuild/netbsd-arm64': 0.25.12 + '@esbuild/netbsd-x64': 0.25.12 + '@esbuild/openbsd-arm64': 0.25.12 + '@esbuild/openbsd-x64': 0.25.12 + '@esbuild/openharmony-arm64': 0.25.12 + '@esbuild/sunos-x64': 0.25.12 + '@esbuild/win32-arm64': 0.25.12 + '@esbuild/win32-ia32': 0.25.12 + '@esbuild/win32-x64': 0.25.12 + + typescript@6.0.3: {} + + undici-types@6.21.0: {} diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts new file mode 100644 index 000000000..59b892446 --- /dev/null +++ b/apps/console/src/entry.ts @@ -0,0 +1,112 @@ +import { createPublicKey } from "node:crypto"; +import { pathToFileURL } from "node:url"; +import { isAbsolute, resolve } from "node:path"; +import { registryFromEnv } from "../../gateway/src/index.ts"; +import type { ConsoleAdapters, Identity } from "./index.ts"; +import { serveConsole } from "./server.ts"; + +type AdapterFactory = (input: { stores: ConsoleAdapters["stores"]; env: NodeJS.ProcessEnv }) => Promise & { read?: ConsoleAdapters["read"] }>; + +function required(name: string): string { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +async function keysAt(url: string): Promise>> { + const response = await fetch(url); + if (!response.ok) throw new Error(`identity key route returned ${response.status}`); + const document: unknown = await response.json(); + if (!document || typeof document !== "object" || !("keys" in document) || !Array.isArray(document.keys)) { + throw new Error("identity key route returned no key set"); + } + const keys = new Map>(); + for (const value of document.keys) { + if (!value || typeof value !== "object" || value.kty !== "RSA" || value.alg !== "RS256" || typeof value.kid !== "string") continue; + keys.set(value.kid, createPublicKey({ key: value, format: "jwk" })); + } + if (keys.size === 0) throw new Error("identity key route returned no RS256 key"); + return keys; +} + +async function identity(): Promise { + if (process.env.CONTEXTFUL_IDENTITY === "cognito") { + const keys = await keysAt(required("CONTEXTFUL_COGNITO_JWKS_URL")); + if (keys.size !== 1) throw new Error("Cognito key route needs one pinned key"); + return { + kind: "cognito", + sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), + queryGroup: required("CONTEXTFUL_COGNITO_QUERY_GROUP"), + adminGroup: required("CONTEXTFUL_COGNITO_ADMIN_GROUP"), + issuer: required("CONTEXTFUL_COGNITO_ISSUER"), + clientId: required("CONTEXTFUL_COGNITO_CLIENT_ID"), + authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), + tokenUrl: required("CONTEXTFUL_COGNITO_TOKEN_URL"), + redirectUri: required("CONTEXTFUL_COGNITO_REDIRECT_URI"), + publicKey: [...keys.values()][0], + }; + } + return { + kind: "access", + issuer: required("CONTEXTFUL_ACCESS_ISSUER"), + queryAudience: required("CONTEXTFUL_QUERY_ACCESS_AUDIENCE"), + adminAudience: required("CONTEXTFUL_ADMIN_ACCESS_AUDIENCE"), + keys: await keysAt(required("CONTEXTFUL_ACCESS_JWKS_URL")), + }; +} + +function unavailable(): never { + throw new Error("ConsoleAdapterUnavailable"); +} + +function unavailableAdapters(): Pick { + return { + turn: async () => unavailable(), + control: { + workflows: async () => unavailable(), + record: async () => unavailable(), + edit: async () => unavailable(), + apply: async () => unavailable(), + }, + }; +} + +async function main(): Promise { + const addressFlag = process.argv.indexOf("--http"); + if (addressFlag < 0 || !process.argv[addressFlag + 1]) throw new Error("--http is required"); + const address = process.argv[addressFlag + 1]; + const separator = address.lastIndexOf(":"); + const host = address.slice(0, separator); + const port = Number(address.slice(separator + 1)); + if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); + const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); + const stores = registry.entries.map(({ id, label }) => ({ id, label })); + let adapters: Pick & { read?: ConsoleAdapters["read"] } = unavailableAdapters(); + const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; + if (modulePath) { + const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); + const module: { createAdapters?: AdapterFactory } = await import(pathToFileURL(absolute).href); + if (!module.createAdapters) throw new Error("console adapter module exports no createAdapters"); + adapters = await module.createAdapters({ stores, env: process.env }); + } + const app = serveConsole({ + identity: await identity(), + stores, + adminCapability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, + turn: adapters.turn, + read: adapters.read ?? { list: async () => stores }, + control: adapters.control, + }, process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`); + await new Promise((resolveListen, rejectListen) => { + app.once("error", rejectListen); + app.listen(port, host, resolveListen); + }); + const bound = app.address(); + if (!bound || typeof bound === "string") throw new Error("console listener has no address"); + process.stderr.write(`listening on http://${host}:${bound.port}\n`); +} + +main().catch((error: unknown) => { + process.stderr.write(`${error instanceof Error ? error.message : "console startup failed"}\n`); + process.exitCode = 1; +}); diff --git a/apps/console/src/server.ts b/apps/console/src/server.ts index 3cdc0fe77..327b0c988 100644 --- a/apps/console/src/server.ts +++ b/apps/console/src/server.ts @@ -26,8 +26,9 @@ export function serveConsole(adapters: ConsoleAdapters, origin: string): Server return createServer(async (message, reply) => { try { await send(reply, await app.fetch(await receive(message, origin))); - } catch { - await send(reply, Response.json({ error: { identifier: "ConsoleServerFailure" } }, { status: 500 })); + } catch (error) { + const unavailable = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; + await send(reply, Response.json({ error: { identifier: unavailable ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable ? 503 : 500 })); } }); } diff --git a/apps/console/tsconfig.json b/apps/console/tsconfig.json new file mode 100644 index 000000000..bb9a739db --- /dev/null +++ b/apps/console/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "allowImportingTsExtensions": true, + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "test/**/*.ts", "../gateway/src/index.ts"] +} From b16b744c3cad8f9d61ed6edc43e47d0e68289d08 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:50:29 +0800 Subject: [PATCH 017/155] fix(console): bind PKCE and Origin to the hosted session --- apps/console/dist/server.js | 37 ++++++++++++++++++--------- apps/console/src/entry.ts | 7 +++--- apps/console/src/index.ts | 32 +++++++++++++++++------ apps/console/src/server.ts | 4 +-- apps/console/test/console.test.ts | 42 ++++++++++++++++++++++++++++--- 5 files changed, 94 insertions(+), 28 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 63c3a70bf..810ef0a69 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -77,7 +77,7 @@ function registryFromEnv(raw) { import { createServer } from "node:http"; // src/index.ts -import { createHmac, createPublicKey, randomBytes, timingSafeEqual, verify } from "node:crypto"; +import { createHash, createHmac, createPublicKey, randomBytes, timingSafeEqual, verify } from "node:crypto"; function encoded(value) { return Buffer.from(JSON.stringify(value)).toString("base64url"); } @@ -163,29 +163,40 @@ function operatorFor(request, identity2) { function cognitoLogin(identity2) { if (!identity2.authorizeUrl || !identity2.clientId || !identity2.redirectUri) return refusal("ConsoleLoginUnconfigured", 503); const state = randomBytes(24).toString("base64url"); + const verifier = randomBytes(32).toString("base64url"); + const issued = Math.floor(Date.now() / 1e3).toString(); + const loginData = `${state}.${verifier}.${issued}`; + const signature = createHmac("sha256", identity2.sessionSecret).update(loginData).digest("base64url"); const location = new URL(identity2.authorizeUrl); location.searchParams.set("response_type", "code"); location.searchParams.set("client_id", identity2.clientId); location.searchParams.set("redirect_uri", identity2.redirectUri); location.searchParams.set("scope", "openid email profile"); location.searchParams.set("state", state); + location.searchParams.set("code_challenge", createHash("sha256").update(verifier).digest("base64url")); + location.searchParams.set("code_challenge_method", "S256"); return new Response(null, { status: 302, headers: { Location: location.toString(), - "Set-Cookie": `console_login_state=${state}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, + "Set-Cookie": `console_login_state=${loginData}.${signature}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, "Cache-Control": "no-store" } }); } async function cognitoCallback(request, identity2) { - if (!identity2.tokenUrl || !identity2.clientId || !identity2.redirectUri || !identity2.issuer || !identity2.publicKey) return refusal("ConsoleLoginUnconfigured", 503); + if (!identity2.tokenUrl || !identity2.clientId || !identity2.redirectUri || !identity2.issuer || !identity2.publicKey && !identity2.keys) return refusal("ConsoleLoginUnconfigured", 503); const url = new URL(request.url); const code = url.searchParams.get("code"); const state = url.searchParams.get("state"); - const cookieState = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); - if (!code || !state || !cookieState || !secureEqual(Buffer.from(state), Buffer.from(cookieState))) return refusal("ConsolePageForbidden"); + const loginCookie = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); + const loginParts = loginCookie?.split("."); + if (!code || !state || !loginParts || loginParts.length !== 4) return refusal("ConsolePageForbidden"); + const [cookieState, verifier, issued, signature] = loginParts; + const expected = createHmac("sha256", identity2.sessionSecret).update(`${cookieState}.${verifier}.${issued}`).digest("base64url"); + const age = Math.floor(Date.now() / 1e3) - Number(issued); + if (!secureEqual(Buffer.from(state), Buffer.from(cookieState)) || !secureEqual(Buffer.from(signature), Buffer.from(expected)) || !Number.isInteger(age) || age < 0 || age > 300 || !/^[A-Za-z0-9_-]{43}$/.test(verifier)) return refusal("ConsolePageForbidden"); const exchange = await (identity2.fetcher ?? fetch)(identity2.tokenUrl, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity2.clientId, redirect_uri: identity2.redirectUri, code }) + body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity2.clientId, redirect_uri: identity2.redirectUri, code, code_verifier: verifier }) }); if (!exchange.ok) return refusal("ConsolePageForbidden"); const tokens = await exchange.json(); @@ -201,7 +212,8 @@ async function cognitoCallback(request, identity2) { return refusal("ConsolePageForbidden"); } if (!object(header) || header.alg !== "RS256" || !object(claims) || claims.iss !== identity2.issuer || claims.aud !== identity2.clientId || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3) || claims.token_use !== "id") return refusal("ConsolePageForbidden"); - const key = typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; + const key = identity2.keys ? typeof header.kid === "string" ? identity2.keys.get(header.kid) : void 0 : typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; + if (!key) return refusal("ConsolePageForbidden"); if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group) => typeof group === "string") : []; const lifetime = Math.min(claims.exp, Math.floor(Date.now() / 1e3) + 3600); @@ -254,7 +266,7 @@ function createConsole(adapters) { } if (!operator) return refusal("ConsolePageForbidden", 401); if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); - if (adapters.identity.kind === "cognito" && request.method === "POST" && request.headers.get("origin") !== url.origin) { + if (request.method === "POST" && (grant === "admin" || adapters.identity.kind === "cognito") && request.headers.get("origin") !== url.origin) { return refusal("ConsolePageForbidden"); } if (request.method === "GET" && path === `/${grant}`) return page(grant); @@ -332,7 +344,7 @@ function serveConsole(adapters, origin) { const app = createConsole(adapters); return createServer(async (message, reply) => { try { - await send(reply, await app.fetch(await receive(message, origin))); + await send(reply, await app.fetch(await receive(message, typeof origin === "string" ? origin : origin()))); } catch (error) { const unavailable2 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; await send(reply, Response.json({ error: { identifier: unavailable2 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable2 ? 503 : 500 })); @@ -364,7 +376,6 @@ async function keysAt(url) { async function identity() { if (process.env.CONTEXTFUL_IDENTITY === "cognito") { const keys = await keysAt(required("CONTEXTFUL_COGNITO_JWKS_URL")); - if (keys.size !== 1) throw new Error("Cognito key route needs one pinned key"); return { kind: "cognito", sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), @@ -375,7 +386,7 @@ async function identity() { authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), tokenUrl: required("CONTEXTFUL_COGNITO_TOKEN_URL"), redirectUri: required("CONTEXTFUL_COGNITO_REDIRECT_URI"), - publicKey: [...keys.values()][0] + keys }; } return { @@ -418,6 +429,7 @@ async function main() { if (!module.createAdapters) throw new Error("console adapter module exports no createAdapters"); adapters = await module.createAdapters({ stores, env: process.env }); } + let origin = process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`; const app = serveConsole({ identity: await identity(), stores, @@ -425,13 +437,14 @@ async function main() { turn: adapters.turn, read: adapters.read ?? { list: async () => stores }, control: adapters.control - }, process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`); + }, () => origin); await new Promise((resolveListen, rejectListen) => { app.once("error", rejectListen); app.listen(port, host, resolveListen); }); const bound = app.address(); if (!bound || typeof bound === "string") throw new Error("console listener has no address"); + if (!process.env.CONTEXTFUL_CONSOLE_ORIGIN) origin = `http://${host}:${bound.port}`; process.stderr.write(`listening on http://${host}:${bound.port} `); } diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 59b892446..0ee90294c 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -32,7 +32,6 @@ async function keysAt(url: string): Promise { if (process.env.CONTEXTFUL_IDENTITY === "cognito") { const keys = await keysAt(required("CONTEXTFUL_COGNITO_JWKS_URL")); - if (keys.size !== 1) throw new Error("Cognito key route needs one pinned key"); return { kind: "cognito", sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), @@ -43,7 +42,7 @@ async function identity(): Promise { authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), tokenUrl: required("CONTEXTFUL_COGNITO_TOKEN_URL"), redirectUri: required("CONTEXTFUL_COGNITO_REDIRECT_URI"), - publicKey: [...keys.values()][0], + keys, }; } return { @@ -89,6 +88,7 @@ async function main(): Promise { if (!module.createAdapters) throw new Error("console adapter module exports no createAdapters"); adapters = await module.createAdapters({ stores, env: process.env }); } + let origin = process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`; const app = serveConsole({ identity: await identity(), stores, @@ -96,13 +96,14 @@ async function main(): Promise { turn: adapters.turn, read: adapters.read ?? { list: async () => stores }, control: adapters.control, - }, process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`); + }, () => origin); await new Promise((resolveListen, rejectListen) => { app.once("error", rejectListen); app.listen(port, host, resolveListen); }); const bound = app.address(); if (!bound || typeof bound === "string") throw new Error("console listener has no address"); + if (!process.env.CONTEXTFUL_CONSOLE_ORIGIN) origin = `http://${host}:${bound.port}`; process.stderr.write(`listening on http://${host}:${bound.port}\n`); } diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 1a6408c7a..04d3486d1 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -1,4 +1,4 @@ -import { createHmac, createPublicKey, randomBytes, timingSafeEqual, verify, type KeyObject } from "node:crypto"; +import { createHash, createHmac, createPublicKey, randomBytes, timingSafeEqual, verify, type KeyObject } from "node:crypto"; export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin"> }; export type Store = { id: string; label: string }; @@ -21,6 +21,7 @@ export type CognitoIdentity = { tokenUrl?: string; redirectUri?: string; publicKey?: KeyObject | string; + keys?: ReadonlyMap; fetcher?: typeof fetch; }; export type Identity = AccessIdentity | CognitoIdentity; @@ -124,30 +125,43 @@ function operatorFor(request: Request, identity: Identity): Operator | null { function cognitoLogin(identity: CognitoIdentity): Response { if (!identity.authorizeUrl || !identity.clientId || !identity.redirectUri) return refusal("ConsoleLoginUnconfigured", 503); const state = randomBytes(24).toString("base64url"); + const verifier = randomBytes(32).toString("base64url"); + const issued = Math.floor(Date.now() / 1000).toString(); + const loginData = `${state}.${verifier}.${issued}`; + const signature = createHmac("sha256", identity.sessionSecret).update(loginData).digest("base64url"); const location = new URL(identity.authorizeUrl); location.searchParams.set("response_type", "code"); location.searchParams.set("client_id", identity.clientId); location.searchParams.set("redirect_uri", identity.redirectUri); location.searchParams.set("scope", "openid email profile"); location.searchParams.set("state", state); + location.searchParams.set("code_challenge", createHash("sha256").update(verifier).digest("base64url")); + location.searchParams.set("code_challenge_method", "S256"); return new Response(null, { status: 302, headers: { Location: location.toString(), - "Set-Cookie": `console_login_state=${state}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, + "Set-Cookie": `console_login_state=${loginData}.${signature}; HttpOnly; Secure; SameSite=Lax; Path=/auth/callback; Max-Age=300`, "Cache-Control": "no-store", } }); } async function cognitoCallback(request: Request, identity: CognitoIdentity): Promise { - if (!identity.tokenUrl || !identity.clientId || !identity.redirectUri || !identity.issuer || !identity.publicKey) return refusal("ConsoleLoginUnconfigured", 503); + if (!identity.tokenUrl || !identity.clientId || !identity.redirectUri || !identity.issuer || (!identity.publicKey && !identity.keys)) return refusal("ConsoleLoginUnconfigured", 503); const url = new URL(request.url); const code = url.searchParams.get("code"); const state = url.searchParams.get("state"); - const cookieState = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); - if (!code || !state || !cookieState || !secureEqual(Buffer.from(state), Buffer.from(cookieState))) return refusal("ConsolePageForbidden"); + const loginCookie = request.headers.get("cookie")?.split(/;\s*/).find((part) => part.startsWith("console_login_state="))?.slice("console_login_state=".length); + const loginParts = loginCookie?.split("."); + if (!code || !state || !loginParts || loginParts.length !== 4) return refusal("ConsolePageForbidden"); + const [cookieState, verifier, issued, signature] = loginParts; + const expected = createHmac("sha256", identity.sessionSecret).update(`${cookieState}.${verifier}.${issued}`).digest("base64url"); + const age = Math.floor(Date.now() / 1000) - Number(issued); + if (!secureEqual(Buffer.from(state), Buffer.from(cookieState)) || + !secureEqual(Buffer.from(signature), Buffer.from(expected)) || !Number.isInteger(age) || age < 0 || age > 300 || + !/^[A-Za-z0-9_-]{43}$/.test(verifier)) return refusal("ConsolePageForbidden"); const exchange = await (identity.fetcher ?? fetch)(identity.tokenUrl, { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, - body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity.clientId, redirect_uri: identity.redirectUri, code }), + body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity.clientId, redirect_uri: identity.redirectUri, code, code_verifier: verifier }), }); if (!exchange.ok) return refusal("ConsolePageForbidden"); const tokens: unknown = await exchange.json(); @@ -160,7 +174,9 @@ async function cognitoCallback(request: Request, identity: CognitoIdentity): Pro if (!object(header) || header.alg !== "RS256" || !object(claims) || claims.iss !== identity.issuer || claims.aud !== identity.clientId || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || (claims.exp as number) <= Math.floor(Date.now() / 1000) || claims.token_use !== "id") return refusal("ConsolePageForbidden"); - const key = typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; + const key = identity.keys ? (typeof header.kid === "string" ? identity.keys.get(header.kid) : undefined) : + typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; + if (!key) return refusal("ConsolePageForbidden"); if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group): group is string => typeof group === "string") : []; const lifetime = Math.min(claims.exp as number, Math.floor(Date.now() / 1000) + 3600); @@ -215,7 +231,7 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req try { operator = operatorFor(request, adapters.identity); } catch { operator = null; } if (!operator) return refusal("ConsolePageForbidden", 401); if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); - if (adapters.identity.kind === "cognito" && request.method === "POST" && request.headers.get("origin") !== url.origin) { + if (request.method === "POST" && (grant === "admin" || adapters.identity.kind === "cognito") && request.headers.get("origin") !== url.origin) { return refusal("ConsolePageForbidden"); } if (request.method === "GET" && path === `/${grant}`) return page(grant); diff --git a/apps/console/src/server.ts b/apps/console/src/server.ts index 327b0c988..ab4ba0f65 100644 --- a/apps/console/src/server.ts +++ b/apps/console/src/server.ts @@ -21,11 +21,11 @@ async function send(reply: ServerResponse, response: Response): Promise { reply.end(Buffer.from(await response.arrayBuffer())); } -export function serveConsole(adapters: ConsoleAdapters, origin: string): Server { +export function serveConsole(adapters: ConsoleAdapters, origin: string | (() => string)): Server { const app = createConsole(adapters); return createServer(async (message, reply) => { try { - await send(reply, await app.fetch(await receive(message, origin))); + await send(reply, await app.fetch(await receive(message, typeof origin === "string" ? origin : origin()))); } catch (error) { const unavailable = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; await send(reply, Response.json({ error: { identifier: unavailable ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable ? 503 : 500 })); diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index b92fbb7ca..4a66e9dd8 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -84,13 +84,17 @@ test("Admin exposes workflows and requires a server-held capability for edit and const { request, calls } = fixture(); const workflows = await request("/admin/api/workflows", access(adminAudience)); assert.equal((await workflows.json() as { pipelines: unknown[] }).pipelines.length, 1); - const edit = await request("/admin/api/edit", access(adminAudience), { method: "POST", body: "{}" }); + const origin = "https://console.example"; + const edit = await request("/admin/api/edit", { ...access(adminAudience), origin }, { method: "POST", body: "{}" }); assert.equal(edit.status, 200); - const apply = await request("/admin/api/apply", access(adminAudience), { method: "POST", body: "{}" }); + const apply = await request("/admin/api/apply", { ...access(adminAudience), origin }, { method: "POST", body: "{}" }); assert.equal(apply.status, 200); assert.deepEqual(calls, ["workflows", "edit:server-secret", "apply:server-secret"]); + assert.equal((await request("/admin/api/edit", access(adminAudience), { method: "POST", body: "{}" })).status, 403); + assert.equal((await request("/admin/api/edit", { ...access(adminAudience), origin: "https://other.example" }, { method: "POST", body: "{}" })).status, 403); + assert.deepEqual(calls, ["workflows", "edit:server-secret", "apply:server-secret"]); const withoutCapability = fixture(""); - const refused = await withoutCapability.request("/admin/api/apply", access(adminAudience), { method: "POST", body: "{}" }); + const refused = await withoutCapability.request("/admin/api/apply", { ...access(adminAudience), origin }, { method: "POST", body: "{}" }); assert.equal(refused.status, 403); assert.match(await refused.text(), /ConsoleAdminGrantMissing/); assert.deepEqual(withoutCapability.calls, []); @@ -126,6 +130,7 @@ test("Cognito hosted login exchanges a code before issuing a first-party session fetcher: async (_url, init) => { assert.equal(init?.method, "POST"); assert.match(String(init?.body), /code=accepted/); + assert.match(String(init?.body), /code_verifier=/); return Response.json({ id_token: jwt({ iss: "https://cognito.example/pool", aud: "console-client", sub: "operator-3", token_use: "id", "cognito:groups": ["console-admin"], exp: Math.floor(Date.now() / 1000) + 300 }) }); }, }, @@ -137,6 +142,8 @@ test("Cognito hosted login exchanges a code before issuing a first-party session const location = new URL(login.headers.get("location")!); const state = location.searchParams.get("state"); assert.ok(state); + assert.equal(location.searchParams.get("code_challenge_method"), "S256"); + assert.match(location.searchParams.get("code_challenge") ?? "", /^[A-Za-z0-9_-]{43}$/); const stateCookie = login.headers.get("set-cookie")!.split(";")[0]; const callback = await app.fetch(new Request(`https://console.example/auth/callback?code=accepted&state=${state}`, { headers: { cookie: stateCookie } })); assert.equal(callback.status, 302); @@ -146,6 +153,35 @@ test("Cognito hosted login exchanges a code before issuing a first-party session assert.equal((await app.fetch(new Request("https://console.example/query", { headers: { cookie: sessionCookie } }))).status, 403); const badState = await app.fetch(new Request("https://console.example/auth/callback?code=accepted&state=wrong", { headers: { cookie: stateCookie } })); assert.equal(badState.status, 403); + const missingVerifier = await app.fetch(new Request(`https://console.example/auth/callback?code=accepted&state=${state}`, { headers: { cookie: `console_login_state=${state}` } })); + assert.equal(missingVerifier.status, 403); + const [cookieName, cookieValue] = stateCookie.split("="); + const tampered = cookieValue.split("."); + tampered[1] = `${tampered[1].slice(0, -1)}${tampered[1].endsWith("A") ? "B" : "A"}`; + const wrongVerifier = await app.fetch(new Request(`https://console.example/auth/callback?code=accepted&state=${state}`, { headers: { cookie: `${cookieName}=${tampered.join(".")}` } })); + assert.equal(wrongVerifier.status, 403); +}); + +test("Cognito Admin POST uses the listener's bound port for its Origin", async () => { + let edits = 0; + const app = serveConsole({ + identity: { kind: "cognito", sessionSecret: "session-secret", queryGroup: "query", adminGroup: "admin" }, + adminCapability: "server-secret", + stores: [], turn: async () => ({ answer: "", sources: [], widgets: [] }), read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => { edits++; return { version: 1 }; }, apply: async () => ({}) }, + }, () => origin); + await new Promise((resolve) => app.listen(0, "127.0.0.1", resolve)); + const address = app.address(); + assert.ok(address && typeof address !== "string"); + const origin = `http://127.0.0.1:${address.port}`; + const session = issueCognitoSession({ subject: "operator", groups: ["admin"], expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + try { + const response = await fetch(`${origin}/admin/api/edit`, { + method: "POST", headers: { cookie: `console_session=${session}`, origin, "content-type": "application/json" }, body: "{}", + }); + assert.equal(response.status, 200); + assert.equal(edits, 1); + } finally { app.close(); } }); test("the Node HTTP listener verifies a page request before serving content", async () => { From 89cc82381ecf347623ad1ffa5b889ada0a70b76b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:55:11 +0800 Subject: [PATCH 018/155] test(console): reject oversized HTTP request bodies --- apps/console/test/console.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 4a66e9dd8..015dee6f1 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -212,6 +212,28 @@ test("the Node HTTP listener verifies a page request before serving content", as } }); +test("the Node HTTP listener rejects an oversized body before dispatch", async () => { + let turns = 0; + const server = serveConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => { turns++; return { answer: "", sources: [], widgets: [] }; }, + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }, "http://127.0.0.1"); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + assert.ok(address && typeof address !== "string"); + const response = await fetch(`http://127.0.0.1:${address.port}/query/api/ask`, { + method: "POST", headers: { "content-type": "application/json" }, body: Buffer.alloc(1_048_577, "x"), + }); + assert.equal(response.status, 413); + assert.equal((await response.json() as { error: { identifier: string } }).error.identifier, "ConsoleBodyTooLarge"); + assert.equal(turns, 0); + } finally { server.close(); } +}); + test("Admin pack listing reaches only the injected listing adapter", async () => { const calls: string[] = []; const app = createConsole({ From fc9e1cb5e3d5104929d43350e9318f68e8dceac9 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:55:35 +0800 Subject: [PATCH 019/155] fix(console): cap incoming HTTP bodies at one MiB --- apps/console/src/server.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/apps/console/src/server.ts b/apps/console/src/server.ts index ab4ba0f65..c5aa76f6c 100644 --- a/apps/console/src/server.ts +++ b/apps/console/src/server.ts @@ -2,10 +2,17 @@ import { createServer, type IncomingMessage, type Server, type ServerResponse } import type { ConsoleAdapters } from "./index.ts"; import { createConsole } from "./index.ts"; +const maxBodyBytes = 1_048_576; + async function receive(message: IncomingMessage, origin: string): Promise { const method = message.method ?? "GET"; const chunks: Buffer[] = []; - for await (const chunk of message) chunks.push(Buffer.from(chunk)); + let size = 0; + for await (const chunk of message) { + size += chunk.length; + if (size > maxBodyBytes) throw new Error("ConsoleBodyTooLarge"); + chunks.push(Buffer.from(chunk)); + } const body = method === "GET" || method === "HEAD" ? undefined : Buffer.concat(chunks); const headers = new Headers(); for (const [key, value] of Object.entries(message.headers)) { @@ -27,6 +34,10 @@ export function serveConsole(adapters: ConsoleAdapters, origin: string | (() => try { await send(reply, await app.fetch(await receive(message, typeof origin === "string" ? origin : origin()))); } catch (error) { + if (error instanceof Error && error.message === "ConsoleBodyTooLarge") { + await send(reply, Response.json({ error: { identifier: "ConsoleBodyTooLarge" } }, { status: 413 })); + return; + } const unavailable = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; await send(reply, Response.json({ error: { identifier: unavailable ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable ? 503 : 500 })); } From 8e67fc5f4e2089200560b2783b5f86cf4864cd60 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:56:01 +0800 Subject: [PATCH 020/155] build(console): refresh server bundle with request cap --- apps/console/dist/server.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 810ef0a69..8fc15d7f7 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -323,10 +323,16 @@ const canvas=document.getElementById('canvas'),record=document.getElementById('r `; // src/server.ts +var maxBodyBytes = 1048576; async function receive(message, origin) { const method = message.method ?? "GET"; const chunks = []; - for await (const chunk of message) chunks.push(Buffer.from(chunk)); + let size = 0; + for await (const chunk of message) { + size += chunk.length; + if (size > maxBodyBytes) throw new Error("ConsoleBodyTooLarge"); + chunks.push(Buffer.from(chunk)); + } const body2 = method === "GET" || method === "HEAD" ? void 0 : Buffer.concat(chunks); const headers = new Headers(); for (const [key, value] of Object.entries(message.headers)) { @@ -346,6 +352,10 @@ function serveConsole(adapters, origin) { try { await send(reply, await app.fetch(await receive(message, typeof origin === "string" ? origin : origin()))); } catch (error) { + if (error instanceof Error && error.message === "ConsoleBodyTooLarge") { + await send(reply, Response.json({ error: { identifier: "ConsoleBodyTooLarge" } }, { status: 413 })); + return; + } const unavailable2 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; await send(reply, Response.json({ error: { identifier: unavailable2 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable2 ? 503 : 500 })); } From f6c27ca8f46cb64afbd55b024f3e76615e034855 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:58:59 +0800 Subject: [PATCH 021/155] test(console): reject Access Query POST without same-origin Origin --- apps/console/test/console.test.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 015dee6f1..514c30c9b 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -74,12 +74,21 @@ test("Query keeps one composer and transcript and delegates a sourced answer", a assert.match(page, /id="composer"/); assert.match(page, /id="transcript"/); assert.match(page, /id="widgets"/); - const answer = await request("/query/api/ask", access(queryAudience), { method: "POST", body: JSON.stringify({ store: "field-notes", question: "What arrived?" }) }); + const answer = await request("/query/api/ask", { ...access(queryAudience), origin: "https://console.example" }, { method: "POST", body: JSON.stringify({ store: "field-notes", question: "What arrived?" }) }); assert.equal(answer.status, 200); assert.equal((await answer.json() as { sources: unknown[] }).sources.length, 1); assert.deepEqual(calls, ["turn:operator-1:field-notes:What arrived?"]); }); +test("Access Query rejects cross-origin and originless POST before a turn", async () => { + const { request, calls } = fixture(); + const body = JSON.stringify({ store: "field-notes", question: "What arrived?" }); + for (const headers of [access(queryAudience), { ...access(queryAudience), origin: "https://other.example" }]) { + assert.equal((await request("/query/api/ask", headers, { method: "POST", body })).status, 403); + } + assert.deepEqual(calls, []); +}); + test("Admin exposes workflows and requires a server-held capability for edit and apply", async () => { const { request, calls } = fixture(); const workflows = await request("/admin/api/workflows", access(adminAudience)); From 01cbffb2e26d836376ef6b9c827f6b47a0f69255 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:59:36 +0800 Subject: [PATCH 022/155] fix(console): require same-origin Origin for Access Query writes --- apps/console/dist/server.js | 2 +- apps/console/src/index.ts | 2 +- crates/acceptance/tests/integration/m12.rs | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 8fc15d7f7..221f1910f 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -266,7 +266,7 @@ function createConsole(adapters) { } if (!operator) return refusal("ConsolePageForbidden", 401); if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); - if (request.method === "POST" && (grant === "admin" || adapters.identity.kind === "cognito") && request.headers.get("origin") !== url.origin) { + if (request.method === "POST" && request.headers.get("origin") !== url.origin) { return refusal("ConsolePageForbidden"); } if (request.method === "GET" && path === `/${grant}`) return page(grant); diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 04d3486d1..4b1ece282 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -231,7 +231,7 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req try { operator = operatorFor(request, adapters.identity); } catch { operator = null; } if (!operator) return refusal("ConsolePageForbidden", 401); if (!operator.grants.has(grant)) return refusal("ConsolePageForbidden"); - if (request.method === "POST" && (grant === "admin" || adapters.identity.kind === "cognito") && request.headers.get("origin") !== url.origin) { + if (request.method === "POST" && request.headers.get("origin") !== url.origin) { return refusal("ConsolePageForbidden"); } if (request.method === "GET" && path === `/${grant}`) return page(grant); diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index 24924593b..985a42875 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -65,9 +65,10 @@ fn request(address: &str, method: &str, path: &str, token: Option<&str>, body: O let mut stream = TcpStream::connect(address).unwrap(); let data = body.map(Value::to_string).unwrap_or_default(); let auth = token.map(|t| format!("Cf-Access-Jwt-Assertion: {t}\r\n")).unwrap_or_default(); + let origin = if method == "POST" { format!("Origin: http://{address}\r\n") } else { String::new() }; write!( stream, - "{method} {path} HTTP/1.1\r\nHost: {address}\r\n{auth}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{data}", + "{method} {path} HTTP/1.1\r\nHost: {address}\r\n{auth}{origin}Content-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{data}", data.len() ) .unwrap(); From 6675dde84013f92fe0cbef1ba258c899c1f814dc Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:01:15 +0800 Subject: [PATCH 023/155] test(serve): require admin for published workflows and apply --- .../contextful-cli/tests/integration/serve.rs | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index dafcfb36c..6b8c36425 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -106,6 +106,45 @@ fn query() -> Value { json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } }) } +fn control(addr: &str, method: &str, path: &str, token: Option<&str>, body: &str) -> (u16, Value) { + let mut stream = TcpStream::connect(addr).unwrap(); + let auth = token.map(|token| format!("Authorization: Bearer {token}\r\n")).unwrap_or_default(); + write!(stream, "{method} {path} HTTP/1.1\r\n{auth}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); + let mut raw = String::new(); + stream.read_to_string(&mut raw).unwrap(); + let (head, answer) = raw.split_once("\r\n\r\n").unwrap(); + (head.split(' ').nth(1).unwrap().parse().unwrap(), serde_json::from_str(answer).unwrap()) +} + +#[test] +fn control_routes_require_admin_and_project_applied_workflows() { + let (dir, public) = project(); + let root = dir.path(); + let pipeline = "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1h\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), pipeline).unwrap(); + stdout(&run(root, &["pipeline", "import", "--project", "research"])); + let read = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--action", "read", "--table", "*", "--ttl", "900"])); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--action", "admin", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + + assert_eq!(control(&addr, "GET", "/control/workflows", None, "").0, 401); + assert_eq!(control(&addr, "GET", "/control/workflows", Some(&read), "").0, 403); + assert_eq!(control(&addr, "POST", "/control/apply", Some(&read), "{}").0, 403); + let (status, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); + assert_eq!(status, 200, "{view}"); + assert_eq!(view["applied"], json!(1)); + assert_eq!(view["pipelines"][0]["id"], "filings-flow"); + assert_eq!(view["pipelines"][0]["tables"], json!(["research/notes"])); + + std::fs::write(root.join("pipelines/filings.toml"), pipeline.replace("every 1h", "every 1d")).unwrap(); + let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":\"filings-flow\"}"); + assert_eq!(status, 200, "{applied}"); + assert_eq!(applied["applied"], json!(2)); + let (_, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); + assert_eq!(view["pipelines"][0]["schedule"], "every 1d"); +} + /// The network transport refuses to start without its audience, its ceiling, or an issuer key that resolves and parses. // spec: topology.publish-hostname.issuer-key@a7736a7f #[test] From 262c9d9db34828f57a777e7df0fd2acebfd864a7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:08:07 +0800 Subject: [PATCH 024/155] feat(serve): expose authenticated published workflow control --- crates/contextful-agent/src/http.rs | 72 ++++++++++++++----- crates/contextful-cli/src/cadence.rs | 11 +++ crates/contextful-cli/src/serve.rs | 35 ++++++++- .../contextful-cli/tests/integration/serve.rs | 11 ++- 4 files changed, 105 insertions(+), 24 deletions(-) diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 37062fc43..365c2d705 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -15,6 +15,7 @@ use crate::mcp::{build_identity, Caller, ReadRecord, Tools}; use contextful_context::read::Face; use contextful_core::ports::Clock; +use contextful_core::grant::{Action, TablePattern}; use contextful_core::read::{ReadError, Refusal}; use contextful_core::AuthorityError; use contextful_policy::enforce::refuse::payload; @@ -41,6 +42,10 @@ pub const MCP_PATH: &str = "/mcp"; /// The readiness route (`read.register.health`). pub const HEALTH_PATH: &str = "/health"; +/// The store-published workflow state and validated control claim routes. +pub const WORKFLOWS_PATH: &str = "/control/workflows"; +pub const APPLY_PATH: &str = "/control/apply"; + /// Largest request head the face reads; a head past it answers `431`. const REQUEST_HEAD_BYTES: usize = 64 * 1024; @@ -108,7 +113,7 @@ pub struct HttpResponse { } impl HttpResponse { - fn json(status: u16, body: &Value) -> HttpResponse { + pub fn json(status: u16, body: &Value) -> HttpResponse { HttpResponse { status, headers: vec![("Content-Type".into(), "application/json".into())], body: body.to_string().into_bytes() } } @@ -193,6 +198,7 @@ pub struct Admitting<'a, C> { pub struct HttpFace<'a, C> { tools: Tools<'a>, admitting: Admitting<'a, C>, + control: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, ceiling: usize, in_flight: AtomicUsize, } @@ -241,7 +247,13 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { let ceiling = ceiling(max_in_flight)?; audience(Some(admitting.audience))?; let tools = Tools::new(face, clock, record)?; - Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0) }) + Ok(HttpFace { tools, admitting, control: None, ceiling, in_flight: AtomicUsize::new(0) }) + } + + /// Attach the store's control API without adding tools to the closed read face. + pub fn with_control(mut self, control: &'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)) -> Self { + self.control = Some(control); + self } /// Accept connections on `listener` until it fails. Each accepted connection takes a @@ -329,28 +341,60 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { (MCP_PATH, _) => { HttpResponse::message(405, "`/mcp` answers POST; the face holds no session and opens no server stream").with("Allow", "POST") } + (WORKFLOWS_PATH, "GET") | (APPLY_PATH, "POST") if self.control.is_some() => self.control(request), + (WORKFLOWS_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/workflows` answers GET").with("Allow", "GET"), + (APPLY_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/apply` answers POST").with("Allow", "POST"), (other, _) => HttpResponse::message(404, format!("no route `{other}`; the protocol endpoint is `{MCP_PATH}`")), } } /// Admit the request's credential, then answer its one message. fn message(&self, request: &HttpRequest) -> HttpResponse { + let (authority, revocation) = match self.admit(request) { + Ok(a) => a, + Err(response) => return response, + }; + let message: Value = match serde_json::from_slice(&request.body) { + Ok(m @ Value::Object(_)) => m, + Ok(_) => return HttpResponse::rpc_error(INVALID_REQUEST, "a request body holds one JSON-RPC message object"), + Err(e) => return HttpResponse::rpc_error(PARSE_ERROR, format!("the request body is not JSON: {e}")), + }; + let admission = Admission::new(self.tools.clock().now(), &revocation).expecting(self.admitting.audience); + let boundary = |a: &AdmittedAuthority| effect_boundary(a, &admission); + match self.tools.handle(Caller { authority: &authority, boundary: &boundary }, &message) { + Some(answer) => HttpResponse::json(200, &answer), + None => HttpResponse::empty(202), + } + } + + fn control(&self, request: &HttpRequest) -> HttpResponse { + let (authority, _) = match self.admit(request) { + Ok(a) => a, + Err(response) => return response, + }; + if !authority.grants().iter().any(|grant| grant.actions.contains(&Action::Admin) && grant.tables.contains(&TablePattern::All)) { + return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlAdminGrantMissing" } })); + } + self.control.expect("the route is installed only with a control handler")(request) + } + + fn admit(&self, request: &HttpRequest) -> Result<(AdmittedAuthority, RevocationState), HttpResponse> { let Some(credential) = request.credential() else { let missing = ReadError::HttpCredentialMissing( "a request carries `Authorization: Bearer `, or `Authorization: DPoP ` with a `DPoP` proof from the credential's holder key".into(), ); - return HttpResponse::json(401, &payload(&Refusal::Read(missing))).with("WWW-Authenticate", CHALLENGE); + return Err(HttpResponse::json(401, &payload(&Refusal::Read(missing))).with("WWW-Authenticate", CHALLENGE)); }; let revocation = match (self.admitting.revocation)() { Ok(r) => r, - Err(why) => return HttpResponse::unavailable(format!("the revocation denylist is unreadable: {why}")), + Err(why) => return Err(HttpResponse::unavailable(format!("the revocation denylist is unreadable: {why}"))), }; let admission = Admission::new(self.tools.clock().now(), &revocation).expecting(self.admitting.audience); let covered = ProofRequest { method: &request.method, target: &request.target, body: &request.body }; let checkpoint = self.admitting.checkpoint; let keys = match checkpoint.keys() { Ok(k) => k, - Err(e) => return HttpResponse::unavailable(e.to_string()), + Err(e) => return Err(HttpResponse::unavailable(e.to_string())), }; let authority = verify_network(credential, &keys, &admission, |jkt| match request.header("DPoP") { Some(proof) => checkpoint.verify_request(jkt, proof, &covered), @@ -358,20 +402,10 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { "the credential binds a holder key, and the request carries no `DPoP` proof".into(), ))), }); - let authority = match authority { - Ok(a) => a, - Err(ProofRefusal::NonceCacheFull) => return HttpResponse::unavailable(ProofRefusal::NonceCacheFull.to_string()), - Err(ProofRefusal::Refused(e)) => return unadmitted(&e), - }; - let message: Value = match serde_json::from_slice(&request.body) { - Ok(m @ Value::Object(_)) => m, - Ok(_) => return HttpResponse::rpc_error(INVALID_REQUEST, "a request body holds one JSON-RPC message object"), - Err(e) => return HttpResponse::rpc_error(PARSE_ERROR, format!("the request body is not JSON: {e}")), - }; - let boundary = |a: &AdmittedAuthority| effect_boundary(a, &admission); - match self.tools.handle(Caller { authority: &authority, boundary: &boundary }, &message) { - Some(answer) => HttpResponse::json(200, &answer), - None => HttpResponse::empty(202), + match authority { + Ok(a) => Ok((a, revocation)), + Err(ProofRefusal::NonceCacheFull) => Err(HttpResponse::unavailable(ProofRefusal::NonceCacheFull.to_string())), + Err(ProofRefusal::Refused(e)) => Err(unadmitted(&e)), } } } diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index d8a888d43..ba3605d5f 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -27,6 +27,7 @@ use contextful_engine::worker::{Relay, WorkerDispatch}; use contextful_outbound::egress::{system, Outbound, Transport}; use serde::Serialize; use serde_json::json; +use serde_json::Value; use std::collections::BTreeMap; use std::net::SocketAddr; use std::path::{Path, PathBuf}; @@ -228,6 +229,16 @@ fn applied(snaps: &Source) -> Result<(Option, BTreeMap Result { + let text = std::fs::read_to_string(declaration).unwrap_or_default(); + let control = control_config(&text, project)?; + let (version, pipelines) = applied(&control.source)?; + let store = contextful_context::Store::open(&project.dir, &project.name)?; + let runs = contextful_sync::run_state::run_states(&store)?; + Ok(json!({ "applied": version, "pipelines": pipelines.into_values().collect::>(), "runs": runs })) +} + /// The snapshot document: every specification as a `[[pipeline]]` block, sorted by id. fn render(specs: &BTreeMap) -> Result { #[derive(Serialize)] diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 05953d645..ecfe191ad 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -12,13 +12,15 @@ use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; -use contextful_agent::http::{audience, ceiling, Admitting, HttpFace}; +use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, WORKFLOWS_PATH}; +use contextful_core::surface::SurfaceError; use contextful_policy::audit::AuditLog; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; use std::net::TcpListener; use std::path::PathBuf; use std::sync::Arc; +use serde_json::{json, Value}; /// The refusals of starting the network transport. `Display` begins with the identifier. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -108,7 +110,36 @@ pub fn run(args: ServeArgs) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; - let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)).map_err(anyhow::Error::msg)?; + let control = |request: &HttpRequest| -> HttpResponse { + let answer = match request.target.split('?').next().unwrap_or_default() { + WORKFLOWS_PATH => crate::cadence::published(&located.project, &located.declaration), + APPLY_PATH => { + let body: Value = match serde_json::from_slice(&request.body) { + Ok(Value::Object(body)) => Value::Object(body), + _ => return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })), + }; + let Some(fields) = body.as_object() else { unreachable!() }; + if fields.len() > 1 || fields.keys().any(|field| field != "id") || fields.get("id").is_some_and(|id| !id.is_string()) { + return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); + } + let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; + crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), &Default::default()) + .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) + } + _ => unreachable!(), + }; + match answer { + Ok(state) => HttpResponse::json(200, &state), + Err(error) => { + let surface = error.chain().find_map(|part| part.downcast_ref::()); + let status = surface.map_or(503, SurfaceError::status); + let identifier = surface.map(|refusal| refusal.to_string().split(':').next().unwrap_or("ControlUnavailable").to_string()) + .unwrap_or_else(|| "ControlUnavailable".into()); + HttpResponse::json(status, &json!({ "error": { "identifier": identifier } })) + } + } + }; + let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)).map_err(anyhow::Error::msg)?.with_control(&control); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); http.serve(listener)?; diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 6b8c36425..ee39f579d 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -124,19 +124,24 @@ fn control_routes_require_admin_and_project_applied_workflows() { std::fs::create_dir_all(root.join("pipelines")).unwrap(); std::fs::write(root.join("pipelines/filings.toml"), pipeline).unwrap(); stdout(&run(root, &["pipeline", "import", "--project", "research"])); - let read = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--action", "read", "--table", "*", "--ttl", "900"])); - let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--action", "admin", "--table", "*", "--ttl", "900"])); + let read = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "900"])); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let narrow_admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "unrelated/*", "--ttl", "900"])); let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); assert_eq!(control(&addr, "GET", "/control/workflows", None, "").0, 401); - assert_eq!(control(&addr, "GET", "/control/workflows", Some(&read), "").0, 403); + let (status, answer) = control(&addr, "GET", "/control/workflows", Some(&read), ""); + assert_eq!(status, 403, "{answer}"); assert_eq!(control(&addr, "POST", "/control/apply", Some(&read), "{}").0, 403); + assert_eq!(control(&addr, "GET", "/control/workflows", Some(&narrow_admin), "").0, 403); let (status, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); assert_eq!(status, 200, "{view}"); assert_eq!(view["applied"], json!(1)); assert_eq!(view["pipelines"][0]["id"], "filings-flow"); assert_eq!(view["pipelines"][0]["tables"], json!(["research/notes"])); + assert_eq!(control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":1}").0, 400); + std::fs::write(root.join("pipelines/filings.toml"), pipeline.replace("every 1h", "every 1d")).unwrap(); let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":\"filings-flow\"}"); assert_eq!(status, 200, "{applied}"); From 0b596a4581507e831d4cb504d36f46825e0ff98d Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:11:20 +0800 Subject: [PATCH 025/155] test(serve): recheck authority before control apply --- .../tests/integration/http.rs | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/crates/contextful-agent/tests/integration/http.rs b/crates/contextful-agent/tests/integration/http.rs index 7b7951782..6e19e38fa 100644 --- a/crates/contextful-agent/tests/integration/http.rs +++ b/crates/contextful-agent/tests/integration/http.rs @@ -101,10 +101,14 @@ fn holder(n: u8) -> SigningKey { /// A credential reading `tables` for `ttl` seconds, bound to `holder` when one is given. fn credential(signer: &SeedSigner, tables: &str, ttl: u64, holder: Option<&SigningKey>) -> String { + credential_with_action(signer, Action::Read, tables, ttl, holder) +} + +fn credential_with_action(signer: &SeedSigner, action: Action, tables: &str, ttl: u64, holder: Option<&SigningKey>) -> String { let policy = IssuancePolicy::parse(&format!("default_audience = \"{AUD}\"\nmax_lifetime_secs = 86400\n")).unwrap(); let subject = Subject { on_behalf_of: Some("user://dana@acme.example".into()), zone: Some("on-prem:hq".into()), ..Subject::default() }; let grant = Grant { - actions: vec![Action::Read], + actions: vec![action], tables: vec![TablePattern::parse(tables).unwrap()], tenant: None, aggregate: None, @@ -119,6 +123,33 @@ fn credential(signer: &SeedSigner, tables: &str, ttl: u64, holder: Option<&Signi mint(&plan, &MintClaims { confirmation, ..MintClaims::default() }, signer).unwrap() } +#[test] +fn control_apply_rechecks_revocation_before_dispatch() { + let f = fixture(); + let token = credential_with_action(&f.signer, Action::Admin, "*", 900, None); + let clock = FixedClock(at(NOW)); + let checks = std::sync::atomic::AtomicUsize::new(0); + let revocation = || { + if checks.fetch_add(1, Ordering::SeqCst) == 0 { Ok(RevocationState::default()) } + else { Err("revocation source changed".into()) } + }; + let admitting = Admitting { checkpoint: &f.checkpoint, audience: AUD, revocation: &revocation }; + let dispatched = AtomicBool::new(false); + let handler = |_: &HttpRequest| { + dispatched.store(true, Ordering::SeqCst); + contextful_agent::http::HttpResponse::json(200, &json!({ "applied": 2 })) + }; + let face = HttpFace::new(&f.face, &clock, &f.audit, admitting, Some(2)).unwrap().with_control(&handler); + let request = HttpRequest { + method: "POST".into(), target: "/control/apply".into(), + headers: vec![("Authorization".into(), format!("Bearer {token}"))], body: b"{}".to_vec(), + }; + let answer = face.answer(&request); + assert_eq!(answer.status, 503); + assert!(!dispatched.load(Ordering::SeqCst)); + assert_eq!(checks.load(Ordering::SeqCst), 2); +} + fn call(tool: &str, arguments: Value) -> Value { json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": tool, "arguments": arguments } }) } From 4cf97a6b798e6caa07869c3dfffdf742f36fd0fd Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:15:37 +0800 Subject: [PATCH 026/155] fix(serve): recheck control authority and project workflow state --- crates/contextful-agent/src/http.rs | 8 ++++++++ crates/contextful-cli/src/cadence.rs | 20 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 365c2d705..254edb89d 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -375,6 +375,14 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { if !authority.grants().iter().any(|grant| grant.actions.contains(&Action::Admin) && grant.tables.contains(&TablePattern::All)) { return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlAdminGrantMissing" } })); } + let revocation = match (self.admitting.revocation)() { + Ok(r) => r, + Err(why) => return HttpResponse::unavailable(format!("the revocation denylist is unreadable: {why}")), + }; + let boundary = Admission::new(self.tools.clock().now(), &revocation).expecting(self.admitting.audience); + if let Err(error) = effect_boundary(&authority, &boundary) { + return unadmitted(&error); + } self.control.expect("the route is installed only with a control handler")(request) } diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index ba3605d5f..f119ede97 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -15,6 +15,7 @@ use crate::run::{boot_id, wire_at, ProjectArgs}; use anyhow::{bail, Context, Result}; use contextful_context::project::Project; use contextful_core::pipeline::declare::{collect, dependent_runs, ManifestFile, PipelineSpec}; +use contextful_core::pipeline::transform::TransformOp; use contextful_core::run::derive::task::Tasks; use contextful_core::surface::arm::{Schedule, Trigger, TICK_INTERVAL_MS, WAKE_ANSWER_SECS}; use contextful_core::surface::control::{admit_loopback, control_url, parse_pointer, poll_schedule, snapshot_file, source_file, POINTER_FILE}; @@ -231,12 +232,27 @@ fn applied(snaps: &Source) -> Result<(Option, BTreeMap Result { - let text = std::fs::read_to_string(declaration).unwrap_or_default(); + let text = std::fs::read_to_string(declaration)?; let control = control_config(&text, project)?; let (version, pipelines) = applied(&control.source)?; let store = contextful_context::Store::open(&project.dir, &project.name)?; let runs = contextful_sync::run_state::run_states(&store)?; - Ok(json!({ "applied": version, "pipelines": pipelines.into_values().collect::>(), "runs": runs })) + let pipelines: Vec = pipelines.into_values().map(|spec| json!({ + "id": spec.id, + "schedule": spec.schedule, + "tables": spec.tables.iter().map(|table| table.name()).collect::>(), + "source": spec.source.name, + "after": spec.after, + "steps": spec.transforms.iter().map(|step| match step { + TransformOp::Select { .. } => "select", + TransformOp::Rename { .. } => "rename", + TransformOp::Cast { .. } => "cast", + TransformOp::Filter { .. } => "filter", + TransformOp::Extract { .. } => "extract", + }).collect::>(), + })).collect(); + let runs: BTreeMap<_, _> = runs.into_iter().map(|(node, state)| (node, state.runs)).collect(); + Ok(json!({ "applied": version, "pipelines": pipelines, "runs": runs })) } /// The snapshot document: every specification as a `[[pipeline]]` block, sorted by id. From 556f811096fa97ecd4e38277eca6e8b5bc397899 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:29:04 +0800 Subject: [PATCH 027/155] test(serve): cap published workflow listing --- .../contextful-cli/tests/integration/serve.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index ee39f579d..6992abf53 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -150,6 +150,24 @@ fn control_routes_require_admin_and_project_applied_workflows() { assert_eq!(view["pipelines"][0]["schedule"], "every 1d"); } +#[test] +fn published_workflow_listing_caps_entries_and_flags_truncation() { + let (dir, public) = project(); + let root = dir.path(); + let snapshot = contextful_engine::control::SnapshotDir::open(&root.join(".contextful/control/research")); + let document = (0..1001).map(|index| format!( + "[[pipeline]]\nid = \"flow-{index:04}\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = {{ endpoint = \"https://example.test/filings\" }}\n" + )).collect::>().join("\n"); + snapshot.import(&document).unwrap(); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let (status, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); + assert_eq!(status, 200, "{view}"); + assert_eq!(view["pipelines"].as_array().unwrap().len(), 1000); + assert_eq!(view["truncated"], true); + assert_eq!(view["declined"], 0); +} + /// The network transport refuses to start without its audience, its ceiling, or an issuer key that resolves and parses. // spec: topology.publish-hostname.issuer-key@a7736a7f #[test] From dbfe5ad158d159a2d7669e73be62f5a164ade1e0 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:31:11 +0800 Subject: [PATCH 028/155] fix(serve): bound published workflow listing --- crates/contextful-cli/src/cadence.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index f119ede97..53d8352fd 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -29,7 +29,7 @@ use contextful_outbound::egress::{system, Outbound, Transport}; use serde::Serialize; use serde_json::json; use serde_json::Value; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::io::Read; @@ -235,9 +235,20 @@ pub(crate) fn published(project: &Project, declaration: &Path) -> Result let text = std::fs::read_to_string(declaration)?; let control = control_config(&text, project)?; let (version, pipelines) = applied(&control.source)?; + let truncated = pipelines.len() > 1000; + let pipelines: Vec = pipelines.into_values().take(1000).collect(); + let selected: BTreeSet<&str> = pipelines.iter().map(|spec| spec.id.as_str()).collect(); let store = contextful_context::Store::open(&project.dir, &project.name)?; - let runs = contextful_sync::run_state::run_states(&store)?; - let pipelines: Vec = pipelines.into_values().map(|spec| json!({ + let states = contextful_sync::run_state::run_states(&store)?; + let mut runs: BTreeMap = BTreeMap::new(); + for state in states.into_values() { + for (id, mark) in state.runs { + if selected.contains(id.as_str()) && runs.get(&id).is_none_or(|earlier| mark.started_at > earlier.started_at) { + runs.insert(id, mark); + } + } + } + let pipelines: Vec = pipelines.into_iter().map(|spec| json!({ "id": spec.id, "schedule": spec.schedule, "tables": spec.tables.iter().map(|table| table.name()).collect::>(), @@ -251,8 +262,7 @@ pub(crate) fn published(project: &Project, declaration: &Path) -> Result TransformOp::Extract { .. } => "extract", }).collect::>(), })).collect(); - let runs: BTreeMap<_, _> = runs.into_iter().map(|(node, state)| (node, state.runs)).collect(); - Ok(json!({ "applied": version, "pipelines": pipelines, "runs": runs })) + Ok(json!({ "applied": version, "pipelines": pipelines, "runs": runs, "truncated": truncated, "declined": 0 })) } /// The snapshot document: every specification as a `[[pipeline]]` block, sorted by id. From 907253d30a170e33205abfbccb9b1dccc10d034a Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:39:45 +0800 Subject: [PATCH 029/155] test(serve): show completed local workflow runs --- .../contextful-cli/tests/integration/serve.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 6992abf53..1df95d2c5 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -168,6 +168,24 @@ fn published_workflow_listing_caps_entries_and_flags_truncation() { assert_eq!(view["declined"], 0); } +#[test] +fn published_workflows_include_completed_local_runs_before_sync_push() { + let vendor = super::pipeline::Vendor::start(|_| (200, "[{\"note_id\":\"filed\"}]".into())); + let (dir, public) = project(); + let root = dir.path(); + let pipeline = format!("[[pipeline]]\nid = \"filings-flow\"\ntables = [\"research/filings\"]\n[pipeline.source]\nname = \"http\"\nconfig = {{ endpoint = \"{}\" }}\n", vendor.url("/filings")); + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), pipeline).unwrap(); + stdout(&run(root, &["pipeline", "import", "--project", "research"])); + stdout(&run(root, &["pipeline", "run", "filings-flow", "--project", "research", "--run-id", "run-flow", "--site-id", "site-a"])); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let (status, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); + assert_eq!(status, 200, "{view}"); + assert_eq!(view["runs"]["filings-flow"]["run_id"], "run-flow"); + assert_eq!(view["runs"]["filings-flow"]["status"], "success"); +} + /// The network transport refuses to start without its audience, its ceiling, or an issuer key that resolves and parses. // spec: topology.publish-hostname.issuer-key@a7736a7f #[test] From 79fd368fae88aa708f45db31f273b26d7ff71ae3 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:41:25 +0800 Subject: [PATCH 030/155] test(mcp): require a signed store owner credential --- .../contextful-cli/tests/integration/mcp.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 3ffa7a20e..efdc54b3f 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -114,6 +114,30 @@ fn a_server_with_no_credential_raises_stdio_credential_missing() { } } +#[test] +fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { + let (dir, public, ordinary) = project(); + let root = dir.path(); + let owner = stdout(&run(root, &[ + "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", + "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner-project", "research", + ])); + let args = ["mcp", "--owner", "--project", "research", "--public-key", &public, "--audience", AUD]; + let read = [json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } })]; + for token in [None, Some(ordinary.as_str())] { + let refused = serve(root, &args, token, &read); + assert!(!refused.status.success() && refused.stdout.is_empty(), "ordinary or absent credentials write no framing"); + } + let admitted = serve(root, &args, Some(&owner), &read); + assert!(admitted.status.success(), "{}", String::from_utf8_lossy(&admitted.stderr)); + let answer: Value = serde_json::from_str(String::from_utf8_lossy(&admitted.stdout).trim()).unwrap(); + assert_eq!(answer["result"]["structuredContent"]["rows"], json!([["n1"]])); + + let other = ["mcp", "--owner", "--project", "other", "--public-key", &public, "--audience", AUD]; + let refused = serve(root, &other, Some(&owner), &read); + assert!(!refused.status.success() && refused.stdout.is_empty(), "an owner credential selects one store"); +} + /// The server admits over the stdio pipe it inherited: a credential binding no key admits /// through that pipe, and one binding a holder key and presenting no holder proof refuses. #[test] From aa77ed0471d6632c3511f86373391da9137d909f Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:44:29 +0800 Subject: [PATCH 031/155] test(serve): apply with registered host tasks --- .../tests/integration/derive.rs | 4 ++-- .../contextful-cli/tests/integration/serve.rs | 22 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/contextful-cli/tests/integration/derive.rs b/crates/contextful-cli/tests/integration/derive.rs index db5f63642..a77305b84 100644 --- a/crates/contextful-cli/tests/integration/derive.rs +++ b/crates/contextful-cli/tests/integration/derive.rs @@ -29,7 +29,7 @@ fn the_verb_refuses_a_fetch_engine_and_runs_a_transcriber_over_one_file() { /// The embedding binary under `examples/host_derive.rs`, built on first request: the command /// line with the compiled `word-split` task registered. -fn host_binary() -> std::path::PathBuf { +pub(crate) fn host_binary() -> std::path::PathBuf { static BUILT: std::sync::Once = std::sync::Once::new(); BUILT.call_once(|| { let status = Command::new(env!("CARGO")).args(["build", "-q", "-p", "contextful-cli", "--example", "host_derive"]).status().unwrap(); @@ -49,7 +49,7 @@ fn ok(out: &Output) -> String { /// A `word-split` pipeline over `documents`: `words` retains versions, `stats` does not, and /// `units` holds the markers. -fn host_manifest(task: &str) -> String { +pub(crate) fn host_manifest(task: &str) -> String { format!( "[[pipeline]]\nid = \"split\"\ntables = [\n \ {{ name = \"words\", primary_key = [\"unit_ref\", \"derivation_key\", \"word_seq\"], retain_versions = true }},\n \ diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 1df95d2c5..6a73257b6 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -59,7 +59,11 @@ impl Drop for Listener { /// Start the face; its address, reported on standard error. fn serve(dir: &Path, args: &[&str]) -> (Listener, String) { - let mut child = Command::new(env!("CARGO_BIN_EXE_contextful")) + serve_binary(Path::new(env!("CARGO_BIN_EXE_contextful")), dir, args) +} + +fn serve_binary(binary: &Path, dir: &Path, args: &[&str]) -> (Listener, String) { + let mut child = Command::new(binary) .args(args) .current_dir(dir) .env_remove("CONTEXTFUL_ISSUER_PUBKEY") @@ -80,6 +84,22 @@ fn serve(dir: &Path, args: &[&str]) -> (Listener, String) { panic!("the face never listened") } +#[test] +fn control_apply_uses_the_host_registered_task_set() { + let binary = super::derive::host_binary(); + let (dir, public) = project(); + let root = dir.path(); + std::fs::write(root.join("contextful.toml"), format!("authoring_posture = \"per_request\"\n{}", super::derive::host_manifest("word-split"))).unwrap(); + let imported = Command::new(&binary).args(["pipeline", "import", "--project", "research"]).current_dir(root).output().unwrap(); + stdout(&imported); + std::fs::write(root.join("contextful.toml"), format!("authoring_posture = \"per_request\"\n{}", super::derive::host_manifest("word-split").replace("[pipeline.source]", "schedule = \"every 1h\"\n[pipeline.source]"))).unwrap(); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve_binary(&binary, root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let (status, state) = control(&addr, "POST", "/control/apply", Some(&admin), "{}"); + assert_eq!(status, 200, "{state}"); + assert_eq!(state["applied"], json!(2)); +} + static NONCE: AtomicU64 = AtomicU64::new(0); /// `POST /mcp` carrying `message`, with a proof from `key` or as a bare bearer. From e462696c09c38f452cd17007541e61b41f99b780 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:44:59 +0800 Subject: [PATCH 032/155] test(mcp): discover the signed owner store from the manifest --- crates/contextful-cli/tests/integration/mcp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index efdc54b3f..2d6a17519 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -120,7 +120,7 @@ fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { let root = dir.path(); let owner = stdout(&run(root, &[ "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", - "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner-project", "research", + "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner", ])); let args = ["mcp", "--owner", "--project", "research", "--public-key", &public, "--audience", AUD]; let read = [json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } })]; From ca48f5ad3ac8268213e1c35711c89aad65210a8b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:45:53 +0800 Subject: [PATCH 033/155] test(mcp): name the project selected by owner mint --- crates/contextful-cli/tests/integration/mcp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 2d6a17519..de360a875 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -23,7 +23,7 @@ fn project() -> (tempfile::TempDir, String, String) { let p = dir.path(); std::fs::create_dir_all(p.join(".contextful")).unwrap(); std::fs::write(p.join(".contextful/issuance.toml"), format!("default_audience = \"{AUD}\"\nmax_lifetime_secs = 3600\n")).unwrap(); - std::fs::write(p.join("contextful.toml"), "authoring_posture = \"per_request\"\n[[pipeline.tables]]\nname = \"research/notes\"\n").unwrap(); + std::fs::write(p.join("contextful.toml"), "authoring_posture = \"per_request\"\n[project]\nname = \"research\"\n[[pipeline.tables]]\nname = \"research/notes\"\n").unwrap(); std::fs::write(p.join("notes.jsonl"), "{\"note_id\":\"n1\",\"title\":\"Solar battery storage\"}\n").unwrap(); stdout(&run(p, &["context", "land", "research/notes", "--project", "research", "--rows", "notes.jsonl", "--run-id", "run-0001", "--site-id", "site-a"])); let public = stdout(&run(p, &["token", "keygen", "--out", ".contextful/issuer.seed"])); From 72f80681c00d36289d6ba7025568bc0e08d4fd9a Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:46:27 +0800 Subject: [PATCH 034/155] test(mcp): refuse owner credentials at a second existing store --- crates/contextful-cli/tests/integration/mcp.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index de360a875..55631d563 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -133,6 +133,7 @@ fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { let answer: Value = serde_json::from_str(String::from_utf8_lossy(&admitted.stdout).trim()).unwrap(); assert_eq!(answer["result"]["structuredContent"]["rows"], json!([["n1"]])); + stdout(&run(root, &["context", "land", "research/notes", "--project", "other", "--rows", "notes.jsonl", "--run-id", "run-other", "--site-id", "site-a"])); let other = ["mcp", "--owner", "--project", "other", "--public-key", &public, "--audience", AUD]; let refused = serve(root, &other, Some(&owner), &read); assert!(!refused.status.success() && refused.stdout.is_empty(), "an owner credential selects one store"); From a34359fcc1b7a4f17ba1496e0c0d9cfb75ec8e82 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:48:35 +0800 Subject: [PATCH 035/155] fix(serve): show local runs and use registered apply tasks --- crates/contextful-cli/src/cadence.rs | 12 ++++++++++++ crates/contextful-cli/src/lib.rs | 2 +- crates/contextful-cli/src/serve.rs | 5 +++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index 53d8352fd..07d712aef 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -248,6 +248,18 @@ pub(crate) fn published(project: &Project, declaration: &Path) -> Result } } } + let wired = wire_at(project, &None)?; + for row in wired.engine.catalog.runs(None)? { + if selected.contains(row.pipeline_id.as_str()) && runs.get(&row.pipeline_id).is_none_or(|earlier| row.started_at > earlier.started_at) { + runs.insert(row.pipeline_id.clone(), contextful_sync::RunMark { + run_id: row.run_id, + table: row.table, + status: row.status, + started_at: row.started_at, + ended_at: row.ended_at, + }); + } + } let pipelines: Vec = pipelines.into_iter().map(|spec| json!({ "id": spec.id, "schedule": spec.schedule, diff --git a/crates/contextful-cli/src/lib.rs b/crates/contextful-cli/src/lib.rs index a4984d4a7..db744db41 100644 --- a/crates/contextful-cli/src/lib.rs +++ b/crates/contextful-cli/src/lib.rs @@ -242,7 +242,7 @@ pub fn main_host(host: Host) { #[cfg(feature = "read-plane")] Cmd::Mcp(c) => mcp::run(c), #[cfg(feature = "read-plane")] - Cmd::Serve(c) => serve::run(c), + Cmd::Serve(c) => serve::run(c, &tasks), #[cfg(feature = "data-plane")] Cmd::Derive(c) => derive::run(c), #[cfg(feature = "read-plane")] diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index ecfe191ad..b9bc5efa8 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -14,6 +14,7 @@ use crate::clock::SystemClock; use anyhow::Result; use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, WORKFLOWS_PATH}; use contextful_core::surface::SurfaceError; +use contextful_core::run::derive::task::Tasks; use contextful_policy::audit::AuditLog; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; @@ -75,7 +76,7 @@ fn issuer_pins(flag: Option<&str>) -> Result { StaticPins::parse(text).map_err(|e| ServeError::IssuerKeyUnusable(format!("--public-key / {PUBKEY_VAR}: {e}"))) } -pub fn run(args: ServeArgs) -> Result<()> { +pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { let clock = SystemClock; // The declarations are checked before anything opens (`read.register.serve-declaration`). let audience = audience(args.audience.as_deref()).map_err(anyhow::Error::msg)?; @@ -123,7 +124,7 @@ pub fn run(args: ServeArgs) -> Result<()> { return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); } let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; - crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), &Default::default()) + crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), tasks) .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) } _ => unreachable!(), From 64d32262078f1b87468811c602a5171ec5d93486 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:49:25 +0800 Subject: [PATCH 036/155] test(console): require rendered, learned, briefed, private answers --- apps/console/test/brief.test.ts | 37 +++++++++++++++++++++++++++++++ apps/console/test/learn.test.ts | 32 ++++++++++++++++++++++++++ apps/console/test/publish.test.ts | 19 ++++++++++++++++ apps/console/test/render.test.ts | 29 ++++++++++++++++++++++++ 4 files changed, 117 insertions(+) create mode 100644 apps/console/test/brief.test.ts create mode 100644 apps/console/test/learn.test.ts create mode 100644 apps/console/test/publish.test.ts create mode 100644 apps/console/test/render.test.ts diff --git a/apps/console/test/brief.test.ts b/apps/console/test/brief.test.ts new file mode 100644 index 000000000..d39eb9a2e --- /dev/null +++ b/apps/console/test/brief.test.ts @@ -0,0 +1,37 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { deriveBrief, topicMatch } from "../src/brief.ts"; + +const now = Date.parse("2026-01-08T12:00:00Z"); +const conclusion = { subject: "Acme", text: "Acme filings need review", live: true }; +const row = { id: "a", label: "Acme filing", topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }; + +test("topic tier needs two shared tokens including the subject", () => { + assert.equal(topicMatch(conclusion, row), true); + assert.equal(topicMatch(conclusion, { ...row, label: "filings review", topics: [] }), false); + assert.equal(topicMatch(conclusion, { ...row, label: "Acme", topics: [] }), false); +}); + +test("brief requires a turnless present-time session and live matched arrivals", async () => { + const input = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], arrivals: [row], now }; + assert.equal((await deriveBrief(input))?.subjects[0].subject, "Acme"); + assert.equal(await deriveBrief({ ...input, session: { turns: 1, vantage: "present" } }), null); + assert.equal(await deriveBrief({ ...input, session: { turns: 0, vantage: "2026-01-01" } }), null); + assert.equal(await deriveBrief({ ...input, conclusions: [{ ...conclusion, live: false }] }), null); + assert.equal(await deriveBrief({ ...input, arrivals: [] }), null); +}); + +test("brief caps window at seven days, subjects at three, and articles at three", async () => { + const conclusions = Array.from({ length: 5 }, (_, n) => ({ subject: `Acme${n}`, text: `Acme${n} filings need review`, live: true })); + const arrivals = conclusions.flatMap((c, n) => Array.from({ length: 5 }, (_, m) => ({ id: `${n}-${m}`, label: `${c.subject} filing`, topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }))); + const card = await deriveBrief({ session: { turns: 0, vantage: "present" }, conclusions, arrivals, now, windowDays: 100 }); + assert.equal(card?.windowDays, 7); + assert.equal(card?.subjects.length, 3); + assert.ok(card?.subjects.every((subject) => subject.articles.length === 3)); +}); + +test("brief reports derivation failure and timeout without a card", async () => { + const base = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], now }; + await assert.rejects(deriveBrief({ ...base, loadArrivals: async () => { throw new Error("unavailable"); } }), /ConsoleBriefUnavailable/); + await assert.rejects(deriveBrief({ ...base, loadArrivals: () => new Promise(() => {}), budgetMs: 5 }), /ConsoleBriefUnavailable/); +}); diff --git a/apps/console/test/learn.test.ts b/apps/console/test/learn.test.ts new file mode 100644 index 000000000..49c87f226 --- /dev/null +++ b/apps/console/test/learn.test.ts @@ -0,0 +1,32 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { learnAfterAnswer } from "../src/learn.ts"; + +test("distillation runs after streaming and lands at most three scoped conclusions", async () => { + const events: string[] = []; + const stream = async function* () { events.push("stream"); yield "One"; yield " answer"; }; + const entries = Array.from({ length: 5 }, (_, n) => ({ subject: `Acme ${n}`, key: `k${n}`, learning: `L${n}` })); + const answer = await learnAfterAnswer({ scope: "session-1", question: "What changed?", stream: stream(), + distill: async ({ answer }) => { events.push(`distill:${answer}`); return entries; }, + land: async (scope, entry) => { events.push(`land:${scope}:${entry.subject}`); }, + }); + assert.equal(answer, "One answer"); + assert.deepEqual(events, ["stream", "distill:One answer", "land:session-1:Acme 0", "land:session-1:Acme 1", "land:session-1:Acme 2"]); +}); + +test("unscoped learning refuses before distillation or landing", async () => { + let called = false; + await assert.rejects(learnAfterAnswer({ scope: "", question: "q", stream: (async function* () { yield "a"; })(), + distill: async () => { called = true; return []; }, land: async () => { called = true; }, + }), /ConsoleLearningUnscoped/); + assert.equal(called, false); +}); + +test("distillation preserves observed subject and permits zero entries", async () => { + const landed: unknown[] = []; + const base = { scope: "s", question: "q", stream: (async function* () { yield "a"; })(), land: async (_scope: string, entry: unknown) => { landed.push(entry); } }; + await learnAfterAnswer({ ...base, distill: async () => [{ subject: "Acme Incorporated", key: "priority", learning: "raised" }] }); + assert.deepEqual(landed, [{ subject: "Acme Incorporated", key: "priority", learning: "raised" }]); + await learnAfterAnswer({ ...base, stream: (async function* () { yield "b"; })(), distill: async () => [] }); + assert.equal(landed.length, 1); +}); diff --git a/apps/console/test/publish.test.ts b/apps/console/test/publish.test.ts new file mode 100644 index 000000000..78837456d --- /dev/null +++ b/apps/console/test/publish.test.ts @@ -0,0 +1,19 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { answerDelivery, scheduledAudience } from "../src/publish.ts"; + +test("a normal answer is delivered only to its asking operator", () => { + assert.deepEqual(answerDelivery({ operator: "alice", answer: "grounded", accessExplanation: false }), { recipient: "alice", answer: "grounded", share: false }); +}); + +test("an access explanation cannot offer a share control", () => { + assert.throws(() => answerDelivery({ operator: "alice", answer: "why", accessExplanation: true, share: true }), /VisibilityShareAffordance/); +}); + +test("a scheduled service post refuses an audience and names its destination", () => { + assert.throws(() => scheduledAudience({ identity: "service", destination: "#team", corpus: ["private"] }), /VisibilityAskerlessAudience.*#team/); +}); + +test("a human scheduled post carries only reviewed destination corpus", () => { + assert.deepEqual(scheduledAudience({ identity: "operator", operator: "alice", destination: "team", corpus: ["shared", "private"], reviewedCorpus: ["shared"] }), { destination: "team", corpus: ["shared"] }); +}); diff --git a/apps/console/test/render.test.ts b/apps/console/test/render.test.ts new file mode 100644 index 000000000..732d3f166 --- /dev/null +++ b/apps/console/test/render.test.ts @@ -0,0 +1,29 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { buildView, readView, splitResult } from "../src/render.ts"; + +test("server result shape chooses metric, line, or table with a bar option", () => { + assert.equal(buildView({ columns: ["revenue"], rows: [[42]] }).component, "metric.v1"); + assert.equal(buildView({ columns: ["day", "revenue"], rows: [["2026-01-01", 1], ["2026-01-02", 2], ["2026-01-03", 3]] }).component, "line.v1"); + assert.deepEqual(buildView({ columns: ["day", "revenue"], rows: [["2026-01-01", 1], ["2026-01-01", 2], ["2026-01-03", 3]] }).alternates, ["bar.v1"]); + assert.equal(buildView({ columns: ["name", "value"], rows: [["A", 1]] }).component, "table.v1"); +}); + +test("client and model view specifications never become widgets", () => { + const result = { columns: ["amount"], rows: [[7]] }; + for (const origin of ["client", "model"] as const) { + assert.throws(() => buildView(result, { origin, view: { component: "metric.v1" } }), /ConsoleViewNotServerBuilt/); + } + assert.equal(buildView(result, { origin: "server", hint: { component: "bar.v1", columns: ["missing"] } }).component, "metric.v1"); +}); + +test("older transcripts render an unknown component as a table", () => { + assert.equal(readView({ component: "map.v3", props: { columns: ["x"], rows: [[1]] } }).component, "table.v1"); +}); + +test("tool return channels keep trace out of model grounding and view rows", () => { + const channels = splitResult({ rows: [[1]], columns: ["count"], sources: [{ id: "s" }], trace: { query: "select secret" } }); + assert.deepEqual(channels.grounding, { rows: [[1]], columns: ["count"], sources: [{ id: "s" }] }); + assert.deepEqual(channels.trace, { query: "select secret" }); + assert.equal(channels.view.component, "metric.v1"); +}); From 3340e7c97398629f0daaab46210260eae184d433 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:52:06 +0800 Subject: [PATCH 037/155] test(owner): cover profile version and typed refusals --- .../contextful-cli/tests/integration/mcp.rs | 3 +++ .../tests/integration/profile.rs | 27 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 55631d563..05f1282c1 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -127,6 +127,8 @@ fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { for token in [None, Some(ordinary.as_str())] { let refused = serve(root, &args, token, &read); assert!(!refused.status.success() && refused.stdout.is_empty(), "ordinary or absent credentials write no framing"); + let error = String::from_utf8_lossy(&refused.stderr); + assert!(error.contains(if token.is_none() { "StdioCredentialMissing" } else { "OwnerCredentialInvalid" }), "{error}"); } let admitted = serve(root, &args, Some(&owner), &read); assert!(admitted.status.success(), "{}", String::from_utf8_lossy(&admitted.stderr)); @@ -137,6 +139,7 @@ fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { let other = ["mcp", "--owner", "--project", "other", "--public-key", &public, "--audience", AUD]; let refused = serve(root, &other, Some(&owner), &read); assert!(!refused.status.success() && refused.stdout.is_empty(), "an owner credential selects one store"); + assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); } /// The server admits over the stdio pipe it inherited: a credential binding no key admits diff --git a/crates/contextful-policy/tests/integration/profile.rs b/crates/contextful-policy/tests/integration/profile.rs index 9c7ce97a6..35d10bfec 100644 --- a/crates/contextful-policy/tests/integration/profile.rs +++ b/crates/contextful-policy/tests/integration/profile.rs @@ -6,7 +6,22 @@ use contextful_core::grant::{Action, AggregateGrant}; use contextful_core::issue::SignatureAlgorithm; use contextful_policy::attenuate::{attenuate, Derivation}; use contextful_policy::issue::{mint, MintClaims}; -use contextful_policy::profile::{authority_facts, ENGINE_FACTS, EVALUATOR_FACT_CEILING, PROFILE_VERSION, SUPPORTED_PROFILE_VERSIONS}; +use contextful_policy::profile::{authority_facts, ENGINE_FACTS, EVALUATOR_FACT_CEILING, OWNER_PROFILE_VERSION, PROFILE_VERSION, SUPPORTED_PROFILE_VERSIONS}; + +#[test] +fn a_signed_owner_claim_uses_profile_two_and_cannot_be_promoted_from_profile_one_or_a_child() { + let signer = issuer(); + let claim = "one-local-store"; + let token = mint(&plan_for(&signer, dana(), vec![grant(&[Action::Read], &["*"])]), + &MintClaims { owner_project: Some(claim.into()), ..MintClaims::default() }, &signer).unwrap(); + assert_eq!(introspect(&token).unwrap().profile, OWNER_PROFILE_VERSION); + assert!(admit(&token, &signer, DURING).unwrap().activate_owner(claim).is_some()); + assert!(admit(&token, &signer, DURING).unwrap().activate_owner("second-store").is_none()); + let child = attenuate(&token, &Derivation::default()).unwrap(); + assert!(admit(&child, &signer, DURING).unwrap().activate_owner(claim).is_none()); + let first_profile = craft(&signer, &block(&signer), &[], "owner_project(\"one-local-store\");"); + refused(admit(&first_profile, &signer, DURING), "ProfileElementUnrecognized"); +} use contextful_policy::revoke::RevocationState; use contextful_policy::verify::{introspect, verify_inherited_pipe, Admission}; @@ -234,15 +249,15 @@ fn a_statement_over_two_tables_needs_one_grant_covering_both() { // spec: authority.profile.version-unsupported@0eb5690f #[test] fn a_profile_version_outside_the_supported_set_is_refused() { - assert_eq!(SUPPORTED_PROFILE_VERSIONS, &[PROFILE_VERSION]); + assert_eq!(SUPPORTED_PROFILE_VERSIONS, &[PROFILE_VERSION, OWNER_PROFILE_VERSION]); let signer = issuer(); let b = block(&signer); - let v2 = craft(&signer, &b, &["profile"], "profile(2);"); - refused(admit(&v2, &signer, DURING), "ProfileVersionUnsupported"); + let v3 = craft(&signer, &b, &["profile"], "profile(3);"); + refused(admit(&v3, &signer, DURING), "ProfileVersionUnsupported"); let none = craft(&signer, &b, &["profile"], ""); refused(admit(&none, &signer, DURING), "ProfileVersionUnsupported"); // A checkpoint whose supported set names the version admits it. let revocation = RevocationState::default(); - let widened = Admission { profiles: &[1, 2], ..Admission::new(at(DURING), &revocation) }; - assert_eq!(verify_inherited_pipe(&v2, &keys(&signer), &widened).unwrap().profile(), 2); + let widened = Admission { profiles: &[1, 3], ..Admission::new(at(DURING), &revocation) }; + assert_eq!(verify_inherited_pipe(&v3, &keys(&signer), &widened).unwrap().profile(), 3); } From d8b18df843907648c53541e71d5610d5461f2279 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:52:56 +0800 Subject: [PATCH 038/155] test(console): bound streamed bodies and validate Admin store --- apps/console/test/console.test.ts | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 514c30c9b..26da13e41 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -243,6 +243,42 @@ test("the Node HTTP listener rejects an oversized body before dispatch", async ( } finally { server.close(); } }); +test("Query counts UTF-8 bytes without Content-Length before parsing", async () => { + const { request, calls } = fixture(); + const body = JSON.stringify({ store: "field-notes", question: "é".repeat(524_289) }); + const response = await request("/query/api/ask", { ...access(queryAudience), origin: "https://console.example" }, { method: "POST", body }); + assert.equal(response.status, 413); + assert.equal((await response.json() as { error: { identifier: string } }).error.identifier, "ConsoleBodyTooLarge"); + assert.deepEqual(calls, []); +}); + +test("Query stops reading a chunked body when its byte cap is crossed", async () => { + const { request, calls } = fixture(); + let pulls = 0; + const stream = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(Buffer.alloc(524_289, 120)); + if (pulls === 4) controller.close(); + }, + }, { highWaterMark: 0 }); + const response = await request("/query/api/ask", { ...access(queryAudience), origin: "https://console.example" }, + { method: "POST", body: stream, duplex: "half" } as RequestInit); + assert.equal(response.status, 413); + assert.ok(pulls < 4, `read ${pulls} chunks past the cap`); + assert.deepEqual(calls, []); +}); + +test("Admin rejects an unknown store before workflow and record adapters", async () => { + const { request, calls } = fixture(); + for (const path of ["/admin/api/workflows", "/admin/api/record"]) { + const response = await request(`${path}?store=missing`, access(adminAudience)); + assert.equal(response.status, 400); + assert.equal((await response.json() as { error: { identifier: string } }).error.identifier, "ConsoleRequestMalformed"); + } + assert.deepEqual(calls, []); +}); + test("Admin pack listing reaches only the injected listing adapter", async () => { const calls: string[] = []; const app = createConsole({ From bca2863e395f636363641fd7dd6c3003ce469063 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:53:58 +0800 Subject: [PATCH 039/155] test(owner): reject restricted signed owner claims --- crates/contextful-policy/tests/integration/profile.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/contextful-policy/tests/integration/profile.rs b/crates/contextful-policy/tests/integration/profile.rs index 35d10bfec..c18107326 100644 --- a/crates/contextful-policy/tests/integration/profile.rs +++ b/crates/contextful-policy/tests/integration/profile.rs @@ -19,6 +19,9 @@ fn a_signed_owner_claim_uses_profile_two_and_cannot_be_promoted_from_profile_one assert!(admit(&token, &signer, DURING).unwrap().activate_owner("second-store").is_none()); let child = attenuate(&token, &Derivation::default()).unwrap(); assert!(admit(&child, &signer, DURING).unwrap().activate_owner(claim).is_none()); + let restricted = mint(&plan_for(&signer, dana(), vec![grant(&[Action::Read], &["research/*"])]), + &MintClaims { owner_project: Some(claim.into()), ..MintClaims::default() }, &signer).unwrap(); + assert!(admit(&restricted, &signer, DURING).unwrap().activate_owner(claim).is_none()); let first_profile = craft(&signer, &block(&signer), &[], "owner_project(\"one-local-store\");"); refused(admit(&first_profile, &signer, DURING), "ProfileElementUnrecognized"); } From ff9f13fb1fa9652224c3901d462152637438b7d7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:54:23 +0800 Subject: [PATCH 040/155] fix(console): enforce byte cap while reading requests --- apps/console/dist/server.js | 38 +++++++++++++++++++++++++++++-------- apps/console/src/index.ts | 35 ++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 221f1910f..749033e5d 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -232,10 +232,31 @@ function json(value) { return Response.json(value, { headers: { "Cache-Control": "no-store" } }); } async function body(request) { - if (Number(request.headers.get("content-length") ?? 0) > 1048576) throw new Error("body too large"); - const text = await request.text(); - if (text.length > 1048576) throw new Error("body too large"); - return JSON.parse(text); + const limit = 1048576; + if (Number(request.headers.get("content-length") ?? 0) > limit) throw new Error("ConsoleBodyTooLarge"); + const chunks = []; + let size = 0; + const reader = request.body?.getReader(); + if (!reader) return JSON.parse(""); + try { + for (; ; ) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + void reader.cancel().catch(() => { + }); + throw new Error("ConsoleBodyTooLarge"); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return JSON.parse(Buffer.concat(chunks, size).toString("utf8")); +} +function bodyFailure(error) { + return error instanceof Error && error.message === "ConsoleBodyTooLarge" ? refusal("ConsoleBodyTooLarge", 413) : refusal("ConsoleRequestMalformed", 400); } function page(name) { return new Response(name === "query" ? queryPage : adminPage, { @@ -276,14 +297,15 @@ function createConsole(adapters) { let input; try { input = await body(request); - } catch { - return refusal("ConsoleRequestMalformed", 400); + } catch (error) { + return bodyFailure(error); } if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); return json(await adapters.turn({ operator, store: input.store, question: input.question })); } } else { const store = url.searchParams.get("store"); + if (store !== null && !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); if (request.method === "GET" && path === "/admin/api/workflows") return json(await adapters.control.workflows(operator, store)); if (request.method === "GET" && path === "/admin/api/record") return json(await adapters.control.record(operator, store)); if (request.method === "GET" && path === "/admin/api/packs") { @@ -298,8 +320,8 @@ function createConsole(adapters) { let document; try { document = await body(request); - } catch { - return refusal("ConsoleRequestMalformed", 400); + } catch (error) { + return bodyFailure(error); } return json(path.endsWith("/edit") ? await adapters.control.edit(document, adapters.adminCapability, operator) : await adapters.control.apply(document, adapters.adminCapability, operator)); } diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 4b1ece282..beab27127 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -198,10 +198,32 @@ function json(value: unknown): Response { } async function body(request: Request): Promise { - if (Number(request.headers.get("content-length") ?? 0) > 1_048_576) throw new Error("body too large"); - const text = await request.text(); - if (text.length > 1_048_576) throw new Error("body too large"); - return JSON.parse(text); + const limit = 1_048_576; + if (Number(request.headers.get("content-length") ?? 0) > limit) throw new Error("ConsoleBodyTooLarge"); + const chunks: Buffer[] = []; + let size = 0; + const reader = request.body?.getReader(); + if (!reader) return JSON.parse(""); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + void reader.cancel().catch(() => {}); + throw new Error("ConsoleBodyTooLarge"); + } + chunks.push(Buffer.from(value)); + } + } finally { + reader.releaseLock(); + } + return JSON.parse(Buffer.concat(chunks, size).toString("utf8")); +} + +function bodyFailure(error: unknown): Response { + return error instanceof Error && error.message === "ConsoleBodyTooLarge" ? + refusal("ConsoleBodyTooLarge", 413) : refusal("ConsoleRequestMalformed", 400); } function page(name: "query" | "admin"): Response { @@ -239,13 +261,14 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req if (request.method === "GET" && path === "/query/api/stores") return json(await adapters.read.list(operator)); if (request.method === "POST" && path === "/query/api/ask") { let input: unknown; - try { input = await body(request); } catch { return refusal("ConsoleRequestMalformed", 400); } + try { input = await body(request); } catch (error) { return bodyFailure(error); } if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); return json(await adapters.turn({ operator, store: input.store, question: input.question })); } } else { const store = url.searchParams.get("store"); + if (store !== null && !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); if (request.method === "GET" && path === "/admin/api/workflows") return json(await adapters.control.workflows(operator, store)); if (request.method === "GET" && path === "/admin/api/record") return json(await adapters.control.record(operator, store)); if (request.method === "GET" && path === "/admin/api/packs") { @@ -258,7 +281,7 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req const origin = request.headers.get("origin"); if (origin && origin !== url.origin) return refusal("ConsolePageForbidden"); let document: unknown; - try { document = await body(request); } catch { return refusal("ConsoleRequestMalformed", 400); } + try { document = await body(request); } catch (error) { return bodyFailure(error); } return json(path.endsWith("/edit") ? await adapters.control.edit(document, adapters.adminCapability, operator) : await adapters.control.apply(document, adapters.adminCapability, operator)); From d9dcd85ffc589589ec7604f880776c201476768e Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:55:49 +0800 Subject: [PATCH 041/155] test(console): sanitize widgets and match learning subjects --- apps/console/test/learn.test.ts | 13 ++++++++++++- apps/console/test/render.test.ts | 9 ++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/apps/console/test/learn.test.ts b/apps/console/test/learn.test.ts index 49c87f226..47a5c0713 100644 --- a/apps/console/test/learn.test.ts +++ b/apps/console/test/learn.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { learnAfterAnswer } from "../src/learn.ts"; +import { learnAfterAnswer, recallSubjects } from "../src/learn.ts"; test("distillation runs after streaming and lands at most three scoped conclusions", async () => { const events: string[] = []; @@ -30,3 +30,14 @@ test("distillation preserves observed subject and permits zero entries", async ( await learnAfterAnswer({ ...base, stream: (async function* () { yield "b"; })(), distill: async () => [] }); assert.equal(landed.length, 1); }); + +test("recall resolves preserved subjects through entity matching", () => { + const entries = [{ subject: "Acme Incorporated", key: "priority", learning: "raised" }, { subject: "Beta", key: "priority", learning: "steady" }]; + const matches: string[] = []; + const recalled = recallSubjects(entries, "Acme's priorities", (question, subject) => { + matches.push(`${question}:${subject}`); + return subject === "Acme Incorporated"; + }); + assert.deepEqual(recalled, [entries[0]]); + assert.deepEqual(matches, ["Acme's priorities:Acme Incorporated", "Acme's priorities:Beta"]); +}); diff --git a/apps/console/test/render.test.ts b/apps/console/test/render.test.ts index 732d3f166..45cc613fc 100644 --- a/apps/console/test/render.test.ts +++ b/apps/console/test/render.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { buildView, readView, splitResult } from "../src/render.ts"; +import { buildView, readView, sanitizeView, splitResult } from "../src/render.ts"; test("server result shape chooses metric, line, or table with a bar option", () => { assert.equal(buildView({ columns: ["revenue"], rows: [[42]] }).component, "metric.v1"); @@ -27,3 +27,10 @@ test("tool return channels keep trace out of model grounding and view rows", () assert.deepEqual(channels.trace, { query: "select secret" }); assert.equal(channels.view.component, "metric.v1"); }); + +test("view props pass through a sanitizer walk and invalid shapes produce no widget", () => { + const view = buildView({ columns: ["customer"], rows: [["secret@example.test"]] }); + const clean = sanitizeView(view, (value) => value.replace("secret@example.test", "[redacted]")); + assert.deepEqual(clean?.props.rows, [["[redacted]"]]); + assert.equal(sanitizeView({ component: "table.v1", props: { columns: ["x"], rows: [[1, 2]] } }, (value) => value), null); +}); From c40d705ffc9cda0bb15b99b64bfdae01d18042bb Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:56:42 +0800 Subject: [PATCH 042/155] feat(console): build answer views, learnings, briefs, and delivery guards --- apps/console/src/brief.ts | 51 ++++++++++++++++++++++++ apps/console/src/learn.ts | 23 +++++++++++ apps/console/src/publish.ts | 21 ++++++++++ apps/console/src/render.ts | 79 +++++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+) create mode 100644 apps/console/src/brief.ts create mode 100644 apps/console/src/learn.ts create mode 100644 apps/console/src/publish.ts create mode 100644 apps/console/src/render.ts diff --git a/apps/console/src/brief.ts b/apps/console/src/brief.ts new file mode 100644 index 000000000..8cca11086 --- /dev/null +++ b/apps/console/src/brief.ts @@ -0,0 +1,51 @@ +export type Conclusion = { subject: string; text: string; live: boolean }; +export type Arrival = { id: string; label: string; topics: string[]; arrivedAt: string }; +export type Brief = { windowDays: number; subjects: Array<{ subject: string; articles: Arrival[] }> }; + +function tokens(text: string): Set { + return new Set((text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).map((word) => word.length > 4 && word.endsWith("s") ? word.slice(0, -1) : word)); +} + +export function topicMatch(conclusion: Conclusion, row: Arrival): boolean { + const subject = tokens(conclusion.subject); + const left = tokens(`${conclusion.subject} ${conclusion.text}`); + const right = tokens(`${row.label} ${row.topics.join(" ")}`); + const shared = [...left].filter((word) => right.has(word)); + return shared.length >= 2 && shared.some((word) => subject.has(word)); +} + +export async function deriveBrief(input: { + session: { turns: number; vantage: string }; + conclusions: Conclusion[]; + arrivals?: Arrival[]; + loadArrivals?: () => Promise; + now: number; + windowDays?: number; + budgetMs?: number; +}): Promise { + if (input.session.turns !== 0 || input.session.vantage !== "present") return null; + const live = input.conclusions.filter((entry) => entry.live); + if (live.length === 0) return null; + const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); + let timer: ReturnType | undefined; + try { + const source = input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? []); + const budgetMs = input.budgetMs; + const arrivals = budgetMs === undefined ? await source : await Promise.race([ + source, + new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), Math.max(0, budgetMs)); }), + ]); + const since = input.now - windowDays * 86_400_000; + const recent = arrivals.filter((row) => { + const arrived = Date.parse(row.arrivedAt); + return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + }); + const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => topicMatch(entry, row)).slice(0, 3) })) + .filter((entry) => entry.articles.length > 0).slice(0, 3); + return subjects.length ? { windowDays, subjects } : null; + } catch { + throw new Error("ConsoleBriefUnavailable"); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} diff --git a/apps/console/src/learn.ts b/apps/console/src/learn.ts new file mode 100644 index 000000000..4de824693 --- /dev/null +++ b/apps/console/src/learn.ts @@ -0,0 +1,23 @@ +export type Learning = { subject: string; key: string; learning: string }; + +export function recallSubjects(entries: Learning[], question: string, matchEntity: (question: string, subject: string) => boolean): Learning[] { + return entries.filter((entry) => matchEntity(question, entry.subject)); +} + +export async function learnAfterAnswer(input: { + scope: string; + question: string; + stream: AsyncIterable; + distill: (exchange: { question: string; answer: string }) => Promise; + land: (scope: string, entry: Learning) => Promise; +}): Promise { + if (!input.scope.trim()) throw new Error("ConsoleLearningUnscoped"); + let answer = ""; + for await (const chunk of input.stream) answer += chunk; + const entries = await input.distill({ question: input.question, answer }); + for (const entry of entries.slice(0, 3)) { + if (!entry.subject || !entry.key || !entry.learning) continue; + await input.land(input.scope, { subject: entry.subject, key: entry.key, learning: entry.learning }); + } + return answer; +} diff --git a/apps/console/src/publish.ts b/apps/console/src/publish.ts new file mode 100644 index 000000000..3f856608f --- /dev/null +++ b/apps/console/src/publish.ts @@ -0,0 +1,21 @@ +export function answerDelivery(input: { operator: string; answer: string; accessExplanation: boolean; share?: boolean }): { + recipient: string; answer: string; share: false; +} { + if (!input.operator.trim()) throw new Error("VisibilityAskerlessAudience"); + if (input.accessExplanation && input.share) throw new Error("VisibilityShareAffordance"); + return { recipient: input.operator, answer: input.answer, share: false }; +} + +export function scheduledAudience(input: { + identity: "service" | "operator"; + operator?: string; + destination: string; + corpus: string[]; + reviewedCorpus?: string[]; +}): { destination: string; corpus: string[] } { + if (input.identity === "service" || !input.operator?.trim()) { + throw new Error(`VisibilityAskerlessAudience: ${input.destination}`); + } + const reviewed = new Set(input.reviewedCorpus ?? []); + return { destination: input.destination, corpus: input.corpus.filter((entry) => reviewed.has(entry)) }; +} diff --git a/apps/console/src/render.ts b/apps/console/src/render.ts new file mode 100644 index 000000000..454f924ae --- /dev/null +++ b/apps/console/src/render.ts @@ -0,0 +1,79 @@ +export type ResultRows = { columns: string[]; rows: unknown[][] }; +export type Component = "metric.v1" | "line.v1" | "table.v1" | "bar.v1"; +export type View = { component: Component; props: ResultRows; alternates?: Component[] }; + +function numeric(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function day(value: unknown): boolean { + if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const parsed = new Date(`${value}T00:00:00Z`); + return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value; +} + +export function buildView(result: ResultRows, options: { + origin?: "server" | "client" | "model"; + view?: unknown; + hint?: { component: Component; columns: string[] }; +} = {}): View { + if (options.origin === "client" || options.origin === "model" || options.view !== undefined) { + throw new Error("ConsoleViewNotServerBuilt"); + } + const props = { columns: [...result.columns], rows: result.rows.map((row) => [...row]) }; + const oneMeasure = props.columns.length === 1 && props.rows.length === 1 && numeric(props.rows[0][0]); + const dateIndex = props.columns.findIndex((column) => /^(date|day|.*_date|.*_day)$/i.test(column)); + const measureIndex = props.columns.findIndex((_, index) => index !== dateIndex && props.rows.every((row) => numeric(row[index]))); + const dates = dateIndex < 0 ? [] : props.rows.map((row) => row[dateIndex]); + const line = dateIndex >= 0 && measureIndex >= 0 && props.rows.length >= 3 && dates.every(day) && new Set(dates).size === dates.length; + const chosen: Component = oneMeasure ? "metric.v1" : line ? "line.v1" : "table.v1"; + const hint = options.hint; + const binds = hint && hint.columns.length > 0 && hint.columns.every((column) => props.columns.includes(column)); + const component = binds && hint.component === "bar.v1" && chosen === "table.v1" ? "bar.v1" : chosen; + return component === "table.v1" ? { component, props, alternates: ["bar.v1"] } : { component, props }; +} + +export function readView(view: { component: string; props: ResultRows }): View { + const known = new Set(["metric.v1", "line.v1", "table.v1", "bar.v1"]); + if (known.has(view.component as Component)) return view as View; + return { component: "table.v1", props: view.props, alternates: ["bar.v1"] }; +} + +export function sanitizeView(view: View, redact: (value: string) => string): View | null { + const { columns, rows } = view.props; + if (!Array.isArray(columns) || !columns.every((column) => typeof column === "string") || + !Array.isArray(rows) || !rows.every((row) => Array.isArray(row) && row.length === columns.length)) return null; + const seen = new WeakSet(); + const walk = (value: unknown): unknown => { + if (typeof value === "string") return redact(value); + if (Array.isArray(value)) { + if (seen.has(value)) throw new TypeError("cyclic view props"); + seen.add(value); + const result = value.map(walk); + seen.delete(value); + return result; + } + if (value !== null && typeof value === "object") { + if (seen.has(value)) throw new TypeError("cyclic view props"); + seen.add(value); + const result = Object.fromEntries(Object.entries(value).map(([key, item]) => [key, walk(item)])); + seen.delete(value); + return result; + } + return value; + }; + try { + return { ...view, props: { columns: columns.map(redact), rows: rows.map((row) => row.map(walk)) } }; + } catch { + return null; + } +} + +export function splitResult(result: ResultRows & { sources: TSource[]; trace: TTrace }): { + grounding: ResultRows & { sources: TSource[] }; + trace: TTrace; + view: View; +} { + const { columns, rows, sources, trace } = result; + return { grounding: { columns, rows, sources }, trace, view: buildView({ columns, rows }) }; +} From 7e3252f96b81a559b47a60ce246fb2ab07d6f3fd Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 14:58:05 +0800 Subject: [PATCH 043/155] feat(owner): admit signed store-bound owner credentials --- crates/contextful-cli/src/admit.rs | 3 ++ crates/contextful-cli/src/differential.rs | 2 +- crates/contextful-cli/src/mcp.rs | 11 +++++- crates/contextful-cli/src/project.rs | 7 ++++ crates/contextful-cli/src/token.rs | 17 +++++++-- crates/contextful-core/src/claims.rs | 3 ++ crates/contextful-policy/src/issue.rs | 3 ++ crates/contextful-policy/src/profile.rs | 24 +++++++++---- crates/contextful-policy/src/verify.rs | 35 ++++++++++++++++++- .../tests/integration/attenuate.rs | 2 +- .../tests/integration/decide.rs | 2 +- .../tests/integration/issue.rs | 2 +- .../tests/integration/revoke.rs | 2 +- .../tests/integration/verify.rs | 4 +-- spec/50-authority.md | 2 ++ spec/71-console.md | 2 +- spec/adr/A-authority.md | 2 +- spec/adr/A-surface.md | 2 +- spec/terms/surface.toml | 1 + 19 files changed, 106 insertions(+), 20 deletions(-) diff --git a/crates/contextful-cli/src/admit.rs b/crates/contextful-cli/src/admit.rs index 11204b098..8303f697a 100644 --- a/crates/contextful-cli/src/admit.rs +++ b/crates/contextful-cli/src/admit.rs @@ -66,6 +66,9 @@ pub enum AdmitError { #[cfg_attr(not(feature = "read-plane"), allow(dead_code))] #[error("StdioCredentialMissing: {0}")] StdioCredentialMissing(String), + /// (`surface.package.owner-flag`) + #[error("OwnerCredentialInvalid: {0}")] + OwnerCredentialInvalid(String), /// (`authority.revoke.ledger-unavailable`) #[error("KeySetLedgerUnavailable: {0}")] KeySetLedgerUnavailable(String), diff --git a/crates/contextful-cli/src/differential.rs b/crates/contextful-cli/src/differential.rs index 274b19d6f..3384197ac 100644 --- a/crates/contextful-cli/src/differential.rs +++ b/crates/contextful-cli/src/differential.rs @@ -529,7 +529,7 @@ fn gen_verify(rng: &mut Rng, pools: &Pools) -> Value { expires_at: At::from_unix_secs(expires).expect("a fixed instant"), lifetime_secs: lifetime, }; - let claims = MintClaims { confirmation: rng.chance(10).then(|| "holder-thumbprint".to_string()), epoch: 0 }; + let claims = MintClaims { confirmation: rng.chance(10).then(|| "holder-thumbprint".to_string()), epoch: 0, ..MintClaims::default() }; let mut seed = [0u8; 32]; for chunk in seed.chunks_mut(8) { chunk.copy_from_slice(&rng.next().to_le_bytes()); diff --git a/crates/contextful-cli/src/mcp.rs b/crates/contextful-cli/src/mcp.rs index 90bd97a1b..7fb7f5bff 100644 --- a/crates/contextful-cli/src/mcp.rs +++ b/crates/contextful-cli/src/mcp.rs @@ -5,7 +5,7 @@ //! unanchored (`disclosure.record.read-chain`), and hands them to the tool server. Every value is resolved before the first protocol line is written, so a //! process that cannot serve exits with nothing on standard output. -use crate::admit::{face, AdmitArgs}; +use crate::admit::{face, AdmitArgs, AdmitError}; use crate::project::locate; use crate::clock::SystemClock; use anyhow::Result; @@ -18,6 +18,9 @@ use std::path::PathBuf; #[derive(clap::Args)] pub struct McpArgs { + /// Admit the selected local project under a verified signed owner claim. + #[arg(long)] + owner: bool, /// The project whose store root is `.contextful/context//` under the working /// directory; absent, the nearest `contextful.toml` upward names it. #[arg(long)] @@ -33,6 +36,12 @@ pub struct McpArgs { pub fn run(args: McpArgs) -> Result<()> { let (authority, revocation) = args.admit.admit(args.project.as_deref(), "the tool server")?; let located = locate(args.project.as_deref(), args.declaration)?; + let authority = if args.owner { + authority.activate_owner(&crate::project::owner_identity(&located.project)?) + .ok_or_else(|| AdmitError::OwnerCredentialInvalid("the verified credential does not own the selected local store".into()))? + } else { + authority + }; crate::sync::pull_before_run(&located)?; let face = face(&located)?; let audit = AuditLog::unanchored(located.project.audit_dir())?; diff --git a/crates/contextful-cli/src/project.rs b/crates/contextful-cli/src/project.rs index eea0d330b..a24519035 100644 --- a/crates/contextful-cli/src/project.rs +++ b/crates/contextful-cli/src/project.rs @@ -8,6 +8,7 @@ use contextful_context::Store; use contextful_core::pipeline::declare::ManifestFile; use contextful_policy::enforce::mask::Pepper; use std::path::Path; +use sha2::{Digest, Sha256}; #[cfg(feature = "data-plane")] use anyhow::anyhow; #[cfg(feature = "data-plane")] @@ -40,6 +41,12 @@ pub struct Located { pub declaration: PathBuf, } +/// The canonical local store root binds an owner credential to one store instance. +pub(crate) fn owner_identity(project: &Project) -> Result { + let root = std::fs::canonicalize(project.store_root())?; + Ok(format!("{:x}", Sha256::digest(root.to_string_lossy().as_bytes()))) +} + /// Resolve the project from `--project` or by discovery, and the declaration from /// `--declaration` or beside the project (`store.init.default-declaration`). pub fn locate(project: Option<&str>, declaration: Option) -> Result { diff --git a/crates/contextful-cli/src/token.rs b/crates/contextful-cli/src/token.rs index 0482e79fb..8354b5b94 100644 --- a/crates/contextful-cli/src/token.rs +++ b/crates/contextful-cli/src/token.rs @@ -121,6 +121,9 @@ pub enum TokenCmd { /// credential's confirmation claim (`authority.verify.possession-binding`). #[arg(long)] holder: Option, + /// Sign a credential for explicit local owner admission of the discovered project. + #[arg(long)] + owner: bool, /// Issue instant (RFC 3339); absent reads the system clock. #[arg(long)] now: Option, @@ -267,6 +270,7 @@ pub fn run(cmd: TokenCmd) -> Result<()> { ttl, audience, holder, + owner, now, } => { let subject = Subject { on_behalf_of, agent, host, task, zone, incognito }; @@ -279,7 +283,7 @@ pub fn run(cmd: TokenCmd) -> Result<()> { max_rows, }; let lifetime = ttl.map_or(Lifetime::Default, Lifetime::Requested); - let token = mint_one(issuer_key.as_deref(), subject, grant, lifetime, audience, holder, now.as_deref())?; + let token = mint_one(issuer_key.as_deref(), subject, grant, lifetime, audience, holder, owner, now.as_deref())?; println!("{token}"); Ok(()) } @@ -572,6 +576,7 @@ fn mint_one( lifetime: Lifetime, audience: Option, holder: Option, + owner: bool, now: Option<&str>, ) -> Result { let root = Root::find()?; @@ -585,6 +590,14 @@ fn mint_one( let clock = FixedClock(instant_or_now(now)?); let ctx = MintContext { node: NodeRole::Primary, signer: &signer as &dyn SigningPort, clock: &clock as &dyn Clock }; let plan = policy.check(&request, &ctx)?; + let owner_project = if owner { + let project = crate::project::locate(None, None)?.project; + let identity = crate::project::owner_identity(&project)?; + if !contextful_policy::verify::owner_grant(&plan.grants) { + bail!("an owner credential requires an unrestricted read grant over `*`"); + } + Some(identity) + } else { None }; if let Some(jkt) = holder.as_deref() { let alphabet = |c: char| c.is_ascii_alphanumeric() || c == '-' || c == '_'; if jkt.len() != 43 || !jkt.chars().all(alphabet) { @@ -592,7 +605,7 @@ fn mint_one( } } let epoch = mint_epoch(&plan, &root.ledger().read()?.current_epochs()); - Ok(mint(&plan, &MintClaims { confirmation: holder, epoch }, &signer)?) + Ok(mint(&plan, &MintClaims { confirmation: holder, epoch, owner_project }, &signer)?) } fn instant_or_now(text: Option<&str>) -> Result { diff --git a/crates/contextful-core/src/claims.rs b/crates/contextful-core/src/claims.rs index 241a2382f..65502a606 100644 --- a/crates/contextful-core/src/claims.rs +++ b/crates/contextful-core/src/claims.rs @@ -22,6 +22,9 @@ pub struct AuthorityBlock { pub sub: Subject, pub att: BTreeMap, pub grants: Vec, + /// The one local project this signed authority block may open as owner. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_project: Option, pub rev: Revocation, } diff --git a/crates/contextful-policy/src/issue.rs b/crates/contextful-policy/src/issue.rs index ddccb2651..8c99fa5a0 100644 --- a/crates/contextful-policy/src/issue.rs +++ b/crates/contextful-policy/src/issue.rs @@ -116,6 +116,8 @@ pub struct MintClaims { pub confirmation: Option, /// The scoped revocation epoch the credential is minted under (`authority.revoke.epoch`). pub epoch: u64, + /// The local project authorized for explicit owner admission. + pub owner_project: Option, } /// The authority block a plan mints: a fresh credential identifier, the subject @@ -141,6 +143,7 @@ fn block_with_jti(plan: &MintPlan, claims: &MintClaims, jti: [u8; 16]) -> Author att: subject.attestations(), sub: subject.to_subject(), grants: plan.grants.clone(), + owner_project: claims.owner_project.clone(), } } diff --git a/crates/contextful-policy/src/profile.rs b/crates/contextful-policy/src/profile.rs index d4bd5156e..4ebcd2f09 100644 --- a/crates/contextful-policy/src/profile.rs +++ b/crates/contextful-policy/src/profile.rs @@ -10,6 +10,8 @@ //! profile(1); iss(s); aud(s)?; jti(s); iat(n); exp(n); alg(s); cnf(s)?; //! sub(member, value)*; incognito(b)?; att(member, attestation)*; grant(i, json)*; rev(id, epoch); //! ``` +//! Profile 2 adds `owner_project(s)?` to the root authority block; profile 1 remains +//! readable without that fact. //! //! An attenuation block carries only `grant(i, json)*`, `exp(n)`, `sub(member, value)*`, //! `incognito(b)` and `cnf(s)`; each proposes a narrowing its parent is checked against, @@ -33,9 +35,11 @@ use std::time::{Duration, UNIX_EPOCH}; /// The profile version this engine mints. pub const PROFILE_VERSION: i64 = 1; +/// Profile 2 adds the authority-only signed local owner claim. +pub const OWNER_PROFILE_VERSION: i64 = 2; /// The profile versions a checkpoint admits by default (`authority.profile.version-unsupported`). -pub const SUPPORTED_PROFILE_VERSIONS: &[i64] = &[PROFILE_VERSION]; +pub const SUPPORTED_PROFILE_VERSIONS: &[i64] = &[PROFILE_VERSION, OWNER_PROFILE_VERSION]; /// Facts one authorization holds, token and engine facts together (`authority.profile.fact-ceiling`). pub const EVALUATOR_FACT_CEILING: u64 = 1000; @@ -170,7 +174,7 @@ pub fn read_chain(bytes: &[u8], supported: &[i64]) -> Result>()?; Ok(Chain { version, authority, hops, fact_count }) } @@ -303,7 +307,7 @@ fn slots(i: usize, facts: &[RawFact], allowed: &[&'static str]) -> Result { + ("iss" | "aud" | "jti" | "alg" | "cnf" | "owner_project", [t @ Term::Str(_)]) => { if out.scalars.insert(name, t.clone()).is_some() { return Err(twice()); } @@ -346,11 +350,14 @@ impl Slots { } const AUTHORITY_FACTS: &[&str] = + &["profile", "iss", "aud", "jti", "iat", "exp", "alg", "cnf", "owner_project", "sub", "incognito", "att", "grant", "rev"]; +const PROFILE_ONE_AUTHORITY_FACTS: &[&str] = &["profile", "iss", "aud", "jti", "iat", "exp", "alg", "cnf", "sub", "incognito", "att", "grant", "rev"]; const HOP_FACTS: &[&str] = &["grant", "exp", "sub", "incognito", "cnf"]; -fn read_authority(facts: &[RawFact]) -> Result { - let s = slots(0, facts, AUTHORITY_FACTS)?; +fn read_authority(facts: &[RawFact], version: i64) -> Result { + let allowed = if version == OWNER_PROFILE_VERSION { AUTHORITY_FACTS } else { PROFILE_ONE_AUTHORITY_FACTS }; + let s = slots(0, facts, allowed)?; let missing = |name: &str| unrecognized(format!("the authority block names no `{name}`")); let timestamp = |name: &str| -> Result { let secs = s.int(name).ok_or_else(|| missing(name))?; @@ -379,6 +386,7 @@ fn read_authority(facts: &[RawFact]) -> Result { sub, att: s.att.clone(), grants: s.grants(), + owner_project: s.str("owner_project"), rev: Revocation { id: rev_id, epoch }, }) } @@ -470,8 +478,9 @@ fn grant_facts(out: &mut Vec, grants: &[Grant]) -> Result<(), AuthorityErr /// The authority block's facts. pub fn authority_facts(block: &AuthorityBlock) -> Result, AuthorityError> { + let version = if block.owner_project.is_some() { OWNER_PROFILE_VERSION } else { PROFILE_VERSION }; let mut out = vec![ - fact("profile", &[int(PROFILE_VERSION)]), + fact("profile", &[int(version)]), fact("iss", &[string(&block.iss)]), fact("aud", &[string(&block.aud)]), fact("jti", &[string(&block.jti)]), @@ -482,6 +491,9 @@ pub fn authority_facts(block: &AuthorityBlock) -> Result, AuthorityErr if let Some(cnf) = &block.cnf { out.push(fact("cnf", &[string(&cnf.jkt)])); } + if let Some(project) = &block.owner_project { + out.push(fact("owner_project", &[string(project)])); + } let subject = block.subject(); sub_facts(&mut out, Member::ALL.map(|m| (m, subject.get(m).map(str::to_string)))); out.push(fact("incognito", &[boolean(subject.incognito())])); diff --git a/crates/contextful-policy/src/verify.rs b/crates/contextful-policy/src/verify.rs index 4d2f90674..edddaeefe 100644 --- a/crates/contextful-policy/src/verify.rs +++ b/crates/contextful-policy/src/verify.rs @@ -17,7 +17,7 @@ use base64::engine::DecodePaddingMode; use base64::Engine; use biscuit_auth::Biscuit; use contextful_core::claims::AuthorityBlock; -use contextful_core::grant::{Action, Grant}; +use contextful_core::grant::{Action, Grant, TablePattern}; use contextful_core::identify::NormalizedSubject; use contextful_core::issue::{MintPlan, SignatureAlgorithm}; use contextful_core::ports::SigningPort; @@ -84,9 +84,28 @@ pub struct AdmittedAuthority { profile: i64, key_version: String, format: &'static str, + owner_project: Option, + owner_active: bool, } impl AdmittedAuthority { + /// The owner claim is signed in the root block and never inferred from grants. + pub fn owner_project(&self) -> Option<&str> { + self.owner_project.as_deref() + } + + /// Activate the signed owner claim only for the selected local project. + pub fn activate_owner(mut self, project: &str) -> Option { + if self.owner_project.as_deref() != Some(project) || !owner_grant(&self.grants) { + return None; + } + self.owner_active = true; + Some(self) + } + + pub fn owner_active(&self) -> bool { + self.owner_active + } pub fn subject(&self) -> &NormalizedSubject { &self.subject } @@ -187,6 +206,18 @@ impl AdmittedAuthority { } } +/// An owner claim names one store and carries an unrestricted read grant over it. +pub fn owner_grant(grants: &[Grant]) -> bool { + grants.iter().any(|grant| { + grant.actions.contains(&Action::Read) + && grant.tables.contains(&TablePattern::All) + && grant.tenant.is_none() + && grant.aggregate.is_none() + && grant.templates.is_none() + && grant.max_rows.is_none() + }) +} + /// Decode transmitted text into the library's bytes. Text that is not the encoding /// carries no signature anybody made. pub(crate) fn token_bytes(credential: &str) -> Result, AuthorityError> { @@ -420,6 +451,8 @@ fn admit>( profile: chain.version, key_version, format: BISCUIT_FORMAT, + owner_project: chain.hops.is_empty().then(|| block.owner_project.clone()).flatten(), + owner_active: false, }; admission.revocation.check(&admitted, admission.at)?; admitted.subject.require_member(ADMISSION_SURFACE)?; diff --git a/crates/contextful-policy/tests/integration/attenuate.rs b/crates/contextful-policy/tests/integration/attenuate.rs index 08b05e3d0..516ac48b9 100644 --- a/crates/contextful-policy/tests/integration/attenuate.rs +++ b/crates/contextful-policy/tests/integration/attenuate.rs @@ -70,7 +70,7 @@ fn each_sub_agent_child_binds_that_sub_agents_own_key() { let signer = issuer(); let agent = ed25519_dalek::SigningKey::from_bytes(&[1; 32]); let agent_jkt = jwk_thumbprint(agent.verifying_key().as_bytes()); - let parent = mint(&plan(&signer), &MintClaims { confirmation: Some(agent_jkt.clone()), epoch: 0 }, &signer).unwrap(); + let parent = mint(&plan(&signer), &MintClaims { confirmation: Some(agent_jkt.clone()), epoch: 0, ..MintClaims::default() }, &signer).unwrap(); let sub_agents = [ed25519_dalek::SigningKey::from_bytes(&[2; 32]), ed25519_dalek::SigningKey::from_bytes(&[3; 32])]; let request = ProofRequest { method: "POST", target: "https://store.example/v1/query", body: b"{}" }; let clock = contextful_core::ports::FixedClock(at(DURING)); diff --git a/crates/contextful-policy/tests/integration/decide.rs b/crates/contextful-policy/tests/integration/decide.rs index 3fba73bba..1dac9b703 100644 --- a/crates/contextful-policy/tests/integration/decide.rs +++ b/crates/contextful-policy/tests/integration/decide.rs @@ -112,7 +112,7 @@ fn a_verification_case_refuses_as_the_checkpoint_does() { assert_eq!(decided(&case(&lasting(&signer, 7200), &signer)), refused("BearerLifetimeExceeded")); assert_eq!(decided(&case(&lasting(&signer, 3600), &signer)), verdict("admitted")); - let bound = mint(&plan(&signer), &MintClaims { confirmation: Some("thumbprint".into()), epoch: 0 }, &signer).unwrap(); + let bound = mint(&plan(&signer), &MintClaims { confirmation: Some("thumbprint".into()), epoch: 0, ..MintClaims::default() }, &signer).unwrap(); assert_eq!(decided(&case(&bound, &signer)), refused("PossessionProofInvalid"), "the case carries no proof"); let mut unpinned = case(&credential, &signer); diff --git a/crates/contextful-policy/tests/integration/issue.rs b/crates/contextful-policy/tests/integration/issue.rs index 46ad5ca74..d01cfedb0 100644 --- a/crates/contextful-policy/tests/integration/issue.rs +++ b/crates/contextful-policy/tests/integration/issue.rs @@ -63,7 +63,7 @@ fn a_seed_file_round_trips_and_its_public_key_is_a_static_pin() { fn a_mint_carries_the_plan_the_policy_checked() { let signer = issuer(); let p = plan(&signer); - let credential = mint(&p, &MintClaims { confirmation: Some("jkt-1".into()), epoch: 7 }, &signer).unwrap(); + let credential = mint(&p, &MintClaims { confirmation: Some("jkt-1".into()), epoch: 7, ..MintClaims::default() }, &signer).unwrap(); let block = introspect(&credential).unwrap().authority; assert_eq!(block.aud, AUD); assert_eq!(block.iss, AUD); diff --git a/crates/contextful-policy/tests/integration/revoke.rs b/crates/contextful-policy/tests/integration/revoke.rs index f39ba4d57..b142e26a0 100644 --- a/crates/contextful-policy/tests/integration/revoke.rs +++ b/crates/contextful-policy/tests/integration/revoke.rs @@ -25,7 +25,7 @@ fn the_credential_identifier_withdraws_the_whole_chain() { #[test] fn an_epoch_below_the_current_scoped_epoch_is_refused_at_admission() { let signer = issuer(); - let credential = mint(&plan(&signer), &MintClaims { confirmation: None, epoch: 1 }, &signer).unwrap(); + let credential = mint(&plan(&signer), &MintClaims { confirmation: None, epoch: 1, ..MintClaims::default() }, &signer).unwrap(); let admitted = admit(&credential, &signer, DURING).unwrap(); let claims = revocation_claims(&admitted); assert_eq!(claims.project, AUD); diff --git a/crates/contextful-policy/tests/integration/verify.rs b/crates/contextful-policy/tests/integration/verify.rs index 5cb53ad4d..88184a399 100644 --- a/crates/contextful-policy/tests/integration/verify.rs +++ b/crates/contextful-policy/tests/integration/verify.rs @@ -201,7 +201,7 @@ fn the_authority_core_flow_admits_narrows_and_re_reads() { /// A credential bound to `holder`'s key, and the request and proof a local client sends. fn key_bound(signer: &contextful_policy::issue::SeedSigner, holder: &ed25519_dalek::SigningKey) -> String { let jkt = contextful_policy::possession::jwk_thumbprint(holder.verifying_key().as_bytes()); - contextful_policy::issue::mint(&plan(signer), &MintClaims { confirmation: Some(jkt), epoch: 0 }, signer).unwrap() + contextful_policy::issue::mint(&plan(signer), &MintClaims { confirmation: Some(jkt), epoch: 0, ..MintClaims::default() }, signer).unwrap() } const LOCAL_TARGET: &str = "unix:///run/contextful.sock/v1/query"; @@ -370,7 +370,7 @@ fn a_bearer_living_past_3600_s_admits_nothing_over_a_network() { let clock = FixedClock(at(MINTED)); let plan = policy.check(&req, &MintContext { node: NodeRole::Primary, signer: &signer, clock: &clock }).unwrap(); let jkt = contextful_policy::possession::jwk_thumbprint(holder.verifying_key().as_bytes()); - let day = contextful_policy::issue::mint(&plan, &MintClaims { confirmation: Some(jkt), epoch: 0 }, &signer).unwrap(); + let day = contextful_policy::issue::mint(&plan, &MintClaims { confirmation: Some(jkt), epoch: 0, ..MintClaims::default() }, &signer).unwrap(); let mut nonces = NonceCache::new(); let proof = sign_proof(&holder, &local_request(), at(DURING), "n-day"); assert!(verify_network(&day, &keys(&signer), &admission, holder_proof(&proof, &mut nonces)).is_ok()); diff --git a/spec/50-authority.md b/spec/50-authority.md index 94e5d5433..5c480222c 100644 --- a/spec/50-authority.md +++ b/spec/50-authority.md @@ -97,6 +97,8 @@ The versioned delegation profile over the attenuable-credential library: admitte - `delegation-profile` — Delegated authority travels in one attenuable, chain-signed library format. The library owns serialization, signatures, block chaining and evaluation; a versioned profile names every fact, check and restriction the engine admits. *A-authority* +- `owner-claim` — Profile 2 permits a signed owner store identity only in the root authority block; profile 1 and attenuation blocks reject it, and activation requires an unrestricted read grant over the selected store. + *A-authority* - `unrecognized-element` — A credential carrying a block version, predicate, rule or restriction the profile does not name raises `ProfileElementUnrecognized`. *P1* - `reserved-fact` — Current time, audience, resolved resources and authenticated request identity are reserved facts the engine supplies. A token block introducing one raises `ProfileReservedFact`. diff --git a/spec/71-console.md b/spec/71-console.md index b55f49836..f6d4d627c 100644 --- a/spec/71-console.md +++ b/spec/71-console.md @@ -149,7 +149,7 @@ The client library, its four deployment shapes and transports, and the credentia *A-read* - `store-selector` — The child's working directory selects the store by walking up to the project manifest; finding none raises `StoreSelectorAbsent` and exits before writing any protocol framing. *A-topology* -- `owner-flag` — An explicit owner flag over the process transport admits the local owner context only with the store's owner credential. +- `owner-flag` — A process `--owner` admits the local owner context only with a verified signed claim bound to the selected store; an ordinary or different-store credential raises `OwnerCredentialInvalid` before protocol framing. *A-surface* The client library's four shapes, and who holds the credential in each: diff --git a/spec/adr/A-authority.md b/spec/adr/A-authority.md index 0acf7e4a5..ba10bcab6 100644 --- a/spec/adr/A-authority.md +++ b/spec/adr/A-authority.md @@ -33,7 +33,7 @@ Revisit: a directory's identifiers exceed 256 B; provisioning latency dominates ## Delegation is a versioned profile over an attenuable-credential library -The library is Biscuit, and its own format is the one wire format; the engine owns a versioned profile and no envelope. `authority.profile` names the admitted facts, checks and restriction tuples, and refuses any other element; the library owns serialization, signatures, chaining and evaluation. A credential is an authority block plus N attenuation blocks; the supported chain depth is unsettled under `authority.attenuate`. Time, audience, resolved resources and request identity are reserved engine-supplied facts. Evaluation admits no third-party block, external function, recursion or regular expression, under fact and iteration ceilings that refuse past their bound. +The library is Biscuit, and its own format is the one wire format; the engine owns a versioned profile and no envelope. `authority.profile` names the admitted facts, checks and restriction tuples, and refuses any other element; the library owns serialization, signatures, chaining and evaluation. Profile 2 adds the signed local owner store identity to the root block; profile 1 stays readable, and attenuation cannot acquire ownership. A credential is an authority block plus N attenuation blocks; the supported chain depth is unsettled under `authority.attenuate`. Time, audience, resolved resources and request identity are reserved engine-supplied facts. Evaluation admits no third-party block, external function, recursion or regular expression, under fact and iteration ceilings that refuse past their bound. | Option | Lost on | Cost | | --- | --- | --- | diff --git a/spec/adr/A-surface.md b/spec/adr/A-surface.md index 9d3fb45ea..89398f4e1 100644 --- a/spec/adr/A-surface.md +++ b/spec/adr/A-surface.md @@ -71,7 +71,7 @@ Consequences: Query and Admin use the same store registry while retaining separa ## Turn compatibility preserves the stored transcript and reader scope -An explicit process owner flag needs the store's owner credential; it does not turn an empty credential into ownership. A store overlay reaches synthesis text alone, so planner tool selection does not inherit a store-authored persona. Distillation records the observed subject and recall resolves it through entity matching. A client with an older component union renders an unknown view as a table. One operator's credential defines an answer; a room-intersection principal is absent. +An explicit process owner flag needs a signed owner claim bound to the selected local store; it does not turn an ordinary or empty credential into ownership. A store overlay reaches synthesis text alone, so planner tool selection does not inherit a store-authored persona. Distillation records the observed subject and recall resolves it through entity matching. A client with an older component union renders an unknown view as a table. One operator's credential defines an answer; a room-intersection principal is absent. | Option | Lost on | Cost | | --- | --- | --- | diff --git a/spec/terms/surface.toml b/spec/terms/surface.toml index 37b185793..ec3d38361 100644 --- a/spec/terms/surface.toml +++ b/spec/terms/surface.toml @@ -56,6 +56,7 @@ StoreEntryMalformed = { clause = "surface.register-store.malformed-ent StoreIdReserved = { clause = "surface.register-store.reserved-id", gloss = "A configured store claiming a built-in id." } StoreNameAuthored = { clause = "surface.register-store.authored-name", gloss = "A store entry carrying its own credential or binding name." } StdioCredentialMissing = { clause = "surface.package.stdio-credential", gloss = "A process-transport client with no credential." } +OwnerCredentialInvalid = { clause = "surface.package.owner-flag", gloss = "A process owner flag with no signed claim for the selected local store." } StoreSelectorAbsent = { clause = "surface.package.store-selector", gloss = "A spawned engine finding no project manifest above its working directory." } ConsoleSuggestionUnanswerable = { clause = "surface.speak.unanswerable-suggestion", gloss = "A suggested prompt whose answer is a decline." } ConsoleUngroundedAnswer = { clause = "surface.ground.ungrounded-answer", gloss = "Answer prose composed in a turn holding no tool result." } From a0ded8cf21b84976870f1c0db7ef7a5d28df7755 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:12:42 +0800 Subject: [PATCH 044/155] fix(owner): gate mint support to read-capable builds --- crates/contextful-cli/src/admit.rs | 1 + crates/contextful-cli/src/token.rs | 15 +- .../contextful-cli/tests/integration/mcp.rs | 2 +- spec/71-console.md | 4 +- spec/adr/A-read.md | 2 +- spec/cards/authority.md | 2 +- spec/cards/surface.md | 1 + spec/spec.lock.json | 305 +++++++++--------- spec/status.md | 8 +- 9 files changed, 181 insertions(+), 159 deletions(-) diff --git a/crates/contextful-cli/src/admit.rs b/crates/contextful-cli/src/admit.rs index 8303f697a..e94c4e6ee 100644 --- a/crates/contextful-cli/src/admit.rs +++ b/crates/contextful-cli/src/admit.rs @@ -67,6 +67,7 @@ pub enum AdmitError { #[error("StdioCredentialMissing: {0}")] StdioCredentialMissing(String), /// (`surface.package.owner-flag`) + #[cfg(feature = "read-plane")] #[error("OwnerCredentialInvalid: {0}")] OwnerCredentialInvalid(String), /// (`authority.revoke.ledger-unavailable`) diff --git a/crates/contextful-cli/src/token.rs b/crates/contextful-cli/src/token.rs index 8354b5b94..b0e9bcd79 100644 --- a/crates/contextful-cli/src/token.rs +++ b/crates/contextful-cli/src/token.rs @@ -591,12 +591,17 @@ fn mint_one( let ctx = MintContext { node: NodeRole::Primary, signer: &signer as &dyn SigningPort, clock: &clock as &dyn Clock }; let plan = policy.check(&request, &ctx)?; let owner_project = if owner { - let project = crate::project::locate(None, None)?.project; - let identity = crate::project::owner_identity(&project)?; - if !contextful_policy::verify::owner_grant(&plan.grants) { - bail!("an owner credential requires an unrestricted read grant over `*`"); + #[cfg(feature = "read-plane")] + { + let project = crate::project::locate(None, None)?.project; + let identity = crate::project::owner_identity(&project)?; + if !contextful_policy::verify::owner_grant(&plan.grants) { + bail!("an owner credential requires an unrestricted read grant over `*`"); + } + Some(identity) } - Some(identity) + #[cfg(not(feature = "read-plane"))] + bail!("an owner credential requires the read-plane build profile"); } else { None }; if let Some(jkt) = holder.as_deref() { let alphabet = |c: char| c.is_ascii_alphanumeric() || c == '-' || c == '_'; diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 05f1282c1..defbe140c 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -92,7 +92,7 @@ fn a_server_with_no_admissible_credential_writes_no_framing() { assert!(!unmanifested.status.success() && unmanifested.stdout.is_empty()); } -/// Over the process transport a credential is mandatory, a capability token or an explicit owner flag; an unset one raises `StdioCredentialMissing` and does not resolve to the owner context. +/// The process transport requires a verified capability token; `--owner` selects its signed owner claim. An absent token raises `StdioCredentialMissing` before protocol framing and never resolves to owner context. #[test] fn a_server_with_no_credential_raises_stdio_credential_missing() { let (dir, public, _token) = project(); diff --git a/spec/71-console.md b/spec/71-console.md index f6d4d627c..d16cf41b5 100644 --- a/spec/71-console.md +++ b/spec/71-console.md @@ -145,11 +145,11 @@ The Admin page's operational views. The client library, its four deployment shapes and transports, and the credential each shape carries. -- `stdio-credential` — Over the process transport a credential is mandatory, a capability token or an explicit owner flag; an unset one raises `StdioCredentialMissing` and does not resolve to the owner context. +- `stdio-credential` — The process transport requires a verified capability token; `--owner` selects its signed owner claim. An absent token raises `StdioCredentialMissing` before protocol framing and never resolves to owner context. *A-read* - `store-selector` — The child's working directory selects the store by walking up to the project manifest; finding none raises `StoreSelectorAbsent` and exits before writing any protocol framing. *A-topology* -- `owner-flag` — A process `--owner` admits the local owner context only with a verified signed claim bound to the selected store; an ordinary or different-store credential raises `OwnerCredentialInvalid` before protocol framing. +- `owner-flag` — A process `--owner` admits the local owner context only with a verified signed claim bound to the selected store and an unrestricted read grant; any other credential raises `OwnerCredentialInvalid` before protocol framing. *A-surface* The client library's four shapes, and who holds the credential in each: diff --git a/spec/adr/A-read.md b/spec/adr/A-read.md index bf4e99151..bd737d1b6 100644 --- a/spec/adr/A-read.md +++ b/spec/adr/A-read.md @@ -4,7 +4,7 @@ ## The read surface admits one read-only statement over registered relations -`read.guard` walks the syntax tree the executor itself serializes and admits exactly one read-only `SELECT` whose base relations are views registered for the caller or declared common table expressions; table functions and catalog reaches refuse, and a refusal never echoes another caller's relation. `read.register` lists only granted templates and binds arguments strictly. A filter's budget applies to the whole filter. `read.respond` reads a preview through its table's relation. The process transport requires a token or an explicit owner flag; the run-stream socket authenticates before upgrade and carries snapshots only. +`read.guard` walks the syntax tree the executor itself serializes and admits exactly one read-only `SELECT` whose base relations are views registered for the caller or declared common table expressions; table functions and catalog reaches refuse, and a refusal never echoes another caller's relation. `read.register` lists only granted templates and binds arguments strictly. A filter's budget applies to the whole filter. `read.respond` reads a preview through its table's relation. The process transport requires a token, with an explicit owner flag for its signed store claim; the run-stream socket authenticates before upgrade and carries snapshots only. | Option | Lost on | Cost | | --- | --- | --- | diff --git a/spec/cards/authority.md b/spec/cards/authority.md index b36d90eb5..1728c5c6f 100644 --- a/spec/cards/authority.md +++ b/spec/cards/authority.md @@ -7,7 +7,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/a | Operation | Covers | Clauses | | --- | --- | --- | | `identify` | The subject tuple, its per-member attestation, hygiene at the mint, and the identity links connecting a subject to a source's principals. | 14 | -| `profile` | The versioned delegation profile over the attenuable-credential library: admitted elements, reserved facts, the bounded evaluator, the scoped session. | 12 | +| `profile` | The versioned delegation profile over the attenuable-credential library: admitted elements, reserved facts, the bounded evaluator, the scoped session. | 13 | | `grant` | What a credential says a holder does: actions, table patterns, tenant scope, the template allowlist, ceilings, and resources answering by name. | 22 | | `attenuate` | Deriving a narrower child offline, and the narrowing legality of each grant dimension. | 8 | | `issue` | Minting: the persisted lifetime ceiling, the principal a row-landing grant needs, signing custody, and the authoring posture. | 32 | diff --git a/spec/cards/surface.md b/spec/cards/surface.md index b367d1ea7..a504194ff 100644 --- a/spec/cards/surface.md +++ b/spec/cards/surface.md @@ -65,6 +65,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `StoreNameAuthored` | A store entry carrying its own credential or binding name. | `surface.register-store.authored-name` | | `StdioCredentialMissing` | A process-transport client with no credential. | `surface.package.stdio-credential` | | `StoreSelectorAbsent` | A spawned engine finding no project manifest above its working directory. | `surface.package.store-selector` | +| `OwnerCredentialInvalid` | A process owner flag with no signed claim for the selected local store. | `surface.package.owner-flag` | | `ConsoleSuggestionUnanswerable` | A suggested prompt whose answer is a decline. | `surface.speak.unanswerable-suggestion` | | `ConsoleUngroundedAnswer` | Answer prose composed in a turn holding no tool result. | `surface.ground.ungrounded-answer` | | `ConsoleToolNotAdmitted` | A tool call outside the turn's admitted packs. | `surface.ground.unadmitted-tool` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 50c8feba2..ff2d34ce9 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -14455,6 +14455,17 @@ "file": "spec/50-authority.md", "line": 98 }, + { + "id": "authority.profile.owner-claim", + "contract": "authority", + "operation": "profile", + "subject": "owner-claim", + "kind": "behavior", + "statement": "Profile 2 permits a signed owner store identity only in the root authority block; profile 1 and attenuation blocks reject it, and activation requires an unrestricted read grant over the selected store.", + "why": "A-authority", + "file": "spec/50-authority.md", + "line": 100 + }, { "id": "authority.profile.unrecognized-element", "contract": "authority", @@ -14464,7 +14475,7 @@ "statement": "A credential carrying a block version, predicate, rule or restriction the profile does not name raises `ProfileElementUnrecognized`.", "why": "P1", "file": "spec/50-authority.md", - "line": 100 + "line": 102 }, { "id": "authority.profile.reserved-fact", @@ -14475,7 +14486,7 @@ "statement": "Current time, audience, resolved resources and authenticated request identity are reserved facts the engine supplies. A token block introducing one raises `ProfileReservedFact`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 102 + "line": 104 }, { "id": "authority.profile.evaluator-bound", @@ -14486,7 +14497,7 @@ "statement": "The evaluator runs with no third-party block, external function, recursion or regular-expression predicate. Input past {{authority.profile.fact-ceiling}} or {{authority.profile.iteration-ceiling}} raises `ProfileEvaluationBudget`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 104 + "line": 106 }, { "id": "authority.profile.fact-ceiling", @@ -14497,7 +14508,7 @@ "statement": "One authorization holds at most 1000 entries in its fact set.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 106 + "line": 108 }, { "id": "authority.profile.iteration-ceiling", @@ -14508,7 +14519,7 @@ "statement": "One authorization runs at most 100 iterations of the evaluator.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 108 + "line": 110 }, { "id": "authority.profile.appended-block", @@ -14519,7 +14530,7 @@ "statement": "A block after the first contributes no authority fact to an allow decision; appending narrows a credential or adds nothing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 110 + "line": 112 }, { "id": "authority.profile.unevaluated-restriction", @@ -14530,7 +14541,7 @@ "statement": "A grant carrying a row restriction or aggregate bound with no read evaluator raises `ProfileRestrictionUnevaluated` at the mint, at derivation on parent and child, and at admission.", "why": "P1", "file": "spec/50-authority.md", - "line": 112 + "line": 114 }, { "id": "authority.profile.declared-field", @@ -14541,7 +14552,7 @@ "statement": "A restriction field the engine refuses stays declared in the profile and is parsed at every admission.", "why": "P1", "file": "spec/50-authority.md", - "line": 114 + "line": 116 }, { "id": "authority.profile.declared-scope", @@ -14552,7 +14563,7 @@ "statement": "Introspection reports the scope a credential declares without evaluating it. Table policy, source access lists, time bounds and revocation settle effective authority afterwards.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 116 + "line": 118 }, { "id": "authority.profile.scoped-session", @@ -14563,7 +14574,7 @@ "statement": "Authorization yields a scoped session carrying every restriction the credential expressed. A statement over two tables needs both accesses in one coherent grant; actions and tables never flatten into independent allowlists.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 118 + "line": 120 }, { "id": "authority.profile.version-unsupported", @@ -14574,7 +14585,7 @@ "statement": "A checkpoint reading a profile version outside its supported set raises `ProfileVersionUnsupported`. Widening the profile mints a new version.", "why": "P1", "file": "spec/50-authority.md", - "line": 120 + "line": 122 }, { "id": "authority.grant.fields", @@ -14585,7 +14596,7 @@ "statement": "A grant names actions, table patterns, and optionally a tenant scope, aggregate constraints, a template allowlist and a row ceiling. An absent constraint leaves its dimension unconstrained; an absent allowlist confers no template.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 129 + "line": 131 }, { "id": "authority.grant.actions", @@ -14596,7 +14607,7 @@ "statement": "The action vocabulary is `read` for every row-returning surface, `write` to land rows, `execute` to fire a run, and `admin` to mint.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 131 + "line": 133 }, { "id": "authority.grant.unknown-action", @@ -14607,7 +14618,7 @@ "statement": "An action outside the vocabulary raises `GrantActionUnknown` at the mint and at admission.", "why": "P1", "file": "spec/50-authority.md", - "line": 133 + "line": 135 }, { "id": "authority.grant.pattern-forms", @@ -14618,7 +14629,7 @@ "statement": "A table pattern is `*`, covering every table; a prefix ending in `*`, covering every name beginning with that prefix; or any other string, matched exactly. A concrete pattern never covers `*`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 135 + "line": 137 }, { "id": "authority.grant.malformed-pattern", @@ -14629,7 +14640,7 @@ "statement": "A pattern with `*` anywhere but its final position raises `GrantPatternMalformed`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 137 + "line": 139 }, { "id": "authority.grant.one-matcher", @@ -14640,7 +14651,7 @@ "statement": "One pattern matcher serves derivation legality, view registration, tool visibility and replica object selection.", "why": "P5", "file": "spec/50-authority.md", - "line": 139 + "line": 141 }, { "id": "authority.grant.star", @@ -14651,7 +14662,7 @@ "statement": "`*` is an ordinary grant, bounded by the credential's actions, audience and expiry and visible in the audit record.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 141 + "line": 143 }, { "id": "authority.grant.tenant-scope", @@ -14662,7 +14673,7 @@ "statement": "A tenant scope pairs a table with an opaque byte string bound to that table's outermost partition column.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 143 + "line": 145 }, { "id": "authority.grant.tenant-bytes", @@ -14673,7 +14684,7 @@ "statement": "The grant's tenant value, the partition key on disk and the consumer's tenant identifier are identical bytes, compared by bound-parameter equality with no trimming, case folding, collation or Unicode normalization.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 145 + "line": 147 }, { "id": "authority.grant.tenant-unbindable", @@ -14684,7 +14695,7 @@ "statement": "A tenant-scoped grant on a table declaring no bare outermost partition column raises `GrantTenantUnbindable` at admission.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 147 + "line": 149 }, { "id": "authority.grant.tenant-child-lifetime", @@ -14695,7 +14706,7 @@ "statement": "A tenant-scoped child credential is derived per query with a lifetime of 15 min.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 149 + "line": 151 }, { "id": "authority.grant.held-parent", @@ -14706,7 +14717,7 @@ "statement": "A consumer's service holds one broader parent and derives each per-query child locally. A durable run step derives its child afresh from a parent the run owner holds, never from a persisted child.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 151 + "line": 153 }, { "id": "authority.grant.template-allowlist", @@ -14717,7 +14728,7 @@ "statement": "A template allowlist denies by default: absent authorizes none, `*` authorizes every template the manifest declares, and a child names only identifiers its parent named or covered.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 153 + "line": 155 }, { "id": "authority.grant.template-tables", @@ -14728,7 +14739,7 @@ "statement": "A covered template reads the tables its statement names inside that template's session, under every other layer; its coverage adds no table to a raw read.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 155 + "line": 157 }, { "id": "authority.grant.template-not-allowed", @@ -14739,7 +14750,7 @@ "statement": "Invoking a template outside the allowlist raises `GrantTemplateNotAllowed`, and an uncovered template is absent from the projected tool listing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 157 + "line": 159 }, { "id": "authority.grant.pipeline-resource", @@ -14750,7 +14761,7 @@ "statement": "A pipeline identifier is a read resource: listing it, describing it and tracing its runs each need a read grant covering it. A grant over its landed tables confers none.", "why": "P2", "file": "spec/50-authority.md", - "line": 159 + "line": 161 }, { "id": "authority.grant.pipeline-not-covered", @@ -14761,7 +14772,7 @@ "statement": "Describing an uncovered pipeline raises `GrantPipelineNotCovered`, naming it. Listing filters to covered identifiers and raises it when none is covered; a project declaring no pipelines lists empty.", "why": "P2", "file": "spec/50-authority.md", - "line": 161 + "line": 163 }, { "id": "authority.grant.run-trace-denied", @@ -14772,7 +14783,7 @@ "statement": "Tracing a run gates on the pipeline resolved from the run record and raises `GrantRunTraceDenied` without naming that pipeline.", "why": "P2", "file": "spec/50-authority.md", - "line": 163 + "line": 165 }, { "id": "authority.grant.write-not-covered", @@ -14783,7 +14794,7 @@ "statement": "A credentialed table write whose grants carry no `write` over a destination table raises `GrantWriteNotCovered`, naming the table, before its first effect.", "why": "because a write grant names the tables a principal authors, and a row landed outside them carries a principal who never authorized it", "file": "spec/50-authority.md", - "line": 165 + "line": 167 }, { "id": "authority.grant.row-ceiling", @@ -14794,7 +14805,7 @@ "statement": "A read's row ceiling is the least of the grant's, the request's, the template's, each touched table's published ceiling and {{read.respond.face-ceiling}}; an undeclared component imposes none, and the face ceiling is always declared.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 167 + "line": 169 }, { "id": "authority.grant.aggregate", @@ -14805,7 +14816,7 @@ "statement": "An aggregate grant carries a minimum group size, a maximum single-contributor share, permitted functions, a groups-per-query ceiling and a row ceiling. A write-only or aggregate-only grant contributes no table to a raw read.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 169 + "line": 171 }, { "id": "authority.grant.group-ceiling", @@ -14816,7 +14827,7 @@ "statement": "An aggregate grant's groups-per-query ceiling is at least 1 entries; a query producing more groups than the ceiling is cut at it.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 171 + "line": 173 }, { "id": "authority.attenuate.offline", @@ -14827,7 +14838,7 @@ "statement": "A holder derives a narrower child with no issuer round trip by appending a signed block. The parent's bytes stay unchanged, independently verifiable and independently withdrawable.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 182 + "line": 184 }, { "id": "authority.attenuate.narrowing", @@ -14838,7 +14849,7 @@ "statement": "Every dimension of a child obeys the narrowing table in Shapes, checked by the deriving holder and again across the whole chain at admission.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 184 + "line": 186 }, { "id": "authority.attenuate.widens", @@ -14849,7 +14860,7 @@ "statement": "A child broader than its parent on actions, tables, templates or aggregate constraints raises `AttenuationWidens`, naming the dimension.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 186 + "line": 188 }, { "id": "authority.attenuate.expiry-extended", @@ -14860,7 +14871,7 @@ "statement": "A child whose expiry falls past its parent's raises `AttenuationExpiryExtended`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 188 + "line": 190 }, { "id": "authority.attenuate.tenant-dropped", @@ -14871,7 +14882,7 @@ "statement": "A child dropping its parent's tenant scope or naming another tenant raises `AttenuationTenantDropped`. Adding a scope to an unscoped parent narrows.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 190 + "line": 192 }, { "id": "authority.attenuate.truncation", @@ -14882,7 +14893,7 @@ "statement": "The chain-final proof advances an ephemeral key per block; a chain truncated to a broader prefix verifies as nothing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 192 + "line": 194 }, { "id": "authority.attenuate.bearer", @@ -14893,7 +14904,7 @@ "statement": "Derivation proves the chain and not the new holder's identity. A hop authenticating its new actor runs through the exchange.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 194 + "line": 196 }, { "id": "authority.attenuate.per-sub-agent", @@ -14904,7 +14915,7 @@ "statement": "An agent derives one child per sub-agent, each bound by its confirmation claim to that sub-agent's own key pair.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 196 + "line": 198 }, { "id": "authority.issue.ceiling", @@ -14915,7 +14926,7 @@ "statement": "A project persists in version control an issuance policy naming the default audience and the longest lifetime any mint produces, at most 24 h.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 227 + "line": 229 }, { "id": "authority.issue.above-ceiling", @@ -14926,7 +14937,7 @@ "statement": "An explicitly requested lifetime above the persisted ceiling raises `IssuanceLifetimeAboveCeiling`; an exchange policy's lifetime clamps down to the ceiling instead.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 229 + "line": 231 }, { "id": "authority.issue.ceiling-lowering", @@ -14937,7 +14948,7 @@ "statement": "Lowering the ceiling records the previous value and its instant; rotation-grace validation uses the recorded value until the last credential minted under it lapses.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 231 + "line": 233 }, { "id": "authority.issue.policy-init", @@ -14948,7 +14959,7 @@ "statement": "`contextful token policy init` writes the issuance policy naming its audience with a ceiling of the {{authority.verify.bearer-lifetime}} bound; over an existing policy it raises `IssuancePolicyExists` and writes nothing.", "why": "because a fresh project mints under a policy it commits, and the served faces admit nothing longer-lived by default", "file": "spec/50-authority.md", - "line": 233 + "line": 235 }, { "id": "authority.issue.lower-only", @@ -14959,7 +14970,7 @@ "statement": "`contextful token policy lower-ceiling` naming a value at or above the current ceiling raises `IssuanceCeilingNotLowered` and leaves the policy unchanged.", "why": "because a raised ceiling mints credentials outliving the grace windows validated against the lower one", "file": "spec/50-authority.md", - "line": 235 + "line": 237 }, { "id": "authority.issue.out-of-tree-mint", @@ -14970,7 +14981,7 @@ "statement": "Ceiling enforcement binds a mint run inside the project tree; a mint run outside it is bounded at the checkpoint by rotation-grace validation.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 237 + "line": 239 }, { "id": "authority.issue.principal-required", @@ -14981,7 +14992,7 @@ "statement": "A mint granting `write` or `execute` whose subject names no `on_behalf_of` raises `IssuancePrincipalRequired`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 239 + "line": 241 }, { "id": "authority.issue.default-read", @@ -14992,7 +15003,7 @@ "statement": "A mint's default action set is `read` alone.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 241 + "line": 243 }, { "id": "authority.issue.absent-author", @@ -15003,7 +15014,7 @@ "statement": "An absent authorship value on a landed row means no credential authorized the write: the uncredentialed owner or a timer-fired run.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 243 + "line": 245 }, { "id": "authority.issue.authoring-posture", @@ -15014,7 +15025,7 @@ "statement": "Every table write verb {{authority.verify.write-verbs}} runs under its manifest's authoring posture. `session` authors every write by one verified ambient credential's principal; `per_request` holds no ambient principal, leaving an unaccompanied write unauthored.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 245 + "line": 247 }, { "id": "authority.issue.posture-key", @@ -15025,7 +15036,7 @@ "statement": "A manifest declares its posture as a top-level `authoring_posture` of `session` or `per_request`. A table write verb reading a manifest whose key is absent or names another value raises `AuthoringPostureUndeclared` before its first effect.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 247 + "line": 249 }, { "id": "authority.issue.session-credential", @@ -15036,7 +15047,7 @@ "statement": "Under `session`, a table write verb run with no credential raises `AuthoringCredentialMissing` before its first effect.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 249 + "line": 251 }, { "id": "authority.issue.one-credential", @@ -15047,7 +15058,7 @@ "statement": "A served face admits a verified capability credential, presented as a bearer or bound to a holder key, and nothing else: no static bearer, no gateway shared secret, no unauthenticated owner path.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 251 + "line": 253 }, { "id": "authority.issue.missing-key", @@ -15058,7 +15069,7 @@ "statement": "A face configured with no issuer key raises `IssuerKeyMissing`, prints the command that fixes it, and binds no port.", "why": "P3", "file": "spec/50-authority.md", - "line": 253 + "line": 255 }, { "id": "authority.issue.default-key", @@ -15069,7 +15080,7 @@ "statement": "A mint naming no issuer key reads `.contextful/issuer.seed`; no file there raises {{authority.issue.missing-key}}.", "why": "because the seed `contextful token keygen` writes by default is the one a mint in the same tree reads", "file": "spec/50-authority.md", - "line": 255 + "line": 257 }, { "id": "authority.issue.project-root", @@ -15080,7 +15091,7 @@ "statement": "The issuance policy, the default issuer seed and the key-set ledger sit under the directory {{store.init.discovery}} bases project paths on; with no `contextful.toml` upward, under the working directory.", "why": "because a ledger read from a subdirectory hides every retirement and epoch bump the root records", "file": "spec/50-authority.md", - "line": 257 + "line": 259 }, { "id": "authority.issue.unresolvable-key", @@ -15091,7 +15102,7 @@ "statement": "A set issuer key reference that resolves to no material raises `IssuerKeyUnresolvable`; no local key is fabricated.", "why": "P3", "file": "spec/50-authority.md", - "line": 259 + "line": 261 }, { "id": "authority.issue.signing-port", @@ -15102,7 +15113,7 @@ "statement": "One signing port signs every credential's authority block and every audit root and tip; no mint path reads a private key. Seed files, secret references resolved at mint time and remote signing oracles are its adapters.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 261 + "line": 263 }, { "id": "authority.issue.oracle-custody", @@ -15113,7 +15124,7 @@ "statement": "Under the oracle adapter the seed stays inside the custodian, which records each mint as a signing call.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 263 + "line": 265 }, { "id": "authority.issue.signature-encoding", @@ -15124,7 +15135,7 @@ "statement": "A signing port names its encoding: `ed25519`, a 32-byte key and 64-byte signature; `es256-der` or `es256-raw`, a SEC1 P-256 point signing ECDSA over SHA-256 of the message as ASN.1 DER or 64-byte `r ‖ s`.", "why": "because an enclave, a cloud KMS and a browser key each answer ES256 in one encoding, and the tag lets every adapter answer in its own", "file": "spec/50-authority.md", - "line": 265 + "line": 267 }, { "id": "authority.issue.der-at-the-edge", @@ -15135,7 +15146,7 @@ "statement": "An `es256-raw` signature converts to DER where it leaves the port; a credential and an audit root or tip carry ES256 only as DER.", "why": "because the credential format verifies ES256 only in DER, and one stored encoding holds a verifier to one decoder", "file": "spec/50-authority.md", - "line": 267 + "line": 269 }, { "id": "authority.issue.encoding-invalid", @@ -15146,7 +15157,7 @@ "statement": "A port signature that does not decode under its encoding tag, or does not verify under the port's public key, raises `SignatureEncodingInvalid`; nothing is minted and no root or tip is written.", "why": "because a mislabelled signature yields a credential every checkpoint rejects, and refusing at the port names the adapter at fault", "file": "spec/50-authority.md", - "line": 269 + "line": 271 }, { "id": "authority.issue.public-key-text", @@ -15157,7 +15168,7 @@ "statement": "A port's public key prints as `ed25519:` or `es256:` of its SEC1 point; either form, with a compressed or uncompressed point, parses back to the key it names.", "why": "", "file": "spec/50-authority.md", - "line": 271 + "line": 273 }, { "id": "authority.issue.algorithm", @@ -15168,7 +15179,7 @@ "statement": "A credential names Ed25519, the default, or ECDSA over P-256; the pinned key's scheme is authoritative.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 272 + "line": 274 }, { "id": "authority.issue.algorithm-mismatch", @@ -15179,7 +15190,7 @@ "statement": "A credential naming a scheme other than its pinned key's raises `SignatureAlgorithmMismatch`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 274 + "line": 276 }, { "id": "authority.issue.unauthorized-mint", @@ -15190,7 +15201,7 @@ "statement": "A mint request presenting no grant carrying `admin` raises `IssuanceUnauthorized`.", "why": "because issuance authorizes itself with the same primitive as every other action, and a separate mint secret is a second perimeter to guard", "file": "spec/50-authority.md", - "line": 276 + "line": 278 }, { "id": "authority.issue.replica-mint", @@ -15201,7 +15212,7 @@ "statement": "A replica holds the issuer's public key and no signing material; a mint there raises `ReplicaCannotIssue`.", "why": "because a replica verifies every credential the primary mints and needs no power to produce one", "file": "spec/50-authority.md", - "line": 278 + "line": 280 }, { "id": "authority.issue.zone-wildcard", @@ -15212,7 +15223,7 @@ "statement": "A mint whose subject declares a wildcard inference zone raises `IssuanceZoneWildcard`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 280 + "line": 282 }, { "id": "authority.issue.key-rotation", @@ -15223,7 +15234,7 @@ "statement": "The issuer signing key rotates every 90 d, and at once on suspected compromise.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 282 + "line": 284 }, { "id": "authority.issue.rotation-not-due", @@ -15234,7 +15245,7 @@ "statement": "A rotation without `--compromise` before {{authority.issue.key-rotation}} falls due raises `IssuerKeyRotationNotDue`, naming the instant it falls due.", "why": "because a scheduled rotation ahead of its cadence churns every verifier's pins with nothing withdrawn", "file": "spec/50-authority.md", - "line": 284 + "line": 286 }, { "id": "authority.issue.unrecorded-key", @@ -15245,7 +15256,7 @@ "statement": "A rotation without `--compromise` of an issuer key the key-set ledger does not date raises `IssuerKeyUnrecorded`; `contextful token record --since` dates it.", "why": "because an undated key otherwise rotates at once, skipping the cadence the ledger enforces", "file": "spec/50-authority.md", - "line": 286 + "line": 288 }, { "id": "authority.issue.bearer-cap-fixed", @@ -15256,7 +15267,7 @@ "statement": "No issuance policy, face or command setting raises the cap {{authority.verify.bearer-lifetime}} sets; a credential living longer admits over a network only bound to a holder key.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 288 + "line": 290 }, { "id": "authority.verify.admitted-authority", @@ -15267,7 +15278,7 @@ "statement": "Verification yields an admitted-authority value carrying the normalized subject tuple and its grants. Every read surface and row-landing effect takes that value as an argument; nothing downstream re-parses a credential or reads ambient state.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 297 + "line": 299 }, { "id": "authority.verify.write-verbs", @@ -15278,7 +15289,7 @@ "statement": "`context land`, `run start` and `pipeline run` are the table write verbs. Each admits its accompanying credential before its first effect and stamps that credential's `on_behalf_of` as every landed row's `_authored_by`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 299 + "line": 301 }, { "id": "authority.verify.write-commit", @@ -15289,7 +15300,7 @@ "statement": "A credentialed table write reads its denylist afresh and checks expiry immediately before each commit; a credential revoked or lapsed since admission commits nothing, as {{authority.revoke.revoked}} and {{authority.verify.expired}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 301 + "line": 303 }, { "id": "authority.verify.no-bypass-constructor", @@ -15300,7 +15311,7 @@ "statement": "The admitted-authority type has no constructor that produces a value without a verification.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 303 + "line": 305 }, { "id": "authority.verify.effect-boundary", @@ -15311,7 +15322,7 @@ "statement": "A statement's start and each commit are effect boundaries. Each boundary re-reads expiry, revocation and policy version against the carried value; a lapse between boundaries stops the effect at the next one.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 305 + "line": 307 }, { "id": "authority.verify.format-interface", @@ -15322,7 +15333,7 @@ "statement": "The credential format sits behind one interface of issue, attenuate, verify and introspect; no enforcement call site names a credential type.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 307 + "line": 309 }, { "id": "authority.verify.bad-signature", @@ -15333,7 +15344,7 @@ "statement": "A credential with any block signature failing against a pinned key raises `SignatureInvalid` and admits nothing, no verified prefix included.", "why": "because a partially verified chain is an unauthenticated credential, and admitting its prefix hands out authority nobody signed as presented", "file": "spec/50-authority.md", - "line": 309 + "line": 311 }, { "id": "authority.verify.audience-mismatch", @@ -15344,7 +15355,7 @@ "statement": "A checkpoint declaring an expected audience raises `AudienceMismatch` for a credential with a different audience or none; a checkpoint declaring none performs no audience check.", "why": "because a credential minted for one store is otherwise replayable against another; the undeclared case is local development", "file": "spec/50-authority.md", - "line": 311 + "line": 313 }, { "id": "authority.verify.timestamps", @@ -15355,7 +15366,7 @@ "statement": "Every checkpoint decodes timestamps into UTC instants under one grammar and compares instants.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 313 + "line": 315 }, { "id": "authority.verify.malformed-timestamp", @@ -15366,7 +15377,7 @@ "statement": "A timestamp that does not decode under that grammar raises `TimestampMalformed` at every checkpoint alike.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 315 + "line": 317 }, { "id": "authority.verify.expired", @@ -15377,7 +15388,7 @@ "statement": "A credential whose expiry precedes the evaluation instant raises `AuthorityExpired` at admission and at each later effect boundary.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 317 + "line": 319 }, { "id": "authority.verify.possession-binding", @@ -15388,7 +15399,7 @@ "statement": "A confirmation claim holds a client public-key thumbprint. Each request under a credential carrying one bears a proof signed by the matching private key over method, target, body digest, issue instant and nonce; a credential carrying none requires no proof.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 319 + "line": 321 }, { "id": "authority.verify.possession-invalid", @@ -15399,7 +15410,7 @@ "statement": "A request whose proof does not verify against the confirmation thumbprint raises `PossessionProofInvalid`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 321 + "line": 323 }, { "id": "authority.verify.replay-window", @@ -15410,7 +15421,7 @@ "statement": "A proof issued more than 300 s before the checkpoint clock is refused as {{authority.verify.possession-invalid}}; the nonce cache retains each nonce for that window.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 323 + "line": 325 }, { "id": "authority.verify.clock-skew", @@ -15421,7 +15432,7 @@ "statement": "A proof issued more than 30 s after the checkpoint clock is refused as {{authority.verify.possession-invalid}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 325 + "line": 327 }, { "id": "authority.verify.replayed-nonce", @@ -15432,7 +15443,7 @@ "statement": "A proof whose nonce repeats inside the replay window raises `PossessionProofReplayed`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 327 + "line": 329 }, { "id": "authority.verify.local-transport", @@ -15443,7 +15454,7 @@ "statement": "A local transport is a stdio pipe the checkpoint inherited from the process that spawned it, or a Unix socket; every other transport is a network transport.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 329 + "line": 331 }, { "id": "authority.verify.local-holder-proof", @@ -15454,7 +15465,7 @@ "statement": "On a local transport, a credential whose confirmation claim holds a thumbprint admits only with a proof as {{authority.verify.possession-binding}}; the peer fallback never applies to it, and a missing or failing proof is refused as {{authority.verify.possession-invalid}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 331 + "line": 333 }, { "id": "authority.verify.local-proof-channel", @@ -15465,7 +15476,7 @@ "statement": "A credentialed command-line verb and the stdio tool server sign one holder proof at admission with the Ed25519 seed in `--holder-key`, else take the proof in `CONTEXTFUL_DPOP`, else sign with the seed in `CONTEXTFUL_HOLDER_KEY`; a blank variable is unset.", "why": "because a bound credential is otherwise unusable off the network face, and a proof on the argument list is readable through the process table", "file": "spec/50-authority.md", - "line": 333 + "line": 335 }, { "id": "authority.verify.local-proof-request", @@ -15476,7 +15487,7 @@ "statement": "A local proof covers method `CLI`, the verb's command path as target, such as `context land`, `memory write` or `mcp`, and an empty body.", "why": "because binding the verb keeps a proof minted for a read from admitting a write", "file": "spec/50-authority.md", - "line": 335 + "line": 337 }, { "id": "authority.verify.local-nonce-store", @@ -15487,7 +15498,7 @@ "statement": "A command-line verb checks its holder proof against the nonce store `.contextful/proof-nonces` under the project root, shared by every invocation on that project under an exclusive lock, so a repeated nonce raises {{authority.verify.replayed-nonce}}.", "why": "because each invocation is a fresh process, and a per-process cache lets a proof copied from the environment replay for the whole window", "file": "spec/50-authority.md", - "line": 337 + "line": 339 }, { "id": "authority.verify.holder-keygen", @@ -15498,7 +15509,7 @@ "statement": "`token keygen --holder` writes an Ed25519 holder seed and prints its thumbprint, the value `token mint --holder` binds as the confirmation claim.", "why": "", "file": "spec/50-authority.md", - "line": 339 + "line": 341 }, { "id": "authority.verify.local-peer-fallback", @@ -15509,7 +15520,7 @@ "statement": "On a local transport, a credential with no confirmation claim admits only through the operating system's peer authentication: an inherited stdio pipe, or a socket peer whose kernel-reported uid equals the checkpoint process's uid.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 340 + "line": 342 }, { "id": "authority.verify.peer-mismatch", @@ -15520,7 +15531,7 @@ "statement": "A socket peer presenting a credential with no confirmation claim, whose kernel-reported uid differs from the checkpoint process's uid or which the platform cannot report, raises `TransportPeerMismatch` and admits nothing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 342 + "line": 344 }, { "id": "authority.verify.network-bearer", @@ -15531,7 +15542,7 @@ "statement": "A network checkpoint admits a credential with no confirmation claim as a bearer when it names the checkpoint's declared audience and meets {{authority.verify.bearer-lifetime}}; the peer fallback never applies over a network.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 344 + "line": 346 }, { "id": "authority.verify.bearer-lifetime", @@ -15542,7 +15553,7 @@ "statement": "A bearer whose expiry falls more than 3600 s after its issue instant raises `BearerLifetimeExceeded` at a network checkpoint and admits nothing; its holder refreshes through {{authority.exchange.surface}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 346 + "line": 348 }, { "id": "authority.verify.connection-scoped", @@ -15553,7 +15564,7 @@ "statement": "A local admission binds to the pipe or socket connection that presented the credential; a request on any other connection is refused as {{authority.verify.peer-mismatch}} until that connection presents the credential and admits itself.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 348 + "line": 350 }, { "id": "authority.verify.nonce-cache", @@ -15564,7 +15575,7 @@ "statement": "A checkpoint's nonce cache holds at most 100000 entries; a proof arriving while it is full answers `503` and admits nothing.", "why": "because a proof the cache cannot record is a replay it cannot detect", "file": "spec/50-authority.md", - "line": 350 + "line": 352 }, { "id": "authority.verify.public-key-only", @@ -15575,7 +15586,7 @@ "statement": "A checkpoint holds public key material and no signing secret, and admits with no call to an issuer, identity provider or policy service.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 352 + "line": 354 }, { "id": "authority.verify.key-set", @@ -15586,7 +15597,7 @@ "statement": "A checkpoint accepts a set of issuer keys: comma-separated static pins, or a project-published unauthenticated key route returning the current and non-retired versions, opted into per key.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 354 + "line": 356 }, { "id": "authority.verify.pin-source", @@ -15597,7 +15608,7 @@ "statement": "A command-line checkpoint reads its pins from `--public-key`, else `CONTEXTFUL_ISSUER_PUBKEY`, and its audience from `--audience`, else `CONTEXTFUL_AUDIENCE`. A bare 64-hex pin reads as `ed25519/`; an unknown scheme prefix is refused as {{authority.verify.key-set-unavailable}}, naming the accepted forms.", "why": "because an operator exports the issuer key once per shell, and key tools commonly print a bare Ed25519 key with no scheme prefix", "file": "spec/50-authority.md", - "line": 356 + "line": 358 }, { "id": "authority.verify.key-set-refresh", @@ -15608,7 +15619,7 @@ "statement": "A published key set refreshes every 300 s, single-flight, plus one refresh on a signature failure, with failure-forced refreshes throttled.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 358 + "line": 360 }, { "id": "authority.verify.key-set-unavailable", @@ -15619,7 +15630,7 @@ "statement": "A checkpoint opted into a published key set that obtains none, or holding a malformed pin, raises `KeySetUnavailable`: the engine declines to start and a gateway answers `503`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 360 + "line": 362 }, { "id": "authority.verify.key-set-stale", @@ -15630,7 +15641,7 @@ "statement": "After a failed fetch the last-known-good set serves while its age since the last successful fetch stays under 1 h; past that the checkpoint raises `KeySetStale`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 362 + "line": 364 }, { "id": "authority.revoke.short-lifetime", @@ -15641,7 +15652,7 @@ "statement": "A short lifetime is the primary withdrawal: a credential's remaining lifetime bounds its exposure after a provider sign-out or before a denylist entry propagates.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 371 + "line": 373 }, { "id": "authority.revoke.denylist", @@ -15652,7 +15663,7 @@ "statement": "A checkpoint reads a denylist keyed on credential identifier. An entry ages out once its identifier verifies under no live key version.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 373 + "line": 375 }, { "id": "authority.revoke.revocation-id", @@ -15663,7 +15674,7 @@ "statement": "Every derivation carries its own revocation identifier, withdrawing one subtree without ending the root it descends from.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 375 + "line": 377 }, { "id": "authority.revoke.epoch", @@ -15674,7 +15685,7 @@ "statement": "A revocation epoch scoped on project, tenant and principal class invalidates that slice of outstanding authority.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 377 + "line": 379 }, { "id": "authority.revoke.epoch-store", @@ -15685,7 +15696,7 @@ "statement": "A project persists each scope's current epoch and each issuer key version's retirement in `.contextful/keyset.toml` beside the issuer seed; a mint stamps its scope's current epoch, and a checkpoint re-reads the file at each admission.", "why": "because a file the checkpoint reads locally keeps a policy service off the admission path, and the seed's directory already holds the issuer's custody", "file": "spec/50-authority.md", - "line": 379 + "line": 381 }, { "id": "authority.revoke.ledger-unavailable", @@ -15696,7 +15707,7 @@ "statement": "A checkpoint pointed at a key-set ledger, or that has read one, raises `KeySetLedgerUnavailable` and admits nothing once the file is absent or malformed.", "why": "because a vanished ledger read as empty lifts every key retirement and epoch bump", "file": "spec/50-authority.md", - "line": 381 + "line": 383 }, { "id": "authority.revoke.unknown-principal-class", @@ -15707,7 +15718,7 @@ "statement": "`contextful token revoke --principal-class` naming neither `delegated` nor `unattributed` raises `RevocationPrincipalClassUnknown` and bumps nothing.", "why": "because an epoch under a class no credential carries withdraws nothing while reporting success", "file": "spec/50-authority.md", - "line": 383 + "line": 385 }, { "id": "authority.revoke.revoked", @@ -15718,7 +15729,7 @@ "statement": "A credential on the denylist, or carrying an epoch below the current scoped epoch, raises `AuthorityRevoked` at the next effect boundary.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 385 + "line": 387 }, { "id": "authority.revoke.compromise", @@ -15729,7 +15740,7 @@ "statement": "Suspected compromise of signing material runs a project-wide epoch bump together with immediate retirement of the key version. Waiting out a grace window withdraws nothing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 387 + "line": 389 }, { "id": "authority.revoke.rotation-grace", @@ -15740,7 +15751,7 @@ "statement": "A rotation policy declares a cadence and a grace window, validated against the persisted issuance policy, during which the retiring key version still verifies.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 389 + "line": 391 }, { "id": "authority.revoke.short-grace", @@ -15751,7 +15762,7 @@ "statement": "A declared or overridden grace window shorter than the effective issuance ceiling raises `RotationGraceTooShort`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 391 + "line": 393 }, { "id": "authority.revoke.immediate-retire", @@ -15762,7 +15773,7 @@ "statement": "Immediate retirement removes a key version from the published set and every static pin; a checkpoint drops it at its next refresh.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 393 + "line": 395 }, { "id": "authority.revoke.format-withdrawn", @@ -15773,7 +15784,7 @@ "statement": "From a declared cutover instant, every admission path raises `CredentialFormatWithdrawn` for a credential in a withdrawn format. Only an explicit declaration restores acceptance.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 395 + "line": 397 }, { "id": "authority.exchange.surface", @@ -15784,7 +15795,7 @@ "statement": "`contextful token exchange` and `POST /auth/exchange` trade a verified external assertion for a capability credential, configured per project by `.contextful/exchange/policy.toml` and `.contextful/exchange/verify.key`. The command reads the assertion from `--jwt` or standard input, and a holder proof from `--dpop`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 416 + "line": 418 }, { "id": "authority.exchange.wire", @@ -15795,7 +15806,7 @@ "statement": "An exchange request body is the JSON object `{\"jwt\": \"\"}`, which the command line builds from its assertion; a body of any other shape raises `ExchangeRequestMalformed` and mints nothing. A served face answers a mint `{\"token\": \"\"}`.", "why": "because the command line and every served face parse one body through one function, so a refusal reads the same on both", "file": "spec/50-authority.md", - "line": 418 + "line": 420 }, { "id": "authority.exchange.body-ceiling", @@ -15806,7 +15817,7 @@ "statement": "An exchange request body holds at most 64 KiB; a longer body raises `ExchangeBodyTooLarge`, mints nothing and is not parsed.", "why": "because an assertion is a few KiB, and parsing an unbounded body is work an unauthenticated caller spends", "file": "spec/50-authority.md", - "line": 420 + "line": 422 }, { "id": "authority.exchange.unconfigured", @@ -15817,7 +15828,7 @@ "statement": "An exchange in a project declaring no exchange policy raises `ExchangeUnconfigured`, naming `.contextful/exchange/policy.toml`, before the body is read.", "why": "because an unconfigured project exposes no exchange, whatever a caller sends", "file": "spec/50-authority.md", - "line": 422 + "line": 424 }, { "id": "authority.exchange.status-map", @@ -15828,7 +15839,7 @@ "statement": "Over HTTP, {{authority.exchange.unconfigured}} answers 404, {{authority.exchange.wire}} and {{authority.exchange.holder-proof-invalid}} 400, {{authority.exchange.body-ceiling}} 413, {{authority.exchange.material-missing}} 503, {{authority.exchange.assertion-invalid}} 401, and any other refusal 403; every refusal body is `{\"error\": {\"http\", \"identifier\", \"message\"}}`.", "why": "because a caller repairs a 400 or 413, retries a 503, re-authenticates on a 401, and repairs nothing on a 403", "file": "spec/50-authority.md", - "line": 424 + "line": 426 }, { "id": "authority.exchange.signing-fault", @@ -15839,7 +15850,7 @@ "statement": "Over HTTP, {{authority.issue.missing-key}}, {{authority.issue.unresolvable-key}}, {{authority.issue.algorithm-mismatch}} or {{authority.issue.replica-mint}} during an exchange answers 500, the body naming the identifier and withholding the refusal's detail.", "why": "because the face's operator repairs its configuration, and the detail describes the face's keys to an unauthenticated caller", "file": "spec/50-authority.md", - "line": 426 + "line": 428 }, { "id": "authority.exchange.policy", @@ -15850,7 +15861,7 @@ "statement": "An exchange policy declares `expected_iss`, optional `expected_aud`, `subject_map`, `tenant_claim`, `role_claim`, `role_grants`, `default_grants`, `ttl_secs`, `minted_iss` and `minted_aud`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 428 + "line": 430 }, { "id": "authority.exchange.injected-material", @@ -15861,7 +15872,7 @@ "statement": "Verifying material is operator-injected: a shared secret, an RS256 public key in PEM form, or a key-set document selected by the assertion's `kid`. The exchange makes no network call.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 430 + "line": 432 }, { "id": "authority.exchange.minted-grants", @@ -15872,7 +15883,7 @@ "statement": "Minted grants come from `role_grants` and `default_grants` alone. A verified role matching no entry earns `default_grants`, which is empty unless declared.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 432 + "line": 434 }, { "id": "authority.exchange.tenant", @@ -15883,7 +15894,7 @@ "statement": "The assertion claim `tenant_claim` names becomes the tenant scope of every minted grant; no subject member carries tenancy.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 434 + "line": 436 }, { "id": "authority.exchange.lifetime-default", @@ -15894,7 +15905,7 @@ "statement": "A minted credential lives 900 s where the policy declares no `ttl_secs`.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 436 + "line": 438 }, { "id": "authority.exchange.lifetime-ceiling", @@ -15905,7 +15916,7 @@ "statement": "A configured `ttl_secs` clamps down to 3600 s and never extends.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 438 + "line": 440 }, { "id": "authority.exchange.audience", @@ -15916,7 +15927,7 @@ "statement": "A minted credential carries `minted_aud`, falling back to the project's persisted default audience.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 440 + "line": 442 }, { "id": "authority.exchange.assertion-invalid", @@ -15927,7 +15938,7 @@ "statement": "A bad signature, a lapsed assertion, an untrusted issuer or audience, or a mapped claim absent from the assertion raises `ExchangeAssertionInvalid` and mints nothing.", "why": "because no credential exists whose subject the assertion did not verify", "file": "spec/50-authority.md", - "line": 442 + "line": 444 }, { "id": "authority.exchange.material-missing", @@ -15938,7 +15949,7 @@ "statement": "An exchange configured with no verifying material raises `ExchangeMaterialMissing` and mints nothing.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 444 + "line": 446 }, { "id": "authority.exchange.per-reader", @@ -15949,7 +15960,7 @@ "statement": "An embedding application holds no project-wide credential; per request it exchanges its signed-in reader's assertion for that reader's short-lived credential and reads as them.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 446 + "line": 448 }, { "id": "authority.exchange.no-cross-reader", @@ -15960,7 +15971,7 @@ "statement": "An exchanged credential serves only the reader it was minted for.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 448 + "line": 450 }, { "id": "authority.exchange.holder-binding", @@ -15971,7 +15982,7 @@ "statement": "An exchange request carrying a `DPoP` proof over `POST /auth/exchange` and its body mints a credential whose `cnf.jkt` is the proof key's thumbprint, the proof checked as {{authority.verify.possession-binding}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 450 + "line": 452 }, { "id": "authority.exchange.bearer-mint", @@ -15982,7 +15993,7 @@ "statement": "An exchange request carrying no proof mints a credential with no confirmation claim, its lifetime clamped by {{authority.exchange.lifetime-ceiling}}, so a network face admits it under {{authority.verify.bearer-lifetime}}.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 452 + "line": 454 }, { "id": "authority.exchange.holder-proof-invalid", @@ -15993,7 +16004,7 @@ "statement": "An exchange request whose proof fails any check of {{authority.exchange.holder-binding}}, a replayed nonce included, raises `ExchangeHolderProofInvalid` and mints nothing.", "why": "because a proof the face cannot verify proves no key, and a bearer minted in its place hands replayable bytes to a caller asking for a bound credential", "file": "spec/50-authority.md", - "line": 454 + "line": 456 }, { "id": "authority.exchange.refresh", @@ -16004,7 +16015,7 @@ "statement": "A holder renews a served-face credential by exchanging a fresh assertion for a new credential with its own identifier and a full lifetime from the exchange instant.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 456 + "line": 458 }, { "id": "authority.exchange.no-extension", @@ -16015,7 +16026,7 @@ "statement": "No surface extends a minted credential's expiry: a minted credential presented as the exchange assertion raises {{authority.exchange.assertion-invalid}}, and {{authority.attenuate.expiry-extended}} bounds every derived child.", "why": "A-authority", "file": "spec/50-authority.md", - "line": 458 + "line": 460 }, { "id": "authority.redact.write-time", @@ -19338,7 +19349,7 @@ "operation": "package", "subject": "stdio-credential", "kind": "refusal", - "statement": "Over the process transport a credential is mandatory, a capability token or an explicit owner flag; an unset one raises `StdioCredentialMissing` and does not resolve to the owner context.", + "statement": "The process transport requires a verified capability token; `--owner` selects its signed owner claim. An absent token raises `StdioCredentialMissing` before protocol framing and never resolves to owner context.", "why": "A-read", "file": "spec/71-console.md", "line": 148 @@ -19359,8 +19370,8 @@ "contract": "surface", "operation": "package", "subject": "owner-flag", - "kind": "behavior", - "statement": "An explicit owner flag over the process transport admits the local owner context only with the store's owner credential.", + "kind": "refusal", + "statement": "A process `--owner` admits the local owner context only with a verified signed claim bound to the selected store and an unrestricted read grant; any other credential raises `OwnerCredentialInvalid` before protocol framing.", "why": "A-surface", "file": "spec/71-console.md", "line": 152 @@ -28487,6 +28498,10 @@ "clause": "surface.apply.version-race", "gloss": "An apply whose pointer compare-and-swap lost to a concurrent apply." }, + "OwnerCredentialInvalid": { + "clause": "surface.package.owner-flag", + "gloss": "A process owner flag with no signed claim for the selected local store." + }, "ResidencySitesDiverge": { "clause": "surface.reside.site-regions", "gloss": "A push whose residency allow-set differs from the one another site recorded in the bucket manifest." diff --git a/spec/status.md b/spec/status.md index 4991c91f7..f072857ad 100644 --- a/spec/status.md +++ b/spec/status.md @@ -6,16 +6,16 @@ An unpinned clause is `committed`; a pinned one is `performed` when its test res | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `assurance` | 2 | 15 | 227 | 59 | 22 | 12 | 135 | 0 | 135 | -| `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | +| `authority` | 2 | 16 | 268 | 77 | 27 | 10 | 204 | 0 | 204 | | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 207 | | `corpus` | 1 | 10 | 77 | 37 | 7 | 0 | 38 | 0 | 38 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 51 | 0 | 51 | +| `surface` | 2 | 20 | 106 | 50 | 22 | 0 | 51 | 0 | 51 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2007 | 581 | 188 | 127 | 1362 | 0 | | +| **total** | 19 | 155 | 2008 | 582 | 188 | 127 | 1362 | 0 | | Decision records: 18. @@ -26,7 +26,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | Milestone | Operations | Clauses | Performed | Acceptance | Closed | | --- | --- | --- | --- | --- | --- | | 0 — The test-first gate | 2 | 18 | 6 | passing | open | -| 1 — The authority core | 14 | 236 | 184 | passing | closed | +| 1 — The authority core | 14 | 237 | 184 | passing | closed | | 2 — The store | 9 | 164 | 132 | passing | closed | | 3 — The run path | 11 | 207 | 144 | passing | closed | | 4 — Ingest | 25 | 496 | 313 | passing | open | From b18352b79d7e245b462eb48fe91fad9c44bc7695 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:13:16 +0800 Subject: [PATCH 045/155] test(console): require server views, private answers, and brief route --- apps/console/test/console.test.ts | 67 +++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 26da13e41..12046de0e 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -297,3 +297,70 @@ test("Admin pack listing reaches only the injected listing adapter", async () => assert.deepEqual(await response.json(), { entries: ["packs/a.toml"], truncated: false, declined: 2 }); assert.deepEqual(calls, ["field-notes:packs/"]); }); + +test("Query builds its widget from rows and sanitizes view props before delivery", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ + answer: "One account arrived.", sources: [], + widgets: [{ component: "injected.v1", props: { value: "secret@example.test" } }], + resultRows: { columns: ["account"], rows: [["secret@example.test"]] }, + }), + redactView: (_operator, value) => value.replace("secret@example.test", "[redacted]"), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { ...access(queryAudience), origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", question: "What arrived?" }), + })); + assert.equal(response.status, 200); + const answer = await response.json() as { widgets: Array<{ component: string; props: { rows: unknown[][] } }> }; + assert.equal(answer.widgets.length, 1); + assert.equal(answer.widgets[0].component, "table.v1"); + assert.deepEqual(answer.widgets[0].props.rows, [["[redacted]"]]); +}); + +test("Query refuses a client view and an access-explanation share control", async () => { + let turns = 0; + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => { turns++; return { answer: "Private rationale", sources: [], widgets: [], accessExplanation: true, share: true }; }, + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const ask = (extra: Record = {}) => app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { ...access(queryAudience), origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", question: "Why?", ...extra }), + })); + const authored = await ask({ view: { component: "metric.v1" } }); + assert.equal(authored.status, 400); + assert.equal((await authored.json() as { error: { identifier: string } }).error.identifier, "ConsoleViewNotServerBuilt"); + assert.equal(turns, 0); + const shared = await ask(); + assert.equal(shared.status, 403); + assert.equal((await shared.json() as { error: { identifier: string } }).error.identifier, "VisibilityShareAffordance"); +}); + +test("Query brief derives a private card only from a turnless present-time session", async () => { + const now = Date.parse("2026-01-08T12:00:00Z"); + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + brief: async () => ({ session: { turns: 0, vantage: "present" }, now, + conclusions: [{ subject: "Acme", text: "Acme filings need review", live: true }], + arrivals: [{ id: "a", label: "Acme filing", topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }], + }), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const url = "https://console.example/query/api/brief?store=field-notes"; + assert.equal((await app.fetch(new Request(url))).status, 401); + const response = await app.fetch(new Request(url, { headers: access(queryAudience) })); + assert.equal(response.status, 200); + assert.equal((await response.json() as { subjects: Array<{ subject: string }> }).subjects[0].subject, "Acme"); + assert.equal((await app.fetch(new Request("https://console.example/query/api/brief?store=missing", { headers: access(queryAudience) }))).status, 400); +}); From 1a41bff646a2f547e6e49d41cfdb388bfdcfdb33 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:32:34 +0800 Subject: [PATCH 046/155] test(console): specify grounded turn policy --- apps/console/test/turn.test.ts | 125 +++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 apps/console/test/turn.test.ts diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts new file mode 100644 index 000000000..d1e929d78 --- /dev/null +++ b/apps/console/test/turn.test.ts @@ -0,0 +1,125 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + ConsoleError, + OverlayCache, + StreamingRedactor, + createTurn, + parseVantage, + validatePack, +} from "../src/turn.ts"; + +const readTool = { name: "query", pack: "data", kind: "read" as const }; + +function harness(overrides: Record = {}) { + const calls: string[] = []; + const turn = createTurn({ + tools: [readTool], + planner: async () => [{ tool: "query", arguments: { question: "count" } }], + transport: { + call: async (call: { tool: string }) => { + calls.push(call.tool); + return { rows: [{ count: 3 }], sources: [{ id: "row-1", label: "Count", url: "https://example.test/1" }] }; + }, + }, + synthesize: async function* () { yield "Three rows."; }, + ...overrides, + }); + return { turn, calls }; +} + +test("a grounded turn cites only governed tool results", async () => { + const { turn, calls } = harness(); + const answer = await turn.ask({ question: "How many?", packs: ["data"] }); + assert.deepEqual(calls, ["query"]); + assert.match(answer.text, /Three rows\./); + assert.match(answer.text, /Sources\n- Count/); + assert.deepEqual(answer.sources.map((source: { id: string }) => source.id), ["row-1"]); +}); + +test("a turn with no tool result refuses model prose", async () => { + const { turn } = harness({ + transport: { call: async () => ({ rows: [], sources: [] }) }, + }); + await assert.rejects(turn.ask({ question: "Guess", packs: ["data"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleUngroundedAnswer"); +}); + +test("unadmitted and mutating calls dispatch nothing", async () => { + const { turn, calls } = harness({ + planner: async () => [{ tool: "query", arguments: {} }], + }); + await assert.rejects(turn.ask({ question: "Read", packs: [] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleToolNotAdmitted"); + assert.deepEqual(calls, []); + const write = harness({ tools: [{ name: "erase", pack: "data", kind: "write" }], planner: async () => [{ tool: "erase", arguments: {} }] }); + await assert.rejects(write.turn.ask({ question: "Erase", packs: ["data"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleMutatingToolRequested"); + assert.deepEqual(write.calls, []); +}); + +test("organization packs refuse writes at startup", () => { + assert.throws(() => validatePack({ name: "shared", face: "organization", tools: [{ name: "erase", kind: "write" }] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleWriteToolOnOrgFace"); +}); + +test("direct file table functions refuse before dispatch", async () => { + const { turn, calls } = harness({ tools: [{ name: "query", pack: "data", kind: "read", access: "direct-file" }] }); + await assert.rejects(turn.ask({ question: "Read", packs: ["data"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleFileAccessDirect"); + assert.deepEqual(calls, []); +}); + +test("source list caps at eight and excludes synthetic citations", async () => { + const { turn } = harness({ + transport: { call: async () => ({ rows: [{ value: 1 }], sources: Array.from({ length: 10 }, (_, i) => ({ id: `s${i}`, label: `Source ${i}` })) }) }, + synthesize: async function* () { yield "Answer [invented]."; }, + }); + const answer = await turn.ask({ question: "List", packs: ["data"] }); + assert.equal(answer.sources.length, 8); + assert.doesNotMatch(answer.text, /invented/); +}); + +test("a failed first round replans once and the overlay reaches synthesis alone", async () => { + const plannerInputs: unknown[] = []; + const synthesisInputs: unknown[] = []; + let reads = 0; + const { turn } = harness({ + planner: async (input: unknown) => { plannerInputs.push(input); return [{ tool: "query", arguments: {} }]; }, + transport: { call: async () => (++reads === 1 ? { rows: [], sources: [] } : { rows: [{ value: 1 }], sources: [{ id: "one", label: "One" }] }) }, + overlay: async () => "Operator voice", + synthesize: async function* (input: unknown) { synthesisInputs.push(input); yield "One."; }, + }); + await turn.ask({ question: "Find", packs: ["data"] }); + assert.equal(plannerInputs.length, 2); + assert.equal(reads, 2); + assert.doesNotMatch(JSON.stringify(plannerInputs), /Operator voice/); + assert.match(JSON.stringify(synthesisInputs), /Operator voice/); +}); + +test("vantage accepts calendar days and instants, and rejects invalid dates", () => { + assert.equal(parseVantage("2026-10-06"), "2026-10-06"); + assert.equal(parseVantage("2026-10-06T12:30:00Z"), "2026-10-06T12:30:00Z"); + assert.throws(() => parseVantage("2026-02-30"), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleVantageUnparseable" && error.status === 400); +}); + +test("overlay cache includes misses, expires after five minutes and truncates at 8000 chars", async () => { + let now = 0; + let reads = 0; + const cache = new OverlayCache(async () => { reads++; return "x".repeat(9000); }, () => now); + assert.equal((await cache.get("store")).length, 8000); + now = 299_999; + await cache.get("store"); + assert.equal(reads, 1); + now = 300_000; + await cache.get("store"); + assert.equal(reads, 2); +}); + +test("streaming redactor masks identifiers split across chunks", () => { + const redactor = new StreamingRedactor(["SECRET-ID"]); + const output = redactor.push("before SEC") + redactor.push("RET-ID after") + redactor.finish(); + assert.equal(output, "before [redacted] after"); + assert.throws(() => new StreamingRedactor(["x".repeat(129)]), RangeError); +}); From d57385cee4c167e15b02d35c7da98148daab84ca Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:37:27 +0800 Subject: [PATCH 047/155] feat(console): enforce grounded operator turns --- apps/console/src/turn.ts | 231 +++++++++++++++++++++++++++++++++ apps/console/test/turn.test.ts | 75 +++++++++++ 2 files changed, 306 insertions(+) create mode 100644 apps/console/src/turn.ts diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts new file mode 100644 index 000000000..24c194abc --- /dev/null +++ b/apps/console/src/turn.ts @@ -0,0 +1,231 @@ +export type ToolKind = "read" | "write"; + +export type Tool = { + name: string; + pack: string; + kind: ToolKind; + access?: "governed" | "direct-file"; + table?: string; + leg?: "store" | "web"; +}; + +export type ToolCall = { tool: string; arguments: Record; publicationBound?: string }; +export type Source = { id: string; label: string; url?: string }; +export type ToolResult = { rows: unknown[]; sources: Source[] }; +export type TurnRequest = { question: string; packs: string[]; store?: string; vantage?: string }; +export type DataTable = { name: string; kind: "data" | "memory" }; +export type Pack = { name: string; face: "organization" | "store"; tools: Array<{ name: string; kind: ToolKind }> }; + +export class ConsoleError extends Error { + readonly code: string; + readonly status: number; + + constructor(code: string, message = code, status = 400) { + super(message); + this.name = "ConsoleError"; + this.code = code; + this.status = status; + } +} + +export function validatePack(pack: Pack): void { + if (pack.face === "organization") { + const write = pack.tools.find((tool) => tool.kind === "write"); + if (write) throw new ConsoleError("ConsoleWriteToolOnOrgFace", `${pack.name}: ${write.name}`); + } +} + +export function parseVantage(value: string): string { + const year = Number(value.slice(0, 4)); + const month = Number(value.slice(5, 7)); + const day = Number(value.slice(8, 10)); + const validDay = Number.isInteger(year) && Number.isInteger(month) && Number.isInteger(day) + && new Date(Date.UTC(year, month - 1, day)).toISOString().slice(0, 10) === value.slice(0, 10); + if (/^\d{4}-\d{2}-\d{2}$/.test(value)) { + const date = new Date(`${value}T00:00:00Z`); + if (validDay && !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value) return value; + } else if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value)) { + const date = new Date(value); + if (validDay && !Number.isNaN(date.getTime())) return value; + } + throw new ConsoleError("ConsoleVantageUnparseable", value, 400); +} + +export function parseWebBound(value: string): string { + try { return parseVantage(value); } + catch { throw new ConsoleError("ConsoleWebBoundUnparseable", value); } +} + +export function sampleArrivals(arrivals: Array<{ table: string; labels: string[] }>) { + return arrivals.map((arrival) => ({ table: arrival.table, labels: arrival.labels.slice(0, 3) })); +} + +export async function resolveReaderCredential(options: { + mint: () => Promise; + shared?: string; +}): Promise { + try { + const credential = await options.mint(); + if (credential) return credential; + } catch { + // A refused mint reaches the shared credential only when the store has one. + } + if (options.shared) return options.shared; + throw new ConsoleError("ConsoleTokenExchangeRefused"); +} + +export class OverlayCache { + private entries = new Map(); + private readonly read: (store: string) => Promise; + private readonly clock: () => number; + + constructor( + read: (store: string) => Promise, + clock: () => number = Date.now, + ) { + this.read = read; + this.clock = clock; + } + + async get(store: string): Promise { + const cached = this.entries.get(store); + if (cached && this.clock() < cached.until) return cached.value ?? ""; + const value = (await this.read(store))?.slice(0, 8000) ?? null; + this.entries.set(store, { until: this.clock() + 300_000, value }); + return value ?? ""; + } +} + +export class StreamingRedactor { + private pending = ""; + private readonly denylist: string[]; + + constructor(denylist: string[]) { + if (denylist.some((entry) => entry.length > 128)) throw new RangeError("denylist entry exceeds 128 chars"); + this.denylist = [...new Set(denylist.filter(Boolean))].sort((a, b) => b.length - a.length); + } + + push(chunk: string): string { + this.pending += chunk; + return this.release(Math.max(0, this.pending.length - 128)); + } + + finish(): string { + return this.release(this.pending.length); + } + + private release(until: number): string { + let output = ""; + let index = 0; + while (index < until) { + const denied = this.denylist.find((entry) => this.pending.startsWith(entry, index)); + if (denied) { + output += "[redacted]"; + index += denied.length; + } else { + output += this.pending[index]; + index++; + } + } + this.pending = this.pending.slice(index); + return output; + } +} + +type PlannerInput = { + question: string; + vantage?: string; + tools: Tool[]; + tables: DataTable[]; + previous: ToolResult[]; +}; +type SynthesisInput = { + question: string; + vantage?: string; + results: ToolResult[]; + overlay: string; + sources: Source[]; +}; + +export type TurnOptions = { + tools: Tool[]; + packs?: Pack[]; + tables?: DataTable[]; + planner: (input: PlannerInput) => Promise; + transport: { call: (call: ToolCall & { vantage?: string; maxRows: number; signal: AbortSignal }) => Promise }; + synthesize: (input: SynthesisInput) => AsyncIterable; + overlay?: (store: string) => Promise; + denylist?: string[]; + clock?: () => number; + timeoutMs?: number; +}; + +export function createTurn(options: TurnOptions) { + options.packs?.forEach(validatePack); + const overlay = new OverlayCache(options.overlay ?? (async () => null), options.clock); + + return { + async ask(request: TurnRequest): Promise<{ text: string; sources: Source[] }> { + const vantage = request.vantage === undefined ? undefined : parseVantage(request.vantage); + if (options.tables?.some((table) => table.kind === "memory")) throw new ConsoleError("ConsolePlannerReachedMemory"); + const admitted = options.tools.filter((tool) => request.packs.includes(tool.pack)); + const results: ToolResult[] = []; + const rowsByTable = new Map(); + const deadline = Date.now() + Math.min(options.timeoutMs ?? 10_000, 10_000); + for (let round = 0; round < 2; round++) { + const calls = await options.planner({ question: request.question, vantage, tools: admitted.filter((tool) => tool.kind === "read"), tables: options.tables ?? [], previous: results }); + for (const call of calls) { + const tool = options.tools.find((candidate) => candidate.name === call.tool); + if (!tool || !request.packs.includes(tool.pack)) throw new ConsoleError("ConsoleToolNotAdmitted", call.tool); + if (tool.kind !== "read") throw new ConsoleError("ConsoleMutatingToolRequested", call.tool); + if (tool.access === "direct-file") throw new ConsoleError("ConsoleFileAccessDirect", call.tool); + if (tool.leg === "web" && call.publicationBound !== undefined) parseWebBound(call.publicationBound); + const table = tool.table ?? tool.name; + const consumed = rowsByTable.get(table) ?? 0; + const remaining = 5000 - consumed; + if (remaining <= 0) throw new RangeError("tool exceeded 5000 rows per data table"); + const controller = new AbortController(); + const milliseconds = Math.max(0, deadline - Date.now()); + let timeout: ReturnType | undefined; + try { + const result = await Promise.race([ + options.transport.call({ ...call, vantage, maxRows: remaining, signal: controller.signal }), + new Promise((_, reject) => { timeout = setTimeout(() => { + controller.abort(); + reject(new Error("code path exceeded 10 s")); + }, milliseconds); }), + ]); + if (result.rows.length > remaining) throw new RangeError("tool returned more than 5000 rows per data table"); + rowsByTable.set(table, consumed + result.rows.length); + results.push(result); + } finally { + if (timeout) clearTimeout(timeout); + } + } + if (results.some((result) => result.rows.length > 0)) break; + } + if (!results.some((result) => result.rows.length > 0)) throw new ConsoleError("ConsoleUngroundedAnswer", "The store holds no matching information."); + + const sources = [...new Map(results.flatMap((result) => result.sources).map((source) => [source.id, source])).values()].slice(0, 8); + if (sources.length === 0) throw new ConsoleError("ConsoleUngroundedAnswer", "The store holds no sourced information."); + const redactor = new StreamingRedactor(options.denylist ?? []); + let answer = ""; + for await (const chunk of options.synthesize({ question: request.question, vantage, results, overlay: await overlay.get(request.store ?? "default"), sources })) { + answer += redactor.push(chunk); + } + answer += redactor.finish(); + const admittedIds = new Set(sources.map((source) => source.id)); + answer = answer.replace(/\[([^\]\n]+)\]/g, (match, id: string) => id === "redacted" || admittedIds.has(id) ? match : ""); + const safeSources = sources.map((source) => { + const labelRedactor = new StreamingRedactor(options.denylist ?? []); + const label = labelRedactor.push(source.label) + labelRedactor.finish(); + const safeLabel = label.replace(/[\r\n]/g, " "); + const url = source.url && /^https?:\/\//.test(source.url) + && !(options.denylist ?? []).some((entry) => entry && source.url?.includes(entry)) ? source.url : undefined; + return { id: source.id, label: safeLabel, url }; + }); + const sourceLines = safeSources.map((source) => `- ${source.label}${source.url ? ` (${source.url})` : ""}`); + return { text: `${answer.trim()}\n\nSources\n${sourceLines.join("\n")}`, sources: safeSources }; + }, + }; +} diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index d1e929d78..f73d97bae 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -6,6 +6,9 @@ import { StreamingRedactor, createTurn, parseVantage, + parseWebBound, + resolveReaderCredential, + sampleArrivals, validatePack, } from "../src/turn.ts"; @@ -80,6 +83,22 @@ test("source list caps at eight and excludes synthetic citations", async () => { assert.doesNotMatch(answer.text, /invented/); }); +test("rows without governed provenance cannot become answer prose", async () => { + const { turn } = harness({ transport: { call: async () => ({ rows: [{ value: 1 }], sources: [] }) } }); + await assert.rejects(turn.ask({ question: "Explain", packs: ["data"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleUngroundedAnswer"); +}); + +test("a web leg with an invalid publication bound fails before dispatch", async () => { + const { turn, calls } = harness({ + tools: [{ name: "web", pack: "web", kind: "read", leg: "web" }], + planner: async () => [{ tool: "web", arguments: {}, publicationBound: "no-date" }], + }); + await assert.rejects(turn.ask({ question: "Search", packs: ["web"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleWebBoundUnparseable"); + assert.deepEqual(calls, []); +}); + test("a failed first round replans once and the overlay reaches synthesis alone", async () => { const plannerInputs: unknown[] = []; const synthesisInputs: unknown[] = []; @@ -102,6 +121,62 @@ test("vantage accepts calendar days and instants, and rejects invalid dates", () assert.equal(parseVantage("2026-10-06T12:30:00Z"), "2026-10-06T12:30:00Z"); assert.throws(() => parseVantage("2026-02-30"), (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleVantageUnparseable" && error.status === 400); + assert.throws(() => parseVantage("2026-02-30T12:30:00Z"), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleVantageUnparseable"); + assert.throws(() => parseWebBound("unknown"), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleWebBoundUnparseable"); +}); + +test("refused per-reader mint falls back only to an installed shared credential", async () => { + const mint = async () => { throw new Error("denied"); }; + assert.equal(await resolveReaderCredential({ mint, shared: "shared-token" }), "shared-token"); + await assert.rejects(resolveReaderCredential({ mint }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused"); +}); + +test("planner scaffolding refuses memory relations", async () => { + const { turn, calls } = harness({ tables: [{ name: "memory_entries", kind: "memory" }] }); + await assert.rejects(turn.ask({ question: "Recall", packs: ["data"] }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsolePlannerReachedMemory"); + assert.deepEqual(calls, []); +}); + +test("the code path refuses an unresponsive transport within ten seconds", async () => { + const { turn } = harness({ + transport: { call: async () => new Promise(() => {}) }, + timeoutMs: 5, + }); + await assert.rejects(turn.ask({ question: "Slow", packs: ["data"] }), + /code path exceeded 10 s/); +}); + +test("the transport receives a bounded read and cannot return more than 5000 rows", async () => { + let maxRows = 0; + const { turn } = harness({ transport: { call: async (call: { maxRows: number }) => { + maxRows = call.maxRows; + return { rows: Array.from({ length: 5001 }, () => ({})), sources: [] }; + } } }); + await assert.rejects(turn.ask({ question: "Many", packs: ["data"] }), RangeError); + assert.equal(maxRows, 5000); +}); + +test("one table cannot exceed the row budget through repeated calls", async () => { + const limits: number[] = []; + const { turn } = harness({ + tools: [{ ...readTool, table: "events" }], + planner: async () => [{ tool: "query", arguments: {} }, { tool: "query", arguments: {} }], + transport: { call: async (call: { maxRows: number }) => { + limits.push(call.maxRows); + return { rows: Array.from({ length: 3000 }, () => ({})), sources: [{ id: "events", label: "Events" }] }; + } }, + }); + await assert.rejects(turn.ask({ question: "Many", packs: ["data"] }), RangeError); + assert.deepEqual(limits, [5000, 2000]); +}); + +test("a table contributes at most three sampled arrival labels", () => { + assert.deepEqual(sampleArrivals([{ table: "events", labels: ["a", "b", "c", "d"] }]), + [{ table: "events", labels: ["a", "b", "c"] }]); }); test("overlay cache includes misses, expires after five minutes and truncates at 8000 chars", async () => { From 8285944ee0661646e952def92d5f3dff3be4329c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:39:33 +0800 Subject: [PATCH 048/155] fix(console): keep memory relations outside planner scaffolding --- apps/console/src/turn.ts | 11 +++++++++-- apps/console/test/turn.test.ts | 17 +++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts index 24c194abc..56745ac58 100644 --- a/apps/console/src/turn.ts +++ b/apps/console/src/turn.ts @@ -167,17 +167,24 @@ export function createTurn(options: TurnOptions) { return { async ask(request: TurnRequest): Promise<{ text: string; sources: Source[] }> { const vantage = request.vantage === undefined ? undefined : parseVantage(request.vantage); - if (options.tables?.some((table) => table.kind === "memory")) throw new ConsoleError("ConsolePlannerReachedMemory"); const admitted = options.tools.filter((tool) => request.packs.includes(tool.pack)); const results: ToolResult[] = []; const rowsByTable = new Map(); const deadline = Date.now() + Math.min(options.timeoutMs ?? 10_000, 10_000); for (let round = 0; round < 2; round++) { - const calls = await options.planner({ question: request.question, vantage, tools: admitted.filter((tool) => tool.kind === "read"), tables: options.tables ?? [], previous: results }); + const memoryTable = (name: string | undefined) => options.tables?.some((table) => table.kind === "memory" && table.name === name); + const calls = await options.planner({ + question: request.question, + vantage, + tools: admitted.filter((tool) => tool.kind === "read" && !memoryTable(tool.table)), + tables: (options.tables ?? []).filter((table) => table.kind === "data"), + previous: results, + }); for (const call of calls) { const tool = options.tools.find((candidate) => candidate.name === call.tool); if (!tool || !request.packs.includes(tool.pack)) throw new ConsoleError("ConsoleToolNotAdmitted", call.tool); if (tool.kind !== "read") throw new ConsoleError("ConsoleMutatingToolRequested", call.tool); + if (memoryTable(tool.table)) throw new ConsoleError("ConsolePlannerReachedMemory", call.tool); if (tool.access === "direct-file") throw new ConsoleError("ConsoleFileAccessDirect", call.tool); if (tool.leg === "web" && call.publicationBound !== undefined) parseWebBound(call.publicationBound); const table = tool.table ?? tool.name; diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index f73d97bae..3243ba741 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -134,8 +134,21 @@ test("refused per-reader mint falls back only to an installed shared credential" (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused"); }); -test("planner scaffolding refuses memory relations", async () => { - const { turn, calls } = harness({ tables: [{ name: "memory_entries", kind: "memory" }] }); +test("planner scaffolding lists data tables while memory relations remain outside it", async () => { + let plannerTables: unknown[] = []; + const { turn } = harness({ + tables: [{ name: "events", kind: "data" }, { name: "memory_entries", kind: "memory" }], + planner: async (input: { tables: unknown[] }) => { plannerTables = input.tables; return [{ tool: "query", arguments: {} }]; }, + }); + await turn.ask({ question: "Count", packs: ["data"] }); + assert.deepEqual(plannerTables, [{ name: "events", kind: "data" }]); +}); + +test("a planner call targeting a memory relation dispatches nothing", async () => { + const { turn, calls } = harness({ + tables: [{ name: "memory_entries", kind: "memory" }], + tools: [{ ...readTool, table: "memory_entries" }], + }); await assert.rejects(turn.ask({ question: "Recall", packs: ["data"] }), (error: unknown) => error instanceof ConsoleError && error.code === "ConsolePlannerReachedMemory"); assert.deepEqual(calls, []); From b16dc278da1ae607603c0c59c8fbfb7458dfd031 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:53:29 +0800 Subject: [PATCH 049/155] test(console): expose source identifiers and unbound row budgets --- apps/console/test/turn.test.ts | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index 3243ba741..211ef2276 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -12,7 +12,7 @@ import { validatePack, } from "../src/turn.ts"; -const readTool = { name: "query", pack: "data", kind: "read" as const }; +const readTool = { name: "query", pack: "data", kind: "read" as const, table: "events" }; function harness(overrides: Record = {}) { const calls: string[] = []; @@ -83,6 +83,17 @@ test("source list caps at eight and excludes synthetic citations", async () => { assert.doesNotMatch(answer.text, /invented/); }); +test("source identifiers returned to Query never expose denylisted material", async () => { + const { turn } = harness({ + denylist: ["SECRET-ID"], + transport: { call: async () => ({ rows: [{ value: 1 }], sources: [{ id: "SECRET-ID", label: "Public filing" }] }) }, + synthesize: async function* () { yield "Answer [SECRET-ID]."; }, + }); + const answer = await turn.ask({ question: "Explain", packs: ["data"] }); + assert.doesNotMatch(JSON.stringify(answer), /SECRET-ID/); + assert.match(answer.sources[0].id, /^source-\d+$/); +}); + test("rows without governed provenance cannot become answer prose", async () => { const { turn } = harness({ transport: { call: async () => ({ rows: [{ value: 1 }], sources: [] }) } }); await assert.rejects(turn.ask({ question: "Explain", packs: ["data"] }), @@ -187,6 +198,23 @@ test("one table cannot exceed the row budget through repeated calls", async () = assert.deepEqual(limits, [5000, 2000]); }); +test("unbound store tools cannot claim a per-table row budget", async () => { + const dispatched: string[] = []; + const { turn } = harness({ + tools: [{ name: "query-a", pack: "data", kind: "read" }, { name: "query-b", pack: "data", kind: "read" }], + planner: async () => [ + { tool: "query-a", arguments: { sql: "SELECT * FROM events" } }, + { tool: "query-b", arguments: { sql: "SELECT * FROM events" } }, + ], + transport: { call: async (call: { tool: string }) => { + dispatched.push(call.tool); + return { rows: Array.from({ length: 5000 }, () => ({})), sources: [{ id: "events", label: "Events" }] }; + } }, + }); + await assert.rejects(turn.ask({ question: "Many", packs: ["data"] }), /registered data table/); + assert.deepEqual(dispatched, []); +}); + test("a table contributes at most three sampled arrival labels", () => { assert.deepEqual(sampleArrivals([{ table: "events", labels: ["a", "b", "c", "d"] }]), [{ table: "events", labels: ["a", "b", "c"] }]); From 5add15923c66ac2023523ba4dfafe487457bd4e8 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:55:44 +0800 Subject: [PATCH 050/155] fix(console): redact source IDs and require table-bound reads --- apps/console/src/turn.ts | 27 ++++++++++++++++----------- apps/console/test/turn.test.ts | 25 ++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts index 56745ac58..12dab8468 100644 --- a/apps/console/src/turn.ts +++ b/apps/console/src/turn.ts @@ -1,13 +1,15 @@ export type ToolKind = "read" | "write"; -export type Tool = { +type ToolBase = { name: string; pack: string; - kind: ToolKind; access?: "governed" | "direct-file"; - table?: string; - leg?: "store" | "web"; }; +export type Tool = ToolBase & ( + | { kind: "read"; leg?: "store"; table: string } + | { kind: "read"; leg: "web"; table?: never } + | { kind: "write"; leg?: "store" | "web"; table?: string } +); export type ToolCall = { tool: string; arguments: Record; publicationBound?: string }; export type Source = { id: string; label: string; url?: string }; @@ -187,8 +189,11 @@ export function createTurn(options: TurnOptions) { if (memoryTable(tool.table)) throw new ConsoleError("ConsolePlannerReachedMemory", call.tool); if (tool.access === "direct-file") throw new ConsoleError("ConsoleFileAccessDirect", call.tool); if (tool.leg === "web" && call.publicationBound !== undefined) parseWebBound(call.publicationBound); - const table = tool.table ?? tool.name; - const consumed = rowsByTable.get(table) ?? 0; + const table = tool.leg === "web" ? undefined : tool.table; + if (tool.leg !== "web" && (!table || !options.tables?.some((entry) => entry.kind === "data" && entry.name === table))) { + throw new RangeError(`tool ${call.tool} has no registered data table`); + } + const consumed = table === undefined ? 0 : rowsByTable.get(table) ?? 0; const remaining = 5000 - consumed; if (remaining <= 0) throw new RangeError("tool exceeded 5000 rows per data table"); const controller = new AbortController(); @@ -203,7 +208,7 @@ export function createTurn(options: TurnOptions) { }, milliseconds); }), ]); if (result.rows.length > remaining) throw new RangeError("tool returned more than 5000 rows per data table"); - rowsByTable.set(table, consumed + result.rows.length); + if (table !== undefined) rowsByTable.set(table, consumed + result.rows.length); results.push(result); } finally { if (timeout) clearTimeout(timeout); @@ -221,15 +226,15 @@ export function createTurn(options: TurnOptions) { answer += redactor.push(chunk); } answer += redactor.finish(); - const admittedIds = new Set(sources.map((source) => source.id)); - answer = answer.replace(/\[([^\]\n]+)\]/g, (match, id: string) => id === "redacted" || admittedIds.has(id) ? match : ""); - const safeSources = sources.map((source) => { + const citations = new Map(sources.map((source, index) => [source.id, `source-${index + 1}`])); + answer = answer.replace(/\[([^\]\n]+)\]/g, (match, id: string) => id === "redacted" ? match : citations.has(id) ? `[${citations.get(id)}]` : ""); + const safeSources = sources.map((source, index) => { const labelRedactor = new StreamingRedactor(options.denylist ?? []); const label = labelRedactor.push(source.label) + labelRedactor.finish(); const safeLabel = label.replace(/[\r\n]/g, " "); const url = source.url && /^https?:\/\//.test(source.url) && !(options.denylist ?? []).some((entry) => entry && source.url?.includes(entry)) ? source.url : undefined; - return { id: source.id, label: safeLabel, url }; + return { id: `source-${index + 1}`, label: safeLabel, url }; }); const sourceLines = safeSources.map((source) => `- ${source.label}${source.url ? ` (${source.url})` : ""}`); return { text: `${answer.trim()}\n\nSources\n${sourceLines.join("\n")}`, sources: safeSources }; diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index 211ef2276..00a8c8dc8 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -18,6 +18,7 @@ function harness(overrides: Record = {}) { const calls: string[] = []; const turn = createTurn({ tools: [readTool], + tables: [{ name: "events", kind: "data" }], planner: async () => [{ tool: "query", arguments: { question: "count" } }], transport: { call: async (call: { tool: string }) => { @@ -32,12 +33,13 @@ function harness(overrides: Record = {}) { } test("a grounded turn cites only governed tool results", async () => { - const { turn, calls } = harness(); + const { turn, calls } = harness({ synthesize: async function* () { yield "Three rows [row-1]."; } }); const answer = await turn.ask({ question: "How many?", packs: ["data"] }); assert.deepEqual(calls, ["query"]); - assert.match(answer.text, /Three rows\./); + assert.match(answer.text, /Three rows/); + assert.match(answer.text, /\[source-1\]/); assert.match(answer.text, /Sources\n- Count/); - assert.deepEqual(answer.sources.map((source: { id: string }) => source.id), ["row-1"]); + assert.deepEqual(answer.sources.map((source: { id: string }) => source.id), ["source-1"]); }); test("a turn with no tool result refuses model prose", async () => { @@ -215,6 +217,23 @@ test("unbound store tools cannot claim a per-table row budget", async () => { assert.deepEqual(dispatched, []); }); +test("distinct registered tools share one data table's row budget", async () => { + const limits: number[] = []; + const { turn } = harness({ + tools: [ + { name: "query-a", pack: "data", kind: "read", table: "events" }, + { name: "query-b", pack: "data", kind: "read", table: "events" }, + ], + planner: async () => [{ tool: "query-a", arguments: {} }, { tool: "query-b", arguments: {} }], + transport: { call: async (call: { maxRows: number }) => { + limits.push(call.maxRows); + return { rows: Array.from({ length: 3000 }, () => ({})), sources: [{ id: "events", label: "Events" }] }; + } }, + }); + await assert.rejects(turn.ask({ question: "Many", packs: ["data"] }), RangeError); + assert.deepEqual(limits, [5000, 2000]); +}); + test("a table contributes at most three sampled arrival labels", () => { assert.deepEqual(sampleArrivals([{ table: "events", labels: ["a", "b", "c", "d"] }]), [{ table: "events", labels: ["a", "b", "c"] }]); From fa401681f3ec5ffd6b099c6e34424be69828446c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:16:16 +0800 Subject: [PATCH 051/155] feat(console): serve private briefs and server-built answer views --- apps/console/dist/server.js | 143 +++++++++++++++++++++++++++++++++--- apps/console/src/brief.ts | 6 +- apps/console/src/index.ts | 41 +++++++++-- 3 files changed, 172 insertions(+), 18 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 749033e5d..0cd071317 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -78,6 +78,109 @@ import { createServer } from "node:http"; // src/index.ts import { createHash, createHmac, createPublicKey, randomBytes, timingSafeEqual, verify } from "node:crypto"; + +// src/brief.ts +function tokens(text) { + return new Set((text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []).map((word) => word.length > 4 && word.endsWith("s") ? word.slice(0, -1) : word)); +} +function topicMatch(conclusion, row) { + const subject = tokens(conclusion.subject); + const left = tokens(`${conclusion.subject} ${conclusion.text}`); + const right = tokens(`${row.label} ${row.topics.join(" ")}`); + const shared = [...left].filter((word) => right.has(word)); + return shared.length >= 2 && shared.some((word) => subject.has(word)); +} +async function deriveBrief(input) { + if (input.session.turns !== 0 || input.session.vantage !== "present") return null; + const live = input.conclusions.filter((entry) => entry.live); + if (live.length === 0) return null; + const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); + let timer; + try { + const source = input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? []); + const budgetMs = input.budgetMs; + const arrivals = budgetMs === void 0 ? await source : await Promise.race([ + source, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), Math.max(0, budgetMs)); + }) + ]); + const since = input.now - windowDays * 864e5; + const recent = arrivals.filter((row) => { + const arrived = Date.parse(row.arrivedAt); + return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + }); + const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => topicMatch(entry, row)).slice(0, 3) })).filter((entry) => entry.articles.length > 0).slice(0, 3); + return subjects.length ? { windowDays, subjects } : null; + } catch { + throw new Error("ConsoleBriefUnavailable"); + } finally { + if (timer !== void 0) clearTimeout(timer); + } +} + +// src/publish.ts +function answerDelivery(input) { + if (!input.operator.trim()) throw new Error("VisibilityAskerlessAudience"); + if (input.accessExplanation && input.share) throw new Error("VisibilityShareAffordance"); + return { recipient: input.operator, answer: input.answer, share: false }; +} + +// src/render.ts +function numeric(value) { + return typeof value === "number" && Number.isFinite(value); +} +function day(value) { + if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const parsed = /* @__PURE__ */ new Date(`${value}T00:00:00Z`); + return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value; +} +function buildView(result, options = {}) { + if (options.origin === "client" || options.origin === "model" || options.view !== void 0) { + throw new Error("ConsoleViewNotServerBuilt"); + } + const props = { columns: [...result.columns], rows: result.rows.map((row) => [...row]) }; + const oneMeasure = props.columns.length === 1 && props.rows.length === 1 && numeric(props.rows[0][0]); + const dateIndex = props.columns.findIndex((column) => /^(date|day|.*_date|.*_day)$/i.test(column)); + const measureIndex = props.columns.findIndex((_, index) => index !== dateIndex && props.rows.every((row) => numeric(row[index]))); + const dates = dateIndex < 0 ? [] : props.rows.map((row) => row[dateIndex]); + const line = dateIndex >= 0 && measureIndex >= 0 && props.rows.length >= 3 && dates.every(day) && new Set(dates).size === dates.length; + const chosen = oneMeasure ? "metric.v1" : line ? "line.v1" : "table.v1"; + const hint = options.hint; + const binds = hint && hint.columns.length > 0 && hint.columns.every((column) => props.columns.includes(column)); + const component = binds && hint.component === "bar.v1" && chosen === "table.v1" ? "bar.v1" : chosen; + return component === "table.v1" ? { component, props, alternates: ["bar.v1"] } : { component, props }; +} +function sanitizeView(view, redact) { + const { columns, rows } = view.props; + if (!Array.isArray(columns) || !columns.every((column) => typeof column === "string") || !Array.isArray(rows) || !rows.every((row) => Array.isArray(row) && row.length === columns.length)) return null; + const seen = /* @__PURE__ */ new WeakSet(); + const walk = (value) => { + if (typeof value === "string") return redact(value); + if (Array.isArray(value)) { + if (seen.has(value)) throw new TypeError("cyclic view props"); + seen.add(value); + const result = value.map(walk); + seen.delete(value); + return result; + } + if (value !== null && typeof value === "object") { + if (seen.has(value)) throw new TypeError("cyclic view props"); + seen.add(value); + const result = Object.fromEntries(Object.entries(value).map(([key, item]) => [key, walk(item)])); + seen.delete(value); + return result; + } + return value; + }; + try { + return { ...view, props: { columns: columns.map(redact), rows: rows.map((row) => row.map(walk)) } }; + } catch { + return null; + } +} + +// src/index.ts function encoded(value) { return Buffer.from(JSON.stringify(value)).toString("base64url"); } @@ -199,9 +302,9 @@ async function cognitoCallback(request, identity2) { body: new URLSearchParams({ grant_type: "authorization_code", client_id: identity2.clientId, redirect_uri: identity2.redirectUri, code, code_verifier: verifier }) }); if (!exchange.ok) return refusal("ConsolePageForbidden"); - const tokens = await exchange.json(); - if (!object(tokens) || typeof tokens.id_token !== "string") return refusal("ConsolePageForbidden"); - const parts = tokens.id_token.split("."); + const tokens2 = await exchange.json(); + if (!object(tokens2) || typeof tokens2.id_token !== "string") return refusal("ConsolePageForbidden"); + const parts = tokens2.id_token.split("."); if (parts.length !== 3) return refusal("ConsolePageForbidden"); let header; let claims; @@ -293,6 +396,17 @@ function createConsole(adapters) { if (request.method === "GET" && path === `/${grant}`) return page(grant); if (grant === "query") { if (request.method === "GET" && path === "/query/api/stores") return json(await adapters.read.list(operator)); + if (request.method === "GET" && path === "/query/api/brief") { + const store = url.searchParams.get("store"); + if (!store || !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); + if (!adapters.brief) return refusal("ConsoleAdapterUnavailable", 503); + try { + const card = await deriveBrief(await adapters.brief(operator, store)); + return card ? json(card) : new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); + } catch { + return new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); + } + } if (request.method === "POST" && path === "/query/api/ask") { let input; try { @@ -301,7 +415,16 @@ function createConsole(adapters) { return bodyFailure(error); } if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); - return json(await adapters.turn({ operator, store: input.store, question: input.question })); + if ("view" in input || "widgets" in input) return refusal("ConsoleViewNotServerBuilt", 400); + const turn = await adapters.turn({ operator, store: input.store, question: input.question }); + let answer; + try { + answer = answerDelivery({ operator: operator.subject, answer: turn.answer, accessExplanation: turn.accessExplanation ?? false, share: turn.share }); + } catch (error) { + return refusal(error instanceof Error ? error.message : "VisibilityShareAffordance"); + } + const built = turn.resultRows ? sanitizeView(buildView(turn.resultRows), (value) => adapters.redactView?.(operator, value) ?? value) : null; + return json({ answer: answer.answer, sources: turn.sources, widgets: built ? [built] : [] }); } } else { const store = url.searchParams.get("store"); @@ -333,11 +456,13 @@ function createConsole(adapters) { var sharedStyle = ``; -var queryPage = `Query \xB7 Contextful${sharedStyle}

Ask the store

Answers cite the rows your access permits.

Conversation

Your answer appears here.

Results

`; var adminPage = `Admin \xB7 Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; const adminPage = `Admin · Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index ad0bb053f..906d97e1c 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -326,7 +326,7 @@ const form=document.getElementById('composer'),store=document.getElementById('st async function showBrief(){brief.hidden=true;const response=await fetch('/query/api/brief?store='+encodeURIComponent(store.value));if(!response.ok||response.status===204)return;const card=await response.json(),body=document.getElementById('brief-body');body.replaceChildren();for(const subject of card.subjects??[]){const item=document.createElement('article'),heading=document.createElement('h3'),list=document.createElement('ul');heading.textContent=subject.subject;item.append(heading);for(const article of subject.articles??[]){const entry=document.createElement('li');entry.textContent=article.label;list.append(entry)}item.append(list);body.append(item)}brief.hidden=!body.children.length} fetch('/query/api/stores').then(r=>r.json()).then(rows=>{for(const row of rows){const option=document.createElement('option');option.value=row.id;option.textContent=row.label;store.append(option)}if(store.value)showBrief()});store.addEventListener('change',showBrief); function tableView(props){const table=document.createElement('table'),head=document.createElement('thead'),header=document.createElement('tr'),body=document.createElement('tbody');for(const column of props.columns){const cell=document.createElement('th');cell.textContent=String(column);header.append(cell)}head.append(header);for(const row of props.rows){const line=document.createElement('tr');for(const value of row){const cell=document.createElement('td');cell.textContent=String(value??'');line.append(cell)}body.append(line)}table.append(head,body);return table} -function draw(widget){const props=widget.props??{},known=['table.v1','metric.v1','line.v1','bar.v1'];if(!known.includes(widget.component))widget={...widget,component:'table.v1'};if(!Array.isArray(props.columns)||!Array.isArray(props.rows))return document.createElement('span');if(widget.component==='metric.v1'){const metric=document.createElement('p');metric.style.fontSize='2rem';metric.textContent=String(props.rows[0]?.[0]??'');return metric}if(widget.component==='line.v1'){const values=props.rows.map(row=>Number(row[1])),svg=document.createElementNS('http://www.w3.org/2000/svg','svg');svg.setAttribute('viewBox','0 0 300 120');svg.setAttribute('role','img');svg.setAttribute('aria-label','Trend over time');const path=document.createElementNS('http://www.w3.org/2000/svg','polyline'),low=Math.min(...values),span=Math.max(...values)-low||1;path.setAttribute('fill','none');path.setAttribute('stroke','#3c7c92');path.setAttribute('stroke-width','3');path.setAttribute('points',values.map((value,index)=>(20+index*260/(values.length-1))+','+(100-(value-low)*80/span)).join(' '));svg.append(path);return svg}if(widget.component==='bar.v1'){const list=document.createElement('div');for(const row of props.rows){const entry=document.createElement('p');entry.textContent=String(row[0]??'')+' · '+String(row[1]??row[0]??'');list.append(entry)}return list}const frame=document.createElement('div');frame.append(tableView(props));if(widget.alternates?.includes('bar.v1')){const button=document.createElement('button');button.type='button';button.textContent='Bar view';button.addEventListener('click',()=>frame.replaceChildren(draw({...widget,component:'bar.v1'})));frame.append(button)}return frame} +function draw(widget){const props=widget.props??{},known=['table.v1','metric.v1','line.v1','bar.v1'];if(!known.includes(widget.component))widget={...widget,component:'table.v1'};if(!Array.isArray(props.columns)||!Array.isArray(props.rows))return document.createElement('span');if(widget.component==='metric.v1'){const metric=document.createElement('p');metric.style.fontSize='2rem';metric.textContent=String(props.rows[0]?.[0]??'');return metric}if(widget.component==='line.v1'){const dateIndex=props.columns.findIndex(column=>/^(date|day|.*_date|.*_day)$/i.test(column)),measureIndex=props.columns.findIndex((_,index)=>index!==dateIndex&&props.rows.every(row=>typeof row[index]==="number"));if(measureIndex<0)return tableView(props);const values=props.rows.map(row=>Number(row[measureIndex])),svg=document.createElementNS('http://www.w3.org/2000/svg','svg');svg.setAttribute('viewBox','0 0 300 120');svg.setAttribute('role','img');svg.setAttribute('aria-label','Trend over time');const path=document.createElementNS('http://www.w3.org/2000/svg','polyline'),low=Math.min(...values),span=Math.max(...values)-low||1;path.setAttribute('fill','none');path.setAttribute('stroke','#3c7c92');path.setAttribute('stroke-width','3');path.setAttribute('points',values.map((value,index)=>(20+index*260/(values.length-1))+','+(100-(value-low)*80/span)).join(' '));svg.append(path);return svg}if(widget.component==='bar.v1'){const list=document.createElement('div');for(const row of props.rows){const entry=document.createElement('p');entry.textContent=String(row[0]??'')+' · '+String(row[1]??row[0]??'');list.append(entry)}return list}const frame=document.createElement('div');frame.append(tableView(props));if(widget.alternates?.includes('bar.v1')){const button=document.createElement('button');button.type='button';button.textContent='Bar view';button.addEventListener('click',()=>frame.replaceChildren(draw({...widget,component:'bar.v1'})));frame.append(button)}return frame} form.addEventListener('submit',async event=>{event.preventDefault();const question=document.getElementById('question').value.trim();if(!question)return;brief.hidden=true;const response=await fetch('/query/api/ask',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({store:store.value,question})});const answer=await response.json();transcript.replaceChildren();const item=document.createElement('article');item.textContent=question+'\n'+(answer.answer??answer.error?.identifier??'No answer');transcript.append(item);widgets.replaceChildren();for(const widget of answer.widgets??[])widgets.append(draw(widget));if(answer.sources?.length){const sources=document.createElement('p');sources.textContent='Sources: '+answer.sources.map(x=>x.title??x.url).join(', ');transcript.append(sources)}}); `; From bd2d4b18e959b3a69abf58aa3ee195e250707c50 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:45:48 +0800 Subject: [PATCH 057/155] test(query): require typed memory exclusion from read listing --- apps/console/test/live.test.ts | 73 +++++++++++++++++-- .../tests/integration/read/respond.rs | 13 ++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index f79ae51b4..85e53ed8d 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -1,6 +1,8 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { createLiveTurn } from "../src/live.ts"; +import { ConsoleError } from "../src/turn.ts"; +import { createConsole, issueCognitoSession } from "../src/index.ts"; import type { StoreEntry } from "../../gateway/src/index.ts"; const store: StoreEntry = { @@ -9,9 +11,10 @@ const store: StoreEntry = { bindingName: "FIELD_NOTES", }; -function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[][] } = {}) { +function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[][]; tables?: Array<{ table: string; kind: string }> } = {}) { const calls: Array<{ url: string; authorization: string | null; body: Record }> = []; let modelCalls = 0; + let modelBody: Record | null = null; const fetcher: typeof fetch = async (input, init) => { const url = String(input); const body = JSON.parse(String(init?.body ?? "{}")) as Record; @@ -19,7 +22,9 @@ function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[ if (url.endsWith("/auth/exchange")) return Response.json(options.exchange === false ? { error: { identifier: "ExchangeAssertionInvalid" } } : { token: "reader-token" }, { status: options.exchange === false ? 403 : 200 }); if (url.endsWith("/mcp")) { const params = body.params as Record; - if (params.name === "context.describe") return Response.json({ jsonrpc: "2.0", id: body.id, result: { structuredContent: { tables: [{ table: "filings" }] } } }); + if (params.name === "context.describe") return Response.json({ jsonrpc: "2.0", id: body.id, result: { structuredContent: { + tables: options.tables ?? [{ table: "filings", kind: "data" }], + } } }); assert.equal(params.name, "context.query"); return Response.json({ jsonrpc: "2.0", id: body.id, result: { structuredContent: { columns: ["filing_id", "summary", "source_url"], @@ -28,6 +33,7 @@ function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[ } if (url.endsWith("/chat/completions")) { modelCalls++; + modelBody = body; return Response.json({ choices: [{ message: { content: "Northwind filed on Monday [filing-1]." } }] }); } throw new Error(`unexpected request ${url}`); @@ -36,7 +42,7 @@ function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[ CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture", ...(options.shared ? { FIELD_NOTES_QUERY_TOKEN: options.shared } : {}), }, fetcher }); - return { turn, calls, modelCalls: () => modelCalls }; + return { turn, calls, modelCalls: () => modelCalls, modelBody: () => modelBody }; } test("Query reads governed rows under the exchanged reader credential and cites their provenance", async () => { @@ -46,11 +52,12 @@ test("Query reads governed rows under the exchanged reader credential and cites assert.match(answer.answer, /Northwind filed on Monday/); assert.match(JSON.stringify(answer.sources), /filing-1/); assert.equal(modelCalls(), 1); - assert.equal(calls.find((call) => call.url.endsWith("/auth/exchange"))?.body.assertion, "verified-access-jwt"); + assert.equal(calls.find((call) => call.url.endsWith("/auth/exchange"))?.body.jwt, "verified-access-jwt"); assert.equal(calls.find((call) => (call.body.params as Record | undefined)?.name === "context.query")?.authorization, "Bearer reader-token"); assert.deepEqual(calls.filter((call) => call.url.endsWith("/mcp")).map((call) => (call.body.params as Record).name), ["context.describe", "context.query"]); }); +// spec: surface.ground.mint-refused@1ca62976 test("Query falls back to a configured shared credential only after exchange refusal", async () => { const fallback = fixture({ exchange: false, shared: "shared-token" }); await fallback.turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, @@ -66,6 +73,62 @@ test("Query falls back to a configured shared credential only after exchange ref test("Query never sends rows without provenance to the model", async () => { const { turn, modelCalls } = fixture({ rows: [["", "Northwind filed on Monday", ""]] }); await assert.rejects(turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, - store: "field-notes", question: "Which filing arrived?" }), /ConsoleUngroundedAnswer/); + store: "field-notes", question: "Which filing arrived?" }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleUngroundedAnswer"); assert.equal(modelCalls(), 0); }); + +test("Query excludes an unsourced sibling row from model context", async () => { + const { turn, modelBody } = fixture({ rows: [ + ["filing-1", "Northwind filed on Monday", "https://example.test/filing-1"], + ["", "PRIVATE UNSOURCED ROW", ""], + ] }); + await turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" }); + assert.doesNotMatch(JSON.stringify(modelBody()), /PRIVATE UNSOURCED ROW/); +}); + +test("Query never scaffolds or dispatches a memory relation", async () => { + const { turn, calls } = fixture({ tables: [{ table: "research/insights", kind: "memory" }, { table: "filings", kind: "data" }] }); + await turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" }); + assert.deepEqual(calls.filter((call) => (call.body.params as Record | undefined)?.name === "context.query") + .map((call) => (call.body.params as Record).arguments), + [{ sql: 'SELECT * FROM "filings"', limit: 5000 }]); +}); + +test("Query returns a typed exchange refusal through its hosted API", async () => { + const { turn, calls } = fixture({ exchange: false }); + const app = createConsole({ + identity: { kind: "cognito", sessionSecret: "session-secret", queryGroup: "query", adminGroup: "admin" }, + stores: [{ id: store.id, label: store.label }], turn, + read: { list: async () => [{ id: store.id, label: store.label }] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const token = issueCognitoSession({ subject: "reader", groups: ["query"], expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { cookie: `console_session=${token}`, origin: "https://console.example" }, + body: JSON.stringify({ store: store.id, question: "Which filing arrived?" }), + })); + assert.equal(response.status, 403); + assert.deepEqual(await response.json(), { error: { identifier: "ConsoleTokenExchangeRefused" } }); + assert.equal(calls.filter((call) => call.url.endsWith("/mcp")).length, 0); +}); + +test("Cognito Query exchanges the signed-in reader's assertion", async () => { + const { turn, calls } = fixture(); + const app = createConsole({ + identity: { kind: "cognito", sessionSecret: "session-secret", queryGroup: "query", adminGroup: "admin" }, + stores: [{ id: store.id, label: store.label }], turn, + read: { list: async () => [{ id: store.id, label: store.label }] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const token = issueCognitoSession({ subject: "reader", groups: ["query"], assertion: "cognito-id-token", + expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { cookie: `console_session=${token}`, origin: "https://console.example" }, + body: JSON.stringify({ store: store.id, question: "Which filing arrived?" }), + })); + assert.equal(response.status, 200); + assert.equal(calls.find((call) => call.url.endsWith("/auth/exchange"))?.body.jwt, "cognito-id-token"); +}); diff --git a/crates/contextful-context/tests/integration/read/respond.rs b/crates/contextful-context/tests/integration/read/respond.rs index 40ac8bea9..ca54b1def 100644 --- a/crates/contextful-context/tests/integration/read/respond.rs +++ b/crates/contextful-context/tests/integration/read/respond.rs @@ -102,6 +102,19 @@ fn describe_reports_the_session_zone_and_each_tables_admission() { assert_eq!((vendor["session_zone"].clone(), vendor["zone_admitted"].clone()), (json!("public-cloud:us-east-1"), json!(true))); } +#[test] +fn describe_marks_declared_memory_tables_regardless_of_name() { + let manifest = format!("{MANIFEST}\n[[table]]\nname = \"research/insights\"\nshape = \"memory_facts\"\ncolumns = [\"claim_id\", \"subject\", \"predicate\", \"object\", \"scope\", \"tier\", \"confidence\", \"valid_from\", \"valid_to\", \"evidence\", \"superseded_by\", \"grant_id\", \"agent\"]\n"); + let r = Reads::with_manifest(&manifest); + let s = r.session(&["research/*"], None, None); + let listing = r.face.describe(&s, None, Bounds::default()).unwrap(); + let tables = listing["tables"].as_array().unwrap(); + let insights = tables.iter().find(|item| item["table"] == "research/insights").unwrap(); + assert_eq!(insights["kind"], json!("memory")); + let notes = tables.iter().find(|item| item["table"] == "research/notes").unwrap(); + assert_eq!(notes["kind"], json!("data")); +} + /// An arm whose table the session's zone excludes contributes no candidate, and the ranked response names that table in {{read.respond.restriction-block}}. // spec: read.retrieve.excluded-arm@12f1c8ad #[test] From 670bd789b8387eed7e064b6c577b345dc7d812d7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:51:25 +0800 Subject: [PATCH 058/155] test: prove owner reads retain row zone and mask policies --- .../contextful-cli/tests/integration/mcp.rs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 9cafcf8ab..5f704a864 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -144,6 +144,54 @@ fn explicit_owner_requires_a_signed_credential_for_the_selected_store() { assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); } +#[test] +fn owner_credential_still_applies_row_zone_and_column_policies() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + std::fs::create_dir_all(root.join(".contextful")).unwrap(); + std::fs::write(root.join(".contextful/issuance.toml"), format!("default_audience = \"{AUD}\"\nmax_lifetime_secs = 3600\n")).unwrap(); + std::fs::write(root.join("contextful.toml"), r#"authoring_posture = "per_request" +[project] +name = "research" + +[[pipeline.tables]] +name = "research/notes" +[pipeline.tables.policy.rows] +predicate = "owner = subject.on_behalf_of" +[pipeline.tables.policy.columns] +title = { strategy = "drop" } + +[[pipeline.tables]] +name = "research/vendor" +[pipeline.tables.policy.zone] +allow = ["public-cloud:*"] +"#).unwrap(); + std::fs::write(root.join("notes.jsonl"), concat!( + "{\"note_id\":\"n1\",\"owner\":\"user://dana@acme.example\",\"title\":\"private one\"}\n", + "{\"note_id\":\"n2\",\"owner\":\"user://other@acme.example\",\"title\":\"private two\"}\n", + )).unwrap(); + std::fs::write(root.join("vendor.jsonl"), "{\"vendor_id\":\"v1\"}\n").unwrap(); + for (table, rows, run_id) in [("research/notes", "notes.jsonl", "run-notes"), ("research/vendor", "vendor.jsonl", "run-vendor")] { + stdout(&run(root, &["context", "land", table, "--project", "research", "--rows", rows, "--run-id", run_id, "--site-id", "site-a"])); + } + let public = stdout(&run(root, &["token", "keygen", "--out", ".contextful/issuer.seed"])); + let owner = stdout(&run(root, &[ + "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", + "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner", + ])); + let args = ["mcp", "--owner", "--project", "research", "--public-key", &public, "--audience", AUD]; + let query = |id, sql| json!({ "jsonrpc": "2.0", "id": id, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": sql } } }); + let out = serve(root, &args, Some(&owner), &[ + query(1, "SELECT note_id, title FROM \"research/notes\" ORDER BY note_id"), + query(2, "SELECT vendor_id FROM \"research/vendor\""), + ]); + assert!(out.status.success(), "{}", String::from_utf8_lossy(&out.stderr)); + let answers: Vec = String::from_utf8_lossy(&out.stdout).lines().map(|line| serde_json::from_str(line).unwrap()).collect(); + assert_eq!(answers.len(), 2, "{answers:?}"); + assert_eq!(answers[0]["result"]["structuredContent"]["rows"], json!([["n1", ""]]), "row predicate and column mask hold under owner"); + assert_eq!(answers[1]["result"]["structuredContent"]["rows"], json!([]), "zone exclusion holds under owner"); +} + /// The server admits over the stdio pipe it inherited: a credential binding no key admits /// through that pipe, and one binding a holder key and presenting no holder proof refuses. #[test] From 529228be64dac58d80975f91a60049609d0451ca Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 15:59:58 +0800 Subject: [PATCH 059/155] test(console): refuse unsafe views and unbounded briefs --- apps/console/test/brief.test.ts | 15 +++++++-- apps/console/test/console.test.ts | 51 +++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/apps/console/test/brief.test.ts b/apps/console/test/brief.test.ts index b7bcedc2f..bd3a2d0b8 100644 --- a/apps/console/test/brief.test.ts +++ b/apps/console/test/brief.test.ts @@ -15,7 +15,7 @@ test("topic tier needs two shared tokens including the subject", () => { // spec: surface.brief.greeting@945ee5ee test("brief requires a turnless present-time session and live matched arrivals", async () => { - const input = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], arrivals: [row], now }; + const input = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], arrivals: [row], now, budgetMs: 1000 }; assert.equal((await deriveBrief(input))?.subjects[0].subject, "Acme"); assert.equal(await deriveBrief({ ...input, session: { turns: 1, vantage: "present" } }), null); assert.equal(await deriveBrief({ ...input, session: { turns: 0, vantage: "2026-01-01" } }), null); @@ -26,14 +26,23 @@ test("brief requires a turnless present-time session and live matched arrivals", test("brief caps window at seven days, subjects at three, and articles at three", async () => { const conclusions = Array.from({ length: 5 }, (_, n) => ({ subject: `Acme${n}`, text: `Acme${n} filings need review`, live: true })); const arrivals = conclusions.flatMap((c, n) => Array.from({ length: 5 }, (_, m) => ({ id: `${n}-${m}`, label: `${c.subject} filing`, topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }))); - const card = await deriveBrief({ session: { turns: 0, vantage: "present" }, conclusions, arrivals, now, windowDays: 100 }); + const card = await deriveBrief({ session: { turns: 0, vantage: "present" }, conclusions, arrivals, now, windowDays: 100, budgetMs: 1000 }); assert.equal(card?.windowDays, 7); assert.equal(card?.subjects.length, 3); assert.ok(card?.subjects.every((subject) => subject.articles.length === 3)); }); test("brief reports derivation failure and timeout without a card", async () => { - const base = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], now }; + const base = { session: { turns: 0, vantage: "present" }, conclusions: [conclusion], now, budgetMs: 1000 }; await assert.rejects(deriveBrief({ ...base, loadArrivals: async () => { throw new Error("unavailable"); } }), /ConsoleBriefUnavailable/); await assert.rejects(deriveBrief({ ...base, loadArrivals: () => new Promise(() => {}), budgetMs: 5 }), /ConsoleBriefUnavailable/); }); + +test("brief refuses a missing derivation budget before loading arrivals", async () => { + const pending = deriveBrief({ session: { turns: 0, vantage: "present" }, conclusions: [conclusion], now, + loadArrivals: () => new Promise(() => {}), budgetMs: undefined as unknown as number }); + await Promise.race([ + assert.rejects(pending, /ConsoleBriefUnavailable/), + new Promise((_, reject) => setTimeout(() => reject(new Error("budget missing hung")), 50)), + ]); +}); diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 12046de0e..0825cbcb5 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -322,6 +322,41 @@ test("Query builds its widget from rows and sanitizes view props before delivery assert.deepEqual(answer.widgets[0].props.rows, [["[redacted]"]]); }); +test("Query omits result widgets when no view redactor is available", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "One account arrived.", sources: [], widgets: [], + resultRows: { columns: ["account"], rows: [["secret@example.test"]] } }), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { ...access(queryAudience), origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", question: "What arrived?" }), + })); + assert.equal(response.status, 200); + assert.deepEqual((await response.json() as { widgets: unknown[] }).widgets, []); +}); + +test("Query keeps malformed result rows prose only", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "One account arrived.", sources: [], widgets: [], + resultRows: { columns: ["account"], rows: [null] } as unknown as { columns: string[]; rows: unknown[][] } }), + redactView: (_operator, value) => value, + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { ...access(queryAudience), origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", question: "What arrived?" }), + })); + assert.equal(response.status, 200); + assert.deepEqual((await response.json() as { widgets: unknown[] }).widgets, []); +}); + test("Query refuses a client view and an access-explanation share control", async () => { let turns = 0; const app = createConsole({ @@ -350,6 +385,7 @@ test("Query brief derives a private card only from a turnless present-time sessi identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, stores: [{ id: "field-notes", label: "Field notes" }], turn: async () => ({ answer: "", sources: [], widgets: [] }), + briefBudgetMs: 1000, brief: async () => ({ session: { turns: 0, vantage: "present" }, now, conclusions: [{ subject: "Acme", text: "Acme filings need review", live: true }], arrivals: [{ id: "a", label: "Acme filing", topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }], @@ -364,3 +400,18 @@ test("Query brief derives a private card only from a turnless present-time sessi assert.equal((await response.json() as { subjects: Array<{ subject: string }> }).subjects[0].subject, "Acme"); assert.equal((await app.fetch(new Request("https://console.example/query/api/brief?store=missing", { headers: access(queryAudience) }))).status, 400); }); + +test("Query brief expires while its adapter is still pending", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + briefBudgetMs: 5, + brief: async () => new Promise(() => {}), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const request = app.fetch(new Request("https://console.example/query/api/brief?store=field-notes", { headers: access(queryAudience) })); + const response = await Promise.race([request, new Promise((_, reject) => setTimeout(() => reject(new Error("brief hung")), 50))]); + assert.equal(response.status, 204); +}); From 690256c93f4737510584f7851e22399bf49e7f96 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:01:18 +0800 Subject: [PATCH 060/155] feat(query): serve governed console turns and reader exchange --- apps/console/dist/server.js | 298 +++++++++++++++++- apps/console/src/entry.ts | 6 +- apps/console/src/index.ts | 21 +- apps/console/src/live.ts | 103 ++++++ apps/console/src/turn.ts | 2 +- apps/console/test/turn.test.ts | 5 + crates/contextful-agent/src/http.rs | 14 +- crates/contextful-cli/src/serve.rs | 28 +- .../contextful-cli/tests/integration/serve.rs | 4 +- crates/contextful-context/src/read/face.rs | 1 + spec/pins.toml | 4 +- spec/status.md | 14 +- 12 files changed, 474 insertions(+), 26 deletions(-) create mode 100644 apps/console/src/live.ts diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 749033e5d..97b947135 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -73,6 +73,284 @@ function registryFromEnv(raw) { return { entries, problems }; } +// src/turn.ts +var ConsoleError = class extends Error { + code; + status; + constructor(code, message = code, status = 400) { + super(message); + this.name = "ConsoleError"; + this.code = code; + this.status = status; + } +}; +function validatePack(pack) { + if (pack.face === "organization") { + const write = pack.tools.find((tool) => tool.kind === "write"); + if (write) throw new ConsoleError("ConsoleWriteToolOnOrgFace", `${pack.name}: ${write.name}`); + } +} +function parseVantage(value) { + const year = Number(value.slice(0, 4)); + const month = Number(value.slice(5, 7)); + const day = Number(value.slice(8, 10)); + const validDay = Number.isInteger(year) && Number.isInteger(month) && Number.isInteger(day) && new Date(Date.UTC(year, month - 1, day)).toISOString().slice(0, 10) === value.slice(0, 10); + if (/^\d{4}-\d{2}-\d{2}$/.test(value)) { + const date = /* @__PURE__ */ new Date(`${value}T00:00:00Z`); + if (validDay && !Number.isNaN(date.getTime()) && date.toISOString().slice(0, 10) === value) return value; + } else if (/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.test(value)) { + const date = new Date(value); + if (validDay && !Number.isNaN(date.getTime())) return value; + } + throw new ConsoleError("ConsoleVantageUnparseable", value, 400); +} +function parseWebBound(value) { + try { + return parseVantage(value); + } catch { + throw new ConsoleError("ConsoleWebBoundUnparseable", value); + } +} +async function resolveReaderCredential(options) { + try { + const credential = await options.mint(); + if (credential) return credential; + } catch { + } + if (options.shared) return options.shared; + throw new ConsoleError("ConsoleTokenExchangeRefused", "ConsoleTokenExchangeRefused", 403); +} +var OverlayCache = class { + entries = /* @__PURE__ */ new Map(); + read; + clock; + constructor(read, clock = Date.now) { + this.read = read; + this.clock = clock; + } + async get(store) { + const cached = this.entries.get(store); + if (cached && this.clock() < cached.until) return cached.value ?? ""; + const value = (await this.read(store))?.slice(0, 8e3) ?? null; + this.entries.set(store, { until: this.clock() + 3e5, value }); + return value ?? ""; + } +}; +var StreamingRedactor = class { + pending = ""; + denylist; + constructor(denylist) { + if (denylist.some((entry) => entry.length > 128)) throw new RangeError("denylist entry exceeds 128 chars"); + this.denylist = [...new Set(denylist.filter(Boolean))].sort((a, b) => b.length - a.length); + } + push(chunk) { + this.pending += chunk; + return this.release(Math.max(0, this.pending.length - 128)); + } + finish() { + return this.release(this.pending.length); + } + release(until) { + let output = ""; + let index = 0; + while (index < until) { + const denied = this.denylist.find((entry) => this.pending.startsWith(entry, index)); + if (denied) { + output += "[redacted]"; + index += denied.length; + } else { + output += this.pending[index]; + index++; + } + } + this.pending = this.pending.slice(index); + return output; + } +}; +function createTurn(options) { + options.packs?.forEach(validatePack); + const overlay = new OverlayCache(options.overlay ?? (async () => null), options.clock); + return { + async ask(request) { + const vantage = request.vantage === void 0 ? void 0 : parseVantage(request.vantage); + const admitted = options.tools.filter((tool) => request.packs.includes(tool.pack)); + const results = []; + const rowsByTable = /* @__PURE__ */ new Map(); + const deadline = Date.now() + Math.min(options.timeoutMs ?? 1e4, 1e4); + for (let round = 0; round < 2; round++) { + const memoryTable = (name) => options.tables?.some((table) => table.kind === "memory" && table.name === name); + const calls = await options.planner({ + question: request.question, + vantage, + tools: admitted.filter((tool) => tool.kind === "read" && !memoryTable(tool.table)), + tables: (options.tables ?? []).filter((table) => table.kind === "data"), + previous: results + }); + for (const call of calls) { + const tool = options.tools.find((candidate) => candidate.name === call.tool); + if (!tool || !request.packs.includes(tool.pack)) throw new ConsoleError("ConsoleToolNotAdmitted", call.tool); + if (tool.kind !== "read") throw new ConsoleError("ConsoleMutatingToolRequested", call.tool); + if (memoryTable(tool.table)) throw new ConsoleError("ConsolePlannerReachedMemory", call.tool); + if (tool.access === "direct-file") throw new ConsoleError("ConsoleFileAccessDirect", call.tool); + if (tool.leg === "web" && call.publicationBound !== void 0) parseWebBound(call.publicationBound); + const table = tool.leg === "web" ? void 0 : tool.table; + if (tool.leg !== "web" && (!table || !options.tables?.some((entry) => entry.kind === "data" && entry.name === table))) { + throw new RangeError(`tool ${call.tool} has no registered data table`); + } + const consumed = table === void 0 ? 0 : rowsByTable.get(table) ?? 0; + const remaining = 5e3 - consumed; + if (remaining <= 0) throw new RangeError("tool exceeded 5000 rows per data table"); + const controller = new AbortController(); + const milliseconds = Math.max(0, deadline - Date.now()); + let timeout; + try { + const result = await Promise.race([ + options.transport.call({ ...call, vantage, maxRows: remaining, signal: controller.signal }), + new Promise((_, reject) => { + timeout = setTimeout(() => { + controller.abort(); + reject(new Error("code path exceeded 10 s")); + }, milliseconds); + }) + ]); + if (result.rows.length > remaining) throw new RangeError("tool returned more than 5000 rows per data table"); + if (table !== void 0) rowsByTable.set(table, consumed + result.rows.length); + results.push(result); + } finally { + if (timeout) clearTimeout(timeout); + } + } + if (results.some((result) => result.rows.length > 0)) break; + } + if (!results.some((result) => result.rows.length > 0)) throw new ConsoleError("ConsoleUngroundedAnswer", "The store holds no matching information."); + const sources = [...new Map(results.flatMap((result) => result.sources).map((source2) => [source2.id, source2])).values()].slice(0, 8); + if (sources.length === 0) throw new ConsoleError("ConsoleUngroundedAnswer", "The store holds no sourced information."); + const redactor = new StreamingRedactor(options.denylist ?? []); + let answer = ""; + for await (const chunk of options.synthesize({ question: request.question, vantage, results, overlay: await overlay.get(request.store ?? "default"), sources })) { + answer += redactor.push(chunk); + } + answer += redactor.finish(); + const citations = new Map(sources.map((source2, index) => [source2.id, `source-${index + 1}`])); + answer = answer.replace(/\[([^\]\n]+)\]/g, (match, id) => id === "redacted" ? match : citations.has(id) ? `[${citations.get(id)}]` : ""); + const safeSources = sources.map((source2, index) => { + const labelRedactor = new StreamingRedactor(options.denylist ?? []); + const label = labelRedactor.push(source2.label) + labelRedactor.finish(); + const safeLabel = label.replace(/[\r\n]/g, " "); + const url = source2.url && /^https?:\/\//.test(source2.url) && !(options.denylist ?? []).some((entry) => entry && source2.url?.includes(entry)) ? source2.url : void 0; + return { id: `source-${index + 1}`, label: safeLabel, url }; + }); + const sourceLines = safeSources.map((source2) => `- ${source2.label}${source2.url ? ` (${source2.url})` : ""}`); + return { text: `${answer.trim()} + +Sources +${sourceLines.join("\n")}`, sources: safeSources }; + } + }; +} + +// src/live.ts +function record(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +function strings(value) { + return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; +} +function source(columns, row, table) { + const at = (name) => row[columns.indexOf(name)]; + const idColumn = columns.find((name) => name.endsWith("_id") || name === "id"); + const urlColumn = columns.find((name) => name === "source_url" || name === "url"); + const id = idColumn && at(idColumn); + const url = urlColumn && at(urlColumn); + if (typeof id !== "string" || !id) return null; + const label = columns.includes("title") ? at("title") : columns.includes("summary") ? at("summary") : id; + return { + id, + label: typeof label === "string" && label ? label : `${table}: ${id}`, + ...typeof url === "string" && /^https?:\/\//.test(url) ? { url } : {} + }; +} +function createLiveTurn({ stores, env, fetcher = fetch }) { + const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; + const modelId = env.CONTEXTFUL_MODEL_ID; + if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); + return async (input) => { + const store = stores.find((entry) => entry.id === input.store); + if (!store) throw new ConsoleError("ConsoleRequestMalformed"); + const shared = env[store.credentialName]; + const credential = store.auth === "exchange" ? await resolveReaderCredential({ + shared, + mint: async () => { + if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeRefused"); + const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jwt: input.operator.assertion }) + }); + if (!response.ok) throw new ConsoleError("ConsoleTokenExchangeRefused"); + const value = await response.json(); + if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeRefused"); + return value.token; + } + }) : shared; + if (!credential) throw new ConsoleError("ConsoleTokenExchangeRefused"); + let id = 0; + const call = async (name, args, signal) => { + const response = await fetcher(new URL("/mcp", store.endpoint), { + method: "POST", + signal, + headers: { "Authorization": `Bearer ${credential}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }) + }); + if (!response.ok) throw new ConsoleError("ConsoleStoreReadRefused", `store answered ${response.status}`, response.status); + const message = await response.json(); + if (!record(message) || !record(message.result) || message.result.isError === true || !record(message.result.structuredContent)) { + throw new ConsoleError("ConsoleStoreReadRefused"); + } + return message.result.structuredContent; + }; + const description = await call("context.describe", {}); + const tables = Array.isArray(description.tables) ? description.tables.flatMap((item) => record(item) && typeof item.table === "string" && item.kind === "data" ? [item.table] : []) : []; + const turn = createTurn({ + tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read", table })), + tables: tables.map((name) => ({ name, kind: "data" })), + planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), + transport: { call: async ({ tool, maxRows, signal }) => { + const table = tool.slice("read:".length); + const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: maxRows }, signal); + const columns = strings(result.columns); + const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; + const sourced = rawRows.flatMap((row) => { + const citation = source(columns, row, table); + return citation ? [{ row, citation }] : []; + }); + return { + rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), + sources: sourced.map(({ citation }) => citation) + }; + } }, + synthesize: async function* ({ question, results, sources }) { + const response = await fetcher(new URL("chat/completions", `${modelEndpoint.replace(/\/$/, "")}/`), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: modelId, messages: [ + { role: "system", content: "Answer only from the supplied governed rows. Cite source IDs in square brackets. Decline unsupported claims." }, + { role: "user", content: JSON.stringify({ question, rows: results.flatMap((result) => result.rows), sources }) } + ] }) + }); + if (!response.ok) throw new ConsoleError("ConsoleModelUnavailable", `model answered ${response.status}`, 503); + const body2 = await response.json(); + const choices = record(body2) && Array.isArray(body2.choices) ? body2.choices : []; + const message = choices.length && record(choices[0]) ? choices[0].message : void 0; + if (!record(message) || typeof message.content !== "string") throw new ConsoleError("ConsoleModelUnavailable", "model answer absent", 503); + yield message.content; + } + }); + const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); + return { answer: answer.text, sources: answer.sources, widgets: [] }; + }; +} + // src/server.ts import { createServer } from "node:http"; @@ -117,11 +395,11 @@ function verifyCognitoSession(cookie, identity2) { } catch { return null; } - if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string")) return null; + if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string") || session.assertion !== void 0 && typeof session.assertion !== "string") return null; const grants = /* @__PURE__ */ new Set(); if (session.groups.includes(identity2.queryGroup)) grants.add("query"); if (session.groups.includes(identity2.adminGroup)) grants.add("admin"); - return { subject: session.subject, grants }; + return { subject: session.subject, grants, assertion: typeof session.assertion === "string" ? session.assertion : void 0 }; } function verifyAccess(assertion, identity2) { const parts = assertion.split("."); @@ -151,7 +429,7 @@ function verifyAccess(assertion, identity2) { const key = identity2.keys ? typeof header.kid === "string" ? identity2.keys.get(header.kid) : void 0 : typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; if (!key) return null; if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, signature)) return null; - return { subject: claims.sub, grants }; + return { subject: claims.sub, grants, assertion }; } function operatorFor(request, identity2) { if (identity2.kind === "access") { @@ -217,7 +495,7 @@ async function cognitoCallback(request, identity2) { if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group) => typeof group === "string") : []; const lifetime = Math.min(claims.exp, Math.floor(Date.now() / 1e3) + 3600); - const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime }, identity2.sessionSecret); + const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime, assertion: tokens.id_token }, identity2.sessionSecret); const destination = groups.includes(identity2.adminGroup) ? "/admin" : "/query"; return new Response(null, { status: 302, headers: { Location: destination, @@ -301,7 +579,12 @@ function createConsole(adapters) { return bodyFailure(error); } if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); - return json(await adapters.turn({ operator, store: input.store, question: input.question })); + try { + return json(await adapters.turn({ operator, store: input.store, question: input.question })); + } catch (error) { + if (error instanceof ConsoleError) return refusal(error.code, error.status); + throw error; + } } } else { const store = url.searchParams.get("store"); @@ -453,8 +736,11 @@ async function main() { if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); - let adapters = unavailableAdapters(); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; + let adapters = { + ...unavailableAdapters(), + turn: modulePath ? unavailableAdapters().turn : createLiveTurn({ stores: registry.entries, env: process.env }) + }; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); const module = await import(pathToFileURL(absolute).href); diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 0ee90294c..a86974d02 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -2,6 +2,7 @@ import { createPublicKey } from "node:crypto"; import { pathToFileURL } from "node:url"; import { isAbsolute, resolve } from "node:path"; import { registryFromEnv } from "../../gateway/src/index.ts"; +import { createLiveTurn } from "./live.ts"; import type { ConsoleAdapters, Identity } from "./index.ts"; import { serveConsole } from "./server.ts"; @@ -80,8 +81,11 @@ async function main(): Promise { if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); - let adapters: Pick & { read?: ConsoleAdapters["read"] } = unavailableAdapters(); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; + let adapters: Pick & { read?: ConsoleAdapters["read"] } = { + ...unavailableAdapters(), + turn: modulePath ? unavailableAdapters().turn : createLiveTurn({ stores: registry.entries, env: process.env }), + }; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); const module: { createAdapters?: AdapterFactory } = await import(pathToFileURL(absolute).href); diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index beab27127..3aa79291f 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -1,6 +1,7 @@ import { createHash, createHmac, createPublicKey, randomBytes, timingSafeEqual, verify, type KeyObject } from "node:crypto"; +import { ConsoleError } from "./turn.ts"; -export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin"> }; +export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin">; assertion?: string }; export type Store = { id: string; label: string }; export type AccessIdentity = { kind: "access"; @@ -43,7 +44,7 @@ export type ConsoleAdapters = { }; }; -type Session = { subject: string; groups: string[]; expiresAt: number }; +type Session = { subject: string; groups: string[]; expiresAt: number; assertion?: string }; function encoded(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString("base64url"); @@ -83,11 +84,12 @@ function verifyCognitoSession(cookie: string, identity: CognitoIdentity): Operat try { session = decode(parts[0]); } catch { return null; } if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || (session.expiresAt as number) <= Math.floor(Date.now() / 1000) || - !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string")) return null; + !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string") || + (session.assertion !== undefined && typeof session.assertion !== "string")) return null; const grants = new Set<"query" | "admin">(); if (session.groups.includes(identity.queryGroup)) grants.add("query"); if (session.groups.includes(identity.adminGroup)) grants.add("admin"); - return { subject: session.subject, grants }; + return { subject: session.subject, grants, assertion: typeof session.assertion === "string" ? session.assertion : undefined }; } function verifyAccess(assertion: string, identity: AccessIdentity): Operator | null { @@ -111,7 +113,7 @@ function verifyAccess(assertion: string, identity: AccessIdentity): Operator | n typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; if (!key) return null; if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, signature)) return null; - return { subject: claims.sub, grants }; + return { subject: claims.sub, grants, assertion }; } function operatorFor(request: Request, identity: Identity): Operator | null { @@ -180,7 +182,7 @@ async function cognitoCallback(request: Request, identity: CognitoIdentity): Pro if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group): group is string => typeof group === "string") : []; const lifetime = Math.min(claims.exp as number, Math.floor(Date.now() / 1000) + 3600); - const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime }, identity.sessionSecret); + const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime, assertion: tokens.id_token }, identity.sessionSecret); const destination = groups.includes(identity.adminGroup) ? "/admin" : "/query"; return new Response(null, { status: 302, headers: { Location: destination, @@ -264,7 +266,12 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req try { input = await body(request); } catch (error) { return bodyFailure(error); } if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); - return json(await adapters.turn({ operator, store: input.store, question: input.question })); + try { + return json(await adapters.turn({ operator, store: input.store, question: input.question })); + } catch (error) { + if (error instanceof ConsoleError) return refusal(error.code, error.status); + throw error; + } } } else { const store = url.searchParams.get("store"); diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts new file mode 100644 index 000000000..7c5c88c63 --- /dev/null +++ b/apps/console/src/live.ts @@ -0,0 +1,103 @@ +import type { StoreEntry } from "../../gateway/src/index.ts"; +import type { TurnInput, TurnResult } from "./index.ts"; +import { ConsoleError, createTurn, resolveReaderCredential, type Source, type ToolResult } from "./turn.ts"; + +type LiveOptions = { stores: StoreEntry[]; env: NodeJS.ProcessEnv; fetcher?: typeof fetch }; + +function record(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function strings(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; +} + +function source(columns: string[], row: unknown[], table: string): Source | null { + const at = (name: string) => row[columns.indexOf(name)]; + const idColumn = columns.find((name) => name.endsWith("_id") || name === "id"); + const urlColumn = columns.find((name) => name === "source_url" || name === "url"); + const id = idColumn && at(idColumn); + const url = urlColumn && at(urlColumn); + if (typeof id !== "string" || !id) return null; + const label = columns.includes("title") ? at("title") : columns.includes("summary") ? at("summary") : id; + return { id, label: typeof label === "string" && label ? label : `${table}: ${id}`, + ...(typeof url === "string" && /^https?:\/\//.test(url) ? { url } : {}) }; +} + +export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): (input: TurnInput) => Promise { + const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; + const modelId = env.CONTEXTFUL_MODEL_ID; + if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); + return async (input) => { + const store = stores.find((entry) => entry.id === input.store); + if (!store) throw new ConsoleError("ConsoleRequestMalformed"); + const shared = env[store.credentialName]; + const credential = store.auth === "exchange" ? await resolveReaderCredential({ + shared, + mint: async () => { + if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeRefused"); + const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jwt: input.operator.assertion }), + }); + if (!response.ok) throw new ConsoleError("ConsoleTokenExchangeRefused"); + const value: unknown = await response.json(); + if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeRefused"); + return value.token; + }, + }) : shared; + if (!credential) throw new ConsoleError("ConsoleTokenExchangeRefused"); + + let id = 0; + const call = async (name: string, args: Record, signal?: AbortSignal): Promise> => { + const response = await fetcher(new URL("/mcp", store.endpoint), { + method: "POST", signal, + headers: { "Authorization": `Bearer ${credential}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }), + }); + if (!response.ok) throw new ConsoleError("ConsoleStoreReadRefused", `store answered ${response.status}`, response.status); + const message: unknown = await response.json(); + if (!record(message) || !record(message.result) || message.result.isError === true || !record(message.result.structuredContent)) { + throw new ConsoleError("ConsoleStoreReadRefused"); + } + return message.result.structuredContent; + }; + + const description = await call("context.describe", {}); + const tables = Array.isArray(description.tables) ? description.tables.flatMap((item) => + record(item) && typeof item.table === "string" && item.kind === "data" ? [item.table] : []) : []; + const turn = createTurn({ + tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read" as const, table })), + tables: tables.map((name) => ({ name, kind: "data" as const })), + planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), + transport: { call: async ({ tool, maxRows, signal }): Promise => { + const table = tool.slice("read:".length); + const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: maxRows }, signal); + const columns = strings(result.columns); + const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; + const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; }); + return { + rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), + sources: sourced.map(({ citation }) => citation), + }; + } }, + synthesize: async function* ({ question, results, sources }) { + const response = await fetcher(new URL("chat/completions", `${modelEndpoint.replace(/\/$/, "")}/`), { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: modelId, messages: [ + { role: "system", content: "Answer only from the supplied governed rows. Cite source IDs in square brackets. Decline unsupported claims." }, + { role: "user", content: JSON.stringify({ question, rows: results.flatMap((result) => result.rows), sources }) }, + ] }), + }); + if (!response.ok) throw new ConsoleError("ConsoleModelUnavailable", `model answered ${response.status}`, 503); + const body: unknown = await response.json(); + const choices = record(body) && Array.isArray(body.choices) ? body.choices : []; + const message = choices.length && record(choices[0]) ? choices[0].message : undefined; + if (!record(message) || typeof message.content !== "string") throw new ConsoleError("ConsoleModelUnavailable", "model answer absent", 503); + yield message.content; + }, + }); + const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); + return { answer: answer.text, sources: answer.sources, widgets: [] }; + }; +} diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts index 12dab8468..d4e808969 100644 --- a/apps/console/src/turn.ts +++ b/apps/console/src/turn.ts @@ -73,7 +73,7 @@ export async function resolveReaderCredential(options: { // A refused mint reaches the shared credential only when the store has one. } if (options.shared) return options.shared; - throw new ConsoleError("ConsoleTokenExchangeRefused"); + throw new ConsoleError("ConsoleTokenExchangeRefused", "ConsoleTokenExchangeRefused", 403); } export class OverlayCache { diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index 00a8c8dc8..7e60aac6e 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -63,11 +63,13 @@ test("unadmitted and mutating calls dispatch nothing", async () => { assert.deepEqual(write.calls, []); }); +// spec: surface.ground.org-face-read-only@030074cb test("organization packs refuse writes at startup", () => { assert.throws(() => validatePack({ name: "shared", face: "organization", tools: [{ name: "erase", kind: "write" }] }), (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleWriteToolOnOrgFace"); }); +// spec: surface.ground.direct-file-read@4edf9c49 test("direct file table functions refuse before dispatch", async () => { const { turn, calls } = harness({ tools: [{ name: "query", pack: "data", kind: "read", access: "direct-file" }] }); await assert.rejects(turn.ask({ question: "Read", packs: ["data"] }), @@ -157,6 +159,7 @@ test("planner scaffolding lists data tables while memory relations remain outsid assert.deepEqual(plannerTables, [{ name: "events", kind: "data" }]); }); +// spec: surface.plan-turn.planner-reached-memory@a6f057a5 test("a planner call targeting a memory relation dispatches nothing", async () => { const { turn, calls } = harness({ tables: [{ name: "memory_entries", kind: "memory" }], @@ -234,6 +237,7 @@ test("distinct registered tools share one data table's row budget", async () => assert.deepEqual(limits, [5000, 2000]); }); +// spec: surface.set-vantage.sample-labels@95b1fd75 test("a table contributes at most three sampled arrival labels", () => { assert.deepEqual(sampleArrivals([{ table: "events", labels: ["a", "b", "c", "d"] }]), [{ table: "events", labels: ["a", "b", "c"] }]); @@ -252,6 +256,7 @@ test("overlay cache includes misses, expires after five minutes and truncates at assert.equal(reads, 2); }); +// spec: surface.speak.redactor-lookahead@2d3a1762 test("streaming redactor masks identifiers split across chunks", () => { const redactor = new StreamingRedactor(["SECRET-ID"]); const output = redactor.push("before SEC") + redactor.push("RET-ID after") + redactor.finish(); diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 37062fc43..49ce70d54 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -108,7 +108,7 @@ pub struct HttpResponse { } impl HttpResponse { - fn json(status: u16, body: &Value) -> HttpResponse { + pub fn json(status: u16, body: &Value) -> HttpResponse { HttpResponse { status, headers: vec![("Content-Type".into(), "application/json".into())], body: body.to_string().into_bytes() } } @@ -195,6 +195,7 @@ pub struct HttpFace<'a, C> { admitting: Admitting<'a, C>, ceiling: usize, in_flight: AtomicUsize, + exchange: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, } /// A request slot held while one request is in flight. @@ -241,7 +242,14 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { let ceiling = ceiling(max_in_flight)?; audience(Some(admitting.audience))?; let tools = Tools::new(face, clock, record)?; - Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0) }) + Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0), exchange: None }) + } + + /// The binary's exchange route mints the reader credential without putting issuer + /// signing material in the read-transport package. + pub fn with_exchange(mut self, exchange: &'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)) -> Self { + self.exchange = Some(exchange); + self } /// Accept connections on `listener` until it fails. Each accepted connection takes a @@ -329,6 +337,8 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { (MCP_PATH, _) => { HttpResponse::message(405, "`/mcp` answers POST; the face holds no session and opens no server stream").with("Allow", "POST") } + ("/auth/exchange", "POST") if self.exchange.is_some() => self.exchange.expect("checked above")(request), + ("/auth/exchange", _) if self.exchange.is_some() => HttpResponse::message(405, "`/auth/exchange` answers POST").with("Allow", "POST"), (other, _) => HttpResponse::message(404, format!("no route `{other}`; the protocol endpoint is `{MCP_PATH}`")), } } diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 05953d645..2578fc638 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -13,6 +13,11 @@ use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; use contextful_agent::http::{audience, ceiling, Admitting, HttpFace}; +use contextful_core::issue::{IssuancePolicy, MintContext, NodeRole}; +use contextful_core::ports::{Clock, SigningPort}; +use contextful_policy::exchange::answer as exchange_answer; +use contextful_policy::issue::{SeedSigner, DEFAULT_SEED_PATH}; +use contextful_policy::possession::ProofChecker; use contextful_policy::audit::AuditLog; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; @@ -108,7 +113,28 @@ pub fn run(args: ServeArgs) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; - let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)).map_err(anyhow::Error::msg)?; + let exchange = crate::token::configured_exchange(&root)?; + let mint_material = if exchange.is_some() { + let policy = std::fs::read_to_string(root.join(IssuancePolicy::PATH))?; + let issuance = IssuancePolicy::parse(&policy).map_err(|e| anyhow::anyhow!("{e:?}"))?; + let signer = SeedSigner::resolve(Some(&root.join(DEFAULT_SEED_PATH)))?; + Some((issuance, signer)) + } else { + None + }; + let proofs = ProofChecker::new(SystemClock); + let exchange_route = |request: &contextful_agent::http::HttpRequest| { + let Some((issuance, signer)) = mint_material.as_ref() else { + return contextful_agent::http::HttpResponse::json(503, + &serde_json::json!({ "error": { "http": 503, "identifier": "ExchangeUnconfigured" } })); + }; + let ctx = MintContext { node: NodeRole::Primary, signer: signer as &dyn SigningPort, clock: &clock as &dyn Clock }; + let result = exchange_answer(exchange.as_ref(), &request.body, request.header("DPoP"), &proofs, issuance, &ctx); + contextful_agent::http::HttpResponse::json(result.status, &result.body) + }; + let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)) + .map_err(anyhow::Error::msg)? + .with_exchange(&exchange_route); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); http.serve(listener)?; diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index f5b5761d0..53f464396 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -113,13 +113,13 @@ fn served_exchange_mints_a_reader_credential_for_the_read_face() { std::fs::create_dir_all(p.join(".contextful/exchange")).unwrap(); std::fs::write(p.join(".contextful/exchange/policy.toml"), "expected_iss = \"https://login.example.test/\"\nexpected_aud = \"console\"\nrole_claim = \"roles\"\n\ - [subject_map]\non_behalf_of = { claim = \"sub\", template = \"user://{}\" }\n\ + [subject_map]\non_behalf_of = { claim = \"sub\", template = \"user://{}\" }\nzone = { claim = \"zone\" }\n\ [[role_grants.reader]]\nactions = [\"read\"]\ntables = [\"research/*\"]\n").unwrap(); std::fs::write(p.join(".contextful/exchange/verify.key"), "exchange-secret").unwrap(); let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs(); let jwt = jsonwebtoken::encode(&jsonwebtoken::Header::new(jsonwebtoken::Algorithm::HS256), &json!({ "iss": "https://login.example.test/", "aud": "console", "exp": now + 300, - "sub": "reader@example.test", "roles": ["reader"] }), + "sub": "reader@example.test", "zone": "on-prem:hq", "roles": ["reader"] }), &jsonwebtoken::EncodingKey::from_secret(b"exchange-secret")).unwrap(); let (_listener, addr) = serve(p, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); diff --git a/crates/contextful-context/src/read/face.rs b/crates/contextful-context/src/read/face.rs index b746307a3..91014a282 100644 --- a/crates/contextful-context/src/read/face.rs +++ b/crates/contextful-context/src/read/face.rs @@ -473,6 +473,7 @@ impl Face { .map(|r| { json!({ "table": r.name(), + "kind": if self.memory.table(r.name()).is_some() { "memory" } else { "data" }, "description": self.decl(r.name()).agent_description, "zone_admitted": session.zone_admitted(r.name()), }) diff --git a/spec/pins.toml b/spec/pins.toml index e4f2f2b3b..bb44c6eaa 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -57,11 +57,11 @@ [floor] assurance = 136 authority = 204 -connector = 207 +connector = 218 corpus = 40 disclosure = 54 read = 163 run = 261 store = 200 -surface = 51 +surface = 57 topology = 38 diff --git a/spec/status.md b/spec/status.md index 28f95df48..aaff60ee5 100644 --- a/spec/status.md +++ b/spec/status.md @@ -7,15 +7,15 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `assurance` | 2 | 15 | 231 | 59 | 22 | 12 | 136 | 0 | 136 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | -| `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 207 | +| `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 218 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 40 | 0 | 40 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 51 | 0 | 51 | +| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 57 | 0 | 57 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1365 | 0 | | +| **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1371 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 49 | 6 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -1365,6 +1365,10 @@ Unscheduled operations: 10. | `surface.fire.store-driven-body` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_naming_an_unregistered_body_raises_job_body_unregistered` | performed | | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | +| `surface.ground.direct-file-read` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::direct file table functions refuse before dispatch` | performed | +| `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | +| `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | +| `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | | `surface.reconcile.loopback-only` | `crates/contextful-cli/tests/integration/pipeline.rs::a_control_url_outside_loopback_arms_nothing` | performed | @@ -1374,6 +1378,8 @@ Unscheduled operations: 10. | `surface.reside.region-entries` | `crates/contextful-core/tests/integration/surface/reside.rs::an_allow_set_holds_sixteen_entries` | performed | | `surface.reside.region-mismatch` | `crates/contextful-cli/tests/integration/pipeline.rs::a_resource_outside_the_residency_allow_set_serves_nothing` | performed | | `surface.reside.site-regions` | `crates/contextful-cli/tests/integration/sync/mod.rs::two_sites_declaring_different_residency_diverge_at_push` | performed | +| `surface.set-vantage.sample-labels` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a table contributes at most three sampled arrival labels` | performed | +| `surface.speak.redactor-lookahead` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::streaming redactor masks identifiers split across chunks` | performed | | `topology.compose.script-runtime` | `tools/ci/tests/integration/topology.rs::a_crate_linking_a_javascript_runtime_is_refused` | performed | | `topology.compose.undeclared-crossing` | `tools/ci/tests/integration/topology.rs::a_run_path_crate_reaching_a_read_path_crate_is_refused` | performed | | `topology.compose.vendor-sdk` | `tools/ci/tests/integration/topology.rs::a_crate_declaring_a_model_vendor_sdk_is_refused` | performed | From 9af3783ccd3078efc731190eef5bcb9f7ac980a0 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:02:30 +0800 Subject: [PATCH 061/155] fix(console): bound briefs and withhold unsafe views --- apps/console/dist/server.js | 43 +++++++++++++++++++++++++------------ apps/console/src/brief.ts | 28 +++++++++++++++--------- apps/console/src/entry.ts | 9 ++++++-- apps/console/src/index.ts | 19 +++++++++++----- 4 files changed, 68 insertions(+), 31 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 4c1529d72..0856b5034 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -90,21 +90,28 @@ function topicMatch(conclusion, row) { const shared = [...left].filter((word) => right.has(word)); return shared.length >= 2 && shared.some((word) => subject.has(word)); } +async function withinBriefBudget(budgetMs, action) { + if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); + let timer; + try { + return await Promise.race([ + Promise.resolve().then(action), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), budgetMs); + }) + ]); + } finally { + if (timer !== void 0) clearTimeout(timer); + } +} async function deriveBrief(input) { + if (!Number.isFinite(input.budgetMs) || input.budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); if (input.session.turns !== 0 || input.session.vantage !== "present") return null; const live = input.conclusions.filter((entry) => entry.live); if (live.length === 0) return null; const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); - let timer; try { - const source = input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? []); - const budgetMs = input.budgetMs; - const arrivals = budgetMs === void 0 ? await source : await Promise.race([ - source, - new Promise((_, reject) => { - timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), Math.max(0, budgetMs)); - }) - ]); + const arrivals = await withinBriefBudget(input.budgetMs, () => input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? [])); const since = input.now - windowDays * 864e5; const recent = arrivals.filter((row) => { const arrived = Date.parse(row.arrivedAt); @@ -114,8 +121,6 @@ async function deriveBrief(input) { return subjects.length ? { windowDays, subjects } : null; } catch { throw new Error("ConsoleBriefUnavailable"); - } finally { - if (timer !== void 0) clearTimeout(timer); } } @@ -399,9 +404,14 @@ function createConsole(adapters) { if (request.method === "GET" && path === "/query/api/brief") { const store = url.searchParams.get("store"); if (!store || !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); - if (!adapters.brief) return refusal("ConsoleAdapterUnavailable", 503); + const brief = adapters.brief; + if (!brief) return refusal("ConsoleAdapterUnavailable", 503); + const budgetMs = adapters.briefBudgetMs ?? NaN; try { - const card = await deriveBrief(await adapters.brief(operator, store)); + const card = await withinBriefBudget(budgetMs, async () => { + const input = await brief(operator, store); + return deriveBrief({ ...input, budgetMs }); + }); return card ? json(card) : new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); } catch { return new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); @@ -423,7 +433,9 @@ function createConsole(adapters) { } catch (error) { return refusal(error instanceof Error ? error.message : "VisibilityShareAffordance"); } - const built = turn.resultRows ? sanitizeView(buildView(turn.resultRows), (value) => adapters.redactView?.(operator, value) ?? value) : null; + const redactView = adapters.redactView; + const clean = turn.resultRows && redactView ? sanitizeView({ component: "table.v1", props: turn.resultRows }, (value) => redactView(operator, value)) : null; + const built = clean ? buildView(clean.props) : null; return json({ answer: answer.answer, sources: turn.sources, widgets: built ? [built] : [] }); } } else { @@ -592,6 +604,9 @@ async function main() { stores, adminCapability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, turn: adapters.turn, + brief: adapters.brief, + briefBudgetMs: adapters.briefBudgetMs, + redactView: adapters.redactView, read: adapters.read ?? { list: async () => stores }, control: adapters.control }, () => origin); diff --git a/apps/console/src/brief.ts b/apps/console/src/brief.ts index 76de076ca..7a143e468 100644 --- a/apps/console/src/brief.ts +++ b/apps/console/src/brief.ts @@ -21,22 +21,32 @@ export type BriefInput = { loadArrivals?: () => Promise; now: number; windowDays?: number; - budgetMs?: number; + budgetMs: number; }; +export async function withinBriefBudget(budgetMs: number, action: () => Promise): Promise { + if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + Promise.resolve().then(action), + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), budgetMs); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + export async function deriveBrief(input: BriefInput): Promise { + if (!Number.isFinite(input.budgetMs) || input.budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); if (input.session.turns !== 0 || input.session.vantage !== "present") return null; const live = input.conclusions.filter((entry) => entry.live); if (live.length === 0) return null; const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); - let timer: ReturnType | undefined; try { - const source = input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? []); - const budgetMs = input.budgetMs; - const arrivals = budgetMs === undefined ? await source : await Promise.race([ - source, - new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), Math.max(0, budgetMs)); }), - ]); + const arrivals = await withinBriefBudget(input.budgetMs, () => input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? [])); const since = input.now - windowDays * 86_400_000; const recent = arrivals.filter((row) => { const arrived = Date.parse(row.arrivedAt); @@ -47,7 +57,5 @@ export async function deriveBrief(input: BriefInput): Promise { return subjects.length ? { windowDays, subjects } : null; } catch { throw new Error("ConsoleBriefUnavailable"); - } finally { - if (timer !== undefined) clearTimeout(timer); } } diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 0ee90294c..67b9566fa 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -5,7 +5,9 @@ import { registryFromEnv } from "../../gateway/src/index.ts"; import type { ConsoleAdapters, Identity } from "./index.ts"; import { serveConsole } from "./server.ts"; -type AdapterFactory = (input: { stores: ConsoleAdapters["stores"]; env: NodeJS.ProcessEnv }) => Promise & { read?: ConsoleAdapters["read"] }>; +type HostedAdapters = Pick & + Partial>; +type AdapterFactory = (input: { stores: ConsoleAdapters["stores"]; env: NodeJS.ProcessEnv }) => Promise; function required(name: string): string { const value = process.env[name]; @@ -80,7 +82,7 @@ async function main(): Promise { if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); - let adapters: Pick & { read?: ConsoleAdapters["read"] } = unavailableAdapters(); + let adapters: HostedAdapters = unavailableAdapters(); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); @@ -94,6 +96,9 @@ async function main(): Promise { stores, adminCapability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, turn: adapters.turn, + brief: adapters.brief, + briefBudgetMs: adapters.briefBudgetMs, + redactView: adapters.redactView, read: adapters.read ?? { list: async () => stores }, control: adapters.control, }, () => origin); diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 906d97e1c..2784b5248 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -1,5 +1,5 @@ import { createHash, createHmac, createPublicKey, randomBytes, timingSafeEqual, verify, type KeyObject } from "node:crypto"; -import { deriveBrief, type BriefInput } from "./brief.ts"; +import { deriveBrief, withinBriefBudget, type BriefInput } from "./brief.ts"; import { answerDelivery } from "./publish.ts"; import { buildView, sanitizeView, type ResultRows } from "./render.ts"; @@ -36,7 +36,8 @@ export type ConsoleAdapters = { stores: Store[]; adminCapability?: string; turn: (input: TurnInput) => Promise; - brief?: (operator: Operator, store: string) => Promise; + brief?: (operator: Operator, store: string) => Promise>; + briefBudgetMs?: number; redactView?: (operator: Operator, value: string) => string; read: { list: (operator: Operator) => Promise }; control: { @@ -267,9 +268,14 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req if (request.method === "GET" && path === "/query/api/brief") { const store = url.searchParams.get("store"); if (!store || !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); - if (!adapters.brief) return refusal("ConsoleAdapterUnavailable", 503); + const brief = adapters.brief; + if (!brief) return refusal("ConsoleAdapterUnavailable", 503); + const budgetMs = adapters.briefBudgetMs ?? NaN; try { - const card = await deriveBrief(await adapters.brief(operator, store)); + const card = await withinBriefBudget(budgetMs, async () => { + const input = await brief(operator, store); + return deriveBrief({ ...input, budgetMs }); + }); return card ? json(card) : new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); } catch { return new Response(null, { status: 204, headers: { "Cache-Control": "no-store" } }); @@ -288,7 +294,10 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req } catch (error) { return refusal(error instanceof Error ? error.message : "VisibilityShareAffordance"); } - const built = turn.resultRows ? sanitizeView(buildView(turn.resultRows), (value) => adapters.redactView?.(operator, value) ?? value) : null; + const redactView = adapters.redactView; + const clean = turn.resultRows && redactView ? + sanitizeView({ component: "table.v1", props: turn.resultRows }, (value) => redactView(operator, value)) : null; + const built = clean ? buildView(clean.props) : null; return json({ answer: answer.answer, sources: turn.sources, widgets: built ? [built] : [] }); } } else { From 6130cdba64b7dbbfcced50b4665a9b5d8a2e3f8b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:06:36 +0800 Subject: [PATCH 062/155] test(console): expire synchronous brief work --- apps/console/test/console.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 0825cbcb5..20968ab3d 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -415,3 +415,25 @@ test("Query brief expires while its adapter is still pending", async () => { const response = await Promise.race([request, new Promise((_, reject) => setTimeout(() => reject(new Error("brief hung")), 50))]); assert.equal(response.status, 204); }); + +test("Query brief discards a card after synchronous adapter work exceeds its budget", async () => { + const now = Date.parse("2026-01-08T12:00:00Z"); + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + briefBudgetMs: 1, + brief: async () => { + const until = performance.now() + 25; + while (performance.now() < until) { /* synchronous adapter work */ } + return { session: { turns: 0, vantage: "present" }, now, + conclusions: [{ subject: "Acme", text: "Acme filings need review", live: true }], + arrivals: [{ id: "a", label: "Acme filing", topics: ["filings"], arrivedAt: "2026-01-08T11:00:00Z" }], + }; + }, + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const response = await app.fetch(new Request("https://console.example/query/api/brief?store=field-notes", { headers: access(queryAudience) })); + assert.equal(response.status, 204); +}); From 307f5341422289a37925a4283a05a0d3dcc065d9 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:08:15 +0800 Subject: [PATCH 063/155] fix(console): check elapsed brief budget after work --- apps/console/dist/server.js | 40 ++++++++++++++++++++++++++----------- apps/console/src/brief.ts | 38 ++++++++++++++++++++++------------- 2 files changed, 52 insertions(+), 26 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 0856b5034..b21ec367f 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -92,10 +92,17 @@ function topicMatch(conclusion, row) { } async function withinBriefBudget(budgetMs, action) { if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); + const deadline = performance.now() + budgetMs; + const checkDeadline = () => { + if (performance.now() >= deadline) throw new Error("ConsoleBriefUnavailable"); + }; let timer; try { return await Promise.race([ - Promise.resolve().then(action), + Promise.resolve().then(() => action(checkDeadline)).then((value) => { + checkDeadline(); + return value; + }), new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), budgetMs); }) @@ -106,19 +113,28 @@ async function withinBriefBudget(budgetMs, action) { } async function deriveBrief(input) { if (!Number.isFinite(input.budgetMs) || input.budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); - if (input.session.turns !== 0 || input.session.vantage !== "present") return null; - const live = input.conclusions.filter((entry) => entry.live); - if (live.length === 0) return null; - const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); try { - const arrivals = await withinBriefBudget(input.budgetMs, () => input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? [])); - const since = input.now - windowDays * 864e5; - const recent = arrivals.filter((row) => { - const arrived = Date.parse(row.arrivedAt); - return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + return await withinBriefBudget(input.budgetMs, async (checkDeadline) => { + if (input.session.turns !== 0 || input.session.vantage !== "present") return null; + const live = input.conclusions.filter((entry) => { + checkDeadline(); + return entry.live; + }); + if (live.length === 0) return null; + const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); + const arrivals = input.loadArrivals ? await input.loadArrivals() : input.arrivals ?? []; + const since = input.now - windowDays * 864e5; + const recent = arrivals.filter((row) => { + checkDeadline(); + const arrived = Date.parse(row.arrivedAt); + return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + }); + const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => { + checkDeadline(); + return topicMatch(entry, row); + }).slice(0, 3) })).filter((entry) => entry.articles.length > 0).slice(0, 3); + return subjects.length ? { windowDays, subjects } : null; }); - const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => topicMatch(entry, row)).slice(0, 3) })).filter((entry) => entry.articles.length > 0).slice(0, 3); - return subjects.length ? { windowDays, subjects } : null; } catch { throw new Error("ConsoleBriefUnavailable"); } diff --git a/apps/console/src/brief.ts b/apps/console/src/brief.ts index 7a143e468..e9b947b5f 100644 --- a/apps/console/src/brief.ts +++ b/apps/console/src/brief.ts @@ -24,12 +24,19 @@ export type BriefInput = { budgetMs: number; }; -export async function withinBriefBudget(budgetMs: number, action: () => Promise): Promise { +export async function withinBriefBudget(budgetMs: number, action: (checkDeadline: () => void) => Promise): Promise { if (!Number.isFinite(budgetMs) || budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); + const deadline = performance.now() + budgetMs; + const checkDeadline = () => { + if (performance.now() >= deadline) throw new Error("ConsoleBriefUnavailable"); + }; let timer: ReturnType | undefined; try { return await Promise.race([ - Promise.resolve().then(action), + Promise.resolve().then(() => action(checkDeadline)).then((value) => { + checkDeadline(); + return value; + }), new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("ConsoleBriefUnavailable")), budgetMs); }), @@ -41,20 +48,23 @@ export async function withinBriefBudget(budgetMs: number, action: () => Promi export async function deriveBrief(input: BriefInput): Promise { if (!Number.isFinite(input.budgetMs) || input.budgetMs <= 0) throw new Error("ConsoleBriefUnavailable"); - if (input.session.turns !== 0 || input.session.vantage !== "present") return null; - const live = input.conclusions.filter((entry) => entry.live); - if (live.length === 0) return null; - const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); try { - const arrivals = await withinBriefBudget(input.budgetMs, () => input.loadArrivals ? input.loadArrivals() : Promise.resolve(input.arrivals ?? [])); - const since = input.now - windowDays * 86_400_000; - const recent = arrivals.filter((row) => { - const arrived = Date.parse(row.arrivedAt); - return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + return await withinBriefBudget(input.budgetMs, async (checkDeadline) => { + if (input.session.turns !== 0 || input.session.vantage !== "present") return null; + const live = input.conclusions.filter((entry) => { checkDeadline(); return entry.live; }); + if (live.length === 0) return null; + const windowDays = Math.max(0, Math.min(input.windowDays ?? 7, 7)); + const arrivals = input.loadArrivals ? await input.loadArrivals() : input.arrivals ?? []; + const since = input.now - windowDays * 86_400_000; + const recent = arrivals.filter((row) => { + checkDeadline(); + const arrived = Date.parse(row.arrivedAt); + return Number.isFinite(arrived) && arrived >= since && arrived <= input.now; + }); + const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => { checkDeadline(); return topicMatch(entry, row); }).slice(0, 3) })) + .filter((entry) => entry.articles.length > 0).slice(0, 3); + return subjects.length ? { windowDays, subjects } : null; }); - const subjects = live.map((entry) => ({ subject: entry.subject, articles: recent.filter((row) => topicMatch(entry, row)).slice(0, 3) })) - .filter((entry) => entry.articles.length > 0).slice(0, 3); - return subjects.length ? { windowDays, subjects } : null; } catch { throw new Error("ConsoleBriefUnavailable"); } From ecb74723385061a1c1607f6cf1d855890d3c7e32 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:11:53 +0800 Subject: [PATCH 064/155] test(query): require scoped reads and fail-closed exchange --- apps/console/test/live.test.ts | 38 ++++++++++++++++++- apps/console/test/turn.test.ts | 11 +++++- .../contextful-cli/tests/integration/serve.rs | 16 ++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index 85e53ed8d..5e0f5af13 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -11,7 +11,7 @@ const store: StoreEntry = { bindingName: "FIELD_NOTES", }; -function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[][]; tables?: Array<{ table: string; kind: string }> } = {}) { +function fixture(options: { exchange?: boolean; exchangeStatus?: number; shared?: string; rows?: unknown[][]; tables?: Array<{ table: string; kind: string }> } = {}) { const calls: Array<{ url: string; authorization: string | null; body: Record }> = []; let modelCalls = 0; let modelBody: Record | null = null; @@ -19,7 +19,10 @@ function fixture(options: { exchange?: boolean; shared?: string; rows?: unknown[ const url = String(input); const body = JSON.parse(String(init?.body ?? "{}")) as Record; calls.push({ url, authorization: new Headers(init?.headers).get("authorization"), body }); - if (url.endsWith("/auth/exchange")) return Response.json(options.exchange === false ? { error: { identifier: "ExchangeAssertionInvalid" } } : { token: "reader-token" }, { status: options.exchange === false ? 403 : 200 }); + if (url.endsWith("/auth/exchange")) { + const status = options.exchangeStatus ?? (options.exchange === false ? 403 : 200); + return Response.json(status === 200 ? { token: "reader-token" } : { error: { identifier: status === 503 ? "ExchangeMaterialMissing" : "ExchangeAssertionInvalid" } }, { status }); + } if (url.endsWith("/mcp")) { const params = body.params as Record; if (params.name === "context.describe") return Response.json({ jsonrpc: "2.0", id: body.id, result: { structuredContent: { @@ -70,6 +73,37 @@ test("Query falls back to a configured shared credential only after exchange ref assert.equal(refused.calls.filter((call) => call.url.endsWith("/mcp")).length, 0); }); +test("Query does not use a shared credential after exchange service failure", async () => { + const { turn, calls } = fixture({ exchangeStatus: 503, shared: "shared-token" }); + await assert.rejects(turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" })); + assert.equal(calls.filter((call) => call.url.endsWith("/mcp")).length, 0); +}); + +test("Query reads only a question-matched data table and refuses an unclear selection", async () => { + const tables = [{ table: "filings", kind: "data" }, { table: "salaries", kind: "data" }]; + const matched = fixture({ tables }); + await matched.turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" }); + assert.deepEqual(matched.calls.filter((call) => (call.body.params as Record | undefined)?.name === "context.query") + .map((call) => (call.body.params as Record).arguments), + [{ sql: 'SELECT * FROM "filings"', limit: 5000 }]); + const unclear = fixture({ tables }); + await assert.rejects(unclear.turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "What changed?" }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleUngroundedAnswer"); + assert.equal(unclear.calls.filter((call) => call.url.endsWith("/mcp") && (call.body.params as Record).name === "context.query").length, 0); +}); + +test("Query sends the model only rows represented by its eight visible sources", async () => { + const rows = Array.from({ length: 9 }, (_, index) => [`filing-${index + 1}`, `Filing ${index + 1}`, `https://example.test/filing-${index + 1}`]); + const { turn, modelBody } = fixture({ rows }); + const result = await turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" }); + assert.equal(result.sources.length, 8); + assert.doesNotMatch(JSON.stringify(modelBody()), /filing-9/); +}); + test("Query never sends rows without provenance to the model", async () => { const { turn, modelCalls } = fixture({ rows: [["", "Northwind filed on Monday", ""]] }); await assert.rejects(turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index 7e60aac6e..2667323ed 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -143,12 +143,21 @@ test("vantage accepts calendar days and instants, and rejects invalid dates", () }); test("refused per-reader mint falls back only to an installed shared credential", async () => { - const mint = async () => { throw new Error("denied"); }; + const mint = async () => { throw new ConsoleError("ConsoleTokenExchangeRefused", "denied", 403); }; assert.equal(await resolveReaderCredential({ mint, shared: "shared-token" }), "shared-token"); await assert.rejects(resolveReaderCredential({ mint }), (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused"); }); +test("reader credential resolution preserves transport and service failures", async () => { + const outage = new ConsoleError("ConsoleTokenExchangeUnavailable", "outage", 503); + await assert.rejects(resolveReaderCredential({ mint: async () => { throw outage; }, shared: "shared-token" }), + (error: unknown) => error === outage); + const network = new TypeError("network offline"); + await assert.rejects(resolveReaderCredential({ mint: async () => { throw network; }, shared: "shared-token" }), + (error: unknown) => error === network); +}); + test("planner scaffolding lists data tables while memory relations remain outside it", async () => { let plannerTables: unknown[] = []; const { turn } = harness({ diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 53f464396..2b21a4891 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -106,6 +106,22 @@ fn query() -> Value { json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } }) } +#[test] +fn served_exchange_without_policy_returns_the_unconfigured_refusal() { + let (dir, public) = project(); + let (_listener, addr) = serve(dir.path(), &["serve", "--http", "127.0.0.1:0", "--audience", AUD, + "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let body = "not-json"; + let mut stream = TcpStream::connect(&addr).unwrap(); + write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); + let mut raw = String::new(); + stream.read_to_string(&mut raw).unwrap(); + let (head, data) = raw.split_once("\r\n\r\n").unwrap(); + assert_eq!(head.split(' ').nth(1), Some("404")); + let answer: Value = serde_json::from_str(data).unwrap(); + assert_eq!(answer["error"]["identifier"], "ExchangeUnconfigured"); +} + #[test] fn served_exchange_mints_a_reader_credential_for_the_read_face() { let (dir, public) = project(); From c768c2714c755a2dff5d6cd4bce10059f5766696 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:14:28 +0800 Subject: [PATCH 065/155] test(console): require live Admin workflow adapter and applied fixture --- apps/console/test/control.test.ts | 51 ++++++++++++++++++++++ crates/acceptance/tests/integration/m12.rs | 17 ++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 apps/console/test/control.test.ts diff --git a/apps/console/test/control.test.ts b/apps/console/test/control.test.ts new file mode 100644 index 000000000..769104e11 --- /dev/null +++ b/apps/console/test/control.test.ts @@ -0,0 +1,51 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { generateKeyPairSync, sign } from "node:crypto"; +import { createConsole } from "../src/index.ts"; +import { createLiveControl } from "../src/control.ts"; + +const { privateKey, publicKey } = generateKeyPairSync("rsa", { modulusLength: 2048 }); +const token = (() => { + const header = Buffer.from(JSON.stringify({ alg: "RS256" })).toString("base64url"); + const claims = Buffer.from(JSON.stringify({ iss: "https://access.example", aud: "admin-app", sub: "operator-1", exp: Math.floor(Date.now() / 1000) + 300 })).toString("base64url"); + return `${header}.${claims}.${sign("RSA-SHA256", Buffer.from(`${header}.${claims}`), privateKey).toString("base64url")}`; +})(); + +test("Admin route retrieves applied workflow state and applies through the server capability", async () => { + const calls: { path: string; authorization: string | null; body: string }[] = []; + const fetcher: typeof fetch = async (input, init) => { + const url = new URL(String(input)); + calls.push({ path: url.pathname, authorization: new Headers(init?.headers).get("authorization"), body: String(init?.body ?? "") }); + return Response.json(url.pathname === "/control/workflows" ? + { applied: 1, pipelines: [{ id: "filings-flow", tables: ["filings"], schedule: "every 1h" }], runs: {} } : + { applied: 2 }); + }; + const control = createLiveControl({ stores: [{ id: "field-notes", endpoint: "http://127.0.0.1:9999" }], capability: "engine-admin-secret", fetcher }); + const app = createConsole({ + identity: { kind: "access", issuer: "https://access.example", queryAudience: "query-app", adminAudience: "admin-app", publicKey }, + stores: [{ id: "field-notes", label: "Field notes" }], adminCapability: "engine-admin-secret", + turn: async () => ({ answer: "", sources: [], widgets: [] }), read: { list: async () => [] }, control, + }); + const headers = { "cf-access-jwt-assertion": token }; + const listing = await app.fetch(new Request("https://console.example/admin/api/workflows", { headers })); + assert.equal(listing.status, 200); + assert.equal((await listing.json() as { pipelines: { id: string }[] }).pipelines[0].id, "filings-flow"); + const apply = await app.fetch(new Request("https://console.example/admin/api/apply", { + method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", id: "filings-flow" }), + })); + assert.equal(apply.status, 200); + assert.deepEqual(calls, [ + { path: "/control/workflows", authorization: "Bearer engine-admin-secret", body: "" }, + { path: "/control/apply", authorization: "Bearer engine-admin-secret", body: '{"id":"filings-flow"}' }, + ]); + assert.doesNotMatch(JSON.stringify(await apply.json()), /engine-admin-secret/); +}); + +test("Admin control refuses ambiguous stores and keeps edit unavailable", async () => { + const control = createLiveControl({ + stores: [{ id: "one", endpoint: "http://127.0.0.1:1" }, { id: "two", endpoint: "http://127.0.0.1:2" }], + capability: "secret", fetcher: async () => { throw new Error("network must not be called"); }, + }); + await assert.rejects(control.workflows({ subject: "operator", grants: new Set(["admin"]) }, null), /ConsoleStoreSelectionRequired/); + await assert.rejects(control.edit({}, "secret", { subject: "operator", grants: new Set(["admin"]) }), /ConsoleAdapterUnavailable/); +}); diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index 985a42875..1335f0c55 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -80,22 +80,28 @@ fn request(address: &str, method: &str, path: &str, token: Option<&str>, body: O } #[test] -#[ignore = "milestone 12 is open: the console server and client library are absent"] +#[ignore = "milestone 12 is open: Query live turn integration remains separate"] fn m12_console() { let cf = bin("contextful"); let repo = GitRepo::init(); - repo.write("contextful.toml", "authoring_posture = \"per_request\"\n[[pipeline]]\nid = \"filings-flow\"\n[[pipeline.tables]]\nname = \"filings\"\n"); + repo.write("contextful.toml", "authoring_posture = \"per_request\"\n"); + repo.write("pipelines/filings.toml", "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1h\"\ntables = [\"filings\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"); repo.write("filings.jsonl", &json!({ "filing_id": "filing-1", "publisher": "Northwind", "summary": "Northwind filed on Monday", "source_url": "https://example.test/filing-1" }).to_string()); ok(&repo.run(&cf, &["context", "land", "filings", "--project", "research", "--rows", "filings.jsonl", "--run-id", "load-1", "--site-id", "site-a"])); + ok(&repo.run(&cf, &["pipeline", "import", "--project", "research"])); repo.write(".contextful/issuance.toml", &format!("default_audience = \"{STORE_AUDIENCE}\"\nmax_lifetime_secs = 3600\n")); let public = ok(&repo.run(&cf, &["token", "keygen", "--out", ".contextful/issuer.seed"])); let read_token = ok(&repo.run(&cf, &[ "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://operator@example.test", "--zone", "on-prem:hq", "--action", "read", "--table", "filings", "--ttl", "3600", ])); + let admin_capability = ok(&repo.run(&cf, &[ + "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://operator@example.test", + "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "3600", + ])); let read = Command::new(&cf) .args(["serve", "--http", "127.0.0.1:0", "--max-in-flight", "2", "--project", "research", "--public-key", &public, "--audience", STORE_AUDIENCE]) .current_dir(&repo.root) @@ -124,6 +130,7 @@ fn m12_console() { .args(["--http", "127.0.0.1:0"]) .env("CONTEXTFUL_STORES_JSON", stores.to_string()) .env("FIELD_NOTES_QUERY_TOKEN", &read_token) + .env("CONTEXTFUL_ADMIN_CAPABILITY", &admin_capability) .env("CONTEXTFUL_ACCESS_JWKS_URL", jwks.url("/certs")) .env("CONTEXTFUL_ACCESS_ISSUER", ACCESS_ISSUER) .env("CONTEXTFUL_QUERY_ACCESS_AUDIENCE", QUERY_ACCESS_AUDIENCE) @@ -154,6 +161,8 @@ fn m12_console() { assert_eq!(request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); let (status, workflows) = request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); assert_eq!(status, 200, "{}", String::from_utf8_lossy(&workflows)); - let workflows = String::from_utf8(workflows).unwrap(); - assert!(workflows.contains("filings-flow") && workflows.contains("filings"), "{workflows}"); + let workflows: Value = serde_json::from_slice(&workflows).unwrap(); + assert_eq!(workflows["applied"], 1, "{workflows}"); + assert_eq!(workflows["pipelines"][0]["id"], "filings-flow", "{workflows}"); + assert_eq!(workflows["pipelines"][0]["tables"], json!(["filings"]), "{workflows}"); } From 3a04f3f61b72dae6fbe4d7fc347964aae6629947 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:16:50 +0800 Subject: [PATCH 066/155] feat(console): proxy Admin control with server-held credential --- apps/console/dist/server.js | 59 ++++++++++++++++++++++++++++++++----- apps/console/src/control.ts | 48 ++++++++++++++++++++++++++++++ apps/console/src/entry.ts | 2 ++ 3 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 apps/console/src/control.ts diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 749033e5d..bfccff3c9 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -73,6 +73,48 @@ function registryFromEnv(raw) { return { entries, problems }; } +// src/control.ts +function unavailable() { + throw new Error("ConsoleAdapterUnavailable"); +} +function selected(stores, id) { + if (id !== null) { + const store = stores.find((candidate) => candidate.id === id); + if (store) return store; + throw new Error("ConsoleStoreUnknown"); + } + if (stores.length === 1) return stores[0]; + throw new Error("ConsoleStoreSelectionRequired"); +} +function applyDocument(document) { + if (document === null || typeof document !== "object" || Array.isArray(document)) throw new Error("ConsoleRequestMalformed"); + const value = document; + if (value.store !== void 0 && typeof value.store !== "string") throw new Error("ConsoleRequestMalformed"); + if (value.id !== void 0 && typeof value.id !== "string") throw new Error("ConsoleRequestMalformed"); + return { store: typeof value.store === "string" ? value.store : null, body: typeof value.id === "string" ? { id: value.id } : {} }; +} +function createLiveControl({ stores, capability, fetcher = fetch }) { + async function call(store, path, token, body2) { + const response = await fetcher(new URL(path, store.endpoint), { + method: body2 ? "POST" : "GET", + headers: { Authorization: `Bearer ${token}`, ...body2 ? { "Content-Type": "application/json" } : {} }, + ...body2 ? { body: JSON.stringify(body2) } : {} + }); + if (!response.ok) throw new Error(`ConsoleControlRefused:${response.status}`); + return response.json(); + } + return { + workflows: async (_operator, id) => capability ? call(selected(stores, id), "/control/workflows", capability) : unavailable(), + record: async () => unavailable(), + edit: async () => unavailable(), + apply: async (document, token) => { + if (!capability || token !== capability) return unavailable(); + const input = applyDocument(document); + return call(selected(stores, input.store), "/control/apply", token, input.body); + } + }; +} + // src/server.ts import { createServer } from "node:http"; @@ -378,8 +420,8 @@ function serveConsole(adapters, origin) { await send(reply, Response.json({ error: { identifier: "ConsoleBodyTooLarge" } }, { status: 413 })); return; } - const unavailable2 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; - await send(reply, Response.json({ error: { identifier: unavailable2 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable2 ? 503 : 500 })); + const unavailable3 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; + await send(reply, Response.json({ error: { identifier: unavailable3 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable3 ? 503 : 500 })); } }); } @@ -429,17 +471,17 @@ async function identity() { keys: await keysAt(required("CONTEXTFUL_ACCESS_JWKS_URL")) }; } -function unavailable() { +function unavailable2() { throw new Error("ConsoleAdapterUnavailable"); } function unavailableAdapters() { return { - turn: async () => unavailable(), + turn: async () => unavailable2(), control: { - workflows: async () => unavailable(), - record: async () => unavailable(), - edit: async () => unavailable(), - apply: async () => unavailable() + workflows: async () => unavailable2(), + record: async () => unavailable2(), + edit: async () => unavailable2(), + apply: async () => unavailable2() } }; } @@ -454,6 +496,7 @@ async function main() { const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); let adapters = unavailableAdapters(); + adapters.control = createLiveControl({ stores: registry.entries, capability: process.env.CONTEXTFUL_ADMIN_CAPABILITY }); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); diff --git a/apps/console/src/control.ts b/apps/console/src/control.ts new file mode 100644 index 000000000..0abcac79a --- /dev/null +++ b/apps/console/src/control.ts @@ -0,0 +1,48 @@ +import type { ConsoleAdapters } from "./index.ts"; + +type ControlStore = { id: string; endpoint: string }; +type ControlOptions = { stores: ControlStore[]; capability?: string; fetcher?: typeof fetch }; + +function unavailable(): never { + throw new Error("ConsoleAdapterUnavailable"); +} + +function selected(stores: ControlStore[], id: string | null): ControlStore { + if (id !== null) { + const store = stores.find((candidate) => candidate.id === id); + if (store) return store; + throw new Error("ConsoleStoreUnknown"); + } + if (stores.length === 1) return stores[0]; + throw new Error("ConsoleStoreSelectionRequired"); +} + +function applyDocument(document: unknown): { store: string | null; body: { id?: string } } { + if (document === null || typeof document !== "object" || Array.isArray(document)) throw new Error("ConsoleRequestMalformed"); + const value = document as Record; + if (value.store !== undefined && typeof value.store !== "string") throw new Error("ConsoleRequestMalformed"); + if (value.id !== undefined && typeof value.id !== "string") throw new Error("ConsoleRequestMalformed"); + return { store: typeof value.store === "string" ? value.store : null, body: typeof value.id === "string" ? { id: value.id } : {} }; +} + +export function createLiveControl({ stores, capability, fetcher = fetch }: ControlOptions): ConsoleAdapters["control"] { + async function call(store: ControlStore, path: string, token: string, body?: object): Promise { + const response = await fetcher(new URL(path, store.endpoint), { + method: body ? "POST" : "GET", + headers: { Authorization: `Bearer ${token}`, ...(body ? { "Content-Type": "application/json" } : {}) }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + if (!response.ok) throw new Error(`ConsoleControlRefused:${response.status}`); + return response.json(); + } + return { + workflows: async (_operator, id) => capability ? call(selected(stores, id), "/control/workflows", capability) : unavailable(), + record: async () => unavailable(), + edit: async () => unavailable(), + apply: async (document, token) => { + if (!capability || token !== capability) return unavailable(); + const input = applyDocument(document); + return call(selected(stores, input.store), "/control/apply", token, input.body); + }, + }; +} diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 0ee90294c..51af14e74 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -3,6 +3,7 @@ import { pathToFileURL } from "node:url"; import { isAbsolute, resolve } from "node:path"; import { registryFromEnv } from "../../gateway/src/index.ts"; import type { ConsoleAdapters, Identity } from "./index.ts"; +import { createLiveControl } from "./control.ts"; import { serveConsole } from "./server.ts"; type AdapterFactory = (input: { stores: ConsoleAdapters["stores"]; env: NodeJS.ProcessEnv }) => Promise & { read?: ConsoleAdapters["read"] }>; @@ -81,6 +82,7 @@ async function main(): Promise { const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); let adapters: Pick & { read?: ConsoleAdapters["read"] } = unavailableAdapters(); + adapters.control = createLiveControl({ stores: registry.entries, capability: process.env.CONTEXTFUL_ADMIN_CAPABILITY }); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); From 8d20178cc1914c7197d3d5fe44e2af177582a3af Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:18:27 +0800 Subject: [PATCH 067/155] test(exchange): require unconfigured refusal before body read --- crates/contextful-cli/tests/integration/serve.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 2b21a4891..01f436f66 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -120,6 +120,12 @@ fn served_exchange_without_policy_returns_the_unconfigured_refusal() { assert_eq!(head.split(' ').nth(1), Some("404")); let answer: Value = serde_json::from_str(data).unwrap(); assert_eq!(answer["error"]["identifier"], "ExchangeUnconfigured"); + let mut stream = TcpStream::connect(&addr).unwrap(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap(); + write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n").unwrap(); + let mut raw = String::new(); + stream.read_to_string(&mut raw).expect("unconfigured exchange answers before waiting for its body"); + assert!(raw.starts_with("HTTP/1.1 404"), "{raw}"); } #[test] From 6866713c4de409e7133471de97ee7b11fed3ec12 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:20:17 +0800 Subject: [PATCH 068/155] test(console): prove live Admin apply before pending Query turn --- crates/acceptance/tests/integration/m12.rs | 24 +++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index 1335f0c55..642d42f58 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -144,6 +144,23 @@ fn m12_console() { assert_eq!(request(&console_address, "GET", "/query", None, None).0, 401); assert_eq!(request(&console_address, "GET", "/admin", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); + assert_eq!(request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); + let (status, workflows) = request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&workflows)); + let workflows: Value = serde_json::from_slice(&workflows).unwrap(); + assert_eq!(workflows["applied"], 1, "{workflows}"); + assert_eq!(workflows["pipelines"][0]["id"], "filings-flow", "{workflows}"); + assert_eq!(workflows["pipelines"][0]["tables"], json!(["filings"]), "{workflows}"); + repo.write("pipelines/filings.toml", "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1d\"\ntables = [\"filings\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"); + let (status, applied) = request(&console_address, "POST", "/admin/api/apply", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), + Some(&json!({ "store": "field-notes", "id": "filings-flow" }))); + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&applied)); + let applied: Value = serde_json::from_slice(&applied).unwrap(); + assert_eq!(applied["applied"], 2, "{applied}"); + let (status, workflows) = request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); + assert_eq!(status, 200); + let workflows: Value = serde_json::from_slice(&workflows).unwrap(); + assert_eq!(workflows["pipelines"][0]["schedule"], "every 1d", "{workflows}"); let (status, answer) = request( &console_address, "POST", "/query/api/ask", Some(&access_token(QUERY_ACCESS_AUDIENCE)), Some(&json!({ "store": "field-notes", "question": "Which filing arrived?" })), @@ -158,11 +175,4 @@ fn m12_console() { "the sources must identify the filing behind the answer: {answer}" ); assert!(!model.received("/v1/chat/completions").is_empty(), "the answer uses the model endpoint"); - assert_eq!(request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); - let (status, workflows) = request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); - assert_eq!(status, 200, "{}", String::from_utf8_lossy(&workflows)); - let workflows: Value = serde_json::from_slice(&workflows).unwrap(); - assert_eq!(workflows["applied"], 1, "{workflows}"); - assert_eq!(workflows["pipelines"][0]["id"], "filings-flow", "{workflows}"); - assert_eq!(workflows["pipelines"][0]["tables"], json!(["filings"]), "{workflows}"); } From e58e17f66b28d8fbb61ba50c4dad3249a803e14b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:28:49 +0800 Subject: [PATCH 069/155] fix(query): scope reads and fail closed on exchange outages --- apps/console/dist/server.js | 41 +++++++++++++++++++++++----- apps/console/src/live.ts | 42 ++++++++++++++++++++++++----- apps/console/src/turn.ts | 4 +-- apps/console/test/live.test.ts | 7 ++--- crates/contextful-agent/src/http.rs | 17 +++++++++--- crates/contextful-cli/src/serve.rs | 6 ++--- 6 files changed, 92 insertions(+), 25 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 97b947135..abb2658b3 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -115,7 +115,8 @@ async function resolveReaderCredential(options) { try { const credential = await options.mint(); if (credential) return credential; - } catch { + } catch (error) { + if (!(error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused" && error.status === 403)) throw error; } if (options.shared) return options.shared; throw new ConsoleError("ConsoleTokenExchangeRefused", "ConsoleTokenExchangeRefused", 403); @@ -256,6 +257,21 @@ function record(value) { function strings(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; } +function terms(value) { + return new Set((value.toLowerCase().match(/[a-z0-9]+/g) ?? []).map((word) => word.endsWith("ies") && word.length > 4 ? `${word.slice(0, -3)}y` : word.endsWith("s") && word.length > 3 ? word.slice(0, -1) : word)); +} +function selectTable(question, listing) { + const asked = terms(question); + const entries = Array.isArray(listing) ? listing : []; + const ranked = entries.flatMap((item) => { + if (!record(item) || typeof item.table !== "string" || item.kind !== "data") return []; + const name = terms(item.table.split("/").at(-1) ?? ""); + const description = terms(typeof item.description === "string" ? item.description : ""); + const score = [...asked].reduce((sum, word) => sum + (name.has(word) ? 2 : description.has(word) ? 1 : 0), 0); + return score > 0 ? [{ table: item.table, score }] : []; + }).sort((a, b) => b.score - a.score); + return ranked.length > 0 && (ranked.length === 1 || ranked[0].score > ranked[1].score) ? ranked[0].table : null; +} function source(columns, row, table) { const at = (name) => row[columns.indexOf(name)]; const idColumn = columns.find((name) => name.endsWith("_id") || name === "id"); @@ -281,15 +297,22 @@ function createLiveTurn({ stores, env, fetcher = fetch }) { const credential = store.auth === "exchange" ? await resolveReaderCredential({ shared, mint: async () => { - if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jwt: input.operator.assertion }) }); - if (!response.ok) throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!response.ok) { + const refusal2 = await response.json(); + const error = record(refusal2) ? refusal2.error : void 0; + if ((response.status === 401 || response.status === 403) && record(error) && typeof error.identifier === "string") { + throw new ConsoleError("ConsoleTokenExchangeRefused", error.identifier, 403); + } + throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); + } const value = await response.json(); - if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); return value.token; } }) : shared; @@ -310,20 +333,24 @@ function createLiveTurn({ stores, env, fetcher = fetch }) { return message.result.structuredContent; }; const description = await call("context.describe", {}); - const tables = Array.isArray(description.tables) ? description.tables.flatMap((item) => record(item) && typeof item.table === "string" && item.kind === "data" ? [item.table] : []) : []; + const selected = selectTable(input.question, description.tables); + if (!selected) throw new ConsoleError("ConsoleUngroundedAnswer", "No data table matches this question."); + const tables = [selected]; + let remainingSources = 8; const turn = createTurn({ tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read", table })), tables: tables.map((name) => ({ name, kind: "data" })), planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), transport: { call: async ({ tool, maxRows, signal }) => { const table = tool.slice("read:".length); - const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: maxRows }, signal); + const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: Math.min(maxRows, remainingSources) }, signal); const columns = strings(result.columns); const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; - }); + }).slice(0, remainingSources); + remainingSources -= sourced.length; return { rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), sources: sourced.map(({ citation }) => citation) diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index 7c5c88c63..bcc2c8fe6 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -12,6 +12,24 @@ function strings(value: unknown): string[] { return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []; } +function terms(value: string): Set { + return new Set((value.toLowerCase().match(/[a-z0-9]+/g) ?? []).map((word) => + word.endsWith("ies") && word.length > 4 ? `${word.slice(0, -3)}y` : word.endsWith("s") && word.length > 3 ? word.slice(0, -1) : word)); +} + +function selectTable(question: string, listing: unknown): string | null { + const asked = terms(question); + const entries = Array.isArray(listing) ? listing : []; + const ranked = entries.flatMap((item) => { + if (!record(item) || typeof item.table !== "string" || item.kind !== "data") return []; + const name = terms(item.table.split("/").at(-1) ?? ""); + const description = terms(typeof item.description === "string" ? item.description : ""); + const score = [...asked].reduce((sum, word) => sum + (name.has(word) ? 2 : description.has(word) ? 1 : 0), 0); + return score > 0 ? [{ table: item.table, score }] : []; + }).sort((a, b) => b.score - a.score); + return ranked.length > 0 && (ranked.length === 1 || ranked[0].score > ranked[1].score) ? ranked[0].table : null; +} + function source(columns: string[], row: unknown[], table: string): Source | null { const at = (name: string) => row[columns.indexOf(name)]; const idColumn = columns.find((name) => name.endsWith("_id") || name === "id"); @@ -35,14 +53,21 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( const credential = store.auth === "exchange" ? await resolveReaderCredential({ shared, mint: async () => { - if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ jwt: input.operator.assertion }), }); - if (!response.ok) throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!response.ok) { + const refusal: unknown = await response.json(); + const error = record(refusal) ? refusal.error : undefined; + if ((response.status === 401 || response.status === 403) && record(error) && typeof error.identifier === "string") { + throw new ConsoleError("ConsoleTokenExchangeRefused", error.identifier, 403); + } + throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); + } const value: unknown = await response.json(); - if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeRefused"); + if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); return value.token; }, }) : shared; @@ -64,18 +89,21 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( }; const description = await call("context.describe", {}); - const tables = Array.isArray(description.tables) ? description.tables.flatMap((item) => - record(item) && typeof item.table === "string" && item.kind === "data" ? [item.table] : []) : []; + const selected = selectTable(input.question, description.tables); + if (!selected) throw new ConsoleError("ConsoleUngroundedAnswer", "No data table matches this question."); + const tables = [selected]; + let remainingSources = 8; const turn = createTurn({ tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read" as const, table })), tables: tables.map((name) => ({ name, kind: "data" as const })), planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), transport: { call: async ({ tool, maxRows, signal }): Promise => { const table = tool.slice("read:".length); - const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: maxRows }, signal); + const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: Math.min(maxRows, remainingSources) }, signal); const columns = strings(result.columns); const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; - const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; }); + const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; }).slice(0, remainingSources); + remainingSources -= sourced.length; return { rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), sources: sourced.map(({ citation }) => citation), diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts index d4e808969..927d76a7e 100644 --- a/apps/console/src/turn.ts +++ b/apps/console/src/turn.ts @@ -69,8 +69,8 @@ export async function resolveReaderCredential(options: { try { const credential = await options.mint(); if (credential) return credential; - } catch { - // A refused mint reaches the shared credential only when the store has one. + } catch (error) { + if (!(error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused" && error.status === 403)) throw error; } if (options.shared) return options.shared; throw new ConsoleError("ConsoleTokenExchangeRefused", "ConsoleTokenExchangeRefused", 403); diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index 5e0f5af13..93d26e26b 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -87,7 +87,7 @@ test("Query reads only a question-matched data table and refuses an unclear sele store: "field-notes", question: "Which filing arrived?" }); assert.deepEqual(matched.calls.filter((call) => (call.body.params as Record | undefined)?.name === "context.query") .map((call) => (call.body.params as Record).arguments), - [{ sql: 'SELECT * FROM "filings"', limit: 5000 }]); + [{ sql: 'SELECT * FROM "filings"', limit: 8 }]); const unclear = fixture({ tables }); await assert.rejects(unclear.turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, store: "field-notes", question: "What changed?" }), @@ -128,7 +128,7 @@ test("Query never scaffolds or dispatches a memory relation", async () => { store: "field-notes", question: "Which filing arrived?" }); assert.deepEqual(calls.filter((call) => (call.body.params as Record | undefined)?.name === "context.query") .map((call) => (call.body.params as Record).arguments), - [{ sql: 'SELECT * FROM "filings"', limit: 5000 }]); + [{ sql: 'SELECT * FROM "filings"', limit: 8 }]); }); test("Query returns a typed exchange refusal through its hosted API", async () => { @@ -139,7 +139,8 @@ test("Query returns a typed exchange refusal through its hosted API", async () = read: { list: async () => [{ id: store.id, label: store.label }] }, control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, }); - const token = issueCognitoSession({ subject: "reader", groups: ["query"], expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + const token = issueCognitoSession({ subject: "reader", groups: ["query"], assertion: "cognito-id-token", + expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); const response = await app.fetch(new Request("https://console.example/query/api/ask", { method: "POST", headers: { cookie: `console_session=${token}`, origin: "https://console.example" }, body: JSON.stringify({ store: store.id, question: "Which filing arrived?" }), diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 49ce70d54..27074f4e4 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -196,6 +196,7 @@ pub struct HttpFace<'a, C> { ceiling: usize, in_flight: AtomicUsize, exchange: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, + exchange_unconfigured: bool, } /// A request slot held while one request is in flight. @@ -242,13 +243,14 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { let ceiling = ceiling(max_in_flight)?; audience(Some(admitting.audience))?; let tools = Tools::new(face, clock, record)?; - Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0), exchange: None }) + Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0), exchange: None, exchange_unconfigured: false }) } /// The binary's exchange route mints the reader credential without putting issuer /// signing material in the read-transport package. - pub fn with_exchange(mut self, exchange: &'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)) -> Self { + pub fn with_exchange(mut self, exchange: &'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync), unconfigured: bool) -> Self { self.exchange = Some(exchange); + self.exchange_unconfigured = unconfigured; self } @@ -291,7 +293,9 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { fn connection(&self, mut stream: TcpStream, slot: Slot<'_>) -> std::io::Result<()> { stream.set_read_timeout(Some(REQUEST_READ_TIMEOUT))?; stream.set_write_timeout(Some(REQUEST_READ_TIMEOUT))?; - let response = match read_request(&mut stream) { + let response = match read_request_preflight(&mut stream, |head| { + (self.exchange_unconfigured && head.path() == "/auth/exchange").then(|| self.answer(head)) + }) { Ok(request) => self.answer(&request), Err(response) => response, }; @@ -389,6 +393,10 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { /// Read one request: the head, then a `Content-Length` body of at most /// [`REQUEST_BODY_BYTES`]. A malformed or oversized request is answered, not read on. pub fn read_request(stream: &mut impl Read) -> Result { + read_request_preflight(stream, |_| None) +} + +fn read_request_preflight(stream: &mut impl Read, preflight: impl Fn(&HttpRequest) -> Option) -> Result { let mut buf = Vec::with_capacity(1024); let mut chunk = [0u8; 4096]; let (mut request, head_len) = loop { @@ -424,6 +432,9 @@ pub fn read_request(stream: &mut impl Read) -> Result Err(e) => return Err(HttpResponse::message(400, format!("the request head does not parse: {e}"))), } }; + if let Some(response) = preflight(&request) { + return Err(response); + } if request.header("Transfer-Encoding").is_some() { return Err(HttpResponse::message(411, "the face reads a `Content-Length` body")); } diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 2578fc638..526e6d560 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -125,8 +125,8 @@ pub fn run(args: ServeArgs) -> Result<()> { let proofs = ProofChecker::new(SystemClock); let exchange_route = |request: &contextful_agent::http::HttpRequest| { let Some((issuance, signer)) = mint_material.as_ref() else { - return contextful_agent::http::HttpResponse::json(503, - &serde_json::json!({ "error": { "http": 503, "identifier": "ExchangeUnconfigured" } })); + return contextful_agent::http::HttpResponse::json(404, + &serde_json::json!({ "error": { "http": 404, "identifier": "ExchangeUnconfigured", "message": ".contextful/exchange/policy.toml is absent" } })); }; let ctx = MintContext { node: NodeRole::Primary, signer: signer as &dyn SigningPort, clock: &clock as &dyn Clock }; let result = exchange_answer(exchange.as_ref(), &request.body, request.header("DPoP"), &proofs, issuance, &ctx); @@ -134,7 +134,7 @@ pub fn run(args: ServeArgs) -> Result<()> { }; let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)) .map_err(anyhow::Error::msg)? - .with_exchange(&exchange_route); + .with_exchange(&exchange_route, exchange.is_none()); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); http.serve(listener)?; From f38bece088eded1e9b72cb5263d1fd2076837792 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:32:14 +0800 Subject: [PATCH 070/155] test(exchange): reject empty minted reader credential --- apps/console/test/live.test.ts | 12 ++++++++++-- apps/console/test/turn.test.ts | 5 +++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index 93d26e26b..5e03361c3 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -11,7 +11,7 @@ const store: StoreEntry = { bindingName: "FIELD_NOTES", }; -function fixture(options: { exchange?: boolean; exchangeStatus?: number; shared?: string; rows?: unknown[][]; tables?: Array<{ table: string; kind: string }> } = {}) { +function fixture(options: { exchange?: boolean; exchangeStatus?: number; emptyToken?: boolean; shared?: string; rows?: unknown[][]; tables?: Array<{ table: string; kind: string }> } = {}) { const calls: Array<{ url: string; authorization: string | null; body: Record }> = []; let modelCalls = 0; let modelBody: Record | null = null; @@ -21,7 +21,7 @@ function fixture(options: { exchange?: boolean; exchangeStatus?: number; shared? calls.push({ url, authorization: new Headers(init?.headers).get("authorization"), body }); if (url.endsWith("/auth/exchange")) { const status = options.exchangeStatus ?? (options.exchange === false ? 403 : 200); - return Response.json(status === 200 ? { token: "reader-token" } : { error: { identifier: status === 503 ? "ExchangeMaterialMissing" : "ExchangeAssertionInvalid" } }, { status }); + return Response.json(status === 200 ? { token: options.emptyToken ? "" : "reader-token" } : { error: { identifier: status === 503 ? "ExchangeMaterialMissing" : "ExchangeAssertionInvalid" } }, { status }); } if (url.endsWith("/mcp")) { const params = body.params as Record; @@ -80,6 +80,14 @@ test("Query does not use a shared credential after exchange service failure", as assert.equal(calls.filter((call) => call.url.endsWith("/mcp")).length, 0); }); +test("Query refuses an empty minted credential without using a shared credential", async () => { + const { turn, calls } = fixture({ emptyToken: true, shared: "shared-token" }); + await assert.rejects(turn({ operator: { subject: "operator-1", grants: new Set(["query"]), assertion: "verified-access-jwt" }, + store: "field-notes", question: "Which filing arrived?" }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleTokenExchangeUnavailable"); + assert.equal(calls.filter((call) => call.url.endsWith("/mcp")).length, 0); +}); + test("Query reads only a question-matched data table and refuses an unclear selection", async () => { const tables = [{ table: "filings", kind: "data" }, { table: "salaries", kind: "data" }]; const matched = fixture({ tables }); diff --git a/apps/console/test/turn.test.ts b/apps/console/test/turn.test.ts index 2667323ed..75411cae3 100644 --- a/apps/console/test/turn.test.ts +++ b/apps/console/test/turn.test.ts @@ -158,6 +158,11 @@ test("reader credential resolution preserves transport and service failures", as (error: unknown) => error === network); }); +test("an empty minted credential never activates a shared fallback", async () => { + await assert.rejects(resolveReaderCredential({ mint: async () => "", shared: "shared-token" }), + (error: unknown) => error instanceof ConsoleError && error.code === "ConsoleTokenExchangeUnavailable"); +}); + test("planner scaffolding lists data tables while memory relations remain outside it", async () => { let plannerTables: unknown[] = []; const { turn } = harness({ From c63eb5275813b762efe7268ac6828fe8a33b0136 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 16:33:32 +0800 Subject: [PATCH 071/155] fix(exchange): refuse empty minted reader tokens --- apps/console/dist/server.js | 5 +++-- apps/console/src/live.ts | 2 +- apps/console/src/turn.ts | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index abb2658b3..8097858e6 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -114,7 +114,8 @@ function parseWebBound(value) { async function resolveReaderCredential(options) { try { const credential = await options.mint(); - if (credential) return credential; + if (credential.trim()) return credential; + throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned an empty credential", 503); } catch (error) { if (!(error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused" && error.status === 403)) throw error; } @@ -312,7 +313,7 @@ function createLiveTurn({ stores, env, fetcher = fetch }) { throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); } const value = await response.json(); - if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); + if (!record(value) || typeof value.token !== "string" || !value.token.trim()) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); return value.token; } }) : shared; diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index bcc2c8fe6..7f5f3119d 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -67,7 +67,7 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); } const value: unknown = await response.json(); - if (!record(value) || typeof value.token !== "string") throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); + if (!record(value) || typeof value.token !== "string" || !value.token.trim()) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); return value.token; }, }) : shared; diff --git a/apps/console/src/turn.ts b/apps/console/src/turn.ts index 927d76a7e..32005926c 100644 --- a/apps/console/src/turn.ts +++ b/apps/console/src/turn.ts @@ -68,7 +68,8 @@ export async function resolveReaderCredential(options: { }): Promise { try { const credential = await options.mint(); - if (credential) return credential; + if (credential.trim()) return credential; + throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned an empty credential", 503); } catch (error) { if (!(error instanceof ConsoleError && error.code === "ConsoleTokenExchangeRefused" && error.status === 403)) throw error; } From 77efd0f4557c5979525f64c1d6a138b754c318c9 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:06:58 +0800 Subject: [PATCH 072/155] Pin governed browse tests to console clauses --- apps/console/test/browse.test.ts | 4 ++++ spec/pins.toml | 2 +- spec/status.md | 10 +++++++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/apps/console/test/browse.test.ts b/apps/console/test/browse.test.ts index 0c6531018..542101a39 100644 --- a/apps/console/test/browse.test.ts +++ b/apps/console/test/browse.test.ts @@ -28,6 +28,7 @@ function fixture() { return { browse: createBrowse(transport), calls }; } +// spec: surface.browse.discovered-chips@def23028 test("browse advertises admitted tables as humanized chips", async () => { const { browse, calls } = fixture(); const result = await browse.discover({ asOf: "2030-01-01T00:00:00Z" }); @@ -39,12 +40,14 @@ test("browse advertises admitted tables as humanized chips", async () => { assert(!calls.some((call) => call.arguments.table === "research/private_payroll")); }); +// spec: surface.browse.insights-panel@b1d70f29 test("insights show governed table row counts", async () => { const { browse } = fixture(); const result = await browse.discover({}); assert.deepEqual(result.insights, [{ table: "research/vendor_filings", label: "Vendor Filings", rows: 3 }]); }); +// spec: surface.browse.gallery-path-unlisted@d8eb9504 test("a gallery preview refuses paths absent from the governed listing", async () => { const { browse, calls } = fixture(); await assert.rejects(browse.preview({ path: "research/vendor_filings/runs/run-1/other.parquet" }), @@ -52,6 +55,7 @@ test("a gallery preview refuses paths absent from the governed listing", async ( assert(!calls.some((call) => call.tool === "context.file")); }); +// spec: surface.browse.file-gallery@4cfe68e1 test("a gallery preview uses context.file with the listing's snapshot bound", async () => { const { browse, calls } = fixture(); const preview = await browse.preview({ path: "research/vendor_filings/runs/run-1/report.parquet", asOf: "2030-01-01T00:00:00Z" }); diff --git a/spec/pins.toml b/spec/pins.toml index 5e43b6541..92bb5c22c 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -63,5 +63,5 @@ disclosure = 54 read = 163 run = 261 store = 200 -surface = 51 +surface = 55 topology = 38 diff --git a/spec/status.md b/spec/status.md index 7c75ce91f..fe8a05d1a 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 110 | 50 | 22 | 0 | 51 | 0 | 51 | +| `surface` | 2 | 20 | 110 | 50 | 22 | 0 | 55 | 0 | 55 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2017 | 582 | 188 | 127 | 1365 | 0 | | +| **total** | 19 | 155 | 2017 | 582 | 188 | 127 | 1369 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 53 | 0 | open | open | +| 12 — The operator console | 12 | 53 | 4 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -1342,6 +1342,10 @@ Unscheduled operations: 10. | `surface.arm.unknown-trigger` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unknown_trigger_arms_nothing` | performed | | `surface.arm.unreadable-schedule` | `crates/contextful-core/tests/integration/surface/arm.rs::an_unreadable_schedule_names_its_diagnostic` | performed | | `surface.arm.wake-answer` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_answers_within_its_bound_naming_what_still_runs` | performed | +| `surface.browse.discovered-chips` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::browse advertises admitted tables as humanized chips` | performed | +| `surface.browse.file-gallery` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::a gallery preview uses context.file with the listing's snapshot bound` | performed | +| `surface.browse.gallery-path-unlisted` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::a gallery preview refuses paths absent from the governed listing` | performed | +| `surface.browse.insights-panel` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::insights show governed table row counts` | performed | | `surface.dispatch.callback-rejected` | `crates/contextful-core/tests/integration/surface/worker.rs::a_superseded_or_skewed_callback_changes_no_step` | performed | | `surface.dispatch.callback-skew` | `crates/contextful-core/tests/integration/surface/worker.rs::the_relay_accepts_a_timestamp_within_the_skew` | performed | | `surface.dispatch.children-reaped` | `crates/contextful-cli/tests/integration/pipeline.rs::serve_ends_every_child_it_dispatched` | performed | From ab1d1319de4c70bea309bc2e12e2734b846c9fc8 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:12:43 +0800 Subject: [PATCH 073/155] test: refuse owner token after same-path store recreation --- .../contextful-cli/tests/integration/mcp.rs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 5f704a864..b8863ff3b 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -192,6 +192,28 @@ allow = ["public-cloud:*"] assert_eq!(answers[1]["result"]["structuredContent"]["rows"], json!([]), "zone exclusion holds under owner"); } +#[test] +fn owner_credential_does_not_follow_a_recreated_store_at_the_same_path() { + let (dir, public, _) = project(); + let root = dir.path(); + let owner = stdout(&run(root, &[ + "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", + "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner", + ])); + let identity_path = root.join(".contextful/context/research/store-id"); + let first_identity = std::fs::read_to_string(&identity_path).unwrap_or_default(); + std::fs::remove_dir_all(root.join(".contextful/context/research")).unwrap(); + stdout(&run(root, &["context", "land", "research/notes", "--project", "research", "--rows", "notes.jsonl", "--run-id", "run-0002", "--site-id", "site-a"])); + let second_identity = std::fs::read_to_string(&identity_path).unwrap_or_default(); + assert!(!first_identity.is_empty() && !second_identity.is_empty() && first_identity != second_identity, + "a recreated store receives a new persistent identity"); + let args = ["mcp", "--owner", "--project", "research", "--public-key", &public, "--audience", AUD]; + let read = [json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } })]; + let refused = serve(root, &args, Some(&owner), &read); + assert!(!refused.status.success() && refused.stdout.is_empty()); + assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); +} + /// The server admits over the stdio pipe it inherited: a credential binding no key admits /// through that pipe, and one binding a holder key and presenting no holder proof refuses. #[test] From 1642de40a65d919b34a1f11bac8dcd9162dae3fc Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:14:37 +0800 Subject: [PATCH 074/155] test(console): require hosted live governed browse --- apps/console/test/browse_hosted.test.ts | 51 +++++++++++++++++++++++++ apps/console/test/live_browse.test.ts | 48 +++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 apps/console/test/browse_hosted.test.ts create mode 100644 apps/console/test/live_browse.test.ts diff --git a/apps/console/test/browse_hosted.test.ts b/apps/console/test/browse_hosted.test.ts new file mode 100644 index 000000000..afc8c049c --- /dev/null +++ b/apps/console/test/browse_hosted.test.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createConsole, issueCognitoSession } from "../src/index.ts"; + +function hosted() { + const calls: string[] = []; + const app = createConsole({ + identity: { kind: "cognito", sessionSecret: "session-secret", queryGroup: "query", adminGroup: "admin" }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + browse: { + discover: async ({ operator, store, asOf }) => { calls.push(`discover:${operator.subject}:${store}:${asOf ?? ""}`); return { + chips: [{ table: "filings", label: "Filings" }], insights: [{ table: "filings", label: "Filings", rows: 3 }], files: [{ table: "filings", path: "filings/runs/run-1/report.parquet", label: "Report" }], + }; }, + preview: async ({ operator, store, path, asOf }) => { calls.push(`preview:${operator.subject}:${store}:${path}:${asOf ?? ""}`); return { columns: ["id"], rows: [["one"]] }; }, + }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const session = issueCognitoSession({ subject: "reader", groups: ["query"], expiresAt: Math.floor(Date.now() / 1000) + 60 }, "session-secret"); + const request = (path: string, body?: unknown, origin = "https://console.example") => app.fetch(new Request(`https://console.example${path}`, { + method: body === undefined ? "GET" : "POST", headers: { cookie: `console_session=${session}`, origin }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + })); + return { request, calls }; +} + +test("hosted Query exposes bounded same-origin browse and preview routes", async () => { + const { request, calls } = hosted(); + const page = await (await request("/query")).text(); + for (const id of ["chips", "insights", "file-gallery", "file-preview"]) assert.match(page, new RegExp(`id="${id}"`)); + assert.match(page, /\/query\/api\/browse/); + assert.match(page, /\/query\/api\/preview/); + const asOf = "2030-01-01T00:00:00Z"; + const browse = await request("/query/api/browse", { store: "field-notes", asOf }); + assert.equal(browse.status, 200); + assert.equal((await browse.json() as { chips: unknown[] }).chips.length, 1); + const preview = await request("/query/api/preview", { store: "field-notes", path: "filings/runs/run-1/report.parquet", asOf }); + assert.equal(preview.status, 200); + assert.deepEqual((await preview.json() as { rows: unknown[] }).rows, [["one"]]); + assert.deepEqual(calls, [`discover:reader:field-notes:${asOf}`, `preview:reader:field-notes:filings/runs/run-1/report.parquet:${asOf}`]); +}); + +test("hosted browse rejects foreign origins, invalid bounds, and oversized paths before dispatch", async () => { + const { request, calls } = hosted(); + assert.equal((await request("/query/api/browse", { store: "field-notes" }, "https://foreign.example")).status, 403); + assert.equal((await request("/query/api/browse", { store: "field-notes", asOf: "not-a-date" })).status, 400); + assert.equal((await request("/query/api/preview", { store: "field-notes", path: "x".repeat(4097) })).status, 400); + assert.equal((await request("/query/api/browse", { store: "unknown" })).status, 400); + assert.deepEqual(calls, []); +}); diff --git a/apps/console/test/live_browse.test.ts b/apps/console/test/live_browse.test.ts new file mode 100644 index 000000000..392624bd7 --- /dev/null +++ b/apps/console/test/live_browse.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createLiveBrowse } from "../src/live_browse.ts"; +import type { StoreEntry } from "../../gateway/src/index.ts"; + +const store: StoreEntry = { + id: "field-notes", label: "Field notes", endpoint: "https://store.example", + auth: "exchange", exchangeRoute: "/auth/exchange", credentialName: "FIELD_NOTES_QUERY_TOKEN", bindingName: "FIELD_NOTES", +}; + +test("live browse uses one exchanged reader credential for governed discovery and preview", async () => { + const calls: Array<{ name: string; args: Record; authorization: string | null }> = []; + let exchanges = 0; + const fetcher: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/auth/exchange")) { exchanges++; return Response.json({ token: "reader-token" }); } + assert.equal(url, "https://store.example/mcp"); + const body = JSON.parse(String(init?.body)) as { id: number; params: { name: string; arguments: Record } }; + calls.push({ name: body.params.name, args: body.params.arguments, authorization: new Headers(init?.headers).get("authorization") }); + const value = body.params.name === "context.describe" && !body.params.arguments.table ? { tables: [ + { table: "filings", kind: "data", zone_admitted: true, description: "Filed reports" }, + { table: "payroll", kind: "data", zone_admitted: false }, + { table: "insights", kind: "memory", zone_admitted: true }, + ] } : body.params.name === "context.describe" ? { row_count: 3 } : body.params.name === "context.files" ? { + columns: ["table", "path"], rows: [["filings", "filings/runs/run-1/report.parquet"], ["payroll", "payroll/runs/run-1/secret.parquet"]], + } : body.params.name === "context.file" ? { columns: ["filing_id"], rows: [["filing-1"]] } : null; + assert(value); + return Response.json({ jsonrpc: "2.0", id: body.id, result: { structuredContent: value } }); + }; + const browse = createLiveBrowse({ stores: [store], env: {}, fetcher }); + const input = { operator: { subject: "reader", grants: new Set(["query" as const]), assertion: "verified-jwt" }, store: store.id, + asOf: "2030-01-01T00:00:00Z" }; + const discovered = await browse.discover(input); + assert.deepEqual(discovered.chips.map((chip: { table: string }) => chip.table), ["filings"]); + assert.deepEqual(discovered.insights.map((insight: { rows: number }) => insight.rows), [3]); + assert.deepEqual(discovered.files.map((file: { path: string }) => file.path), ["filings/runs/run-1/report.parquet"]); + assert.equal(exchanges, 1); + assert(calls.every((call) => call.authorization === "Bearer reader-token")); + assert(calls.every((call) => call.args.as_of === input.asOf)); + const preview = await browse.preview({ ...input, path: "filings/runs/run-1/report.parquet" }); + assert.deepEqual(preview, { columns: ["filing_id"], rows: [["filing-1"]] }); + assert.equal(exchanges, 2); + assert.equal(calls.at(-1)?.name, "context.file"); + const before = calls.length; + await assert.rejects(browse.preview({ ...input, path: "filings/runs/run-1/other.parquet" }), /ConsoleGalleryPathUnlisted/); + assert.equal(calls.filter((call) => call.name === "context.file").length, 1); + assert(calls.length > before, "the refusal rechecks the governed file list"); +}); From a48b2183cb08cb14be2c6d33faa7134e689e5483 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:15:22 +0800 Subject: [PATCH 075/155] test: require persisted UUID and sync identity conflict refusal --- .../tests/integration/init.rs | 21 +++++++++++++++++++ .../contextful-sync/tests/integration/pull.rs | 21 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/crates/contextful-context/tests/integration/init.rs b/crates/contextful-context/tests/integration/init.rs index 689b4310f..666060bc0 100644 --- a/crates/contextful-context/tests/integration/init.rs +++ b/crates/contextful-context/tests/integration/init.rs @@ -22,6 +22,27 @@ fn an_init_writes_the_declaration_and_the_store_root() { assert!(dir.path().join(".contextful/context/research").is_dir()); } +#[test] +fn init_persists_a_distinct_store_uuid_across_repetition_and_recreation() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join(".contextful/context/research"); + init(dir.path(), "research").unwrap(); + let first = fs::read_to_string(root.join("store-id")).unwrap(); + let valid = |value: &str| { + let text = value.trim(); + text.len() == 36 && text.chars().enumerate().all(|(i, c)| + if [8, 13, 18, 23].contains(&i) { c == '-' } else { c.is_ascii_hexdigit() }) && + text.as_bytes()[14] == b'4' + }; + assert!(valid(&first), "{first}"); + init(dir.path(), "research").unwrap(); + assert_eq!(fs::read_to_string(root.join("store-id")).unwrap(), first); + fs::remove_dir_all(&root).unwrap(); + init(dir.path(), "research").unwrap(); + let second = fs::read_to_string(root.join("store-id")).unwrap(); + assert!(valid(&second) && second != first, "recreating the store changes its UUID"); +} + /// A project name that is not path-safe segments raises `StoreProjectNameInvalid` before any file is read or written. // spec: store.init.name-shape@93c80b33 #[test] diff --git a/crates/contextful-sync/tests/integration/pull.rs b/crates/contextful-sync/tests/integration/pull.rs index a9da30163..58882c8de 100644 --- a/crates/contextful-sync/tests/integration/pull.rs +++ b/crates/contextful-sync/tests/integration/pull.rs @@ -30,6 +30,27 @@ fn pushed() -> (tempfile::TempDir, Arc, crate::support::Node) { (dir, b, a) } +#[test] +fn a_clone_keeps_the_store_uuid_and_an_independent_store_refuses_it() { + let dir = tempfile::tempdir().unwrap(); + let b = bucket(dir.path()); + let original = node("ingest-a", b.clone(), ""); + let identity = "11111111-1111-4111-8111-111111111111\n"; + std::fs::write(original.root().join("store-id"), identity).unwrap(); + original.syncer.push(at(NOW)).unwrap(); + + let clone = node("ingest-b", b.clone(), ""); + clone.syncer.pull(&PullScope::default()).unwrap(); + assert_eq!(std::fs::read_to_string(clone.root().join("store-id")).unwrap(), identity); + + let independent = node("ingest-c", b, ""); + let other = "22222222-2222-4222-8222-222222222222\n"; + std::fs::write(independent.root().join("store-id"), other).unwrap(); + let refused = independent.syncer.pull(&PullScope::default()).unwrap_err(); + assert!(refused.to_string().contains("StoreIdentityConflict"), "{refused}"); + assert_eq!(std::fs::read_to_string(independent.root().join("store-id")).unwrap(), other); +} + /// A downloaded object whose digest differs from its entry raises `SyncObjectDigestMismatch` and is discarded. // spec: store.pull.digest-mismatch@5acf85aa #[test] From b0416f0bf778ecfc2cf5f0522676eff3f5a386fd Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:18:38 +0800 Subject: [PATCH 076/155] test(control): require validated store draft and versioned apply --- apps/console/test/control.test.ts | 16 ++++++---- .../contextful-cli/tests/integration/serve.rs | 30 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/apps/console/test/control.test.ts b/apps/console/test/control.test.ts index 769104e11..6fb1a8177 100644 --- a/apps/console/test/control.test.ts +++ b/apps/console/test/control.test.ts @@ -18,7 +18,7 @@ test("Admin route retrieves applied workflow state and applies through the serve calls.push({ path: url.pathname, authorization: new Headers(init?.headers).get("authorization"), body: String(init?.body ?? "") }); return Response.json(url.pathname === "/control/workflows" ? { applied: 1, pipelines: [{ id: "filings-flow", tables: ["filings"], schedule: "every 1h" }], runs: {} } : - { applied: 2 }); + url.pathname === "/control/edit" ? { expected: 1 } : { applied: 2 }); }; const control = createLiveControl({ stores: [{ id: "field-notes", endpoint: "http://127.0.0.1:9999" }], capability: "engine-admin-secret", fetcher }); const app = createConsole({ @@ -30,22 +30,28 @@ test("Admin route retrieves applied workflow state and applies through the serve const listing = await app.fetch(new Request("https://console.example/admin/api/workflows", { headers })); assert.equal(listing.status, 200); assert.equal((await listing.json() as { pipelines: { id: string }[] }).pipelines[0].id, "filings-flow"); + const edit = await app.fetch(new Request("https://console.example/admin/api/edit", { + method: "POST", headers: { ...headers, origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", expected: 1, document: "[[pipeline]]\nid = \"filings-flow\"" }), + })); + assert.equal(edit.status, 200); const apply = await app.fetch(new Request("https://console.example/admin/api/apply", { - method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", id: "filings-flow" }), + method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", expected: 1 }), })); assert.equal(apply.status, 200); assert.deepEqual(calls, [ { path: "/control/workflows", authorization: "Bearer engine-admin-secret", body: "" }, - { path: "/control/apply", authorization: "Bearer engine-admin-secret", body: '{"id":"filings-flow"}' }, + { path: "/control/edit", authorization: "Bearer engine-admin-secret", body: '{"expected":1,"document":"[[pipeline]]\\nid = \\"filings-flow\\""}' }, + { path: "/control/apply", authorization: "Bearer engine-admin-secret", body: '{"expected":1}' }, ]); assert.doesNotMatch(JSON.stringify(await apply.json()), /engine-admin-secret/); }); -test("Admin control refuses ambiguous stores and keeps edit unavailable", async () => { +test("Admin control refuses ambiguous stores", async () => { const control = createLiveControl({ stores: [{ id: "one", endpoint: "http://127.0.0.1:1" }, { id: "two", endpoint: "http://127.0.0.1:2" }], capability: "secret", fetcher: async () => { throw new Error("network must not be called"); }, }); await assert.rejects(control.workflows({ subject: "operator", grants: new Set(["admin"]) }, null), /ConsoleStoreSelectionRequired/); - await assert.rejects(control.edit({}, "secret", { subject: "operator", grants: new Set(["admin"]) }), /ConsoleAdapterUnavailable/); + await assert.rejects(control.edit({ expected: 1, document: "" }, "secret", { subject: "operator", grants: new Set(["admin"]) }), /ConsoleStoreSelectionRequired/); }); diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 1b5483ebf..86bd50075 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -170,6 +170,36 @@ fn control_routes_require_admin_and_project_applied_workflows() { assert_eq!(view["pipelines"][0]["schedule"], "every 1d"); } +#[test] +fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() { + let (dir, public) = project(); + let root = dir.path(); + let original = "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1h\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); + stdout(&run(root, &["pipeline", "import", "--project", "research"])); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let read = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let changed = original.replace("every 1h", "every 1d"); + let edit = json!({ "expected": 1, "document": changed }).to_string(); + assert_eq!(control(&addr, "POST", "/control/edit", None, &edit).0, 401); + assert_eq!(control(&addr, "POST", "/control/edit", Some(&read), &edit).0, 403); + let invalid = json!({ "expected": 1, "document": changed.replace("every 1d", "invalid schedule") }).to_string(); + assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &invalid).0, 422); + let (status, draft) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); + assert_eq!(status, 200, "{draft}"); + assert_eq!(draft["expected"], 1); + let (_, before) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); + assert_eq!(before["applied"], 1); + assert_eq!(before["pipelines"][0]["schedule"], "every 1h"); + let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + assert_eq!(status, 200, "{applied}"); + assert_eq!(applied["applied"], 2); + assert_eq!(applied["pipelines"][0]["schedule"], "every 1d"); + assert_eq!(control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}").0, 409); +} + #[test] fn published_workflow_listing_caps_entries_and_flags_truncation() { let (dir, public) = project(); From 37164d8af12d734c8719d08719ddbebdd77353c2 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:19:01 +0800 Subject: [PATCH 077/155] feat(console): host governed browse in Query --- apps/console/dist/server.js | 226 ++++++++++++++++++++------ apps/console/src/browse.ts | 14 +- apps/console/src/entry.ts | 7 +- apps/console/src/index.ts | 40 ++++- apps/console/src/live.ts | 18 +- apps/console/src/live_browse.ts | 21 +++ apps/console/test/browse.test.ts | 2 +- apps/console/test/live_browse.test.ts | 4 +- 8 files changed, 259 insertions(+), 73 deletions(-) create mode 100644 apps/console/src/live_browse.ts diff --git a/apps/console/dist/server.js b/apps/console/dist/server.js index 8097858e6..da4698bd3 100644 --- a/apps/console/dist/server.js +++ b/apps/console/dist/server.js @@ -287,52 +287,57 @@ function source(columns, row, table) { ...typeof url === "string" && /^https?:\/\//.test(url) ? { url } : {} }; } -function createLiveTurn({ stores, env, fetcher = fetch }) { +async function openReader({ stores, env, fetcher = fetch }, input) { + const store = stores.find((entry) => entry.id === input.store); + if (!store) throw new ConsoleError("ConsoleRequestMalformed"); + const shared = env[store.credentialName]; + const credential = store.auth === "exchange" ? await resolveReaderCredential({ + shared, + mint: async () => { + if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); + const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ jwt: input.operator.assertion }) + }); + if (!response.ok) { + const refusal2 = await response.json(); + const error = record(refusal2) ? refusal2.error : void 0; + if ((response.status === 401 || response.status === 403) && record(error) && typeof error.identifier === "string") { + throw new ConsoleError("ConsoleTokenExchangeRefused", error.identifier, 403); + } + throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); + } + const value = await response.json(); + if (!record(value) || typeof value.token !== "string" || !value.token.trim()) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); + return value.token; + } + }) : shared; + if (!credential) throw new ConsoleError("ConsoleTokenExchangeRefused"); + let id = 0; + const call = async (name, args, signal) => { + const response = await fetcher(new URL("/mcp", store.endpoint), { + method: "POST", + signal, + headers: { "Authorization": `Bearer ${credential}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }) + }); + if (!response.ok) throw new ConsoleError("ConsoleStoreReadRefused", `store answered ${response.status}`, response.status); + const message = await response.json(); + if (!record(message) || !record(message.result) || message.result.isError === true || !record(message.result.structuredContent)) { + throw new ConsoleError("ConsoleStoreReadRefused"); + } + return message.result.structuredContent; + }; + return { store, call }; +} +function createLiveTurn(options) { + const { env, fetcher = fetch } = options; const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; const modelId = env.CONTEXTFUL_MODEL_ID; if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); return async (input) => { - const store = stores.find((entry) => entry.id === input.store); - if (!store) throw new ConsoleError("ConsoleRequestMalformed"); - const shared = env[store.credentialName]; - const credential = store.auth === "exchange" ? await resolveReaderCredential({ - shared, - mint: async () => { - if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); - const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ jwt: input.operator.assertion }) - }); - if (!response.ok) { - const refusal2 = await response.json(); - const error = record(refusal2) ? refusal2.error : void 0; - if ((response.status === 401 || response.status === 403) && record(error) && typeof error.identifier === "string") { - throw new ConsoleError("ConsoleTokenExchangeRefused", error.identifier, 403); - } - throw new ConsoleError("ConsoleTokenExchangeUnavailable", `exchange answered ${response.status}`, 503); - } - const value = await response.json(); - if (!record(value) || typeof value.token !== "string" || !value.token.trim()) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); - return value.token; - } - }) : shared; - if (!credential) throw new ConsoleError("ConsoleTokenExchangeRefused"); - let id = 0; - const call = async (name, args, signal) => { - const response = await fetcher(new URL("/mcp", store.endpoint), { - method: "POST", - signal, - headers: { "Authorization": `Bearer ${credential}`, "Content-Type": "application/json" }, - body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }) - }); - if (!response.ok) throw new ConsoleError("ConsoleStoreReadRefused", `store answered ${response.status}`, response.status); - const message = await response.json(); - if (!record(message) || !record(message.result) || message.result.isError === true || !record(message.result.structuredContent)) { - throw new ConsoleError("ConsoleStoreReadRefused"); - } - return message.result.structuredContent; - }; + const { store, call } = await openReader(options, input); const description = await call("context.describe", {}); const selected = selectTable(input.question, description.tables); if (!selected) throw new ConsoleError("ConsoleUngroundedAnswer", "No data table matches this question."); @@ -379,6 +384,89 @@ function createLiveTurn({ stores, env, fetcher = fetch }) { }; } +// src/browse.ts +var MAX_TABLES = 64; +var MAX_FILES = 128; +var MAX_PREVIEW_ROWS = 100; +var BrowseError = class extends Error { + code; + constructor(code, message = code) { + super(message); + this.name = "BrowseError"; + this.code = code; + } +}; +var abbreviations = /* @__PURE__ */ new Set(["API", "CSV", "PDF", "SEC", "SQL", "URL"]); +function humanizeLabel(identifier) { + const tail = identifier.split("/").at(-1) ?? identifier; + return tail.replace(/\.[^.]+$/, "").replace(/([a-z])([A-Z])/g, "$1 $2").split(/[-_\s]+/).filter(Boolean).map((part) => { + const upper = part.toUpperCase(); + return abbreviations.has(upper) || /^[Qq]\d+$/.test(part) ? upper : part[0].toUpperCase() + part.slice(1).toLowerCase(); + }).join(" "); +} +function object(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + return value; +} +function bounds(request) { + return request.asOf ? { as_of: request.asOf } : {}; +} +function createBrowse(transport) { + async function read(tool, arguments_) { + const result = await transport.call(tool, arguments_); + if (result.isError) { + const error = object(object(result.structuredContent).error); + throw new BrowseError(String(error.identifier ?? "ConsoleBrowseReadRefused")); + } + return object(result.structuredContent); + } + async function listed(request) { + const listing = await read("context.describe", bounds(request)); + if (!Array.isArray(listing.tables)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + const tables = listing.tables.map((item) => object(item)).filter((item) => item.zone_admitted === true && item.kind !== "memory" && typeof item.table === "string").slice(0, MAX_TABLES).map((item) => ({ table: item.table, description: typeof item.description === "string" ? item.description : void 0 })); + const names2 = new Set(tables.map((table) => table.table)); + const fileListing = await read("context.files", bounds(request)); + if (!Array.isArray(fileListing.rows)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + const files = fileListing.rows.filter((row) => Array.isArray(row) && typeof row[0] === "string" && typeof row[1] === "string" && names2.has(row[0])).slice(0, MAX_FILES).map(([table, path]) => ({ table, path, label: humanizeLabel(path) })); + return { tables, files }; + } + return { + async discover(request) { + const { tables, files } = await listed(request); + const chips = tables.map((table) => ({ table: table.table, label: humanizeLabel(table.table), description: table.description })); + const descriptions = await Promise.all(tables.map((table) => read("context.describe", { table: table.table, ...bounds(request) }))); + const insights = tables.flatMap((table, index) => { + const count = descriptions[index].row_count; + return typeof count === "number" && Number.isFinite(count) && count >= 0 ? [{ table: table.table, label: humanizeLabel(table.table), rows: count }] : []; + }); + return { chips, insights, files }; + }, + async preview(request) { + const { files } = await listed(request); + if (!files.some((file) => file.path === request.path)) throw new BrowseError("ConsoleGalleryPathUnlisted", request.path); + const response = await read("context.file", { path: request.path, limit: MAX_PREVIEW_ROWS, ...bounds(request) }); + if (!Array.isArray(response.columns) || !Array.isArray(response.rows)) throw new BrowseError("ConsoleBrowseResponseMalformed"); + return { columns: response.columns.slice(0, 64), rows: response.rows.slice(0, MAX_PREVIEW_ROWS).map((row) => Array.isArray(row) ? row.slice(0, 64) : []) }; + } + }; +} + +// src/live_browse.ts +function createLiveBrowse(options) { + async function session(input) { + const { call } = await openReader(options, input); + return createBrowse({ call: async (name, args) => ({ structuredContent: await call(name, args) }) }); + } + return { + async discover(input) { + return (await session(input)).discover({ asOf: input.asOf }); + }, + async preview(input) { + return (await session(input)).preview({ asOf: input.asOf, path: input.path }); + } + }; +} + // src/server.ts import { createServer } from "node:http"; @@ -390,7 +478,7 @@ function encoded(value) { function decode(value) { return JSON.parse(Buffer.from(value, "base64url").toString("utf8")); } -function object(value) { +function object2(value) { return value !== null && typeof value === "object" && !Array.isArray(value); } function secureEqual(left, right) { @@ -423,7 +511,7 @@ function verifyCognitoSession(cookie, identity2) { } catch { return null; } - if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string") || session.assertion !== void 0 && typeof session.assertion !== "string") return null; + if (!object2(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || session.expiresAt <= Math.floor(Date.now() / 1e3) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string") || session.assertion !== void 0 && typeof session.assertion !== "string") return null; const grants = /* @__PURE__ */ new Set(); if (session.groups.includes(identity2.queryGroup)) grants.add("query"); if (session.groups.includes(identity2.adminGroup)) grants.add("admin"); @@ -440,7 +528,7 @@ function verifyAccess(assertion, identity2) { } catch { return null; } - if (!object(header) || header.alg !== "RS256" || !object(claims)) return null; + if (!object2(header) || header.alg !== "RS256" || !object2(claims)) return null; if (claims.iss !== identity2.issuer || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3)) return null; if (typeof claims.nbf === "number" && claims.nbf > Math.floor(Date.now() / 1e3)) return null; const audience = typeof claims.aud === "string" ? [claims.aud] : Array.isArray(claims.aud) ? claims.aud : []; @@ -506,7 +594,7 @@ async function cognitoCallback(request, identity2) { }); if (!exchange.ok) return refusal("ConsolePageForbidden"); const tokens = await exchange.json(); - if (!object(tokens) || typeof tokens.id_token !== "string") return refusal("ConsolePageForbidden"); + if (!object2(tokens) || typeof tokens.id_token !== "string") return refusal("ConsolePageForbidden"); const parts = tokens.id_token.split("."); if (parts.length !== 3) return refusal("ConsolePageForbidden"); let header; @@ -517,7 +605,7 @@ async function cognitoCallback(request, identity2) { } catch { return refusal("ConsolePageForbidden"); } - if (!object(header) || header.alg !== "RS256" || !object(claims) || claims.iss !== identity2.issuer || claims.aud !== identity2.clientId || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3) || claims.token_use !== "id") return refusal("ConsolePageForbidden"); + if (!object2(header) || header.alg !== "RS256" || !object2(claims) || claims.iss !== identity2.issuer || claims.aud !== identity2.clientId || typeof claims.sub !== "string" || !claims.sub || !Number.isSafeInteger(claims.exp) || claims.exp <= Math.floor(Date.now() / 1e3) || claims.token_use !== "id") return refusal("ConsolePageForbidden"); const key = identity2.keys ? typeof header.kid === "string" ? identity2.keys.get(header.kid) : void 0 : typeof identity2.publicKey === "string" ? createPublicKey(identity2.publicKey) : identity2.publicKey; if (!key) return refusal("ConsolePageForbidden"); if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); @@ -599,6 +687,31 @@ function createConsole(adapters) { if (request.method === "GET" && path === `/${grant}`) return page(grant); if (grant === "query") { if (request.method === "GET" && path === "/query/api/stores") return json(await adapters.read.list(operator)); + if (request.method === "POST" && (path === "/query/api/browse" || path === "/query/api/preview")) { + let input; + try { + input = await body(request); + } catch (error) { + return bodyFailure(error); + } + if (!object2(input) || typeof input.store !== "string" || !adapters.stores.some((store) => store.id === input.store) || input.asOf !== void 0 && typeof input.asOf !== "string" || path.endsWith("/preview") && (typeof input.path !== "string" || !input.path || input.path.length > 4096)) { + return refusal("ConsoleRequestMalformed", 400); + } + let asOf; + try { + asOf = input.asOf === void 0 ? void 0 : parseVantage(input.asOf); + } catch { + return refusal("ConsoleVantageUnparseable", 400); + } + if (!adapters.browse) return refusal("ConsoleAdapterUnavailable", 503); + try { + return json(path.endsWith("/preview") ? await adapters.browse.preview({ operator, store: input.store, asOf, path: input.path }) : await adapters.browse.discover({ operator, store: input.store, asOf })); + } catch (error) { + if (error instanceof BrowseError) return refusal(error.code, error.code === "ConsoleGalleryPathUnlisted" ? 403 : 400); + if (error instanceof ConsoleError) return refusal(error.code, error.status); + throw error; + } + } if (request.method === "POST" && path === "/query/api/ask") { let input; try { @@ -606,7 +719,7 @@ function createConsole(adapters) { } catch (error) { return bodyFailure(error); } - if (!object(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); + if (!object2(input) || typeof input.store !== "string" || typeof input.question !== "string" || !input.question.trim() || !adapters.stores.some((store) => store.id === input.store)) return refusal("ConsoleRequestMalformed", 400); try { return json(await adapters.turn({ operator, store: input.store, question: input.question })); } catch (error) { @@ -644,12 +757,15 @@ function createConsole(adapters) { var sharedStyle = ``; -var queryPage = `Query \xB7 Contextful${sharedStyle}

Ask the store

Answers cite the rows your access permits.

Conversation

Your answer appears here.

Results

`; var adminPage = `Admin \xB7 Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; const adminPage = `Admin · Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; -var adminPage = `Admin \xB7 Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; - -// src/server.ts -var maxBodyBytes = 1048576; -async function receive(message, origin) { - const method = message.method ?? "GET"; - const chunks = []; - let size = 0; - for await (const chunk of message) { - size += chunk.length; - if (size > maxBodyBytes) throw new Error("ConsoleBodyTooLarge"); - chunks.push(Buffer.from(chunk)); - } - const body2 = method === "GET" || method === "HEAD" ? void 0 : Buffer.concat(chunks); - const headers = new Headers(); - for (const [key, value] of Object.entries(message.headers)) { - if (typeof value === "string") headers.set(key, value); - else if (Array.isArray(value)) headers.set(key, value.join(", ")); - } - return new Request(new URL(message.url ?? "/", origin), { method, headers, body: body2 }); -} -async function send(reply, response) { - reply.statusCode = response.status; - response.headers.forEach((value, key) => reply.setHeader(key, value)); - reply.end(Buffer.from(await response.arrayBuffer())); -} -function serveConsole(adapters, origin) { - const app = createConsole(adapters); - return createServer(async (message, reply) => { - try { - await send(reply, await app.fetch(await receive(message, typeof origin === "string" ? origin : origin()))); - } catch (error) { - if (error instanceof Error && error.message === "ConsoleBodyTooLarge") { - await send(reply, Response.json({ error: { identifier: "ConsoleBodyTooLarge" } }, { status: 413 })); - return; - } - const unavailable2 = error instanceof Error && error.message === "ConsoleAdapterUnavailable"; - await send(reply, Response.json({ error: { identifier: unavailable2 ? "ConsoleAdapterUnavailable" : "ConsoleServerFailure" } }, { status: unavailable2 ? 503 : 500 })); - } - }); -} - -// src/entry.ts -function required(name) { - const value = process.env[name]; - if (!value) throw new Error(`${name} is required`); - return value; -} -async function keysAt(url) { - const response = await fetch(url); - if (!response.ok) throw new Error(`identity key route returned ${response.status}`); - const document = await response.json(); - if (!document || typeof document !== "object" || !("keys" in document) || !Array.isArray(document.keys)) { - throw new Error("identity key route returned no key set"); - } - const keys = /* @__PURE__ */ new Map(); - for (const value of document.keys) { - if (!value || typeof value !== "object" || value.kty !== "RSA" || value.alg !== "RS256" || typeof value.kid !== "string") continue; - keys.set(value.kid, createPublicKey2({ key: value, format: "jwk" })); - } - if (keys.size === 0) throw new Error("identity key route returned no RS256 key"); - return keys; -} -async function identity() { - if (process.env.CONTEXTFUL_IDENTITY === "cognito") { - const keys = await keysAt(required("CONTEXTFUL_COGNITO_JWKS_URL")); - return { - kind: "cognito", - sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), - queryGroup: required("CONTEXTFUL_COGNITO_QUERY_GROUP"), - adminGroup: required("CONTEXTFUL_COGNITO_ADMIN_GROUP"), - issuer: required("CONTEXTFUL_COGNITO_ISSUER"), - clientId: required("CONTEXTFUL_COGNITO_CLIENT_ID"), - authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), - tokenUrl: required("CONTEXTFUL_COGNITO_TOKEN_URL"), - redirectUri: required("CONTEXTFUL_COGNITO_REDIRECT_URI"), - keys - }; - } - return { - kind: "access", - issuer: required("CONTEXTFUL_ACCESS_ISSUER"), - queryAudience: required("CONTEXTFUL_QUERY_ACCESS_AUDIENCE"), - adminAudience: required("CONTEXTFUL_ADMIN_ACCESS_AUDIENCE"), - keys: await keysAt(required("CONTEXTFUL_ACCESS_JWKS_URL")) - }; -} -function unavailable() { - throw new Error("ConsoleAdapterUnavailable"); -} -function unavailableAdapters() { - return { - turn: async () => unavailable(), - control: { - workflows: async () => unavailable(), - record: async () => unavailable(), - edit: async () => unavailable(), - apply: async () => unavailable() - } - }; -} -async function main() { - const addressFlag = process.argv.indexOf("--http"); - if (addressFlag < 0 || !process.argv[addressFlag + 1]) throw new Error("--http is required"); - const address = process.argv[addressFlag + 1]; - const separator = address.lastIndexOf(":"); - const host = address.slice(0, separator); - const port = Number(address.slice(separator + 1)); - if (!host || !Number.isInteger(port) || port < 0 || port > 65535) throw new Error("invalid --http address"); - const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); - const stores = registry.entries.map(({ id, label }) => ({ id, label })); - let adapters = unavailableAdapters(); - const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; - if (modulePath) { - const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); - const module = await import(pathToFileURL(absolute).href); - if (!module.createAdapters) throw new Error("console adapter module exports no createAdapters"); - adapters = await module.createAdapters({ stores, env: process.env }); - } - let origin = process.env.CONTEXTFUL_CONSOLE_ORIGIN ?? `http://${host}:${port}`; - const app = serveConsole({ - identity: await identity(), - stores, - adminCapability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, - turn: adapters.turn, - read: adapters.read ?? { list: async () => stores }, - control: adapters.control - }, () => origin); - await new Promise((resolveListen, rejectListen) => { - app.once("error", rejectListen); - app.listen(port, host, resolveListen); - }); - const bound = app.address(); - if (!bound || typeof bound === "string") throw new Error("console listener has no address"); - if (!process.env.CONTEXTFUL_CONSOLE_ORIGIN) origin = `http://${host}:${bound.port}`; - process.stderr.write(`listening on http://${host}:${bound.port} -`); -} -main().catch((error) => { - process.stderr.write(`${error instanceof Error ? error.message : "console startup failed"} -`); - process.exitCode = 1; -}); From c3e63a47ba71f7ce1ad459d12d878b16bd55ce95 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:30:47 +0800 Subject: [PATCH 081/155] feat(console): connect live answer modules to governed Query --- apps/console/src/answer_live.ts | 108 ++++++++++++++++++++++++++ apps/console/src/entry.ts | 8 +- apps/console/src/live.ts | 57 ++++++++++---- apps/console/test/answer_live.test.ts | 8 ++ 4 files changed, 164 insertions(+), 17 deletions(-) create mode 100644 apps/console/src/answer_live.ts diff --git a/apps/console/src/answer_live.ts b/apps/console/src/answer_live.ts new file mode 100644 index 000000000..6d989e8de --- /dev/null +++ b/apps/console/src/answer_live.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; +import type { Operator, TurnInput } from "./index.ts"; +import type { Arrival, Conclusion } from "./brief.ts"; +import { learnAfterAnswer, type Learning } from "./learn.ts"; +import { consoleDenylist, createLiveTurn, openReader, redactText, selectTable, type LiveOptions } from "./live.ts"; + +type AnswerOptions = LiveOptions & { clock?: () => number }; + +function object(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function scope(...parts: string[]): string { + return createHash("sha256").update(JSON.stringify(parts)).digest("hex"); +} + +function observed(subject: string, question: string, answer: string): boolean { + const word = subject.trim().toLowerCase(); + return word.length >= 3 && (`${question} ${answer}`).toLowerCase().includes(word); +} + +export function createLiveAnswer(options: AnswerOptions) { + const { env, fetcher = fetch, clock = Date.now } = options; + const endpoint = env.CONTEXTFUL_MODEL_ENDPOINT; + const modelId = env.CONTEXTFUL_MODEL_ID; + if (!endpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); + const modelUrl = new URL("chat/completions", `${endpoint.replace(/\/$/, "")}/`); + const configuredDenylist = consoleDenylist(env); + const liveTurn = createLiveTurn(options); + const learnings = new Map>(); + const turns = new Map(); + const actorScope = (operator: Operator, store: string) => scope(operator.subject, store); + const sessionScope = (operator: Operator, store: string) => scope(operator.subject, store, operator.assertion ?? ""); + const denied = (operator: Operator) => [...configuredDenylist, ...(operator.assertion ? [operator.assertion] : [])]; + + async function distill(question: string, answer: string, operator: Operator): Promise { + try { + const response = await fetcher(modelUrl, { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: modelId, messages: [ + { role: "system", content: "Distil at most three observed facts into JSON entries with subject, key, learning. Only subjects present in the question or answer qualify." }, + { role: "user", content: JSON.stringify({ question, answer }) }, + ] }), + }); + if (!response.ok) return []; + const body: unknown = await response.json(); + const choices = object(body) && Array.isArray(body.choices) ? body.choices : []; + const message = choices.length && object(choices[0]) ? choices[0].message : undefined; + if (!object(message) || typeof message.content !== "string") return []; + const parsed: unknown = JSON.parse(message.content); + const entries = object(parsed) && Array.isArray(parsed.entries) ? parsed.entries : []; + return entries.flatMap((entry): Learning[] => { + if (!object(entry) || typeof entry.subject !== "string" || typeof entry.key !== "string" || typeof entry.learning !== "string") return []; + if (!observed(entry.subject, question, answer)) return []; + return [{ subject: redactText(entry.subject, denied(operator)), key: redactText(entry.key, denied(operator)), + learning: redactText(entry.learning, denied(operator)) }]; + }); + } catch { return []; } + } + + async function turn(input: TurnInput) { + const result = await liveTurn(input); + const reader = actorScope(input.operator, input.store); + const session = sessionScope(input.operator, input.store); + const stream = (async function* () { yield result.answer; })(); + await learnAfterAnswer({ scope: reader, question: input.question, stream, + distill: ({ question, answer }) => distill(question, answer, input.operator), + land: async (key, entry) => { + const found = learnings.get(key) ?? new Map(); + found.set(`${entry.subject}\0${entry.key}`, entry); + learnings.set(key, found); + }, + }); + turns.set(session, (turns.get(session) ?? 0) + 1); + return result; + } + + async function brief(operator: Operator, store: string) { + const entries = [...(learnings.get(actorScope(operator, store))?.values() ?? [])]; + const conclusions: Conclusion[] = entries.map((entry) => ({ subject: entry.subject, text: entry.learning, live: true })); + return { + session: { turns: turns.get(sessionScope(operator, store)) ?? 0, vantage: "present" }, + conclusions, + now: clock(), + loadArrivals: async (): Promise => { + const { call } = await openReader(options, operator, store); + const description = await call("context.describe", {}); + const question = entries.map((entry) => `${entry.subject} ${entry.key} ${entry.learning}`).join(" "); + const table = selectTable(question, description.tables); + if (!table) return []; + const result = await call("context.query", { sql: `SELECT * FROM "${table.replaceAll('"', '""')}"`, limit: 20 }); + const columns = Array.isArray(result.columns) ? result.columns.filter((value): value is string => typeof value === "string") : []; + const rows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; + const at = (row: unknown[], names: string[]) => names.map((name) => row[columns.indexOf(name)]).find((value) => typeof value === "string" && value) as string | undefined; + return rows.flatMap((row): Arrival[] => { + const id = at(row, ["filing_id", "id", "document_id"]); + const label = at(row, ["title", "label", "summary"]); + const arrivedAt = at(row, ["published_at", "arrived_at", "date"]); + if (!id || !label || !arrivedAt) return []; + const topics = [label, at(row, ["summary"]) ?? ""].map((value) => redactText(value, denied(operator))); + return [{ id: redactText(id, denied(operator)), label: redactText(label, denied(operator)), topics, arrivedAt }]; + }).slice(0, 20); + }, + }; + } + + return { turn, brief, briefBudgetMs: 250, redactView: (operator: Operator, value: string) => redactText(value, denied(operator)) }; +} diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 51cfd115d..6554daf08 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -2,7 +2,7 @@ import { createPublicKey } from "node:crypto"; import { pathToFileURL } from "node:url"; import { isAbsolute, resolve } from "node:path"; import { registryFromEnv } from "../../gateway/src/index.ts"; -import { createLiveTurn } from "./live.ts"; +import { createLiveAnswer } from "./answer_live.ts"; import type { ConsoleAdapters, Identity } from "./index.ts"; import { serveConsole } from "./server.ts"; @@ -84,9 +84,13 @@ async function main(): Promise { const registry = registryFromEnv(process.env.CONTEXTFUL_STORES_JSON); const stores = registry.entries.map(({ id, label }) => ({ id, label })); const modulePath = process.env.CONTEXTFUL_CONSOLE_ADAPTER_MODULE; + const live = modulePath ? null : createLiveAnswer({ stores: registry.entries, env: process.env }); let adapters: HostedAdapters = { ...unavailableAdapters(), - turn: modulePath ? unavailableAdapters().turn : createLiveTurn({ stores: registry.entries, env: process.env }), + turn: live?.turn ?? unavailableAdapters().turn, + brief: live?.brief, + briefBudgetMs: live?.briefBudgetMs, + redactView: live?.redactView, }; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index 7f5f3119d..470999b2e 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -1,8 +1,22 @@ import type { StoreEntry } from "../../gateway/src/index.ts"; -import type { TurnInput, TurnResult } from "./index.ts"; -import { ConsoleError, createTurn, resolveReaderCredential, type Source, type ToolResult } from "./turn.ts"; +import type { Operator, TurnInput, TurnResult } from "./index.ts"; +import { ConsoleError, createTurn, resolveReaderCredential, StreamingRedactor, type Source, type ToolResult } from "./turn.ts"; -type LiveOptions = { stores: StoreEntry[]; env: NodeJS.ProcessEnv; fetcher?: typeof fetch }; +export type LiveOptions = { stores: StoreEntry[]; env: NodeJS.ProcessEnv; fetcher?: typeof fetch }; + +export function consoleDenylist(env: NodeJS.ProcessEnv): string[] { + if (!env.CONTEXTFUL_CONSOLE_DENYLIST) return []; + const parsed: unknown = JSON.parse(env.CONTEXTFUL_CONSOLE_DENYLIST); + if (!Array.isArray(parsed) || !parsed.every((entry) => typeof entry === "string")) throw new Error("ConsoleDenylistMalformed"); + return parsed; +} + +export function redactText(value: string, denylist: string[]): string { + const long = denylist.filter((entry) => entry.length > 128); + const exact = long.reduce((text, entry) => text.replaceAll(entry, "[redacted]"), value); + const redactor = new StreamingRedactor(denylist.filter((entry) => entry.length <= 128)); + return redactor.push(exact) + redactor.finish(); +} function record(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); @@ -17,7 +31,7 @@ function terms(value: string): Set { word.endsWith("ies") && word.length > 4 ? `${word.slice(0, -3)}y` : word.endsWith("s") && word.length > 3 ? word.slice(0, -1) : word)); } -function selectTable(question: string, listing: unknown): string | null { +export function selectTable(question: string, listing: unknown): string | null { const asked = terms(question); const entries = Array.isArray(listing) ? listing : []; const ranked = entries.flatMap((item) => { @@ -42,21 +56,17 @@ function source(columns: string[], row: unknown[], table: string): Source | null ...(typeof url === "string" && /^https?:\/\//.test(url) ? { url } : {}) }; } -export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): (input: TurnInput) => Promise { - const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; - const modelId = env.CONTEXTFUL_MODEL_ID; - if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); - return async (input) => { - const store = stores.find((entry) => entry.id === input.store); +export async function openReader({ stores, env, fetcher = fetch }: LiveOptions, operator: Operator, storeId: string) { + const store = stores.find((entry) => entry.id === storeId); if (!store) throw new ConsoleError("ConsoleRequestMalformed"); const shared = env[store.credentialName]; const credential = store.auth === "exchange" ? await resolveReaderCredential({ shared, mint: async () => { - if (!input.operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); + if (!operator.assertion || !store.exchangeRoute) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "reader assertion or exchange route absent", 503); const response = await fetcher(new URL(store.exchangeRoute, store.endpoint), { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ jwt: input.operator.assertion }), + body: JSON.stringify({ jwt: operator.assertion }), }); if (!response.ok) { const refusal: unknown = await response.json(); @@ -88,12 +98,27 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( return message.result.structuredContent; }; + return { store, call, credential }; +} + +export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Promise { + const { env, fetcher = fetch } = options; + const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; + const modelId = env.CONTEXTFUL_MODEL_ID; + if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); + const configuredDenylist = consoleDenylist(env); + return async (input) => { + const { store, call, credential } = await openReader(options, input.operator, input.store); + const denylist = [...configuredDenylist, credential, ...(input.operator.assertion ? [input.operator.assertion] : [])]; + const description = await call("context.describe", {}); const selected = selectTable(input.question, description.tables); if (!selected) throw new ConsoleError("ConsoleUngroundedAnswer", "No data table matches this question."); const tables = [selected]; let remainingSources = 8; + let resultRows: TurnResult["resultRows"]; const turn = createTurn({ + denylist: denylist.filter((entry) => entry.length <= 128), tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read" as const, table })), tables: tables.map((name) => ({ name, kind: "data" as const })), planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), @@ -104,9 +129,11 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; }).slice(0, remainingSources); remainingSources -= sourced.length; + const cleanRows = sourced.map(({ row }) => row.map((value) => typeof value === "string" ? redactText(value, denylist) : value)); + resultRows = { columns, rows: cleanRows }; return { - rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), - sources: sourced.map(({ citation }) => citation), + rows: cleanRows.map((row) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), + sources: sourced.map(({ citation }) => ({ ...citation, label: redactText(citation.label, denylist) })), }; } }, synthesize: async function* ({ question, results, sources }) { @@ -126,6 +153,6 @@ export function createLiveTurn({ stores, env, fetcher = fetch }: LiveOptions): ( }, }); const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); - return { answer: answer.text, sources: answer.sources, widgets: [] }; + return { answer: answer.text, sources: answer.sources, widgets: [], resultRows }; }; } diff --git a/apps/console/test/answer_live.test.ts b/apps/console/test/answer_live.test.ts index 136265fae..f5224d354 100644 --- a/apps/console/test/answer_live.test.ts +++ b/apps/console/test/answer_live.test.ts @@ -88,3 +88,11 @@ test("hosted Query publishes only to the verified operator and sanitizes its ser assert.equal(body.widgets[0].component, "table.v1"); assert.doesNotMatch(JSON.stringify(body.widgets), /secret@example.test/); }); + +test("long reader assertions and minted credentials preserve governed Query", async () => { + const { live } = fixture(); + const assertion = "a".repeat(512); + const answer = await live.turn({ operator: { subject: "alice", grants: new Set(["query"]), assertion }, + store: store.id, question: "Which Northwind filing arrived?" }); + assert.match(answer.answer, /filing-1/); +}); From ec2585f0eb04b81c9f35a52837eee4988a57f858 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:32:40 +0800 Subject: [PATCH 082/155] feat(control): validate and claim versioned store drafts --- apps/console/src/control.ts | 18 ++++-- apps/console/src/index.ts | 6 +- crates/acceptance/tests/integration/m12.rs | 9 ++- crates/contextful-agent/src/http.rs | 4 +- crates/contextful-cli/src/cadence.rs | 47 ++++++++++++++- crates/contextful-cli/src/serve.rs | 32 ++++++++--- .../contextful-cli/tests/integration/serve.rs | 27 ++++++++- crates/contextful-core/src/surface/mod.rs | 5 +- crates/contextful-engine/src/control.rs | 55 ++++++++++++++++++ spec/70-control-plane.md | 6 ++ spec/adr/A-surface.md | 2 +- spec/cards/surface.md | 5 +- spec/spec.lock.json | 57 +++++++++++++++---- spec/status.md | 6 +- spec/terms/surface.toml | 1 + 15 files changed, 240 insertions(+), 40 deletions(-) diff --git a/apps/console/src/control.ts b/apps/console/src/control.ts index 0abcac79a..cf8d2a032 100644 --- a/apps/console/src/control.ts +++ b/apps/console/src/control.ts @@ -17,12 +17,16 @@ function selected(stores: ControlStore[], id: string | null): ControlStore { throw new Error("ConsoleStoreSelectionRequired"); } -function applyDocument(document: unknown): { store: string | null; body: { id?: string } } { +function controlDocument(document: unknown, editing: boolean): { store: string | null; body: { expected: number; document?: string } } { if (document === null || typeof document !== "object" || Array.isArray(document)) throw new Error("ConsoleRequestMalformed"); const value = document as Record; if (value.store !== undefined && typeof value.store !== "string") throw new Error("ConsoleRequestMalformed"); - if (value.id !== undefined && typeof value.id !== "string") throw new Error("ConsoleRequestMalformed"); - return { store: typeof value.store === "string" ? value.store : null, body: typeof value.id === "string" ? { id: value.id } : {} }; + if (!Number.isSafeInteger(value.expected) || (value.expected as number) < 1) throw new Error("ConsoleRequestMalformed"); + if (editing && typeof value.document !== "string") throw new Error("ConsoleRequestMalformed"); + const keys = Object.keys(value); + if (keys.some((key) => key !== "store" && key !== "expected" && (key !== "document" || !editing))) throw new Error("ConsoleRequestMalformed"); + return { store: typeof value.store === "string" ? value.store : null, + body: editing ? { expected: value.expected as number, document: value.document as string } : { expected: value.expected as number } }; } export function createLiveControl({ stores, capability, fetcher = fetch }: ControlOptions): ConsoleAdapters["control"] { @@ -38,10 +42,14 @@ export function createLiveControl({ stores, capability, fetcher = fetch }: Contr return { workflows: async (_operator, id) => capability ? call(selected(stores, id), "/control/workflows", capability) : unavailable(), record: async () => unavailable(), - edit: async () => unavailable(), + edit: async (document, token) => { + if (!capability || token !== capability) return unavailable(); + const input = controlDocument(document, true); + return call(selected(stores, input.store), "/control/edit", token, input.body); + }, apply: async (document, token) => { if (!capability || token !== capability) return unavailable(); - const input = applyDocument(document); + const input = controlDocument(document, false); return call(selected(stores, input.store), "/control/apply", token, input.body); }, }; diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 3aa79291f..bde6507a8 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -310,6 +310,8 @@ function draw(widget){const props=widget.props??{};if(widget.component==='table. form.addEventListener('submit',async event=>{event.preventDefault();const question=document.getElementById('question').value.trim();if(!question)return;const response=await fetch('/query/api/ask',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({store:store.value,question})});const answer=await response.json();transcript.replaceChildren();const item=document.createElement('article');item.textContent=question+'\n'+(answer.answer??answer.error?.identifier??'No answer');transcript.append(item);widgets.replaceChildren();for(const widget of answer.widgets??[])widgets.append(draw(widget));if(answer.sources?.length){const sources=document.createElement('p');sources.textContent='Sources: '+answer.sources.map(x=>x.title??x.url).join(', ');transcript.append(sources)}}); `; -const adminPage = `Admin · Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index 642d42f58..b2db1382b 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -151,9 +151,14 @@ fn m12_console() { assert_eq!(workflows["applied"], 1, "{workflows}"); assert_eq!(workflows["pipelines"][0]["id"], "filings-flow", "{workflows}"); assert_eq!(workflows["pipelines"][0]["tables"], json!(["filings"]), "{workflows}"); - repo.write("pipelines/filings.toml", "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1d\"\ntables = [\"filings\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"); + let draft = "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1d\"\ntables = [\"filings\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + let (status, edited) = request(&console_address, "POST", "/admin/api/edit", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), + Some(&json!({ "store": "field-notes", "expected": 1, "document": draft }))); + assert_eq!(status, 200, "{}", String::from_utf8_lossy(&edited)); + let edited: Value = serde_json::from_slice(&edited).unwrap(); + assert_eq!(edited["expected"], 1, "{edited}"); let (status, applied) = request(&console_address, "POST", "/admin/api/apply", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), - Some(&json!({ "store": "field-notes", "id": "filings-flow" }))); + Some(&json!({ "store": "field-notes", "expected": 1 }))); assert_eq!(status, 200, "{}", String::from_utf8_lossy(&applied)); let applied: Value = serde_json::from_slice(&applied).unwrap(); assert_eq!(applied["applied"], 2, "{applied}"); diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index a8adb6ad3..651e00c38 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -44,6 +44,7 @@ pub const HEALTH_PATH: &str = "/health"; /// The store-published workflow state and validated control claim routes. pub const WORKFLOWS_PATH: &str = "/control/workflows"; +pub const EDIT_PATH: &str = "/control/edit"; pub const APPLY_PATH: &str = "/control/apply"; /// Largest request head the face reads; a head past it answers `431`. @@ -353,8 +354,9 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { (MCP_PATH, _) => { HttpResponse::message(405, "`/mcp` answers POST; the face holds no session and opens no server stream").with("Allow", "POST") } - (WORKFLOWS_PATH, "GET") | (APPLY_PATH, "POST") if self.control.is_some() => self.control(request), + (WORKFLOWS_PATH, "GET") | (EDIT_PATH, "POST") | (APPLY_PATH, "POST") if self.control.is_some() => self.control(request), (WORKFLOWS_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/workflows` answers GET").with("Allow", "GET"), + (EDIT_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/edit` answers POST").with("Allow", "POST"), (APPLY_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/apply` answers POST").with("Allow", "POST"), ("/auth/exchange", "POST") if self.exchange.is_some() => self.exchange.expect("checked above")(request), ("/auth/exchange", _) if self.exchange.is_some() => HttpResponse::message(405, "`/auth/exchange` answers POST").with("Allow", "POST"), diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index 07d712aef..57009bace 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -22,7 +22,7 @@ use contextful_core::surface::control::{admit_loopback, control_url, parse_point use contextful_core::surface::edit::check_document; use contextful_core::surface::dispatch::{CHILD_GRACE_SECS, DEFAULT_POOL}; use contextful_core::surface::SurfaceError; -use contextful_engine::control::{ControlError, SnapshotDir}; +use contextful_engine::control::{ControlError, Draft, SnapshotDir}; use contextful_engine::scheduler::{Beat, Dispatch, Entry, Fired, LeaseState, Scheduler}; use contextful_engine::worker::{Relay, WorkerDispatch}; use contextful_outbound::egress::{system, Outbound, Transport}; @@ -292,6 +292,51 @@ fn render(specs: &BTreeMap) -> Result { Ok(format!("# An applied snapshot, claimed by `contextful pipeline apply`; immutable once claimed.\n\n{body}")) } +fn validated_draft(document: &str, declaration: &Path, tasks: &Tasks) -> Result { + let specs = collect(&[ManifestFile { path: "manifest@draft.toml".into(), text: document.to_owned() }]) + .map_err(|e| SurfaceError::ApplyValidationRefused(e.to_string()))?; + let specs: BTreeMap = specs.into_iter().map(|entry| (entry.spec.id.clone(), entry.spec)).collect(); + let rendered = render(&specs)?; + for spec in specs.values() { + if let Some(schedule) = spec.schedule.as_deref() { + Schedule::parse(schedule).map_err(|e| SurfaceError::ApplyValidationRefused(format!("pipeline `{}`: {e}", spec.id)))?; + } + check(spec, declaration, tasks).map_err(|e| SurfaceError::ApplyValidationRefused(format!("pipeline `{}`: {e:#}", spec.id)))?; + } + Ok(rendered) +} + +/// Store one validated, version-bound control draft without changing the applied pointer. +pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected: u64, document: &str, tasks: &Tasks) -> Result { + let (located, _, control) = located(project, declaration)?; + let snapshots = owner(&control)?; + snapshots.initialized()?; + let text = validated_draft(document, &located.declaration, tasks)?; + snapshots.save_draft(&Draft { expected, document: text })?; + Ok(json!({ "expected": expected })) +} + +/// Revalidate the saved draft and claim it only at the version the editor read. +pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, tasks: &Tasks) -> Result<()> { + let (initial, _, control) = located(project, declaration.clone())?; + let snapshots = owner(&control)?; + snapshots.initialized()?; + let draft = snapshots.read_draft()?; + if draft.expected != expected { + return Err(SurfaceError::ManifestVersionConflict(format!("the draft read v{} and apply named v{expected}", draft.expected)).into()); + } + if validated_draft(&draft.document, &initial.declaration, tasks)? != draft.document { + return Err(SurfaceError::ApplyValidationRefused("the saved draft is not the validated snapshot".into()).into()); + } + let (fresh, _, control) = located(project, declaration)?; + let owner = owner(&control)?; + if owner.root() != snapshots.root() || fresh.declaration != initial.declaration { + return Err(SurfaceError::ConfigOwnerUnconfigured("the store's control owner changed during validation".into()).into()); + } + owner.claim_draft(&draft)?; + Ok(()) +} + #[derive(Serialize)] struct Change { id: String, diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index d08e4fd1a..40d2b4e4a 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -12,9 +12,10 @@ use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; -use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, WORKFLOWS_PATH}; +use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, EDIT_PATH, WORKFLOWS_PATH}; use contextful_core::surface::SurfaceError; use contextful_core::run::derive::task::Tasks; +use contextful_engine::control::ControlError; use contextful_core::issue::{IssuancePolicy, MintContext, NodeRole}; use contextful_core::ports::{Clock, SigningPort}; use contextful_policy::exchange::answer as exchange_answer; @@ -117,27 +118,40 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; let audit = AuditLog::unanchored(located.project.audit_dir())?; let control = |request: &HttpRequest| -> HttpResponse { + let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let answer = match request.target.split('?').next().unwrap_or_default() { WORKFLOWS_PATH => crate::cadence::published(&located.project, &located.declaration), - APPLY_PATH => { + EDIT_PATH | APPLY_PATH => { let body: Value = match serde_json::from_slice(&request.body) { Ok(Value::Object(body)) => Value::Object(body), - _ => return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })), + _ => return malformed(), }; let Some(fields) = body.as_object() else { unreachable!() }; - if fields.len() > 1 || fields.keys().any(|field| field != "id") || fields.get("id").is_some_and(|id| !id.is_string()) { - return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); - } + let Some(expected) = fields.get("expected").and_then(Value::as_u64) else { return malformed() }; let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; - crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), tasks) - .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) + if request.target.split('?').next() == Some(EDIT_PATH) { + if fields.len() != 2 || fields.keys().any(|field| field != "expected" && field != "document") { + return malformed(); + } + let Some(document) = fields.get("document").and_then(Value::as_str) else { return malformed() }; + crate::cadence::edit(&project, Some(located.declaration.clone()), expected, document, tasks) + } else { + if fields.len() != 1 { return malformed(); } + crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, tasks) + .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) + } } _ => unreachable!(), }; match answer { Ok(state) => HttpResponse::json(200, &state), Err(error) => { - let surface = error.chain().find_map(|part| part.downcast_ref::()); + let surface = error.chain().find_map(|part| part.downcast_ref::().or_else(|| { + match part.downcast_ref::() { + Some(ControlError::Surface(refusal)) => Some(refusal), + _ => None, + } + })); let status = surface.map_or(503, SurfaceError::status); let identifier = surface.map(|refusal| refusal.to_string().split(':').next().unwrap_or("ControlUnavailable").to_string()) .unwrap_or_else(|| "ControlUnavailable".into()); diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 86bd50075..d9eeb0d20 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -162,8 +162,9 @@ fn control_routes_require_admin_and_project_applied_workflows() { assert_eq!(control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":1}").0, 400); - std::fs::write(root.join("pipelines/filings.toml"), pipeline.replace("every 1h", "every 1d")).unwrap(); - let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":\"filings-flow\"}"); + let edited = json!({ "expected": 1, "document": pipeline.replace("every 1h", "every 1d") }).to_string(); + assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edited).0, 200); + let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); assert_eq!(status, 200, "{applied}"); assert_eq!(applied["applied"], json!(2)); let (_, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); @@ -183,6 +184,9 @@ fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); let changed = original.replace("every 1h", "every 1d"); let edit = json!({ "expected": 1, "document": changed }).to_string(); + let (status, absent) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + assert_eq!(status, 409, "{absent}"); + assert_eq!(absent["error"]["identifier"], "ControlDraftAbsent"); assert_eq!(control(&addr, "POST", "/control/edit", None, &edit).0, 401); assert_eq!(control(&addr, "POST", "/control/edit", Some(&read), &edit).0, 403); let invalid = json!({ "expected": 1, "document": changed.replace("every 1d", "invalid schedule") }).to_string(); @@ -190,6 +194,13 @@ fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() let (status, draft) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); assert_eq!(status, 200, "{draft}"); assert_eq!(draft["expected"], 1); + let declaration = root.join("contextful.toml"); + let owner_text = std::fs::read_to_string(&declaration).unwrap(); + std::fs::write(&declaration, format!("{owner_text}\n[control]\nurl = \"http://127.0.0.1:12345/\"\n")).unwrap(); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + assert_eq!(status, 503, "{refused}"); + assert_eq!(refused["error"]["identifier"], "ConfigOwnerUnconfigured"); + std::fs::write(&declaration, owner_text).unwrap(); let (_, before) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); assert_eq!(before["applied"], 1); assert_eq!(before["pipelines"][0]["schedule"], "every 1h"); @@ -197,7 +208,17 @@ fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() assert_eq!(status, 200, "{applied}"); assert_eq!(applied["applied"], 2); assert_eq!(applied["pipelines"][0]["schedule"], "every 1d"); - assert_eq!(control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}").0, 409); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + assert_eq!(status, 409, "{refused}"); + assert_eq!(refused["error"]["identifier"], "ControlDraftAbsent"); + let later = original.replace("every 1h", "every 2d"); + let edit = json!({ "expected": 2, "document": later }).to_string(); + assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edit).0, 200); + std::fs::write(root.join("pipelines/filings.toml"), original.replace("every 1h", "every 3d")).unwrap(); + stdout(&run(root, &["pipeline", "apply", "filings-flow", "--project", "research"])); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":2}"); + assert_eq!(status, 409, "{refused}"); + assert_eq!(refused["error"]["identifier"], "ManifestVersionConflict"); } #[test] diff --git a/crates/contextful-core/src/surface/mod.rs b/crates/contextful-core/src/surface/mod.rs index 760c43bfd..5b7057bf3 100644 --- a/crates/contextful-core/src/surface/mod.rs +++ b/crates/contextful-core/src/surface/mod.rs @@ -34,6 +34,9 @@ pub enum SurfaceError { /// (`surface.apply.version-race`) #[error("ManifestVersionConflict: {0}")] ManifestVersionConflict(String), + /// (`surface.apply.draft-absent`) + #[error("ControlDraftAbsent: {0}")] + ControlDraftAbsent(String), /// (`surface.apply.validation`) #[error("ApplyValidationRefused: {0}")] ApplyValidationRefused(String), @@ -94,7 +97,7 @@ impl SurfaceError { | SurfaceError::ControlSourceNotLoopback(_) | SurfaceError::CycleControlSourceUnresolved(_) | SurfaceError::ConditionalWriteUnsupported(_) => 503, - SurfaceError::StoreNotInitialized(_) | SurfaceError::ManifestVersionConflict(_) | SurfaceError::DispatchCallbackRejected(_) => 409, + SurfaceError::StoreNotInitialized(_) | SurfaceError::ManifestVersionConflict(_) | SurfaceError::ControlDraftAbsent(_) | SurfaceError::DispatchCallbackRejected(_) => 409, SurfaceError::EnforceRegionMismatch(_) | SurfaceError::ResidencySitesDiverge(_) => 503, SurfaceError::DispatchSubmitRejected(_) => 401, SurfaceError::ScheduleUnreadable(_) diff --git a/crates/contextful-engine/src/control.rs b/crates/contextful-engine/src/control.rs index a8a488f71..b74171c32 100644 --- a/crates/contextful-engine/src/control.rs +++ b/crates/contextful-engine/src/control.rs @@ -10,9 +10,18 @@ use contextful_core::surface::control::{admit_conditional, parse_pointer, snapsh use contextful_core::surface::SurfaceError; use std::io::ErrorKind; use std::path::{Path, PathBuf}; +use serde::{Deserialize, Serialize}; /// The lock file serializing claims. const LOCK_FILE: &str = "manifest.lock"; +const DRAFT_FILE: &str = "manifest@draft.json"; + +/// A validated store draft bound to the applied version it read. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Draft { + pub expected: u64, + pub document: String, +} /// A snapshot-directory refusal or a storage failure beneath it. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -85,12 +94,58 @@ impl SnapshotDir { std::fs::read_to_string(&path).map_err(|e| SurfaceError::ControlSnapshotUnreadable(format!("{}: {e}", path.display())).into()) } + /// Save one store draft only while the applied version still matches its base. + pub fn save_draft(&self, draft: &Draft) -> Result<(), ControlError> { + let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); + let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; + self.expect_version(draft.expected)?; + let body = serde_json::to_vec(draft).map_err(|e| ControlError::Storage(e.to_string()))?; + replace(&self.root.join(DRAFT_FILE), &body).map_err(storage)?; + Ok(()) + } + + /// Read the saved draft without changing the applied pointer. + pub fn read_draft(&self) -> Result { + let path = self.root.join(DRAFT_FILE); + let body = std::fs::read(&path).map_err(|e| match e.kind() { + ErrorKind::NotFound => ControlError::Surface(SurfaceError::ControlDraftAbsent(format!("`{}` holds no validated draft", self.root.display()))), + _ => ControlError::Storage(format!("{}: {e}", path.display())), + })?; + serde_json::from_slice(&body).map_err(|e| ControlError::Storage(format!("{}: {e}", path.display()))) + } + + /// Claim the exact draft revalidated by the caller under the existing pointer lock. + pub fn claim_draft(&self, draft: &Draft) -> Result { + let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); + let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; + self.expect_version(draft.expected)?; + if self.read_draft()? != *draft { + return Err(SurfaceError::ManifestVersionConflict("the validated draft changed before apply; reload and reapply".into()).into()); + } + let version = self.claim_locked(Some(draft.expected), &draft.document)?; + let _ = std::fs::remove_file(self.root.join(DRAFT_FILE)); + Ok(version) + } + + fn expect_version(&self, expected: u64) -> Result<(), ControlError> { + let current = self.initialized()?; + if current != expected { + return Err(SurfaceError::ManifestVersionConflict(format!("the pointer names v{current} and the draft read v{expected}; reload v{current} and reapply")).into()); + } + Ok(()) + } + /// Claim the version after `expected` holding `text` and advance the pointer to it. A /// pointer no longer at `expected` raises `ManifestVersionConflict` and writes nothing; /// a version file already present is never overwritten, the claim taking the next free one. pub fn claim(&self, expected: Option, text: &str) -> Result { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; + self.claim_locked(expected, text) + } + + fn claim_locked(&self, expected: Option, text: &str) -> Result { + let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let current = self.current()?; if current != expected { let name = |v: Option| v.map(|v| format!("v{v}")).unwrap_or_else(|| "no version".into()); diff --git a/spec/70-control-plane.md b/spec/70-control-plane.md index 6529707e4..00f7ce90f 100644 --- a/spec/70-control-plane.md +++ b/spec/70-control-plane.md @@ -236,6 +236,8 @@ The configuration document, the records it presents read-only, the structured sc *because a value in the document reaches every daemon and replica that reads a snapshot* - `connector-upload` — An artifact uploaded through the operator surface raises `ConnectorUploadRefused`; the surface references registered connectors by id and version. *because publishing into the registry runs a separate signed path* +- `store-draft` — An Admin edit validates a complete control document and saves one store-scoped draft bound to the applied version it read, leaving the applied pointer unchanged. + *A-surface* ## apply @@ -254,6 +256,10 @@ Validation, the immutable version claim, the pointer advance, the owner's storag - `local-claim` — A local control plane validates and claims `manifest@v.toml` in its snapshot directory, `.contextful/control//` unless `[control] snapshot_dir` names one, on its own; `contextful pipeline apply` is that apply, and no hosted plane sits on its path. *because {{topology.coordinate.air-gap}} holds a single-node deployment to reach no process outside itself* - `guarded-import` — `contextful pipeline import` claims v1 from the declared pipelines while the snapshot directory holds no version; a second import claims nothing. +- `draft-claim` — An Admin apply rechecks the configured store owner, validates its saved draft again, and claims that draft through the owner's version compare-and-swap. + *A-surface* +- `draft-absent` — An Admin apply finding no validated store draft raises `ControlDraftAbsent` and changes no applied version. + *because an absent draft supplies no document for the version claim* One apply through the engine's store-scoped API: diff --git a/spec/adr/A-surface.md b/spec/adr/A-surface.md index 9d3fb45ea..6a3a5fb76 100644 --- a/spec/adr/A-surface.md +++ b/spec/adr/A-surface.md @@ -4,7 +4,7 @@ ## The control document is CAS-versioned, validated per entry, and fails static -One hand-edited control document arms every scheduled entry unattended. `surface.apply` claims a version by compare-and-swap on an engine-assigned version; a loser raises `ManifestVersionConflict`, reloads and reapplies. The engine owns the control-state model and raises `ConfigOwnerUnconfigured`, `StoreNotInitialized` or `ConditionalWriteUnsupported` rather than substitute a local writer. `surface.arm` holds back an invalid entry alone, by name. `surface.reconcile` keeps the armed set unchanged on a failed poll. `surface.dispatch` starts one instance per due unit; a dependent run's step refuses as a unit. +One hand-edited control document arms every scheduled entry unattended. Admin edit validates and saves one store draft at the version it read; Admin apply rechecks ownership and validates before claiming that draft by compare-and-swap. A loser raises `ManifestVersionConflict`, reloads and reapplies. The engine owns the control-state model and raises `ConfigOwnerUnconfigured`, `StoreNotInitialized` or `ConditionalWriteUnsupported` rather than substitute a local writer. `surface.arm` holds back an invalid entry alone, by name. `surface.reconcile` keeps the armed set unchanged on a failed poll. `surface.dispatch` starts one instance per due unit; a dependent run's step refuses as a unit. | Option | Lost on | Cost | | --- | --- | --- | diff --git a/spec/cards/surface.md b/spec/cards/surface.md index b367d1ea7..508206282 100644 --- a/spec/cards/surface.md +++ b/spec/cards/surface.md @@ -10,8 +10,8 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `reconcile` | The control source, the snapshot pointer and its versions, the pure schedule diff, and what a failed poll leaves running. | 6 | | `fire` | Job declaration, the closed kind union, same-tick order, the fire watermark, and one-shot evaluation. | 7 | | `dispatch` | The bounded fire pool and its exclusion keys, the reconciler's hold on the cadence lease, step results, and the worker adapter with its attempt-fenced callback. | 15 | -| `edit` | The configuration document, the records it presents read-only, the structured schedule specification, and connector configuration. | 2 | -| `apply` | Validation, the immutable version claim, the pointer advance, the owner's storage, and the authority an apply carries. | 7 | +| `edit` | The configuration document, the records it presents read-only, the structured schedule specification, and connector configuration. | 3 | +| `apply` | Validation, the immutable version claim, the pointer advance, the owner's storage, and the authority an apply carries. | 9 | | `reside` | Where the data plane runs, the region allow-set gating placement, and operation with no external reach. | 3 | | `open-console` | The hosted console's two pages, their API routes, identity gates, page grants and engine capabilities. | 8 | | `register-store` | Which stores a deployment serves, how each store's credential and binding names derive, and how a request reaches a store's origin. | 5 | @@ -54,6 +54,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `ConfigOwnerUnconfigured` | A configuration owner with no storage or no credential behind it. | `surface.apply.owner-unconfigured` | | `ConditionalWriteUnsupported` | A configuration owner or catalog backend lacking linearizable conditional replacement. | `surface.apply.weak-conditional-backend` | | `StoreNotInitialized` | An edit or apply against a store that has taken no guarded import. | `surface.apply.uninitialized-store` | +| `ControlDraftAbsent` | An Admin apply with no validated store draft. | `surface.apply.draft-absent` | | `EnforceRegionMismatch` | A configured resource resolving outside the residency allow-set. | `surface.reside.region-mismatch` | | `ResidencySitesDiverge` | A push whose residency allow-set differs from the one another site recorded in the bucket manifest. | `surface.reside.site-regions` | | `ConsoleRouteUngated` | An anonymous hosted route serving content instead of its declared identity gate. | `surface.open-console.ungated-route` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index e8bebd87a..c54eb4524 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -19053,6 +19053,17 @@ "file": "spec/70-control-plane.md", "line": 237 }, + { + "id": "surface.edit.store-draft", + "contract": "surface", + "operation": "edit", + "subject": "store-draft", + "kind": "behavior", + "statement": "An Admin edit validates a complete control document and saves one store-scoped draft bound to the applied version it read, leaving the applied pointer unchanged.", + "why": "A-surface", + "file": "spec/70-control-plane.md", + "line": 239 + }, { "id": "surface.apply.version-race", "contract": "surface", @@ -19062,7 +19073,7 @@ "statement": "An apply whose compare-and-swap loses raises `ManifestVersionConflict`, reloads the winning version and reapplies its pending edits onto it, overwriting no applied version.", "why": "A-surface", "file": "spec/70-control-plane.md", - "line": 244 + "line": 246 }, { "id": "surface.apply.validation", @@ -19073,7 +19084,7 @@ "statement": "A document failing engine validation raises `ApplyValidationRefused` and claims no version.", "why": "because a snapshot store holding a version daemons refuse to arm stalls every daemon reading it", "file": "spec/70-control-plane.md", - "line": 246 + "line": 248 }, { "id": "surface.apply.owner-unconfigured", @@ -19084,7 +19095,7 @@ "statement": "An owner with no storage configured or no credential raises `ConfigOwnerUnconfigured`, answered `503`; no local writer substitutes for the store-scoped API.", "why": "P3", "file": "spec/70-control-plane.md", - "line": 248 + "line": 250 }, { "id": "surface.apply.weak-conditional-backend", @@ -19095,7 +19106,7 @@ "statement": "A configuration owner or catalog backend whose conditional replacement is not linearizable raises `ConditionalWriteUnsupported` at startup or open.", "why": "A-surface", "file": "spec/70-control-plane.md", - "line": 250 + "line": 252 }, { "id": "surface.apply.uninitialized-store", @@ -19106,7 +19117,7 @@ "statement": "An edit or apply against a store that has taken no explicit guarded import, an empty store included, raises `StoreNotInitialized`, answered `409`.", "why": "P3", "file": "spec/70-control-plane.md", - "line": 252 + "line": 254 }, { "id": "surface.apply.local-claim", @@ -19117,7 +19128,7 @@ "statement": "A local control plane validates and claims `manifest@v.toml` in its snapshot directory, `.contextful/control//` unless `[control] snapshot_dir` names one, on its own; `contextful pipeline apply` is that apply, and no hosted plane sits on its path.", "why": "because {{topology.coordinate.air-gap}} holds a single-node deployment to reach no process outside itself", "file": "spec/70-control-plane.md", - "line": 254 + "line": 256 }, { "id": "surface.apply.guarded-import", @@ -19128,7 +19139,29 @@ "statement": "`contextful pipeline import` claims v1 from the declared pipelines while the snapshot directory holds no version; a second import claims nothing.", "why": "", "file": "spec/70-control-plane.md", - "line": 256 + "line": 258 + }, + { + "id": "surface.apply.draft-claim", + "contract": "surface", + "operation": "apply", + "subject": "draft-claim", + "kind": "behavior", + "statement": "An Admin apply rechecks the configured store owner, validates its saved draft again, and claims that draft through the owner's version compare-and-swap.", + "why": "A-surface", + "file": "spec/70-control-plane.md", + "line": 259 + }, + { + "id": "surface.apply.draft-absent", + "contract": "surface", + "operation": "apply", + "subject": "draft-absent", + "kind": "refusal", + "statement": "An Admin apply finding no validated store draft raises `ControlDraftAbsent` and changes no applied version.", + "why": "because an absent draft supplies no document for the version claim", + "file": "spec/70-control-plane.md", + "line": 261 }, { "id": "surface.reside.region-entries", @@ -19139,7 +19172,7 @@ "statement": "A residency allow-set holds at most 16 entries.", "why": "", "file": "spec/70-control-plane.md", - "line": 292 + "line": 298 }, { "id": "surface.reside.region-mismatch", @@ -19150,7 +19183,7 @@ "statement": "A configured resource resolving to a region the policy omits raises `EnforceRegionMismatch` at startup, and the runtime serves nothing.", "why": "P3", "file": "spec/70-control-plane.md", - "line": 293 + "line": 299 }, { "id": "surface.reside.site-regions", @@ -19161,7 +19194,7 @@ "statement": "A push records the pushing site's residency allow-set in the bucket manifest; a push finding a differing set another site recorded, an absent one included, raises `ResidencySitesDiverge` and commits nothing.", "why": "A-surface", "file": "spec/70-control-plane.md", - "line": 295 + "line": 301 }, { "id": "surface.open-console.page-routes", @@ -28514,6 +28547,10 @@ "clause": "surface.ground.org-face-read-only", "gloss": "A write tool registered on an organization-wide face." }, + "ControlDraftAbsent": { + "clause": "surface.apply.draft-absent", + "gloss": "An Admin apply with no validated store draft." + }, "ControlPointerMalformed": { "clause": "surface.reconcile.pointer-malformed", "gloss": "A snapshot pointer body that is not wholly a version." diff --git a/spec/status.md b/spec/status.md index aaff60ee5..5283266b9 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 57 | 0 | 57 | +| `surface` | 2 | 20 | 109 | 50 | 22 | 0 | 57 | 0 | 57 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1371 | 0 | | +| **total** | 19 | 155 | 2016 | 582 | 188 | 127 | 1371 | 0 | | Decision records: 18. @@ -35,7 +35,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 7 — Memory | 6 | 37 | 37 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | -| 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | +| 10 — Cadence and the operator plane | 11 | 71 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | | 12 — The operator console | 12 | 49 | 6 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | diff --git a/spec/terms/surface.toml b/spec/terms/surface.toml index 37b185793..59012b6ba 100644 --- a/spec/terms/surface.toml +++ b/spec/terms/surface.toml @@ -44,6 +44,7 @@ DispatchCallbackRejected = { clause = "surface.dispatch.callback-rejected", DispatchSubmitRejected = { clause = "surface.dispatch.submit-signed", gloss = "A worker submission unsigned, skewed, or naming a callback off the worker's relay." } SecretMaterialInDocument = { clause = "surface.edit.secret-in-document", gloss = "A credential value written into the configuration document." } ConnectorUploadRefused = { clause = "surface.edit.connector-upload", gloss = "A connector artifact offered through the operator surface." } +ControlDraftAbsent = { clause = "surface.apply.draft-absent", gloss = "An Admin apply with no validated store draft." } ManifestVersionConflict = { clause = "surface.apply.version-race", gloss = "An apply whose pointer compare-and-swap lost to a concurrent apply." } ApplyValidationRefused = { clause = "surface.apply.validation", gloss = "A document engine validation refused; no version was claimed." } ConfigOwnerUnconfigured = { clause = "surface.apply.owner-unconfigured", gloss = "A configuration owner with no storage or no credential behind it." } From 60063ad11b15ec6e683d2349288b40245f677a9c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:34:18 +0800 Subject: [PATCH 083/155] chore(spec): raise verified console and connector pin floors --- spec/pins.toml | 4 ++-- spec/status.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/pins.toml b/spec/pins.toml index 8366751f4..f2a97931c 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -57,11 +57,11 @@ [floor] assurance = 136 authority = 204 -connector = 207 +connector = 218 corpus = 40 disclosure = 54 read = 163 run = 261 store = 200 -surface = 59 +surface = 65 topology = 38 diff --git a/spec/status.md b/spec/status.md index 9f0feaa72..433edd1ef 100644 --- a/spec/status.md +++ b/spec/status.md @@ -7,13 +7,13 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | `assurance` | 2 | 15 | 231 | 59 | 22 | 12 | 136 | 0 | 136 | | `authority` | 2 | 16 | 267 | 77 | 27 | 10 | 204 | 0 | 204 | -| `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 207 | +| `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 218 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 40 | 0 | 40 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 65 | 0 | 59 | +| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 65 | 0 | 65 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | | **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1379 | 0 | | From af59dcf257a5182a99d015578bbf6cbe9c74dcd1 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:34:39 +0800 Subject: [PATCH 084/155] test: preserve store UUID on scoped pull --- crates/contextful-sync/tests/integration/pull.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/contextful-sync/tests/integration/pull.rs b/crates/contextful-sync/tests/integration/pull.rs index 58882c8de..de04f3951 100644 --- a/crates/contextful-sync/tests/integration/pull.rs +++ b/crates/contextful-sync/tests/integration/pull.rs @@ -43,11 +43,15 @@ fn a_clone_keeps_the_store_uuid_and_an_independent_store_refuses_it() { clone.syncer.pull(&PullScope::default()).unwrap(); assert_eq!(std::fs::read_to_string(clone.root().join("store-id")).unwrap(), identity); + let scoped = node("ingest-d", b.clone(), ""); + scoped.syncer.pull(&PullScope { tables: ["filings".to_string()].into(), ..PullScope::default() }).unwrap(); + assert_eq!(std::fs::read_to_string(scoped.root().join("store-id")).unwrap(), identity); + let independent = node("ingest-c", b, ""); let other = "22222222-2222-4222-8222-222222222222\n"; std::fs::write(independent.root().join("store-id"), other).unwrap(); let refused = independent.syncer.pull(&PullScope::default()).unwrap_err(); - assert!(refused.to_string().contains("StoreIdentityConflict"), "{refused}"); + assert!(matches!(refused, SyncError::Store(StoreError::StoreIdentityConflict(_))), "{refused}"); assert_eq!(std::fs::read_to_string(independent.root().join("store-id")).unwrap(), other); } From 565133a15ca9bd1bc005e516c6c3c740d6f1a5cb Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:45:37 +0800 Subject: [PATCH 085/155] Bind signed owner claim to synced store UUID --- crates/contextful-cli/src/mcp.rs | 4 +- crates/contextful-cli/src/project.rs | 11 +- crates/contextful-cli/src/token.rs | 2 +- .../contextful-cli/tests/integration/mcp.rs | 23 + crates/contextful-context/src/project.rs | 40 +- .../tests/integration/init.rs | 23 +- crates/contextful-core/src/store/error.rs | 6 + crates/contextful-core/src/store/lay_out.rs | 1 + crates/contextful-sync/src/sync.rs | 18 + spec/10-store.md | 8 +- spec/adr/A-store.md | 12 + spec/cards/store.md | 6 +- spec/guide/store.md | 4 +- spec/pins.toml | 5 +- spec/spec.lock.json | 461 ++++++++++-------- spec/status.md | 11 +- spec/terms/store.toml | 2 + 17 files changed, 412 insertions(+), 225 deletions(-) diff --git a/crates/contextful-cli/src/mcp.rs b/crates/contextful-cli/src/mcp.rs index 7fb7f5bff..ab4e48d81 100644 --- a/crates/contextful-cli/src/mcp.rs +++ b/crates/contextful-cli/src/mcp.rs @@ -37,7 +37,9 @@ pub fn run(args: McpArgs) -> Result<()> { let (authority, revocation) = args.admit.admit(args.project.as_deref(), "the tool server")?; let located = locate(args.project.as_deref(), args.declaration)?; let authority = if args.owner { - authority.activate_owner(&crate::project::owner_identity(&located.project)?) + let identity = crate::project::owner_identity(&located.project) + .map_err(|_| AdmitError::OwnerCredentialInvalid("the selected local store has no readable UUID".into()))?; + authority.activate_owner(&identity) .ok_or_else(|| AdmitError::OwnerCredentialInvalid("the verified credential does not own the selected local store".into()))? } else { authority diff --git a/crates/contextful-cli/src/project.rs b/crates/contextful-cli/src/project.rs index a24519035..4bb010c3f 100644 --- a/crates/contextful-cli/src/project.rs +++ b/crates/contextful-cli/src/project.rs @@ -8,7 +8,6 @@ use contextful_context::Store; use contextful_core::pipeline::declare::ManifestFile; use contextful_policy::enforce::mask::Pepper; use std::path::Path; -use sha2::{Digest, Sha256}; #[cfg(feature = "data-plane")] use anyhow::anyhow; #[cfg(feature = "data-plane")] @@ -41,10 +40,14 @@ pub struct Located { pub declaration: PathBuf, } -/// The canonical local store root binds an owner credential to one store instance. +/// The synced UUID binds an owner credential to one store instance and its clones. pub(crate) fn owner_identity(project: &Project) -> Result { - let root = std::fs::canonicalize(project.store_root())?; - Ok(format!("{:x}", Sha256::digest(root.to_string_lossy().as_bytes()))) + Ok(contextful_context::project::store_id(&project.store_root())?) +} + +/// Legacy stores acquire the synced UUID before minting their first owner claim. +pub(crate) fn mint_owner_identity(project: &Project) -> Result { + Ok(contextful_context::project::ensure_store_id(&project.store_root())?) } /// Resolve the project from `--project` or by discovery, and the declaration from diff --git a/crates/contextful-cli/src/token.rs b/crates/contextful-cli/src/token.rs index b0e9bcd79..b1c8d2aeb 100644 --- a/crates/contextful-cli/src/token.rs +++ b/crates/contextful-cli/src/token.rs @@ -594,7 +594,7 @@ fn mint_one( #[cfg(feature = "read-plane")] { let project = crate::project::locate(None, None)?.project; - let identity = crate::project::owner_identity(&project)?; + let identity = crate::project::mint_owner_identity(&project)?; if !contextful_policy::verify::owner_grant(&plan.grants) { bail!("an owner credential requires an unrestricted read grant over `*`"); } diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index b8863ff3b..e9d0e28cd 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -203,6 +203,7 @@ fn owner_credential_does_not_follow_a_recreated_store_at_the_same_path() { let identity_path = root.join(".contextful/context/research/store-id"); let first_identity = std::fs::read_to_string(&identity_path).unwrap_or_default(); std::fs::remove_dir_all(root.join(".contextful/context/research")).unwrap(); + stdout(&run(root, &["init", "research"])); stdout(&run(root, &["context", "land", "research/notes", "--project", "research", "--rows", "notes.jsonl", "--run-id", "run-0002", "--site-id", "site-a"])); let second_identity = std::fs::read_to_string(&identity_path).unwrap_or_default(); assert!(!first_identity.is_empty() && !second_identity.is_empty() && first_identity != second_identity, @@ -214,6 +215,28 @@ fn owner_credential_does_not_follow_a_recreated_store_at_the_same_path() { assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); } +#[test] +fn owner_credential_does_not_cross_independent_stores_with_one_issuer() { + let (first, public, _) = project(); + let owner = stdout(&run(first.path(), &[ + "token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", + "--zone", "on-prem:hq", "--action", "read", "--table", "*", "--ttl", "600", "--owner", + ])); + let second = tempfile::tempdir().unwrap(); + let root = second.path(); + std::fs::create_dir_all(root.join(".contextful")).unwrap(); + for path in ["contextful.toml", "notes.jsonl", ".contextful/issuance.toml", ".contextful/issuer.seed"] { + std::fs::copy(first.path().join(path), root.join(path)).unwrap(); + } + stdout(&run(root, &["init", "research"])); + stdout(&run(root, &["context", "land", "research/notes", "--project", "research", "--rows", "notes.jsonl", "--run-id", "run-0001", "--site-id", "site-a"])); + let args = ["mcp", "--owner", "--project", "research", "--public-key", &public, "--audience", AUD]; + let read = [json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } })]; + let refused = serve(root, &args, Some(&owner), &read); + assert!(!refused.status.success() && refused.stdout.is_empty()); + assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); +} + /// The server admits over the stdio pipe it inherited: a credential binding no key admits /// through that pipe, and one binding a holder key and presenting no holder proof refuses. #[test] diff --git a/crates/contextful-context/src/project.rs b/crates/contextful-context/src/project.rs index c94e8f1e8..487707a3e 100644 --- a/crates/contextful-context/src/project.rs +++ b/crates/contextful-context/src/project.rs @@ -3,7 +3,7 @@ use crate::error::{IoPath, Result}; use contextful_core::issue::AuthoringPosture; -use contextful_core::store::lay_out::{is_path_segment, store_root}; +use contextful_core::store::lay_out::{is_path_segment, store_root, STORE_ID_FILE}; use contextful_core::store::StoreError; use std::fs; use std::path::{Path, PathBuf}; @@ -46,6 +46,43 @@ impl Project { } } +/// Read one persistent UUID from the synced store root. +pub fn store_id(root: &Path) -> Result { + let path = root.join(STORE_ID_FILE); + let raw = fs::read_to_string(&path).at(&path)?; + let id = raw.trim(); + let valid = id.len() == 36 && id.bytes().enumerate().all(|(i, b)| match i { + 8 | 13 | 18 | 23 => b == b'-', + 14 => b == b'4', + 19 => matches!(b, b'8' | b'9' | b'a' | b'b'), + _ => b.is_ascii_digit() || (b'a'..=b'f').contains(&b), + }); + if !valid { + return Err(StoreError::StoreIdentityInvalid(format!("`{}` holds no canonical version-4 UUID", path.display())).into()); + } + Ok(id.to_string()) +} + +/// Keep an existing UUID, or publish a new one by exclusive creation. +pub fn ensure_store_id(root: &Path) -> Result { + match store_id(root) { + Ok(id) => return Ok(id), + Err(crate::ContextError::Io { source, .. }) if source.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + if !root.is_dir() { + return Err(StoreError::StoreIdentityInvalid(format!("`{}` is not an initialized store root", root.display())).into()); + } + let mut bytes = [0u8; 16]; + getrandom::fill(&mut bytes).map_err(|e| crate::ContextError::Invalid(format!("store UUID entropy unavailable: {e}")))?; + bytes[6] = (bytes[6] & 0x0f) | 0x40; + bytes[8] = (bytes[8] & 0x3f) | 0x80; + let hex = |slice: &[u8]| slice.iter().map(|b| format!("{b:02x}")).collect::(); + let id = format!("{}-{}-{}-{}-{}", hex(&bytes[..4]), hex(&bytes[4..6]), hex(&bytes[6..8]), hex(&bytes[8..10]), hex(&bytes[10..])); + crate::store::create_new_file(&root.join(STORE_ID_FILE), format!("{id}\n").as_bytes())?; + store_id(root) +} + /// What an init did to the directory. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Initialized { @@ -131,6 +168,7 @@ pub fn init(dir: &Path, name: &str) -> Result { }; let root = dir.join(store_root(name)); fs::create_dir_all(&root).at(&root)?; + ensure_store_id(&root)?; Ok(outcome) } diff --git a/crates/contextful-context/tests/integration/init.rs b/crates/contextful-context/tests/integration/init.rs index 666060bc0..9b11b2684 100644 --- a/crates/contextful-context/tests/integration/init.rs +++ b/crates/contextful-context/tests/integration/init.rs @@ -43,6 +43,22 @@ fn init_persists_a_distinct_store_uuid_across_repetition_and_recreation() { assert!(valid(&second) && second != first, "recreating the store changes its UUID"); } +#[test] +fn malformed_store_uuid_and_absent_owner_root_refuse_identity() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join(".contextful/context/research"); + match store_err(contextful_context::project::ensure_store_id(&root).unwrap_err()) { + StoreError::StoreIdentityInvalid(message) => assert!(message.contains("research")), + other => panic!("{other}"), + } + init(dir.path(), "research").unwrap(); + fs::write(root.join("store-id"), "not-a-uuid\n").unwrap(); + match store_err(contextful_context::project::ensure_store_id(&root).unwrap_err()) { + StoreError::StoreIdentityInvalid(message) => assert!(message.contains("store-id")), + other => panic!("{other}"), + } +} + /// A project name that is not path-safe segments raises `StoreProjectNameInvalid` before any file is read or written. // spec: store.init.name-shape@93c80b33 #[test] @@ -60,17 +76,20 @@ fn a_traversing_or_unsafe_name_refuses_before_any_write() { assert!(dir.path().join(".contextful/context/team/research").is_dir()); } -/// An init against a `contextful.toml` already declaring the same name rewrites nothing and succeeds. -// spec: store.init.repeat@db5b16da +/// An init against the same declaration preserves its existing store content and UUID. +// spec: store.init.repeat@6387e88e #[test] fn a_repeated_init_rewrites_nothing() { let dir = tempfile::tempdir().unwrap(); init(dir.path(), "research").unwrap(); + let identity_path = dir.path().join(".contextful/context/research/store-id"); + let identity = fs::read_to_string(&identity_path).unwrap(); let path = dir.path().join(DECLARATION_FILE); let edited = format!("{}\n[[pipeline.tables]]\nname = \"filings\"\n", fs::read_to_string(&path).unwrap()); fs::write(&path, &edited).unwrap(); assert_eq!(init(dir.path(), "research").unwrap(), Initialized::Unchanged); assert_eq!(fs::read_to_string(&path).unwrap(), edited); + assert_eq!(fs::read_to_string(&identity_path).unwrap(), identity); } /// An init against a `contextful.toml` declaring no `project` key appends the `[project]` table and keeps every existing byte. diff --git a/crates/contextful-core/src/store/error.rs b/crates/contextful-core/src/store/error.rs index 2858716c8..c370524f0 100644 --- a/crates/contextful-core/src/store/error.rs +++ b/crates/contextful-core/src/store/error.rs @@ -5,6 +5,12 @@ #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum StoreError { + /// A persisted store UUID is malformed or cannot identify a store. (`store.init.identity-invalid`) + #[error("StoreIdentityInvalid: {0}")] + StoreIdentityInvalid(String), + /// A pull would replace one store UUID with another. (`store.pull.identity-conflict`) + #[error("StoreIdentityConflict: {0}")] + StoreIdentityConflict(String), /// An `as_of` precedes the oldest retained snapshot of a table whose history was collected. (`store.bound-time.as-of-unretained`) #[error("StoreAsOfUnretained: {0}")] StoreAsOfUnretained(String), diff --git a/crates/contextful-core/src/store/lay_out.rs b/crates/contextful-core/src/store/lay_out.rs index 969176bf5..6c898c380 100644 --- a/crates/contextful-core/src/store/lay_out.rs +++ b/crates/contextful-core/src/store/lay_out.rs @@ -25,6 +25,7 @@ pub const SCHEMA_FILE: &str = "schema.json"; pub const STAGING_SUFFIX: &str = ".staging"; pub const CONFIG_FILE: &str = "config.toml"; +pub const STORE_ID_FILE: &str = "store-id"; pub const TABLES_DIR: &str = "tables"; pub const CURSORS_DIR: &str = "cursors"; pub const RUNS_DIR: &str = "data/runs"; diff --git a/crates/contextful-sync/src/sync.rs b/crates/contextful-sync/src/sync.rs index d84c92b78..f9676b7d6 100644 --- a/crates/contextful-sync/src/sync.rs +++ b/crates/contextful-sync/src/sync.rs @@ -787,7 +787,11 @@ impl Syncer { .into()); } } + let identity_key = format!("{}/{}", self.project, contextful_core::store::lay_out::STORE_ID_FILE); let reaches = |key: &str| -> bool { + if key == identity_key { + return true; + } match table_of(key) { None => scope.tables.is_empty(), Some(t) => (scope.tables.is_empty() || scope.tables.contains(&t)) && !(replica && scope.replicate_off.contains(&t)), @@ -821,6 +825,20 @@ impl Syncer { Some((listed, _)) => listed.clone(), None => self.manifest()?.0, }; + if let Some(remote) = manifest.entries.get(&identity_key) { + let local_path = self.store.root().join(contextful_core::store::lay_out::STORE_ID_FILE); + match std::fs::read(&local_path) { + Ok(local) if sha256_hex(&local) != remote.sha256 => { + return Err(StoreError::StoreIdentityConflict(format!( + "`{}` holds a different store UUID from the bucket; clone into an empty store root", + local_path.display() + )).into()); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(io(&local_path, error)), + Ok(_) => {} + } + } for (key, entry) in manifest.entries.iter().filter(|(k, _)| in_project(k) && reaches(k)) { // This node's own keys are authoritative here, except in a restore. if generation.is_none() && owner_of(key).as_deref() == Some(self.node.as_str()) { diff --git a/spec/10-store.md b/spec/10-store.md index 70e7f002e..3d15ebbe8 100644 --- a/spec/10-store.md +++ b/spec/10-store.md @@ -139,9 +139,13 @@ unsettled: How does a consumer discover the format version of a run or snapshot A project's declaration file: what `contextful init` writes, what a repeated init does, and how a command given no `--project` finds its project from the working directory. - `declaration-file` — `contextful init ` writes `contextful.toml` in the working directory declaring `[project]` with `name = ""`, and creates the store root {{store.lay-out.store-root}} beside it. +- `store-identity` — An initialized store holds one synced, canonical version-4 UUID in `store-id`; repeated init and clones keep it, while a recreated root receives a new UUID. + *A-store* +- `identity-invalid` — A malformed `store-id`, or an owner mint naming no initialized store root, raises `StoreIdentityInvalid`. + *because an ambiguous or absent UUID cannot bind an owner credential* - `name-shape` — A project name that is not `/`-separated segments of `[A-Za-z0-9._-]`, or that holds a `.` or `..` segment, raises `StoreProjectNameInvalid` before any file is read or written. *because the name is interpolated into every project path, and a traversing segment places a store outside its project* -- `repeat` — An init against a `contextful.toml` already declaring the same `[project] name` rewrites nothing but a posture {{store.init.posture}}, leaves the store root as it stands, and succeeds. +- `repeat` — An init against a `contextful.toml` already declaring the same `[project] name` rewrites nothing but a posture {{store.init.posture}} and a missing {{store.init.store-identity}}, preserves existing store content, and succeeds. *because a repeated init converges on the state the first one wrote, so scripts and onboarding run it unconditionally* - `adopt` — An init against a `contextful.toml` declaring no `project` key appends the `[project]` table and keeps every existing byte of the file. - `posture` — `contextful init --authoring-posture ` prepends a top-level `authoring_posture = ""` {{authority.issue.posture-key}} to a `contextful.toml` declaring none, keeping every other byte; a declared posture stands, and an init without the flag writes none. @@ -539,6 +543,8 @@ Fetching a bucket into a store: the digest diff, the parallel download, and the - `digest-mismatch` — A downloaded object whose digest differs from its entry raises `SyncObjectDigestMismatch` and is discarded. *P4* +- `identity-conflict` — A pull into a store holding a different {{store.init.store-identity}} raises `StoreIdentityConflict` before downloading files. + *because importing another UUID lets that store's owner credential read local rows* - `convergence` — When a named key disappears mid-download, the pull re-fetches the manifest and retries the shortfall, up to 3 attempts. - `unconverged` — Exhausting those retries raises `SyncPullDidNotConverge`, naming the key that kept moving, and writes no pointer. *P4* diff --git a/spec/adr/A-store.md b/spec/adr/A-store.md index 9a7d8eef9..815ac5d2c 100644 --- a/spec/adr/A-store.md +++ b/spec/adr/A-store.md @@ -2,6 +2,18 @@ **Status:** accepted +## A synced UUID binds a store across clones + +`store.init` publishes a random version-4 UUID in `store-id` inside the store root. Owner mint on a root lacking an ID creates it once. Bucket sync carries it as an immutable object: an empty clone accepts the UUID, while a pull into an independently identified store refuses before copying data. Removing and reinitializing a root creates another UUID, so a signed owner claim from the removed root does not follow its filesystem path. + +| Option | Lost on | Cost | +| --- | --- | --- | +| Synced immutable UUID *(chosen)* | — | Roots lacking an ID receive one before owner mint; independently initialized stores cannot merge by pull. | +| Digest of the absolute root path | Recreation | A token follows a new store placed at the same path and differs across clones. | +| Issuer key fingerprint | Shared issuers | Two stores signed by one project issuer have the same fingerprint. | + +Consequences: a clone holds the original store identity; a new store at an old path does not. + ## An empty complete snapshot carries a replacement frontier A zero-row result has two meanings for a replacing table. A source can finish enumerating its inventory and find no rows, or it can skip unchanged input without reading it. The run records a replacement frontier only for the complete result. The marker lives in the run manifest, survives bucket sync as committed data, and remains effective when a fold publishes an empty snapshot. An incomplete or failed pull commits neither rows nor frontier. A bounded read before the marker still reaches earlier runs while retention keeps them. diff --git a/spec/cards/store.md b/spec/cards/store.md index e5b831095..18a9786da 100644 --- a/spec/cards/store.md +++ b/spec/cards/store.md @@ -7,7 +7,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | Operation | Covers | Clauses | | --- | --- | --- | | `lay-out` | The directory tree, run and snapshot manifests, the table pointer, the two catalogs, and node identity. | 31 | -| `init` | A project's declaration file: what `contextful init` writes, what a repeated init does, and how a command given no `--project` finds its project from the working directory. | 12 | +| `init` | A project's declaration file: what `contextful init` writes, what a repeated init does, and how a command given no `--project` finds its project from the working directory. | 14 | | `declare` | A table's declaration block: its key, ordering column and write mode, and what a read returns for a withdrawn row. | 21 | | `reserve` | The column and table namespaces the engine holds, the provenance columns it injects, and the request ledger. | 20 | | `reconcile` | Schema evolution across a table's file set: the type lattice, additive columns, and what a scan may invent. | 18 | @@ -18,7 +18,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `endpoint` | Opening a store's bucket: the endpoint schemes, the transport, request addressing and signing, credentials, and what a backend's answers mean. | 9 | | `emit` | Planning a push from the store alone: the manifest a push commits, computed from local files without reading or writing the bucket. | 2 | | `push` | Uploading the store to a bucket: the wire format, the bucket manifest, prefix confinement and the manifest write that commits it. | 21 | -| `pull` | Fetching a bucket into a store: the digest diff, the parallel download, and the pointer written last. | 14 | +| `pull` | Fetching a bucket into a store: the digest diff, the parallel download, and the pointer written last. | 15 | | `probe` | Measuring a backend's conditional-write behavior with a live sentinel, and the coordination mode that outcome resolves. | 4 | | `merge` | Reconciling one bucket manifest between writers: per-entry ownership, tombstones, the rebase loop and the consumer-facing section. | 7 | | `lease` | Single-writer exclusion over a pipeline or a table's compaction: two implementations, the fence, and the commit log carrying the cursor. | 12 | @@ -33,6 +33,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `StoreUnknownTable` | A table name resolves to no `schema.json` in the tree. | `store.lay-out.unknown-table` | | `StoreNodeIdInvalid` | A resolved node id is too long or outside the path-safe pattern. | `store.lay-out.node-id-shape` | | `StoreNodeIdShared` | A node id is declared in a control-plane configuration applied to many machines. | `store.lay-out.node-id-shared` | +| `StoreIdentityInvalid` | A store UUID is malformed or owner mint names no initialized root. | `store.init.identity-invalid` | | `StoreProjectNameInvalid` | A project name is not path-safe `/`-separated segments. | `store.init.name-shape` | | `StoreProjectConflict` | An init meets a `contextful.toml` declaring another project, or a project with no name. | `store.init.name-conflict` | | `StoreProjectUndiscovered` | A command given no `--project` finds no `contextful.toml` naming a project from its working directory up. | `store.init.undiscovered` | @@ -66,6 +67,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `SyncManifestFormatUnsupported` | A bucket or generation manifest carries a format newer than the reader parses. | `store.push.format-unsupported` | | `SyncGenerationConflict` | A generation manifest already holds another commit's bytes under the number a push committed. | `store.push.generation-conflict` | | `SyncObjectDigestMismatch` | A downloaded object's digest differs from its entry. | `store.pull.digest-mismatch` | +| `StoreIdentityConflict` | A pull would replace one store UUID with another. | `store.pull.identity-conflict` | | `SyncPullDidNotConverge` | A pull exhausted its retries against keys moving beneath it. | `store.pull.unconverged` | | `SyncGenerationAbsent` | A pull names a generation the bucket holds no generation manifest for. | `store.pull.generation-absent` | | `SyncGenerationDiverged` | A generation pull meets a local file the generation does not list. | `store.pull.generation-diverged` | diff --git a/spec/guide/store.md b/spec/guide/store.md index 4112ec3f0..3e86b5ce0 100644 --- a/spec/guide/store.md +++ b/spec/guide/store.md @@ -10,7 +10,7 @@ The store is the canonical corpus of **Contextful**: Parquet any SQL tool opens, ## How it works -`contextful init` names the project ({{store.init.declaration-file}}) and its authoring posture ({{store.init.posture}}); commands below it find it ({{store.init.discovery}}). +`contextful init` names the project ({{store.init.declaration-file}}), gives its store a synced identity ({{store.init.store-identity}}), and declares its authoring posture ({{store.init.posture}}); commands below it find it ({{store.init.discovery}}). Files never change once written ({{store.lay-out.immutable-files}}). A write adds a run directory and commits by conditionally creating that run's manifest ({{store.lay-out.run-manifest}}); until the manifest exists, the run's parts join no read ({{store.lay-out.uncommitted-run}}). The pipeline's cursor rides inside the same commit, so rows and position land together ({{store.lay-out.cursor-in-commit}}). @@ -22,7 +22,7 @@ Runs accumulate until a fold compacts them. A pass writes Parquet and every side Two SQLite catalogs sit beside the files. `derived.sqlite` is a disposable cache rebuilt from the tree ({{store.lay-out.derived-catalog}}); `machine.sqlite` holds one machine's journal, cursor cache and lease rows, and no pull or rebuild touches it ({{store.lay-out.machine-catalog}}). Both sit behind ports ({{store.lay-out.catalog-ports}}). -A bucket mirrors the tree. A push uploads changed files, commits the bucket manifest by compare-and-set, and re-merges a lost race within a bound ({{store.merge.retries}}). A pull checks every digest ({{store.pull.digest-mismatch}}) and writes no pointer unless it converges ({{store.pull.unconverged}}). Leases keep writers on different machines apart, with a fence the storage itself checks on commit ({{store.lease.stale-fence}}). A replica is read-only ({{store.replicate.write-refused}}) and holds a snapshot whole or not at all ({{store.replicate.partial-parquet}}). +A bucket mirrors the tree. A push uploads changed files, commits the bucket manifest by compare-and-set, and re-merges a lost race within a bound ({{store.merge.retries}}). A pull checks every digest ({{store.pull.digest-mismatch}}), refuses a different store identity ({{store.pull.identity-conflict}}), and writes no pointer unless it converges ({{store.pull.unconverged}}). Leases keep writers on different machines apart, with a fence the storage itself checks on commit ({{store.lease.stale-fence}}). A replica is read-only ({{store.replicate.write-refused}}) and holds a snapshot whole or not at all ({{store.replicate.partial-parquet}}). ## Worked example diff --git a/spec/pins.toml b/spec/pins.toml index 74ec2ff28..3fdfa5daa 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -27,6 +27,9 @@ # `contextful-spec state` reads this file and the tags and writes spec/status.md. [pin] +"store.init.store-identity" = { test = "contextful_context::init::init_persists_a_distinct_store_uuid_across_repetition_and_recreation" } +"store.init.identity-invalid" = { test = "contextful_context::init::malformed_store_uuid_and_absent_owner_root_refuse_identity" } +"store.pull.identity-conflict" = { test = "contextful_sync::pull::a_clone_keeps_the_store_uuid_and_an_independent_store_refuses_it" } "corpus.state.typescript-tag-pin" = { test = "contextful_spec::ts_pins::a_running_surface_test_tag_performs_its_clause" } "corpus.state.typescript-unfinished" = { test = "contextful_spec::ts_pins::a_skipped_or_todo_surface_test_is_broken" } "assurance.gate.native-test-files" = { test = "contextful_ci::stages::the_surfaces_stage_runs_each_native_typescript_test_file_even_without_a_test_script" } @@ -62,6 +65,6 @@ corpus = 40 disclosure = 54 read = 163 run = 261 -store = 200 +store = 203 surface = 52 topology = 38 diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 3309eedb4..08c7e6405 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -1991,6 +1991,28 @@ "file": "spec/10-store.md", "line": 141 }, + { + "id": "store.init.store-identity", + "contract": "store", + "operation": "init", + "subject": "store-identity", + "kind": "behavior", + "statement": "An initialized store holds one synced, canonical version-4 UUID in `store-id`; repeated init and clones keep it, while a recreated root receives a new UUID.", + "why": "A-store", + "file": "spec/10-store.md", + "line": 142 + }, + { + "id": "store.init.identity-invalid", + "contract": "store", + "operation": "init", + "subject": "identity-invalid", + "kind": "refusal", + "statement": "A malformed `store-id`, or an owner mint naming no initialized store root, raises `StoreIdentityInvalid`.", + "why": "because an ambiguous or absent UUID cannot bind an owner credential", + "file": "spec/10-store.md", + "line": 144 + }, { "id": "store.init.name-shape", "contract": "store", @@ -2000,7 +2022,7 @@ "statement": "A project name that is not `/`-separated segments of `[A-Za-z0-9._-]`, or that holds a `.` or `..` segment, raises `StoreProjectNameInvalid` before any file is read or written.", "why": "because the name is interpolated into every project path, and a traversing segment places a store outside its project", "file": "spec/10-store.md", - "line": 142 + "line": 146 }, { "id": "store.init.repeat", @@ -2008,10 +2030,10 @@ "operation": "init", "subject": "repeat", "kind": "behavior", - "statement": "An init against a `contextful.toml` already declaring the same `[project] name` rewrites nothing but a posture {{store.init.posture}}, leaves the store root as it stands, and succeeds.", + "statement": "An init against a `contextful.toml` already declaring the same `[project] name` rewrites nothing but a posture {{store.init.posture}} and a missing {{store.init.store-identity}}, preserves existing store content, and succeeds.", "why": "because a repeated init converges on the state the first one wrote, so scripts and onboarding run it unconditionally", "file": "spec/10-store.md", - "line": 144, + "line": 148, "scenarios": [ "WHEN `contextful init research` runs twice in one directory, THEN the second run succeeds and `contextful.toml` holds the bytes the first wrote." ] @@ -2025,7 +2047,7 @@ "statement": "An init against a `contextful.toml` declaring no `project` key appends the `[project]` table and keeps every existing byte of the file.", "why": "", "file": "spec/10-store.md", - "line": 146 + "line": 150 }, { "id": "store.init.posture", @@ -2036,7 +2058,7 @@ "statement": "`contextful init --authoring-posture ` prepends a top-level `authoring_posture = \"\"` {{authority.issue.posture-key}} to a `contextful.toml` declaring none, keeping every other byte; a declared posture stands, and an init without the flag writes none.", "why": "because the posture decides whether a write without a credential lands, so no default chooses it for the operator", "file": "spec/10-store.md", - "line": 147, + "line": 151, "scenarios": [ "WHEN `contextful init research` runs without `--authoring-posture`, THEN a following `contextful context land` raises `AuthoringPostureUndeclared`, and after `contextful init research --authoring-posture per_request` the same land commits." ] @@ -2050,7 +2072,7 @@ "statement": "An init against a `contextful.toml` whose `project` key declares another name, or no string `name`, raises `StoreProjectConflict`, naming the declared name when one exists and the init's name, and writes nothing.", "why": "because renaming a project orphans its store root, which an overwrite hides", "file": "spec/10-store.md", - "line": 149, + "line": 153, "scenarios": [ "WHEN `contextful init archive` runs beside a `contextful.toml` naming `research`, THEN it raises `StoreProjectConflict` and the file is unchanged." ] @@ -2064,7 +2086,7 @@ "statement": "A command given no `--project` reads the nearest `contextful.toml` in the working directory or an ancestor, takes its `[project] name` as the project, and bases every project path on that file's directory.", "why": "", "file": "spec/10-store.md", - "line": 151, + "line": 155, "scenarios": [ "WHEN `contextful context land` runs in `notes/` below a directory where `contextful init research --authoring-posture per_request` ran, THEN the run commits under that directory's `.contextful/context/research/`." ] @@ -2078,7 +2100,7 @@ "statement": "A command given `--project` bases every project path on the working directory and runs no discovery.", "why": "", "file": "spec/10-store.md", - "line": 152 + "line": 156 }, { "id": "store.init.project-paths", @@ -2089,7 +2111,7 @@ "statement": "The project paths are the store root, the run state `.contextful/run//` and the memory pass state `.contextful/memory//`.", "why": "", "file": "spec/10-store.md", - "line": 153 + "line": 157 }, { "id": "store.init.default-declaration", @@ -2100,7 +2122,7 @@ "statement": "A command given no `--declaration` reads the discovered `contextful.toml`, or `contextful.toml` in the working directory under `--project`.", "why": "", "file": "spec/10-store.md", - "line": 154 + "line": 158 }, { "id": "store.init.declaration-base", @@ -2111,7 +2133,7 @@ "statement": "Under a located project, a relative path its declaration names — a derive binding's `media_root`, a path-form `command[0]` — resolves against the directory its project paths are based on, never the working directory.", "why": "because one pipeline otherwise reads different media or programs depending on the subdirectory it fires from", "file": "spec/10-store.md", - "line": 155, + "line": 159, "scenarios": [ "WHEN `contextful pipeline run` fires in `notes/` below a `contextful.toml` binding `media_root = \"media\"`, THEN media resolves under `media/` beside that file, not `notes/media/`." ] @@ -2125,7 +2147,7 @@ "statement": "A command given no `--project` whose working directory and ancestors hold no `contextful.toml`, or whose nearest one declares no `[project] name`, raises `StoreProjectUndiscovered`, naming the starting directory.", "why": "because a store opened at a guessed root splits one project's rows across two trees", "file": "spec/10-store.md", - "line": 157 + "line": 161 }, { "id": "store.declare.table-block", @@ -2136,7 +2158,7 @@ "statement": "A table block declares any of `primary_key`, `order_by`, `write_mode`, `replicate`, `subject_id`, `class`, `policy`, `visibility`, `valid_time`, `cluster_by`, `partition_by`, `retain_runs`, `columns`, `indexes`, `agent_description`, `agent_hint`, `example_queries`, `content_hash_column`, `result_cache` and `private`; an unset key is absent from the canonical serialization.", "why": "", "file": "spec/10-store.md", - "line": 172 + "line": 176 }, { "id": "store.declare.column-types", @@ -2147,7 +2169,7 @@ "statement": "`columns` maps a column to a type spelled as {{store.reconcile.typed-landing}} reads it; every landing into the table, a pipeline run included, lands that column in the declared type.", "why": "because a JSON value alone cannot say it carries bytes or a vector", "file": "spec/10-store.md", - "line": 173 + "line": 177 }, { "id": "store.declare.two-genres", @@ -2158,7 +2180,7 @@ "statement": "A table holds items, landed by connectors, or artifacts, synthesized and tagged by an open kind string the engine does not enumerate. Both append, dedupe on content and carry a timestamp.", "why": "", "file": "spec/10-store.md", - "line": 175 + "line": 179 }, { "id": "store.declare.unkeyed-union", @@ -2169,7 +2191,7 @@ "statement": "A table declaring no `primary_key` reads as the byte-identical union of its visible committed runs, selected under {{store.declare.replace-frontier}} when replacing.", "why": "", "file": "spec/10-store.md", - "line": 176 + "line": 180 }, { "id": "store.declare.dedup-view", @@ -2180,7 +2202,7 @@ "statement": "A table declaring `primary_key` reads through `ROW_NUMBER() OVER (PARTITION BY ORDER BY DESC, _ingested_at DESC, _run_id DESC, _row_seq DESC) = 1` over its current snapshot, if any, unioned with the committed runs that snapshot omits.", "why": "because a keyed table read as a union before its first fold inflates every aggregate silently", "file": "spec/10-store.md", - "line": 177 + "line": 181 }, { "id": "store.declare.order-by-default", @@ -2191,7 +2213,7 @@ "statement": "`order_by` names the column picking the surviving row per key, and defaults to `_ingested_at`.", "why": "", "file": "spec/10-store.md", - "line": 179 + "line": 183 }, { "id": "store.declare.order-by-unknown", @@ -2202,7 +2224,7 @@ "statement": "An `order_by` naming a column neither declared nor injected raises `StoreOrderByUnknownColumn` at validation, before the first batch.", "why": "because an ordering column absent from every file reads as null and picks survivors arbitrarily", "file": "spec/10-store.md", - "line": 180 + "line": 184 }, { "id": "store.declare.key-unknown", @@ -2213,7 +2235,7 @@ "statement": "A `primary_key` naming a column neither declared nor injected raises `StoreKeyUnknownColumn` at validation, before the first batch.", "why": "because a key column absent from every file partitions every row into one group and collapses the table to a single row", "file": "spec/10-store.md", - "line": 182 + "line": 186 }, { "id": "store.declare.nested-key", @@ -2224,7 +2246,7 @@ "statement": "A struct, list or map column named by `primary_key`, `order_by`, `cluster_by`, `partition_by` or `valid_time` raises `StoreNestedKeyColumn` at validation, before any Parquet.", "why": "A-store", "file": "spec/10-store.md", - "line": 184 + "line": 188 }, { "id": "store.declare.write-mode", @@ -2235,7 +2257,7 @@ "statement": "`write_mode` is `append`, the default, keeping the last write per key and retiring no key, or `replace`.", "why": "", "file": "spec/10-store.md", - "line": 186 + "line": 190 }, { "id": "store.declare.replace-frontier", @@ -2246,7 +2268,7 @@ "statement": "Under `replace`, a read covers the newest run carrying the source's complete state plus every run committed after it.", "why": "", "file": "spec/10-store.md", - "line": 187 + "line": 191 }, { "id": "store.declare.replace-retains", @@ -2257,7 +2279,7 @@ "statement": "A replacing run leaves the runs it displaced on disk until `retain_runs` passes, writes no erasure receipt and walks no lineage.", "why": "", "file": "spec/10-store.md", - "line": 188 + "line": 192 }, { "id": "store.declare.empty-run", @@ -2268,7 +2290,7 @@ "statement": "A run landing zero rows commits a manifest with no parts; without {{store.declare.empty-replacement}} it replaces nothing. A table with no rows registers as a zero-row relation over its declared and injected columns.", "why": "", "file": "spec/10-store.md", - "line": 189 + "line": 193 }, { "id": "store.declare.empty-replacement", @@ -2279,7 +2301,7 @@ "statement": "A complete zero-row `replace` run marks its manifest as a replacement frontier; current reads cover no earlier files, while bounded reads before its commit retain earlier rows within retention.", "why": "A-store", "file": "spec/10-store.md", - "line": 190 + "line": 194 }, { "id": "store.declare.empty-frontier-preserved", @@ -2290,7 +2312,7 @@ "statement": "Sync push, sync pull and fold preserve an empty replacement frontier without writing a data row.", "why": "A-store", "file": "spec/10-store.md", - "line": 192 + "line": 196 }, { "id": "store.declare.fold-job", @@ -2301,7 +2323,7 @@ "statement": "An enabled compaction job covering a table is a `[[job]]` block of kind `fold` carrying a `schedule`, whose `enabled` is absent or true and whose `target` is absent or names the table's destination name.", "why": "A-store", "file": "spec/10-store.md", - "line": 194 + "line": 198 }, { "id": "store.declare.fold-coverage", @@ -2312,7 +2334,7 @@ "statement": "`pipeline validate` warns on stderr, naming the table, where a table declares `primary_key` and no job meeting {{store.declare.fold-job}} covers it; the warning alone fails nothing.", "why": "A-store", "file": "spec/10-store.md", - "line": 196 + "line": 200 }, { "id": "store.declare.declaration-set", @@ -2323,7 +2345,7 @@ "statement": "A project's table blocks are the declaration's and those of every `pipelines/*.toml` and `pipelines/*.json` beside it — a `[pipeline]` table's as named, each specification's under {{run.declare.table-name}}. The read face and every `sync` verb read this set.", "why": "because a key or `replicate` declared only where `pipeline run` reads it leaves reads, pulls and folds treating the table as unkeyed and replicated", "file": "spec/10-store.md", - "line": 198 + "line": 202 }, { "id": "store.declare.read-side-keys", @@ -2334,7 +2356,7 @@ "statement": "A declaration key changes what a read returns and rewrites no committed part; a key added after rows land applies from the next read.", "why": "", "file": "spec/10-store.md", - "line": 200 + "line": 204 }, { "id": "store.declare.view-definition", @@ -2345,7 +2367,7 @@ "statement": "`view` holds one `SELECT` naming store tables and other views, admitted by {{read.guard.single-read-only-statement}} at validation; the engine builds it into staging and publishes it as a model through {{run.publish.staging}}.", "why": "A-store", "file": "spec/10-store.md", - "line": 201 + "line": 205 }, { "id": "store.declare.view-policy", @@ -2356,7 +2378,7 @@ "statement": "A `view` reading a table that carries a row policy or column masks declares its own `policy`; one declaring none raises `ViewPolicyUndeclared` at validation.", "why": "A-store", "file": "spec/10-store.md", - "line": 203 + "line": 207 }, { "id": "store.reserve.underscore-namespace", @@ -2367,7 +2389,7 @@ "statement": "Column names beginning `_` belong to the engine; the injected set and the reserved optional set are its whole content.", "why": "", "file": "spec/10-store.md", - "line": 214 + "line": 218 }, { "id": "store.reserve.injected", @@ -2378,7 +2400,7 @@ "statement": "The engine injects `_ingested_at` as a non-null Parquet `TIMESTAMP(UTC, NANOS)`, `_run_id`, `_batch_seq` as int32 where a batch scope exists, `_site_id`, and `_authored_by` where an authenticated subject authorized the write, replacing any producer value.", "why": "because a string instant does not sort by time once fractions or offsets appear", "file": "spec/10-store.md", - "line": 215 + "line": 219 }, { "id": "store.reserve.row-seq", @@ -2389,7 +2411,7 @@ "statement": "The engine injects `_row_seq`, a non-null int64 numbering a run's rows from 0 in batch order, replacing any producer value.", "why": "because a run's rows share `_ingested_at` and `_run_id`, and keeping the last write per key needs an order among them", "file": "spec/10-store.md", - "line": 217 + "line": 221 }, { "id": "store.reserve.commit-seq", @@ -2400,7 +2422,7 @@ "statement": "The engine injects `_commit_seq`, a non-null int64 its commit assigns above every value the table holds; a run committing after a read carries a value above every row that read returned, whatever its `_ingested_at`.", "why": "A-topology", "file": "spec/10-store.md", - "line": 219 + "line": 223 }, { "id": "store.reserve.commit-order", @@ -2411,7 +2433,7 @@ "statement": "Commits of one table serialize from assigning `_commit_seq`, through the part write, to the step that makes the run readable, so the readable runs of a table always hold a prefix of its commit sequence.", "why": "A-topology", "file": "spec/10-store.md", - "line": 221 + "line": 225 }, { "id": "store.reserve.commit-seq-seed", @@ -2422,7 +2444,7 @@ "statement": "A commit assigns one above the greatest of the node's counter, every run manifest's `commit_seq` and the current snapshot's, so a store restored by a pull, or holding no counter, never reissues a value.", "why": "A-topology", "file": "spec/10-store.md", - "line": 223 + "line": 227 }, { "id": "store.reserve.no-placeholder", @@ -2433,7 +2455,7 @@ "statement": "A path with no batch scope or no authenticated subject omits that column instead of writing nulls.", "why": "", "file": "spec/10-store.md", - "line": 225 + "line": 229 }, { "id": "store.reserve.taint", @@ -2444,7 +2466,7 @@ "statement": "The engine injects `_taint`, a label under {{connector.infer.provenance-order}}, on each row a model's output lands as, replacing any producer value; a row no model produced omits it.", "why": "because a label a producer sets is one injected text can forge", "file": "spec/10-store.md", - "line": 226 + "line": 230 }, { "id": "store.reserve.optional", @@ -2455,7 +2477,7 @@ "statement": "A producer sets any of `_modality`, `_lang`, `_provenance` and `_prompt_hash`, and each surfaces in the provenance envelope where present.", "why": "", "file": "spec/10-store.md", - "line": 228 + "line": 232 }, { "id": "store.reserve.modality", @@ -2466,7 +2488,7 @@ "statement": "`_modality` takes one of text, image, audio, structured or mixed; another value fails validation of its batch.", "why": "", "file": "spec/10-store.md", - "line": 229 + "line": 233 }, { "id": "store.reserve.lang-and-provenance", @@ -2477,7 +2499,7 @@ "statement": "`_lang` carries a BCP-47 tag; `_provenance` carries the evidence rows the row derives from, addressed by the identity the source table keys on.", "why": "", "file": "spec/10-store.md", - "line": 230 + "line": 234 }, { "id": "store.reserve.prompt-hash", @@ -2488,7 +2510,7 @@ "statement": "`_prompt_hash` is `sha256:` over the prompt template, not the rendered prompt.", "why": "", "file": "spec/10-store.md", - "line": 231 + "line": 235 }, { "id": "store.reserve.column-name", @@ -2499,7 +2521,7 @@ "statement": "A producer column inside the `_` namespace and outside the optional set raises `StoreReservedColumnName` at reconciliation, before any Parquet.", "why": "P1", "file": "spec/10-store.md", - "line": 232 + "line": 236 }, { "id": "store.reserve.table-namespaces", @@ -2510,7 +2532,7 @@ "statement": "The engine reserves two table namespaces: `_runs`, the durable run record, and `_visibility`, under which {{disclosure.mirror.grants-table}} lands each source's mirrored access tables.", "why": "", "file": "spec/10-store.md", - "line": 234 + "line": 238 }, { "id": "store.reserve.table-name", @@ -2521,7 +2543,7 @@ "statement": "A pipeline declaring a table inside a reserved namespace raises `StoreReservedTableName` when its manifest is assembled, naming the reservation.", "why": "P1", "file": "spec/10-store.md", - "line": 235 + "line": 239 }, { "id": "store.reserve.ledger-suffix", @@ -2532,7 +2554,7 @@ "statement": "A table name ending in `__requests` is reserved to request-ledger relations and refuses as {{store.reserve.table-name}}.", "why": "", "file": "spec/10-store.md", - "line": 237 + "line": 241 }, { "id": "store.reserve.ledger-path", @@ -2543,7 +2565,7 @@ "statement": "A run's request ledger is `requests/..parquet`, disjoint per run and per writing node.", "why": "", "file": "spec/10-store.md", - "line": 238 + "line": 242 }, { "id": "store.reserve.ledger-append", @@ -2554,7 +2576,7 @@ "statement": "Appends to one run's ledger file serialize under a lock on it, and an append returns once its rows and the rename are synced to disk.", "why": "", "file": "spec/10-store.md", - "line": 239 + "line": 243 }, { "id": "store.reserve.ledger-fold", @@ -2565,7 +2587,7 @@ "statement": "Each fold pass merges a table's committed ledger files into `requests/folded-.parquet`, and a replica carries ledgers with their table.", "why": "because one ledger file per run per node per table grows listing and diff cost without bound", "file": "spec/10-store.md", - "line": 240 + "line": 244 }, { "id": "store.reserve.ledger-retention", @@ -2576,7 +2598,7 @@ "statement": "A ledger row is collected 365 d after its run committed.", "why": "", "file": "spec/10-store.md", - "line": 242 + "line": 246 }, { "id": "store.reconcile.explicit-file-list", @@ -2587,7 +2609,7 @@ "statement": "Every read hands `read_parquet` an explicit sorted file list resolved from the pointer and the manifests, never a glob; a stray file joins nothing.", "why": "", "file": "spec/10-store.md", - "line": 252 + "line": 256 }, { "id": "store.reconcile.union-by-name", @@ -2598,7 +2620,7 @@ "statement": "Every read passes `union_by_name=true`; a column resolves to the common supertype of the files carrying it, and the generated relation casts no column but a vector, per {{store.reconcile.half-width}}.", "why": "", "file": "spec/10-store.md", - "line": 253 + "line": 257 }, { "id": "store.reconcile.lattice", @@ -2609,7 +2631,7 @@ "statement": "The type lattice holds one promotion, `Int64` with `Float64` to `Float64`; a JSON type absorbs `Utf8`.", "why": "A-store", "file": "spec/10-store.md", - "line": 254 + "line": 258 }, { "id": "store.reconcile.incompatible", @@ -2620,7 +2642,7 @@ "statement": "Any other pair of types observed for one column raises `StoreSchemaIncompatible` at the write, naming the column, the stored type and the arriving type.", "why": "A-store", "file": "spec/10-store.md", - "line": 256 + "line": 260 }, { "id": "store.reconcile.float-loss", @@ -2631,7 +2653,7 @@ "statement": "An `Int64` value above 9007199254740992 loses precision once a `Float64` batch lands on its column.", "why": "", "file": "spec/10-store.md", - "line": 258 + "line": 262 }, { "id": "store.reconcile.binary-and-vector", @@ -2642,7 +2664,7 @@ "statement": "The lattice holds `Binary`, `FixedSizeBinary(n)` and a `FixedSizeList` of `Float32` or `Float16` at dimension n; none takes a promotion, so a width, item or dimension change meets {{store.reconcile.incompatible}}.", "why": "A-store", "file": "spec/10-store.md", - "line": 259 + "line": 263 }, { "id": "store.reconcile.typed-landing", @@ -2653,7 +2675,7 @@ "statement": "A producer declares these types; a JSON batch carries bytes as padded base64 and a vector as a number array of its dimension, and any other value meets {{store.reconcile.incompatible}}.", "why": "A-store", "file": "spec/10-store.md", - "line": 261 + "line": 265 }, { "id": "store.reconcile.nested-lattice", @@ -2664,7 +2686,7 @@ "statement": "A struct gains fields additively, a file without a field reading it as null; list items and map values reconcile by {{store.reconcile.lattice}}; a kind change, scalar against nested or list against map, meets {{store.reconcile.incompatible}} naming the column path.", "why": "A-store", "file": "spec/10-store.md", - "line": 263 + "line": 267 }, { "id": "store.reconcile.nested-landing", @@ -2675,7 +2697,7 @@ "statement": "A producer or declaration types a column `struct<…>`, `list<…>` or `map`; a JSON batch carries a struct and a map as an object and a list as an array, a struct key naming no field meeting {{store.reconcile.incompatible}}.", "why": "A-store", "file": "spec/10-store.md", - "line": 265 + "line": 269 }, { "id": "store.reconcile.stored-type", @@ -2686,7 +2708,7 @@ "statement": "A column `schema.json` holds as a binary, vector or nested type lands a later undeclared JSON value in that type, so a run after the first needs no declaration.", "why": "A-store", "file": "spec/10-store.md", - "line": 267 + "line": 271 }, { "id": "store.reconcile.half-width", @@ -2697,7 +2719,7 @@ "statement": "A `Float16` vector stores each element at half width in Parquet. The engine reads its elements as `FLOAT`, either binary type as `BLOB`, and a vector as an `ARRAY` of its dimension through one relation cast.", "why": "A-store", "file": "spec/10-store.md", - "line": 269 + "line": 273 }, { "id": "store.reconcile.key-widening", @@ -2708,7 +2730,7 @@ "statement": "A primary-key column takes no `Float64` promotion: the widening batch raises `StoreKeyWidened` before any Parquet, as does a fold meeting a key already reconciled to `Float64`.", "why": "A-store", "file": "spec/10-store.md", - "line": 271 + "line": 275 }, { "id": "store.reconcile.additive", @@ -2719,7 +2741,7 @@ "statement": "An unseen column joins the merged schema, and files written before it read it as null.", "why": "", "file": "spec/10-store.md", - "line": 273 + "line": 277 }, { "id": "store.reconcile.no-invented-column", @@ -2730,7 +2752,7 @@ "statement": "A scan invents no column: an ordering column absent from every file enters through a zero-row branch, and a column added after a table's first rows projects as a literal null.", "why": "", "file": "spec/10-store.md", - "line": 274 + "line": 278 }, { "id": "store.reconcile.first-sight", @@ -2741,7 +2763,7 @@ "statement": "A destination creates a table on the first schema it sees and merges each later schema into `schema.json`.", "why": "", "file": "spec/10-store.md", - "line": 275 + "line": 279 }, { "id": "store.reconcile.fold-never-narrows", @@ -2752,7 +2774,7 @@ "statement": "A fold backfills nulls and widens types, a struct's fields included; it narrows no type and drops no column or field.", "why": "", "file": "spec/10-store.md", - "line": 276 + "line": 280 }, { "id": "store.reconcile.no-history", @@ -2763,7 +2785,7 @@ "statement": "Reconciliation keeps the current shape alone; a past widening is attributed from the run files.", "why": "", "file": "spec/10-store.md", - "line": 277 + "line": 281 }, { "id": "store.reconcile.reserved-set-versioned", @@ -2774,7 +2796,7 @@ "statement": "Adding an injected column advances the semantics version, whose fingerprint recipe names the column.", "why": "", "file": "spec/10-store.md", - "line": 278 + "line": 282 }, { "id": "store.fold.pass", @@ -2785,7 +2807,7 @@ "statement": "A pass selects the committed runs the current snapshot omits, dedupes by key or unions, reconciles the schema, sorts by `cluster_by`, partitions, writes Parquet and every declared sidecar into staging, then commits by {{store.fold.pointer-commit}}.", "why": "", "file": "spec/10-store.md", - "line": 288 + "line": 292 }, { "id": "store.fold.valid-time-line", @@ -2796,7 +2818,7 @@ "statement": "A keyed table declaring `valid_time` partitions on the key together with the valid-time line and keeps one row per line.", "why": "", "file": "spec/10-store.md", - "line": 289 + "line": 293 }, { "id": "store.fold.includes-runs", @@ -2807,7 +2829,7 @@ "statement": "A snapshot's `includes_runs` names each run it folded as `/`, the run's own directory; a run committed afterwards reads on top of it.", "why": "", "file": "spec/10-store.md", - "line": 290 + "line": 294 }, { "id": "store.fold.triggers", @@ -2818,7 +2840,7 @@ "statement": "A pass fires at 50 runs committed on a table, 6 h after the table's previous pass, or on `contextful context compact
`.", "why": "", "file": "spec/10-store.md", - "line": 291 + "line": 295 }, { "id": "store.fold.retention", @@ -2829,7 +2851,7 @@ "statement": "`retain_runs` defaults to 7 d; a folded run, a superseded snapshot and its sidecars are collected once older than the window.", "why": "", "file": "spec/10-store.md", - "line": 292 + "line": 296 }, { "id": "store.fold.result", @@ -2840,7 +2862,7 @@ "statement": "A pass reports each table as folded, nothing-landed or failed, and a nothing-landed table does not stop the pass.", "why": "", "file": "spec/10-store.md", - "line": 293 + "line": 297 }, { "id": "store.fold.collection-failed", @@ -2851,7 +2873,7 @@ "statement": "A collection that fails reports its failure: beside `folded` when the pass published, since the snapshot stays published, and as `failed` otherwise; either way the command exits non-zero.", "why": "", "file": "spec/10-store.md", - "line": 294 + "line": 298 }, { "id": "store.fold.unknown-table", @@ -2862,7 +2884,7 @@ "statement": "A pass naming a table no `schema.json` declares halts the command with {{store.lay-out.unknown-table}}.", "why": "P1", "file": "spec/10-store.md", - "line": 295 + "line": 299 }, { "id": "store.fold.compaction-lease", @@ -2873,7 +2895,7 @@ "statement": "A pass holds the table's compaction lease and stamps its fence into the snapshot manifest and the pointer.", "why": "A-store", "file": "spec/10-store.md", - "line": 297 + "line": 301 }, { "id": "store.fold.pointer-commit", @@ -2884,7 +2906,7 @@ "statement": "A pass publishes by replacing `_pointer.json` conditioned on the ETag it read at pass start, with `If-Match` on an object store and a version-checked rename on a filesystem.", "why": "A-store", "file": "spec/10-store.md", - "line": 299 + "line": 303 }, { "id": "store.fold.partial-snapshot", @@ -2895,7 +2917,7 @@ "statement": "A reader observes a snapshot and every declared sidecar together or neither; a commit exposing one without the other raises `StorePartialSnapshot`.", "why": "P4", "file": "spec/10-store.md", - "line": 301 + "line": 305 }, { "id": "store.fold.lost-pointer", @@ -2906,7 +2928,7 @@ "statement": "A pass whose pointer replace loses its condition publishes nothing, and its staged snapshot is collected.", "why": "", "file": "spec/10-store.md", - "line": 303 + "line": 307 }, { "id": "store.fold.staging-collected", @@ -2917,7 +2939,7 @@ "statement": "A staging directory, and a snapshot directory no pointer chain reaches, are collected by the next pass and read by nobody.", "why": "", "file": "spec/10-store.md", - "line": 304 + "line": 308 }, { "id": "store.fold.non-blocking", @@ -2928,7 +2950,7 @@ "statement": "A statement running during a pass reads the snapshot the pointer named when it started; statements starting after the commit read the new one.", "why": "", "file": "spec/10-store.md", - "line": 305 + "line": 309 }, { "id": "store.fold.supersedes", @@ -2939,7 +2961,7 @@ "statement": "A new snapshot supersedes the previous one without deleting it, and a bounded read reaches the older one until retention collects it.", "why": "", "file": "spec/10-store.md", - "line": 306 + "line": 310 }, { "id": "store.index.kinds", @@ -2950,7 +2972,7 @@ "statement": "Five index kinds exist: Parquet footer zone maps, a sorted-Parquet sparse-map primary-key lookup, an HNSW vector graph, a positional full-text index, and opt-in per-column bloom filters.", "why": "", "file": "spec/10-store.md", - "line": 342 + "line": 346 }, { "id": "store.index.declaration", @@ -2961,7 +2983,7 @@ "statement": "A sidecar is declared per table under `indexes` with a kind, a `column`, an `id_column` and builder parameters; the `vector` kind takes `model`, `dim`, the `cosine` metric, `m` and `ef_construction`, the `fulltext` kind a `tokenizer`.", "why": "", "file": "spec/10-store.md", - "line": 343 + "line": 347 }, { "id": "store.index.id-column", @@ -2972,7 +2994,7 @@ "statement": "A sidecar's `id_column` defaults to a single-column primary key on a table declaring no `valid_time`, and every sidecar of a table shares it, so keyed, composite-key and unkeyed tables each take one.", "why": "A-store", "file": "spec/10-store.md", - "line": 344 + "line": 348 }, { "id": "store.index.id-column-unresolved", @@ -2983,7 +3005,7 @@ "statement": "A declaration naming no `id_column` on a table without a single-column primary key or declaring `valid_time`, one naming a valid-time table's single key, or two sidecars naming different ones, raises `StoreIndexIdColumnUnresolved` at manifest validation.", "why": "A-store", "file": "spec/10-store.md", - "line": 346 + "line": 350 }, { "id": "store.index.id-unique", @@ -2994,7 +3016,7 @@ "statement": "A fold meeting one `id_column` value on two rows of the snapshot it stages raises `StoreIndexIdNotUnique`, naming the table, the column and the value, and publishes nothing.", "why": "A-store", "file": "spec/10-store.md", - "line": 348 + "line": 352 }, { "id": "store.index.paths", @@ -3005,7 +3027,7 @@ "statement": "A vector sidecar sits at `indexes/vec--/zone=-/`, inside the snapshot directory it indexes.", "why": "", "file": "spec/10-store.md", - "line": 350 + "line": 354 }, { "id": "store.index.path-collision", @@ -3016,7 +3038,7 @@ "statement": "Two sidecar declarations of one table resolving to one path under {{store.index.paths}} raise `StoreIndexPathCollision`, naming both, at manifest validation and before the fold builds either.", "why": "because a second build overwrites the first sidecar's files, and the manifest then records an entry no reader opens", "file": "spec/10-store.md", - "line": 351 + "line": 355 }, { "id": "store.index.identity", @@ -3027,7 +3049,7 @@ "statement": "A sidecar's identity is `(column, builder, builder-version)`; two builders over one column coexist, the caller picks at query time, and `derived.sqlite` records each builder.", "why": "", "file": "spec/10-store.md", - "line": 353 + "line": 357 }, { "id": "store.index.rebuild", @@ -3038,7 +3060,7 @@ "statement": "Swapping a builder rebuilds the sidecar, leaves the Parquet untouched, and callers on the existing identity read through the cutover.", "why": "", "file": "spec/10-store.md", - "line": 354 + "line": 358 }, { "id": "store.index.not-in-file-set", @@ -3049,7 +3071,7 @@ "statement": "`indexes/` joins no table's file set; a snapshot reader lists only the parts its manifest names.", "why": "", "file": "spec/10-store.md", - "line": 355 + "line": 359 }, { "id": "store.index.dies-with-snapshot", @@ -3060,7 +3082,7 @@ "statement": "Collecting a snapshot collects its sidecars in the same step.", "why": "", "file": "spec/10-store.md", - "line": 356 + "line": 360 }, { "id": "store.index.candidate-ids", @@ -3071,7 +3093,7 @@ "statement": "A sidecar yields candidate `id_column` values, not rows; they re-join through the enforced relation on that column before a top-K is final.", "why": "P5", "file": "spec/10-store.md", - "line": 357 + "line": 361 }, { "id": "store.index.vector-by-fold", @@ -3082,7 +3104,7 @@ "statement": "The fold builds each declared vector sidecar over the staged rows holding a non-null identifier and a non-zero vector, from the declared model where the table carries `embedding_model`, and records its entry in the snapshot manifest.", "why": "A-store", "file": "spec/10-store.md", - "line": 359 + "line": 363 }, { "id": "store.index.graph", @@ -3093,7 +3115,7 @@ "statement": "A vector sidecar is an HNSW graph over unit-length `Float32` vectors whose layers draw from a seed of the snapshot id and column, so one staged row set builds one byte-identical graph.", "why": "because a rebuilt sidecar then differs from its predecessor only where the rows do, and a recall figure replays from its snapshot", "file": "spec/10-store.md", - "line": 361 + "line": 365 }, { "id": "store.index.fulltext-by-fold", @@ -3104,7 +3126,7 @@ "statement": "The fold builds each declared full-text sidecar over the staged rows whose identifier is non-null and whose text yields a term, and records its entry in the snapshot manifest.", "why": "A-store", "file": "spec/10-store.md", - "line": 363 + "line": 367 }, { "id": "store.index.postings", @@ -3115,7 +3137,7 @@ "statement": "A full-text sidecar is one file holding a sorted term dictionary and, per term, the rows and positions it occurs at, which a reader binary-searches in mapped or decrypted bytes.", "why": "because one layout then serves a mapped plaintext file and a decrypted sealed one, and a probe reads only its own terms' postings", "file": "spec/10-store.md", - "line": 365 + "line": 369 }, { "id": "store.index.tokenizer", @@ -3126,7 +3148,7 @@ "statement": "A `tokenizer` is `unicode`, the default, indexing each lowercased alphanumeric run as one term, or `cjk`, which indexes each Han, Kana or Hangul stretch of a run as overlapping character bigrams.", "why": "because an unspaced script puts no boundary inside a sentence, so a word within one is never a whole term", "file": "spec/10-store.md", - "line": 367 + "line": 371 }, { "id": "store.index.column-absent", @@ -3137,7 +3159,7 @@ "statement": "An index over a column, or naming an `id_column`, the reconciled schema lacks raises `StoreIndexColumnAbsent` at the fold, before the pass that builds it stages anything.", "why": "because an index over a missing column builds empty and reads as no match", "file": "spec/10-store.md", - "line": 369 + "line": 373 }, { "id": "store.index.column-type", @@ -3148,7 +3170,7 @@ "statement": "An `id_column` typed other than text or integer, a vector sidecar's column other than a vector of its `dim`, or a full-text sidecar's other than text raises `StoreIndexColumnType` at manifest validation where `columns` types it, else before any row lands.", "why": "because a builder reading another width, or an identifier the re-join casts differently, indexes rows no reader finds", "file": "spec/10-store.md", - "line": 371 + "line": 375 }, { "id": "store.index.clustering", @@ -3159,7 +3181,7 @@ "statement": "`cluster_by` sorts rows within a file lexicographically over its columns in declared order; zone maps then skip row groups with no manifest entry and no sidecar.", "why": "", "file": "spec/10-store.md", - "line": 373 + "line": 377 }, { "id": "store.index.partitioning", @@ -3170,7 +3192,7 @@ "statement": "Partitioning is off unless `partition_by` declares it.", "why": "", "file": "spec/10-store.md", - "line": 374 + "line": 378 }, { "id": "store.index.partition-type", @@ -3181,7 +3203,7 @@ "statement": "A `partition_by` column typed binary or vector raises `StorePartitionColumnType` at validation, before any Parquet.", "why": "because bytes name no directory that keeps two distinct values apart", "file": "spec/10-store.md", - "line": 375 + "line": 379 }, { "id": "store.index.partition-warnings", @@ -3192,7 +3214,7 @@ "statement": "Planning warns, naming the column, where a partition specification projects more than 1000 partitions or a median partition below 16 MiB.", "why": "", "file": "spec/10-store.md", - "line": 377 + "line": 381 }, { "id": "store.index.tenant-outermost", @@ -3203,7 +3225,7 @@ "statement": "A multi-tenant table carries the tenant identifier as its outermost partition column.", "why": "because a dropped tenant predicate then reads nothing instead of every tenant", "file": "spec/10-store.md", - "line": 378 + "line": 382 }, { "id": "store.index.tenant-verbatim", @@ -3214,7 +3236,7 @@ "statement": "A tenant value is written and compared byte for byte, with no trimming, case folding or Unicode normalization; a percent-escaped directory name is representation alone.", "why": "", "file": "spec/10-store.md", - "line": 380 + "line": 384 }, { "id": "store.bound-time.two-clocks", @@ -3225,7 +3247,7 @@ "statement": "Transaction time is `_ingested_at`; valid time is a declared pair of the table's own columns. The engine infers no pair and stamps no second transaction clock.", "why": "", "file": "spec/10-store.md", - "line": 394 + "line": 398 }, { "id": "store.bound-time.valid-time-declaration", @@ -3236,7 +3258,7 @@ "statement": "`[pipeline.tables.valid_time]` declares `from` and an optional `to`; a table declaring `from` alone treats each row as valid from that instant onward.", "why": "", "file": "spec/10-store.md", - "line": 395 + "line": 399 }, { "id": "store.bound-time.valid-time-type", @@ -3247,7 +3269,7 @@ "statement": "A declared valid-time column whose type is not a timestamp raises `StoreValidTimeNotTimestamp` at declaration.", "why": "P1", "file": "spec/10-store.md", - "line": 396 + "line": 400 }, { "id": "store.bound-time.instant-comparison", @@ -3258,7 +3280,7 @@ "statement": "Every bound compares instants as timestamps, never as strings; a date-only literal resolves, where it is built, to the start of the next day, exclusive.", "why": "", "file": "spec/10-store.md", - "line": 398 + "line": 402 }, { "id": "store.bound-time.as-of", @@ -3269,7 +3291,7 @@ "statement": "`as_of` resolves each table to the newest reachable snapshot created at or before it, plus the committed runs at or before it that snapshot omits, inside the table's FROM-source.", "why": "because filtering the current files by ingest stamp returns different rows before and after a fold", "file": "spec/10-store.md", - "line": 399 + "line": 403 }, { "id": "store.bound-time.as-of-unretained", @@ -3280,7 +3302,7 @@ "statement": "An `as_of` earlier than the oldest retained snapshot of a table whose history has been collected raises `StoreAsOfUnretained`, naming the oldest answerable instant.", "why": "P4", "file": "spec/10-store.md", - "line": 401 + "line": 405 }, { "id": "store.bound-time.valid-as-of", @@ -3291,7 +3313,7 @@ "statement": "`valid_as_of` wraps the same inner source with `from <= valid_as_of AND (to IS NULL OR to > valid_as_of)` over the declared pair.", "why": "", "file": "spec/10-store.md", - "line": 403 + "line": 407 }, { "id": "store.bound-time.valid-time-undeclared", @@ -3302,7 +3324,7 @@ "statement": "A `valid_as_of` read against a table declaring no pair raises `StoreValidTimeUndeclared`, naming the table.", "why": "P1", "file": "spec/10-store.md", - "line": 404 + "line": 408 }, { "id": "store.bound-time.pin-bound", @@ -3313,7 +3335,7 @@ "statement": "A read carrying `as_of` and a build pin resolves each table named by both by {{read.resolve-pin.earlier-bound-wins}}.", "why": "", "file": "spec/10-store.md", - "line": 406 + "line": 410 }, { "id": "store.bound-time.unbounded-latest", @@ -3324,7 +3346,7 @@ "statement": "A read carrying neither bound resolves each table to its current snapshot plus the committed runs it omits.", "why": "", "file": "spec/10-store.md", - "line": 407 + "line": 411 }, { "id": "store.bound-time.beneath-enforcement", @@ -3335,7 +3357,7 @@ "statement": "Both bounds apply beneath the row predicate, the column mask and the tenant filter, narrowing what enforcement admits and widening nothing.", "why": "P5", "file": "spec/10-store.md", - "line": 408 + "line": 412 }, { "id": "store.bound-time.covering-versions", @@ -3346,7 +3368,7 @@ "statement": "Over an unkeyed table a valid-time bound returns every version whose interval covers the instant.", "why": "", "file": "spec/10-store.md", - "line": 410 + "line": 414 }, { "id": "store.bound-time.echo", @@ -3357,7 +3379,7 @@ "statement": "A bounded read echoes `contextful.bounds` as `{as_of?, valid_as_of?, inclusive}`, each instant in RFC 3339 UTC with nine fractional digits and a `Z` suffix; an unbounded read omits it.", "why": "", "file": "spec/10-store.md", - "line": 411 + "line": 415 }, { "id": "store.encrypt.key-binding", @@ -3368,7 +3390,7 @@ "statement": "At-rest encryption is per project, off unless `[encryption] key_source` names `env:` or a key-management service.", "why": "", "file": "spec/10-store.md", - "line": 419 + "line": 423 }, { "id": "store.encrypt.key-unbound", @@ -3379,7 +3401,7 @@ "statement": "A `key_source` naming a binding the process lacks raises `StoreEncryptionKeyUnbound` at startup, with no cleartext fallback.", "why": "P3", "file": "spec/10-store.md", - "line": 420 + "line": 424 }, { "id": "store.encrypt.cipher", @@ -3390,7 +3412,7 @@ "statement": "Parquet, footers included, encrypts through Parquet modular encryption; every sidecar and ledger file encrypts with AES-256-GCM under a per-file data key wrapped by the project key.", "why": "because a cleartext vector graph admits nearest-neighbour search over the embedding space", "file": "spec/10-store.md", - "line": 422 + "line": 426 }, { "id": "store.encrypt.password-kdf", @@ -3401,7 +3423,7 @@ "statement": "A password-derived project key uses Argon2id with 64 MiB memory, 3 iterations and 4 lanes.", "why": "", "file": "spec/10-store.md", - "line": 424 + "line": 428 }, { "id": "store.encrypt.transport-separate", @@ -3412,7 +3434,7 @@ "statement": "At-rest encryption covers files and TLS covers bucket transport; a cleartext endpoint carries no encrypted-at-rest claim.", "why": "", "file": "spec/10-store.md", - "line": 425 + "line": 429 }, { "id": "store.encrypt.redacted-index", @@ -3423,7 +3445,7 @@ "statement": "An index declared over a column redacted at write time raises `StoreIndexOverRedactedColumn` at manifest validation.", "why": "A-authority", "file": "spec/10-store.md", - "line": 426 + "line": 430 }, { "id": "store.encrypt.at-rest-scope", @@ -3434,7 +3456,7 @@ "statement": "A stolen bucket credential yields ciphertext Parquet and sidecars, no write-time-redacted content, and no key material.", "why": "", "file": "spec/10-store.md", - "line": 428 + "line": 432 }, { "id": "store.encrypt.sidecar-reader", @@ -3445,7 +3467,7 @@ "statement": "In an unencrypted project a reader memory-maps a sidecar read-only; in an encrypted one it decrypts each sealed sidecar file into anonymous process memory and writes no cleartext to disk.", "why": "A-store", "file": "spec/10-store.md", - "line": 429 + "line": 433 }, { "id": "store.encrypt.rotation", @@ -3456,7 +3478,7 @@ "statement": "Rotation writes forward: new files take the new key version, published files keep theirs until collected, and a key version retires once no retained file names it.", "why": "", "file": "spec/10-store.md", - "line": 431 + "line": 435 }, { "id": "store.endpoint.schemes", @@ -3467,7 +3489,7 @@ "statement": "`[sync] endpoint` opens `file://` as a filesystem bucket, `s3://` as that region's AWS S3 endpoint, `r2://` as that account's R2 endpoint at region `auto`, and `https://` at `[sync] region`, default `us-east-1`.", "why": "", "file": "spec/10-store.md", - "line": 439 + "line": 443 }, { "id": "store.endpoint.unsupported-scheme", @@ -3478,7 +3500,7 @@ "statement": "Any other scheme, or an S3 or R2 endpoint in a build without the `s3-sync` feature, raises `SyncEndpointUnsupported`, naming the endpoint.", "why": "because a bucket the binary cannot reach refuses when it opens, before a push uploads anything", "file": "spec/10-store.md", - "line": 440 + "line": 444 }, { "id": "store.endpoint.plaintext", @@ -3489,7 +3511,7 @@ "statement": "An `http://` endpoint opens on a loopback host alone; any other host raises `SyncEndpointInsecure`, naming it.", "why": "because objects and signed requests crossing a network in plaintext are readable and replayable by anyone on the path", "file": "spec/10-store.md", - "line": 442 + "line": 446 }, { "id": "store.endpoint.addressing", @@ -3500,7 +3522,7 @@ "statement": "An S3 or R2 bucket addresses each object path-style, `//` with every key segment percent-encoded, and signs every request with AWS Signature Version 4 for service `s3` at the endpoint's region.", "why": "", "file": "spec/10-store.md", - "line": 444 + "line": 448 }, { "id": "store.endpoint.credentials", @@ -3511,7 +3533,7 @@ "statement": "`[sync] access_key_id`, `secret_access_key` and the optional `session_token` each bind `secret://`, hydrated as {{connector.reference.whole-value-reference}}, or `env://NAME`, read whole from the process environment, as the bucket opens.", "why": "", "file": "spec/10-store.md", - "line": 445 + "line": 449 }, { "id": "store.endpoint.credential-unbound", @@ -3522,7 +3544,7 @@ "statement": "An S3 or R2 endpoint whose `[sync]` omits `access_key_id` or `secret_access_key`, binds a credential key to anything but a reference, or names an unset variable raises `SyncCredentialUnbound`, naming the key.", "why": "because a literal key in `config.toml` sits in plaintext on every disk and backup holding the store root", "file": "spec/10-store.md", - "line": 446 + "line": 450 }, { "id": "store.endpoint.conditional-answers", @@ -3533,7 +3555,7 @@ "statement": "A `412` or `409`, or a `404` naming `NoSuchKey` to an `If-Match` put, answers a failed condition; a `501` answers an unsupported method and a `403` a forbidden credential, both read by {{store.probe.inconclusive}}.", "why": "", "file": "spec/10-store.md", - "line": 448 + "line": 452 }, { "id": "store.endpoint.missing-bucket", @@ -3544,7 +3566,7 @@ "statement": "A `404` naming any code but `NoSuchKey`, `NoSuchBucket` among them, answers no absent object and no failed condition: the get, put, delete or list fails as transport, naming the code.", "why": "because a mistyped or deleted bucket otherwise reads as an empty bucket, a free lease or a lost compare-and-set", "file": "spec/10-store.md", - "line": 449 + "line": 453 }, { "id": "store.endpoint.list-pages", @@ -3555,7 +3577,7 @@ "statement": "A list follows each continuation token until the backend reports the listing complete, and returns every key under the prefix sorted.", "why": "", "file": "spec/10-store.md", - "line": 451 + "line": 455 }, { "id": "store.emit.plan", @@ -3566,7 +3588,7 @@ "statement": "`contextful sync manifest --emit` prints, as a bucket manifest in JSON, each key a push of the store commits with its sha256, size and owner, and reads and writes no bucket object.", "why": "because a pre-push check and a dry run need the commit's content without a credential or a network", "file": "spec/10-store.md", - "line": 459 + "line": 463 }, { "id": "store.emit.plan-scope", @@ -3577,7 +3599,7 @@ "statement": "The plan lists every key {{store.push.wire-format}} uploads and each `schema.json` at its local digest; a key another node owns stays out.", "why": "", "file": "spec/10-store.md", - "line": 461 + "line": 465 }, { "id": "store.push.prefix-escape", @@ -3588,7 +3610,7 @@ "statement": "A key resolving outside the prefix raises `SyncPrefixEscape`, naming the key and the prefix.", "why": "P3", "file": "spec/10-store.md", - "line": 467 + "line": 471 }, { "id": "store.push.prefix-unbound", @@ -3599,7 +3621,7 @@ "statement": "An unset variable named by `prefix_from` raises `SyncPrefixUnbound` at startup, with no bucket-root fallback.", "why": "P3", "file": "spec/10-store.md", - "line": 469 + "line": 473 }, { "id": "store.push.prefix-overspecified", @@ -3610,7 +3632,7 @@ "statement": "Declaring `prefix` and `prefix_from` together raises `SyncPrefixOverspecified`.", "why": "P3", "file": "spec/10-store.md", - "line": 471 + "line": 475 }, { "id": "store.push.in-flight", @@ -3621,7 +3643,7 @@ "statement": "A second push of one store on one machine raises `SyncPushInFlight`, naming the holder of the push guard.", "why": "A-store", "file": "spec/10-store.md", - "line": 473 + "line": 477 }, { "id": "store.push.wire-format", @@ -3632,7 +3654,7 @@ "statement": "A push uploads each file it owns, or no node owns, whose digest the bucket lacks under `//` by a conditional put; machine catalogs, `config.toml`, locks and staging directories stay local, and table pointers travel by {{store.push.pointer-carry}}.", "why": "", "file": "spec/10-store.md", - "line": 475 + "line": 479 }, { "id": "store.push.schema-cas", @@ -3643,7 +3665,7 @@ "statement": "A table's `schema.json` commits by merging into the bucket's copy through the one-promotion lattice and replacing it on the ETag read; no copy overwrites another.", "why": "because two nodes landing different columns into one table both keep them", "file": "spec/10-store.md", - "line": 476 + "line": 480 }, { "id": "store.push.manifest-commit", @@ -3654,7 +3676,7 @@ "statement": "A push commits when the bucket manifest, `/manifest.json` listing each key's sha256, size and owner, replaces the copy it read under `If-Match` on that copy's ETag.", "why": "A-store", "file": "spec/10-store.md", - "line": 478 + "line": 482 }, { "id": "store.push.manifest-format", @@ -3665,7 +3687,7 @@ "statement": "The bucket manifest and every generation manifest carry `format`, `1` for this layout; a manifest without `format` reads as `1`.", "why": "", "file": "spec/10-store.md", - "line": 480 + "line": 484 }, { "id": "store.push.format-unsupported", @@ -3676,7 +3698,7 @@ "statement": "A bucket or generation manifest whose `format` exceeds 1 raises `SyncManifestFormatUnsupported`, naming the key and its format, before the push commits or the pull writes a file.", "why": "because a reader rewriting a manifest it parses only in part drops the fields a newer writer added", "file": "spec/10-store.md", - "line": 481 + "line": 485 }, { "id": "store.push.generation", @@ -3687,7 +3709,7 @@ "statement": "Each manifest commit carries `generation` per {{store.push.generation-floor}} and the project's table pointers as read before it; the push then creates `/manifests/gen-.json` holding the committed bytes under `If-None-Match`.", "why": "because a restore or a pre-push gate names one committed state, and the live manifest and pointers move on", "file": "spec/10-store.md", - "line": 483 + "line": 487 }, { "id": "store.push.generation-floor", @@ -3698,7 +3720,7 @@ "statement": "A commit's `generation` is one past the greater of the read manifest's `generation`, absent reading as 0, and the newest `gen-.json` under the prefix.", "why": "because a format-1 writer predating the field drops `generation`, and reusing a taken number collides with its immutable copy", "file": "spec/10-store.md", - "line": 485 + "line": 489 }, { "id": "store.push.generation-heal", @@ -3709,7 +3731,7 @@ "statement": "Before each manifest commit, a push finding no `gen-.json` for the generation of the manifest it read creates it from the bytes it read.", "why": "because a commit of N+1 landing while `gen-.json` is absent leaves a gap no later push sees", "file": "spec/10-store.md", - "line": 487 + "line": 491 }, { "id": "store.push.generation-conflict", @@ -3720,7 +3742,7 @@ "statement": "A push whose create of `gen-.json` meets another commit's bytes raises `SyncGenerationConflict`, naming N, after its manifest commit applied.", "why": "because the immutable copy keeps the first commit, and a push reporting success names a state `pull --generation` does not restore", "file": "spec/10-store.md", - "line": 489 + "line": 493 }, { "id": "store.push.run-state", @@ -3731,7 +3753,7 @@ "statement": "`sync push` and `sync manifest --emit` first write the node's run state to `nodes//run-state.json`: each pipeline's newest run and each cursor row with its commit marker.", "why": "because run history and cursors live outside the store root, and a node starting cold otherwise sees neither", "file": "spec/10-store.md", - "line": 491 + "line": 495 }, { "id": "store.push.run-state-format", @@ -3742,7 +3764,7 @@ "statement": "A run state carries `format`, `1` for this layout; one whose `format` exceeds 1 contributes no run or cursor to a reader.", "why": "", "file": "spec/10-store.md", - "line": 493 + "line": 497 }, { "id": "store.push.pointer-carry", @@ -3753,7 +3775,7 @@ "statement": "After its manifest commit, a push publishes each local table pointer whose snapshot is whole and whose {{store.lay-out.ancestors}} name the bucket pointer's, or the bucket pointer names none, by a conditional put keeping the bucket's fence.", "why": "because a snapshot a local fold or a build publishes reads on no other node until its pointer reaches the bucket", "file": "spec/10-store.md", - "line": 494 + "line": 498 }, { "id": "store.push.pointer-leased", @@ -3764,7 +3786,7 @@ "statement": "A push publishes no pointer for a table whose compaction lease a holder keeps unexpired; that holder publishes under its fence.", "why": "because the holder publishes under the fence its acquisition raised, and a publish beside it races that commit", "file": "spec/10-store.md", - "line": 496 + "line": 500 }, { "id": "store.push.pointer-fenced", @@ -3775,7 +3797,7 @@ "statement": "A local pointer carrying a fence below the bucket pointer's stays local, and the push reports it as a warning beside its commit.", "why": "A-store", "file": "spec/10-store.md", - "line": 498 + "line": 502 }, { "id": "store.push.pointer-unrooted", @@ -3786,7 +3808,7 @@ "statement": "A local pointer whose snapshot's ancestry ends at a collected manifest before reaching the bucket pointer's snapshot or a root stays local, and the push reports it as a warning beside its commit.", "why": "because a publish over a snapshot it may not descend from drops rows the bucket reaches, and a silent keep leaves every other node reading the older snapshot", "file": "spec/10-store.md", - "line": 500 + "line": 504 }, { "id": "store.push.pointer-recommit", @@ -3797,7 +3819,7 @@ "statement": "A push whose pointer publish the bucket took commits the manifest again, so a generation names a carried pointer only once the bucket's pointer holds it.", "why": "because a generation naming a pointer the bucket never held restores a state no node read", "file": "spec/10-store.md", - "line": 502 + "line": 506 }, { "id": "store.push.stranded", @@ -3808,7 +3830,7 @@ "statement": "A push names on stderr, per owning node, each local key another node owns that the committed bucket manifest neither lists nor tombstones, with the `CONTEXTFUL_NODE_ID` that pushes it.", "why": "because a moved store root re-derives the node id, and runs landed under the old one otherwise stay local without a message", "file": "spec/10-store.md", - "line": 504 + "line": 508 }, { "id": "store.pull.digest-mismatch", @@ -3819,7 +3841,18 @@ "statement": "A downloaded object whose digest differs from its entry raises `SyncObjectDigestMismatch` and is discarded.", "why": "P4", "file": "spec/10-store.md", - "line": 540 + "line": 544 + }, + { + "id": "store.pull.identity-conflict", + "contract": "store", + "operation": "pull", + "subject": "identity-conflict", + "kind": "refusal", + "statement": "A pull into a store holding a different {{store.init.store-identity}} raises `StoreIdentityConflict` before downloading files.", + "why": "because importing another UUID lets that store's owner credential read local rows", + "file": "spec/10-store.md", + "line": 546 }, { "id": "store.pull.convergence", @@ -3830,7 +3863,7 @@ "statement": "When a named key disappears mid-download, the pull re-fetches the manifest and retries the shortfall, up to 3 attempts.", "why": "", "file": "spec/10-store.md", - "line": 542 + "line": 548 }, { "id": "store.pull.unconverged", @@ -3841,7 +3874,7 @@ "statement": "Exhausting those retries raises `SyncPullDidNotConverge`, naming the key that kept moving, and writes no pointer.", "why": "P4", "file": "spec/10-store.md", - "line": 543 + "line": 549 }, { "id": "store.pull.pointer-last", @@ -3852,7 +3885,7 @@ "statement": "A pull writes a table's pointer only after every Parquet part of the snapshot it names is home, so no reader meets a pointer ahead of its data.", "why": "A-store", "file": "spec/10-store.md", - "line": 545 + "line": 551 }, { "id": "store.pull.tombstone-applied", @@ -3863,7 +3896,7 @@ "statement": "A pull deletes the local copy of each key a tombstone names and no entry lists.", "why": "because a deletion that never reaches a consumer resurrects once its tombstone ages out", "file": "spec/10-store.md", - "line": 547 + "line": 553 }, { "id": "store.pull.pointer-advance", @@ -3874,7 +3907,7 @@ "statement": "A pull advances a local pointer only to a bucket pointer carrying a higher fence, or the same fence and a later snapshot, and verifies every advancing table before writing any pointer.", "why": "A-store", "file": "spec/10-store.md", - "line": 549 + "line": 555 }, { "id": "store.pull.schema-merge", @@ -3885,7 +3918,7 @@ "statement": "A pulled `schema.json` differing from the local copy merges into it column by column through {{store.reconcile.incompatible}}'s lattice rather than replacing it.", "why": "because rows landed locally carry columns the bucket's copy may lack", "file": "spec/10-store.md", - "line": 551 + "line": 557 }, { "id": "store.pull.before-run", @@ -3896,7 +3929,7 @@ "statement": "With `[sync] pull_before_run = true`, `run start`, `pipeline run`, `mcp` and `serve` pull every table of the bucket into the store before their first read, and a failed pull stops the command.", "why": "because a container starting on an empty disk otherwise serves and folds against a store missing every other node's runs", "file": "spec/10-store.md", - "line": 553 + "line": 559 }, { "id": "store.pull.generation", @@ -3907,7 +3940,7 @@ "statement": "`contextful sync pull --generation ` reads `gen-.json` in place of the bucket manifest, downloads each listed key in scope at its listed digest, and writes exactly that generation's pointers, whatever their fence.", "why": "", "file": "spec/10-store.md", - "line": 555 + "line": 561 }, { "id": "store.pull.generation-absent", @@ -3918,7 +3951,7 @@ "statement": "A generation the bucket holds no `gen-.json` for raises `SyncGenerationAbsent`, naming N and the newest generation, and writes nothing.", "why": "because a restore falling back to another generation reads a state nobody named", "file": "spec/10-store.md", - "line": 556 + "line": 562 }, { "id": "store.pull.generation-diverged", @@ -3929,7 +3962,7 @@ "statement": "A generation pull into a store holding a syncable file in scope that the generation does not list raises `SyncGenerationDiverged`, naming the file, and writes nothing.", "why": "because a kept unlisted run adds rows the generation never held, and deleting it loses unpushed work", "file": "spec/10-store.md", - "line": 558 + "line": 564 }, { "id": "store.pull.generation-schema", @@ -3940,7 +3973,7 @@ "statement": "A generation pull takes each `schema.json` as the bucket holds it, merged per {{store.pull.schema-merge}}, never refusing on its generation digest.", "why": "because the lattice keeps every column an earlier generation carried, and the merged copy replaces the old one in the bucket", "file": "spec/10-store.md", - "line": 560 + "line": 566 }, { "id": "store.pull.generation-run-state", @@ -3951,7 +3984,7 @@ "statement": "A generation pull holds no `nodes//run-state.json` to {{store.pull.generation-diverged}}: one the generation does not list stays as it is, and any other unlisted file under `nodes/` meets that refusal.", "why": "because a run state summarizes history outside the store root, and a restore predating it otherwise refuses every node that pushed since", "file": "spec/10-store.md", - "line": 562 + "line": 568 }, { "id": "store.pull.run-state-cursor", @@ -3962,7 +3995,7 @@ "statement": "A run opening where a pulled run state records a commit marker for its pipeline and table newer than every local one resumes from that marker's cursor.", "why": "because a collected run takes its manifest's cursor out of the bucket, and a cold node otherwise re-reads the source from its start", "file": "spec/10-store.md", - "line": 564 + "line": 570 }, { "id": "store.probe.inconclusive", @@ -3973,7 +4006,7 @@ "statement": "An unsupported-method response, a forbidden response or a transport error raises `SyncProbeInconclusive` and counts as capability not demonstrated.", "why": "A-store", "file": "spec/10-store.md", - "line": 601 + "line": 607 }, { "id": "store.probe.unproven", @@ -3984,7 +4017,7 @@ "statement": "Declaring `cas` against a backend the probe did not demonstrate raises `SyncCoordinationUnproven` and stops the push.", "why": "A-store", "file": "spec/10-store.md", - "line": 603 + "line": 609 }, { "id": "store.probe.sentinel", @@ -3995,7 +4028,7 @@ "statement": "The probe creates a sentinel under `_contextful/cas-probe/` and demonstrates `cas` when a second create and a stale `If-Match` both fail and a current `If-Match` replaces it, deleting the sentinel after.", "why": "", "file": "spec/10-store.md", - "line": 605 + "line": 611 }, { "id": "store.probe.network-volume", @@ -4006,7 +4039,7 @@ "statement": "A filesystem bucket whose mount type is not apfs, hfs, ext4, xfs, btrfs, zfs, tmpfs or overlay resolves `single-writer` without the sentinel, naming the mount type, and a declared `cas` there meets {{store.probe.unproven}}.", "why": "because the bucket's advisory lock is not a guarantee across the clients of a network share, and a sentinel run from one client cannot observe another", "file": "spec/10-store.md", - "line": 606 + "line": 612 }, { "id": "store.merge.tombstone-owner", @@ -4017,7 +4050,7 @@ "statement": "A tombstone whose owner differs from the owner of the entry it names raises `SyncTombstoneForeign`, and the merge keeps the entry.", "why": "A-store", "file": "spec/10-store.md", - "line": 613 + "line": 619 }, { "id": "store.merge.tombstone-ttl", @@ -4028,7 +4061,7 @@ "statement": "A tombstone leaves the manifest 30 d after its `deleted_at`.", "why": "", "file": "spec/10-store.md", - "line": 615 + "line": 621 }, { "id": "store.merge.retries", @@ -4039,7 +4072,7 @@ "statement": "`[sync] push_retries` bounds the re-commit loop, defaulting to 5 attempts.", "why": "", "file": "spec/10-store.md", - "line": 616 + "line": 622 }, { "id": "store.merge.exhausted", @@ -4050,7 +4083,7 @@ "statement": "Exhausting those retries raises `SyncManifestRebaseExhausted`, reports every uploaded object as already in the bucket, and asks for a re-run.", "why": "A-store", "file": "spec/10-store.md", - "line": 617 + "line": 623 }, { "id": "store.merge.cursor-recency", @@ -4061,7 +4094,7 @@ "statement": "Resolving a cursor by whichever copy was written last raises `SyncCursorConflict`; a cursor resolves through its commit.", "why": "A-run", "file": "spec/10-store.md", - "line": 619 + "line": 625 }, { "id": "store.merge.scoped-union", @@ -4072,7 +4105,7 @@ "statement": "A merge takes each local entry this writer owns or no node owns, and from the remote every other entry; a remote entry it owns and no longer holds leaves with a tombstone.", "why": "A-store", "file": "spec/10-store.md", - "line": 621 + "line": 627 }, { "id": "store.merge.ownership", @@ -4083,7 +4116,7 @@ "statement": "A key's owner is read off the key: a run directory's node segment, a request-ledger file's node, a commit log's node directory, or a run state's node directory. Every other key is unowned and propagates no deletion.", "why": "A-store", "file": "spec/10-store.md", - "line": 623 + "line": 629 }, { "id": "store.lease.ttl", @@ -4094,7 +4127,7 @@ "statement": "A lease is granted for 10 min.", "why": "", "file": "spec/10-store.md", - "line": 632 + "line": 638 }, { "id": "store.lease.renewal", @@ -4105,7 +4138,7 @@ "statement": "A holder renews every 200 s by `If-Match` replace on the ETag it holds.", "why": "", "file": "spec/10-store.md", - "line": 633 + "line": 639 }, { "id": "store.lease.clock-skew", @@ -4116,7 +4149,7 @@ "statement": "Bucket leasing assumes the clocks of two machines differ by 30 s or less, and every expiry judgment reads the judging machine's monotonic clock.", "why": "because correctness rests on the fence, and the skew bound only sets how early a holder stops committing", "file": "spec/10-store.md", - "line": 634 + "line": 640 }, { "id": "store.lease.held", @@ -4127,7 +4160,7 @@ "statement": "A run finding an unexpired lease raises `LeaseHeld`, naming the holder and the expiry, is skipped, and is attempted again at the next reconciliation tick.", "why": "A-store", "file": "spec/10-store.md", - "line": 636 + "line": 642 }, { "id": "store.lease.stale-fence", @@ -4138,7 +4171,7 @@ "statement": "A commit-log create, pointer replace or catalog `UPDATE` losing its condition to a higher fence raises `LeaseFenced`, and the run or snapshot it carried stays unreadable.", "why": "A-store", "file": "spec/10-store.md", - "line": 638 + "line": 644 }, { "id": "store.lease.not-held", @@ -4149,7 +4182,7 @@ "statement": "Releasing a lease another node holds raises `LeaseNotHeld` and leaves the object untouched.", "why": "A-store", "file": "spec/10-store.md", - "line": 640 + "line": 646 }, { "id": "store.lease.cursor-kind", @@ -4160,7 +4193,7 @@ "statement": "A cursor whose kind takes no lease reaching `cursors/` raises `LeaseCursorKindMismatch`.", "why": "A-run", "file": "spec/10-store.md", - "line": 642 + "line": 648 }, { "id": "store.lease.local-node", @@ -4171,7 +4204,7 @@ "statement": "A bucket lease attempted under the node id `local` raises `LeaseNodeIdLocal`, logging the variable that sets a node id; that machine keeps the machine lease.", "why": "P3", "file": "spec/10-store.md", - "line": 644 + "line": 650 }, { "id": "store.lease.acquire", @@ -4182,7 +4215,7 @@ "statement": "Acquisition creates the lease object under `If-None-Match`, or replaces an expired or released one under `If-Match` on its ETag, the fence one past the object's.", "why": "A-store", "file": "spec/10-store.md", - "line": 646 + "line": 652 }, { "id": "store.lease.pointer-fence", @@ -4193,7 +4226,7 @@ "statement": "Taking a table's compaction lease raises the fence stored in its bucket pointer, so a publish carrying a lower fence loses its condition.", "why": "A-store", "file": "spec/10-store.md", - "line": 648 + "line": 654 }, { "id": "store.lease.network-volume", @@ -4204,7 +4237,7 @@ "statement": "Taking a compaction lease on a bucket {{store.probe.network-volume}} resolves to `single-writer` meets {{store.probe.unproven}} naming the mount type, and writes no lease object and no pointer fence.", "why": "because a conditional put holding on one machine only lets two machines each take the lease", "file": "spec/10-store.md", - "line": 650 + "line": 656 }, { "id": "store.lease.commit-log", @@ -4215,7 +4248,7 @@ "statement": "Under a machine lease, a run commits by creating the next `cursors///.json`, and an acquisition creates one carrying its fence; a manifest marked `logged` is readable once that log records it.", "why": "A-store", "file": "spec/10-store.md", - "line": 652 + "line": 658 }, { "id": "store.replicate.write-refused", @@ -4226,7 +4259,7 @@ "statement": "A write verb against a replica raises `ReplicaWriteRefused`, naming the canonical store.", "why": "A-store", "file": "spec/10-store.md", - "line": 670 + "line": 676 }, { "id": "store.replicate.partial-parquet", @@ -4237,7 +4270,7 @@ "statement": "A replica holding a strict subset of a snapshot's Parquet raises `ReplicaPartialParquet` at refresh and leaves that snapshot unpublished.", "why": "A-store", "file": "spec/10-store.md", - "line": 672 + "line": 678 }, { "id": "store.replicate.missing-index", @@ -4248,7 +4281,7 @@ "statement": "A query needing a sidecar or a partition the replica lacks raises `ReplicaMissingIndex`, naming the refresh that supplies it.", "why": "A-store", "file": "spec/10-store.md", - "line": 674 + "line": 680 }, { "id": "store.replicate.sensitive-refused", @@ -4259,7 +4292,7 @@ "statement": "A refresh requesting a replicate-off table raises `ReplicaSensitiveTable`; the consumer reads through the proxying face.", "why": "A-store", "file": "spec/10-store.md", - "line": 676 + "line": 682 }, { "id": "read.register.connection-views", @@ -22784,6 +22817,10 @@ "store.init.repeat", "store.init.posture" ], + [ + "store.init.repeat", + "store.init.store-identity" + ], [ "store.init.posture", "authority.issue.posture-key" @@ -22892,6 +22929,10 @@ "store.push.pointer-carry", "store.lay-out.ancestors" ], + [ + "store.pull.identity-conflict", + "store.init.store-identity" + ], [ "store.pull.schema-merge", "store.reconcile.incompatible" @@ -28062,6 +28103,14 @@ "clause": "store.encrypt.key-unbound", "gloss": "A declared key source names a binding the process lacks." }, + "StoreIdentityConflict": { + "clause": "store.pull.identity-conflict", + "gloss": "A pull would replace one store UUID with another." + }, + "StoreIdentityInvalid": { + "clause": "store.init.identity-invalid", + "gloss": "A store UUID is malformed or owner mint names no initialized root." + }, "StoreIndexColumnAbsent": { "clause": "store.index.column-absent", "gloss": "An index is declared over a column the reconciled schema lacks." diff --git a/spec/status.md b/spec/status.md index 813ea13cb..46b881695 100644 --- a/spec/status.md +++ b/spec/status.md @@ -12,10 +12,10 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | -| `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | +| `store` | 1 | 17 | 240 | 57 | 13 | 20 | 203 | 0 | 203 | | `surface` | 2 | 20 | 106 | 50 | 22 | 0 | 52 | 0 | 52 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2014 | 582 | 188 | 127 | 1367 | 0 | | +| **total** | 19 | 155 | 2017 | 584 | 188 | 127 | 1370 | 0 | | Decision records: 18. @@ -27,11 +27,11 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | --- | --- | --- | --- | --- | --- | | 0 — The test-first gate | 2 | 18 | 6 | passing | open | | 1 — The authority core | 14 | 237 | 185 | passing | closed | -| 2 — The store | 9 | 164 | 132 | passing | closed | +| 2 — The store | 9 | 166 | 134 | passing | closed | | 3 — The run path | 11 | 207 | 144 | passing | closed | | 4 — Ingest | 25 | 496 | 313 | passing | open | | 5 — The read face under enforcement | 17 | 252 | 192 | passing | open | -| 6 — Sync and replicas | 8 | 73 | 68 | passing | closed | +| 6 — Sync and replicas | 8 | 74 | 69 | passing | closed | | 7 — Memory | 6 | 37 | 37 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | @@ -1202,11 +1202,13 @@ Unscheduled operations: 10. | `store.init.default-declaration` | `crates/contextful-cli/tests/integration/init.rs::a_command_without_project_discovers_it_from_a_subdirectory` | performed | | `store.init.discovery` | `crates/contextful-context/tests/integration/init.rs::discovery_takes_the_nearest_declaration_upward` | performed | | `store.init.explicit-project` | `crates/contextful-cli/tests/integration/init.rs::an_explicit_project_uses_the_working_directory` | performed | +| `store.init.identity-invalid` | `spec/pins.toml` | performed | | `store.init.name-conflict` | `crates/contextful-context/tests/integration/init.rs::an_init_naming_another_project_refuses` | performed | | `store.init.name-shape` | `crates/contextful-context/tests/integration/init.rs::a_traversing_or_unsafe_name_refuses_before_any_write` | performed | | `store.init.posture` | `crates/contextful-cli/tests/integration/init.rs::an_init_with_a_posture_lands_its_first_write_unedited` | performed | | `store.init.project-paths` | `crates/contextful-context/tests/integration/init.rs::every_project_path_is_based_on_the_project_directory` | performed | | `store.init.repeat` | `crates/contextful-context/tests/integration/init.rs::a_repeated_init_rewrites_nothing` | performed | +| `store.init.store-identity` | `spec/pins.toml` | performed | | `store.init.undiscovered` | `crates/contextful-context/tests/integration/init.rs::discovery_without_a_named_declaration_refuses` | performed | | `store.lay-out.ancestors` | `crates/contextful-core/tests/integration/store/lay_out.rs::a_snapshot_records_its_ancestors_nearest_first_up_to_the_bound` | performed | | `store.lay-out.catalog-ports` | `crates/contextful-sqlite/tests/integration/derived.rs::both_catalogs_are_reached_through_the_core_ports` | performed | @@ -1262,6 +1264,7 @@ Unscheduled operations: 10. | `store.pull.generation-diverged` | `crates/contextful-sync/tests/integration/generation.rs::a_generation_pull_into_a_store_holding_unlisted_files_refuses` | performed | | `store.pull.generation-run-state` | `crates/contextful-sync/tests/integration/run_state.rs::a_restore_keeps_a_run_state_its_generation_does_not_list` | performed | | `store.pull.generation-schema` | `crates/contextful-sync/tests/integration/generation.rs::a_generation_pull_takes_the_current_schema` | performed | +| `store.pull.identity-conflict` | `spec/pins.toml` | performed | | `store.pull.pointer-advance` | `crates/contextful-sync/tests/integration/converge.rs::a_pull_never_regresses_a_pointer_and_writes_none_until_every_table_verifies` | performed | | `store.pull.pointer-last` | `crates/contextful-sync/tests/integration/pull.rs::a_pointer_is_written_only_once_its_snapshot_is_home` | performed | | `store.pull.run-state-cursor` | `crates/contextful-cli/tests/integration/sync/run_state.rs::a_cold_node_resumes_the_cursor_a_push_carried` | performed | diff --git a/spec/terms/store.toml b/spec/terms/store.toml index b70575f1d..a1ad16d3a 100644 --- a/spec/terms/store.toml +++ b/spec/terms/store.toml @@ -25,6 +25,8 @@ StoreManifestUnreadable = { clause = "store.lay-out.manifest-unreadable", g StoreUnknownTable = { clause = "store.lay-out.unknown-table", gloss = "A table name resolves to no `schema.json` in the tree." } StoreNodeIdInvalid = { clause = "store.lay-out.node-id-shape", gloss = "A resolved node id is too long or outside the path-safe pattern." } StoreProjectNameInvalid = { clause = "store.init.name-shape", gloss = "A project name is not path-safe `/`-separated segments." } +StoreIdentityInvalid = { clause = "store.init.identity-invalid", gloss = "A store UUID is malformed or owner mint names no initialized root." } +StoreIdentityConflict = { clause = "store.pull.identity-conflict", gloss = "A pull would replace one store UUID with another." } StoreProjectConflict = { clause = "store.init.name-conflict", gloss = "An init meets a `contextful.toml` declaring another project, or a project with no name." } StoreProjectUndiscovered = { clause = "store.init.undiscovered", gloss = "A command given no `--project` finds no `contextful.toml` naming a project from its working directory up." } StoreNodeIdShared = { clause = "store.lay-out.node-id-shared", gloss = "A node id is declared in a control-plane configuration applied to many machines." } From 62740a93a6168d8f19b312d9279b1774e7ac3a58 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:53:09 +0800 Subject: [PATCH 086/155] test(memory): require governed served claim writes --- .../contextful-cli/tests/integration/serve.rs | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 01f436f66..46fe6f012 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -106,6 +106,67 @@ fn query() -> Value { json!({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": { "name": "context.query", "arguments": { "sql": "SELECT note_id FROM \"research/notes\"" } } }) } +fn post_claim(addr: &str, token: &str, body: &Value, browser_origin: bool) -> (u16, Value) { + let body = body.to_string(); + let origin = if browser_origin { "Origin: https://console.example\r\n" } else { "" }; + let mut stream = TcpStream::connect(addr).unwrap(); + write!(stream, "POST /memory/claims HTTP/1.1\r\nAuthorization: Bearer {token}\r\n{origin}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); + let mut raw = String::new(); + stream.read_to_string(&mut raw).unwrap(); + let (head, data) = raw.split_once("\r\n\r\n").unwrap(); + (head.split(' ').nth(1).unwrap().parse().unwrap(), serde_json::from_str(data).unwrap_or(Value::Null)) +} + +#[test] +fn served_claim_write_is_durable_actor_bound_and_outside_read_mcp() { + let memory = "\n[[table]]\nname = \"memory/facts\"\nshape = \"memory_facts\"\ncolumns = [\"claim_id\", \"subject\", \"predicate\", \"object\", \"scope\", \"tier\", \"confidence\", \"valid_from\", \"valid_to\", \"evidence\", \"superseded_by\", \"grant_id\", \"agent\"]\n"; + let (dir, public) = project_declaring(memory); + let p = dir.path(); + let mint = |who: &str, task: &str, actions: &[&str]| { + let mut args = vec!["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", who, + "--task", task, "--zone", "on-prem:hq", "--table", "memory/facts", "--ttl", "900"]; + for action in actions { args.extend(["--action", action]); } + stdout(&run(p, &args)) + }; + let alice = mint("user://alice", "session-1", &["read", "write"]); + let bob = mint("user://bob", "session-2", &["read", "write"]); + let reader = mint("user://alice", "session-1", &["read"]); + let claim = json!({ "into": "memory/facts", "actor": "user://alice", "session": "session-1", "dedup_key": "turn-1", + "claim": { "subject": "Northwind", "predicate": "filings", "object": "arrived", "confidence": 0.9, "evidence": [] } }); + let (listener, addr) = serve(p, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, + "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + assert_eq!(post_claim(&addr, &reader, &claim, false).0, 403, "read grant cannot write"); + assert_eq!(post_claim(&addr, &alice, &claim, true).0, 403, "a browser origin cannot reach the write"); + let (status, first) = post_claim(&addr, &alice, &claim, false); + assert_eq!(status, 200, "{first}"); + assert_eq!(first["landed"], true); + assert!(first["scope"].as_str().is_some_and(|scope| scope.contains("alice") && scope.contains("session-1")), "{first}"); + let (status, duplicate) = post_claim(&addr, &alice, &claim, false); + assert_eq!(status, 200, "{duplicate}"); + assert_eq!(duplicate["landed"], false); + let mut borrowed = claim.clone(); + borrowed["actor"] = json!("user://bob"); + assert_eq!(post_claim(&addr, &alice, &borrowed, false).0, 403, "a writer cannot author for another operator"); + borrowed["actor"] = json!("user://alice"); + borrowed["session"] = json!("session-2"); + assert_eq!(post_claim(&addr, &alice, &borrowed, false).0, 403, "a writer cannot pick another session"); + assert_eq!(post_claim(&addr, &bob, &claim, false).0, 403, "the second operator cannot author Alice's claim"); + let write_tool = json!({ "jsonrpc": "2.0", "id": 7, "method": "tools/call", + "params": { "name": "memory.write", "arguments": claim } }); + let (_, mcp) = post(&addr, &write_tool, &alice, None); + assert!(mcp.to_string().contains("no tool"), "{mcp}"); + drop(listener); + let (_restarted, addr) = serve(p, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, + "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let recall = json!({ "jsonrpc": "2.0", "id": 8, "method": "tools/call", + "params": { "name": "memory.recall", "arguments": { "table": "memory/facts", "subject": "Northwind" } } }); + let (status, memory) = post(&addr, &recall, &reader, None); + assert_eq!(status, 200, "{memory}"); + assert_eq!(memory["result"]["structuredContent"]["rows"].as_array().map(Vec::len), Some(1), "{memory}"); + stdout(&run(p, &["token", "revoke", "--principal-class", "delegated"])); + assert_eq!(post_claim(&addr, &alice, &claim, false).0, 401, "revoked authority commits nothing"); +} + #[test] fn served_exchange_without_policy_returns_the_unconfigured_refusal() { let (dir, public) = project(); From 9596d97fb5b3945310604398d982097f4bb254c8 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 17:58:31 +0800 Subject: [PATCH 087/155] test(console): close M12 acceptance and pin draft claims --- crates/acceptance/tests/integration/m12.rs | 12 ++++++++++-- crates/contextful-cli/tests/integration/serve.rs | 8 ++++++-- spec/pins.toml | 5 ++++- spec/status.md | 11 +++++++---- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index b2db1382b..2b247d0c0 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -80,8 +80,13 @@ fn request(address: &str, method: &str, path: &str, token: Option<&str>, body: O } #[test] -#[ignore = "milestone 12 is open: Query live turn integration remains separate"] fn m12_console() { + let console_package = workspace_root().join("apps/console"); + let bundle = console_package.join("dist/server.js"); + let _ = std::fs::remove_file(&bundle); + ok(&Command::new("pnpm").arg("--dir").arg(&console_package).args(["install", "--frozen-lockfile"]).output().unwrap()); + ok(&Command::new("pnpm").arg("--dir").arg(&console_package).arg("build").output().unwrap()); + assert!(bundle.is_file(), "the console build writes its runnable server"); let cf = bin("contextful"); let repo = GitRepo::init(); repo.write("contextful.toml", "authoring_posture = \"per_request\"\n"); @@ -126,7 +131,7 @@ fn m12_console() { }); let stores = json!([{ "id": "field-notes", "label": "Field notes", "endpoint": format!("http://{read_address}"), "auth": "shared" }]); let console = Command::new("node") - .arg(workspace_root().join("apps/console/dist/server.js")) + .arg(&bundle) .args(["--http", "127.0.0.1:0"]) .env("CONTEXTFUL_STORES_JSON", stores.to_string()) .env("FIELD_NOTES_QUERY_TOKEN", &read_token) @@ -144,6 +149,9 @@ fn m12_console() { assert_eq!(request(&console_address, "GET", "/query", None, None).0, 401); assert_eq!(request(&console_address, "GET", "/admin", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); + let (status, admin_page) = request(&console_address, "GET", "/admin", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); + assert_eq!(status, 200); + assert!(String::from_utf8_lossy(&admin_page).contains("id=\"canvas\"")); assert_eq!(request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(QUERY_ACCESS_AUDIENCE)), None).0, 403); let (status, workflows) = request(&console_address, "GET", "/admin/api/workflows", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), None); assert_eq!(status, 200, "{}", String::from_utf8_lossy(&workflows)); diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index d9eeb0d20..c648ab3c9 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -92,10 +92,14 @@ fn control_apply_uses_the_host_registered_task_set() { std::fs::write(root.join("contextful.toml"), format!("authoring_posture = \"per_request\"\n{}", super::derive::host_manifest("word-split"))).unwrap(); let imported = Command::new(&binary).args(["pipeline", "import", "--project", "research"]).current_dir(root).output().unwrap(); stdout(&imported); - std::fs::write(root.join("contextful.toml"), format!("authoring_posture = \"per_request\"\n{}", super::derive::host_manifest("word-split").replace("[pipeline.source]", "schedule = \"every 1h\"\n[pipeline.source]"))).unwrap(); + let draft = super::derive::host_manifest("word-split").replace("[pipeline.source]", "schedule = \"every 1h\"\n[pipeline.source]"); + std::fs::write(root.join("contextful.toml"), format!("authoring_posture = \"per_request\"\n{draft}")).unwrap(); let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); let (_listener, addr) = serve_binary(&binary, root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); - let (status, state) = control(&addr, "POST", "/control/apply", Some(&admin), "{}"); + let edit = json!({ "expected": 1, "document": draft }).to_string(); + let (status, edited) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); + assert_eq!(status, 200, "{edited}"); + let (status, state) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); assert_eq!(status, 200, "{state}"); assert_eq!(state["applied"], json!(2)); } diff --git a/spec/pins.toml b/spec/pins.toml index bb44c6eaa..36c86cc7b 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -27,6 +27,9 @@ # `contextful-spec state` reads this file and the tags and writes spec/status.md. [pin] +"surface.edit.store-draft" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } +"surface.apply.draft-claim" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } +"surface.apply.draft-absent" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } "corpus.state.typescript-tag-pin" = { test = "contextful_spec::ts_pins::a_running_surface_test_tag_performs_its_clause" } "corpus.state.typescript-unfinished" = { test = "contextful_spec::ts_pins::a_skipped_or_todo_surface_test_is_broken" } "assurance.gate.native-test-files" = { test = "contextful_ci::stages::the_surfaces_stage_runs_each_native_typescript_test_file_even_without_a_test_script" } @@ -63,5 +66,5 @@ disclosure = 54 read = 163 run = 261 store = 200 -surface = 57 +surface = 60 topology = 38 diff --git a/spec/status.md b/spec/status.md index 5283266b9..33498627c 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 109 | 50 | 22 | 0 | 57 | 0 | 57 | +| `surface` | 2 | 20 | 109 | 50 | 22 | 0 | 60 | 0 | 60 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2016 | 582 | 188 | 127 | 1371 | 0 | | +| **total** | 19 | 155 | 2016 | 582 | 188 | 127 | 1374 | 0 | | Decision records: 18. @@ -35,9 +35,9 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 7 — Memory | 6 | 37 | 37 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | -| 10 — Cadence and the operator plane | 11 | 71 | 56 | passing | open | +| 10 — Cadence and the operator plane | 11 | 71 | 59 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 49 | 6 | open | open | +| 12 — The operator console | 12 | 49 | 6 | passing | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -1323,6 +1323,8 @@ Unscheduled operations: 10. | `store.reserve.table-namespaces` | `crates/contextful-core/tests/integration/store/reserve.rs::the_visibility_namespace_holds_each_sources_access_tables` | performed | | `store.reserve.taint` | `crates/contextful-context/tests/integration/reserve.rs::a_model_output_row_carries_the_engine_taint_and_no_other_row_does` | performed | | `store.reserve.underscore-namespace` | `crates/contextful-core/tests/integration/store/reserve.rs::the_underscore_namespace_is_the_injected_and_optional_sets` | performed | +| `surface.apply.draft-absent` | `spec/pins.toml` | performed | +| `surface.apply.draft-claim` | `spec/pins.toml` | performed | | `surface.apply.guarded-import` | `crates/contextful-cli/tests/integration/pipeline.rs::the_import_claims_the_first_version_once` | performed | | `surface.apply.local-claim` | `crates/contextful-cli/tests/integration/pipeline.rs::apply_claims_a_version_in_the_local_snapshot_directory` | performed | | `surface.apply.owner-unconfigured` | `crates/contextful-cli/tests/integration/pipeline.rs::an_owner_with_nothing_behind_it_is_refused` | performed | @@ -1358,6 +1360,7 @@ Unscheduled operations: 10. | `surface.dispatch.worker-target` | `crates/contextful-cli/tests/integration/worker.rs::a_killed_worker_s_step_moves_once_and_its_late_callback_is_rejected` | performed | | `surface.edit.connector-upload` | `crates/contextful-cli/tests/integration/pipeline.rs::an_artifact_in_the_document_is_refused` | performed | | `surface.edit.secret-in-document` | `crates/contextful-cli/tests/integration/pipeline.rs::a_credential_in_the_document_is_refused` | performed | +| `surface.edit.store-draft` | `spec/pins.toml` | performed | | `surface.fire.cycle` | `crates/contextful-cli/tests/integration/pipeline.rs::a_cycle_fires_what_is_due_once_and_reports_the_next_instant` | performed | | `surface.fire.cycle-control-source` | `crates/contextful-cli/tests/integration/pipeline.rs::a_cycle_with_no_applied_snapshot_is_refused` | performed | | `surface.fire.cycle-exit` | `crates/contextful-cli/tests/integration/pipeline.rs::a_cycle_with_a_failed_fire_exits_non_zero` | performed | From 264fcfd2aad92b530221dae8d65cc0241465876e Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:31:44 +0800 Subject: [PATCH 088/155] test(surface): cover client transports and listing bounds --- .../contextful-cli/tests/integration/mcp.rs | 13 +++++ packages/client/test/client.test.ts | 48 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 packages/client/test/client.test.ts diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index e9d0e28cd..1cd49aa68 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -237,6 +237,19 @@ fn owner_credential_does_not_cross_independent_stores_with_one_issuer() { assert!(String::from_utf8_lossy(&refused.stderr).contains("OwnerCredentialInvalid")); } +/// The child's working directory selects the store by walking up to the project manifest; finding none raises `StoreSelectorAbsent` and exits before writing any protocol framing. +#[test] +fn a_spawned_server_without_a_project_manifest_refuses_before_framing() { + let (dir, public, token) = project(); + std::fs::remove_file(dir.path().join("contextful.toml")).unwrap(); + let nested = dir.path().join("nested/child"); + std::fs::create_dir_all(&nested).unwrap(); + let hello = [json!({ "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {} })]; + let out = serve(&nested, &["mcp", "--public-key", &public, "--audience", AUD], Some(&token), &hello); + assert!(!out.status.success() && out.stdout.is_empty()); + assert!(String::from_utf8_lossy(&out.stderr).starts_with("StoreSelectorAbsent"), "{}", String::from_utf8_lossy(&out.stderr)); +} + /// The server admits over the stdio pipe it inherited: a credential binding no key admits /// through that pipe, and one binding a holder key and presenting no holder proof refuses. #[test] diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts new file mode 100644 index 000000000..2ebe0fa7a --- /dev/null +++ b/packages/client/test/client.test.ts @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { createClient, listPackKeys, SpawnedClient } from "../src/index.ts"; + +test("four client shapes carry credentials in their assigned transport", async () => { + const calls: Request[] = []; + const fetcher = async (request: Request) => { + calls.push(request); + return Response.json({ jsonrpc: "2.0", id: 1, result: { ok: true } }); + }; + const proxy = createClient({ shape: "same-origin-proxy", baseUrl: "https://console.example.test/query/stores/field-notes", fetch: fetcher }); + const direct = createClient({ shape: "engine-direct", baseUrl: "https://engine.example.test/mcp", token: "viewer-token", fetch: fetcher }); + const binding = createClient({ shape: "service-binding", binding: { fetch: fetcher }, baseUrl: "https://binding.invalid/mcp" }); + assert.deepEqual(await proxy.call("tools/list"), { ok: true }); + assert.deepEqual(await direct.call("tools/list"), { ok: true }); + assert.deepEqual(await binding.call("tools/list"), { ok: true }); + assert.equal(calls[0].headers.get("authorization"), null); + assert.equal(calls[1].headers.get("authorization"), "Bearer viewer-token"); + assert.equal(calls[2].headers.get("authorization"), null); + assert.equal(calls[0].url, "https://console.example.test/query/stores/field-notes/mcp"); +}); + +test("spawned child refuses absent credential and absent ancestor manifest before framing", async () => { + const root = mkdtempSync(join(tmpdir(), "contextful-client-")); + const nested = join(root, "a", "b"); + mkdirSync(nested, { recursive: true }); + assert.throws(() => new SpawnedClient({ cwd: nested, command: "contextful" }), /StdioCredentialMissing/); + assert.throws(() => new SpawnedClient({ cwd: nested, command: "contextful", token: "reader-token" }), /StoreSelectorAbsent/); + writeFileSync(join(root, "contextful.toml"), ""); + const client = new SpawnedClient({ cwd: nested, command: "contextful", token: "reader-token" }); + assert.equal(client.projectRoot, root); + assert.equal(client.command, "contextful"); +}); + +test("listing counts declined keys and flags the first omitted entry", () => { + const keys = Array.from({ length: 1005 }, (_, index) => `packs/${String(index).padStart(4, "0")}`); + keys.push("private/a", "private/b"); + const page = listPackKeys(keys, (key) => !key.startsWith("private/")); + assert.equal(page.entries.length, 1000); + assert.equal(page.truncated, true); + assert.equal(page.declined, 2); + const exact = listPackKeys(keys.slice(0, 1000), () => true); + assert.equal(exact.truncated, false); + assert.equal(exact.entries.length, 1000); +}); From 0da7d3e0eadde459d1becf044513f1225fab5f0b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:39:28 +0800 Subject: [PATCH 089/155] feat(surface): expose four client transports and bounded pack listing --- crates/contextful-cli/src/mcp.rs | 6 +++ packages/client/.gitignore | 1 + packages/client/package.json | 21 ++++++++++ packages/client/pnpm-lock.yaml | 39 +++++++++++++++++++ packages/client/src/browser.ts | 57 ++++++++++++++++++++++++++++ packages/client/src/index.ts | 59 +++++++++++++++++++++++++++++ packages/client/test/client.test.ts | 7 +++- packages/client/tsconfig.json | 16 ++++++++ 8 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 packages/client/.gitignore create mode 100644 packages/client/package.json create mode 100644 packages/client/pnpm-lock.yaml create mode 100644 packages/client/src/browser.ts create mode 100644 packages/client/src/index.ts create mode 100644 packages/client/tsconfig.json diff --git a/crates/contextful-cli/src/mcp.rs b/crates/contextful-cli/src/mcp.rs index ab4e48d81..da951bb50 100644 --- a/crates/contextful-cli/src/mcp.rs +++ b/crates/contextful-cli/src/mcp.rs @@ -35,6 +35,12 @@ pub struct McpArgs { pub fn run(args: McpArgs) -> Result<()> { let (authority, revocation) = args.admit.admit(args.project.as_deref(), "the tool server")?; + if args.project.is_none() { + let cwd = std::env::current_dir()?; + if !cwd.ancestors().any(|dir| dir.join("contextful.toml").is_file()) { + anyhow::bail!("StoreSelectorAbsent: no contextful.toml in {} or any ancestor", cwd.display()); + } + } let located = locate(args.project.as_deref(), args.declaration)?; let authority = if args.owner { let identity = crate::project::owner_identity(&located.project) diff --git a/packages/client/.gitignore b/packages/client/.gitignore new file mode 100644 index 000000000..849ddff3b --- /dev/null +++ b/packages/client/.gitignore @@ -0,0 +1 @@ +dist/ diff --git a/packages/client/package.json b/packages/client/package.json new file mode 100644 index 000000000..87d14c206 --- /dev/null +++ b/packages/client/package.json @@ -0,0 +1,21 @@ +{ + "name": "@contextful/client", + "version": "0.5.0", + "packageManager": "pnpm@10.14.0", + "type": "module", + "exports": { + ".": { + "browser": "./dist/browser.js", + "default": "./dist/index.js" + } + }, + "scripts": { + "typecheck": "tsc --noEmit", + "test": "node --experimental-strip-types --test test/*.test.ts", + "build": "tsc" + }, + "devDependencies": { + "@types/node": "22.20.4", + "typescript": "6.0.3" + } +} diff --git a/packages/client/pnpm-lock.yaml b/packages/client/pnpm-lock.yaml new file mode 100644 index 000000000..f6fa2b274 --- /dev/null +++ b/packages/client/pnpm-lock.yaml @@ -0,0 +1,39 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + '@types/node': + specifier: 22.20.4 + version: 22.20.4 + typescript: + specifier: 6.0.3 + version: 6.0.3 + +packages: + + '@types/node@22.20.4': + resolution: {integrity: sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==} + + typescript@6.0.3: + resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + +snapshots: + + '@types/node@22.20.4': + dependencies: + undici-types: 6.21.0 + + typescript@6.0.3: {} + + undici-types@6.21.0: {} diff --git a/packages/client/src/browser.ts b/packages/client/src/browser.ts new file mode 100644 index 000000000..dbea68a4b --- /dev/null +++ b/packages/client/src/browser.ts @@ -0,0 +1,57 @@ +type Fetch = (request: Request) => Promise; +type Binding = { fetch: Fetch }; + +export type ClientShape = + | { shape: "same-origin-proxy"; baseUrl: string; fetch?: Fetch } + | { shape: "service-binding"; baseUrl: string; binding: Binding } + | { shape: "engine-direct"; baseUrl: string; token: string; fetch?: Fetch }; + +export class ClientError extends Error { + readonly identifier: string; + constructor(identifier: string, message: string) { + super(`${identifier}: ${message}`); + this.identifier = identifier; + this.name = identifier; + } +} + +export interface Client { call(method: string, params?: unknown): Promise } + +function endpoint(shape: ClientShape): string { + const url = new URL(shape.baseUrl); + if (shape.shape === "same-origin-proxy") url.pathname = `${url.pathname.replace(/\/$/, "")}/mcp`; + return url.toString(); +} + +export function createClient(shape: ClientShape): Client { + const request = shape.shape === "service-binding" ? shape.binding.fetch.bind(shape.binding) : shape.fetch ?? fetch; + return { + async call(method, params = {}) { + const headers = new Headers({ "content-type": "application/json", accept: "application/json" }); + if (shape.shape === "engine-direct") { + if (!shape.token.trim()) throw new ClientError("EngineCredentialMissing", "engine-direct requires a scoped viewer token"); + headers.set("authorization", `Bearer ${shape.token}`); + } + const response = await request(new Request(endpoint(shape), { method: "POST", headers, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }) })); + if (!response.ok) throw new ClientError("EngineResponseRefused", `HTTP ${response.status}`); + const message = await response.json() as { result?: unknown; error?: { code?: number; message?: string } }; + if (message.error) throw new ClientError("EngineProtocolRefused", message.error.message ?? String(message.error.code)); + if (!("result" in message)) throw new ClientError("EngineProtocolRefused", "missing JSON-RPC result"); + return message.result; + }, + }; +} + +export type Listing = { entries: T[]; truncated: boolean; declined: number }; + +export function listPackKeys(keys: Iterable, serves: (key: T) => boolean): Listing { + const entries: T[] = []; + let truncated = false; + let declined = 0; + for (const key of keys) { + if (!serves(key)) declined += 1; + else if (entries.length < 1000) entries.push(key); + else truncated = true; + } + return { entries, truncated, declined }; +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts new file mode 100644 index 000000000..e3b1c55e5 --- /dev/null +++ b/packages/client/src/index.ts @@ -0,0 +1,59 @@ +import { spawn } from "node:child_process"; +import { existsSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { ClientError } from "./browser.ts"; +import type { Client } from "./browser.ts"; + +export { ClientError, createClient, listPackKeys } from "./browser.ts"; +export type { Client, ClientShape, Listing } from "./browser.ts"; + +function findProjectRoot(cwd: string): string { + let path = resolve(cwd); + while (true) { + if (existsSync(join(path, "contextful.toml"))) return path; + const parent = dirname(path); + if (parent === path) throw new ClientError("StoreSelectorAbsent", `no contextful.toml above ${cwd}`); + path = parent; + } +} + +export type SpawnedOptions = { cwd: string; command: string; args?: string[]; token?: string }; + +export class SpawnedClient implements Client { + readonly projectRoot: string; + readonly command: string; + private readonly args: string[]; + private readonly token: string; + + constructor(options: SpawnedOptions) { + if (!options.token?.trim()) throw new ClientError("StdioCredentialMissing", "spawned engine requires a capability token"); + this.projectRoot = findProjectRoot(options.cwd); + this.command = options.command; + this.args = options.args ?? []; + this.token = options.token; + } + + async call(method: string, params: unknown = {}): Promise { + const child = spawn(this.command, ["mcp", ...this.args], { + cwd: this.projectRoot, + env: { ...process.env, CONTEXTFUL_TOKEN: this.token }, + stdio: ["pipe", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout.on("data", (data: Buffer) => stdout.push(data)); + child.stderr.on("data", (data: Buffer) => stderr.push(data)); + child.stdin.end(`${JSON.stringify({ jsonrpc: "2.0", id: 1, method, params })}\n`); + const status = await new Promise((resolveExit, reject) => { + child.once("error", reject); + child.once("close", resolveExit); + }); + if (status !== 0) throw new ClientError("EngineProcessRefused", Buffer.concat(stderr).toString("utf8").trim()); + const line = Buffer.concat(stdout).toString("utf8").split("\n").find((text) => text.trim()); + if (!line) throw new ClientError("EngineProtocolRefused", "missing JSON-RPC response"); + const message = JSON.parse(line) as { result?: unknown; error?: { message?: string } }; + if (message.error) throw new ClientError("EngineProtocolRefused", message.error.message ?? "JSON-RPC error"); + if (!("result" in message)) throw new ClientError("EngineProtocolRefused", "missing JSON-RPC result"); + return message.result; + } +} diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts index 2ebe0fa7a..c36d2ee78 100644 --- a/packages/client/test/client.test.ts +++ b/packages/client/test/client.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; @@ -33,6 +33,11 @@ test("spawned child refuses absent credential and absent ancestor manifest befor const client = new SpawnedClient({ cwd: nested, command: "contextful", token: "reader-token" }); assert.equal(client.projectRoot, root); assert.equal(client.command, "contextful"); + const fixture = join(root, "echo-mcp"); + writeFileSync(fixture, "#!/bin/sh\nread line\nprintf '%s\\n' '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"ok\":true}}'\n"); + chmodSync(fixture, 0o700); + const spawned = new SpawnedClient({ cwd: nested, command: fixture, token: "reader-token" }); + assert.deepEqual(await spawned.call("tools/list"), { ok: true }); }); test("listing counts declined keys and flags the first omitted entry", () => { diff --git a/packages/client/tsconfig.json b/packages/client/tsconfig.json new file mode 100644 index 000000000..34bc3856d --- /dev/null +++ b/packages/client/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "strict": true, + "declaration": true, + "rewriteRelativeImportExtensions": true, + "outDir": "dist", + "rootDir": "src", + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} From 0245bef95dc38d8d0a5ba38db864e95190bd0e2a Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:47:49 +0800 Subject: [PATCH 090/155] test(surface): pin process refusals and exercise HTTP client transport --- .../contextful-cli/tests/integration/mcp.rs | 2 ++ packages/client/package.json | 8 ++++++- packages/client/test/client.test.ts | 24 +++++++++++++++++++ spec/pins.toml | 2 +- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/contextful-cli/tests/integration/mcp.rs b/crates/contextful-cli/tests/integration/mcp.rs index 1cd49aa68..a56b463e7 100644 --- a/crates/contextful-cli/tests/integration/mcp.rs +++ b/crates/contextful-cli/tests/integration/mcp.rs @@ -93,6 +93,7 @@ fn a_server_with_no_admissible_credential_writes_no_framing() { } /// The process transport requires a verified capability token; `--owner` selects its signed owner claim. An absent token raises `StdioCredentialMissing` before protocol framing and never resolves to owner context. +// spec: surface.package.stdio-credential@c6b49c5d #[test] fn a_server_with_no_credential_raises_stdio_credential_missing() { let (dir, public, _token) = project(); @@ -238,6 +239,7 @@ fn owner_credential_does_not_cross_independent_stores_with_one_issuer() { } /// The child's working directory selects the store by walking up to the project manifest; finding none raises `StoreSelectorAbsent` and exits before writing any protocol framing. +// spec: surface.package.store-selector@79c745d0 #[test] fn a_spawned_server_without_a_project_manifest_refuses_before_framing() { let (dir, public, token) = project(); diff --git a/packages/client/package.json b/packages/client/package.json index 87d14c206..97fc254a4 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -3,9 +3,15 @@ "version": "0.5.0", "packageManager": "pnpm@10.14.0", "type": "module", + "files": ["dist"], + "types": "./dist/index.d.ts", "exports": { ".": { - "browser": "./dist/browser.js", + "browser": { + "types": "./dist/browser.d.ts", + "default": "./dist/browser.js" + }, + "types": "./dist/index.d.ts", "default": "./dist/index.js" } }, diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts index c36d2ee78..818ec28b1 100644 --- a/packages/client/test/client.test.ts +++ b/packages/client/test/client.test.ts @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { chmodSync, mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; @@ -23,6 +24,29 @@ test("four client shapes carry credentials in their assigned transport", async ( assert.equal(calls[0].url, "https://console.example.test/query/stores/field-notes/mcp"); }); +test("engine-direct sends JSON-RPC to a real HTTP transport with its viewer credential", async () => { + const server = createServer(async (request, response) => { + assert.equal(request.url, "/mcp"); + assert.equal(request.method, "POST"); + assert.equal(request.headers.authorization, "Bearer viewer-token"); + assert.equal(request.headers.accept, "application/json"); + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(chunk); + assert.equal(JSON.parse(Buffer.concat(chunks).toString())["method"], "tools/list"); + response.setHeader("Content-Type", "application/json"); + response.end(JSON.stringify({ jsonrpc: "2.0", id: 1, result: { tools: [] } })); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const address = server.address(); + assert(address && typeof address !== "string"); + const client = createClient({ shape: "engine-direct", baseUrl: `http://127.0.0.1:${address.port}/mcp`, token: "viewer-token" }); + assert.deepEqual(await client.call("tools/list"), { tools: [] }); + } finally { + server.close(); + } +}); + test("spawned child refuses absent credential and absent ancestor manifest before framing", async () => { const root = mkdtempSync(join(tmpdir(), "contextful-client-")); const nested = join(root, "a", "b"); diff --git a/spec/pins.toml b/spec/pins.toml index 3fdfa5daa..d1cc94f53 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -66,5 +66,5 @@ disclosure = 54 read = 163 run = 261 store = 203 -surface = 52 +surface = 53 topology = 38 From 8fd486d756bdf071a2780ca420f7bbce190b86c7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:54:03 +0800 Subject: [PATCH 091/155] test(client): reject in-band MCP tool refusals --- packages/client/test/client.test.ts | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts index 818ec28b1..936d473a9 100644 --- a/packages/client/test/client.test.ts +++ b/packages/client/test/client.test.ts @@ -47,6 +47,24 @@ test("engine-direct sends JSON-RPC to a real HTTP transport with its viewer cred } }); +test("HTTP client rejects an in-band MCP tool refusal", async () => { + const client = createClient({ + shape: "engine-direct", + baseUrl: "https://engine.example.test/mcp", + token: "viewer-token", + fetch: async () => Response.json({ + jsonrpc: "2.0", + id: 1, + result: { + isError: true, + content: [{ type: "text", text: "scope denied" }], + structuredContent: { error: { identifier: "ScopeDenied", message: "scope denied" } }, + }, + }), + }); + await assert.rejects(client.call("tools/call"), { name: "ScopeDenied", message: /scope denied/ }); +}); + test("spawned child refuses absent credential and absent ancestor manifest before framing", async () => { const root = mkdtempSync(join(tmpdir(), "contextful-client-")); const nested = join(root, "a", "b"); @@ -64,6 +82,16 @@ test("spawned child refuses absent credential and absent ancestor manifest befor assert.deepEqual(await spawned.call("tools/list"), { ok: true }); }); +test("spawned client rejects an in-band MCP tool refusal", async () => { + const root = mkdtempSync(join(tmpdir(), "contextful-client-")); + writeFileSync(join(root, "contextful.toml"), ""); + const fixture = join(root, "refused-mcp"); + writeFileSync(fixture, "#!/bin/sh\nread line\nprintf '%s\\n' '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{\"isError\":true,\"content\":[{\"type\":\"text\",\"text\":\"scope denied\"}],\"structuredContent\":{\"error\":{\"identifier\":\"ScopeDenied\",\"message\":\"scope denied\"}}}}'\n"); + chmodSync(fixture, 0o700); + const client = new SpawnedClient({ cwd: root, command: fixture, token: "reader-token" }); + await assert.rejects(client.call("tools/call"), { name: "ScopeDenied", message: /scope denied/ }); +}); + test("listing counts declined keys and flags the first omitted entry", () => { const keys = Array.from({ length: 1005 }, (_, index) => `packs/${String(index).padStart(4, "0")}`); keys.push("private/a", "private/b"); From 8f561fc1dff6058956d902734562ccb37144aca2 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 13:54:32 +0800 Subject: [PATCH 092/155] fix(client): reject in-band MCP tool refusals --- packages/client/src/browser.ts | 14 +++++++++++++- packages/client/src/index.ts | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/packages/client/src/browser.ts b/packages/client/src/browser.ts index dbea68a4b..8c00fd366 100644 --- a/packages/client/src/browser.ts +++ b/packages/client/src/browser.ts @@ -17,6 +17,18 @@ export class ClientError extends Error { export interface Client { call(method: string, params?: unknown): Promise } +export function toolResult(result: unknown): unknown { + if (typeof result !== "object" || result === null || !("isError" in result) || result.isError !== true) return result; + const structured = "structuredContent" in result ? result.structuredContent : undefined; + const error = typeof structured === "object" && structured !== null && "error" in structured ? structured.error : undefined; + const detail = typeof error === "object" && error !== null ? error : undefined; + const identifier = detail && "identifier" in detail && typeof detail.identifier === "string" ? detail.identifier : "EngineToolRefused"; + const content = "content" in result && Array.isArray(result.content) ? result.content : []; + const text = content.find((item: unknown) => typeof item === "object" && item !== null && "type" in item && item.type === "text" && "text" in item && typeof item.text === "string") as { text: string } | undefined; + const message = detail && "message" in detail && typeof detail.message === "string" ? detail.message : text?.text ?? "MCP tool refused"; + throw new ClientError(identifier, message); +} + function endpoint(shape: ClientShape): string { const url = new URL(shape.baseUrl); if (shape.shape === "same-origin-proxy") url.pathname = `${url.pathname.replace(/\/$/, "")}/mcp`; @@ -37,7 +49,7 @@ export function createClient(shape: ClientShape): Client { const message = await response.json() as { result?: unknown; error?: { code?: number; message?: string } }; if (message.error) throw new ClientError("EngineProtocolRefused", message.error.message ?? String(message.error.code)); if (!("result" in message)) throw new ClientError("EngineProtocolRefused", "missing JSON-RPC result"); - return message.result; + return toolResult(message.result); }, }; } diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index e3b1c55e5..3dc36967c 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -1,7 +1,7 @@ import { spawn } from "node:child_process"; import { existsSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; -import { ClientError } from "./browser.ts"; +import { ClientError, toolResult } from "./browser.ts"; import type { Client } from "./browser.ts"; export { ClientError, createClient, listPackKeys } from "./browser.ts"; @@ -54,6 +54,6 @@ export class SpawnedClient implements Client { const message = JSON.parse(line) as { result?: unknown; error?: { message?: string } }; if (message.error) throw new ClientError("EngineProtocolRefused", message.error.message ?? "JSON-RPC error"); if (!("result" in message)) throw new ClientError("EngineProtocolRefused", "missing JSON-RPC result"); - return message.result; + return toolResult(message.result); } } From 815acf5c21cae449967dd300b8d57dafc970f1ab Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:04:53 +0800 Subject: [PATCH 093/155] test(client): require explicit owner flag on spawned child --- packages/client/test/client.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/client/test/client.test.ts b/packages/client/test/client.test.ts index 936d473a9..c1a24776d 100644 --- a/packages/client/test/client.test.ts +++ b/packages/client/test/client.test.ts @@ -82,6 +82,17 @@ test("spawned child refuses absent credential and absent ancestor manifest befor assert.deepEqual(await spawned.call("tools/list"), { ok: true }); }); +test("spawned owner sends an explicit flag with its credential", async () => { + const root = mkdtempSync(join(tmpdir(), "contextful-client-owner-")); + writeFileSync(join(root, "contextful.toml"), ""); + assert.throws(() => new SpawnedClient({ cwd: root, command: "contextful", owner: true }), /StdioCredentialMissing/); + const fixture = join(root, "owner-mcp"); + writeFileSync(fixture, `#!/usr/bin/env node\nprocess.stdin.resume();\nprocess.stdin.once("data", () => process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id: 1, result: { owner: process.argv.includes("--owner"), token: process.env.CONTEXTFUL_TOKEN } }) + "\\n"));\n`); + chmodSync(fixture, 0o700); + const client = new SpawnedClient({ cwd: root, command: fixture, token: "signed-owner-claim", owner: true }); + assert.deepEqual(await client.call("tools/list"), { owner: true, token: "signed-owner-claim" }); +}); + test("spawned client rejects an in-band MCP tool refusal", async () => { const root = mkdtempSync(join(tmpdir(), "contextful-client-")); writeFileSync(join(root, "contextful.toml"), ""); From f953343701b3cc9bd2bacee530fa63ea56d83353 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:13:40 +0800 Subject: [PATCH 094/155] test(console): bind drafts and select admin store --- apps/console/test/console.test.ts | 16 ++++++++++++++++ .../tests/integration/control.rs | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 26da13e41..1d5fd8ca0 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -297,3 +297,19 @@ test("Admin pack listing reaches only the injected listing adapter", async () => assert.deepEqual(await response.json(), { entries: ["packs/a.toml"], truncated: false, declined: 2 }); assert.deepEqual(calls, ["field-notes:packs/"]); }); + +test("Admin names the selected store on every operational request and shows run outcomes", async () => { + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey }, + stores: [{ id: "one", label: "Store one" }, { id: "two", label: "Store two" }], + adminCapability: "server-secret", turn: async () => ({ answer: "", sources: [], widgets: [] }), + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const page = await (await app.fetch(new Request("https://console.example/admin", { headers: access(adminAudience) }))).text(); + assert.match(page, / Value { } fn control(addr: &str, method: &str, path: &str, token: Option<&str>, body: &str) -> (u16, Value) { + control_headers(addr, method, path, token, body, "") +} + +fn control_headers(addr: &str, method: &str, path: &str, token: Option<&str>, body: &str, headers: &str) -> (u16, Value) { let mut stream = TcpStream::connect(addr).unwrap(); let auth = token.map(|token| format!("Authorization: Bearer {token}\r\n")).unwrap_or_default(); - write!(stream, "{method} {path} HTTP/1.1\r\n{auth}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); + write!(stream, "{method} {path} HTTP/1.1\r\n{auth}{headers}Content-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); let mut raw = String::new(); stream.read_to_string(&mut raw).unwrap(); let (head, answer) = raw.split_once("\r\n\r\n").unwrap(); (head.split(' ').nth(1).unwrap().parse().unwrap(), serde_json::from_str(answer).unwrap()) } +fn attestation(path: &str, body: &str, operator: &str, at: i64, nonce: &str) -> String { + let digest = format!("{:x}", Sha256::digest(body.as_bytes())); + let mut mac = Hmac::::new_from_slice(b"separate-attestation-secret").unwrap(); + mac.update(format!("POST\n{path}\n{digest}\n{operator}\n{at}\n{nonce}").as_bytes()); + let signature = mac.finalize().into_bytes().iter().map(|byte| format!("{byte:02x}")).collect::(); + format!("X-Contextful-Operator: {operator}\r\nX-Contextful-Operator-Time: {at}\r\nX-Contextful-Operator-Nonce: {nonce}\r\nX-Contextful-Operator-Signature: {signature}\r\n") +} + +#[test] +fn admin_changes_require_fresh_console_attestation_and_record_the_operator() { + let (dir, public) = project(); + let root = dir.path(); + let original = "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1h\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); + stdout(&run(root, &["pipeline", "import", "--project", "research"])); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://service@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public, "--control-attestation-secret", "separate-attestation-secret"]); + assert_eq!(control(&addr, "GET", "/control/record", Some(&admin), "").0, 200); + let edit = json!({ "expected": 1, "document": original.replace("every 1h", "every 1d") }).to_string(); + assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edit).0, 403); + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + let stale = attestation("/control/edit", &edit, "operator-a", now - 120, "stale"); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &stale).0, 403); + let forged = attestation("/control/edit", &edit, "operator-a", now, "forged").replace("Operator: operator-a", "Operator: operator-b"); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &forged).0, 403); + let signed = attestation("/control/edit", &edit, "operator-a", now, "edit-once"); + let (status, saved) = control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed); + assert_eq!(status, 200, "{saved}"); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); + let apply = json!({ "expected": 1, "nonce": saved["nonce"] }).to_string(); + let signed = attestation("/control/apply", &apply, "operator-a", now, "apply-once"); + let (status, result) = control_headers(&addr, "POST", "/control/apply", Some(&admin), &apply, &signed); + assert_eq!(status, 200, "{result}"); + let entries = contextful_policy::audit::entries(&root.join(".contextful/audit")).unwrap(); + assert!(entries.iter().any(|entry| entry.attributes["contextful.operator.subject"] == "operator-a" && entry.attributes["contextful.control.operation"] == "apply")); + let (status, record) = control(&addr, "GET", "/control/record", Some(&admin), ""); + assert_eq!(status, 200, "{record}"); + assert!(record["entries"].as_array().unwrap().iter().any(|entry| entry["attributes"]["contextful.operator.subject"] == "operator-a")); +} + #[test] fn control_routes_require_admin_and_project_applied_workflows() { let (dir, public) = project(); From 5af25686ce06f90a0f7cdf9d79bc361d1fc05497 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:16:10 +0800 Subject: [PATCH 096/155] feat(client): pass signed owner mode to spawned engine --- packages/client/src/index.ts | 6 ++++-- spec/status.md | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 3dc36967c..e37266fa0 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -17,13 +17,14 @@ function findProjectRoot(cwd: string): string { } } -export type SpawnedOptions = { cwd: string; command: string; args?: string[]; token?: string }; +export type SpawnedOptions = { cwd: string; command: string; args?: string[]; token?: string; owner?: boolean }; export class SpawnedClient implements Client { readonly projectRoot: string; readonly command: string; private readonly args: string[]; private readonly token: string; + private readonly owner: boolean; constructor(options: SpawnedOptions) { if (!options.token?.trim()) throw new ClientError("StdioCredentialMissing", "spawned engine requires a capability token"); @@ -31,10 +32,11 @@ export class SpawnedClient implements Client { this.command = options.command; this.args = options.args ?? []; this.token = options.token; + this.owner = options.owner ?? false; } async call(method: string, params: unknown = {}): Promise { - const child = spawn(this.command, ["mcp", ...this.args], { + const child = spawn(this.command, ["mcp", ...(this.owner ? ["--owner"] : []), ...this.args], { cwd: this.projectRoot, env: { ...process.env, CONTEXTFUL_TOKEN: this.token }, stdio: ["pipe", "pipe", "pipe"], diff --git a/spec/status.md b/spec/status.md index 46b881695..6c232c69d 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 240 | 57 | 13 | 20 | 203 | 0 | 203 | -| `surface` | 2 | 20 | 106 | 50 | 22 | 0 | 52 | 0 | 52 | +| `surface` | 2 | 20 | 106 | 50 | 22 | 0 | 54 | 0 | 53 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2017 | 584 | 188 | 127 | 1370 | 0 | | +| **total** | 19 | 155 | 2017 | 584 | 188 | 127 | 1372 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 49 | 1 | open | open | +| 12 — The operator console | 12 | 49 | 3 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -1370,6 +1370,8 @@ Unscheduled operations: 10. | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | | `surface.package.owner-flag` | `crates/contextful-cli/tests/integration/mcp.rs::explicit_owner_requires_a_signed_credential_for_the_selected_store` | performed | +| `surface.package.stdio-credential` | `crates/contextful-cli/tests/integration/mcp.rs::a_server_with_no_credential_raises_stdio_credential_missing` | performed | +| `surface.package.store-selector` | `crates/contextful-cli/tests/integration/mcp.rs::a_spawned_server_without_a_project_manifest_refuses_before_framing` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | | `surface.reconcile.loopback-only` | `crates/contextful-cli/tests/integration/pipeline.rs::a_control_url_outside_loopback_arms_nothing` | performed | From 4dc5b530d2ce0bb3c78fd59a180e91f802c648b5 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:25:41 +0800 Subject: [PATCH 097/155] feat(console): bind Admin changes to attested editor and show operational record --- Cargo.lock | 1 + apps/console/src/control.ts | 37 ++++++---- apps/console/src/entry.ts | 3 +- apps/console/src/index.ts | 11 ++- apps/console/test/control.test.ts | 33 ++++++++- crates/acceptance/tests/integration/m12.rs | 4 +- crates/contextful-agent/src/http.rs | 10 ++- .../tests/integration/http.rs | 2 +- crates/contextful-cli/src/cadence.rs | 11 +-- crates/contextful-cli/src/serve.rs | 74 +++++++++++++++++-- .../contextful-cli/tests/integration/serve.rs | 43 +++++++---- crates/contextful-engine/src/control.rs | 11 +++ spec/70-control-plane.md | 4 +- spec/cards/surface.md | 3 +- spec/spec.lock.json | 25 +++++-- spec/status.md | 6 +- spec/terms/surface.toml | 1 + 17 files changed, 213 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92784ba16..8524e2252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -835,6 +835,7 @@ dependencies = [ "contextful-sync", "contextful-wasm", "ed25519-dalek", + "hmac 0.12.1", "jsonwebtoken", "serde", "serde_json", diff --git a/apps/console/src/control.ts b/apps/console/src/control.ts index cf8d2a032..543bcb084 100644 --- a/apps/console/src/control.ts +++ b/apps/console/src/control.ts @@ -1,7 +1,8 @@ import type { ConsoleAdapters } from "./index.ts"; +import { createHash, createHmac, randomBytes } from "node:crypto"; type ControlStore = { id: string; endpoint: string }; -type ControlOptions = { stores: ControlStore[]; capability?: string; fetcher?: typeof fetch }; +type ControlOptions = { stores: ControlStore[]; capability?: string; attestationSecret?: string; fetcher?: typeof fetch }; function unavailable(): never { throw new Error("ConsoleAdapterUnavailable"); @@ -17,40 +18,50 @@ function selected(stores: ControlStore[], id: string | null): ControlStore { throw new Error("ConsoleStoreSelectionRequired"); } -function controlDocument(document: unknown, editing: boolean): { store: string | null; body: { expected: number; document?: string } } { +function controlDocument(document: unknown, editing: boolean): { store: string | null; body: { expected: number; document?: string; nonce?: string } } { if (document === null || typeof document !== "object" || Array.isArray(document)) throw new Error("ConsoleRequestMalformed"); const value = document as Record; if (value.store !== undefined && typeof value.store !== "string") throw new Error("ConsoleRequestMalformed"); if (!Number.isSafeInteger(value.expected) || (value.expected as number) < 1) throw new Error("ConsoleRequestMalformed"); if (editing && typeof value.document !== "string") throw new Error("ConsoleRequestMalformed"); + if (!editing && (typeof value.nonce !== "string" || !/^[0-9a-f]{48}$/.test(value.nonce))) throw new Error("ConsoleRequestMalformed"); const keys = Object.keys(value); - if (keys.some((key) => key !== "store" && key !== "expected" && (key !== "document" || !editing))) throw new Error("ConsoleRequestMalformed"); + if (keys.some((key) => key !== "store" && key !== "expected" && (key !== "document" || !editing) && (key !== "nonce" || editing))) throw new Error("ConsoleRequestMalformed"); return { store: typeof value.store === "string" ? value.store : null, - body: editing ? { expected: value.expected as number, document: value.document as string } : { expected: value.expected as number } }; + body: editing ? { expected: value.expected as number, document: value.document as string } : { expected: value.expected as number, nonce: value.nonce as string } }; } -export function createLiveControl({ stores, capability, fetcher = fetch }: ControlOptions): ConsoleAdapters["control"] { - async function call(store: ControlStore, path: string, token: string, body?: object): Promise { +export function createLiveControl({ stores, capability, attestationSecret, fetcher = fetch }: ControlOptions): ConsoleAdapters["control"] { + async function call(store: ControlStore, path: string, token: string, body?: object, subject?: string): Promise { + const serialized = body ? JSON.stringify(body) : undefined; + const headers: Record = { Authorization: `Bearer ${token}`, ...(body ? { "Content-Type": "application/json" } : {}) }; + if (body && subject && attestationSecret) { + const timestamp = Math.floor(Date.now() / 1000).toString(); + const nonce = randomBytes(16).toString("hex"); + const digest = createHash("sha256").update(serialized!).digest("hex"); + const signature = createHmac("sha256", attestationSecret).update(`POST\n${path}\n${digest}\n${subject}\n${timestamp}\n${nonce}`).digest("hex"); + Object.assign(headers, { "X-Contextful-Operator": subject, "X-Contextful-Operator-Time": timestamp, "X-Contextful-Operator-Nonce": nonce, "X-Contextful-Operator-Signature": signature }); + } const response = await fetcher(new URL(path, store.endpoint), { method: body ? "POST" : "GET", - headers: { Authorization: `Bearer ${token}`, ...(body ? { "Content-Type": "application/json" } : {}) }, - ...(body ? { body: JSON.stringify(body) } : {}), + headers, + ...(serialized ? { body: serialized } : {}), }); if (!response.ok) throw new Error(`ConsoleControlRefused:${response.status}`); return response.json(); } return { workflows: async (_operator, id) => capability ? call(selected(stores, id), "/control/workflows", capability) : unavailable(), - record: async () => unavailable(), - edit: async (document, token) => { + record: async (_operator, id) => capability ? call(selected(stores, id), "/control/record", capability) : unavailable(), + edit: async (document, token, operator) => { if (!capability || token !== capability) return unavailable(); const input = controlDocument(document, true); - return call(selected(stores, input.store), "/control/edit", token, input.body); + return call(selected(stores, input.store), "/control/edit", token, input.body, operator.subject); }, - apply: async (document, token) => { + apply: async (document, token, operator) => { if (!capability || token !== capability) return unavailable(); const input = controlDocument(document, false); - return call(selected(stores, input.store), "/control/apply", token, input.body); + return call(selected(stores, input.store), "/control/apply", token, input.body, operator.subject); }, }; } diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index a872dc97a..73f4eec86 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -86,7 +86,8 @@ async function main(): Promise { let adapters: Pick & { read?: ConsoleAdapters["read"] } = { ...unavailableAdapters(), turn: modulePath ? unavailableAdapters().turn : createLiveTurn({ stores: registry.entries, env: process.env }), - control: createLiveControl({ stores: registry.entries, capability: process.env.CONTEXTFUL_ADMIN_CAPABILITY }), + control: createLiveControl({ stores: registry.entries, capability: process.env.CONTEXTFUL_ADMIN_CAPABILITY, + attestationSecret: process.env.CONTEXTFUL_ADMIN_CAPABILITY ? required("CONTEXTFUL_CONTROL_ATTESTATION_SECRET") : undefined }), }; if (modulePath) { const absolute = isAbsolute(modulePath) ? modulePath : resolve(modulePath); diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index bde6507a8..64013f5e6 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -276,6 +276,7 @@ export function createConsole(adapters: ConsoleAdapters): { fetch: (request: Req } else { const store = url.searchParams.get("store"); if (store !== null && !adapters.stores.some((entry) => entry.id === store)) return refusal("ConsoleRequestMalformed", 400); + if (request.method === "GET" && path === "/admin/api/stores") return json(adapters.stores); if (request.method === "GET" && path === "/admin/api/workflows") return json(await adapters.control.workflows(operator, store)); if (request.method === "GET" && path === "/admin/api/record") return json(await adapters.control.record(operator, store)); if (request.method === "GET" && path === "/admin/api/packs") { @@ -310,8 +311,10 @@ function draw(widget){const props=widget.props??{};if(widget.component==='table. form.addEventListener('submit',async event=>{event.preventDefault();const question=document.getElementById('question').value.trim();if(!question)return;const response=await fetch('/query/api/ask',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({store:store.value,question})});const answer=await response.json();transcript.replaceChildren();const item=document.createElement('article');item.textContent=question+'\n'+(answer.answer??answer.error?.identifier??'No answer');transcript.append(item);widgets.replaceChildren();for(const widget of answer.widgets??[])widgets.append(draw(widget));if(answer.sources?.length){const sources=document.createElement('p');sources.textContent='Sources: '+answer.sources.map(x=>x.title??x.url).join(', ');transcript.append(sources)}}); `; -const adminPage = `Admin · Contextful${sharedStyle}

Store operations

Pipelines, schedules, steps and run outcomes come from the store.

Workflow canvas

Operational record

Control document

`; diff --git a/apps/console/test/control.test.ts b/apps/console/test/control.test.ts index 6fb1a8177..b41eed058 100644 --- a/apps/console/test/control.test.ts +++ b/apps/console/test/control.test.ts @@ -1,6 +1,6 @@ import test from "node:test"; import assert from "node:assert/strict"; -import { generateKeyPairSync, sign } from "node:crypto"; +import { createHash, createHmac, generateKeyPairSync, sign } from "node:crypto"; import { createConsole } from "../src/index.ts"; import { createLiveControl } from "../src/control.ts"; @@ -18,7 +18,8 @@ test("Admin route retrieves applied workflow state and applies through the serve calls.push({ path: url.pathname, authorization: new Headers(init?.headers).get("authorization"), body: String(init?.body ?? "") }); return Response.json(url.pathname === "/control/workflows" ? { applied: 1, pipelines: [{ id: "filings-flow", tables: ["filings"], schedule: "every 1h" }], runs: {} } : - url.pathname === "/control/edit" ? { expected: 1 } : { applied: 2 }); + url.pathname === "/control/record" ? { entries: [], truncated: false, declined: 0 } : + url.pathname === "/control/edit" ? { expected: 1, nonce: "a".repeat(48) } : { applied: 2 }); }; const control = createLiveControl({ stores: [{ id: "field-notes", endpoint: "http://127.0.0.1:9999" }], capability: "engine-admin-secret", fetcher }); const app = createConsole({ @@ -30,19 +31,22 @@ test("Admin route retrieves applied workflow state and applies through the serve const listing = await app.fetch(new Request("https://console.example/admin/api/workflows", { headers })); assert.equal(listing.status, 200); assert.equal((await listing.json() as { pipelines: { id: string }[] }).pipelines[0].id, "filings-flow"); + const record = await app.fetch(new Request("https://console.example/admin/api/record", { headers })); + assert.equal(record.status, 200); const edit = await app.fetch(new Request("https://console.example/admin/api/edit", { method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", expected: 1, document: "[[pipeline]]\nid = \"filings-flow\"" }), })); assert.equal(edit.status, 200); const apply = await app.fetch(new Request("https://console.example/admin/api/apply", { - method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", expected: 1 }), + method: "POST", headers: { ...headers, origin: "https://console.example" }, body: JSON.stringify({ store: "field-notes", expected: 1, nonce: "a".repeat(48) }), })); assert.equal(apply.status, 200); assert.deepEqual(calls, [ { path: "/control/workflows", authorization: "Bearer engine-admin-secret", body: "" }, + { path: "/control/record", authorization: "Bearer engine-admin-secret", body: "" }, { path: "/control/edit", authorization: "Bearer engine-admin-secret", body: '{"expected":1,"document":"[[pipeline]]\\nid = \\"filings-flow\\""}' }, - { path: "/control/apply", authorization: "Bearer engine-admin-secret", body: '{"expected":1}' }, + { path: "/control/apply", authorization: "Bearer engine-admin-secret", body: '{"expected":1,"nonce":"' + "a".repeat(48) + '"}' }, ]); assert.doesNotMatch(JSON.stringify(await apply.json()), /engine-admin-secret/); }); @@ -55,3 +59,24 @@ test("Admin control refuses ambiguous stores", async () => { await assert.rejects(control.workflows({ subject: "operator", grants: new Set(["admin"]) }, null), /ConsoleStoreSelectionRequired/); await assert.rejects(control.edit({ expected: 1, document: "" }, "secret", { subject: "operator", grants: new Set(["admin"]) }), /ConsoleStoreSelectionRequired/); }); + +test("the live control adapter signs the verified subject and exact request body", async () => { + let signed = false; + const control = createLiveControl({ stores: [{ id: "one", endpoint: "http://127.0.0.1:1" }], + capability: "admin-capability", attestationSecret: "separate-attestation-secret", + fetcher: async (input, init) => { + const headers = new Headers(init?.headers); + const body = String(init?.body); + const subject = headers.get("x-contextful-operator"); + assert.equal(subject, "verified-operator"); + const canonical = `POST\n${new URL(String(input)).pathname}\n${createHash("sha256").update(body).digest("hex")}\n${subject}\n${headers.get("x-contextful-operator-time")}\n${headers.get("x-contextful-operator-nonce")}`; + assert.equal(headers.get("x-contextful-operator-signature"), createHmac("sha256", "separate-attestation-secret").update(canonical).digest("hex")); + signed = true; + return Response.json({ expected: 1, nonce: "a".repeat(48) }); + }, + }); + await control.edit({ store: "one", expected: 1, document: "" }, "admin-capability", { subject: "verified-operator", grants: new Set(["admin"]) }); + assert.equal(signed, true); + await assert.rejects(control.edit({ store: "one", expected: 1, document: "", operator: "forged" }, "admin-capability", + { subject: "verified-operator", grants: new Set(["admin"]) }), /ConsoleRequestMalformed/); +}); diff --git a/crates/acceptance/tests/integration/m12.rs b/crates/acceptance/tests/integration/m12.rs index 2b247d0c0..7bf6d4fd1 100644 --- a/crates/acceptance/tests/integration/m12.rs +++ b/crates/acceptance/tests/integration/m12.rs @@ -109,6 +109,7 @@ fn m12_console() { ])); let read = Command::new(&cf) .args(["serve", "--http", "127.0.0.1:0", "--max-in-flight", "2", "--project", "research", "--public-key", &public, "--audience", STORE_AUDIENCE]) + .env("CONTEXTFUL_CONTROL_ATTESTATION_SECRET", "separate-attestation-secret") .current_dir(&repo.root) .stderr(Stdio::piped()) .spawn() @@ -136,6 +137,7 @@ fn m12_console() { .env("CONTEXTFUL_STORES_JSON", stores.to_string()) .env("FIELD_NOTES_QUERY_TOKEN", &read_token) .env("CONTEXTFUL_ADMIN_CAPABILITY", &admin_capability) + .env("CONTEXTFUL_CONTROL_ATTESTATION_SECRET", "separate-attestation-secret") .env("CONTEXTFUL_ACCESS_JWKS_URL", jwks.url("/certs")) .env("CONTEXTFUL_ACCESS_ISSUER", ACCESS_ISSUER) .env("CONTEXTFUL_QUERY_ACCESS_AUDIENCE", QUERY_ACCESS_AUDIENCE) @@ -166,7 +168,7 @@ fn m12_console() { let edited: Value = serde_json::from_slice(&edited).unwrap(); assert_eq!(edited["expected"], 1, "{edited}"); let (status, applied) = request(&console_address, "POST", "/admin/api/apply", Some(&access_token(ADMIN_ACCESS_AUDIENCE)), - Some(&json!({ "store": "field-notes", "expected": 1 }))); + Some(&json!({ "store": "field-notes", "expected": 1, "nonce": edited["nonce"] }))); assert_eq!(status, 200, "{}", String::from_utf8_lossy(&applied)); let applied: Value = serde_json::from_slice(&applied).unwrap(); assert_eq!(applied["applied"], 2, "{applied}"); diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 651e00c38..3ef5b8e0d 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -44,6 +44,7 @@ pub const HEALTH_PATH: &str = "/health"; /// The store-published workflow state and validated control claim routes. pub const WORKFLOWS_PATH: &str = "/control/workflows"; +pub const RECORD_PATH: &str = "/control/record"; pub const EDIT_PATH: &str = "/control/edit"; pub const APPLY_PATH: &str = "/control/apply"; @@ -199,7 +200,7 @@ pub struct Admitting<'a, C> { pub struct HttpFace<'a, C> { tools: Tools<'a>, admitting: Admitting<'a, C>, - control: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, + control: Option<&'a (dyn Fn(&HttpRequest, &AdmittedAuthority) -> HttpResponse + Sync)>, ceiling: usize, in_flight: AtomicUsize, exchange: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, @@ -254,7 +255,7 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { } /// Attach the store's control API without adding tools to the closed read face. - pub fn with_control(mut self, control: &'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)) -> Self { + pub fn with_control(mut self, control: &'a (dyn Fn(&HttpRequest, &AdmittedAuthority) -> HttpResponse + Sync)) -> Self { self.control = Some(control); self } @@ -354,8 +355,9 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { (MCP_PATH, _) => { HttpResponse::message(405, "`/mcp` answers POST; the face holds no session and opens no server stream").with("Allow", "POST") } - (WORKFLOWS_PATH, "GET") | (EDIT_PATH, "POST") | (APPLY_PATH, "POST") if self.control.is_some() => self.control(request), + (WORKFLOWS_PATH, "GET") | (RECORD_PATH, "GET") | (EDIT_PATH, "POST") | (APPLY_PATH, "POST") if self.control.is_some() => self.control(request), (WORKFLOWS_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/workflows` answers GET").with("Allow", "GET"), + (RECORD_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/record` answers GET").with("Allow", "GET"), (EDIT_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/edit` answers POST").with("Allow", "POST"), (APPLY_PATH, _) if self.control.is_some() => HttpResponse::message(405, "`/control/apply` answers POST").with("Allow", "POST"), ("/auth/exchange", "POST") if self.exchange.is_some() => self.exchange.expect("checked above")(request), @@ -399,7 +401,7 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { if let Err(error) = effect_boundary(&authority, &boundary) { return unadmitted(&error); } - self.control.expect("the route is installed only with a control handler")(request) + self.control.expect("the route is installed only with a control handler")(request, &authority) } fn admit(&self, request: &HttpRequest) -> Result<(AdmittedAuthority, RevocationState), HttpResponse> { diff --git a/crates/contextful-agent/tests/integration/http.rs b/crates/contextful-agent/tests/integration/http.rs index 6e19e38fa..d8caedc85 100644 --- a/crates/contextful-agent/tests/integration/http.rs +++ b/crates/contextful-agent/tests/integration/http.rs @@ -135,7 +135,7 @@ fn control_apply_rechecks_revocation_before_dispatch() { }; let admitting = Admitting { checkpoint: &f.checkpoint, audience: AUD, revocation: &revocation }; let dispatched = AtomicBool::new(false); - let handler = |_: &HttpRequest| { + let handler = |_: &HttpRequest, _: &contextful_policy::verify::AdmittedAuthority| { dispatched.store(true, Ordering::SeqCst); contextful_agent::http::HttpResponse::json(200, &json!({ "applied": 2 })) }; diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index 57009bace..50cdb28de 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -307,22 +307,23 @@ fn validated_draft(document: &str, declaration: &Path, tasks: &Tasks) -> Result< } /// Store one validated, version-bound control draft without changing the applied pointer. -pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected: u64, document: &str, tasks: &Tasks) -> Result { +pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected: u64, document: &str, operator: &str, tasks: &Tasks) -> Result { let (located, _, control) = located(project, declaration)?; let snapshots = owner(&control)?; snapshots.initialized()?; let text = validated_draft(document, &located.declaration, tasks)?; - snapshots.save_draft(&Draft { expected, document: text })?; - Ok(json!({ "expected": expected })) + let draft = Draft::new(expected, text, operator.to_owned())?; + snapshots.save_draft(&draft)?; + Ok(json!({ "expected": expected, "nonce": draft.nonce })) } /// Revalidate the saved draft and claim it only at the version the editor read. -pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, tasks: &Tasks) -> Result<()> { +pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks) -> Result<()> { let (initial, _, control) = located(project, declaration.clone())?; let snapshots = owner(&control)?; snapshots.initialized()?; let draft = snapshots.read_draft()?; - if draft.expected != expected { + if draft.expected != expected || draft.nonce != nonce || draft.operator != operator { return Err(SurfaceError::ManifestVersionConflict(format!("the draft read v{} and apply named v{expected}", draft.expected)).into()); } if validated_draft(&draft.document, &initial.declaration, tasks)? != draft.document { diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 40d2b4e4a..23e50f570 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -12,7 +12,7 @@ use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; -use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, EDIT_PATH, WORKFLOWS_PATH}; +use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, EDIT_PATH, RECORD_PATH, WORKFLOWS_PATH}; use contextful_core::surface::SurfaceError; use contextful_core::run::derive::task::Tasks; use contextful_engine::control::ControlError; @@ -22,6 +22,12 @@ use contextful_policy::exchange::answer as exchange_answer; use contextful_policy::issue::{SeedSigner, DEFAULT_SEED_PATH}; use contextful_policy::possession::ProofChecker; use contextful_policy::audit::AuditLog; +use contextful_policy::verify::AdmittedAuthority; +use hmac::{Hmac, Mac}; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::sync::Mutex; +use std::time::{SystemTime, UNIX_EPOCH}; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; use std::net::TcpListener; @@ -73,6 +79,31 @@ pub struct ServeArgs { keyset: Option, } +fn operator_attestation(request: &HttpRequest, secret: &str, used: &Mutex>) -> Option { + let subject = request.header("X-Contextful-Operator")?; + let at: i64 = request.header("X-Contextful-Operator-Time")?.parse().ok()?; + let nonce = request.header("X-Contextful-Operator-Nonce")?; + let signature = request.header("X-Contextful-Operator-Signature")?; + if subject.is_empty() || subject.len() > 256 || nonce.len() != 32 || !nonce.bytes().all(|byte| byte.is_ascii_hexdigit()) || signature.len() != 64 { + return None; + } + let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs() as i64; + if now.abs_diff(at) > 60 { return None; } + let digest = format!("{:x}", Sha256::digest(&request.body)); + let message = format!("{}\n{}\n{digest}\n{subject}\n{at}\n{nonce}", request.method, request.target); + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).ok()?; + mac.update(message.as_bytes()); + let mut signature_bytes = [0u8; 32]; + for (index, chunk) in signature.as_bytes().chunks_exact(2).enumerate() { + signature_bytes[index] = u8::from_str_radix(std::str::from_utf8(chunk).ok()?, 16).ok()?; + } + mac.verify_slice(&signature_bytes).ok()?; + let mut seen = used.lock().ok()?; + seen.retain(|_, expires| *expires >= now); + if seen.len() >= 10_000 || seen.insert(format!("{subject}:{nonce}"), at + 60).is_some() { return None; } + Some(subject.to_owned()) +} + /// The issuer key pins, resolved and parsed before the listener binds; no generated key /// substitutes (`topology.publish-hostname.issuer-key`). fn issuer_pins(flag: Option<&str>) -> Result { @@ -117,10 +148,27 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; - let control = |request: &HttpRequest| -> HttpResponse { + let control_attestation_secret = std::env::var("CONTEXTFUL_CONTROL_ATTESTATION_SECRET").ok().filter(|secret| !secret.is_empty()); + let attestations = Mutex::new(HashMap::new()); + let control = |request: &HttpRequest, authority: &AdmittedAuthority| -> HttpResponse { let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); - let answer = match request.target.split('?').next().unwrap_or_default() { + let path = request.target.split('?').next().unwrap_or_default(); + let operator = if path == EDIT_PATH || path == APPLY_PATH { + match control_attestation_secret.as_deref() { + Some(secret) => match operator_attestation(request, secret, &attestations) { + Some(subject) => subject, + None => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), + }, + None => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), + } + } else { String::new() }; + let answer = match path { WORKFLOWS_PATH => crate::cadence::published(&located.project, &located.declaration), + RECORD_PATH => contextful_policy::audit::entries(&located.project.audit_dir()) + .map(|entries| { + let count = entries.len(); + json!({ "entries": entries.into_iter().rev().take(1000).collect::>(), "truncated": count > 1000, "declined": count.saturating_sub(1000) }) + }).map_err(anyhow::Error::from), EDIT_PATH | APPLY_PATH => { let body: Value = match serde_json::from_slice(&request.body) { Ok(Value::Object(body)) => Value::Object(body), @@ -129,15 +177,27 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { let Some(fields) = body.as_object() else { unreachable!() }; let Some(expected) = fields.get("expected").and_then(Value::as_u64) else { return malformed() }; let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; - if request.target.split('?').next() == Some(EDIT_PATH) { + if path == EDIT_PATH { if fields.len() != 2 || fields.keys().any(|field| field != "expected" && field != "document") { return malformed(); } let Some(document) = fields.get("document").and_then(Value::as_str) else { return malformed() }; - crate::cadence::edit(&project, Some(located.declaration.clone()), expected, document, tasks) + if audit.append(json!({ "contextful.control.operation": "edit", "contextful.operator.subject": operator, + "contextful.operator.attestation": "console-hmac", + "contextful.credential": authority.credential_id(), "contextful.control.expected": expected })).is_err() { + return HttpResponse::json(503, &json!({ "error": { "identifier": "AuditEntryUnpersisted" } })); + } + crate::cadence::edit(&project, Some(located.declaration.clone()), expected, document, &operator, tasks) } else { - if fields.len() != 1 { return malformed(); } - crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, tasks) + if fields.len() != 2 { return malformed(); } + let Some(nonce) = fields.get("nonce").and_then(Value::as_str) else { return malformed() }; + if audit.append(json!({ "contextful.control.operation": "apply", "contextful.operator.subject": operator, + "contextful.operator.attestation": "console-hmac", + "contextful.credential": authority.credential_id(), "contextful.control.expected": expected, + "contextful.control.draft_nonce": nonce })).is_err() { + return HttpResponse::json(503, &json!({ "error": { "identifier": "AuditEntryUnpersisted" } })); + } + crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, nonce, &operator, tasks) .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) } } diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 66b94390e..5014216b1 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -67,6 +67,7 @@ fn serve(dir: &Path, args: &[&str]) -> (Listener, String) { fn serve_binary(binary: &Path, dir: &Path, args: &[&str]) -> (Listener, String) { let mut child = Command::new(binary) .args(args) + .env("CONTEXTFUL_CONTROL_ATTESTATION_SECRET", "separate-attestation-secret") .current_dir(dir) .env_remove("CONTEXTFUL_ISSUER_PUBKEY") .env_remove("CONTEXTFUL_AUDIENCE") @@ -101,7 +102,8 @@ fn control_apply_uses_the_host_registered_task_set() { let edit = json!({ "expected": 1, "document": draft }).to_string(); let (status, edited) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); assert_eq!(status, 200, "{edited}"); - let (status, state) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let apply = json!({ "expected": 1, "nonce": edited["nonce"] }).to_string(); + let (status, state) = control(&addr, "POST", "/control/apply", Some(&admin), &apply); assert_eq!(status, 200, "{state}"); assert_eq!(state["applied"], json!(2)); } @@ -133,7 +135,12 @@ fn query() -> Value { } fn control(addr: &str, method: &str, path: &str, token: Option<&str>, body: &str) -> (u16, Value) { - control_headers(addr, method, path, token, body, "") + let headers = if method == "POST" && (path == "/control/edit" || path == "/control/apply") { + let at = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + let nonce = NONCE.fetch_add(1, Ordering::SeqCst); + attestation(path, body, "user://dana@acme.example", at, &format!("{nonce:032x}")) + } else { String::new() }; + control_headers(addr, method, path, token, body, &headers) } fn control_headers(addr: &str, method: &str, path: &str, token: Option<&str>, body: &str, headers: &str) -> (u16, Value) { @@ -163,21 +170,21 @@ fn admin_changes_require_fresh_console_attestation_and_record_the_operator() { std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); stdout(&run(root, &["pipeline", "import", "--project", "research"])); let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://service@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); - let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public, "--control-attestation-secret", "separate-attestation-secret"]); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); assert_eq!(control(&addr, "GET", "/control/record", Some(&admin), "").0, 200); let edit = json!({ "expected": 1, "document": original.replace("every 1h", "every 1d") }).to_string(); - assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edit).0, 403); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, "").0, 403); let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; - let stale = attestation("/control/edit", &edit, "operator-a", now - 120, "stale"); + let stale = attestation("/control/edit", &edit, "operator-a", now - 120, &"1".repeat(32)); assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &stale).0, 403); - let forged = attestation("/control/edit", &edit, "operator-a", now, "forged").replace("Operator: operator-a", "Operator: operator-b"); + let forged = attestation("/control/edit", &edit, "operator-a", now, &"2".repeat(32)).replace("Operator: operator-a", "Operator: operator-b"); assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &forged).0, 403); - let signed = attestation("/control/edit", &edit, "operator-a", now, "edit-once"); + let signed = attestation("/control/edit", &edit, "operator-a", now, &"3".repeat(32)); let (status, saved) = control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed); assert_eq!(status, 200, "{saved}"); assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); let apply = json!({ "expected": 1, "nonce": saved["nonce"] }).to_string(); - let signed = attestation("/control/apply", &apply, "operator-a", now, "apply-once"); + let signed = attestation("/control/apply", &apply, "operator-a", now, &"4".repeat(32)); let (status, result) = control_headers(&addr, "POST", "/control/apply", Some(&admin), &apply, &signed); assert_eq!(status, 200, "{result}"); let entries = contextful_policy::audit::entries(&root.join(".contextful/audit")).unwrap(); @@ -201,8 +208,10 @@ fn control_routes_require_admin_and_project_applied_workflows() { let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); assert_eq!(control(&addr, "GET", "/control/workflows", None, "").0, 401); + assert_eq!(control(&addr, "GET", "/control/record", None, "").0, 401); let (status, answer) = control(&addr, "GET", "/control/workflows", Some(&read), ""); assert_eq!(status, 403, "{answer}"); + assert_eq!(control(&addr, "GET", "/control/record", Some(&read), "").0, 403); assert_eq!(control(&addr, "POST", "/control/apply", Some(&read), "{}").0, 403); assert_eq!(control(&addr, "GET", "/control/workflows", Some(&narrow_admin), "").0, 403); let (status, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); @@ -214,8 +223,9 @@ fn control_routes_require_admin_and_project_applied_workflows() { assert_eq!(control(&addr, "POST", "/control/apply", Some(&admin), "{\"id\":1}").0, 400); let edited = json!({ "expected": 1, "document": pipeline.replace("every 1h", "every 1d") }).to_string(); - assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edited).0, 200); - let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let (_, saved) = control(&addr, "POST", "/control/edit", Some(&admin), &edited); + let apply = json!({ "expected": 1, "nonce": saved["nonce"] }).to_string(); + let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), &apply); assert_eq!(status, 200, "{applied}"); assert_eq!(applied["applied"], json!(2)); let (_, view) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); @@ -235,7 +245,7 @@ fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); let changed = original.replace("every 1h", "every 1d"); let edit = json!({ "expected": 1, "document": changed }).to_string(); - let (status, absent) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let (status, absent) = control(&addr, "POST", "/control/apply", Some(&admin), &json!({ "expected": 1, "nonce": "000000000000000000000000000000000000000000000000" }).to_string()); assert_eq!(status, 409, "{absent}"); assert_eq!(absent["error"]["identifier"], "ControlDraftAbsent"); assert_eq!(control(&addr, "POST", "/control/edit", None, &edit).0, 401); @@ -248,26 +258,27 @@ fn control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version() let declaration = root.join("contextful.toml"); let owner_text = std::fs::read_to_string(&declaration).unwrap(); std::fs::write(&declaration, format!("{owner_text}\n[control]\nurl = \"http://127.0.0.1:12345/\"\n")).unwrap(); - let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let apply = json!({ "expected": 1, "nonce": draft["nonce"] }).to_string(); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), &apply); assert_eq!(status, 503, "{refused}"); assert_eq!(refused["error"]["identifier"], "ConfigOwnerUnconfigured"); std::fs::write(&declaration, owner_text).unwrap(); let (_, before) = control(&addr, "GET", "/control/workflows", Some(&admin), ""); assert_eq!(before["applied"], 1); assert_eq!(before["pipelines"][0]["schedule"], "every 1h"); - let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let (status, applied) = control(&addr, "POST", "/control/apply", Some(&admin), &apply); assert_eq!(status, 200, "{applied}"); assert_eq!(applied["applied"], 2); assert_eq!(applied["pipelines"][0]["schedule"], "every 1d"); - let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":1}"); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), &apply); assert_eq!(status, 409, "{refused}"); assert_eq!(refused["error"]["identifier"], "ControlDraftAbsent"); let later = original.replace("every 1h", "every 2d"); let edit = json!({ "expected": 2, "document": later }).to_string(); - assert_eq!(control(&addr, "POST", "/control/edit", Some(&admin), &edit).0, 200); + let (_, later_draft) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); std::fs::write(root.join("pipelines/filings.toml"), original.replace("every 1h", "every 3d")).unwrap(); stdout(&run(root, &["pipeline", "apply", "filings-flow", "--project", "research"])); - let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), "{\"expected\":2}"); + let (status, refused) = control(&addr, "POST", "/control/apply", Some(&admin), &json!({ "expected": 2, "nonce": later_draft["nonce"] }).to_string()); assert_eq!(status, 409, "{refused}"); assert_eq!(refused["error"]["identifier"], "ManifestVersionConflict"); } diff --git a/crates/contextful-engine/src/control.rs b/crates/contextful-engine/src/control.rs index b74171c32..d83406250 100644 --- a/crates/contextful-engine/src/control.rs +++ b/crates/contextful-engine/src/control.rs @@ -21,6 +21,17 @@ const DRAFT_FILE: &str = "manifest@draft.json"; pub struct Draft { pub expected: u64, pub document: String, + pub operator: String, + pub nonce: String, +} + +impl Draft { + pub fn new(expected: u64, document: String, operator: String) -> Result { + let mut nonce = [0u8; 24]; + getrandom::fill(&mut nonce).map_err(|e| ControlError::Storage(e.to_string()))?; + let nonce = nonce.iter().map(|byte| format!("{byte:02x}")).collect(); + Ok(Self { expected, document, operator, nonce }) + } } /// A snapshot-directory refusal or a storage failure beneath it. diff --git a/spec/70-control-plane.md b/spec/70-control-plane.md index 00f7ce90f..6112a2e37 100644 --- a/spec/70-control-plane.md +++ b/spec/70-control-plane.md @@ -236,7 +236,7 @@ The configuration document, the records it presents read-only, the structured sc *because a value in the document reaches every daemon and replica that reads a snapshot* - `connector-upload` — An artifact uploaded through the operator surface raises `ConnectorUploadRefused`; the surface references registered connectors by id and version. *because publishing into the registry runs a separate signed path* -- `store-draft` — An Admin edit validates a complete control document and saves one store-scoped draft bound to the applied version it read, leaving the applied pointer unchanged. +- `store-draft` — An Admin edit validates a complete control document and saves one store-scoped draft bound to its applied version, verified operator and random nonce, leaving the applied pointer unchanged. *A-surface* ## apply @@ -258,6 +258,8 @@ Validation, the immutable version claim, the pointer advance, the owner's storag - `guarded-import` — `contextful pipeline import` claims v1 from the declared pipelines while the snapshot directory holds no version; a second import claims nothing. - `draft-claim` — An Admin apply rechecks the configured store owner, validates its saved draft again, and claims that draft through the owner's version compare-and-swap. *A-surface* +- `operator-attestation` — An Admin mutation lacking a fresh, single-use console signature over its verified operator, route and body raises `ControlOperatorAttestationInvalid` before changing the control document. + *because a shared store capability cannot identify the person who used the console* - `draft-absent` — An Admin apply finding no validated store draft raises `ControlDraftAbsent` and changes no applied version. *because an absent draft supplies no document for the version claim* diff --git a/spec/cards/surface.md b/spec/cards/surface.md index 508206282..638665386 100644 --- a/spec/cards/surface.md +++ b/spec/cards/surface.md @@ -11,7 +11,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `fire` | Job declaration, the closed kind union, same-tick order, the fire watermark, and one-shot evaluation. | 7 | | `dispatch` | The bounded fire pool and its exclusion keys, the reconciler's hold on the cadence lease, step results, and the worker adapter with its attempt-fenced callback. | 15 | | `edit` | The configuration document, the records it presents read-only, the structured schedule specification, and connector configuration. | 3 | -| `apply` | Validation, the immutable version claim, the pointer advance, the owner's storage, and the authority an apply carries. | 9 | +| `apply` | Validation, the immutable version claim, the pointer advance, the owner's storage, and the authority an apply carries. | 10 | | `reside` | Where the data plane runs, the region allow-set gating placement, and operation with no external reach. | 3 | | `open-console` | The hosted console's two pages, their API routes, identity gates, page grants and engine capabilities. | 8 | | `register-store` | Which stores a deployment serves, how each store's credential and binding names derive, and how a request reaches a store's origin. | 5 | @@ -54,6 +54,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `ConfigOwnerUnconfigured` | A configuration owner with no storage or no credential behind it. | `surface.apply.owner-unconfigured` | | `ConditionalWriteUnsupported` | A configuration owner or catalog backend lacking linearizable conditional replacement. | `surface.apply.weak-conditional-backend` | | `StoreNotInitialized` | An edit or apply against a store that has taken no guarded import. | `surface.apply.uninitialized-store` | +| `ControlOperatorAttestationInvalid` | An Admin mutation without a fresh single-use console signature for its verified operator and request. | `surface.apply.operator-attestation` | | `ControlDraftAbsent` | An Admin apply with no validated store draft. | `surface.apply.draft-absent` | | `EnforceRegionMismatch` | A configured resource resolving outside the residency allow-set. | `surface.reside.region-mismatch` | | `ResidencySitesDiverge` | A push whose residency allow-set differs from the one another site recorded in the bucket manifest. | `surface.reside.site-regions` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index c54eb4524..a0dd1299f 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -19059,7 +19059,7 @@ "operation": "edit", "subject": "store-draft", "kind": "behavior", - "statement": "An Admin edit validates a complete control document and saves one store-scoped draft bound to the applied version it read, leaving the applied pointer unchanged.", + "statement": "An Admin edit validates a complete control document and saves one store-scoped draft bound to its applied version, verified operator and random nonce, leaving the applied pointer unchanged.", "why": "A-surface", "file": "spec/70-control-plane.md", "line": 239 @@ -19152,6 +19152,17 @@ "file": "spec/70-control-plane.md", "line": 259 }, + { + "id": "surface.apply.operator-attestation", + "contract": "surface", + "operation": "apply", + "subject": "operator-attestation", + "kind": "refusal", + "statement": "An Admin mutation lacking a fresh, single-use console signature over its verified operator, route and body raises `ControlOperatorAttestationInvalid` before changing the control document.", + "why": "because a shared store capability cannot identify the person who used the console", + "file": "spec/70-control-plane.md", + "line": 261 + }, { "id": "surface.apply.draft-absent", "contract": "surface", @@ -19161,7 +19172,7 @@ "statement": "An Admin apply finding no validated store draft raises `ControlDraftAbsent` and changes no applied version.", "why": "because an absent draft supplies no document for the version claim", "file": "spec/70-control-plane.md", - "line": 261 + "line": 263 }, { "id": "surface.reside.region-entries", @@ -19172,7 +19183,7 @@ "statement": "A residency allow-set holds at most 16 entries.", "why": "", "file": "spec/70-control-plane.md", - "line": 298 + "line": 300 }, { "id": "surface.reside.region-mismatch", @@ -19183,7 +19194,7 @@ "statement": "A configured resource resolving to a region the policy omits raises `EnforceRegionMismatch` at startup, and the runtime serves nothing.", "why": "P3", "file": "spec/70-control-plane.md", - "line": 299 + "line": 301 }, { "id": "surface.reside.site-regions", @@ -19194,7 +19205,7 @@ "statement": "A push records the pushing site's residency allow-set in the bucket manifest; a push finding a differing set another site recorded, an absent one included, raises `ResidencySitesDiverge` and commits nothing.", "why": "A-surface", "file": "spec/70-control-plane.md", - "line": 301 + "line": 303 }, { "id": "surface.open-console.page-routes", @@ -28551,6 +28562,10 @@ "clause": "surface.apply.draft-absent", "gloss": "An Admin apply with no validated store draft." }, + "ControlOperatorAttestationInvalid": { + "clause": "surface.apply.operator-attestation", + "gloss": "An Admin mutation without a fresh single-use console signature for its verified operator and request." + }, "ControlPointerMalformed": { "clause": "surface.reconcile.pointer-malformed", "gloss": "A snapshot pointer body that is not wholly a version." diff --git a/spec/status.md b/spec/status.md index 33498627c..b948f7575 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 109 | 50 | 22 | 0 | 60 | 0 | 60 | +| `surface` | 2 | 20 | 110 | 51 | 22 | 0 | 60 | 0 | 60 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2016 | 582 | 188 | 127 | 1374 | 0 | | +| **total** | 19 | 155 | 2017 | 583 | 188 | 127 | 1374 | 0 | | Decision records: 18. @@ -35,7 +35,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 7 — Memory | 6 | 37 | 37 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | -| 10 — Cadence and the operator plane | 11 | 71 | 59 | passing | open | +| 10 — Cadence and the operator plane | 11 | 72 | 59 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | | 12 — The operator console | 12 | 49 | 6 | passing | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | diff --git a/spec/terms/surface.toml b/spec/terms/surface.toml index 59012b6ba..3d5c19a2c 100644 --- a/spec/terms/surface.toml +++ b/spec/terms/surface.toml @@ -45,6 +45,7 @@ DispatchSubmitRejected = { clause = "surface.dispatch.submit-signed", glo SecretMaterialInDocument = { clause = "surface.edit.secret-in-document", gloss = "A credential value written into the configuration document." } ConnectorUploadRefused = { clause = "surface.edit.connector-upload", gloss = "A connector artifact offered through the operator surface." } ControlDraftAbsent = { clause = "surface.apply.draft-absent", gloss = "An Admin apply with no validated store draft." } +ControlOperatorAttestationInvalid = { clause = "surface.apply.operator-attestation", gloss = "An Admin mutation without a fresh single-use console signature for its verified operator and request." } ManifestVersionConflict = { clause = "surface.apply.version-race", gloss = "An apply whose pointer compare-and-swap lost to a concurrent apply." } ApplyValidationRefused = { clause = "surface.apply.validation", gloss = "A document engine validation refused; no version was claimed." } ConfigOwnerUnconfigured = { clause = "surface.apply.owner-unconfigured", gloss = "A configuration owner with no storage or no credential behind it." } From b65915a7498f23a2ba5ca38ced3564081982a3df Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:27:52 +0800 Subject: [PATCH 098/155] test(memory): cover served claim admission and refusal --- .../tests/integration/http.rs | 33 ++++++++++++++++++- .../contextful-cli/tests/integration/serve.rs | 15 +++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/crates/contextful-agent/tests/integration/http.rs b/crates/contextful-agent/tests/integration/http.rs index 7b7951782..9daa0358b 100644 --- a/crates/contextful-agent/tests/integration/http.rs +++ b/crates/contextful-agent/tests/integration/http.rs @@ -4,7 +4,7 @@ //! per-request admission of short-lived bearers and of holder-bound credentials, each //! request under its own proof, the in-flight ceiling and concurrency. -use contextful_agent::http::{audience, ceiling, read_request, Admitting, HttpFace, HttpRequest, Revocation}; +use contextful_agent::http::{audience, ceiling, read_request, Admitting, HttpFace, HttpRequest, HttpResponse, Revocation}; use contextful_agent::mcp::Server; use contextful_context::land::{land, Batch, RunContext}; use contextful_context::read::Face; @@ -220,6 +220,37 @@ fn stdio_answer(f: &Fixture, token: &str, message: &Value) -> Value { server.handle(&message.to_string()).unwrap() } +#[test] +fn server_held_claim_route_admits_each_request_and_keeps_browser_and_mcp_writes_out() { + let f: &'static Fixture = Box::leak(Box::new(fixture())); + let clock: &'static FixedClock = Box::leak(Box::new(FixedClock(at(NOW)))); + let admitting = Admitting { checkpoint: &f.checkpoint, audience: AUD, revocation: &no_revocation }; + let write = |_: &HttpRequest, authority: &AdmittedAuthority, boundary: &dyn Fn() -> Result<(), AuthorityError>| { + boundary().unwrap(); + HttpResponse::json(200, &json!({ "actor": authority.subject().on_behalf_of() })) + }; + let write: &'static _ = Box::leak(Box::new(write)); + let face = Box::leak(Box::new(HttpFace::new(&f.face, clock, &f.audit, admitting, Some(4)).unwrap().with_claim_write(write))); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + std::thread::spawn(move || face.serve(listener)); + let token = credential(&f.signer, "research/*", 900, None); + let mut request = HttpRequest { method: "POST".into(), target: "/memory/claims".into(), + headers: vec![("Authorization".into(), format!("Bearer {token}"))], body: b"{}".to_vec() }; + let (status, _, response) = send(addr, &request); + assert_eq!(status, 200); + assert_eq!(body(&response)["actor"], "user://dana@acme.example"); + request.headers.push(("Origin".into(), "https://console.example".into())); + let (status, _, _) = send(addr, &request); + assert_eq!(status, 403); + request.headers.clear(); + let (status, _, _) = send(addr, &request); + assert_eq!(status, 401); + let (status, _, response) = send(addr, &unproven(&call("memory.write", json!({})), "Bearer", &token)); + assert_eq!(status, 200); + assert!(body(&response).to_string().contains("no tool")); +} + /// `contextful serve --http --audience --max-in-flight ` answers MCP Streamable HTTP at `POST /mcp`: one JSON-RPC message per request, answered as `application/json` by the tool server the stdio transport runs. // spec: read.register.network-transport@38ec986b #[test] diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 46fe6f012..f87c22ccd 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -124,18 +124,29 @@ fn served_claim_write_is_durable_actor_bound_and_outside_read_mcp() { let p = dir.path(); let mint = |who: &str, task: &str, actions: &[&str]| { let mut args = vec!["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", who, - "--task", task, "--zone", "on-prem:hq", "--table", "memory/facts", "--ttl", "900"]; + "--task", task, "--zone", "on-prem:hq", "--table", "memory/facts", "--table", "research/notes", "--ttl", "900"]; for action in actions { args.extend(["--action", action]); } stdout(&run(p, &args)) }; let alice = mint("user://alice", "session-1", &["read", "write"]); let bob = mint("user://bob", "session-2", &["read", "write"]); let reader = mint("user://alice", "session-1", &["read"]); + let wrong_table = stdout(&run(p, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://alice", + "--task", "session-1", "--zone", "on-prem:hq", "--action", "write", "--table", "research/notes", "--ttl", "900"])); + let expired = stdout(&run(p, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://alice", + "--task", "session-1", "--zone", "on-prem:hq", "--action", "read", "--action", "write", + "--table", "memory/facts", "--table", "research/notes", "--ttl", "900", "--now", "2020-01-01T00:00:00Z"])); let claim = json!({ "into": "memory/facts", "actor": "user://alice", "session": "session-1", "dedup_key": "turn-1", - "claim": { "subject": "Northwind", "predicate": "filings", "object": "arrived", "confidence": 0.9, "evidence": [] } }); + "claim": { "subject": "Northwind", "predicate": "filings", "object": "arrived", "confidence": 0.9, + "evidence": [{ "table": "research/notes", "run": "run-0001", "seq": 0 }] } }); let (listener, addr) = serve(p, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); assert_eq!(post_claim(&addr, &reader, &claim, false).0, 403, "read grant cannot write"); + assert_eq!(post_claim(&addr, &wrong_table, &claim, false).0, 403, "a write grant on another table cannot land memory"); + assert_eq!(post_claim(&addr, &expired, &claim, false).0, 401, "expired authority cannot write"); + let mut client_scope = claim.clone(); + client_scope["claim"]["scope"] = json!("shared"); + assert_eq!(post_claim(&addr, &alice, &client_scope, false).0, 400, "the caller cannot set claim scope"); assert_eq!(post_claim(&addr, &alice, &claim, true).0, 403, "a browser origin cannot reach the write"); let (status, first) = post_claim(&addr, &alice, &claim, false); assert_eq!(status, 200, "{first}"); From 2477a4dcb8532ce90bc0a9dd65fc492fe6fec4aa Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:29:23 +0800 Subject: [PATCH 099/155] feat(memory): serve scoped durable claim writes --- crates/contextful-agent/src/http.rs | 51 ++++++++--- crates/contextful-cli/src/serve.rs | 69 +++++++++++++++ spec/21-memory.md | 12 +++ spec/adr/A-read.md | 15 ++++ spec/cards/read.md | 6 +- spec/guide/read.md | 4 +- spec/pins.toml | 7 +- spec/spec.lock.json | 128 +++++++++++++++++++++++----- spec/status.md | 11 ++- spec/terms/read.toml | 4 + 10 files changed, 269 insertions(+), 38 deletions(-) diff --git a/crates/contextful-agent/src/http.rs b/crates/contextful-agent/src/http.rs index 27074f4e4..46302cd20 100644 --- a/crates/contextful-agent/src/http.rs +++ b/crates/contextful-agent/src/http.rs @@ -197,6 +197,7 @@ pub struct HttpFace<'a, C> { in_flight: AtomicUsize, exchange: Option<&'a (dyn Fn(&HttpRequest) -> HttpResponse + Sync)>, exchange_unconfigured: bool, + claim_write: Option<&'a (dyn Fn(&HttpRequest, &AdmittedAuthority, &dyn Fn() -> Result<(), AuthorityError>) -> HttpResponse + Sync)>, } /// A request slot held while one request is in flight. @@ -243,7 +244,7 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { let ceiling = ceiling(max_in_flight)?; audience(Some(admitting.audience))?; let tools = Tools::new(face, clock, record)?; - Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0), exchange: None, exchange_unconfigured: false }) + Ok(HttpFace { tools, admitting, ceiling, in_flight: AtomicUsize::new(0), exchange: None, exchange_unconfigured: false, claim_write: None }) } /// The binary's exchange route mints the reader credential without putting issuer @@ -254,6 +255,12 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { self } + /// The binary owns the claim writer; this route never joins the read MCP tool set. + pub fn with_claim_write(mut self, write: &'a (dyn Fn(&HttpRequest, &AdmittedAuthority, &dyn Fn() -> Result<(), AuthorityError>) -> HttpResponse + Sync)) -> Self { + self.claim_write = Some(write); + self + } + /// Accept connections on `listener` until it fails. Each accepted connection takes a /// slot before its thread starts and holds it until its answer is written, so at most /// the ceiling's count of connection threads run. One accepted with every slot held @@ -341,6 +348,8 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { (MCP_PATH, _) => { HttpResponse::message(405, "`/mcp` answers POST; the face holds no session and opens no server stream").with("Allow", "POST") } + ("/memory/claims", "POST") if self.claim_write.is_some() => self.claim_write_request(request), + ("/memory/claims", _) if self.claim_write.is_some() => HttpResponse::message(405, "`/memory/claims` answers POST").with("Allow", "POST"), ("/auth/exchange", "POST") if self.exchange.is_some() => self.exchange.expect("checked above")(request), ("/auth/exchange", _) if self.exchange.is_some() => HttpResponse::message(405, "`/auth/exchange` answers POST").with("Allow", "POST"), (other, _) => HttpResponse::message(404, format!("no route `{other}`; the protocol endpoint is `{MCP_PATH}`")), @@ -349,6 +358,35 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { /// Admit the request's credential, then answer its one message. fn message(&self, request: &HttpRequest) -> HttpResponse { + self.admitted(request, |authority, admission| { + let message: Value = match serde_json::from_slice(&request.body) { + Ok(m @ Value::Object(_)) => m, + Ok(_) => return HttpResponse::rpc_error(INVALID_REQUEST, "a request body holds one JSON-RPC message object"), + Err(e) => return HttpResponse::rpc_error(PARSE_ERROR, format!("the request body is not JSON: {e}")), + }; + let boundary = |a: &AdmittedAuthority| effect_boundary(a, admission); + match self.tools.handle(Caller { authority, boundary: &boundary }, &message) { + Some(answer) => HttpResponse::json(200, &answer), + None => HttpResponse::empty(202), + } + }) + } + + fn claim_write_request(&self, request: &HttpRequest) -> HttpResponse { + if request.header("Origin").is_some() { + return HttpResponse::json(403, &json!({ "error": { "http": 403, "identifier": "MemoryClaimBrowserRefused" } })); + } + self.admitted(request, |authority, _| { + let boundary = || { + let revocation = (self.admitting.revocation)().map_err(AuthorityError::AuthorityRevoked)?; + let admission = Admission::new(self.tools.clock().now(), &revocation).expecting(self.admitting.audience); + effect_boundary(authority, &admission) + }; + self.claim_write.expect("route exists")(request, authority, &boundary) + }) + } + + fn admitted(&self, request: &HttpRequest, answer: impl FnOnce(&AdmittedAuthority, &Admission<'_>) -> HttpResponse) -> HttpResponse { let Some(credential) = request.credential() else { let missing = ReadError::HttpCredentialMissing( "a request carries `Authorization: Bearer `, or `Authorization: DPoP ` with a `DPoP` proof from the credential's holder key".into(), @@ -377,16 +415,7 @@ impl<'a, C: Clock + Sync> HttpFace<'a, C> { Err(ProofRefusal::NonceCacheFull) => return HttpResponse::unavailable(ProofRefusal::NonceCacheFull.to_string()), Err(ProofRefusal::Refused(e)) => return unadmitted(&e), }; - let message: Value = match serde_json::from_slice(&request.body) { - Ok(m @ Value::Object(_)) => m, - Ok(_) => return HttpResponse::rpc_error(INVALID_REQUEST, "a request body holds one JSON-RPC message object"), - Err(e) => return HttpResponse::rpc_error(PARSE_ERROR, format!("the request body is not JSON: {e}")), - }; - let boundary = |a: &AdmittedAuthority| effect_boundary(a, &admission); - match self.tools.handle(Caller { authority: &authority, boundary: &boundary }, &message) { - Some(answer) => HttpResponse::json(200, &answer), - None => HttpResponse::empty(202), - } + answer(&authority, &admission) } } diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 526e6d560..b84a4df27 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -15,6 +15,12 @@ use anyhow::Result; use contextful_agent::http::{audience, ceiling, Admitting, HttpFace}; use contextful_core::issue::{IssuancePolicy, MintContext, NodeRole}; use contextful_core::ports::{Clock, SigningPort}; +#[cfg(feature = "data-plane")] +use contextful_core::memory::synthesize::CandidateClaim; +#[cfg(feature = "data-plane")] +use contextful_memory::write::{write_observed, Observation}; +#[cfg(feature = "data-plane")] +use contextful_memory::MemoryFault; use contextful_policy::exchange::answer as exchange_answer; use contextful_policy::issue::{SeedSigner, DEFAULT_SEED_PATH}; use contextful_policy::possession::ProofChecker; @@ -25,6 +31,31 @@ use std::net::TcpListener; use std::path::PathBuf; use std::sync::Arc; +#[cfg(feature = "data-plane")] +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct ClaimWriteRequest { + into: String, + actor: String, + session: String, + dedup_key: String, + claim: CandidateClaim, +} + +#[cfg(feature = "data-plane")] +fn claim_write_error(error: MemoryFault) -> contextful_agent::http::HttpResponse { + let (status, identifier): (u16, String) = match &error { + MemoryFault::Denied(_) => (403, "GrantWriteNotCovered".into()), + MemoryFault::Authority(why) => (401, why.to_string().split(':').next().unwrap_or("AuthorityRevoked").to_owned()), + MemoryFault::Invalid(_) => (400, "MemoryClaimMalformed".into()), + MemoryFault::Memory(why) => (400, why.identifier().into()), + _ => (503, "MemoryClaimWriteRefused".into()), + }; + contextful_agent::http::HttpResponse::json(status, &serde_json::json!({ + "error": { "http": status, "identifier": identifier, "message": error.to_string() } + })) +} + /// The refusals of starting the network transport. `Display` begins with the identifier. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum ServeError { @@ -123,6 +154,42 @@ pub fn run(args: ServeArgs) -> Result<()> { None }; let proofs = ProofChecker::new(SystemClock); + #[cfg(feature = "data-plane")] + let (node, _) = contextful_context::node::resolve(face.store(), |k| std::env::var(k).ok())?; + #[cfg(feature = "data-plane")] + let claim_write = |request: &contextful_agent::http::HttpRequest, + authority: &contextful_policy::verify::AdmittedAuthority, + boundary: &dyn Fn() -> Result<(), contextful_core::AuthorityError>| { + let body: ClaimWriteRequest = match serde_json::from_slice(&request.body) { + Ok(body) => body, + Err(_) => return contextful_agent::http::HttpResponse::json(400, &serde_json::json!({ + "error": { "http": 400, "identifier": "MemoryClaimMalformed" } + })), + }; + if body.actor.is_empty() || body.session.is_empty() || + authority.subject().on_behalf_of() != Some(body.actor.as_str()) || + authority.subject().task() != Some(body.session.as_str()) { + return contextful_agent::http::HttpResponse::json(403, &serde_json::json!({ + "error": { "http": 403, "identifier": "MemoryClaimScopeRefused" } + })); + } + let mut claim = body.claim; + if claim.scope.is_some() || body.dedup_key.trim().is_empty() { + return contextful_agent::http::HttpResponse::json(400, &serde_json::json!({ + "error": { "http": 400, "identifier": "MemoryClaimMalformed" } + })); + } + claim.scope = Some(format!("console:{}:{}", body.actor, body.session)); + let scope = claim.scope.clone().expect("assigned"); + match write_observed(&face, authority, &body.into, claim, + &Observation { observed_at: None, dedup_key: Some(body.dedup_key) }, &node, clock.now(), boundary) { + Ok(written) => contextful_agent::http::HttpResponse::json(200, &serde_json::json!({ + "landed": written.claim.is_some(), "scope": scope, + "claim_id": written.claim.map(|claim| claim.claim_id) + })), + Err(error) => claim_write_error(error), + } + }; let exchange_route = |request: &contextful_agent::http::HttpRequest| { let Some((issuance, signer)) = mint_material.as_ref() else { return contextful_agent::http::HttpResponse::json(404, @@ -135,6 +202,8 @@ pub fn run(args: ServeArgs) -> Result<()> { let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)) .map_err(anyhow::Error::msg)? .with_exchange(&exchange_route, exchange.is_none()); + #[cfg(feature = "data-plane")] + let http = http.with_claim_write(&claim_write); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); http.serve(listener)?; diff --git a/spec/21-memory.md b/spec/21-memory.md index 37a76ad6e..a0f38608d 100644 --- a/spec/21-memory.md +++ b/spec/21-memory.md @@ -113,6 +113,18 @@ Supersession within one validity line, confidence decay, the direct write and it *because retiring the later claim at the earlier instant ends it before it starts, and it then answers at no instant* - `citation-live` — A direct write citing a keyed table's row that reads through the writer's session as a version its key has since replaced raises `MemoryCitationNotLive`, and nothing lands. *because such a claim rests on a replaced version from its first read, while a citation no readable row carries lands undigested and recall withholds it* +- `served-write` — A writable served face answers `POST /memory/claims` under a per-request network credential, lands a declared claim through the direct write, and keeps the read MCP tool set closed. + *A-read* +- `served-scope` — A claim write whose `actor` or `session` differs from the admitted credential's `on_behalf_of` or `task` raises `MemoryClaimScopeRefused`; accepted claims take scope from those members, never from the payload. + *A-read* +- `browser-request` — An `Origin` header on the served claim-write route raises `MemoryClaimBrowserRefused` before credential admission and commits nothing. + *A-read* +- `malformed-request` — Malformed JSON, unknown fields, a client-selected claim scope, or an empty dedup key on the served claim-write route raises `MemoryClaimMalformed` and commits nothing. + *A-read* +- `served-dedup` — A served claim write passes its `dedup_key` to {{read.revise.dedup-key}}, returns `landed: false` on restatement, and returns the bound scope on both outcomes. + *A-read* +- `served-fault` — A served claim write whose store effect refuses raises `MemoryClaimWriteRefused` and lands nothing. + *A-read* unsettled: Does a claim observed before a live contradicting claim of its line land beneath it with a bounded end, rather than refuse? owner: memory affects: read.revise diff --git a/spec/adr/A-read.md b/spec/adr/A-read.md index bf4e99151..3e4f250a2 100644 --- a/spec/adr/A-read.md +++ b/spec/adr/A-read.md @@ -85,6 +85,21 @@ Criteria: MCP-client compatibility, then one surface per guarantee, then revocat Consequences: admission sits on every read, and a possession-proof check sits on every read under a holder-bound credential; an MCP client that signs nothing reads with a short-lived audience-bound bearer and re-exchanges it before expiry. Revisit: a caller needs server-initiated messages or resumable streams. +## A served claim write stays outside read MCP + +Context: an operator answer distils claims after its read turn, while a hosted console owns no local store root. Criteria: preserve the closed read tool set, the writer's actor and session, and the memory table's commit and evidence gates. + +Decision: a separate served route admits the server-held credential, binds the actor and session to its subject tuple, and calls the direct memory write. The credential needs a narrow write grant on the declared claims table. The browser holds no writer credential; an Origin header refuses. The read-only edge profile mounts no write route. + +| Option | Lost on | Cost | +| --- | --- | --- | +| Separate route and direct memory write *(chosen)* | — | The hosted console needs a writing credential whose task names its reading session. | +| Add `memory.write` to read MCP | Closed read tool set | A client-reachable tool can mutate a store. | +| Spawn the CLI from the console | Hosted reach | The console needs the store's local project root and signing configuration. | +| Keep claims in the console process | Durability | Restart loses conclusions and bypasses memory recall and erasure. | + +Consequences: claim writes share memory's evidence, supersession, attribution and commit path. A credential with only read grants cannot land a claim. + ## Memory writes validate or dead-letter, and outcomes settle under their source `read.synthesize` validates every candidate against the declared output schema, retries with the error up to 3 attempts per batch, then dead-letters the response, template hash and drop reason with the cursor held. The relation vocabulary is a reserved core plus declared types; an undeclared edge dead-letters while the batch lands. `read.resolve-entity` dead-letters ambiguous mentions and dangling endpoints. `read.settle` requires one resolution form and one source — `metric`, `adjudicator` or `manual`; metric comparators evaluate outside the engine, verdicts carry an `http`/`https` citation, self-rated outcomes carry a null verdict, and the scored and unresolved views partition the join. diff --git a/spec/cards/read.md b/spec/cards/read.md index ebe5e4402..c351f9c75 100644 --- a/spec/cards/read.md +++ b/spec/cards/read.md @@ -17,7 +17,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `embed` | The embedding port and its default, the model identifier beside each vector, the linked-backend identity a handshake reports, and refusals on an absent backend. | 6 | | `declare` | The five memory table shapes, their canonical columns, tier and scope, and the relation vocabulary with its cardinality. | 2 | | `synthesize` | Extract, Resolve and Consolidate; the dedup key, evidence support, the audit record and coverage gaps. | 7 | -| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 7 | +| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 13 | | `recall` | Serving memory: the ranked arm at the read's anchor, the keyed read at an observed instant, the two clocks, tier-first ordering, the scope filter, the tombstone filter, the evidence gate. | 15 | | `resolve-entity` | Mention-to-entity matching, aliases, the knowledge card, place identity and nearness, edges, and ownership answers. | 2 | | `settle` | Predictions, observations, the registration, the citation a verdict owes, and the label views. | 4 | @@ -47,6 +47,10 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `MemoryDirectWriteShapeRefused` | A direct write targeted a memory shape other than claims. | `read.revise.direct-write` | | `MemoryObservationOutOfOrder` | A claim was observed before a live contradicting claim of its line. | `read.revise.observed-order` | | `MemoryCitationNotLive` | A direct write cites a keyed row that is not its key's live version. | `read.revise.citation-live` | +| `MemoryClaimScopeRefused` | A served claim write names another actor or reading session. | `read.revise.served-scope` | +| `MemoryClaimBrowserRefused` | A browser-origin request reached the served claim-write route. | `read.revise.browser-request` | +| `MemoryClaimMalformed` | A served claim write holds a malformed or client-scoped body. | `read.revise.malformed-request` | +| `MemoryClaimWriteRefused` | A served claim write's store effect refused. | `read.revise.served-fault` | | `MemoryEvidenceUnresolved` | A claim's evidence does not resolve through the caller's session. | `read.recall.evidence-unresolved` | | `MemoryEvidenceOverflow` | A claim names more evidence references than the bound. | `read.recall.evidence-references` | | `MemoryRecallNotClaims` | A keyed recall named a table declaring no `memory_facts` shape. | `read.recall.keyed-not-claims` | diff --git a/spec/guide/read.md b/spec/guide/read.md index 6034aa574..d852c0fcc 100644 --- a/spec/guide/read.md +++ b/spec/guide/read.md @@ -63,7 +63,9 @@ sequenceDiagram Every transport serializes one projection ({{read.respond.one-projection}}). `truncated` is exact, set by an over-fetched probe row ({{read.respond.truncation-is-exact}}), and zero rows is a success ({{read.respond.zero-rows-is-success}}). A caller may pin tables to published builds, and a pin never silently widens to the latest state ({{read.resolve-pin.unknown-build}}). -Memory keeps episodes, facts, entities, edges and preferences as ordinary store tables ({{topology.compose.memory-substrate}}). Synthesis extracts, resolves and consolidates. An extraction failing validation is re-prompted a bounded number of times ({{read.synthesize.extract-attempts}}), then dead-lettered with the cursor held ({{read.synthesize.dead-letter}}). An ambiguous entity mention dead-letters instead of merging ({{read.resolve-entity.ambiguous-mention}}). Recall serves a claim only when all of its evidence reads through the caller's own session ({{read.recall.evidence-unresolved}}), so memory never tells a caller what that caller could not read directly. Settlement scores registered predictions against observed outcomes ({{read.settle.registration}}). +Memory uses store tables ({{topology.compose.memory-substrate}}). Invalid extraction dead-letters with its cursor held ({{read.synthesize.dead-letter}}); ambiguous entities dead-letter ({{read.resolve-entity.ambiguous-mention}}). Recall serves claims whose evidence reads through the caller's session ({{read.recall.evidence-unresolved}}). Settlement scores outcomes against registered predictions ({{read.settle.registration}}). + +Operator claims land through {{read.revise.served-write}} under credential-bound scope ({{read.revise.served-scope}}); browser requests refuse ({{read.revise.browser-request}}). ```mermaid sequenceDiagram diff --git a/spec/pins.toml b/spec/pins.toml index bb44c6eaa..50c9c96fd 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -27,6 +27,11 @@ # `contextful-spec state` reads this file and the tags and writes spec/status.md. [pin] +"read.revise.served-write" = { test = "contextful_cli::serve::served_claim_write_is_durable_actor_bound_and_outside_read_mcp" } +"read.revise.served-scope" = { test = "contextful_cli::serve::served_claim_write_is_durable_actor_bound_and_outside_read_mcp" } +"read.revise.browser-request" = { test = "contextful_cli::serve::served_claim_write_is_durable_actor_bound_and_outside_read_mcp" } +"read.revise.malformed-request" = { test = "contextful_cli::serve::served_claim_write_is_durable_actor_bound_and_outside_read_mcp" } +"read.revise.served-dedup" = { test = "contextful_cli::serve::served_claim_write_is_durable_actor_bound_and_outside_read_mcp" } "corpus.state.typescript-tag-pin" = { test = "contextful_spec::ts_pins::a_running_surface_test_tag_performs_its_clause" } "corpus.state.typescript-unfinished" = { test = "contextful_spec::ts_pins::a_skipped_or_todo_surface_test_is_broken" } "assurance.gate.native-test-files" = { test = "contextful_ci::stages::the_surfaces_stage_runs_each_native_typescript_test_file_even_without_a_test_script" } @@ -60,7 +65,7 @@ authority = 204 connector = 218 corpus = 40 disclosure = 54 -read = 163 +read = 168 run = 261 store = 200 surface = 57 diff --git a/spec/spec.lock.json b/spec/spec.lock.json index e8bebd87a..d27c6a241 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -5999,6 +5999,72 @@ "file": "spec/21-memory.md", "line": 114 }, + { + "id": "read.revise.served-write", + "contract": "read", + "operation": "revise", + "subject": "served-write", + "kind": "behavior", + "statement": "A writable served face answers `POST /memory/claims` under a per-request network credential, lands a declared claim through the direct write, and keeps the read MCP tool set closed.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 116 + }, + { + "id": "read.revise.served-scope", + "contract": "read", + "operation": "revise", + "subject": "served-scope", + "kind": "refusal", + "statement": "A claim write whose `actor` or `session` differs from the admitted credential's `on_behalf_of` or `task` raises `MemoryClaimScopeRefused`; accepted claims take scope from those members, never from the payload.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 118 + }, + { + "id": "read.revise.browser-request", + "contract": "read", + "operation": "revise", + "subject": "browser-request", + "kind": "refusal", + "statement": "An `Origin` header on the served claim-write route raises `MemoryClaimBrowserRefused` before credential admission and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 120 + }, + { + "id": "read.revise.malformed-request", + "contract": "read", + "operation": "revise", + "subject": "malformed-request", + "kind": "refusal", + "statement": "Malformed JSON, unknown fields, a client-selected claim scope, or an empty dedup key on the served claim-write route raises `MemoryClaimMalformed` and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 122 + }, + { + "id": "read.revise.served-dedup", + "contract": "read", + "operation": "revise", + "subject": "served-dedup", + "kind": "behavior", + "statement": "A served claim write passes its `dedup_key` to {{read.revise.dedup-key}}, returns `landed: false` on restatement, and returns the bound scope on both outcomes.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 124 + }, + { + "id": "read.revise.served-fault", + "contract": "read", + "operation": "revise", + "subject": "served-fault", + "kind": "refusal", + "statement": "A served claim write whose store effect refuses raises `MemoryClaimWriteRefused` and lands nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 126 + }, { "id": "read.recall.evidence-unresolved", "contract": "read", @@ -6008,7 +6074,7 @@ "statement": "An unreadable or masked source row, an unknown table, a reference into another memory row, or malformed lineage suppresses the claim and raises `MemoryEvidenceUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 127 + "line": 139 }, { "id": "read.recall.evidence-references", @@ -6019,7 +6085,7 @@ "statement": "A claim naming more than 256 entries of evidence is suppressed unresolved, raising `MemoryEvidenceOverflow`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 129 + "line": 141 }, { "id": "read.recall.evidence-key", @@ -6030,7 +6096,7 @@ "statement": "The write landing a claim stamps each citation into a keyed table with a pepper-keyed digest of the cited row's key; recall resolves a citation whose row no longer reads through the digest's live version.", "why": "A-read", "file": "spec/21-memory.md", - "line": 131 + "line": 143 }, { "id": "read.recall.evidence-key-masked", @@ -6041,7 +6107,7 @@ "statement": "A citation whose key column reads masked or nulled by zone in the writer's session carries no digest; a masked or nulled column outside the key leaves the digest stamped.", "why": "because a masked key cell digests a value no other session reads, while a cell outside the key plays no part in matching it", "file": "spec/21-memory.md", - "line": 133 + "line": 145 }, { "id": "read.recall.evidence-no-value", @@ -6052,7 +6118,7 @@ "statement": "A citation stored in a memory row carries the cited table, run, sequence and key digest, and no cell value of the cited row.", "why": "A-read", "file": "spec/21-memory.md", - "line": 135 + "line": 147 }, { "id": "read.recall.evidence-stale", @@ -6063,7 +6129,7 @@ "statement": "A claim served through its key digest while its cited run and sequence no longer read counts under `stale` in the `contextful.recall` block, which names no claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 137 + "line": 149 }, { "id": "read.recall.ranked-arm", @@ -6074,7 +6140,7 @@ "statement": "A `corpus.retrieve` arm over a `memory_facts` table serves only live claims — no `superseded_by`, and a `valid_to` null or past the read's anchor — whose evidence passes the gate.", "why": "", "file": "spec/21-memory.md", - "line": 139 + "line": 151 }, { "id": "read.recall.suppression-count", @@ -6085,7 +6151,7 @@ "statement": "A suppressed claim is absent from the rows; the `contextful.recall` block counts suppressions per error identifier and names no claim.", "why": "because a count discloses that a conclusion was withheld, never what it concluded", "file": "spec/21-memory.md", - "line": 140 + "line": 152 }, { "id": "read.recall.keyed", @@ -6096,7 +6162,7 @@ "statement": "`memory.recall` takes `table`, `subject`, `observed_at`, `as_of_ingest` and `limit`, and returns the claims whose `subject` equals `subject` exactly and whose `valid_from` and `valid_to` cover `observed_at` as {{store.bound-time.valid-as-of}} does.", "why": "", "file": "spec/21-memory.md", - "line": 142 + "line": 154 }, { "id": "read.recall.keyed-clocks", @@ -6107,7 +6173,7 @@ "statement": "`as_of_ingest` bounds the read as {{store.bound-time.as-of}} does; absent, the read takes the latest committed state, and an absent `observed_at` the call's instant. `contextful.bounds` echoes each supplied bound under its argument name, with a per-name `inclusive` map.", "why": "", "file": "spec/21-memory.md", - "line": 143 + "line": 155 }, { "id": "read.recall.keyed-history", @@ -6118,7 +6184,7 @@ "statement": "A claim a successor retired still answers an `observed_at` inside its validity; the keyed read filters on validity alone, never on `superseded_by`.", "why": "because the question is what held at that instant, and the retired claim is the answer its interval records", "file": "spec/21-memory.md", - "line": 144 + "line": 156 }, { "id": "read.recall.keyed-gate", @@ -6129,7 +6195,7 @@ "statement": "Every keyed claim passes {{read.recall.evidence-unresolved}} and {{read.recall.evidence-references}}, and the response carries {{read.recall.suppression-count}}.", "why": "", "file": "spec/21-memory.md", - "line": 146 + "line": 158 }, { "id": "read.recall.keyed-window", @@ -6140,7 +6206,7 @@ "statement": "The keyed read gates claims in order and stops once it keeps one claim past the row ceiling, so the suppression count covers the claims gated before that stop.", "why": "because a one-row recall over a subject with thousands of claims otherwise resolves every claim's evidence", "file": "spec/21-memory.md", - "line": 147 + "line": 159 }, { "id": "read.recall.keyed-order", @@ -6151,7 +6217,7 @@ "statement": "Keyed claims order by tier, `curated` first, then `valid_from`, newest first, then `claim_id`; `limit` and the table's ceilings bound them under {{read.respond.row-ceiling}}.", "why": "", "file": "spec/21-memory.md", - "line": 149 + "line": 161 }, { "id": "read.recall.keyed-not-claims", @@ -6162,7 +6228,7 @@ "statement": "A granted `table` declaring a shape other than `memory_facts`, or no memory shape, raises `MemoryRecallNotClaims`.", "why": "because a keyed read answers with a claim's subject, validity and evidence, which no other table carries", "file": "spec/21-memory.md", - "line": 150 + "line": 162 }, { "id": "read.resolve-entity.ambiguous-mention", @@ -6173,7 +6239,7 @@ "statement": "A mention matching two canonical identities with no deterministic key separating them is recorded as an ambiguous skip, raises `MemoryEntityAmbiguous`, and dead-letters its candidate claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 171 + "line": 183 }, { "id": "read.resolve-entity.edge-endpoint", @@ -6184,7 +6250,7 @@ "statement": "A candidate edge whose source or target resolves to no identity is dead-lettered, raising `MemoryEdgeEndpointUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 173 + "line": 185 }, { "id": "read.settle.registration", @@ -6195,7 +6261,7 @@ "statement": "A registration names exactly one form (relative horizon, absolute deadline, open watch) and one source (`metric`, `adjudicator`, `manual`), with a comparator exactly when the source is `metric`; otherwise it raises `OutcomeRegistrationInvalid`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 184 + "line": 196 }, { "id": "read.settle.source-mismatch", @@ -6206,7 +6272,7 @@ "statement": "An observation carrying a verdict whose resolution source is absent or differs from the registration's raises `OutcomeSourceMismatch`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 186 + "line": 198 }, { "id": "read.settle.settling-citation", @@ -6217,7 +6283,7 @@ "statement": "An `adjudicator` or `manual` verdict without an `http` or `https` settling citation raises `OutcomeCitationMissing`; the citation rides the label view.", "why": "A-read", "file": "spec/21-memory.md", - "line": 188 + "line": 200 }, { "id": "read.settle.grace-window", @@ -6228,7 +6294,7 @@ "statement": "The label join keeps an observation from the prediction instant through the deadline plus an inclusive grace of 86400 s.", "why": "", "file": "spec/21-memory.md", - "line": 190 + "line": 202 }, { "id": "run.journal.step-output", @@ -23053,6 +23119,10 @@ "read.synthesize.claim-taint", "store.reserve.taint" ], + [ + "read.revise.served-dedup", + "read.revise.dedup-key" + ], [ "read.recall.keyed", "store.bound-time.valid-as-of" @@ -26785,6 +26855,22 @@ "clause": "read.revise.citation-live", "gloss": "A direct write cites a keyed row that is not its key's live version." }, + "MemoryClaimBrowserRefused": { + "clause": "read.revise.browser-request", + "gloss": "A browser-origin request reached the served claim-write route." + }, + "MemoryClaimMalformed": { + "clause": "read.revise.malformed-request", + "gloss": "A served claim write holds a malformed or client-scoped body." + }, + "MemoryClaimScopeRefused": { + "clause": "read.revise.served-scope", + "gloss": "A served claim write names another actor or reading session." + }, + "MemoryClaimWriteRefused": { + "clause": "read.revise.served-fault", + "gloss": "A served claim write's store effect refused." + }, "MemoryDirectWriteShapeRefused": { "clause": "read.revise.direct-write", "gloss": "A direct write targeted a memory shape other than claims." diff --git a/spec/status.md b/spec/status.md index aaff60ee5..3156d8944 100644 --- a/spec/status.md +++ b/spec/status.md @@ -10,12 +10,12 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 218 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 40 | 0 | 40 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | -| `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | +| `read` | 2 | 15 | 185 | 33 | 22 | 21 | 168 | 0 | 168 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | | `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 57 | 0 | 57 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1371 | 0 | | +| **total** | 19 | 155 | 2019 | 585 | 188 | 127 | 1376 | 0 | | Decision records: 18. @@ -32,7 +32,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 4 — Ingest | 25 | 496 | 313 | passing | open | | 5 — The read face under enforcement | 17 | 252 | 192 | passing | open | | 6 — Sync and replicas | 8 | 73 | 68 | passing | closed | -| 7 — Memory | 6 | 37 | 37 | passing | closed | +| 7 — Memory | 6 | 43 | 42 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | @@ -844,11 +844,16 @@ Unscheduled operations: 10. | `read.retrieve.token-cap` | `crates/contextful-core/tests/integration/read/retrieve.rs::the_token_set_is_capped_and_an_empty_set_omits_the_floor` | performed | | `read.retrieve.token-length-floor` | `crates/contextful-core/tests/integration/read/retrieve.rs::an_ascii_run_below_the_length_floor_leaves` | performed | | `read.retrieve.unsatisfiable-arm-drops` | `crates/contextful-context/tests/integration/read/retrieve.rs::a_filter_binds_named_columns_and_a_table_lacking_one_drops_its_arm` | performed | +| `read.revise.browser-request` | `spec/pins.toml` | performed | | `read.revise.citation-live` | `crates/contextful-memory/tests/integration/evidence.rs::a_write_citing_a_superseded_version_refuses` | performed | | `read.revise.dedup-key` | `crates/contextful-memory/tests/integration/write.rs::a_dedup_key_seeds_the_claim_id_and_a_retry_lands_nothing` | performed | | `read.revise.direct-write` | `crates/contextful-core/tests/integration/memory/revise.rs::the_direct_write_accepts_claims_alone` | performed | +| `read.revise.malformed-request` | `spec/pins.toml` | performed | | `read.revise.observed-at` | `crates/contextful-memory/tests/integration/write.rs::an_observed_write_lands_valid_from_its_observed_instant` | performed | | `read.revise.observed-order` | `crates/contextful-memory/tests/integration/write.rs::an_observation_before_a_live_contradicting_prior_refuses` | performed | +| `read.revise.served-dedup` | `spec/pins.toml` | performed | +| `read.revise.served-scope` | `spec/pins.toml` | performed | +| `read.revise.served-write` | `spec/pins.toml` | performed | | `read.revise.supersede` | `crates/contextful-core/tests/integration/memory/revise.rs::an_equal_or_higher_claim_retires_its_prior_on_one_line` | performed | | `read.revise.tier` | `crates/contextful-core/tests/integration/memory/revise.rs::tier_follows_the_write_path_and_the_lowest_grounding` | performed | | `read.settle.grace-window` | `crates/contextful-core/tests/integration/memory/settle.rs::the_label_window_runs_through_the_deadline_plus_a_day` | performed | diff --git a/spec/terms/read.toml b/spec/terms/read.toml index e013b46a4..fa674c41b 100644 --- a/spec/terms/read.toml +++ b/spec/terms/read.toml @@ -38,6 +38,10 @@ MemoryUndeclaredRelation = { clause = "read.declare.undeclared-relation", g MemoryExtractExhausted = { clause = "read.synthesize.dead-letter", gloss = "An extract batch exhausted its attempts and was dead-lettered." } MemoryDirectWriteShapeRefused = { clause = "read.revise.direct-write", gloss = "A direct write targeted a memory shape other than claims." } MemoryCitationNotLive = { clause = "read.revise.citation-live", gloss = "A direct write cites a keyed row that is not its key's live version." } +MemoryClaimScopeRefused = { clause = "read.revise.served-scope", gloss = "A served claim write names another actor or reading session." } +MemoryClaimBrowserRefused = { clause = "read.revise.browser-request", gloss = "A browser-origin request reached the served claim-write route." } +MemoryClaimMalformed = { clause = "read.revise.malformed-request", gloss = "A served claim write holds a malformed or client-scoped body." } +MemoryClaimWriteRefused = { clause = "read.revise.served-fault", gloss = "A served claim write's store effect refused." } MemoryEvidenceUnresolved = { clause = "read.recall.evidence-unresolved", gloss = "A claim's evidence does not resolve through the caller's session." } MemoryEvidenceOverflow = { clause = "read.recall.evidence-references", gloss = "A claim names more evidence references than the bound." } MemoryObservationOutOfOrder = { clause = "read.revise.observed-order", gloss = "A claim was observed before a live contradicting claim of its line." } From 67d963a8445e9f1abbc5a8f942f4944661b7a7b4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:34:29 +0800 Subject: [PATCH 100/155] spec(surface): pin Admin operator attestation --- spec/pins.toml | 3 ++- spec/status.md | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/spec/pins.toml b/spec/pins.toml index 36c86cc7b..a6adad4aa 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -30,6 +30,7 @@ "surface.edit.store-draft" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } "surface.apply.draft-claim" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } "surface.apply.draft-absent" = { test = "contextful_cli::serve::control_edit_saves_a_validated_draft_and_apply_claims_its_expected_version" } +"surface.apply.operator-attestation" = { test = "contextful_cli::serve::admin_changes_require_fresh_console_attestation_and_record_the_operator" } "corpus.state.typescript-tag-pin" = { test = "contextful_spec::ts_pins::a_running_surface_test_tag_performs_its_clause" } "corpus.state.typescript-unfinished" = { test = "contextful_spec::ts_pins::a_skipped_or_todo_surface_test_is_broken" } "assurance.gate.native-test-files" = { test = "contextful_ci::stages::the_surfaces_stage_runs_each_native_typescript_test_file_even_without_a_test_script" } @@ -66,5 +67,5 @@ disclosure = 54 read = 163 run = 261 store = 200 -surface = 60 +surface = 61 topology = 38 diff --git a/spec/status.md b/spec/status.md index b948f7575..e3cd16ddd 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 110 | 51 | 22 | 0 | 60 | 0 | 60 | +| `surface` | 2 | 20 | 110 | 51 | 22 | 0 | 61 | 0 | 61 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2017 | 583 | 188 | 127 | 1374 | 0 | | +| **total** | 19 | 155 | 2017 | 583 | 188 | 127 | 1375 | 0 | | Decision records: 18. @@ -35,7 +35,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 7 — Memory | 6 | 37 | 37 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | -| 10 — Cadence and the operator plane | 11 | 72 | 59 | passing | open | +| 10 — Cadence and the operator plane | 11 | 72 | 60 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | | 12 — The operator console | 12 | 49 | 6 | passing | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | @@ -1327,6 +1327,7 @@ Unscheduled operations: 10. | `surface.apply.draft-claim` | `spec/pins.toml` | performed | | `surface.apply.guarded-import` | `crates/contextful-cli/tests/integration/pipeline.rs::the_import_claims_the_first_version_once` | performed | | `surface.apply.local-claim` | `crates/contextful-cli/tests/integration/pipeline.rs::apply_claims_a_version_in_the_local_snapshot_directory` | performed | +| `surface.apply.operator-attestation` | `spec/pins.toml` | performed | | `surface.apply.owner-unconfigured` | `crates/contextful-cli/tests/integration/pipeline.rs::an_owner_with_nothing_behind_it_is_refused` | performed | | `surface.apply.uninitialized-store` | `crates/contextful-cli/tests/integration/pipeline.rs::an_apply_before_the_import_is_refused` | performed | | `surface.apply.validation` | `crates/contextful-cli/tests/integration/pipeline.rs::an_invalid_document_claims_no_version` | performed | From a7c3a66f7253ec48c974669b5da1e15bb74f3bf3 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:40:37 +0800 Subject: [PATCH 101/155] test(console): require scoped durable learning before planning --- apps/console/test/answer_memory.test.ts | 70 +++++++++++++++++++++++++ apps/console/test/console.test.ts | 20 +++++++ 2 files changed, 90 insertions(+) create mode 100644 apps/console/test/answer_memory.test.ts diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts new file mode 100644 index 000000000..1defbed7e --- /dev/null +++ b/apps/console/test/answer_memory.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { createLiveAnswer } from "../src/answer_live.ts"; +import type { StoreEntry } from "../../gateway/src/index.ts"; + +test("verified reading session lands evidence-backed claims and recalls them before planning after restart", async () => { + const store: StoreEntry = { id: "field-notes", label: "Field notes", endpoint: "https://store.example", auth: "exchange", + exchangeRoute: "/auth/exchange", credentialName: "FIELD_NOTES_QUERY_TOKEN", bindingName: "FIELD_NOTES" }; + const events: string[] = []; + const claims: Array<{ subject: string; predicate: string; object: string; scope: string }> = []; + const writes: Array<{ authorization: string | null; body: Record }> = []; + const fetcher: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/auth/exchange")) return Response.json({ token: "reader-writer-alice" }); + if (url.endsWith("/memory/claims")) { + events.push("write"); + const body = JSON.parse(String(init?.body)) as Record; + writes.push({ authorization: new Headers(init?.headers).get("authorization"), body }); + const claim = body.claim as { subject: string; predicate: string; object: string }; + claims.push({ ...claim, scope: `console:${body.actor}:${body.session}` }); + return Response.json({ landed: true, scope: claims.at(-1)?.scope, claim_id: "claim-1" }); + } + if (url.endsWith("/mcp")) { + const message = JSON.parse(String(init?.body)) as { id: number; params: { name: string; arguments: Record } }; + events.push(message.params.name); + let result: unknown; + if (message.params.name === "context.describe") result = { tables: [ + { table: "filings", kind: "data", description: "Northwind filings" }, + { table: "memory/facts", kind: "memory" }, + ] }; + else if (message.params.name === "memory.recall") result = { columns: ["subject", "predicate", "object", "scope"], + rows: claims.filter((claim) => claim.subject === message.params.arguments.subject) + .map((claim) => [claim.subject, claim.predicate, claim.object, claim.scope]) }; + else { + assert.equal(message.params.name, "context.query"); + const sql = String(message.params.arguments.sql); + result = sql.includes("DISTINCT subject") ? { columns: ["subject"], rows: [["Northwind"]] } : { + columns: ["filing_id", "title", "summary", "_run_id", "_row_seq"], + rows: [["filing-1", "Northwind filing", "Northwind filings need review", "run-1", 0]], + }; + } + return Response.json({ jsonrpc: "2.0", id: message.id, result: { structuredContent: result } }); + } + assert.equal(url, "https://model.example/v1/chat/completions"); + events.push("model"); + const prompt = String(init?.body); + const content = prompt.includes("Distil") ? JSON.stringify({ entries: [ + { subject: "Northwind", key: "filings", learning: "Northwind filings need review" }, + ] }) : "Northwind filings need review [filing-1]."; + if (claims.length > 0 && !prompt.includes("Distil")) assert.match(prompt, /Northwind filings need review/); + return Response.json({ choices: [{ message: { content } }] }); + }; + const options = { stores: [store], env: { CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture" }, fetcher }; + const operator = { subject: "alice", session: "reading-1", grants: new Set(["query" as const]), assertion: "verified-assertion" }; + await createLiveAnswer(options).turn({ operator, store: store.id, question: "Which Northwind filing arrived?" }); + assert.equal(writes.length, 1); + assert.equal(writes[0].authorization, "Bearer reader-writer-alice"); + assert.equal(writes[0].body.actor, "alice"); + assert.equal(writes[0].body.session, "reading-1"); + assert.equal(writes[0].body.into, "memory/facts"); + assert.equal(typeof writes[0].body.dedup_key, "string"); + assert.deepEqual((writes[0].body.claim as { evidence: unknown; scope?: string }).evidence, + [{ table: "filings", run: "run-1", seq: 0 }]); + assert.equal((writes[0].body.claim as { scope?: string }).scope, undefined); + + events.length = 0; + await createLiveAnswer(options).turn({ operator, store: store.id, question: "What changed for Northwind?" }); + assert(events.indexOf("memory.recall") >= 0 && events.indexOf("memory.recall") < events.lastIndexOf("context.query")); + assert(!events.includes("memory.write"), "read MCP remains closed to mutation"); +}); diff --git a/apps/console/test/console.test.ts b/apps/console/test/console.test.ts index 20968ab3d..86501c270 100644 --- a/apps/console/test/console.test.ts +++ b/apps/console/test/console.test.ts @@ -62,6 +62,26 @@ test("verified Access audiences grant only their page and API", async () => { assert.deepEqual(calls, ["workflows"]); }); +test("Access passes only its configured signed reading-session claim to Query", async () => { + const sessions: Array = []; + const app = createConsole({ + identity: { kind: "access", issuer, queryAudience, adminAudience, publicKey, memorySessionClaim: "sid" }, + stores: [{ id: "field-notes", label: "Field notes" }], + turn: async ({ operator }) => { sessions.push(operator.session); return { answer: "Grounded.", sources: [], widgets: [] }; }, + read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) }, + }); + const ask = (headers: HeadersInit) => app.fetch(new Request("https://console.example/query/api/ask", { + method: "POST", headers: { ...headers, origin: "https://console.example" }, + body: JSON.stringify({ store: "field-notes", question: "What changed?" }), + })); + assert.equal((await ask(access(queryAudience, { sid: "stable-reading-session", jti: "rotating-token-id" }))).status, 200); + assert.equal((await ask(access(queryAudience, { jti: "another-token-id" }))).status, 200); + assert.deepEqual(sessions, ["stable-reading-session", undefined]); + assert.equal((await ask({ "cf-access-jwt-assertion": "forged" })).status, 401); + assert.equal(sessions.length, 2); +}); + test("expired or wrong-issuer assertions cannot open a page", async () => { const { request } = fixture(); assert.equal((await request("/query", access(queryAudience, { exp: 1 }))).status, 401); From 37943b06351ae9299754fb6939d699f511982c97 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:41:56 +0800 Subject: [PATCH 102/155] test(control): refuse signed edit replay after restart --- crates/contextful-cli/tests/integration/serve.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 5014216b1..37d68543f 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -170,7 +170,7 @@ fn admin_changes_require_fresh_console_attestation_and_record_the_operator() { std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); stdout(&run(root, &["pipeline", "import", "--project", "research"])); let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://service@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); - let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let (listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); assert_eq!(control(&addr, "GET", "/control/record", Some(&admin), "").0, 200); let edit = json!({ "expected": 1, "document": original.replace("every 1h", "every 1d") }).to_string(); assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, "").0, 403); @@ -183,13 +183,16 @@ fn admin_changes_require_fresh_console_attestation_and_record_the_operator() { let (status, saved) = control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed); assert_eq!(status, 200, "{saved}"); assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); + drop(listener); + let (_restart, restarted_addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + assert_eq!(control_headers(&restarted_addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); let apply = json!({ "expected": 1, "nonce": saved["nonce"] }).to_string(); let signed = attestation("/control/apply", &apply, "operator-a", now, &"4".repeat(32)); - let (status, result) = control_headers(&addr, "POST", "/control/apply", Some(&admin), &apply, &signed); + let (status, result) = control_headers(&restarted_addr, "POST", "/control/apply", Some(&admin), &apply, &signed); assert_eq!(status, 200, "{result}"); let entries = contextful_policy::audit::entries(&root.join(".contextful/audit")).unwrap(); assert!(entries.iter().any(|entry| entry.attributes["contextful.operator.subject"] == "operator-a" && entry.attributes["contextful.control.operation"] == "apply")); - let (status, record) = control(&addr, "GET", "/control/record", Some(&admin), ""); + let (status, record) = control(&restarted_addr, "GET", "/control/record", Some(&admin), ""); assert_eq!(status, 200, "{record}"); assert!(record["entries"].as_array().unwrap().iter().any(|entry| entry["attributes"]["contextful.operator.subject"] == "operator-a")); } From 8d2de581b9c33e07fcfbf6a5ab3d0086a1d6c08c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:43:21 +0800 Subject: [PATCH 103/155] test(console): ignore reserved run ID as citation source --- apps/console/test/answer_memory.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index 1defbed7e..64b88480b 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -35,8 +35,8 @@ test("verified reading session lands evidence-backed claims and recalls them bef assert.equal(message.params.name, "context.query"); const sql = String(message.params.arguments.sql); result = sql.includes("DISTINCT subject") ? { columns: ["subject"], rows: [["Northwind"]] } : { - columns: ["filing_id", "title", "summary", "_run_id", "_row_seq"], - rows: [["filing-1", "Northwind filing", "Northwind filings need review", "run-1", 0]], + columns: ["_run_id", "_row_seq", "filing_id", "title", "summary"], + rows: [["run-1", 0, "filing-1", "Northwind filing", "Northwind filings need review"]], }; } return Response.json({ jsonrpc: "2.0", id: message.id, result: { structuredContent: result } }); @@ -52,7 +52,8 @@ test("verified reading session lands evidence-backed claims and recalls them bef }; const options = { stores: [store], env: { CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture" }, fetcher }; const operator = { subject: "alice", session: "reading-1", grants: new Set(["query" as const]), assertion: "verified-assertion" }; - await createLiveAnswer(options).turn({ operator, store: store.id, question: "Which Northwind filing arrived?" }); + const first = await createLiveAnswer(options).turn({ operator, store: store.id, question: "Which Northwind filing arrived?" }); + assert.match(first.answer, /\[source-1\]/); assert.equal(writes.length, 1); assert.equal(writes[0].authorization, "Bearer reader-writer-alice"); assert.equal(writes[0].body.actor, "alice"); From a8dea1b41f7d5e7cef2f1ab3e518a88f956f3f63 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:44:07 +0800 Subject: [PATCH 104/155] test(console): keep evidence coordinates off model and view --- apps/console/test/answer_memory.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index 64b88480b..bb6140293 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -44,6 +44,7 @@ test("verified reading session lands evidence-backed claims and recalls them bef assert.equal(url, "https://model.example/v1/chat/completions"); events.push("model"); const prompt = String(init?.body); + assert.doesNotMatch(prompt, /run-1/, "reserved provenance stays out of model input"); const content = prompt.includes("Distil") ? JSON.stringify({ entries: [ { subject: "Northwind", key: "filings", learning: "Northwind filings need review" }, ] }) : "Northwind filings need review [filing-1]."; @@ -54,6 +55,7 @@ test("verified reading session lands evidence-backed claims and recalls them bef const operator = { subject: "alice", session: "reading-1", grants: new Set(["query" as const]), assertion: "verified-assertion" }; const first = await createLiveAnswer(options).turn({ operator, store: store.id, question: "Which Northwind filing arrived?" }); assert.match(first.answer, /\[source-1\]/); + assert.deepEqual(first.resultRows?.columns, ["filing_id", "title", "summary"]); assert.equal(writes.length, 1); assert.equal(writes[0].authorization, "Bearer reader-writer-alice"); assert.equal(writes[0].body.actor, "alice"); From 703a832942da2d0d8fe1011ab899c3ab0d5fc914 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:45:14 +0800 Subject: [PATCH 105/155] test(console): reload scoped brief conclusions from durable memory --- apps/console/test/answer_memory.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index bb6140293..47e52d5ae 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -48,7 +48,10 @@ test("verified reading session lands evidence-backed claims and recalls them bef const content = prompt.includes("Distil") ? JSON.stringify({ entries: [ { subject: "Northwind", key: "filings", learning: "Northwind filings need review" }, ] }) : "Northwind filings need review [filing-1]."; - if (claims.length > 0 && !prompt.includes("Distil")) assert.match(prompt, /Northwind filings need review/); + if (claims.length > 0 && !prompt.includes("Distil")) { + assert.match(prompt, /Northwind filings need review/); + assert.doesNotMatch(prompt, /Bob only/); + } return Response.json({ choices: [{ message: { content } }] }); }; const options = { stores: [store], env: { CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture" }, fetcher }; @@ -66,8 +69,13 @@ test("verified reading session lands evidence-backed claims and recalls them bef [{ table: "filings", run: "run-1", seq: 0 }]); assert.equal((writes[0].body.claim as { scope?: string }).scope, undefined); + claims.push({ subject: "Northwind", predicate: "private", object: "Bob only", scope: "console:bob:another-session" }); + const restarted = createLiveAnswer(options); + const brief = await restarted.brief(operator, store.id); + assert.deepEqual(brief.conclusions.map((entry) => entry.subject), ["Northwind"]); + assert.deepEqual(brief.conclusions.map((entry) => entry.text), ["Northwind filings need review"]); events.length = 0; - await createLiveAnswer(options).turn({ operator, store: store.id, question: "What changed for Northwind?" }); + await restarted.turn({ operator, store: store.id, question: "What changed for Northwind?" }); assert(events.indexOf("memory.recall") >= 0 && events.indexOf("memory.recall") < events.lastIndexOf("context.query")); assert(!events.includes("memory.write"), "read MCP remains closed to mutation"); }); From 488177cc7509dcf196eaba18cf91793f27a477a7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:46:30 +0800 Subject: [PATCH 106/155] test(console): omit transient memory without verified session --- apps/console/test/answer_live.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/apps/console/test/answer_live.test.ts b/apps/console/test/answer_live.test.ts index f5224d354..bb3c779c2 100644 --- a/apps/console/test/answer_live.test.ts +++ b/apps/console/test/answer_live.test.ts @@ -42,14 +42,14 @@ function fixture() { return { live, calls, modelPrompts }; } -test("live answer renders redacted governed rows and distils only observed scoped learning", async () => { +test("live answer with no verified memory session leaves no transient learning", async () => { const { live, calls, modelPrompts } = fixture(); const alice = { subject: "alice", grants: new Set(["query" as const]), assertion: "alice-session-1" }; const answer = await live.turn({ operator: alice, store: store.id, question: "Which Northwind filing arrived?" }); assert.match(answer.answer, /Northwind filings need review/); assert.deepEqual(answer.resultRows?.columns, ["filing_id", "title", "summary", "published_at", "source_url", "owner_email"]); assert.doesNotMatch(JSON.stringify(answer.resultRows), /secret@example.test/); - assert.equal(modelPrompts.length, 2, "the second model pass distils the completed answer"); + assert.equal(modelPrompts.length, 1, "no signed memory session means no learning pass"); assert.doesNotMatch(modelPrompts[0], /secret@example.test/); assert(calls.every((call) => call.authorization === "Bearer reader-alice-session-1")); @@ -58,10 +58,9 @@ test("live answer renders redacted governed rows and distils only observed scope assert.equal(await deriveBrief({ ...sameSession, budgetMs: 100 }), null); const nextSession = await live.brief({ ...alice, assertion: "alice-session-2" }, store.id); assert.equal(nextSession.session.turns, 0); - assert.deepEqual(nextSession.conclusions.map((entry) => entry.subject), ["Northwind"]); + assert.deepEqual(nextSession.conclusions, []); const card = await deriveBrief({ ...nextSession, budgetMs: 100 }); - assert.equal(card?.subjects[0].subject, "Northwind"); - assert(calls.some((call) => call.name === "context.query" && call.authorization === "Bearer reader-alice-session-2")); + assert.equal(card, null); const bob = await live.brief({ subject: "bob", grants: new Set(["query"]), assertion: "bob-session" }, store.id); assert.deepEqual(bob.conclusions, []); }); From a78ad9e4cb0da46b23a68d29df09771cb5a05e9c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:49:18 +0800 Subject: [PATCH 107/155] fix(control): persist operator request nonces across restarts --- crates/contextful-cli/src/cadence.rs | 6 +++ crates/contextful-cli/src/serve.rs | 48 ++++++++++--------- crates/contextful-engine/src/control.rs | 38 +++++++++++++++ .../tests/integration/control.rs | 11 +++++ spec/70-control-plane.md | 2 +- spec/spec.lock.json | 2 +- 6 files changed, 82 insertions(+), 25 deletions(-) diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index 50cdb28de..e970aec64 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -317,6 +317,12 @@ pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected Ok(json!({ "expected": expected, "nonce": draft.nonce })) } +/// Claim a verified control request's nonce in the selected store owner's snapshot state. +pub(crate) fn claim_operator_nonce(project: &ProjectArgs, declaration: Option, nonce: &str, signed_at: i64, now: i64) -> Result { + let (_, _, control) = located(project, declaration)?; + owner(&control)?.claim_attestation_nonce(nonce, signed_at, now).map_err(Into::into) +} + /// Revalidate the saved draft and claim it only at the version the editor read. pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks) -> Result<()> { let (initial, _, control) = located(project, declaration.clone())?; diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 23e50f570..eb59ccc06 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -25,8 +25,6 @@ use contextful_policy::audit::AuditLog; use contextful_policy::verify::AdmittedAuthority; use hmac::{Hmac, Mac}; use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use std::sync::Mutex; use std::time::{SystemTime, UNIX_EPOCH}; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; @@ -79,7 +77,7 @@ pub struct ServeArgs { keyset: Option, } -fn operator_attestation(request: &HttpRequest, secret: &str, used: &Mutex>) -> Option { +fn operator_attestation(request: &HttpRequest, secret: &str) -> Option<(String, String, i64, i64)> { let subject = request.header("X-Contextful-Operator")?; let at: i64 = request.header("X-Contextful-Operator-Time")?.parse().ok()?; let nonce = request.header("X-Contextful-Operator-Nonce")?; @@ -98,10 +96,20 @@ fn operator_attestation(request: &HttpRequest, secret: &str, used: &Mutex= now); - if seen.len() >= 10_000 || seen.insert(format!("{subject}:{nonce}"), at + 60).is_some() { return None; } - Some(subject.to_owned()) + Some((subject.to_owned(), nonce.to_owned(), at, now)) +} + +fn control_failure(error: anyhow::Error) -> HttpResponse { + let surface = error.chain().find_map(|part| part.downcast_ref::().or_else(|| { + match part.downcast_ref::() { + Some(ControlError::Surface(refusal)) => Some(refusal), + _ => None, + } + })); + let status = surface.map_or(503, SurfaceError::status); + let identifier = surface.map(|refusal| refusal.to_string().split(':').next().unwrap_or("ControlUnavailable").to_string()) + .unwrap_or_else(|| "ControlUnavailable".into()); + HttpResponse::json(status, &json!({ "error": { "identifier": identifier } })) } /// The issuer key pins, resolved and parsed before the listener binds; no generated key @@ -149,14 +157,20 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; let audit = AuditLog::unanchored(located.project.audit_dir())?; let control_attestation_secret = std::env::var("CONTEXTFUL_CONTROL_ATTESTATION_SECRET").ok().filter(|secret| !secret.is_empty()); - let attestations = Mutex::new(HashMap::new()); let control = |request: &HttpRequest, authority: &AdmittedAuthority| -> HttpResponse { let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let path = request.target.split('?').next().unwrap_or_default(); + let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; let operator = if path == EDIT_PATH || path == APPLY_PATH { match control_attestation_secret.as_deref() { - Some(secret) => match operator_attestation(request, secret, &attestations) { - Some(subject) => subject, + Some(secret) => match operator_attestation(request, secret) { + Some((subject, nonce, signed_at, now)) => { + match crate::cadence::claim_operator_nonce(&project, Some(located.declaration.clone()), &nonce, signed_at, now) { + Ok(true) => subject, + Ok(false) => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), + Err(error) => return control_failure(error), + } + } None => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), }, None => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), @@ -176,7 +190,6 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; let Some(fields) = body.as_object() else { unreachable!() }; let Some(expected) = fields.get("expected").and_then(Value::as_u64) else { return malformed() }; - let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; if path == EDIT_PATH { if fields.len() != 2 || fields.keys().any(|field| field != "expected" && field != "document") { return malformed(); @@ -205,18 +218,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; match answer { Ok(state) => HttpResponse::json(200, &state), - Err(error) => { - let surface = error.chain().find_map(|part| part.downcast_ref::().or_else(|| { - match part.downcast_ref::() { - Some(ControlError::Surface(refusal)) => Some(refusal), - _ => None, - } - })); - let status = surface.map_or(503, SurfaceError::status); - let identifier = surface.map(|refusal| refusal.to_string().split(':').next().unwrap_or("ControlUnavailable").to_string()) - .unwrap_or_else(|| "ControlUnavailable".into()); - HttpResponse::json(status, &json!({ "error": { "identifier": identifier } })) - } + Err(error) => control_failure(error), } }; let exchange = crate::token::configured_exchange(&root)?; diff --git a/crates/contextful-engine/src/control.rs b/crates/contextful-engine/src/control.rs index d83406250..5b94bd8e1 100644 --- a/crates/contextful-engine/src/control.rs +++ b/crates/contextful-engine/src/control.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; /// The lock file serializing claims. const LOCK_FILE: &str = "manifest.lock"; const DRAFT_FILE: &str = "manifest@draft.json"; +const ATTESTATION_NONCES: &str = "attestation-nonces"; /// A validated store draft bound to the applied version it read. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -105,6 +106,43 @@ impl SnapshotDir { std::fs::read_to_string(&path).map_err(|e| SurfaceError::ControlSnapshotUnreadable(format!("{}: {e}", path.display())).into()) } + /// Claim one signed control request under the same lock that serializes draft and + /// applied-pointer mutations. A claim stays until its signed time is no longer + /// admissible, including across a listener restart. + pub fn claim_attestation_nonce(&self, nonce: &str, signed_at: i64, now: i64) -> Result { + if nonce.len() != 32 || !nonce.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(ControlError::Storage("an attestation nonce is 32 hexadecimal bytes".into())); + } + let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); + let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; + let directory = self.root.join(ATTESTATION_NONCES); + std::fs::create_dir_all(&directory).map_err(|e| ControlError::Storage(format!("{}: {e}", directory.display())))?; + let first_live = now.saturating_sub(60); + for entry in std::fs::read_dir(&directory).map_err(|e| ControlError::Storage(format!("{}: {e}", directory.display())))? { + let entry = entry.map_err(|e| ControlError::Storage(format!("{}: {e}", directory.display())))?; + if !entry.file_type().map_err(|e| ControlError::Storage(format!("{}: {e}", entry.path().display())))?.is_dir() { + return Err(ControlError::Storage(format!("{}: an attestation bucket is a directory", entry.path().display()))); + } + let signed_second = entry.file_name().to_str().and_then(|name| name.parse::().ok()) + .ok_or_else(|| ControlError::Storage(format!("{}: malformed attestation bucket", entry.path().display())))?; + if signed_second < first_live { + std::fs::remove_dir_all(entry.path()).map_err(|e| ControlError::Storage(format!("{}: {e}", entry.path().display())))?; + } + } + let marker = directory.join(signed_at.to_string()).join(nonce); + let claimed = create_new(&marker, b"").map_err(storage)?; + if claimed { + std::fs::File::open(&marker).and_then(|file| file.sync_all()) + .map_err(|e| ControlError::Storage(format!("{}: {e}", marker.display())))?; + #[cfg(unix)] + for path in [marker.parent().expect("nonce marker has a bucket"), directory.as_path(), self.root.as_path()] { + std::fs::File::open(path).and_then(|dir| dir.sync_all()) + .map_err(|e| ControlError::Storage(format!("{}: {e}", path.display())))?; + } + } + Ok(claimed) + } + /// Save one store draft only while the applied version still matches its base. pub fn save_draft(&self, draft: &Draft) -> Result<(), ControlError> { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); diff --git a/crates/contextful-engine/tests/integration/control.rs b/crates/contextful-engine/tests/integration/control.rs index 6241f4530..25d36630b 100644 --- a/crates/contextful-engine/tests/integration/control.rs +++ b/crates/contextful-engine/tests/integration/control.rs @@ -18,6 +18,17 @@ fn a_saved_draft_is_bound_to_its_editor_and_exact_nonce() { assert_eq!(snaps.read(2).unwrap(), "# bob\n"); } +#[test] +fn an_attestation_nonce_survives_reopen_and_expires_after_its_signed_window() { + let dir = tempfile::tempdir().unwrap(); + let first = SnapshotDir::open(dir.path()); + assert!(first.claim_attestation_nonce(&"a".repeat(32), 100, 100).unwrap()); + let reopened = SnapshotDir::open(dir.path()); + assert!(!reopened.claim_attestation_nonce(&"a".repeat(32), 100, 160).unwrap()); + assert!(reopened.claim_attestation_nonce(&"b".repeat(32), 161, 161).unwrap()); + assert!(!dir.path().join("attestation-nonces/100").exists()); +} + /// An apply whose compare-and-swap loses raises `ManifestVersionConflict`, reloads the winning version and /// reapplies its pending edits onto it, overwriting no applied version. // spec: surface.apply.version-race@30fc0e6c diff --git a/spec/70-control-plane.md b/spec/70-control-plane.md index 6112a2e37..5dd51a246 100644 --- a/spec/70-control-plane.md +++ b/spec/70-control-plane.md @@ -258,7 +258,7 @@ Validation, the immutable version claim, the pointer advance, the owner's storag - `guarded-import` — `contextful pipeline import` claims v1 from the declared pipelines while the snapshot directory holds no version; a second import claims nothing. - `draft-claim` — An Admin apply rechecks the configured store owner, validates its saved draft again, and claims that draft through the owner's version compare-and-swap. *A-surface* -- `operator-attestation` — An Admin mutation lacking a fresh, single-use console signature over its verified operator, route and body raises `ControlOperatorAttestationInvalid` before changing the control document. +- `operator-attestation` — An Admin mutation lacking a fresh console signature over its verified operator, route and body, or reusing a nonce held in store control state across restarts, raises `ControlOperatorAttestationInvalid` before changing the document. *because a shared store capability cannot identify the person who used the console* - `draft-absent` — An Admin apply finding no validated store draft raises `ControlDraftAbsent` and changes no applied version. *because an absent draft supplies no document for the version claim* diff --git a/spec/spec.lock.json b/spec/spec.lock.json index a0dd1299f..47e94e1a0 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -19158,7 +19158,7 @@ "operation": "apply", "subject": "operator-attestation", "kind": "refusal", - "statement": "An Admin mutation lacking a fresh, single-use console signature over its verified operator, route and body raises `ControlOperatorAttestationInvalid` before changing the control document.", + "statement": "An Admin mutation lacking a fresh console signature over its verified operator, route and body, or reusing a nonce held in store control state across restarts, raises `ControlOperatorAttestationInvalid` before changing the document.", "why": "because a shared store capability cannot identify the person who used the console", "file": "spec/70-control-plane.md", "line": 261 From f617cff2757dda3828475a16e4e7b6f33527a186 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:49:26 +0800 Subject: [PATCH 108/155] test(console): cold memory relation does not block first learning --- apps/console/test/answer_memory.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index 47e52d5ae..d291ec09a 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -34,6 +34,9 @@ test("verified reading session lands evidence-backed claims and recalls them bef else { assert.equal(message.params.name, "context.query"); const sql = String(message.params.arguments.sql); + if (sql.includes("DISTINCT subject") && claims.length === 0) { + return Response.json({ error: { identifier: "EmptyMemoryRelation" } }, { status: 400 }); + } result = sql.includes("DISTINCT subject") ? { columns: ["subject"], rows: [["Northwind"]] } : { columns: ["_run_id", "_row_seq", "filing_id", "title", "summary"], rows: [["run-1", 0, "filing-1", "Northwind filing", "Northwind filings need review"]], From f3c09d96aae68b5f38cee2f71421022e150f26f4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:50:55 +0800 Subject: [PATCH 109/155] test(console): type served learning refusal at Query boundary --- apps/console/test/answer_memory.test.ts | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index d291ec09a..3537e2872 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { createLiveAnswer } from "../src/answer_live.ts"; +import { createConsole, issueCognitoSession } from "../src/index.ts"; import type { StoreEntry } from "../../gateway/src/index.ts"; test("verified reading session lands evidence-backed claims and recalls them before planning after restart", async () => { @@ -82,3 +83,39 @@ test("verified reading session lands evidence-backed claims and recalls them bef assert(events.indexOf("memory.recall") >= 0 && events.indexOf("memory.recall") < events.lastIndexOf("context.query")); assert(!events.includes("memory.write"), "read MCP remains closed to mutation"); }); + +test("hosted Query reports a served learning refusal without exposing its credential", async () => { + const store: StoreEntry = { id: "field-notes", label: "Field notes", endpoint: "https://store.example", auth: "exchange", + exchangeRoute: "/auth/exchange", credentialName: "FIELD_NOTES_QUERY_TOKEN", bindingName: "FIELD_NOTES" }; + const fetcher: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/auth/exchange")) return Response.json({ token: "secret-reader-writer" }); + if (url.endsWith("/memory/claims")) return Response.json({ error: { identifier: "GrantWriteNotCovered" } }, { status: 403 }); + if (url.endsWith("/mcp")) { + const message = JSON.parse(String(init?.body)) as { id: number; params: { name: string } }; + const value = message.params.name === "context.describe" ? { tables: [ + { table: "filings", kind: "data", description: "Northwind filings" }, { table: "memory/facts", kind: "memory" }, + ] } : message.params.name === "memory.recall" ? { columns: [], rows: [] } : { + columns: ["_run_id", "_row_seq", "filing_id", "title"], rows: [["run-1", 0, "filing-1", "Northwind filing"]], + }; + return Response.json({ jsonrpc: "2.0", id: message.id, result: { structuredContent: value } }); + } + const prompt = String(init?.body); + return Response.json({ choices: [{ message: { content: prompt.includes("Distil") ? JSON.stringify({ entries: [ + { subject: "Northwind", key: "filings", learning: "Northwind filing arrived" }, + ] }) : "Northwind filing arrived [filing-1]." } }] }); + }; + const live = createLiveAnswer({ stores: [store], env: { CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", + CONTEXTFUL_MODEL_ID: "fixture" }, fetcher }); + const app = createConsole({ identity: { kind: "cognito", sessionSecret: "secret", queryGroup: "query", adminGroup: "admin" }, + stores: [{ id: store.id, label: store.label }], turn: live.turn, read: { list: async () => [] }, + control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) } }); + const cookie = issueCognitoSession({ subject: "alice", groups: ["query"], session: "stable-reading-session", + assertion: "verified-assertion", expiresAt: Math.floor(Date.now() / 1000) + 60 }, "secret"); + const response = await app.fetch(new Request("https://console.example/query/api/ask", { method: "POST", + headers: { cookie: `console_session=${cookie}`, origin: "https://console.example" }, + body: JSON.stringify({ store: store.id, question: "Which Northwind filing arrived?" }), + })); + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { error: { identifier: "ConsoleLearningWriteRefused" } }); +}); From ba994bdf6129e4ed423f0800ff02398738552495 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:54:54 +0800 Subject: [PATCH 110/155] test(console): cover shared Admin trust across registered stores --- apps/console/test/control.test.ts | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/apps/console/test/control.test.ts b/apps/console/test/control.test.ts index b41eed058..4a707f037 100644 --- a/apps/console/test/control.test.ts +++ b/apps/console/test/control.test.ts @@ -60,6 +60,28 @@ test("Admin control refuses ambiguous stores", async () => { await assert.rejects(control.edit({ expected: 1, document: "" }, "secret", { subject: "operator", grants: new Set(["admin"]) }), /ConsoleStoreSelectionRequired/); }); +test("registered stores share one Admin capability and console attestation domain", async () => { + const seen: string[] = []; + const control = createLiveControl({ + stores: [{ id: "one", endpoint: "https://one.example" }, { id: "two", endpoint: "https://two.example" }], + capability: "deployment-admin", attestationSecret: "deployment-attestation", + fetcher: async (input, init) => { + const url = new URL(String(input)); + const headers = new Headers(init?.headers); + assert.equal(headers.get("authorization"), "Bearer deployment-admin"); + const body = String(init?.body); + const canonical = `POST\n${url.pathname}\n${createHash("sha256").update(body).digest("hex")}\noperator\n${headers.get("x-contextful-operator-time")}\n${headers.get("x-contextful-operator-nonce")}`; + assert.equal(headers.get("x-contextful-operator-signature"), createHmac("sha256", "deployment-attestation").update(canonical).digest("hex")); + seen.push(url.hostname); + return Response.json({ expected: 1, nonce: "a".repeat(48) }); + }, + }); + const operator = { subject: "operator", grants: new Set<"admin">(["admin"]) }; + await control.edit({ store: "one", expected: 1, document: "" }, "deployment-admin", operator); + await control.edit({ store: "two", expected: 1, document: "" }, "deployment-admin", operator); + assert.deepEqual(seen, ["one.example", "two.example"]); +}); + test("the live control adapter signs the verified subject and exact request body", async () => { let signed = false; const control = createLiveControl({ stores: [{ id: "one", endpoint: "http://127.0.0.1:1" }], From d6cc2227b6187a4374a75abd5351ae0b8372c6a6 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:55:41 +0800 Subject: [PATCH 111/155] test(console): refuse invented durable memory from distillation --- apps/console/test/answer_memory.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index 3537e2872..0c3f53729 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -51,6 +51,7 @@ test("verified reading session lands evidence-backed claims and recalls them bef assert.doesNotMatch(prompt, /run-1/, "reserved provenance stays out of model input"); const content = prompt.includes("Distil") ? JSON.stringify({ entries: [ { subject: "Northwind", key: "filings", learning: "Northwind filings need review" }, + { subject: "Northwind", key: "invented", learning: "Northwind has a secret unreported acquisition" }, ] }) : "Northwind filings need review [filing-1]."; if (claims.length > 0 && !prompt.includes("Distil")) { assert.match(prompt, /Northwind filings need review/); From 0760d722c417ea71033ac173d9cc2431bc47c3e9 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 18:59:07 +0800 Subject: [PATCH 112/155] feat(console): persist scoped learning through served memory --- apps/console/src/answer_live.ts | 110 +++++++++++++++++++++++++++----- apps/console/src/entry.ts | 2 + apps/console/src/index.ts | 22 +++++-- apps/console/src/live.ts | 37 ++++++++--- spec/71-console.md | 2 + spec/adr/A-read.md | 2 + spec/cards/surface.md | 3 +- spec/guide/surface.md | 7 +- spec/spec.lock.json | 41 ++++++++---- spec/status.md | 15 +++-- spec/terms/surface.toml | 1 + 11 files changed, 188 insertions(+), 54 deletions(-) diff --git a/apps/console/src/answer_live.ts b/apps/console/src/answer_live.ts index 6d989e8de..31874c7a3 100644 --- a/apps/console/src/answer_live.ts +++ b/apps/console/src/answer_live.ts @@ -3,6 +3,7 @@ import type { Operator, TurnInput } from "./index.ts"; import type { Arrival, Conclusion } from "./brief.ts"; import { learnAfterAnswer, type Learning } from "./learn.ts"; import { consoleDenylist, createLiveTurn, openReader, redactText, selectTable, type LiveOptions } from "./live.ts"; +import { ConsoleError } from "./turn.ts"; type AnswerOptions = LiveOptions & { clock?: () => number }; @@ -19,6 +20,33 @@ function observed(subject: string, question: string, answer: string): boolean { return word.length >= 3 && (`${question} ${answer}`).toLowerCase().includes(word); } +function observedLearning(learning: string, answer: string): boolean { + const statement = learning.trim().replace(/\s+/g, " ").toLowerCase(); + return statement.length >= 3 && answer.replace(/\s+/g, " ").toLowerCase().includes(statement); +} + +function memoryTable(description: unknown): string | null { + const tables = object(description) && Array.isArray(description.tables) ? description.tables : []; + const memory = tables.flatMap((table) => object(table) && table.kind === "memory" && typeof table.table === "string" ? [table.table] : []); + return memory.length === 1 ? memory[0] : null; +} + +function rows(value: unknown): { columns: string[]; rows: unknown[][] } { + if (!object(value)) return { columns: [], rows: [] }; + return { columns: Array.isArray(value.columns) ? value.columns.filter((column): column is string => typeof column === "string") : [], + rows: Array.isArray(value.rows) ? value.rows.filter(Array.isArray) : [] }; +} + +function matchesEntity(question: string, subject: string): boolean { + const asked = new Set(question.toLowerCase().match(/[a-z0-9]{3,}/g) ?? []); + return (subject.toLowerCase().match(/[a-z0-9]{3,}/g) ?? []).some((part) => asked.has(part)); +} + +function namedSubjects(question: string): string[] { + const stop = new Set(["Which", "What", "Who", "When", "Where", "Why", "How", "The"]); + return (question.match(/\b[A-Z][A-Za-z0-9]{2,}\b/g) ?? []).filter((name) => !stop.has(name)); +} + export function createLiveAnswer(options: AnswerOptions) { const { env, fetcher = fetch, clock = Date.now } = options; const endpoint = env.CONTEXTFUL_MODEL_ENDPOINT; @@ -27,9 +55,7 @@ export function createLiveAnswer(options: AnswerOptions) { const modelUrl = new URL("chat/completions", `${endpoint.replace(/\/$/, "")}/`); const configuredDenylist = consoleDenylist(env); const liveTurn = createLiveTurn(options); - const learnings = new Map>(); const turns = new Map(); - const actorScope = (operator: Operator, store: string) => scope(operator.subject, store); const sessionScope = (operator: Operator, store: string) => scope(operator.subject, store, operator.assertion ?? ""); const denied = (operator: Operator) => [...configuredDenylist, ...(operator.assertion ? [operator.assertion] : [])]; @@ -51,32 +77,86 @@ export function createLiveAnswer(options: AnswerOptions) { const entries = object(parsed) && Array.isArray(parsed.entries) ? parsed.entries : []; return entries.flatMap((entry): Learning[] => { if (!object(entry) || typeof entry.subject !== "string" || typeof entry.key !== "string" || typeof entry.learning !== "string") return []; - if (!observed(entry.subject, question, answer)) return []; + if (!observed(entry.subject, question, answer) || !observedLearning(entry.learning, answer)) return []; return [{ subject: redactText(entry.subject, denied(operator)), key: redactText(entry.key, denied(operator)), learning: redactText(entry.learning, denied(operator)) }]; }); } catch { return []; } } + async function readMemory(operator: Operator, storeId: string, question?: string) { + if (!operator.session || !operator.assertion) return null; + const reader = await openReader(options, operator, storeId); + if (!reader.exchanged) return null; + const table = memoryTable(await reader.call("context.describe", {})); + if (!table) return null; + const expectedScope = `console:${operator.subject}:${operator.session}`; + let names = { columns: [] as string[], rows: [] as unknown[][] }; + try { + names = rows(await reader.call("context.query", { + sql: `SELECT DISTINCT subject FROM "${table.replaceAll('"', '""')}" LIMIT 100`, limit: 100, + })); + } catch (error) { + if (!(error instanceof ConsoleError && error.code === "ConsoleStoreReadRefused" && error.status === 400)) throw error; + } + const subjectAt = names.columns.indexOf("subject"); + const known = names.rows.map((row) => row[subjectAt]).filter((value): value is string => + typeof value === "string" && (question === undefined || matchesEntity(question, value))); + const subjects = [...new Set([...known, ...(question === undefined ? [] : namedSubjects(question))])].slice(0, 3); + const entries: Learning[] = []; + for (const subject of subjects) { + const response = rows(await reader.call("memory.recall", { table, subject, limit: 3 })); + const index = (name: string) => response.columns.indexOf(name); + for (const row of response.rows) { + if (row[index("scope")] !== expectedScope) continue; + const predicate = row[index("predicate")]; + const value = row[index("object")]; + if (typeof predicate === "string" && typeof value === "string") entries.push({ subject, key: predicate, learning: value }); + } + } + return { table, credential: reader.credential, scope: expectedScope, entries }; + } + async function turn(input: TurnInput) { - const result = await liveTurn(input); - const reader = actorScope(input.operator, input.store); + const memory = await readMemory(input.operator, input.store, input.question); + const recallOverlay = (memory?.entries ?? []).map((entry) => + redactText(`${entry.subject} ${entry.key}: ${entry.learning}`, denied(input.operator))).join("\n").slice(0, 8000); + const result = await liveTurn({ ...input, recallOverlay }); const session = sessionScope(input.operator, input.store); - const stream = (async function* () { yield result.answer; })(); - await learnAfterAnswer({ scope: reader, question: input.question, stream, - distill: ({ question, answer }) => distill(question, answer, input.operator), - land: async (key, entry) => { - const found = learnings.get(key) ?? new Map(); - found.set(`${entry.subject}\0${entry.key}`, entry); - learnings.set(key, found); - }, - }); + if (memory && result.evidence?.length) { + const stream = (async function* () { yield result.answer; })(); + await learnAfterAnswer({ scope: memory.scope, question: input.question, stream, + distill: ({ question, answer }) => distill(question, answer, input.operator), + land: async (_key, entry) => { + const claim = { subject: entry.subject, predicate: entry.key, object: entry.learning, + confidence: 0.8, evidence: result.evidence }; + const dedupKey = scope(input.operator.subject, input.operator.session ?? "", input.question, result.answer, + entry.subject, entry.key, entry.learning); + let response: Response; + try { + response = await fetcher(new URL("/memory/claims", options.stores.find((store) => store.id === input.store)!.endpoint), { + method: "POST", headers: { Authorization: `Bearer ${memory.credential}`, "Content-Type": "application/json" }, + body: JSON.stringify({ into: memory.table, actor: input.operator.subject, session: input.operator.session, + dedup_key: dedupKey, claim }), + }); + } catch { throw new ConsoleError("ConsoleLearningWriteRefused", "memory write unavailable", 503); } + if (!response.ok) throw new ConsoleError("ConsoleLearningWriteRefused", "memory write refused", 503); + let receipt: unknown; + try { receipt = await response.json(); } + catch { throw new ConsoleError("ConsoleLearningWriteRefused", "memory receipt malformed", 503); } + if (!object(receipt) || receipt.scope !== memory.scope || typeof receipt.landed !== "boolean") { + throw new ConsoleError("ConsoleLearningWriteRefused", "memory receipt unbound", 503); + } + }, + }); + } turns.set(session, (turns.get(session) ?? 0) + 1); return result; } async function brief(operator: Operator, store: string) { - const entries = [...(learnings.get(actorScope(operator, store))?.values() ?? [])]; + const durable = await readMemory(operator, store); + const entries = durable?.entries ?? []; const conclusions: Conclusion[] = entries.map((entry) => ({ subject: entry.subject, text: entry.learning, live: true })); return { session: { turns: turns.get(sessionScope(operator, store)) ?? 0, vantage: "present" }, diff --git a/apps/console/src/entry.ts b/apps/console/src/entry.ts index 6554daf08..42eb91363 100644 --- a/apps/console/src/entry.ts +++ b/apps/console/src/entry.ts @@ -40,6 +40,7 @@ async function identity(): Promise { sessionSecret: required("CONTEXTFUL_COGNITO_SESSION_SECRET"), queryGroup: required("CONTEXTFUL_COGNITO_QUERY_GROUP"), adminGroup: required("CONTEXTFUL_COGNITO_ADMIN_GROUP"), + memorySessionClaim: process.env.CONTEXTFUL_CONSOLE_MEMORY_SESSION_CLAIM, issuer: required("CONTEXTFUL_COGNITO_ISSUER"), clientId: required("CONTEXTFUL_COGNITO_CLIENT_ID"), authorizeUrl: required("CONTEXTFUL_COGNITO_AUTHORIZE_URL"), @@ -53,6 +54,7 @@ async function identity(): Promise { issuer: required("CONTEXTFUL_ACCESS_ISSUER"), queryAudience: required("CONTEXTFUL_QUERY_ACCESS_AUDIENCE"), adminAudience: required("CONTEXTFUL_ADMIN_ACCESS_AUDIENCE"), + memorySessionClaim: process.env.CONTEXTFUL_CONSOLE_MEMORY_SESSION_CLAIM, keys: await keysAt(required("CONTEXTFUL_ACCESS_JWKS_URL")), }; } diff --git a/apps/console/src/index.ts b/apps/console/src/index.ts index 80d54f5c3..fe793e935 100644 --- a/apps/console/src/index.ts +++ b/apps/console/src/index.ts @@ -4,13 +4,14 @@ import { answerDelivery } from "./publish.ts"; import { buildView, sanitizeView, type ResultRows } from "./render.ts"; import { ConsoleError } from "./turn.ts"; -export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin">; assertion?: string }; +export type Operator = { subject: string; grants: ReadonlySet<"query" | "admin">; assertion?: string; session?: string }; export type Store = { id: string; label: string }; export type AccessIdentity = { kind: "access"; issuer: string; queryAudience: string; adminAudience: string; + memorySessionClaim?: string; publicKey?: KeyObject | string; keys?: ReadonlyMap; }; @@ -19,6 +20,7 @@ export type CognitoIdentity = { sessionSecret: string; queryGroup: string; adminGroup: string; + memorySessionClaim?: string; issuer?: string; clientId?: string; authorizeUrl?: string; @@ -30,7 +32,8 @@ export type CognitoIdentity = { }; export type Identity = AccessIdentity | CognitoIdentity; export type TurnInput = { operator: Operator; store: string; question: string }; -export type TurnResult = { answer: string; sources: unknown[]; widgets: unknown[]; resultRows?: ResultRows; accessExplanation?: boolean; share?: boolean }; +export type TurnResult = { answer: string; sources: unknown[]; widgets: unknown[]; resultRows?: ResultRows; + evidence?: Array<{ table: string; run: string; seq: number }>; accessExplanation?: boolean; share?: boolean }; export type PackList = { entries: unknown[]; truncated: boolean; declined: number }; export type ConsoleAdapters = { identity: Identity; @@ -50,7 +53,7 @@ export type ConsoleAdapters = { }; }; -type Session = { subject: string; groups: string[]; expiresAt: number; assertion?: string }; +type Session = { subject: string; groups: string[]; expiresAt: number; assertion?: string; session?: string }; function encoded(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString("base64url"); @@ -91,11 +94,13 @@ function verifyCognitoSession(cookie: string, identity: CognitoIdentity): Operat if (!object(session) || typeof session.subject !== "string" || !session.subject || !Number.isSafeInteger(session.expiresAt) || (session.expiresAt as number) <= Math.floor(Date.now() / 1000) || !Array.isArray(session.groups) || !session.groups.every((group) => typeof group === "string") || - (session.assertion !== undefined && typeof session.assertion !== "string")) return null; + (session.assertion !== undefined && typeof session.assertion !== "string") || + (session.session !== undefined && typeof session.session !== "string")) return null; const grants = new Set<"query" | "admin">(); if (session.groups.includes(identity.queryGroup)) grants.add("query"); if (session.groups.includes(identity.adminGroup)) grants.add("admin"); - return { subject: session.subject, grants, assertion: typeof session.assertion === "string" ? session.assertion : undefined }; + return { subject: session.subject, grants, assertion: typeof session.assertion === "string" ? session.assertion : undefined, + session: session.session }; } function verifyAccess(assertion: string, identity: AccessIdentity): Operator | null { @@ -119,7 +124,8 @@ function verifyAccess(assertion: string, identity: AccessIdentity): Operator | n typeof identity.publicKey === "string" ? createPublicKey(identity.publicKey) : identity.publicKey; if (!key) return null; if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, signature)) return null; - return { subject: claims.sub, grants, assertion }; + const session = identity.memorySessionClaim ? claims[identity.memorySessionClaim] : undefined; + return { subject: claims.sub, grants, assertion, session: typeof session === "string" && session.trim() ? session : undefined }; } function operatorFor(request: Request, identity: Identity): Operator | null { @@ -188,7 +194,9 @@ async function cognitoCallback(request: Request, identity: CognitoIdentity): Pro if (!verify("RSA-SHA256", Buffer.from(`${parts[0]}.${parts[1]}`), key, Buffer.from(parts[2], "base64url"))) return refusal("ConsolePageForbidden"); const groups = Array.isArray(claims["cognito:groups"]) ? claims["cognito:groups"].filter((group): group is string => typeof group === "string") : []; const lifetime = Math.min(claims.exp as number, Math.floor(Date.now() / 1000) + 3600); - const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime, assertion: tokens.id_token }, identity.sessionSecret); + const task = identity.memorySessionClaim ? claims[identity.memorySessionClaim] : undefined; + const session = issueCognitoSession({ subject: claims.sub, groups, expiresAt: lifetime, assertion: tokens.id_token, + session: typeof task === "string" && task.trim() ? task : undefined }, identity.sessionSecret); const destination = groups.includes(identity.adminGroup) ? "/admin" : "/query"; return new Response(null, { status: 302, headers: { Location: destination, diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index 470999b2e..d22b7dfdb 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -46,7 +46,7 @@ export function selectTable(question: string, listing: unknown): string | null { function source(columns: string[], row: unknown[], table: string): Source | null { const at = (name: string) => row[columns.indexOf(name)]; - const idColumn = columns.find((name) => name.endsWith("_id") || name === "id"); + const idColumn = columns.find((name) => !name.startsWith("_") && (name.endsWith("_id") || name === "id")); const urlColumn = columns.find((name) => name === "source_url" || name === "url"); const id = idColumn && at(idColumn); const url = urlColumn && at(urlColumn); @@ -60,6 +60,7 @@ export async function openReader({ stores, env, fetcher = fetch }: LiveOptions, const store = stores.find((entry) => entry.id === storeId); if (!store) throw new ConsoleError("ConsoleRequestMalformed"); const shared = env[store.credentialName]; + let exchanged = false; const credential = store.auth === "exchange" ? await resolveReaderCredential({ shared, mint: async () => { @@ -78,6 +79,7 @@ export async function openReader({ stores, env, fetcher = fetch }: LiveOptions, } const value: unknown = await response.json(); if (!record(value) || typeof value.token !== "string" || !value.token.trim()) throw new ConsoleError("ConsoleTokenExchangeUnavailable", "exchange returned no credential", 503); + exchanged = true; return value.token; }, }) : shared; @@ -98,10 +100,10 @@ export async function openReader({ stores, env, fetcher = fetch }: LiveOptions, return message.result.structuredContent; }; - return { store, call, credential }; + return { store, call, credential, exchanged }; } -export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Promise { +export function createLiveTurn(options: LiveOptions): (input: TurnInput & { recallOverlay?: string }) => Promise { const { env, fetcher = fetch } = options; const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; const modelId = env.CONTEXTFUL_MODEL_ID; @@ -117,6 +119,7 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom const tables = [selected]; let remainingSources = 8; let resultRows: TurnResult["resultRows"]; + const evidence: NonNullable = []; const turn = createTurn({ denylist: denylist.filter((entry) => entry.length <= 128), tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read" as const, table })), @@ -128,20 +131,34 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom const columns = strings(result.columns); const rawRows = Array.isArray(result.rows) ? result.rows.filter(Array.isArray) : []; const sourced = rawRows.flatMap((row) => { const citation = source(columns, row, table); return citation ? [{ row, citation }] : []; }).slice(0, remainingSources); + const runIndex = columns.indexOf("_run_id"); + const seqIndex = columns.indexOf("_row_seq"); + for (const { row } of sourced) { + const run = row[runIndex]; + const seq = row[seqIndex]; + if (typeof run === "string" && run && Number.isSafeInteger(seq) && (seq as number) >= 0) { + evidence.push({ table, run, seq: seq as number }); + } + } remainingSources -= sourced.length; - const cleanRows = sourced.map(({ row }) => row.map((value) => typeof value === "string" ? redactText(value, denylist) : value)); - resultRows = { columns, rows: cleanRows }; + const visible = columns.flatMap((column, index) => column.startsWith("_") ? [] : [{ column, index }]); + const cleanRows = sourced.map(({ row }) => visible.map(({ index }) => { + const value = row[index]; + return typeof value === "string" ? redactText(value, denylist) : value; + })); + resultRows = { columns: visible.map(({ column }) => column), rows: cleanRows }; return { - rows: cleanRows.map((row) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), + rows: cleanRows.map((row) => Object.fromEntries(visible.map(({ column }, index) => [column, row[index]]))), sources: sourced.map(({ citation }) => ({ ...citation, label: redactText(citation.label, denylist) })), }; } }, - synthesize: async function* ({ question, results, sources }) { + overlay: async () => input.recallOverlay ?? null, + synthesize: async function* ({ question, results, sources, overlay }) { const response = await fetcher(new URL("chat/completions", `${modelEndpoint.replace(/\/$/, "")}/`), { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ model: modelId, messages: [ - { role: "system", content: "Answer only from the supplied governed rows. Cite source IDs in square brackets. Decline unsupported claims." }, - { role: "user", content: JSON.stringify({ question, rows: results.flatMap((result) => result.rows), sources }) }, + { role: "system", content: "Answer only from the supplied governed rows. Memory may guide context but never supplies evidence. Cite source IDs in square brackets. Decline unsupported claims." }, + { role: "user", content: JSON.stringify({ question, rows: results.flatMap((result) => result.rows), sources, memory: overlay }) }, ] }), }); if (!response.ok) throw new ConsoleError("ConsoleModelUnavailable", `model answered ${response.status}`, 503); @@ -153,6 +170,6 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom }, }); const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); - return { answer: answer.text, sources: answer.sources, widgets: [], resultRows }; + return { answer: answer.text, sources: answer.sources, widgets: [], resultRows, evidence }; }; } diff --git a/spec/71-console.md b/spec/71-console.md index b55f49836..e97dee408 100644 --- a/spec/71-console.md +++ b/spec/71-console.md @@ -283,6 +283,8 @@ The reading loop's memory: recall ahead of planning, the per-turn distillation a - `distillation` — After the answer streams, a second pass distils the exchange into at most 3 entries shaped `{subject, key, learning}`, zero included. - `unscoped` — A distilled conclusion landing without the reading-session scope raises `ConsoleLearningUnscoped`. *A-surface* +- `write-refused` — A served claim write that refuses or returns no scope-bound receipt raises `ConsoleLearningWriteRefused` to Query before answer publication. + *because a failed write cannot count as a durable learning* - `subject-resolution` — Distillation preserves the subject it observes; recall resolves that subject through entity matching. *A-surface* diff --git a/spec/adr/A-read.md b/spec/adr/A-read.md index 3e4f250a2..840763b41 100644 --- a/spec/adr/A-read.md +++ b/spec/adr/A-read.md @@ -100,6 +100,8 @@ Decision: a separate served route admits the server-held credential, binds the a Consequences: claim writes share memory's evidence, supersession, attribution and commit path. A credential with only read grants cannot land a claim. +Integration: `CONTEXTFUL_CONSOLE_MEMORY_SESSION_CLAIM` names a stable signed claim. Exchange maps `sub` to `on_behalf_of`, that claim to `task`, and grants evidence/claims read plus claims write. Missing mappings, tables or evidence suppress landing. Subject-keyed recall admits broad claims-table readers; the console filters its exact actor/session scope before model input. + ## Memory writes validate or dead-letter, and outcomes settle under their source `read.synthesize` validates every candidate against the declared output schema, retries with the error up to 3 attempts per batch, then dead-letters the response, template hash and drop reason with the cursor held. The relation vocabulary is a reserved core plus declared types; an undeclared edge dead-letters while the batch lands. `read.resolve-entity` dead-letters ambiguous mentions and dangling endpoints. `read.settle` requires one resolution form and one source — `metric`, `adjudicator` or `manual`; metric comparators evaluate outside the engine, verdicts carry an `http`/`https` citation, self-rated outcomes carry a null verdict, and the scored and unresolved views partition the join. diff --git a/spec/cards/surface.md b/spec/cards/surface.md index b367d1ea7..8a397f2db 100644 --- a/spec/cards/surface.md +++ b/spec/cards/surface.md @@ -22,7 +22,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `plan-turn` | The shape of one turn: planner scaffolding, the replanning round, the answerability test, the code path, and the store's prompt overlay. | 6 | | `set-vantage` | A session's time basis: its vantage, the bound each leg carries, the snapshot timeline and the arrivals strip. | 3 | | `browse` | What a store advertises before a question: discovered chips, humanized labels, the insights panel and the file gallery. | 0 | -| `learn` | The reading loop's memory: recall ahead of planning, the per-turn distillation and the labels its conclusions inherit. | 3 | +| `learn` | The reading loop's memory: recall ahead of planning, the per-turn distillation and the labels its conclusions inherit. | 4 | | `render` | The three channels, the component union, deterministic component choice, view hints and the sanitizer walk. | 3 | | `brief` | The proactive greeting card: its derivation, payload, matching tiers, client-side clock and the absence it earns. | 6 | | `publish-answer` | The moment an answer computed for one operator reaches a place other people read. | 3 | @@ -76,6 +76,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/s | `ConsoleVantageUnparseable` | A vantage that is neither a calendar day nor an instant. | `surface.set-vantage.unparseable` | | `ConsoleWebBoundUnparseable` | A web leg whose publication bound does not parse. | `surface.set-vantage.web-bound-unparseable` | | `ConsoleLearningUnscoped` | A distilled conclusion landing without the reading-session scope. | `surface.learn.unscoped` | +| `ConsoleLearningWriteRefused` | A served claim write that refuses or returns no scope-bound receipt. | `surface.learn.write-refused` | | `ConsoleViewNotServerBuilt` | A view from a client or the model rather than inferred from a result. | `surface.render.client-authored-view` | | `ConsoleBriefUnavailable` | A greeting derivation that errored or overran its budget. | `surface.brief.absence-is-earned` | | `VisibilityShareAffordance` | A share control offered on access-explanation output. | `surface.publish-answer.share-affordance` | diff --git a/spec/guide/surface.md b/spec/guide/surface.md index 99781706e..e77e4b737 100644 --- a/spec/guide/surface.md +++ b/spec/guide/surface.md @@ -111,9 +111,10 @@ The tool return splits into grounding for the model, a view and internals. Synth streams through the redactor ({{surface.speak.redactor-lookahead}}). The rows carry a date column and a measure over distinct days, so component choice draws a line ({{surface.render.component-choice}}), and a source list follows -({{surface.ground.sources-per-turn}}). After the answer, distillation writes a few -conclusions ({{surface.learn.distillation}}) under the reading-session scope -({{surface.learn.unscoped}}). +({{surface.ground.sources-per-turn}}). After the answer, +{{surface.learn.distillation}} lands through {{read.revise.served-write}} under +{{surface.learn.unscoped}} and {{read.revise.served-scope}}. {{read.recall.keyed}} +precedes planning; {{surface.learn.write-refused}} withholds publication. The following day the greeting card appears only because a new row matches one of those conclusions within budget ({{surface.brief.absence-is-earned}}). If the operator wants the diff --git a/spec/spec.lock.json b/spec/spec.lock.json index d27c6a241..9049d93d4 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -19695,6 +19695,17 @@ "file": "spec/71-console.md", "line": 284 }, + { + "id": "surface.learn.write-refused", + "contract": "surface", + "operation": "learn", + "subject": "write-refused", + "kind": "refusal", + "statement": "A served claim write that refuses or returns no scope-bound receipt raises `ConsoleLearningWriteRefused` to Query before answer publication.", + "why": "because a failed write cannot count as a durable learning", + "file": "spec/71-console.md", + "line": 286 + }, { "id": "surface.learn.subject-resolution", "contract": "surface", @@ -19704,7 +19715,7 @@ "statement": "Distillation preserves the subject it observes; recall resolves that subject through entity matching.", "why": "A-surface", "file": "spec/71-console.md", - "line": 286 + "line": 288 }, { "id": "surface.render.client-authored-view", @@ -19715,7 +19726,7 @@ "statement": "A view specification arriving from a client, or composed by the model, raises `ConsoleViewNotServerBuilt`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 293 + "line": 295 }, { "id": "surface.render.component-choice", @@ -19726,7 +19737,7 @@ "statement": "One row with one measure draws a metric; a date column with a measure over 3 rows on distinct days draws a line; anything else draws a table offering a bar view.", "why": "", "file": "spec/71-console.md", - "line": 295 + "line": 297 }, { "id": "surface.render.older-transcript", @@ -19737,7 +19748,7 @@ "statement": "A client reading a transcript with an unknown component-union member renders its view as a table.", "why": "A-surface", "file": "spec/71-console.md", - "line": 296 + "line": 298 }, { "id": "surface.brief.absence-is-earned", @@ -19748,7 +19759,7 @@ "statement": "The card needs a session with no turns, at present time, a live conclusion, an arrived row matching an interest, and a derivation inside its budget. An error or timeout raises `ConsoleBriefUnavailable`, and no card renders.", "why": "A-surface", "file": "spec/71-console.md", - "line": 332 + "line": 334 }, { "id": "surface.brief.topic-tier", @@ -19759,7 +19770,7 @@ "statement": "The topic tier needs at least 2 tokens shared between the conclusion's subject and text and the row's label and topics, one of them naming the subject.", "why": "", "file": "spec/71-console.md", - "line": 334 + "line": 336 }, { "id": "surface.brief.card-subjects", @@ -19770,7 +19781,7 @@ "statement": "A card carries at most 3 subjects.", "why": "", "file": "spec/71-console.md", - "line": 335 + "line": 337 }, { "id": "surface.brief.articles-per-subject", @@ -19781,7 +19792,7 @@ "statement": "A subject carries at most 3 entries of matched article.", "why": "", "file": "spec/71-console.md", - "line": 336 + "line": 338 }, { "id": "surface.brief.catchup-window", @@ -19792,7 +19803,7 @@ "statement": "The backend caps the requested window at 7 d.", "why": "", "file": "spec/71-console.md", - "line": 337 + "line": 339 }, { "id": "surface.brief.greeting", @@ -19803,7 +19814,7 @@ "statement": "Query derives a greeting card from arrived rows matching live conclusions in a turnless present-time session.", "why": "A-surface", "file": "spec/71-console.md", - "line": 338 + "line": 340 }, { "id": "surface.publish-answer.share-affordance", @@ -19814,7 +19825,7 @@ "statement": "A surface offering a share control on access-explanation output raises `VisibilityShareAffordance`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 345 + "line": 347 }, { "id": "surface.publish-answer.askerless-audience", @@ -19825,7 +19836,7 @@ "statement": "A scheduled job posting to an audience under a service identity raises `VisibilityAskerlessAudience`, naming the destination; a scheduled post's corpus is narrowed in the reviewed manifest to what the destination reaches.", "why": "A-surface", "file": "spec/71-console.md", - "line": 347 + "line": 349 }, { "id": "surface.publish-answer.one-operator-principal", @@ -19836,7 +19847,7 @@ "statement": "An answer uses one operator's credential; the console offers no room-intersection principal.", "why": "A-surface", "file": "spec/71-console.md", - "line": 349 + "line": 351 }, { "id": "assurance.model.package", @@ -28552,6 +28563,10 @@ "clause": "surface.learn.unscoped", "gloss": "A distilled conclusion landing without the reading-session scope." }, + "ConsoleLearningWriteRefused": { + "clause": "surface.learn.write-refused", + "gloss": "A served claim write that refuses or returns no scope-bound receipt." + }, "ConsoleMutatingToolRequested": { "clause": "surface.ground.mutating-tool", "gloss": "A client-reachable path naming a write." diff --git a/spec/status.md b/spec/status.md index 433edd1ef..99305a6a8 100644 --- a/spec/status.md +++ b/spec/status.md @@ -10,12 +10,12 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 218 | 0 | 218 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 40 | 0 | 40 | | `disclosure` | 3 | 16 | 122 | 64 | 13 | 7 | 54 | 0 | 54 | -| `read` | 2 | 15 | 179 | 29 | 22 | 21 | 163 | 0 | 163 | +| `read` | 2 | 15 | 185 | 33 | 22 | 21 | 168 | 0 | 168 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 106 | 49 | 22 | 0 | 65 | 0 | 65 | +| `surface` | 2 | 20 | 107 | 50 | 22 | 0 | 65 | 0 | 65 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2013 | 581 | 188 | 127 | 1379 | 0 | | +| **total** | 19 | 155 | 2020 | 586 | 188 | 127 | 1384 | 0 | | Decision records: 18. @@ -32,12 +32,12 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 4 — Ingest | 25 | 496 | 313 | passing | open | | 5 — The read face under enforcement | 17 | 252 | 192 | passing | open | | 6 — Sync and replicas | 8 | 73 | 68 | passing | closed | -| 7 — Memory | 6 | 37 | 37 | passing | closed | +| 7 — Memory | 6 | 43 | 42 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 49 | 14 | open | open | +| 12 — The operator console | 12 | 50 | 14 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -844,11 +844,16 @@ Unscheduled operations: 10. | `read.retrieve.token-cap` | `crates/contextful-core/tests/integration/read/retrieve.rs::the_token_set_is_capped_and_an_empty_set_omits_the_floor` | performed | | `read.retrieve.token-length-floor` | `crates/contextful-core/tests/integration/read/retrieve.rs::an_ascii_run_below_the_length_floor_leaves` | performed | | `read.retrieve.unsatisfiable-arm-drops` | `crates/contextful-context/tests/integration/read/retrieve.rs::a_filter_binds_named_columns_and_a_table_lacking_one_drops_its_arm` | performed | +| `read.revise.browser-request` | `spec/pins.toml` | performed | | `read.revise.citation-live` | `crates/contextful-memory/tests/integration/evidence.rs::a_write_citing_a_superseded_version_refuses` | performed | | `read.revise.dedup-key` | `crates/contextful-memory/tests/integration/write.rs::a_dedup_key_seeds_the_claim_id_and_a_retry_lands_nothing` | performed | | `read.revise.direct-write` | `crates/contextful-core/tests/integration/memory/revise.rs::the_direct_write_accepts_claims_alone` | performed | +| `read.revise.malformed-request` | `spec/pins.toml` | performed | | `read.revise.observed-at` | `crates/contextful-memory/tests/integration/write.rs::an_observed_write_lands_valid_from_its_observed_instant` | performed | | `read.revise.observed-order` | `crates/contextful-memory/tests/integration/write.rs::an_observation_before_a_live_contradicting_prior_refuses` | performed | +| `read.revise.served-dedup` | `spec/pins.toml` | performed | +| `read.revise.served-scope` | `spec/pins.toml` | performed | +| `read.revise.served-write` | `spec/pins.toml` | performed | | `read.revise.supersede` | `crates/contextful-core/tests/integration/memory/revise.rs::an_equal_or_higher_claim_retires_its_prior_on_one_line` | performed | | `read.revise.tier` | `crates/contextful-core/tests/integration/memory/revise.rs::tier_follows_the_write_path_and_the_lowest_grounding` | performed | | `read.settle.grace-window` | `crates/contextful-core/tests/integration/memory/settle.rs::the_label_window_runs_through_the_deadline_plus_a_day` | performed | diff --git a/spec/terms/surface.toml b/spec/terms/surface.toml index 37b185793..d1fe34548 100644 --- a/spec/terms/surface.toml +++ b/spec/terms/surface.toml @@ -68,6 +68,7 @@ ConsolePlannerReachedMemory = { clause = "surface.plan-turn.planner-reached-me ConsoleVantageUnparseable = { clause = "surface.set-vantage.unparseable", gloss = "A vantage that is neither a calendar day nor an instant." } ConsoleWebBoundUnparseable = { clause = "surface.set-vantage.web-bound-unparseable", gloss = "A web leg whose publication bound does not parse." } ConsoleLearningUnscoped = { clause = "surface.learn.unscoped", gloss = "A distilled conclusion landing without the reading-session scope." } +ConsoleLearningWriteRefused = { clause = "surface.learn.write-refused", gloss = "A served claim write that refuses or returns no scope-bound receipt." } ConsoleViewNotServerBuilt = { clause = "surface.render.client-authored-view", gloss = "A view from a client or the model rather than inferred from a result." } ConsoleBriefUnavailable = { clause = "surface.brief.absence-is-earned", gloss = "A greeting derivation that errored or overran its budget." } VisibilityShareAffordance = { clause = "surface.publish-answer.share-affordance", gloss = "A share control offered on access-explanation output." } From 135dd3a34e8d225fd45a562644eebc8d70438a40 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Tue, 6 Oct 2026 19:03:02 +0800 Subject: [PATCH 113/155] test(console): pin served learning refusal --- apps/console/test/answer_memory.test.ts | 8 +++++++- spec/pins.toml | 2 +- spec/status.md | 7 ++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apps/console/test/answer_memory.test.ts b/apps/console/test/answer_memory.test.ts index 0c3f53729..9efce5bfe 100644 --- a/apps/console/test/answer_memory.test.ts +++ b/apps/console/test/answer_memory.test.ts @@ -85,7 +85,7 @@ test("verified reading session lands evidence-backed claims and recalls them bef assert(!events.includes("memory.write"), "read MCP remains closed to mutation"); }); -test("hosted Query reports a served learning refusal without exposing its credential", async () => { +function servedLearningRefusalFixture() { const store: StoreEntry = { id: "field-notes", label: "Field notes", endpoint: "https://store.example", auth: "exchange", exchangeRoute: "/auth/exchange", credentialName: "FIELD_NOTES_QUERY_TOKEN", bindingName: "FIELD_NOTES" }; const fetcher: typeof fetch = async (input, init) => { @@ -113,6 +113,12 @@ test("hosted Query reports a served learning refusal without exposing its creden control: { workflows: async () => ({}), record: async () => ({}), edit: async () => ({}), apply: async () => ({}) } }); const cookie = issueCognitoSession({ subject: "alice", groups: ["query"], session: "stable-reading-session", assertion: "verified-assertion", expiresAt: Math.floor(Date.now() / 1000) + 60 }, "secret"); + return { app, cookie, store }; +} + +// spec: surface.learn.write-refused@32cba5ac +test("hosted Query reports a served learning refusal without exposing its credential", async () => { + const { app, cookie, store } = servedLearningRefusalFixture(); const response = await app.fetch(new Request("https://console.example/query/api/ask", { method: "POST", headers: { cookie: `console_session=${cookie}`, origin: "https://console.example" }, body: JSON.stringify({ store: store.id, question: "Which Northwind filing arrived?" }), diff --git a/spec/pins.toml b/spec/pins.toml index 1d2bcbf4d..d2ed8ea1a 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -68,5 +68,5 @@ disclosure = 54 read = 168 run = 261 store = 200 -surface = 65 +surface = 66 topology = 38 diff --git a/spec/status.md b/spec/status.md index 99305a6a8..a8fe0a9fd 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 185 | 33 | 22 | 21 | 168 | 0 | 168 | | `run` | 3 | 26 | 373 | 98 | 36 | 37 | 261 | 0 | 261 | | `store` | 1 | 17 | 237 | 55 | 13 | 20 | 200 | 0 | 200 | -| `surface` | 2 | 20 | 107 | 50 | 22 | 0 | 65 | 0 | 65 | +| `surface` | 2 | 20 | 107 | 50 | 22 | 0 | 66 | 0 | 66 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2020 | 586 | 188 | 127 | 1384 | 0 | | +| **total** | 19 | 155 | 2020 | 586 | 188 | 127 | 1385 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 68 | 56 | passing | open | | 11 — The derive tier | 7 | 78 | 55 | passing | open | -| 12 — The operator console | 12 | 50 | 14 | open | open | +| 12 — The operator console | 12 | 50 | 15 | open | open | | 13 — Disclosure | 5 | 21 | 4 | open | open | | 14 — Assurance | 7 | 134 | 84 | passing | closed | @@ -1376,6 +1376,7 @@ Unscheduled operations: 10. | `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | | `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | | `surface.learn.unscoped` | `apps/console/test/learn.test.ts::apps/console/test/learn.test.ts::unscoped learning refuses before distillation or landing` | performed | +| `surface.learn.write-refused` | `apps/console/test/answer_memory.test.ts::apps/console/test/answer_memory.test.ts::hosted Query reports a served learning refusal without exposing its credential` | performed | | `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | | `surface.publish-answer.askerless-audience` | `apps/console/test/publish.test.ts::apps/console/test/publish.test.ts::a scheduled service post refuses an audience and names its destination` | performed | | `surface.publish-answer.share-affordance` | `apps/console/test/publish.test.ts::apps/console/test/publish.test.ts::an access explanation cannot offer a share control` | performed | From 6a1ebb62e1fb30a772ec2d6bb2976c0301dddc8b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:07:22 +0800 Subject: [PATCH 114/155] Keep store guide within corpus word bound --- spec/guide/store.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/guide/store.md b/spec/guide/store.md index f72ccb81b..e94ecca4e 100644 --- a/spec/guide/store.md +++ b/spec/guide/store.md @@ -16,13 +16,13 @@ Files never change once written ({{store.lay-out.immutable-files}}). A write add A read resolves an explicit sorted file list from the pointer and the manifests, never a glob ({{store.reconcile.explicit-file-list}}). An unkeyed table reads as the union of its committed runs ({{store.declare.unkeyed-union}}); a keyed table reads through a view that keeps one survivor per key ({{store.declare.dedup-view}}), picked by `order_by` ({{store.declare.order-by-default}}). Schemas grow additively: a new column joins and older files read it as null ({{store.reconcile.additive}}). The type lattice admits one promotion ({{store.reconcile.lattice}}), and any other clash refuses at the write ({{store.reconcile.incompatible}}). A source with mixed scalar kinds uses {{store.reconcile.variant}}. -A complete empty replacement marks a frontier with no files ({{store.declare.empty-replacement}}). +An empty replacement marks a fileless frontier ({{store.declare.empty-replacement}}). Runs accumulate until a fold compacts them. A pass writes Parquet and every sidecar into staging ({{store.fold.pass}}), then publishes by replacing `_pointer.json` conditioned on the ETag it read at the start ({{store.fold.pointer-commit}}). Readers see a snapshot and its sidecars together or not at all ({{store.fold.partial-snapshot}}), and a statement in flight keeps the snapshot it started on ({{store.fold.non-blocking}}). A table's {{store.declare.retain-rows}} declaration gives rows an age limit. Reads enforce the cutoff at the statement ({{store.bound-time.row-age-cutoff}}), and a fold omits expired rows from the next snapshot ({{store.fold.row-retention}}). The pass reports its cutoff, expired rows, dropped partitions and collected directories ({{store.fold.row-retention-report}}). -The catalogs have distinct lifetimes: `derived.sqlite` rebuilds from the tree ({{store.lay-out.derived-catalog}}); `machine.sqlite` keeps machine-local journal, cursors and leases ({{store.lay-out.machine-catalog}}). Both sit behind ports ({{store.lay-out.catalog-ports}}). +Catalog lifetimes differ: `derived.sqlite` rebuilds from the tree ({{store.lay-out.derived-catalog}}); `machine.sqlite` keeps machine-local journal, cursors and leases ({{store.lay-out.machine-catalog}}). Both use ports ({{store.lay-out.catalog-ports}}). A bucket mirrors the tree. A push uploads changed files, commits the bucket manifest by compare-and-set, and re-merges a lost race within a bound ({{store.merge.retries}}). A pull checks every digest ({{store.pull.digest-mismatch}}), refuses a different store identity ({{store.pull.identity-conflict}}), and writes no pointer unless it converges ({{store.pull.unconverged}}). Leases keep writers apart, with a fence storage checks on commit ({{store.lease.stale-fence}}). A replica is read-only ({{store.replicate.write-refused}}) and holds a snapshot whole or not at all ({{store.replicate.partial-parquet}}). From 57c275afc99946205d05f232ca675a8d7cf2b256 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:09:15 +0800 Subject: [PATCH 115/155] Bound store guide to seven hundred words --- spec/guide/store.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/guide/store.md b/spec/guide/store.md index e94ecca4e..9c43862d2 100644 --- a/spec/guide/store.md +++ b/spec/guide/store.md @@ -26,7 +26,7 @@ Catalog lifetimes differ: `derived.sqlite` rebuilds from the tree ({{store.lay-o A bucket mirrors the tree. A push uploads changed files, commits the bucket manifest by compare-and-set, and re-merges a lost race within a bound ({{store.merge.retries}}). A pull checks every digest ({{store.pull.digest-mismatch}}), refuses a different store identity ({{store.pull.identity-conflict}}), and writes no pointer unless it converges ({{store.pull.unconverged}}). Leases keep writers apart, with a fence storage checks on commit ({{store.lease.stale-fence}}). A replica is read-only ({{store.replicate.write-refused}}) and holds a snapshot whole or not at all ({{store.replicate.partial-parquet}}). -A pull carries the signed control head ({{store.pull.control-head}}) without changing the applied pointer. The control reconciler verifies it before arming. +A pull carries the signed control head ({{store.pull.control-head}}) without changing the applied pointer. The reconciler verifies before arming. ## Worked example From 5614a9574ac9e01075bbd26ccc4213d53af22ec3 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:11:31 +0800 Subject: [PATCH 116/155] Retain apply admission settings on HTTP control Test-First: refactor --- crates/contextful-cli/src/serve.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index b9bc5efa8..34f7c43e2 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -7,7 +7,7 @@ //! network transport, which admits each one on its own credential. Every value is //! resolved before the listener binds, so a process that cannot serve binds nothing. -use crate::admit::{face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; +use crate::admit::{AdmitArgs, face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; @@ -111,6 +111,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; + let control_admit = AdmitArgs { public_key: args.public_key.clone(), audience: args.audience.clone(), denylist: args.denylist.clone(), keyset: args.keyset.clone(), holder_key: None }; let control = |request: &HttpRequest| -> HttpResponse { let answer = match request.target.split('?').next().unwrap_or_default() { WORKFLOWS_PATH => crate::cadence::published(&located.project, &located.declaration), @@ -124,7 +125,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { return HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); } let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; - crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), tasks) + crate::cadence::apply(&project, Some(located.declaration.clone()), body["id"].as_str(), tasks, &control_admit, None) .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) } _ => unreachable!(), From 4c68c9ef51c5058e62cd8ce520ede57ed407bb29 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:14:33 +0800 Subject: [PATCH 117/155] Test signed receipt integrity for console drafts --- .../contextful-cli/tests/integration/serve.rs | 44 +++++++++++++++++++ .../tests/integration/control.rs | 18 ++++++++ 2 files changed, 62 insertions(+) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 562d423f4..8b2b97844 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -691,3 +691,47 @@ fn serve_caches_a_repeated_statement_under_its_declared_budget() { assert_eq!(states(&["--result-cache-bytes", "65536"]), [json!("miss"), json!("hit")]); assert_eq!(states(&[]), [Value::Null, Value::Null], "no declared budget caches nothing"); } + +#[test] +fn synced_admin_draft_apply_keeps_a_signed_receipt_under_network_authority() { + let (dir, public) = project(); + let root = dir.path(); + let original = "[[pipeline]]\nid = \"filings-flow\"\nschedule = \"every 1h\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); + let admin = stdout(&run(root, &["token", "mint", "--issuer-key", ".contextful/issuer.seed", "--on-behalf-of", "user://service@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", "900"])); + let bucket = tempfile::tempdir().unwrap(); + std::fs::write(root.join(".contextful/context/research/config.toml"), format!("[node]\nid = \"site-a\"\n[sync]\nendpoint = \"file://{}\"\nbucket = \"control-test\"\nprefix = \"team\"\ncoordination = \"single-writer\"\n", bucket.path().display())).unwrap(); + let imported = Command::new(env!("CARGO_BIN_EXE_contextful")).args(["pipeline", "import", "--project", "research", "--public-key", &public, "--audience", AUD]).env("CONTEXTFUL_TOKEN", &admin).current_dir(root).output().unwrap(); + stdout(&imported); + let (listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + assert_eq!(control(&addr, "GET", "/control/record", Some(&admin), "").0, 200); + let edit = json!({ "expected": 1, "document": original.replace("every 1h", "every 1d") }).to_string(); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, "").0, 403); + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + let stale = attestation("/control/edit", &edit, "operator-a", now - 120, &"1".repeat(32)); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &stale).0, 403); + let forged = attestation("/control/edit", &edit, "operator-a", now, &"2".repeat(32)).replace("Operator: operator-a", "Operator: operator-b"); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &forged).0, 403); + let signed = attestation("/control/edit", &edit, "operator-a", now, &"3".repeat(32)); + let (status, saved) = control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed); + assert_eq!(status, 200, "{saved}"); + assert_eq!(control_headers(&addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); + drop(listener); + let (_restart, restarted_addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + assert_eq!(control_headers(&restarted_addr, "POST", "/control/edit", Some(&admin), &edit, &signed).0, 403); + let apply = json!({ "expected": 1, "nonce": saved["nonce"] }).to_string(); + let signed = attestation("/control/apply", &apply, "operator-a", now, &"4".repeat(32)); + let (status, result) = control_headers(&restarted_addr, "POST", "/control/apply", Some(&admin), &apply, &signed); + assert_eq!(status, 200, "{result}"); + let receipts = root.join(".contextful/control/research"); + let receipt: contextful_policy::control_receipt::ControlReceipt = serde_json::from_str(&std::fs::read_to_string(receipts.join("receipt@v2.json")).unwrap()).unwrap(); + let snapshot = std::fs::read(receipts.join("manifest@v2.toml")).unwrap(); + receipt.verify("research", &snapshot, &[public.parse().unwrap()]).unwrap(); + assert!(receipt.parent.is_some()); + let entries = contextful_policy::audit::entries(&root.join(".contextful/audit")).unwrap(); + assert!(entries.iter().any(|entry| entry.attributes["contextful.operator.subject"] == "operator-a" && entry.attributes["contextful.control.operation"] == "apply")); + let (status, record) = control(&restarted_addr, "GET", "/control/record", Some(&admin), ""); + assert_eq!(status, 200, "{record}"); + assert!(record["entries"].as_array().unwrap().iter().any(|entry| entry["attributes"]["contextful.operator.subject"] == "operator-a")); +} diff --git a/crates/contextful-engine/tests/integration/control.rs b/crates/contextful-engine/tests/integration/control.rs index 08d896dbb..3e159976e 100644 --- a/crates/contextful-engine/tests/integration/control.rs +++ b/crates/contextful-engine/tests/integration/control.rs @@ -143,3 +143,21 @@ fn an_attested_import_resumes_v1_only_after_receipt_validation() { assert!(snaps.import_attested("first", sign, verify).is_err()); assert!(!dir.path().join("manifest@v2.toml").exists()); } + +#[test] +fn an_attested_draft_keeps_its_receipt_chain_and_refuses_failed_signing() { + let dir = tempfile::tempdir().unwrap(); + let snaps = SnapshotDir::open(dir.path()); + snaps.import_attested("first", |_, _| Ok("receipt one".into()), |_| Ok(())).unwrap(); + let draft = Draft::new(1, "second".into(), "alice".into()).unwrap(); + snaps.save_draft(&draft).unwrap(); + assert!(snaps.claim_draft_attested(&draft, |_, _| Err(ControlError::Storage("signer absent".into()))).is_err()); + assert_eq!(snaps.current().unwrap(), Some(1)); + assert_eq!(snaps.read_draft().unwrap(), draft); + assert_eq!(snaps.claim_draft_attested(&draft, |version, prior| { + assert_eq!((version, prior), (2, Some("receipt one"))); + Ok("receipt two".into()) + }).unwrap(), 2); + assert_eq!(std::fs::read_to_string(dir.path().join("receipt@v2.json")).unwrap(), "receipt two"); + assert_eq!(snaps.current().unwrap(), Some(2)); +} From de0704aa2699e733177f26289a82b17d54d17724 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:15:45 +0800 Subject: [PATCH 118/155] Keep gate discovery and generated corpus aligned --- AGENTS.md | 9 +++++---- spec/spec.lock.json | 2 +- tools/ci/tests/integration/target_dirs.rs | 7 +------ 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 009e8a3c9..50dd4c736 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,21 +121,22 @@ The gate measures commits, so commit before running it. The FlareDispatch GitHub App dispatches `contextful-ci gate` from same-repository pull-request heads. Its `contextful-gate` run publishes the -`flare-dispatch/contextful-gate` parent and 29 child check-runs: +`flare-dispatch/contextful-gate` parent, 29 static child check-runs, and one +test-first child per changed package: `flare-dispatch/check:` for each of `pins`, `toolchain`, `schema`, `workspace`, `acceptance`, `evaluate`, `features`, `crate-graph`, `connectors`, `surfaces`, `formal` and `budget`. The test-first, features and budget stages dispatch one check per part, `flare-dispatch/check:test-first.validate` and `flare-dispatch/check:test-first.`, `flare-dispatch/check:features.` and `flare-dispatch/check:budget.`, so -each fits the sandbox's wall clock; `contextful-ci stages --parts` prints the list, and +each fits the sandbox's wall clock; `contextful-ci stages --parts --base ` prints the list, and `--stage .` runs one part. A selected subset runs in that order and refuses a stage whose predecessor's output is absent; `--predecessors` runs those too. A local run and the remote check invoke the identical command. `contextful-ci`'s suite asserts 29 -dispatchable parts. FlareDispatch runs `contextful-measures` nightly against the default +static dispatchable parts. FlareDispatch runs `contextful-measures` nightly against the default branch and attaches its report to `refs/notes/measures`. A `v*` tag starts `contextful-release`, with `contextful-release-cell` for the ten cells in `contextful-ci release --plan`, `contextful-release-formula` for formulae and SHA256SUMS, and three independently tagged container images. The disabled -ruleset proposal under `.github/rulesets/` lists the parent and all 29 children. +ruleset proposal under `.github/rulesets/` lists the parent and all 29 static children. The CLI differential module runs in `workspace.cli-formal` and `features.formal-`; `workspace.cli` and `features.binary-` run the remaining CLI tests. diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 70be8b5ff..ecd6eb203 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -22187,7 +22187,7 @@ "operation": "gate", "subject": "remote-check", "kind": "behavior", - "statement": "The FlareDispatch pull-request webhook dispatches every part from `contextful-ci stages --parts`, each as `flare-dispatch/check:` on the head commit.", + "statement": "The FlareDispatch pull-request webhook dispatches every part from `contextful-ci stages --parts --base `, each as `flare-dispatch/check:` on the head commit.", "why": "A-assurance", "file": "spec/81-engineering.md", "line": 166 diff --git a/tools/ci/tests/integration/target_dirs.rs b/tools/ci/tests/integration/target_dirs.rs index c128eb4e3..37c12ff32 100644 --- a/tools/ci/tests/integration/target_dirs.rs +++ b/tools/ci/tests/integration/target_dirs.rs @@ -105,18 +105,13 @@ fn remote_workspace_parts_compile_the_union_and_run_each_package_suite() { assert!(calls.contains("test --workspace --exclude contextful-acceptance --no-run"), "{calls}"); assert_eq!(calls.lines().count(), 1, "compile runs no package suite: {calls}"); - std::fs::remove_file(&log).unwrap(); - let tested = r.gate_with_cargo(&recording(&log, 0), &["--stage", "workspace.cli"]); - assert!(tested.status.success(), "{}", stderr(&tested)); - let calls = std::fs::read_to_string(&log).unwrap(); - assert_eq!(calls.lines().count(), 1, "compile runs no package suite: {calls}"); - std::fs::remove_file(&log).unwrap(); let tested = r.gate_with_cargo(&recording(&log, 0), &["--stage", "workspace.cli"]); assert!(tested.status.success(), "{}", stderr(&tested)); let calls = std::fs::read_to_string(&log).unwrap(); assert!(calls.contains("test --package contextful-cli"), "{calls}"); assert!(calls.contains("--skip differential::"), "{calls}"); + assert_eq!(calls.lines().count(), 1, "CLI runs no workspace compile: {calls}"); std::fs::remove_file(&log).unwrap(); let formal = r.gate_with_cargo(&recording(&log, 0), &["--stage", "workspace.cli-formal"]); From 051fb40eff0db8a4a2121e96cd07c480b649b91f Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:20:36 +0800 Subject: [PATCH 119/155] Route scheduled protocol exploration through FlareDispatch --- .github/workflows/protocol-differential.yml | 29 --------------------- 1 file changed, 29 deletions(-) delete mode 100644 .github/workflows/protocol-differential.yml diff --git a/.github/workflows/protocol-differential.yml b/.github/workflows/protocol-differential.yml deleted file mode 100644 index cf331a1f8..000000000 --- a/.github/workflows/protocol-differential.yml +++ /dev/null @@ -1,29 +0,0 @@ -# The fixed seed runs in the workspace and evaluate stages; this schedule explores a -# fresh sequence over the same three nodes and four lease generations. -name: protocol differential - -on: - schedule: - - cron: "17 3 * * 0" - workflow_dispatch: - -permissions: - contents: read - -jobs: - random-seed: - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - - name: Provision the pinned Lean toolchain and run the fixed gate - run: cargo run --locked -q -p contextful-ci -- gate --predecessors --stage formal - - name: Compare a random protocol schedule - run: | - seed=$(od -An -N8 -tu8 /dev/urandom | tr -d ' ') - cargo run --locked -q -p contextful-cli --no-default-features --bin contextful -- formal protocol-differential --seed "$seed" --cases 256 - - if: failure() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: protocol-regressions - path: formal/protocol/regressions.jsonl From 694f65ed747cea58af915879337ff78fae7e28df Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:22:15 +0800 Subject: [PATCH 120/155] Sign synced Admin draft claims under network authority --- crates/contextful-cli/src/cadence.rs | 21 +++++++++++++++++-- crates/contextful-cli/src/serve.rs | 5 +++-- .../contextful-cli/tests/integration/serve.rs | 3 ++- crates/contextful-engine/src/control.rs | 11 +++++++++- 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index c1631f5e5..84b46e9cf 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -467,7 +467,7 @@ pub(crate) fn claim_operator_nonce(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks) -> Result<()> { +pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks, authority: &AdmittedAuthority, admit: &AdmitArgs) -> Result<()> { let (initial, _, control) = located(project, declaration.clone())?; let snapshots = owner(&control)?; snapshots.initialized()?; @@ -483,7 +483,16 @@ pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, e if owner.root() != snapshots.root() || fresh.declaration != initial.declaration { return Err(SurfaceError::ConfigOwnerUnconfigured("the store's control owner changed during validation".into()).into()); } - owner.claim_draft(&draft)?; + match SyncAttestation::for_authority(&fresh, authority.clone(), admit, None)? { + Some(attestation) => { + owner.claim_draft_attested(&draft, |next, previous| { + let prior_snapshot = owner.read(expected)?; + let parent = previous.map(|receipt| (receipt, expected, prior_snapshot.as_str())); + attestation.receipt(&fresh.project.name, next, parent, &draft.document) + })?; + } + None => { owner.claim_draft(&draft)?; } + } Ok(()) } @@ -576,6 +585,14 @@ impl SyncAttestation { let unavailable = |why: String| SurfaceError::ControlAttestationUnavailable(why); let (authority, _) = admit.admit(project.project.as_deref(), "a synced control claim") .map_err(|e| unavailable(format!("admin capability: {e:#}")))?; + Self::for_authority(l, authority, admit, issuer_key) + } + + fn for_authority(l: &Located, authority: AdmittedAuthority, admit: &AdmitArgs, issuer_key: Option<&Path>) -> Result> { + if crate::sync::sync_config(l)?.0.is_none() { + return Ok(None); + } + let unavailable = |why: String| SurfaceError::ControlAttestationUnavailable(why); if !authority.permits(Action::Admin, &[]) { return Err(unavailable("the credential carries no admin grant".into()).into()); } diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index aed52f93f..92418240f 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -7,7 +7,7 @@ //! network transport, which admits each one on its own credential. Every value is //! resolved before the listener binds, so a process that cannot serve binds nothing. -use crate::admit::{face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; +use crate::admit::{AdmitArgs, face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; @@ -188,6 +188,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; let audit = AuditLog::unanchored(located.project.audit_dir())?; let control_attestation_secret = std::env::var("CONTEXTFUL_CONTROL_ATTESTATION_SECRET").ok().filter(|secret| !secret.is_empty()); + let control_admit = AdmitArgs { public_key: args.public_key.clone(), audience: args.audience.clone(), denylist: args.denylist.clone(), keyset: args.keyset.clone(), holder_key: None }; let control = |request: &HttpRequest, authority: &AdmittedAuthority| -> HttpResponse { let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let path = request.target.split('?').next().unwrap_or_default(); @@ -241,7 +242,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { "contextful.control.draft_nonce": nonce })).is_err() { return HttpResponse::json(503, &json!({ "error": { "identifier": "AuditEntryUnpersisted" } })); } - crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, nonce, &operator, tasks) + crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, nonce, &operator, tasks, authority, &control_admit) .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) } } diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 8b2b97844..acf65bd16 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -727,7 +727,8 @@ fn synced_admin_draft_apply_keeps_a_signed_receipt_under_network_authority() { let receipts = root.join(".contextful/control/research"); let receipt: contextful_policy::control_receipt::ControlReceipt = serde_json::from_str(&std::fs::read_to_string(receipts.join("receipt@v2.json")).unwrap()).unwrap(); let snapshot = std::fs::read(receipts.join("manifest@v2.toml")).unwrap(); - receipt.verify("research", &snapshot, &[public.parse().unwrap()]).unwrap(); + let signer = contextful_policy::issue::SeedSigner::resolve(Some(&root.join(".contextful/issuer.seed"))).unwrap(); + receipt.verify("research", &snapshot, &[contextful_policy::issue::SignerKey::of(&signer)]).unwrap(); assert!(receipt.parent.is_some()); let entries = contextful_policy::audit::entries(&root.join(".contextful/audit")).unwrap(); assert!(entries.iter().any(|entry| entry.attributes["contextful.operator.subject"] == "operator-a" && entry.attributes["contextful.control.operation"] == "apply")); diff --git a/crates/contextful-engine/src/control.rs b/crates/contextful-engine/src/control.rs index 5266af1ab..a38a99794 100644 --- a/crates/contextful-engine/src/control.rs +++ b/crates/contextful-engine/src/control.rs @@ -168,13 +168,22 @@ impl SnapshotDir { /// Claim the exact draft revalidated by the caller under the existing pointer lock. pub fn claim_draft(&self, draft: &Draft) -> Result { + self.claim_draft_inner(draft, None) + } + + /// Claim a validated draft with its signed receipt before publishing the pointer. + pub fn claim_draft_attested(&self, draft: &Draft, receipt: impl Fn(u64, Option<&str>) -> Result) -> Result { + self.claim_draft_inner(draft, Some(&receipt)) + } + + fn claim_draft_inner(&self, draft: &Draft, receipt: Option<&ReceiptBuilder<'_>>) -> Result { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; self.expect_version(draft.expected)?; if self.read_draft()? != *draft { return Err(SurfaceError::ManifestVersionConflict("the validated draft changed before apply; reload and reapply".into()).into()); } - let version = self.claim_locked(Some(draft.expected), &draft.document, None, None)?; + let version = self.claim_locked(Some(draft.expected), &draft.document, receipt, None)?; let _ = std::fs::remove_file(self.root.join(DRAFT_FILE)); Ok(version) } From ec02f0413ad76b5eca933c08850e485a6eabbbaf Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:23:58 +0800 Subject: [PATCH 121/155] Regenerate console answer corpus Test-First: refactor --- spec/cards/read.md | 6 +- spec/spec.lock.json | 185 ++++++++++++++++++++++++++++++++++---------- spec/status.md | 30 +++++-- 3 files changed, 173 insertions(+), 48 deletions(-) diff --git a/spec/cards/read.md b/spec/cards/read.md index 3ec1a6f61..4d1c53e07 100644 --- a/spec/cards/read.md +++ b/spec/cards/read.md @@ -17,7 +17,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `embed` | The embedding port and its default, the model identifier beside each vector, the linked-backend identity a handshake reports, and refusals on an absent backend. | 6 | | `declare` | The five memory table shapes, their canonical columns, tier and scope, and the relation vocabulary with its cardinality. | 3 | | `synthesize` | Extract, Resolve and Consolidate; the dedup key, evidence support, the audit record and coverage gaps. | 9 | -| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 9 | +| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 15 | | `recall` | Serving memory: the ranked arm at the read's anchor, the keyed read at an observed instant, the two clocks, tier-first ordering, the scope filter, the tombstone filter, the evidence gate. | 18 | | `resolve-entity` | Mention-to-entity matching, aliases, the knowledge card, place identity and nearness, edges, and ownership answers. | 4 | | `settle` | Predictions, observations, the registration, the citation a verdict owes, and the label views. | 5 | @@ -49,6 +49,10 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `MemoryDirectWriteShapeRefused` | A direct write targeted a memory shape other than claims. | `read.revise.direct-write` | | `MemoryObservationOutOfOrder` | A claim was observed before a live contradicting claim of its line. | `read.revise.observed-order` | | `MemoryCitationNotLive` | A direct write cites a keyed row that is not its key's live version. | `read.revise.citation-live` | +| `MemoryClaimScopeRefused` | A served claim write names another actor or reading session. | `read.revise.served-scope` | +| `MemoryClaimBrowserRefused` | A browser-origin request reached the served claim-write route. | `read.revise.browser-request` | +| `MemoryClaimMalformed` | A served claim write holds a malformed or client-scoped body. | `read.revise.malformed-request` | +| `MemoryClaimWriteRefused` | A served claim write's store effect refused. | `read.revise.served-fault` | | `MemoryEvidenceUnresolved` | A claim's evidence does not resolve through the caller's session. | `read.recall.evidence-unresolved` | | `MemoryEvidenceOverflow` | A claim names more evidence references than the bound. | `read.recall.evidence-references` | | `MemoryRecallNotClaims` | A keyed recall named a table declaring no `memory_facts` shape. | `read.recall.keyed-not-claims` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 2a63d129b..57a6827bf 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -6280,6 +6280,72 @@ "file": "spec/21-memory.md", "line": 116 }, + { + "id": "read.revise.served-write", + "contract": "read", + "operation": "revise", + "subject": "served-write", + "kind": "behavior", + "statement": "A writable served face answers `POST /memory/claims` under a per-request network credential, lands a declared claim through the direct write, and keeps the read MCP tool set closed.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 118 + }, + { + "id": "read.revise.served-scope", + "contract": "read", + "operation": "revise", + "subject": "served-scope", + "kind": "refusal", + "statement": "A claim write whose `actor` or `session` differs from the admitted credential's `on_behalf_of` or `task` raises `MemoryClaimScopeRefused`; accepted claims take scope from those members, never from the payload.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 120 + }, + { + "id": "read.revise.browser-request", + "contract": "read", + "operation": "revise", + "subject": "browser-request", + "kind": "refusal", + "statement": "An `Origin` header on the served claim-write route raises `MemoryClaimBrowserRefused` before credential admission and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 122 + }, + { + "id": "read.revise.malformed-request", + "contract": "read", + "operation": "revise", + "subject": "malformed-request", + "kind": "refusal", + "statement": "Malformed JSON, unknown fields, a client-selected claim scope, or an empty dedup key on the served claim-write route raises `MemoryClaimMalformed` and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 124 + }, + { + "id": "read.revise.served-dedup", + "contract": "read", + "operation": "revise", + "subject": "served-dedup", + "kind": "behavior", + "statement": "A served claim write passes its `dedup_key` to {{read.revise.dedup-key}}, returns `landed: false` on restatement, and returns the bound scope on both outcomes.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 126 + }, + { + "id": "read.revise.served-fault", + "contract": "read", + "operation": "revise", + "subject": "served-fault", + "kind": "refusal", + "statement": "A served claim write whose store effect refuses raises `MemoryClaimWriteRefused` and lands nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 128 + }, { "id": "read.revise.unscoped-collision", "contract": "read", @@ -6289,7 +6355,7 @@ "statement": "Direct writes with no scope, equal subject, predicate and valid-from, and different objects record both claim ids in the dead-letter table; neither claim retires the other until an explicit revision resolves the conflict.", "why": "A-read", "file": "spec/21-memory.md", - "line": 118 + "line": 130 }, { "id": "read.revise.retention-default", @@ -6300,7 +6366,7 @@ "statement": "Claims retain their recorded validity until explicit expiry or erasure; ranking decay is opt-in per shape with a declared half-life, and never deletes a claim or changes its validity interval.", "why": "A-read", "file": "spec/21-memory.md", - "line": 120 + "line": 132 }, { "id": "read.recall.evidence-unresolved", @@ -6311,7 +6377,7 @@ "statement": "An unreadable or masked source row, an unknown table, a reference into another memory row, or malformed lineage suppresses the claim and raises `MemoryEvidenceUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 128 + "line": 140 }, { "id": "read.recall.evidence-references", @@ -6322,7 +6388,7 @@ "statement": "A claim naming more than 256 entries of evidence is suppressed unresolved, raising `MemoryEvidenceOverflow`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 130 + "line": 142 }, { "id": "read.recall.evidence-key", @@ -6333,7 +6399,7 @@ "statement": "The write landing a claim stamps each citation into a keyed table with a pepper-keyed digest of the cited row's key; recall resolves a citation whose row no longer reads through the digest's live version.", "why": "A-read", "file": "spec/21-memory.md", - "line": 132 + "line": 144 }, { "id": "read.recall.evidence-key-masked", @@ -6344,7 +6410,7 @@ "statement": "A citation whose key column reads masked or nulled by zone in the writer's session carries no digest; a masked or nulled column outside the key leaves the digest stamped.", "why": "because a masked key cell digests a value no other session reads, while a cell outside the key plays no part in matching it", "file": "spec/21-memory.md", - "line": 134 + "line": 146 }, { "id": "read.recall.evidence-no-value", @@ -6355,7 +6421,7 @@ "statement": "A citation stored in a memory row carries the cited table, run, sequence and key digest, and no cell value of the cited row.", "why": "A-read", "file": "spec/21-memory.md", - "line": 136 + "line": 148 }, { "id": "read.recall.evidence-stale", @@ -6366,7 +6432,7 @@ "statement": "A claim served through its key digest while its cited run and sequence no longer read counts under `stale` in the `contextful.recall` block, which names no claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 138 + "line": 150 }, { "id": "read.recall.ranked-arm", @@ -6377,7 +6443,7 @@ "statement": "A `corpus.retrieve` arm over a `memory_facts` table serves only live claims — no `superseded_by`, and a `valid_to` null or past the read's anchor — whose evidence passes the gate.", "why": "", "file": "spec/21-memory.md", - "line": 140 + "line": 152 }, { "id": "read.recall.suppression-count", @@ -6388,7 +6454,7 @@ "statement": "A suppressed claim is absent from the rows; the `contextful.recall` block counts suppressions per error identifier and names no claim.", "why": "because a count discloses that a conclusion was withheld, never what it concluded", "file": "spec/21-memory.md", - "line": 141 + "line": 153 }, { "id": "read.recall.keyed", @@ -6399,7 +6465,7 @@ "statement": "`memory.recall` takes `table`, `subject`, `observed_at`, `as_of_ingest` and `limit`, and returns the claims whose `subject` equals `subject` exactly and whose `valid_from` and `valid_to` cover `observed_at` as {{store.bound-time.valid-as-of}} does.", "why": "", "file": "spec/21-memory.md", - "line": 143 + "line": 155 }, { "id": "read.recall.cli-verb", @@ -6410,7 +6476,7 @@ "statement": "The `contextful memory recall` verb passes its arguments through {{read.recall.keyed}} and prints the same JSON response as the `memory.recall` tool.", "why": "", "file": "spec/21-memory.md", - "line": 144 + "line": 156 }, { "id": "read.recall.keyed-clocks", @@ -6421,7 +6487,7 @@ "statement": "`as_of_ingest` bounds the read as {{store.bound-time.as-of}} does; absent, the read takes the latest committed state, and an absent `observed_at` the call's instant. `contextful.bounds` echoes each supplied bound under its argument name, with a per-name `inclusive` map.", "why": "", "file": "spec/21-memory.md", - "line": 145 + "line": 157 }, { "id": "read.recall.keyed-history", @@ -6432,7 +6498,7 @@ "statement": "A claim a successor retired still answers an `observed_at` inside its validity; the keyed read filters on validity alone, never on `superseded_by`.", "why": "because the question is what held at that instant, and the retired claim is the answer its interval records", "file": "spec/21-memory.md", - "line": 146 + "line": 158 }, { "id": "read.recall.keyed-gate", @@ -6443,7 +6509,7 @@ "statement": "Every keyed claim passes {{read.recall.evidence-unresolved}} and {{read.recall.evidence-references}}, and the response carries {{read.recall.suppression-count}}.", "why": "", "file": "spec/21-memory.md", - "line": 148 + "line": 160 }, { "id": "read.recall.keyed-window", @@ -6454,7 +6520,7 @@ "statement": "The keyed read gates claims in order and stops once it keeps one claim past the row ceiling, so the suppression count covers the claims gated before that stop.", "why": "because a one-row recall over a subject with thousands of claims otherwise resolves every claim's evidence", "file": "spec/21-memory.md", - "line": 149 + "line": 161 }, { "id": "read.recall.keyed-order", @@ -6465,7 +6531,7 @@ "statement": "Keyed claims order by tier, `curated` first, then `valid_from`, newest first, then `claim_id`; `limit` and the table's ceilings bound them under {{read.respond.row-ceiling}}.", "why": "", "file": "spec/21-memory.md", - "line": 151 + "line": 163 }, { "id": "read.recall.keyed-not-claims", @@ -6476,7 +6542,7 @@ "statement": "A granted `table` declaring a shape other than `memory_facts`, or no memory shape, raises `MemoryRecallNotClaims`.", "why": "because a keyed read answers with a claim's subject, validity and evidence, which no other table carries", "file": "spec/21-memory.md", - "line": 152 + "line": 164 }, { "id": "read.recall.stale-revision", @@ -6487,7 +6553,7 @@ "statement": "A stale citation remains subject to {{read.recall.keyed-gate}}; only a newly committed source row advancing {{read.synthesize.pass-cursor}} starts synthesis against its live key version.", "why": "A-read", "file": "spec/21-memory.md", - "line": 154 + "line": 166 }, { "id": "read.recall.usage-ledger", @@ -6498,7 +6564,7 @@ "statement": "Each `memory.recall` and ranked memory arm records the ids of claims actually returned in the request ledger, under the caller's admitted authority and the read frontier.", "why": "A-read", "file": "spec/21-memory.md", - "line": 156 + "line": 168 }, { "id": "read.resolve-entity.ambiguous-mention", @@ -6509,7 +6575,7 @@ "statement": "A mention matching two canonical identities with no deterministic key separating them is recorded as an ambiguous skip, raises `MemoryEntityAmbiguous`, and dead-letters its candidate claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 174 + "line": 186 }, { "id": "read.resolve-entity.edge-endpoint", @@ -6520,7 +6586,7 @@ "statement": "A candidate edge whose source or target resolves to no identity is dead-lettered, raising `MemoryEdgeEndpointUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 176 + "line": 188 }, { "id": "read.resolve-entity.graph-index", @@ -6531,7 +6597,7 @@ "statement": "An external graph engine is a derived index of admitted memory edges; an entity or ownership answer resolves through the store's enforced rows at the read frontier.", "why": "A-read", "file": "spec/21-memory.md", - "line": 178 + "line": 190 }, { "id": "read.resolve-entity.ownership-answer", @@ -6542,7 +6608,7 @@ "statement": "An artifact's ownership answer returns every attached principal visible to the caller, newest attachment first, with principal id breaking equal-instant ties.", "why": "A-read", "file": "spec/21-memory.md", - "line": 180 + "line": 192 }, { "id": "read.settle.registration", @@ -6553,7 +6619,7 @@ "statement": "A registration names exactly one form (relative horizon, absolute deadline, open watch) and one source (`metric`, `adjudicator`, `manual`), with a comparator exactly when the source is `metric`; otherwise it raises `OutcomeRegistrationInvalid`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 188 + "line": 200 }, { "id": "read.settle.source-mismatch", @@ -6564,7 +6630,7 @@ "statement": "An observation carrying a verdict whose resolution source is absent or differs from the registration's raises `OutcomeSourceMismatch`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 190 + "line": 202 }, { "id": "read.settle.settling-citation", @@ -6575,7 +6641,7 @@ "statement": "An `adjudicator` or `manual` verdict without an `http` or `https` settling citation raises `OutcomeCitationMissing`; the citation rides the label view.", "why": "A-read", "file": "spec/21-memory.md", - "line": 192 + "line": 204 }, { "id": "read.settle.grace-window", @@ -6586,7 +6652,7 @@ "statement": "The label join keeps an observation from the prediction instant through the deadline plus an inclusive grace of 86400 s.", "why": "", "file": "spec/21-memory.md", - "line": 194 + "line": 206 }, { "id": "read.settle.calibration-report", @@ -6597,7 +6663,7 @@ "statement": "Calibration reports group settled predictions by shape and predicate, and name sample count, Brier score, expected calibration error and the held-out score used by {{read.synthesize.confidence-calibration}}.", "why": "A-read", "file": "spec/21-memory.md", - "line": 195 + "line": 207 }, { "id": "run.journal.step-output", @@ -20665,6 +20731,17 @@ "file": "spec/71-console.md", "line": 284 }, + { + "id": "surface.learn.write-refused", + "contract": "surface", + "operation": "learn", + "subject": "write-refused", + "kind": "refusal", + "statement": "A served claim write that refuses or returns no scope-bound receipt raises `ConsoleLearningWriteRefused` to Query before answer publication.", + "why": "because a failed write cannot count as a durable learning", + "file": "spec/71-console.md", + "line": 286 + }, { "id": "surface.learn.subject-resolution", "contract": "surface", @@ -20674,7 +20751,7 @@ "statement": "Distillation preserves the subject it observes; recall resolves that subject through entity matching.", "why": "A-surface", "file": "spec/71-console.md", - "line": 286 + "line": 288 }, { "id": "surface.render.client-authored-view", @@ -20685,7 +20762,7 @@ "statement": "A view specification arriving from a client, or composed by the model, raises `ConsoleViewNotServerBuilt`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 293 + "line": 295 }, { "id": "surface.render.component-choice", @@ -20696,7 +20773,7 @@ "statement": "One row with one measure draws a metric; a date column with a measure over 3 rows on distinct days draws a line; anything else draws a table offering a bar view.", "why": "", "file": "spec/71-console.md", - "line": 295 + "line": 297 }, { "id": "surface.render.older-transcript", @@ -20707,7 +20784,7 @@ "statement": "A client reading a transcript with an unknown component-union member renders its view as a table.", "why": "A-surface", "file": "spec/71-console.md", - "line": 296 + "line": 298 }, { "id": "surface.brief.absence-is-earned", @@ -20718,7 +20795,7 @@ "statement": "The card needs a session with no turns, at present time, a live conclusion, an arrived row matching an interest, and a derivation inside its budget. An error or timeout raises `ConsoleBriefUnavailable`, and no card renders.", "why": "A-surface", "file": "spec/71-console.md", - "line": 332 + "line": 334 }, { "id": "surface.brief.topic-tier", @@ -20729,7 +20806,7 @@ "statement": "The topic tier needs at least 2 tokens shared between the conclusion's subject and text and the row's label and topics, one of them naming the subject.", "why": "", "file": "spec/71-console.md", - "line": 334 + "line": 336 }, { "id": "surface.brief.card-subjects", @@ -20740,7 +20817,7 @@ "statement": "A card carries at most 3 subjects.", "why": "", "file": "spec/71-console.md", - "line": 335 + "line": 337 }, { "id": "surface.brief.articles-per-subject", @@ -20751,7 +20828,7 @@ "statement": "A subject carries at most 3 entries of matched article.", "why": "", "file": "spec/71-console.md", - "line": 336 + "line": 338 }, { "id": "surface.brief.catchup-window", @@ -20762,7 +20839,7 @@ "statement": "The backend caps the requested window at 7 d.", "why": "", "file": "spec/71-console.md", - "line": 337 + "line": 339 }, { "id": "surface.brief.greeting", @@ -20773,7 +20850,7 @@ "statement": "Query derives a greeting card from arrived rows matching live conclusions in a turnless present-time session.", "why": "A-surface", "file": "spec/71-console.md", - "line": 338 + "line": 340 }, { "id": "surface.publish-answer.share-affordance", @@ -20784,7 +20861,7 @@ "statement": "A surface offering a share control on access-explanation output raises `VisibilityShareAffordance`.", "why": "A-surface", "file": "spec/71-console.md", - "line": 345 + "line": 347 }, { "id": "surface.publish-answer.askerless-audience", @@ -20795,7 +20872,7 @@ "statement": "A scheduled job posting to an audience under a service identity raises `VisibilityAskerlessAudience`, naming the destination; a scheduled post's corpus is narrowed in the reviewed manifest to what the destination reaches.", "why": "A-surface", "file": "spec/71-console.md", - "line": 347 + "line": 349 }, { "id": "surface.publish-answer.one-operator-principal", @@ -20806,7 +20883,7 @@ "statement": "An answer uses one operator's credential; the console offers no room-intersection principal.", "why": "A-surface", "file": "spec/71-console.md", - "line": 349 + "line": 351 }, { "id": "assurance.model.package", @@ -24188,6 +24265,10 @@ "read.revise.supersede", "read.revise.unscoped-collision" ], + [ + "read.revise.served-dedup", + "read.revise.dedup-key" + ], [ "read.recall.keyed", "store.bound-time.valid-as-of" @@ -28028,6 +28109,22 @@ "clause": "read.revise.citation-live", "gloss": "A direct write cites a keyed row that is not its key's live version." }, + "MemoryClaimBrowserRefused": { + "clause": "read.revise.browser-request", + "gloss": "A browser-origin request reached the served claim-write route." + }, + "MemoryClaimMalformed": { + "clause": "read.revise.malformed-request", + "gloss": "A served claim write holds a malformed or client-scoped body." + }, + "MemoryClaimScopeRefused": { + "clause": "read.revise.served-scope", + "gloss": "A served claim write names another actor or reading session." + }, + "MemoryClaimWriteRefused": { + "clause": "read.revise.served-fault", + "gloss": "A served claim write's store effect refused." + }, "MemoryDirectWriteShapeRefused": { "clause": "read.revise.direct-write", "gloss": "A direct write targeted a memory shape other than claims." @@ -29762,6 +29859,10 @@ "clause": "surface.learn.unscoped", "gloss": "A distilled conclusion landing without the reading-session scope." }, + "ConsoleLearningWriteRefused": { + "clause": "surface.learn.write-refused", + "gloss": "A served claim write that refuses or returns no scope-bound receipt." + }, "ConsoleMutatingToolRequested": { "clause": "surface.ground.mutating-tool", "gloss": "A client-reachable path naming a write." diff --git a/spec/status.md b/spec/status.md index bed22f463..922442e7c 100644 --- a/spec/status.md +++ b/spec/status.md @@ -10,12 +10,12 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 229 | 0 | 228 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 43 | 0 | 43 | | `disclosure` | 3 | 16 | 134 | 66 | 13 | 3 | 63 | 0 | 63 | -| `read` | 2 | 15 | 200 | 31 | 22 | 9 | 178 | 0 | 172 | +| `read` | 2 | 15 | 206 | 35 | 22 | 9 | 183 | 0 | 172 | | `run` | 3 | 26 | 406 | 102 | 39 | 28 | 306 | 0 | 279 | | `store` | 1 | 17 | 249 | 57 | 13 | 19 | 213 | 0 | 204 | -| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 59 | 0 | 59 | +| `surface` | 2 | 20 | 115 | 52 | 22 | 0 | 74 | 0 | 66 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2105 | 593 | 191 | 103 | 1476 | 0 | | +| **total** | 19 | 155 | 2112 | 598 | 191 | 103 | 1496 | 0 | | Decision records: 18. @@ -32,12 +32,12 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 4 — Ingest | 25 | 517 | 344 | passing | open | | 5 — The read face under enforcement | 17 | 262 | 204 | passing | open | | 6 — Sync and replicas | 8 | 77 | 72 | passing | closed | -| 7 — Memory | 6 | 48 | 40 | passing | closed | +| 7 — Memory | 6 | 54 | 45 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 76 | 64 | passing | open | | 11 — The derive tier | 7 | 88 | 79 | passing | closed | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 50 | 15 | open | open | | 13 — Disclosure | 5 | 33 | 13 | open | open | | 14 — Assurance | 7 | 138 | 89 | passing | closed | @@ -888,11 +888,16 @@ Unscheduled operations: 10. | `read.retrieve.token-cap` | `crates/contextful-core/tests/integration/read/retrieve.rs::the_token_set_is_capped_and_an_empty_set_omits_the_floor` | performed | | `read.retrieve.token-length-floor` | `crates/contextful-core/tests/integration/read/retrieve.rs::an_ascii_run_below_the_length_floor_leaves` | performed | | `read.retrieve.unsatisfiable-arm-drops` | `crates/contextful-context/tests/integration/read/retrieve.rs::a_filter_binds_named_columns_and_a_table_lacking_one_drops_its_arm` | performed | +| `read.revise.browser-request` | `spec/pins.toml` | performed | | `read.revise.citation-live` | `crates/contextful-memory/tests/integration/evidence.rs::a_write_citing_a_superseded_version_refuses` | performed | | `read.revise.dedup-key` | `crates/contextful-memory/tests/integration/write.rs::a_dedup_key_seeds_the_claim_id_and_a_retry_lands_nothing` | performed | | `read.revise.direct-write` | `crates/contextful-core/tests/integration/memory/revise.rs::the_direct_write_accepts_claims_alone` | performed | +| `read.revise.malformed-request` | `spec/pins.toml` | performed | | `read.revise.observed-at` | `crates/contextful-memory/tests/integration/write.rs::an_observed_write_lands_valid_from_its_observed_instant` | performed | | `read.revise.observed-order` | `crates/contextful-memory/tests/integration/write.rs::an_observation_before_a_live_contradicting_prior_refuses` | performed | +| `read.revise.served-dedup` | `spec/pins.toml` | performed | +| `read.revise.served-scope` | `spec/pins.toml` | performed | +| `read.revise.served-write` | `spec/pins.toml` | performed | | `read.revise.supersede` | `crates/contextful-core/tests/integration/memory/revise.rs::an_equal_or_higher_claim_retires_its_prior_on_one_line` | performed | | `read.revise.tier` | `crates/contextful-core/tests/integration/memory/revise.rs::tier_follows_the_write_path_and_the_lowest_grounding` | performed | | `read.revise.unscoped-collision` | `crates/contextful-memory/tests/integration/write.rs::unscoped_writers_at_one_instant_record_a_conflict_without_retirement` | performed | @@ -1450,6 +1455,8 @@ Unscheduled operations: 10. | `surface.arm.unknown-trigger` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unknown_trigger_arms_nothing` | performed | | `surface.arm.unreadable-schedule` | `crates/contextful-core/tests/integration/surface/arm.rs::an_unreadable_schedule_names_its_diagnostic` | performed | | `surface.arm.wake-answer` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_answers_within_its_bound_naming_what_still_runs` | performed | +| `surface.brief.greeting` | `apps/console/test/brief.test.ts::apps/console/test/brief.test.ts::brief requires a turnless present-time session and live matched arrivals` | performed | +| `surface.brief.topic-tier` | `apps/console/test/brief.test.ts::apps/console/test/brief.test.ts::topic tier needs two shared tokens including the subject` | performed | | `surface.dispatch.callback-rejected` | `crates/contextful-core/tests/integration/surface/worker.rs::a_superseded_or_skewed_callback_changes_no_step` | performed | | `surface.dispatch.callback-skew` | `crates/contextful-core/tests/integration/surface/worker.rs::the_relay_accepts_a_timestamp_within_the_skew` | performed | | `surface.dispatch.children-reaped` | `crates/contextful-cli/tests/integration/pipeline.rs::serve_ends_every_child_it_dispatched` | performed | @@ -1473,6 +1480,14 @@ Unscheduled operations: 10. | `surface.fire.store-driven-body` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_naming_an_unregistered_body_raises_job_body_unregistered` | performed | | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | +| `surface.ground.direct-file-read` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::direct file table functions refuse before dispatch` | performed | +| `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | +| `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | +| `surface.learn.unscoped` | `apps/console/test/learn.test.ts::apps/console/test/learn.test.ts::unscoped learning refuses before distillation or landing` | performed | +| `surface.learn.write-refused` | `apps/console/test/answer_memory.test.ts::apps/console/test/answer_memory.test.ts::hosted Query reports a served learning refusal without exposing its credential` | performed | +| `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | +| `surface.publish-answer.askerless-audience` | `apps/console/test/publish.test.ts::apps/console/test/publish.test.ts::a scheduled service post refuses an audience and names its destination` | performed | +| `surface.publish-answer.share-affordance` | `apps/console/test/publish.test.ts::apps/console/test/publish.test.ts::an access explanation cannot offer a share control` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.issuer-pin` | `crates/contextful-cli/tests/integration/pipeline.rs::a_pulled_receipt_cannot_supply_its_own_trust_pin` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | @@ -1482,9 +1497,14 @@ Unscheduled operations: 10. | `surface.reconcile.pulled-control` | `crates/contextful-cli/tests/integration/pipeline.rs::a_cold_node_adopts_a_pinned_pulled_control_snapshot_after_local_validation` | performed | | `surface.reconcile.pulled-control-untrusted` | `crates/contextful-cli/tests/integration/pipeline.rs::a_bad_signature_or_local_declaration_refuses_pulled_control_without_arming` | performed | | `surface.reconcile.url-layout` | `crates/contextful-cli/tests/integration/pipeline.rs::a_loopback_control_url_serves_the_applied_snapshot` | performed | +| `surface.render.client-authored-view` | `apps/console/test/render.test.ts::apps/console/test/render.test.ts::client and model view specifications never become widgets` | performed | +| `surface.render.component-choice` | `apps/console/test/render.test.ts::apps/console/test/render.test.ts::server result shape chooses metric, line, or table with a bar option` | performed | +| `surface.render.older-transcript` | `apps/console/test/render.test.ts::apps/console/test/render.test.ts::older transcripts render an unknown component as a table` | performed | | `surface.reside.region-entries` | `crates/contextful-core/tests/integration/surface/reside.rs::an_allow_set_holds_sixteen_entries` | performed | | `surface.reside.region-mismatch` | `crates/contextful-cli/tests/integration/pipeline.rs::a_resource_outside_the_residency_allow_set_serves_nothing` | performed | | `surface.reside.site-regions` | `crates/contextful-cli/tests/integration/sync/mod.rs::two_sites_declaring_different_residency_diverge_at_push` | performed | +| `surface.set-vantage.sample-labels` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a table contributes at most three sampled arrival labels` | performed | +| `surface.speak.redactor-lookahead` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::streaming redactor masks identifiers split across chunks` | performed | | `topology.compose.script-runtime` | `tools/ci/tests/integration/topology.rs::a_crate_linking_a_javascript_runtime_is_refused` | performed | | `topology.compose.undeclared-crossing` | `tools/ci/tests/integration/topology.rs::a_run_path_crate_reaching_a_read_path_crate_is_refused` | performed | | `topology.compose.vendor-sdk` | `tools/ci/tests/integration/topology.rs::a_crate_declaring_a_model_vendor_sdk_is_refused` | performed | From ca1df1403edfda4614fe69f4238fe7be29b629a4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:24:16 +0800 Subject: [PATCH 122/155] Regenerate console browse corpus Test-First: refactor --- spec/status.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/spec/status.md b/spec/status.md index bed22f463..9d155e721 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 200 | 31 | 22 | 9 | 178 | 0 | 172 | | `run` | 3 | 26 | 406 | 102 | 39 | 28 | 306 | 0 | 279 | | `store` | 1 | 17 | 249 | 57 | 13 | 19 | 213 | 0 | 204 | -| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 59 | 0 | 59 | +| `surface` | 2 | 20 | 118 | 52 | 22 | 0 | 69 | 0 | 61 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2105 | 593 | 191 | 103 | 1476 | 0 | | +| **total** | 19 | 155 | 2109 | 594 | 191 | 103 | 1486 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 76 | 64 | passing | open | | 11 — The derive tier | 7 | 88 | 79 | passing | closed | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 53 | 10 | open | open | | 13 — Disclosure | 5 | 33 | 13 | open | open | | 14 — Assurance | 7 | 138 | 89 | passing | closed | @@ -1450,6 +1450,10 @@ Unscheduled operations: 10. | `surface.arm.unknown-trigger` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unknown_trigger_arms_nothing` | performed | | `surface.arm.unreadable-schedule` | `crates/contextful-core/tests/integration/surface/arm.rs::an_unreadable_schedule_names_its_diagnostic` | performed | | `surface.arm.wake-answer` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_answers_within_its_bound_naming_what_still_runs` | performed | +| `surface.browse.discovered-chips` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::browse advertises admitted tables as humanized chips` | performed | +| `surface.browse.file-gallery` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::a gallery preview uses context.file with the listing's snapshot bound` | performed | +| `surface.browse.gallery-path-unlisted` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::a gallery preview refuses paths absent from the governed listing` | performed | +| `surface.browse.insights-panel` | `apps/console/test/browse.test.ts::apps/console/test/browse.test.ts::insights show governed table row counts` | performed | | `surface.dispatch.callback-rejected` | `crates/contextful-core/tests/integration/surface/worker.rs::a_superseded_or_skewed_callback_changes_no_step` | performed | | `surface.dispatch.callback-skew` | `crates/contextful-core/tests/integration/surface/worker.rs::the_relay_accepts_a_timestamp_within_the_skew` | performed | | `surface.dispatch.children-reaped` | `crates/contextful-cli/tests/integration/pipeline.rs::serve_ends_every_child_it_dispatched` | performed | @@ -1473,6 +1477,10 @@ Unscheduled operations: 10. | `surface.fire.store-driven-body` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_naming_an_unregistered_body_raises_job_body_unregistered` | performed | | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | +| `surface.ground.direct-file-read` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::direct file table functions refuse before dispatch` | performed | +| `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | +| `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | +| `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.issuer-pin` | `crates/contextful-cli/tests/integration/pipeline.rs::a_pulled_receipt_cannot_supply_its_own_trust_pin` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | @@ -1485,6 +1493,8 @@ Unscheduled operations: 10. | `surface.reside.region-entries` | `crates/contextful-core/tests/integration/surface/reside.rs::an_allow_set_holds_sixteen_entries` | performed | | `surface.reside.region-mismatch` | `crates/contextful-cli/tests/integration/pipeline.rs::a_resource_outside_the_residency_allow_set_serves_nothing` | performed | | `surface.reside.site-regions` | `crates/contextful-cli/tests/integration/sync/mod.rs::two_sites_declaring_different_residency_diverge_at_push` | performed | +| `surface.set-vantage.sample-labels` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a table contributes at most three sampled arrival labels` | performed | +| `surface.speak.redactor-lookahead` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::streaming redactor masks identifiers split across chunks` | performed | | `topology.compose.script-runtime` | `tools/ci/tests/integration/topology.rs::a_crate_linking_a_javascript_runtime_is_refused` | performed | | `topology.compose.undeclared-crossing` | `tools/ci/tests/integration/topology.rs::a_run_path_crate_reaching_a_read_path_crate_is_refused` | performed | | `topology.compose.vendor-sdk` | `tools/ci/tests/integration/topology.rs::a_crate_declaring_a_model_vendor_sdk_is_refused` | performed | From 440382ff1fc2d66c1390ea80dc059a567e4abb96 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:24:33 +0800 Subject: [PATCH 123/155] Regenerate console memory corpus Test-First: refactor --- spec/cards/read.md | 6 +- spec/spec.lock.json | 144 +++++++++++++++++++++++++++++++++++--------- spec/status.md | 21 +++++-- 3 files changed, 136 insertions(+), 35 deletions(-) diff --git a/spec/cards/read.md b/spec/cards/read.md index 3ec1a6f61..4d1c53e07 100644 --- a/spec/cards/read.md +++ b/spec/cards/read.md @@ -17,7 +17,7 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `embed` | The embedding port and its default, the model identifier beside each vector, the linked-backend identity a handshake reports, and refusals on an absent backend. | 6 | | `declare` | The five memory table shapes, their canonical columns, tier and scope, and the relation vocabulary with its cardinality. | 3 | | `synthesize` | Extract, Resolve and Consolidate; the dedup key, evidence support, the audit record and coverage gaps. | 9 | -| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 9 | +| `revise` | Supersession within one validity line, confidence decay, the direct write and its anchor, expiry, retention, promotion. | 15 | | `recall` | Serving memory: the ranked arm at the read's anchor, the keyed read at an observed instant, the two clocks, tier-first ordering, the scope filter, the tombstone filter, the evidence gate. | 18 | | `resolve-entity` | Mention-to-entity matching, aliases, the knowledge card, place identity and nearness, edges, and ownership answers. | 4 | | `settle` | Predictions, observations, the registration, the citation a verdict owes, and the label views. | 5 | @@ -49,6 +49,10 @@ Generated by `contextful-spec state` from the contract's files and `spec/terms/r | `MemoryDirectWriteShapeRefused` | A direct write targeted a memory shape other than claims. | `read.revise.direct-write` | | `MemoryObservationOutOfOrder` | A claim was observed before a live contradicting claim of its line. | `read.revise.observed-order` | | `MemoryCitationNotLive` | A direct write cites a keyed row that is not its key's live version. | `read.revise.citation-live` | +| `MemoryClaimScopeRefused` | A served claim write names another actor or reading session. | `read.revise.served-scope` | +| `MemoryClaimBrowserRefused` | A browser-origin request reached the served claim-write route. | `read.revise.browser-request` | +| `MemoryClaimMalformed` | A served claim write holds a malformed or client-scoped body. | `read.revise.malformed-request` | +| `MemoryClaimWriteRefused` | A served claim write's store effect refused. | `read.revise.served-fault` | | `MemoryEvidenceUnresolved` | A claim's evidence does not resolve through the caller's session. | `read.recall.evidence-unresolved` | | `MemoryEvidenceOverflow` | A claim names more evidence references than the bound. | `read.recall.evidence-references` | | `MemoryRecallNotClaims` | A keyed recall named a table declaring no `memory_facts` shape. | `read.recall.keyed-not-claims` | diff --git a/spec/spec.lock.json b/spec/spec.lock.json index 2a63d129b..342c5e507 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -6280,6 +6280,72 @@ "file": "spec/21-memory.md", "line": 116 }, + { + "id": "read.revise.served-write", + "contract": "read", + "operation": "revise", + "subject": "served-write", + "kind": "behavior", + "statement": "A writable served face answers `POST /memory/claims` under a per-request network credential, lands a declared claim through the direct write, and keeps the read MCP tool set closed.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 118 + }, + { + "id": "read.revise.served-scope", + "contract": "read", + "operation": "revise", + "subject": "served-scope", + "kind": "refusal", + "statement": "A claim write whose `actor` or `session` differs from the admitted credential's `on_behalf_of` or `task` raises `MemoryClaimScopeRefused`; accepted claims take scope from those members, never from the payload.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 120 + }, + { + "id": "read.revise.browser-request", + "contract": "read", + "operation": "revise", + "subject": "browser-request", + "kind": "refusal", + "statement": "An `Origin` header on the served claim-write route raises `MemoryClaimBrowserRefused` before credential admission and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 122 + }, + { + "id": "read.revise.malformed-request", + "contract": "read", + "operation": "revise", + "subject": "malformed-request", + "kind": "refusal", + "statement": "Malformed JSON, unknown fields, a client-selected claim scope, or an empty dedup key on the served claim-write route raises `MemoryClaimMalformed` and commits nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 124 + }, + { + "id": "read.revise.served-dedup", + "contract": "read", + "operation": "revise", + "subject": "served-dedup", + "kind": "behavior", + "statement": "A served claim write passes its `dedup_key` to {{read.revise.dedup-key}}, returns `landed: false` on restatement, and returns the bound scope on both outcomes.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 126 + }, + { + "id": "read.revise.served-fault", + "contract": "read", + "operation": "revise", + "subject": "served-fault", + "kind": "refusal", + "statement": "A served claim write whose store effect refuses raises `MemoryClaimWriteRefused` and lands nothing.", + "why": "A-read", + "file": "spec/21-memory.md", + "line": 128 + }, { "id": "read.revise.unscoped-collision", "contract": "read", @@ -6289,7 +6355,7 @@ "statement": "Direct writes with no scope, equal subject, predicate and valid-from, and different objects record both claim ids in the dead-letter table; neither claim retires the other until an explicit revision resolves the conflict.", "why": "A-read", "file": "spec/21-memory.md", - "line": 118 + "line": 130 }, { "id": "read.revise.retention-default", @@ -6300,7 +6366,7 @@ "statement": "Claims retain their recorded validity until explicit expiry or erasure; ranking decay is opt-in per shape with a declared half-life, and never deletes a claim or changes its validity interval.", "why": "A-read", "file": "spec/21-memory.md", - "line": 120 + "line": 132 }, { "id": "read.recall.evidence-unresolved", @@ -6311,7 +6377,7 @@ "statement": "An unreadable or masked source row, an unknown table, a reference into another memory row, or malformed lineage suppresses the claim and raises `MemoryEvidenceUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 128 + "line": 140 }, { "id": "read.recall.evidence-references", @@ -6322,7 +6388,7 @@ "statement": "A claim naming more than 256 entries of evidence is suppressed unresolved, raising `MemoryEvidenceOverflow`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 130 + "line": 142 }, { "id": "read.recall.evidence-key", @@ -6333,7 +6399,7 @@ "statement": "The write landing a claim stamps each citation into a keyed table with a pepper-keyed digest of the cited row's key; recall resolves a citation whose row no longer reads through the digest's live version.", "why": "A-read", "file": "spec/21-memory.md", - "line": 132 + "line": 144 }, { "id": "read.recall.evidence-key-masked", @@ -6344,7 +6410,7 @@ "statement": "A citation whose key column reads masked or nulled by zone in the writer's session carries no digest; a masked or nulled column outside the key leaves the digest stamped.", "why": "because a masked key cell digests a value no other session reads, while a cell outside the key plays no part in matching it", "file": "spec/21-memory.md", - "line": 134 + "line": 146 }, { "id": "read.recall.evidence-no-value", @@ -6355,7 +6421,7 @@ "statement": "A citation stored in a memory row carries the cited table, run, sequence and key digest, and no cell value of the cited row.", "why": "A-read", "file": "spec/21-memory.md", - "line": 136 + "line": 148 }, { "id": "read.recall.evidence-stale", @@ -6366,7 +6432,7 @@ "statement": "A claim served through its key digest while its cited run and sequence no longer read counts under `stale` in the `contextful.recall` block, which names no claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 138 + "line": 150 }, { "id": "read.recall.ranked-arm", @@ -6377,7 +6443,7 @@ "statement": "A `corpus.retrieve` arm over a `memory_facts` table serves only live claims — no `superseded_by`, and a `valid_to` null or past the read's anchor — whose evidence passes the gate.", "why": "", "file": "spec/21-memory.md", - "line": 140 + "line": 152 }, { "id": "read.recall.suppression-count", @@ -6388,7 +6454,7 @@ "statement": "A suppressed claim is absent from the rows; the `contextful.recall` block counts suppressions per error identifier and names no claim.", "why": "because a count discloses that a conclusion was withheld, never what it concluded", "file": "spec/21-memory.md", - "line": 141 + "line": 153 }, { "id": "read.recall.keyed", @@ -6399,7 +6465,7 @@ "statement": "`memory.recall` takes `table`, `subject`, `observed_at`, `as_of_ingest` and `limit`, and returns the claims whose `subject` equals `subject` exactly and whose `valid_from` and `valid_to` cover `observed_at` as {{store.bound-time.valid-as-of}} does.", "why": "", "file": "spec/21-memory.md", - "line": 143 + "line": 155 }, { "id": "read.recall.cli-verb", @@ -6410,7 +6476,7 @@ "statement": "The `contextful memory recall` verb passes its arguments through {{read.recall.keyed}} and prints the same JSON response as the `memory.recall` tool.", "why": "", "file": "spec/21-memory.md", - "line": 144 + "line": 156 }, { "id": "read.recall.keyed-clocks", @@ -6421,7 +6487,7 @@ "statement": "`as_of_ingest` bounds the read as {{store.bound-time.as-of}} does; absent, the read takes the latest committed state, and an absent `observed_at` the call's instant. `contextful.bounds` echoes each supplied bound under its argument name, with a per-name `inclusive` map.", "why": "", "file": "spec/21-memory.md", - "line": 145 + "line": 157 }, { "id": "read.recall.keyed-history", @@ -6432,7 +6498,7 @@ "statement": "A claim a successor retired still answers an `observed_at` inside its validity; the keyed read filters on validity alone, never on `superseded_by`.", "why": "because the question is what held at that instant, and the retired claim is the answer its interval records", "file": "spec/21-memory.md", - "line": 146 + "line": 158 }, { "id": "read.recall.keyed-gate", @@ -6443,7 +6509,7 @@ "statement": "Every keyed claim passes {{read.recall.evidence-unresolved}} and {{read.recall.evidence-references}}, and the response carries {{read.recall.suppression-count}}.", "why": "", "file": "spec/21-memory.md", - "line": 148 + "line": 160 }, { "id": "read.recall.keyed-window", @@ -6454,7 +6520,7 @@ "statement": "The keyed read gates claims in order and stops once it keeps one claim past the row ceiling, so the suppression count covers the claims gated before that stop.", "why": "because a one-row recall over a subject with thousands of claims otherwise resolves every claim's evidence", "file": "spec/21-memory.md", - "line": 149 + "line": 161 }, { "id": "read.recall.keyed-order", @@ -6465,7 +6531,7 @@ "statement": "Keyed claims order by tier, `curated` first, then `valid_from`, newest first, then `claim_id`; `limit` and the table's ceilings bound them under {{read.respond.row-ceiling}}.", "why": "", "file": "spec/21-memory.md", - "line": 151 + "line": 163 }, { "id": "read.recall.keyed-not-claims", @@ -6476,7 +6542,7 @@ "statement": "A granted `table` declaring a shape other than `memory_facts`, or no memory shape, raises `MemoryRecallNotClaims`.", "why": "because a keyed read answers with a claim's subject, validity and evidence, which no other table carries", "file": "spec/21-memory.md", - "line": 152 + "line": 164 }, { "id": "read.recall.stale-revision", @@ -6487,7 +6553,7 @@ "statement": "A stale citation remains subject to {{read.recall.keyed-gate}}; only a newly committed source row advancing {{read.synthesize.pass-cursor}} starts synthesis against its live key version.", "why": "A-read", "file": "spec/21-memory.md", - "line": 154 + "line": 166 }, { "id": "read.recall.usage-ledger", @@ -6498,7 +6564,7 @@ "statement": "Each `memory.recall` and ranked memory arm records the ids of claims actually returned in the request ledger, under the caller's admitted authority and the read frontier.", "why": "A-read", "file": "spec/21-memory.md", - "line": 156 + "line": 168 }, { "id": "read.resolve-entity.ambiguous-mention", @@ -6509,7 +6575,7 @@ "statement": "A mention matching two canonical identities with no deterministic key separating them is recorded as an ambiguous skip, raises `MemoryEntityAmbiguous`, and dead-letters its candidate claim.", "why": "A-read", "file": "spec/21-memory.md", - "line": 174 + "line": 186 }, { "id": "read.resolve-entity.edge-endpoint", @@ -6520,7 +6586,7 @@ "statement": "A candidate edge whose source or target resolves to no identity is dead-lettered, raising `MemoryEdgeEndpointUnresolved`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 176 + "line": 188 }, { "id": "read.resolve-entity.graph-index", @@ -6531,7 +6597,7 @@ "statement": "An external graph engine is a derived index of admitted memory edges; an entity or ownership answer resolves through the store's enforced rows at the read frontier.", "why": "A-read", "file": "spec/21-memory.md", - "line": 178 + "line": 190 }, { "id": "read.resolve-entity.ownership-answer", @@ -6542,7 +6608,7 @@ "statement": "An artifact's ownership answer returns every attached principal visible to the caller, newest attachment first, with principal id breaking equal-instant ties.", "why": "A-read", "file": "spec/21-memory.md", - "line": 180 + "line": 192 }, { "id": "read.settle.registration", @@ -6553,7 +6619,7 @@ "statement": "A registration names exactly one form (relative horizon, absolute deadline, open watch) and one source (`metric`, `adjudicator`, `manual`), with a comparator exactly when the source is `metric`; otherwise it raises `OutcomeRegistrationInvalid`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 188 + "line": 200 }, { "id": "read.settle.source-mismatch", @@ -6564,7 +6630,7 @@ "statement": "An observation carrying a verdict whose resolution source is absent or differs from the registration's raises `OutcomeSourceMismatch`.", "why": "A-read", "file": "spec/21-memory.md", - "line": 190 + "line": 202 }, { "id": "read.settle.settling-citation", @@ -6575,7 +6641,7 @@ "statement": "An `adjudicator` or `manual` verdict without an `http` or `https` settling citation raises `OutcomeCitationMissing`; the citation rides the label view.", "why": "A-read", "file": "spec/21-memory.md", - "line": 192 + "line": 204 }, { "id": "read.settle.grace-window", @@ -6586,7 +6652,7 @@ "statement": "The label join keeps an observation from the prediction instant through the deadline plus an inclusive grace of 86400 s.", "why": "", "file": "spec/21-memory.md", - "line": 194 + "line": 206 }, { "id": "read.settle.calibration-report", @@ -6597,7 +6663,7 @@ "statement": "Calibration reports group settled predictions by shape and predicate, and name sample count, Brier score, expected calibration error and the held-out score used by {{read.synthesize.confidence-calibration}}.", "why": "A-read", "file": "spec/21-memory.md", - "line": 195 + "line": 207 }, { "id": "run.journal.step-output", @@ -24188,6 +24254,10 @@ "read.revise.supersede", "read.revise.unscoped-collision" ], + [ + "read.revise.served-dedup", + "read.revise.dedup-key" + ], [ "read.recall.keyed", "store.bound-time.valid-as-of" @@ -28028,6 +28098,22 @@ "clause": "read.revise.citation-live", "gloss": "A direct write cites a keyed row that is not its key's live version." }, + "MemoryClaimBrowserRefused": { + "clause": "read.revise.browser-request", + "gloss": "A browser-origin request reached the served claim-write route." + }, + "MemoryClaimMalformed": { + "clause": "read.revise.malformed-request", + "gloss": "A served claim write holds a malformed or client-scoped body." + }, + "MemoryClaimScopeRefused": { + "clause": "read.revise.served-scope", + "gloss": "A served claim write names another actor or reading session." + }, + "MemoryClaimWriteRefused": { + "clause": "read.revise.served-fault", + "gloss": "A served claim write's store effect refused." + }, "MemoryDirectWriteShapeRefused": { "clause": "read.revise.direct-write", "gloss": "A direct write targeted a memory shape other than claims." diff --git a/spec/status.md b/spec/status.md index bed22f463..87c09ab96 100644 --- a/spec/status.md +++ b/spec/status.md @@ -10,12 +10,12 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 229 | 0 | 228 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 43 | 0 | 43 | | `disclosure` | 3 | 16 | 134 | 66 | 13 | 3 | 63 | 0 | 63 | -| `read` | 2 | 15 | 200 | 31 | 22 | 9 | 178 | 0 | 172 | +| `read` | 2 | 15 | 206 | 35 | 22 | 9 | 183 | 0 | 172 | | `run` | 3 | 26 | 406 | 102 | 39 | 28 | 306 | 0 | 279 | | `store` | 1 | 17 | 249 | 57 | 13 | 19 | 213 | 0 | 204 | -| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 59 | 0 | 59 | +| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 65 | 0 | 59 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2105 | 593 | 191 | 103 | 1476 | 0 | | +| **total** | 19 | 155 | 2111 | 597 | 191 | 103 | 1487 | 0 | | Decision records: 18. @@ -32,12 +32,12 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 4 — Ingest | 25 | 517 | 344 | passing | open | | 5 — The read face under enforcement | 17 | 262 | 204 | passing | open | | 6 — Sync and replicas | 8 | 77 | 72 | passing | closed | -| 7 — Memory | 6 | 48 | 40 | passing | closed | +| 7 — Memory | 6 | 54 | 45 | passing | closed | | 8 — Accountability | 5 | 60 | 47 | passing | open | | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 76 | 64 | passing | open | | 11 — The derive tier | 7 | 88 | 79 | passing | closed | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 49 | 6 | open | open | | 13 — Disclosure | 5 | 33 | 13 | open | open | | 14 — Assurance | 7 | 138 | 89 | passing | closed | @@ -888,11 +888,16 @@ Unscheduled operations: 10. | `read.retrieve.token-cap` | `crates/contextful-core/tests/integration/read/retrieve.rs::the_token_set_is_capped_and_an_empty_set_omits_the_floor` | performed | | `read.retrieve.token-length-floor` | `crates/contextful-core/tests/integration/read/retrieve.rs::an_ascii_run_below_the_length_floor_leaves` | performed | | `read.retrieve.unsatisfiable-arm-drops` | `crates/contextful-context/tests/integration/read/retrieve.rs::a_filter_binds_named_columns_and_a_table_lacking_one_drops_its_arm` | performed | +| `read.revise.browser-request` | `spec/pins.toml` | performed | | `read.revise.citation-live` | `crates/contextful-memory/tests/integration/evidence.rs::a_write_citing_a_superseded_version_refuses` | performed | | `read.revise.dedup-key` | `crates/contextful-memory/tests/integration/write.rs::a_dedup_key_seeds_the_claim_id_and_a_retry_lands_nothing` | performed | | `read.revise.direct-write` | `crates/contextful-core/tests/integration/memory/revise.rs::the_direct_write_accepts_claims_alone` | performed | +| `read.revise.malformed-request` | `spec/pins.toml` | performed | | `read.revise.observed-at` | `crates/contextful-memory/tests/integration/write.rs::an_observed_write_lands_valid_from_its_observed_instant` | performed | | `read.revise.observed-order` | `crates/contextful-memory/tests/integration/write.rs::an_observation_before_a_live_contradicting_prior_refuses` | performed | +| `read.revise.served-dedup` | `spec/pins.toml` | performed | +| `read.revise.served-scope` | `spec/pins.toml` | performed | +| `read.revise.served-write` | `spec/pins.toml` | performed | | `read.revise.supersede` | `crates/contextful-core/tests/integration/memory/revise.rs::an_equal_or_higher_claim_retires_its_prior_on_one_line` | performed | | `read.revise.tier` | `crates/contextful-core/tests/integration/memory/revise.rs::tier_follows_the_write_path_and_the_lowest_grounding` | performed | | `read.revise.unscoped-collision` | `crates/contextful-memory/tests/integration/write.rs::unscoped_writers_at_one_instant_record_a_conflict_without_retirement` | performed | @@ -1473,6 +1478,10 @@ Unscheduled operations: 10. | `surface.fire.store-driven-body` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_naming_an_unregistered_body_raises_job_body_unregistered` | performed | | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | +| `surface.ground.direct-file-read` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::direct file table functions refuse before dispatch` | performed | +| `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | +| `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | +| `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.issuer-pin` | `crates/contextful-cli/tests/integration/pipeline.rs::a_pulled_receipt_cannot_supply_its_own_trust_pin` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | @@ -1485,6 +1494,8 @@ Unscheduled operations: 10. | `surface.reside.region-entries` | `crates/contextful-core/tests/integration/surface/reside.rs::an_allow_set_holds_sixteen_entries` | performed | | `surface.reside.region-mismatch` | `crates/contextful-cli/tests/integration/pipeline.rs::a_resource_outside_the_residency_allow_set_serves_nothing` | performed | | `surface.reside.site-regions` | `crates/contextful-cli/tests/integration/sync/mod.rs::two_sites_declaring_different_residency_diverge_at_push` | performed | +| `surface.set-vantage.sample-labels` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a table contributes at most three sampled arrival labels` | performed | +| `surface.speak.redactor-lookahead` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::streaming redactor masks identifiers split across chunks` | performed | | `topology.compose.script-runtime` | `tools/ci/tests/integration/topology.rs::a_crate_linking_a_javascript_runtime_is_refused` | performed | | `topology.compose.undeclared-crossing` | `tools/ci/tests/integration/topology.rs::a_run_path_crate_reaching_a_read_path_crate_is_refused` | performed | | `topology.compose.vendor-sdk` | `tools/ci/tests/integration/topology.rs::a_crate_declaring_a_model_vendor_sdk_is_refused` | performed | From daeaea8634d3c9b1cdcdff26105fdcf62266e023 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:40:12 +0800 Subject: [PATCH 124/155] Keep read HTTP available without control features --- crates/contextful-cli/src/serve.rs | 25 ++++++++-- .../tests/integration/profile.rs | 48 +++++++++++++++++++ 2 files changed, 70 insertions(+), 3 deletions(-) diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 92418240f..c938acf1e 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -7,14 +7,20 @@ //! network transport, which admits each one on its own credential. Every value is //! resolved before the listener binds, so a process that cannot serve binds nothing. -use crate::admit::{AdmitArgs, face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; +#[cfg(feature = "data-plane")] +use crate::admit::AdmitArgs; +use crate::admit::{face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; -use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, EDIT_PATH, RECORD_PATH, WORKFLOWS_PATH}; +use contextful_agent::http::{audience, ceiling, Admitting, HttpFace}; +#[cfg(feature = "data-plane")] +use contextful_agent::http::{HttpRequest, HttpResponse, APPLY_PATH, EDIT_PATH, RECORD_PATH, WORKFLOWS_PATH}; +#[cfg(feature = "data-plane")] use contextful_core::surface::SurfaceError; use contextful_core::run::derive::task::Tasks; +#[cfg(feature = "data-plane")] use contextful_engine::control::ControlError; use contextful_core::issue::{IssuancePolicy, MintContext, NodeRole}; use contextful_core::ports::{Clock, SigningPort}; @@ -28,15 +34,20 @@ use contextful_policy::exchange::answer as exchange_answer; use contextful_policy::issue::{SeedSigner, DEFAULT_SEED_PATH}; use contextful_policy::possession::ProofChecker; use contextful_policy::audit::AuditLog; +#[cfg(feature = "data-plane")] use contextful_policy::verify::AdmittedAuthority; +#[cfg(feature = "data-plane")] use hmac::{Hmac, Mac}; +#[cfg(feature = "data-plane")] use sha2::{Digest, Sha256}; +#[cfg(feature = "data-plane")] use std::time::{SystemTime, UNIX_EPOCH}; use contextful_policy::keyset::{KeyCheckpoint, StaticPins}; use contextful_policy::revoke::RevocationState; use std::net::TcpListener; use std::path::PathBuf; use std::sync::Arc; +#[cfg(feature = "data-plane")] use serde_json::{json, Value}; #[cfg(feature = "data-plane")] @@ -108,6 +119,7 @@ pub struct ServeArgs { keyset: Option, } +#[cfg(feature = "data-plane")] fn operator_attestation(request: &HttpRequest, secret: &str) -> Option<(String, String, i64, i64)> { let subject = request.header("X-Contextful-Operator")?; let at: i64 = request.header("X-Contextful-Operator-Time")?.parse().ok()?; @@ -130,6 +142,7 @@ fn operator_attestation(request: &HttpRequest, secret: &str) -> Option<(String, Some((subject.to_owned(), nonce.to_owned(), at, now)) } +#[cfg(feature = "data-plane")] fn control_failure(error: anyhow::Error) -> HttpResponse { let surface = error.chain().find_map(|part| part.downcast_ref::().or_else(|| { match part.downcast_ref::() { @@ -153,6 +166,8 @@ fn issuer_pins(flag: Option<&str>) -> Result { } pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { + #[cfg(not(feature = "data-plane"))] + let _ = tasks; let clock = SystemClock; // The declarations are checked before anything opens (`read.register.serve-declaration`). let audience = audience(args.audience.as_deref()).map_err(anyhow::Error::msg)?; @@ -187,8 +202,11 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; + #[cfg(feature = "data-plane")] let control_attestation_secret = std::env::var("CONTEXTFUL_CONTROL_ATTESTATION_SECRET").ok().filter(|secret| !secret.is_empty()); + #[cfg(feature = "data-plane")] let control_admit = AdmitArgs { public_key: args.public_key.clone(), audience: args.audience.clone(), denylist: args.denylist.clone(), keyset: args.keyset.clone(), holder_key: None }; + #[cfg(feature = "data-plane")] let control = |request: &HttpRequest, authority: &AdmittedAuthority| -> HttpResponse { let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let path = request.target.split('?').next().unwrap_or_default(); @@ -310,9 +328,10 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { }; let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)) .map_err(anyhow::Error::msg)? - .with_control(&control) .with_exchange(&exchange_route, exchange.is_none()); #[cfg(feature = "data-plane")] + let http = http.with_control(&control); + #[cfg(feature = "data-plane")] let http = http.with_claim_write(&claim_write); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); diff --git a/crates/contextful-cli/tests/integration/profile.rs b/crates/contextful-cli/tests/integration/profile.rs index 8d4727309..a26b4f60e 100644 --- a/crates/contextful-cli/tests/integration/profile.rs +++ b/crates/contextful-cli/tests/integration/profile.rs @@ -110,3 +110,51 @@ fn the_read_replica_reads_and_syncs_and_refuses_every_write_path() { } assert!(!dir.path().join(".contextful").join("context").join("research").exists(), "a refused subcommand created a store root"); } + +/// HTTP admission stays available on the read replica; control routes require the run path. +#[cfg(feature = "read-plane")] +#[test] +fn served_http_keeps_read_admission_and_limits_control_to_the_data_plane() { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpStream; + use std::process::Stdio; + struct Listener(std::process::Child); + impl Drop for Listener { + fn drop(&mut self) { let _ = self.0.kill(); let _ = self.0.wait(); } + } + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("contextful.toml"), "authoring_posture = \"per_request\"\n").unwrap(); + let key = cf(dir.path(), &["token", "keygen", "--out", ".contextful/issuer.seed"]); + assert!(key.status.success(), "{}", String::from_utf8_lossy(&key.stderr)); + let public = String::from_utf8_lossy(&key.stdout).trim().to_owned(); + let mut child = Command::new(env!("CARGO_BIN_EXE_contextful")) + .args(["serve", "--http", "127.0.0.1:0", "--project", "research", "--audience", "contextful://profile-test", "--max-in-flight", "2", "--public-key", &public]) + .current_dir(dir.path()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().unwrap(); + let mut stderr = BufReader::new(child.stderr.take().unwrap()); + let _listener = Listener(child); + let address = loop { + let mut line = String::new(); + assert!(stderr.read_line(&mut line).unwrap() > 0, "HTTP profile face exits before binding"); + if let Some(address) = line.trim().strip_prefix("listening on http://").and_then(|value| value.strip_suffix("/mcp")) { + break address.to_owned(); + } + }; + let request = |method: &str, path: &str| { + let mut stream = TcpStream::connect(&address).unwrap(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(10))).unwrap(); + write!(stream, "{method} {path} HTTP/1.1\r\nHost: {address}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); + let mut answer = String::new(); + stream.read_to_string(&mut answer).unwrap(); + answer + }; + assert!(request("GET", "/health").starts_with("HTTP/1.1 200")); + let refused = request("POST", "/mcp"); + assert!(refused.starts_with("HTTP/1.1 401"), "{refused}"); + assert!(refused.to_ascii_lowercase().contains("www-authenticate:"), "{refused}"); + for (method, path) in [("GET", "/control/workflows"), ("POST", "/control/apply")] { + let answer = request(method, path); + let status = if cfg!(feature = "data-plane") { "401" } else { "404" }; + assert!(answer.starts_with(&format!("HTTP/1.1 {status}")), "{answer}"); + assert!(!answer.contains("filings-flow"), "{answer}"); + } +} From f2611ce834837065e336bae08197006186bda6f4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 00:40:50 +0800 Subject: [PATCH 125/155] Keep read HTTP available without control features --- crates/contextful-cli/src/serve.rs | 18 +++++-- .../tests/integration/profile.rs | 48 +++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index 34f7c43e2..a03a5b0fc 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -7,12 +7,17 @@ //! network transport, which admits each one on its own credential. Every value is //! resolved before the listener binds, so a process that cannot serve binds nothing. -use crate::admit::{AdmitArgs, face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; +#[cfg(feature = "data-plane")] +use crate::admit::AdmitArgs; +use crate::admit::{face, revocation_state, LedgerFile, LivePins, AUDIENCE_VAR, PUBKEY_VAR}; use crate::project::locate; use crate::root::root as project_root; use crate::clock::SystemClock; use anyhow::Result; -use contextful_agent::http::{audience, ceiling, Admitting, HttpFace, HttpRequest, HttpResponse, APPLY_PATH, WORKFLOWS_PATH}; +use contextful_agent::http::{audience, ceiling, Admitting, HttpFace}; +#[cfg(feature = "data-plane")] +use contextful_agent::http::{HttpRequest, HttpResponse, APPLY_PATH, WORKFLOWS_PATH}; +#[cfg(feature = "data-plane")] use contextful_core::surface::SurfaceError; use contextful_core::run::derive::task::Tasks; use contextful_policy::audit::AuditLog; @@ -21,6 +26,7 @@ use contextful_policy::revoke::RevocationState; use std::net::TcpListener; use std::path::PathBuf; use std::sync::Arc; +#[cfg(feature = "data-plane")] use serde_json::{json, Value}; /// The refusals of starting the network transport. `Display` begins with the identifier. @@ -77,6 +83,8 @@ fn issuer_pins(flag: Option<&str>) -> Result { } pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { + #[cfg(not(feature = "data-plane"))] + let _ = tasks; let clock = SystemClock; // The declarations are checked before anything opens (`read.register.serve-declaration`). let audience = audience(args.audience.as_deref()).map_err(anyhow::Error::msg)?; @@ -111,7 +119,9 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { None => face, }; let audit = AuditLog::unanchored(located.project.audit_dir())?; + #[cfg(feature = "data-plane")] let control_admit = AdmitArgs { public_key: args.public_key.clone(), audience: args.audience.clone(), denylist: args.denylist.clone(), keyset: args.keyset.clone(), holder_key: None }; + #[cfg(feature = "data-plane")] let control = |request: &HttpRequest| -> HttpResponse { let answer = match request.target.split('?').next().unwrap_or_default() { WORKFLOWS_PATH => crate::cadence::published(&located.project, &located.declaration), @@ -141,7 +151,9 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { } } }; - let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)).map_err(anyhow::Error::msg)?.with_control(&control); + let http = HttpFace::new(&face, &clock, &audit, admitting, Some(ceiling)).map_err(anyhow::Error::msg)?; + #[cfg(feature = "data-plane")] + let http = http.with_control(&control); let listener = TcpListener::bind(&args.http)?; eprintln!("listening on http://{}/mcp", listener.local_addr()?); http.serve(listener)?; diff --git a/crates/contextful-cli/tests/integration/profile.rs b/crates/contextful-cli/tests/integration/profile.rs index 8d4727309..a26b4f60e 100644 --- a/crates/contextful-cli/tests/integration/profile.rs +++ b/crates/contextful-cli/tests/integration/profile.rs @@ -110,3 +110,51 @@ fn the_read_replica_reads_and_syncs_and_refuses_every_write_path() { } assert!(!dir.path().join(".contextful").join("context").join("research").exists(), "a refused subcommand created a store root"); } + +/// HTTP admission stays available on the read replica; control routes require the run path. +#[cfg(feature = "read-plane")] +#[test] +fn served_http_keeps_read_admission_and_limits_control_to_the_data_plane() { + use std::io::{BufRead, BufReader, Read, Write}; + use std::net::TcpStream; + use std::process::Stdio; + struct Listener(std::process::Child); + impl Drop for Listener { + fn drop(&mut self) { let _ = self.0.kill(); let _ = self.0.wait(); } + } + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("contextful.toml"), "authoring_posture = \"per_request\"\n").unwrap(); + let key = cf(dir.path(), &["token", "keygen", "--out", ".contextful/issuer.seed"]); + assert!(key.status.success(), "{}", String::from_utf8_lossy(&key.stderr)); + let public = String::from_utf8_lossy(&key.stdout).trim().to_owned(); + let mut child = Command::new(env!("CARGO_BIN_EXE_contextful")) + .args(["serve", "--http", "127.0.0.1:0", "--project", "research", "--audience", "contextful://profile-test", "--max-in-flight", "2", "--public-key", &public]) + .current_dir(dir.path()).stdout(Stdio::null()).stderr(Stdio::piped()).spawn().unwrap(); + let mut stderr = BufReader::new(child.stderr.take().unwrap()); + let _listener = Listener(child); + let address = loop { + let mut line = String::new(); + assert!(stderr.read_line(&mut line).unwrap() > 0, "HTTP profile face exits before binding"); + if let Some(address) = line.trim().strip_prefix("listening on http://").and_then(|value| value.strip_suffix("/mcp")) { + break address.to_owned(); + } + }; + let request = |method: &str, path: &str| { + let mut stream = TcpStream::connect(&address).unwrap(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(10))).unwrap(); + write!(stream, "{method} {path} HTTP/1.1\r\nHost: {address}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").unwrap(); + let mut answer = String::new(); + stream.read_to_string(&mut answer).unwrap(); + answer + }; + assert!(request("GET", "/health").starts_with("HTTP/1.1 200")); + let refused = request("POST", "/mcp"); + assert!(refused.starts_with("HTTP/1.1 401"), "{refused}"); + assert!(refused.to_ascii_lowercase().contains("www-authenticate:"), "{refused}"); + for (method, path) in [("GET", "/control/workflows"), ("POST", "/control/apply")] { + let answer = request(method, path); + let status = if cfg!(feature = "data-plane") { "401" } else { "404" }; + assert!(answer.starts_with(&format!("HTTP/1.1 {status}")), "{answer}"); + assert!(!answer.contains("filings-flow"), "{answer}"); + } +} From d643b6d7e9fe37de11717718dbb3c13aa549e6e4 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 01:03:53 +0800 Subject: [PATCH 126/155] fix(ci): report the base listing timeout bound --- tools/ci/src/main.rs | 6 +++++- tools/ci/tests/integration/test_first.rs | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 4f9d44263..706afe03d 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1290,7 +1290,11 @@ fn red_against_base(root: &Path, tree: &Path, target: &Path, tests: &[&str], bou if !matches!(listed, BaseRun::Passed) { eprint!("{}{}", printed.stdout, printed.stderr); unrunnable(&printed)?; - red.push(format!("{label}, whose tests do not list at base")); + red.push(match listed { + BaseRun::Killed => format!("{label}, listing killed at the {} s bound", bound.as_secs()), + BaseRun::Failed => format!("{label}, whose tests do not list at base"), + BaseRun::Passed => unreachable!("a passing listing has no red verdict"), + }); continue; } let names: Vec = printed diff --git a/tools/ci/tests/integration/test_first.rs b/tools/ci/tests/integration/test_first.rs index 26c8ff371..5b2f58254 100644 --- a/tools/ci/tests/integration/test_first.rs +++ b/tools/ci/tests/integration/test_first.rs @@ -272,6 +272,20 @@ fn a_test_quoting_an_infrastructure_fault_still_reads_as_red() { assert!(err.contains("red against the base in crates/demo"), "{err}"); } +#[test] +fn a_base_listing_past_its_bound_reports_the_package_and_bound() { + let r = Repo::init(); + let base = r.head(); + r.write("crates/demo/src/lib.rs", TRIPLE); + r.write("crates/demo/tests/integration/double.rs", "#[test]\nfn doubles() {\n assert_eq!(demo::double(4), 8);\n}\n"); + r.commit("triple, with a changed double test"); + let cargo = "case \" $* \" in\n *\" --list \"*) sleep 30 ;;\nesac\n"; + let o = r.gate_with_cargo(cargo, &["--stage", "test-first", "--base", &base, "--base-bound-secs", "20"]); + let err = stderr(&o); + assert!(o.status.success(), "{err}"); + assert!(err.contains("crates/demo") && err.contains("listing killed at the 20 s bound"), "{err}"); +} + #[test] fn a_slow_base_build_is_not_charged_to_the_bound() { let r = Repo::init(); From 715a6f56e4aca429815d7ce13fc72dbccdef0680 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 01:21:49 +0800 Subject: [PATCH 127/155] Render live console clause pins --- spec/status.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/spec/status.md b/spec/status.md index bed22f463..b00a203d1 100644 --- a/spec/status.md +++ b/spec/status.md @@ -13,9 +13,9 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | `read` | 2 | 15 | 200 | 31 | 22 | 9 | 178 | 0 | 172 | | `run` | 3 | 26 | 406 | 102 | 39 | 28 | 306 | 0 | 279 | | `store` | 1 | 17 | 249 | 57 | 13 | 19 | 213 | 0 | 204 | -| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 59 | 0 | 59 | +| `surface` | 2 | 20 | 114 | 51 | 22 | 0 | 65 | 0 | 59 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | -| **total** | 19 | 155 | 2105 | 593 | 191 | 103 | 1476 | 0 | | +| **total** | 19 | 155 | 2105 | 593 | 191 | 103 | 1482 | 0 | | Decision records: 18. @@ -37,7 +37,7 @@ A milestone reads `closed` when its acceptance test computes `passing` and every | 9 — Visibility | 6 | 41 | 3 | open | open | | 10 — Cadence and the operator plane | 11 | 76 | 64 | passing | open | | 11 — The derive tier | 7 | 88 | 79 | passing | closed | -| 12 — The operator console | 12 | 49 | 0 | open | open | +| 12 — The operator console | 12 | 49 | 6 | open | open | | 13 — Disclosure | 5 | 33 | 13 | open | open | | 14 — Assurance | 7 | 138 | 89 | passing | closed | @@ -1473,6 +1473,10 @@ Unscheduled operations: 10. | `surface.fire.store-driven-body` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_naming_an_unregistered_body_raises_job_body_unregistered` | performed | | `surface.fire.store-driven-concurrency` | `crates/contextful-core/tests/integration/job.rs::a_store_driven_block_without_a_positive_max_in_flight_raises_job_concurrency_unset` | performed | | `surface.fire.target-unbound` | `crates/contextful-cli/tests/integration/job.rs::a_job_target_naming_nothing_produced_is_refused_at_validation` | performed | +| `surface.ground.direct-file-read` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::direct file table functions refuse before dispatch` | performed | +| `surface.ground.mint-refused` | `apps/console/test/live.test.ts::apps/console/test/live.test.ts::Query falls back to a configured shared credential only after exchange refusal` | performed | +| `surface.ground.org-face-read-only` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::organization packs refuse writes at startup` | performed | +| `surface.plan-turn.planner-reached-memory` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a planner call targeting a memory relation dispatches nothing` | performed | | `surface.reconcile.fail-static` | `crates/contextful-cli/tests/integration/pipeline.rs::an_unreadable_snapshot_leaves_the_armed_set_running` | performed | | `surface.reconcile.issuer-pin` | `crates/contextful-cli/tests/integration/pipeline.rs::a_pulled_receipt_cannot_supply_its_own_trust_pin` | performed | | `surface.reconcile.learns-by-reading` | `crates/contextful-cli/tests/integration/pipeline.rs::a_wake_reads_the_applied_version_from_the_control_source` | performed | @@ -1485,6 +1489,8 @@ Unscheduled operations: 10. | `surface.reside.region-entries` | `crates/contextful-core/tests/integration/surface/reside.rs::an_allow_set_holds_sixteen_entries` | performed | | `surface.reside.region-mismatch` | `crates/contextful-cli/tests/integration/pipeline.rs::a_resource_outside_the_residency_allow_set_serves_nothing` | performed | | `surface.reside.site-regions` | `crates/contextful-cli/tests/integration/sync/mod.rs::two_sites_declaring_different_residency_diverge_at_push` | performed | +| `surface.set-vantage.sample-labels` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::a table contributes at most three sampled arrival labels` | performed | +| `surface.speak.redactor-lookahead` | `apps/console/test/turn.test.ts::apps/console/test/turn.test.ts::streaming redactor masks identifiers split across chunks` | performed | | `topology.compose.script-runtime` | `tools/ci/tests/integration/topology.rs::a_crate_linking_a_javascript_runtime_is_refused` | performed | | `topology.compose.undeclared-crossing` | `tools/ci/tests/integration/topology.rs::a_run_path_crate_reaching_a_read_path_crate_is_refused` | performed | | `topology.compose.vendor-sdk` | `tools/ci/tests/integration/topology.rs::a_crate_declaring_a_model_vendor_sdk_is_refused` | performed | From c9f90391397c30cbe643f803bb990a6535357260 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 01:34:53 +0800 Subject: [PATCH 128/155] Cover owner binding and draft refusal contracts --- .../tests/integration/identify.rs | 19 +++++++++++++++++++ .../tests/integration/surface/control.rs | 7 +++++++ 2 files changed, 26 insertions(+) diff --git a/crates/contextful-core/tests/integration/identify.rs b/crates/contextful-core/tests/integration/identify.rs index e1ff67db0..58c5c9e79 100644 --- a/crates/contextful-core/tests/integration/identify.rs +++ b/crates/contextful-core/tests/integration/identify.rs @@ -16,6 +16,25 @@ fn subject(on_behalf_of: Option<&str>, agent: Option<&str>) -> Subject { } } +#[test] +fn an_owner_project_survives_authority_block_round_trips() { + let mut document = serde_json::json!({ + "iss": "issuer", "aud": "contextful://stdio", "jti": "owner", + "iat": 1, "exp": 2, "alg": "Ed25519", "sub": { "on_behalf_of": "operator" }, + "att": {}, "grants": [], "owner_project": "research", + "rev": { "id": "owner", "epoch": 0 } + }); + let block: AuthorityBlock = serde_json::from_value(document.clone()).unwrap(); + let serialized = serde_json::to_value(block).unwrap(); + assert_eq!(serialized["owner_project"], "research"); + let restored: AuthorityBlock = serde_json::from_value(serialized).unwrap(); + assert_eq!(serde_json::to_value(restored).unwrap()["owner_project"], "research"); + + document.as_object_mut().unwrap().remove("owner_project"); + let bearer: AuthorityBlock = serde_json::from_value(document).unwrap(); + assert!(serde_json::to_value(bearer).unwrap().get("owner_project").is_none()); +} + fn link(method: LinkMethod, confidence: f64) -> IdentityLink { IdentityLink { source: "drive".to_string(), diff --git a/crates/contextful-core/tests/integration/surface/control.rs b/crates/contextful-core/tests/integration/surface/control.rs index bf40c92ff..4ed044140 100644 --- a/crates/contextful-core/tests/integration/surface/control.rs +++ b/crates/contextful-core/tests/integration/surface/control.rs @@ -3,6 +3,13 @@ use contextful_core::surface::control::{parse_pointer, receipt_file, receipt_version, snapshot_file, POINTER_FILE}; use contextful_core::surface::SurfaceError; +#[test] +fn an_absent_draft_preserves_its_identifier_and_conflict_status() { + let error = SurfaceError::ControlDraftAbsent("draft nonce is absent".into()); + assert_eq!(error.status(), 409); + assert_eq!(error.to_string(), "ControlDraftAbsent: draft nonce is absent"); +} + #[test] fn an_untrusted_pulled_control_head_names_its_reason() { let error = SurfaceError::ControlSnapshotUntrusted("receipt signature does not verify".into()); From 6890ffe10c55d08dc875e47faa1fb63011ad525c Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 03:08:47 +0800 Subject: [PATCH 129/155] Redact long Query secrets from prose and source URLs --- apps/console/src/live.ts | 5 +++-- apps/console/test/live.test.ts | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index d22b7dfdb..33eb30ed4 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -149,7 +149,8 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput & { reca resultRows = { columns: visible.map(({ column }) => column), rows: cleanRows }; return { rows: cleanRows.map((row) => Object.fromEntries(visible.map(({ column }, index) => [column, row[index]]))), - sources: sourced.map(({ citation }) => ({ ...citation, label: redactText(citation.label, denylist) })), + sources: sourced.map(({ citation }) => ({ ...citation, label: redactText(citation.label, denylist), + url: citation.url && !denylist.some((entry) => entry && citation.url!.includes(entry)) ? citation.url : undefined })), }; } }, overlay: async () => input.recallOverlay ?? null, @@ -166,7 +167,7 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput & { reca const choices = record(body) && Array.isArray(body.choices) ? body.choices : []; const message = choices.length && record(choices[0]) ? choices[0].message : undefined; if (!record(message) || typeof message.content !== "string") throw new ConsoleError("ConsoleModelUnavailable", "model answer absent", 503); - yield message.content; + yield redactText(message.content, denylist); }, }); const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index 5e03361c3..a945116a3 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -175,3 +175,32 @@ test("Cognito Query exchanges the signed-in reader's assertion", async () => { assert.equal(response.status, 200); assert.equal(calls.find((call) => call.url.endsWith("/auth/exchange"))?.body.jwt, "cognito-id-token"); }); + +test("Query redacts long denylisted values and credentials from prose and source URLs", async () => { + const assertion = "assertion-" + "a".repeat(512); + const credential = "credential-" + "c".repeat(512); + const denied = "private-" + "p".repeat(256); + const fetcher: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/auth/exchange")) return Response.json({ token: credential }); + if (url.endsWith("/mcp")) { + const body = JSON.parse(String(init?.body)); + const value = body.params.name === "context.describe" ? { tables: [{ table: "filings", kind: "data" }] } : { + columns: ["filing_id", "summary", "source_url"], + rows: [["filing-1", "A filing arrived", `https://example.test/${credential}`]], + }; + return Response.json({ result: { structuredContent: value } }); + } + return Response.json({ choices: [{ message: { content: `A filing arrived [filing-1]. ${assertion} ${credential} ${denied}` } }] }); + }; + const turn = createLiveTurn({ stores: [store], env: { + CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture", + CONTEXTFUL_CONSOLE_DENYLIST: JSON.stringify([denied]), + }, fetcher }); + const result = await turn({ operator: { subject: "reader", grants: new Set(["query"]), assertion }, + store: store.id, question: "Which filing arrived?" }); + const visible = JSON.stringify(result); + for (const secret of [assertion, credential, denied]) assert.equal(visible.includes(secret), false); + assert.match(result.answer, /A filing arrived \[source-1\]/); + assert.equal((result.sources[0] as { url?: string }).url, undefined); +}); From 8822adba234688f82de1b3d1e01d5907ddb20512 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 03:08:51 +0800 Subject: [PATCH 130/155] Recheck control authority under mutation locks --- crates/contextful-cli/src/cadence.rs | 16 +++--- crates/contextful-cli/src/serve.rs | 30 +++++++++-- .../contextful-cli/tests/integration/serve.rs | 50 +++++++++++++++++++ crates/contextful-engine/src/control.rs | 39 +++++++++++++-- .../tests/integration/control.rs | 25 ++++++++++ 5 files changed, 144 insertions(+), 16 deletions(-) diff --git a/crates/contextful-cli/src/cadence.rs b/crates/contextful-cli/src/cadence.rs index 84b46e9cf..e72c8cd52 100644 --- a/crates/contextful-cli/src/cadence.rs +++ b/crates/contextful-cli/src/cadence.rs @@ -450,24 +450,24 @@ fn validated_draft(document: &str, declaration: &Path, tasks: &Tasks) -> Result< } /// Store one validated, version-bound control draft without changing the applied pointer. -pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected: u64, document: &str, operator: &str, tasks: &Tasks) -> Result { +pub(crate) fn edit(project: &ProjectArgs, declaration: Option, expected: u64, document: &str, operator: &str, tasks: &Tasks, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { let (located, _, control) = located(project, declaration)?; let snapshots = owner(&control)?; snapshots.initialized()?; let text = validated_draft(document, &located.declaration, tasks)?; let draft = Draft::new(expected, text, operator.to_owned())?; - snapshots.save_draft(&draft)?; + snapshots.save_draft_guarded(&draft, boundary)?; Ok(json!({ "expected": expected, "nonce": draft.nonce })) } /// Claim a verified control request's nonce in the selected store owner's snapshot state. -pub(crate) fn claim_operator_nonce(project: &ProjectArgs, declaration: Option, nonce: &str, signed_at: i64, now: i64) -> Result { +pub(crate) fn claim_operator_nonce(project: &ProjectArgs, declaration: Option, nonce: &str, signed_at: i64, now: i64, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { let (_, _, control) = located(project, declaration)?; - owner(&control)?.claim_attestation_nonce(nonce, signed_at, now).map_err(Into::into) + owner(&control)?.claim_attestation_nonce_guarded(nonce, signed_at, now, boundary).map_err(Into::into) } /// Revalidate the saved draft and claim it only at the version the editor read. -pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks, authority: &AdmittedAuthority, admit: &AdmitArgs) -> Result<()> { +pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, expected: u64, nonce: &str, operator: &str, tasks: &Tasks, authority: &AdmittedAuthority, admit: &AdmitArgs, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result<()> { let (initial, _, control) = located(project, declaration.clone())?; let snapshots = owner(&control)?; snapshots.initialized()?; @@ -485,13 +485,13 @@ pub(crate) fn apply_draft(project: &ProjectArgs, declaration: Option, e } match SyncAttestation::for_authority(&fresh, authority.clone(), admit, None)? { Some(attestation) => { - owner.claim_draft_attested(&draft, |next, previous| { + owner.claim_draft_attested_guarded(&draft, |next, previous| { let prior_snapshot = owner.read(expected)?; let parent = previous.map(|receipt| (receipt, expected, prior_snapshot.as_str())); attestation.receipt(&fresh.project.name, next, parent, &draft.document) - })?; + }, boundary)?; } - None => { owner.claim_draft(&draft)?; } + None => { owner.claim_draft_guarded(&draft, boundary)?; } } Ok(()) } diff --git a/crates/contextful-cli/src/serve.rs b/crates/contextful-cli/src/serve.rs index c938acf1e..917fa6d04 100644 --- a/crates/contextful-cli/src/serve.rs +++ b/crates/contextful-cli/src/serve.rs @@ -144,6 +144,17 @@ fn operator_attestation(request: &HttpRequest, secret: &str) -> Option<(String, #[cfg(feature = "data-plane")] fn control_failure(error: anyhow::Error) -> HttpResponse { + if error.chain().any(|part| matches!(part.downcast_ref::(), Some(ControlError::OperatorAttestationInvalid(_)))) { + return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })); + } + if let Some(authority) = error.chain().find_map(|part| { + part.downcast_ref::().or_else(|| match part.downcast_ref::() { + Some(ControlError::Authority(authority)) => Some(authority), + _ => None, + }) + }) { + return HttpResponse::json(401, &json!({ "error": { "identifier": authority.to_string().split(':').next().unwrap_or("AuthorityRevoked") } })); + } let surface = error.chain().find_map(|part| part.downcast_ref::().or_else(|| { match part.downcast_ref::() { Some(ControlError::Surface(refusal)) => Some(refusal), @@ -208,14 +219,25 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { let control_admit = AdmitArgs { public_key: args.public_key.clone(), audience: args.audience.clone(), denylist: args.denylist.clone(), keyset: args.keyset.clone(), holder_key: None }; #[cfg(feature = "data-plane")] let control = |request: &HttpRequest, authority: &AdmittedAuthority| -> HttpResponse { - let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let path = request.target.split('?').next().unwrap_or_default(); + let boundary = || { + let state = revocation().map_err(ControlError::Storage)?; + contextful_policy::verify::effect_boundary(authority, + &contextful_policy::verify::Admission::new(clock.now(), &state).expecting(audience)) + .map_err(ControlError::from)?; + if (path == EDIT_PATH || path == APPLY_PATH) && + control_attestation_secret.as_deref().and_then(|secret| operator_attestation(request, secret)).is_none() { + return Err(ControlError::OperatorAttestationInvalid("the operator signature expired before the control mutation".into())); + } + Ok(()) + }; + let malformed = || HttpResponse::json(400, &json!({ "error": { "identifier": "ControlRequestMalformed" } })); let project = crate::run::ProjectArgs { project: Some(located.project.name.clone()), now: None }; let operator = if path == EDIT_PATH || path == APPLY_PATH { match control_attestation_secret.as_deref() { Some(secret) => match operator_attestation(request, secret) { Some((subject, nonce, signed_at, now)) => { - match crate::cadence::claim_operator_nonce(&project, Some(located.declaration.clone()), &nonce, signed_at, now) { + match crate::cadence::claim_operator_nonce(&project, Some(located.declaration.clone()), &nonce, signed_at, now, &boundary) { Ok(true) => subject, Ok(false) => return HttpResponse::json(403, &json!({ "error": { "identifier": "ControlOperatorAttestationInvalid" } })), Err(error) => return control_failure(error), @@ -250,7 +272,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { "contextful.credential": authority.credential_id(), "contextful.control.expected": expected })).is_err() { return HttpResponse::json(503, &json!({ "error": { "identifier": "AuditEntryUnpersisted" } })); } - crate::cadence::edit(&project, Some(located.declaration.clone()), expected, document, &operator, tasks) + crate::cadence::edit(&project, Some(located.declaration.clone()), expected, document, &operator, tasks, &boundary) } else { if fields.len() != 2 { return malformed(); } let Some(nonce) = fields.get("nonce").and_then(Value::as_str) else { return malformed() }; @@ -260,7 +282,7 @@ pub fn run(args: ServeArgs, tasks: &Tasks) -> Result<()> { "contextful.control.draft_nonce": nonce })).is_err() { return HttpResponse::json(503, &json!({ "error": { "identifier": "AuditEntryUnpersisted" } })); } - crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, nonce, &operator, tasks, authority, &control_admit) + crate::cadence::apply_draft(&project, Some(located.declaration.clone()), expected, nonce, &operator, tasks, authority, &control_admit, &boundary) .and_then(|()| crate::cadence::published(&located.project, &located.declaration)) } } diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index acf65bd16..19b93f03f 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -161,6 +161,56 @@ fn attestation(path: &str, body: &str, operator: &str, at: i64, nonce: &str) -> format!("X-Contextful-Operator: {operator}\r\nX-Contextful-Operator-Time: {at}\r\nX-Contextful-Operator-Nonce: {nonce}\r\nX-Contextful-Operator-Signature: {signature}\r\n") } +#[test] +fn pending_control_edit_refuses_revoked_authority_before_nonce_or_draft_commit() { + pending_control_mutation(false, false); +} + +#[test] +fn pending_control_apply_refuses_revoked_authority_before_nonce_or_pointer_commit() { + pending_control_mutation(true, false); +} + +#[test] +fn pending_control_edit_refuses_expired_authority_before_nonce_or_draft_commit() { + pending_control_mutation(false, true); +} + +fn pending_control_mutation(apply: bool, expired: bool) { + let (dir, public) = project(); + let root = dir.path(); + let original = "[[pipeline]]\nid = \"filings-flow\"\ntables = [\"research/notes\"]\n[pipeline.source]\nname = \"http\"\nconfig = { endpoint = \"https://example.test/filings\" }\n"; + std::fs::create_dir_all(root.join("pipelines")).unwrap(); + std::fs::write(root.join("pipelines/filings.toml"), original).unwrap(); + stdout(&run(root, &["pipeline", "import", "--project", "research"])); + let (_listener, addr) = serve(root, &["serve", "--http", "127.0.0.1:0", "--audience", AUD, "--max-in-flight", "2", "--project", "research", "--public-key", &public]); + let admin = stdout(&run(root, &["token", "mint", "--on-behalf-of", "user://dana@acme.example", "--zone", "on-prem:hq", "--action", "admin", "--table", "*", "--ttl", if expired { "2" } else { "900" }])); + let edit = json!({ "expected": 1, "document": original }).to_string(); + let (path, body) = if apply { + let (status, saved) = control(&addr, "POST", "/control/edit", Some(&admin), &edit); + assert_eq!(status, 200, "{saved}"); + ("/control/apply", json!({ "expected": 1, "nonce": saved["nonce"] }).to_string()) + } else { ("/control/edit", edit) }; + let snapshots = root.join(".contextful/control/research"); + let lock = std::fs::OpenOptions::new().read(true).write(true).open(snapshots.join("manifest.lock")).unwrap(); + lock.lock().unwrap(); + let at = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs() as i64; + let nonce = "f".repeat(32); + let headers = attestation(path, &body, "user://dana@acme.example", at, &nonce); + std::thread::scope(|scope| { + let pending = scope.spawn(|| control_headers(&addr, "POST", path, Some(&admin), &body, &headers)); + std::thread::sleep(std::time::Duration::from_millis(if expired { 3100 } else { 200 })); + if !expired { stdout(&run(root, &["token", "revoke", "--principal-class", "delegated"])); } + drop(lock); + let (status, response) = pending.join().unwrap(); + assert_eq!(status, 401, "{response}"); + assert_eq!(response["error"]["identifier"], if expired { "AuthorityExpired" } else { "AuthorityRevoked" }); + }); + assert!(!snapshots.join(format!("attestation-nonces/{at}/{nonce}")).exists()); + assert_eq!(std::fs::read_to_string(snapshots.join("manifest@current")).unwrap().trim(), "1"); + assert_eq!(snapshots.join("manifest@draft.json").exists(), apply); +} + #[test] fn admin_changes_require_fresh_console_attestation_and_record_the_operator() { let (dir, public) = project(); diff --git a/crates/contextful-engine/src/control.rs b/crates/contextful-engine/src/control.rs index a38a99794..159ccac73 100644 --- a/crates/contextful-engine/src/control.rs +++ b/crates/contextful-engine/src/control.rs @@ -41,6 +41,10 @@ type ReceiptVerifier<'a> = dyn Fn(&str) -> Result<(), ControlError> + 'a; /// A snapshot-directory refusal or a storage failure beneath it. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum ControlError { + #[error("ControlOperatorAttestationInvalid: {0}")] + OperatorAttestationInvalid(String), + #[error(transparent)] + Authority(#[from] contextful_core::AuthorityError), #[error(transparent)] Surface(#[from] SurfaceError), #[error("{0}")] @@ -113,11 +117,17 @@ impl SnapshotDir { /// applied-pointer mutations. A claim stays until its signed time is no longer /// admissible, including across a listener restart. pub fn claim_attestation_nonce(&self, nonce: &str, signed_at: i64, now: i64) -> Result { + self.claim_attestation_nonce_guarded(nonce, signed_at, now, &|| Ok(())) + } + + /// Recheck authority under the nonce lock before changing replay state. + pub fn claim_attestation_nonce_guarded(&self, nonce: &str, signed_at: i64, now: i64, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { if nonce.len() != 32 || !nonce.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(ControlError::Storage("an attestation nonce is 32 hexadecimal bytes".into())); } let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; + boundary()?; let directory = self.root.join(ATTESTATION_NONCES); std::fs::create_dir_all(&directory).map_err(|e| ControlError::Storage(format!("{}: {e}", directory.display())))?; let first_live = now.saturating_sub(60); @@ -148,10 +158,16 @@ impl SnapshotDir { /// Save one store draft only while the applied version still matches its base. pub fn save_draft(&self, draft: &Draft) -> Result<(), ControlError> { + self.save_draft_guarded(draft, &|| Ok(())) + } + + /// Recheck authority under the draft lock immediately before publication. + pub fn save_draft_guarded(&self, draft: &Draft, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result<(), ControlError> { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; self.expect_version(draft.expected)?; let body = serde_json::to_vec(draft).map_err(|e| ControlError::Storage(e.to_string()))?; + boundary()?; replace(&self.root.join(DRAFT_FILE), &body).map_err(storage)?; Ok(()) } @@ -168,22 +184,32 @@ impl SnapshotDir { /// Claim the exact draft revalidated by the caller under the existing pointer lock. pub fn claim_draft(&self, draft: &Draft) -> Result { - self.claim_draft_inner(draft, None) + self.claim_draft_guarded(draft, &|| Ok(())) + } + + /// Recheck authority under the claim lock before an unsigned draft commit. + pub fn claim_draft_guarded(&self, draft: &Draft, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { + self.claim_draft_inner(draft, None, boundary) } /// Claim a validated draft with its signed receipt before publishing the pointer. pub fn claim_draft_attested(&self, draft: &Draft, receipt: impl Fn(u64, Option<&str>) -> Result) -> Result { - self.claim_draft_inner(draft, Some(&receipt)) + self.claim_draft_attested_guarded(draft, receipt, &|| Ok(())) + } + + /// Recheck the control boundary after signing and before publishing an attested draft. + pub fn claim_draft_attested_guarded(&self, draft: &Draft, receipt: impl Fn(u64, Option<&str>) -> Result, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { + self.claim_draft_inner(draft, Some(&receipt), boundary) } - fn claim_draft_inner(&self, draft: &Draft, receipt: Option<&ReceiptBuilder<'_>>) -> Result { + fn claim_draft_inner(&self, draft: &Draft, receipt: Option<&ReceiptBuilder<'_>>, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let _lock = FileLock::acquire(&self.root.join(LOCK_FILE)).map_err(storage)?; self.expect_version(draft.expected)?; if self.read_draft()? != *draft { return Err(SurfaceError::ManifestVersionConflict("the validated draft changed before apply; reload and reapply".into()).into()); } - let version = self.claim_locked(Some(draft.expected), &draft.document, receipt, None)?; + let version = self.claim_locked_guarded(Some(draft.expected), &draft.document, receipt, None, boundary)?; let _ = std::fs::remove_file(self.root.join(DRAFT_FILE)); Ok(version) } @@ -263,6 +289,10 @@ impl SnapshotDir { } fn claim_locked(&self, expected: Option, text: &str, receipt: Option<&ReceiptBuilder<'_>>, recover: Option<&ReceiptVerifier<'_>>) -> Result { + self.claim_locked_guarded(expected, text, receipt, recover, &|| Ok(())) + } + + fn claim_locked_guarded(&self, expected: Option, text: &str, receipt: Option<&ReceiptBuilder<'_>>, recover: Option<&ReceiptVerifier<'_>>, boundary: &dyn Fn() -> Result<(), ControlError>) -> Result { let storage = |e: contextful_core::run::Failure| ControlError::Storage(e.to_string()); let current = self.current()?; if current != expected { @@ -324,6 +354,7 @@ impl SnapshotDir { } let signed = receipt.map(|sign| sign(version, parent.as_deref())).transpose()?; let snapshot = self.root.join(snapshot_file(version)); + boundary()?; if create_new(&snapshot, text.as_bytes()).map_err(storage)? { if let Some(signed) = signed { let receipt_error = match create_new(&self.root.join(receipt_file(version)), signed.as_bytes()) { diff --git a/crates/contextful-engine/tests/integration/control.rs b/crates/contextful-engine/tests/integration/control.rs index 3e159976e..dcc51f154 100644 --- a/crates/contextful-engine/tests/integration/control.rs +++ b/crates/contextful-engine/tests/integration/control.rs @@ -3,6 +3,31 @@ use contextful_core::surface::SurfaceError; use contextful_engine::control::{ControlError, Draft, SnapshotDir}; +#[test] +fn control_mutations_check_authority_under_the_lock_and_refuse_without_writes() { + let dir = tempfile::tempdir().unwrap(); + let snaps = SnapshotDir::open(dir.path()); + snaps.claim_attested(None, "original", |_, _| Ok("first receipt".into())).unwrap(); + let draft = Draft::new(1, "edited".into(), "alice".into()).unwrap(); + let refuse = || { + let lock = std::fs::OpenOptions::new().read(true).write(true).open(dir.path().join("manifest.lock")).unwrap(); + assert!(lock.try_lock().is_err(), "the effect boundary holds the mutation lock"); + Err(ControlError::Storage("authority revoked at commit".into())) + }; + assert!(snaps.claim_attestation_nonce_guarded(&"a".repeat(32), 100, 100, &refuse).is_err()); + assert!(!dir.path().join("attestation-nonces").exists()); + assert!(snaps.save_draft_guarded(&draft, &refuse).is_err()); + assert!(!dir.path().join("manifest@draft.json").exists()); + snaps.save_draft(&draft).unwrap(); + assert!(snaps.claim_draft_guarded(&draft, &refuse).is_err()); + assert_eq!(snaps.current().unwrap(), Some(1)); + assert!(!dir.path().join("manifest@v2.toml").exists()); + assert_eq!(snaps.read_draft().unwrap(), draft); + assert!(snaps.claim_draft_attested_guarded(&draft, |_, _| Ok("receipt".into()), &refuse).is_err()); + assert!(!dir.path().join("receipt@v2.json").exists()); + assert_eq!(snaps.current().unwrap(), Some(1)); +} + #[test] fn a_saved_draft_is_bound_to_its_editor_and_exact_nonce() { let dir = tempfile::tempdir().unwrap(); From 3253959980f69020437f3e01f74504ae7bcdaf71 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 03:38:33 +0800 Subject: [PATCH 131/155] Read exchange test responses by Content-Length The exchange integration client accepts a complete framed response before a later TCP reset and refuses truncated, ambiguous or oversized frames. Server close and admission semantics remain unchanged. --- .../contextful-cli/tests/integration/serve.rs | 88 +++++++++++++++++-- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 19b93f03f..007fb1d2b 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -16,6 +16,85 @@ use std::sync::atomic::{AtomicU64, Ordering}; const AUD: &str = "contextful://acme-research"; +/// One bounded Content-Length response ends at its body, so a later TCP reset has +/// no bearing on the HTTP answer; a reset or EOF inside the frame remains an error. +fn exchange_response(stream: &mut impl Read) -> std::io::Result { + let invalid = |message| std::io::Error::new(std::io::ErrorKind::InvalidData, message); + let mut raw = Vec::new(); + while !raw.ends_with(b"\r\n\r\n") { + if raw.len() == 8192 { return Err(invalid("the test response head exceeds 8 KiB")); } + let mut byte = [0]; + stream.read_exact(&mut byte)?; + raw.push(byte[0]); + } + let head = std::str::from_utf8(&raw).map_err(|_| invalid("the response head is not UTF-8"))?; + let mut lines = head.split("\r\n"); + let mut status = lines.next().unwrap_or_default().split_whitespace(); + if status.next() != Some("HTTP/1.1") || + !status.next().is_some_and(|code| code.len() == 3 && code.bytes().all(|b| b.is_ascii_digit())) { + return Err(invalid("the response status does not parse")); + } + let mut length = None; + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line.split_once(':').ok_or_else(|| invalid("the response header does not parse"))?; + if name.eq_ignore_ascii_case("Transfer-Encoding") { + return Err(invalid("the test response requires Content-Length framing")); + } + if name.eq_ignore_ascii_case("Content-Length") { + let value = value.trim(); + if length.is_some() || value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err(invalid("the response length is ambiguous or malformed")); + } + length = Some(value.parse::().map_err(|_| invalid("the response length exceeds usize"))?); + } + } + let length = length.ok_or_else(|| invalid("the response has no Content-Length"))?; + if length > 1024 * 1024 { return Err(invalid("the test response body exceeds 1 MiB")); } + let start = raw.len(); + raw.resize(start + length, 0); + stream.read_exact(&mut raw[start..])?; + String::from_utf8(raw).map_err(|_| invalid("the response is not UTF-8")) +} + +struct ResetAfter(std::io::Cursor>); + +impl Read for ResetAfter { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self.0.read(out)? { + 0 => Err(std::io::ErrorKind::ConnectionReset.into()), + n => Ok(n), + } + } +} + +#[test] +fn exchange_response_reads_one_complete_frame_before_a_later_reset() { + let wire = b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n\r\n{}"; + assert_eq!(exchange_response(&mut &wire[..]).unwrap(), std::str::from_utf8(wire).unwrap()); + assert_eq!(exchange_response(&mut ResetAfter(std::io::Cursor::new(wire.to_vec()))).unwrap(), std::str::from_utf8(wire).unwrap()); +} + +#[test] +fn exchange_response_refuses_truncated_headers_and_bodies() { + for wire in [b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n".as_slice(), + b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n\r\n{".as_slice()] { + assert!(exchange_response(&mut &wire[..]).is_err(), "EOF inside a frame refuses"); + assert!(exchange_response(&mut ResetAfter(std::io::Cursor::new(wire.to_vec()))).is_err(), "a reset inside a frame refuses"); + } +} + +#[test] +fn exchange_response_refuses_ambiguous_or_unbounded_framing() { + for head in ["not-http\r\nContent-Length: 0", "HTTP/1.1 404 Not Found", + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nContent-Length: 0", + "HTTP/1.1 404 Not Found\r\nContent-Length: -1", + "HTTP/1.1 404 Not Found\r\nContent-Length: 1048577", + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nTransfer-Encoding: chunked"] { + assert!(exchange_response(&mut format!("{head}\r\n\r\n").as_bytes()).is_err(), "{head}"); + } + assert!(exchange_response(&mut vec![b'x'; 8193].as_slice()).is_err()); +} + fn run(dir: &Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_contextful")) .args(args) @@ -452,8 +531,7 @@ fn served_exchange_without_policy_returns_the_unconfigured_refusal() { let body = "not-json"; let mut stream = TcpStream::connect(&addr).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).unwrap(); + let raw = exchange_response(&mut stream).unwrap(); let (head, data) = raw.split_once("\r\n\r\n").unwrap(); assert_eq!(head.split(' ').nth(1), Some("404")); let answer: Value = serde_json::from_str(data).unwrap(); @@ -461,8 +539,7 @@ fn served_exchange_without_policy_returns_the_unconfigured_refusal() { let mut stream = TcpStream::connect(&addr).unwrap(); stream.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n").unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).expect("unconfigured exchange answers before waiting for its body"); + let raw = exchange_response(&mut stream).expect("unconfigured exchange answers before waiting for its body"); assert!(raw.starts_with("HTTP/1.1 404"), "{raw}"); } @@ -487,8 +564,7 @@ fn served_exchange_mints_a_reader_credential_for_the_read_face() { let body = json!({ "jwt": jwt }).to_string(); let mut stream = TcpStream::connect(&addr).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).unwrap(); + let raw = exchange_response(&mut stream).unwrap(); let (head, data) = raw.split_once("\r\n\r\n").unwrap(); (head.split(' ').nth(1).unwrap().parse::().unwrap(), serde_json::from_str::(data).unwrap()) }; From d312c6f4f3ce85aa032a34e285ab46ecbe1397e7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 05:29:53 +0800 Subject: [PATCH 132/155] CI fixture repositories carry a local Git identity --- tools/ci/tests/integration/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/ci/tests/integration/main.rs b/tools/ci/tests/integration/main.rs index c6c6697e0..b07f4f6fe 100644 --- a/tools/ci/tests/integration/main.rs +++ b/tools/ci/tests/integration/main.rs @@ -41,6 +41,9 @@ impl Repo { let root = dir.path().to_path_buf(); let r = Repo { _dir: dir, root }; r.git(&["init", "-q", "-b", "main"]); + r.git(&["config", "--local", "user.name", "t"]); + r.git(&["config", "--local", "user.email", "t@example.com"]); + r.git(&["config", "--local", "commit.gpgsign", "false"]); r.write("Cargo.toml", "[workspace]\nresolver = \"2\"\nmembers = [\"crates/*\"]\n"); r.write("crates/demo/Cargo.toml", &manifest("demo", "")); r.write("crates/demo/src/lib.rs", "pub fn double(x: i32) -> i32 {\n x * 2\n}\n"); @@ -65,7 +68,6 @@ impl Repo { pub fn git(&self, args: &[&str]) -> String { let out = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.name=t", "-c", "user.email=t@example.com"]) .args(args) .current_dir(&self.root) .output() From 85a2095059133999da103389832a64d2b6d22475 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 06:55:04 +0800 Subject: [PATCH 133/155] Console permits the pinned esbuild install script --- apps/console/pnpm-workspace.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 apps/console/pnpm-workspace.yaml diff --git a/apps/console/pnpm-workspace.yaml b/apps/console/pnpm-workspace.yaml new file mode 100644 index 000000000..2f24ed578 --- /dev/null +++ b/apps/console/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +packages: + - . + +allowBuilds: + esbuild@0.25.12: true From c0c14c16050701b446112a245bbaeb8dfdf4acb8 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 04:22:59 +0800 Subject: [PATCH 134/155] Evaluation requires formal tools and read fixtures bound planning --- .../tests/integration/read/respond.rs | 5 +-- tools/ci/src/main.rs | 6 +++- tools/ci/tests/integration/stages.rs | 35 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/contextful-context/tests/integration/read/respond.rs b/crates/contextful-context/tests/integration/read/respond.rs index cf117ea0c..20c3a1552 100644 --- a/crates/contextful-context/tests/integration/read/respond.rs +++ b/crates/contextful-context/tests/integration/read/respond.rs @@ -14,8 +14,9 @@ fn restriction(r: &Response) -> &Value { } fn expensive_statement() -> String { - let tables = (0..14).map(|i| format!("\"research/notes\" t{i}")).collect::>().join(", "); - format!("SELECT sum(random()) FROM {tables}") + // Four materialized arms execute 96^4 random evaluations under the statement deadline. + let rows = (0..32).map(|_| "SELECT note_id FROM \"research/notes\"").collect::>().join(" UNION ALL "); + format!("WITH work AS MATERIALIZED ({rows}) SELECT sum(random()) FROM work t0, work t1, work t2, work t3") } // spec: read.respond.duration-ceiling@cfa152d7 diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index abd3fe81b..2baf5175a 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -478,7 +478,11 @@ fn run_stage(root: &Path, stage: &str, only: Option<&[String]>, base: &str, boun workspace(root, only)? } "acceptance" => acceptance(root)?, - "evaluate" => measure::evaluate(root)?, + "evaluate" => { + provision_lean(root)?; + provision_wasm(root)?; + measure::evaluate(root)? + } "features" => features(root, only)?, "crate-graph" => { committed_lock(root)?; diff --git a/tools/ci/tests/integration/stages.rs b/tools/ci/tests/integration/stages.rs index b6b87b466..812a7bc72 100644 --- a/tools/ci/tests/integration/stages.rs +++ b/tools/ci/tests/integration/stages.rs @@ -289,6 +289,41 @@ fn declared_derived_artifact_is_checked_by_schema_stage() { assert!(err.contains("schema: spec/custom.md differs from its regeneration"), "{err}"); } +#[test] +fn the_evaluate_stage_provisions_required_lean_and_wasm_before_measuring() { + const PIN: &str = "leanprover/lean4:fixture"; + const MEASURED: &str = r###" +#[test] +fn required_tools() { + assert_eq!(std::env::var("CONTEXTFUL_REQUIRE_LEAN").as_deref(), Ok("1")); + assert_eq!(std::env::var("CONTEXTFUL_REQUIRE_WASM").as_deref(), Ok("1")); + assert_eq!(std::env::var("ELAN_TOOLCHAIN").as_deref(), Ok("leanprover/lean4:fixture")); + let lean = std::process::Command::new("lean").arg("--version").output().unwrap(); + assert!(lean.status.success()); + assert_eq!(String::from_utf8(lean.stdout).unwrap().trim(), "fixture Lean"); + let dir = std::path::PathBuf::from(std::env::var_os("CONTEXTFUL_MEASURE_DIR").unwrap()); + std::fs::write(dir.join("required-tools.json"), r##"{"id":"required-tools","value":1,"n":1,"seed":7,"run":{"processor":"t","nproc":1,"memory_limit":null}}"##).unwrap(); +} +"###; + let r = Repo::init(); + r.write("formal/lean-toolchain", &format!("{PIN}\n")); + r.write("spec/spec.lock.json", "{\"clauses\":[{\"id\":\"run.journal.entry-key\"}]}\n"); + r.write("evals/ledger.toml", "[entry.required-tools]\nclause = \"run.journal.entry-key\"\nmetric = \"tools.ready\"\nkind = \"test\"\ntier = \"gate\"\nmethod = { test = \"demo::measured::required_tools\" }\ntarget = { op = \">=\", value = 1 }\nseed = 7\n"); + r.write("crates/demo/tests/integration/main.rs", "mod double;\nmod measured;\n"); + r.write("crates/demo/tests/integration/measured.rs", MEASURED); + r.commit("a measurement requiring Lean and WebAssembly"); + let bin = Bin::new(); + bin.fake("elan", "echo \"elan $*\" >> \"$CALLS\"\n[ \"$*\" = 'toolchain list' ] || exit 1\necho leanprover/lean4:fixture\n"); + bin.fake("lean", "[ \"$ELAN_TOOLCHAIN\" = leanprover/lean4:fixture ] || exit 1\necho 'fixture Lean'\n"); + let elan = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(bin.dir.path(), elan.path().join("bin")).unwrap(); + let o = gate_env(&r, Some(&bin), &[("ELAN_HOME", elan.path().to_str().unwrap())], &["--stage", "evaluate"]); + let err = stderr(&o); + assert!(o.status.success(), "{err}"); + assert!(err.contains("measure: required-tools = 1"), "{err}"); + assert_eq!(bin.calls(), ["elan toolchain list"]); +} + /// The evaluate stage runs every gate-tier ledger entry and the native case set in the deterministic tier, and reports the floor and baseline verdicts. // spec: assurance.gate.evaluate-stage@316d20b1 #[test] From 452cead57ead4fc27c4eab1a0cdf438252e4a74d Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 04:37:42 +0800 Subject: [PATCH 135/155] Extract the connection deadline watcher Test-First: refactor --- crates/contextful-context/src/read/deadline.rs | 10 ++++++++++ crates/contextful-context/src/read/engine.rs | 4 +--- crates/contextful-context/src/read/mod.rs | 1 + 3 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 crates/contextful-context/src/read/deadline.rs diff --git a/crates/contextful-context/src/read/deadline.rs b/crates/contextful-context/src/read/deadline.rs new file mode 100644 index 000000000..fd798d4bc --- /dev/null +++ b/crates/contextful-context/src/read/deadline.rs @@ -0,0 +1,10 @@ +//! The connection's deadline watcher and completion acknowledgement. + +use std::sync::mpsc::{Receiver, RecvTimeoutError}; +use std::time::{Duration, Instant}; + +pub(crate) fn watch(_started: Instant, duration: Duration, receiver: Receiver<()>, mut interrupt: impl FnMut()) { + if receiver.recv_timeout(duration).is_err_and(|e| e == RecvTimeoutError::Timeout) { + interrupt(); + } +} diff --git a/crates/contextful-context/src/read/engine.rs b/crates/contextful-context/src/read/engine.rs index 5daaa209c..de80408c1 100644 --- a/crates/contextful-context/src/read/engine.rs +++ b/crates/contextful-context/src/read/engine.rs @@ -331,9 +331,7 @@ impl SqlEngine { let duration = std::time::Duration::from_millis(milliseconds); let started = std::time::Instant::now(); let watcher = std::thread::spawn(move || { - if receiver.recv_timeout(duration).is_err_and(|e| e == std::sync::mpsc::RecvTimeoutError::Timeout) { - handle.interrupt(); - } + super::deadline::watch(started, duration, receiver, || handle.interrupt()); }); let result = run(self); let elapsed = started.elapsed().as_millis() as u64; diff --git a/crates/contextful-context/src/read/mod.rs b/crates/contextful-context/src/read/mod.rs index 7daf51de7..891be5fb8 100644 --- a/crates/contextful-context/src/read/mod.rs +++ b/crates/contextful-context/src/read/mod.rs @@ -2,6 +2,7 @@ //! and templates, descriptions and file previews, and ranked retrieval. pub mod audit; +mod deadline; pub(crate) mod engine; pub mod evidence; pub mod face; From d368780740f1621de00684a57555e9f8548b0760 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 04:43:56 +0800 Subject: [PATCH 136/155] Keep read cancellation active until statement completion DuckDB preparation clears its interrupt flag. The connection watcher reasserts cancellation after the absolute deadline until completion acknowledgement or disconnection, and joins before connection reuse. --- .../contextful-context/src/read/deadline.rs | 19 ++++- .../tests/integration/main.rs | 2 + .../tests/integration/read_deadline.rs | 72 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 crates/contextful-context/tests/integration/read_deadline.rs diff --git a/crates/contextful-context/src/read/deadline.rs b/crates/contextful-context/src/read/deadline.rs index fd798d4bc..35009e603 100644 --- a/crates/contextful-context/src/read/deadline.rs +++ b/crates/contextful-context/src/read/deadline.rs @@ -3,8 +3,21 @@ use std::sync::mpsc::{Receiver, RecvTimeoutError}; use std::time::{Duration, Instant}; -pub(crate) fn watch(_started: Instant, duration: Duration, receiver: Receiver<()>, mut interrupt: impl FnMut()) { - if receiver.recv_timeout(duration).is_err_and(|e| e == RecvTimeoutError::Timeout) { - interrupt(); +// DuckDB preparation resets its interrupt flag; cancellation persists until completion. +const INTERRUPT_RETRY: Duration = Duration::from_millis(1); + +pub(crate) fn watch( + started: Instant, + duration: Duration, + receiver: Receiver<()>, + mut interrupt: impl FnMut(), +) { + let mut wait = duration.saturating_sub(started.elapsed()); + loop { + match receiver.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => return, + Err(RecvTimeoutError::Timeout) => interrupt(), + } + wait = INTERRUPT_RETRY; } } diff --git a/crates/contextful-context/tests/integration/main.rs b/crates/contextful-context/tests/integration/main.rs index d19cddd36..b11009edf 100644 --- a/crates/contextful-context/tests/integration/main.rs +++ b/crates/contextful-context/tests/integration/main.rs @@ -13,6 +13,8 @@ mod init; mod lay_out; mod ledger; mod read; +#[cfg(feature = "read")] +mod read_deadline; mod reconcile; mod retention; mod reserve; diff --git a/crates/contextful-context/tests/integration/read_deadline.rs b/crates/contextful-context/tests/integration/read_deadline.rs new file mode 100644 index 000000000..d2d716e9a --- /dev/null +++ b/crates/contextful-context/tests/integration/read_deadline.rs @@ -0,0 +1,72 @@ +//! Cancellation persists across preparation's interrupt reset and stops before reuse. + +#[path = "../../src/read/deadline.rs"] +mod deadline; + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{mpsc, Arc}; +use std::time::{Duration, Instant}; + +fn reset_then_stop(acknowledge: bool) { + let interrupted = Arc::new(AtomicBool::new(false)); + let (cancel, receiver) = mpsc::channel(); + let (fired, observed) = mpsc::channel(); + let (resume, barrier) = mpsc::channel(); + let flag = interrupted.clone(); + let watcher = std::thread::spawn(move || { + let mut first = true; + deadline::watch(Instant::now(), Duration::ZERO, receiver, || { + flag.store(true, Ordering::SeqCst); + fired.send(()).unwrap(); + if first { + first = false; + barrier.recv().unwrap(); + } + }); + }); + observed + .recv_timeout(Duration::from_secs(5)) + .expect("the deadline interrupts"); + interrupted.store(false, Ordering::SeqCst); + resume.send(()).unwrap(); + let reasserted = observed.recv_timeout(Duration::from_secs(5)); + if acknowledge { + let _ = cancel.send(()); + } + drop(cancel); + watcher.join().unwrap(); + assert!( + reasserted.is_ok(), + "preparation erased cancellation: {reasserted:?}" + ); + assert!(interrupted.load(Ordering::SeqCst)); + interrupted.store(false, Ordering::SeqCst); + assert!( + !interrupted.load(Ordering::SeqCst), + "the joined watcher cannot interrupt the next statement" + ); +} + +#[test] +fn cancellation_survives_a_preparation_reset_until_the_statement_acknowledges() { + reset_then_stop(true); +} + +#[test] +fn a_disconnected_running_statement_stops_cancellation_before_connection_reuse() { + reset_then_stop(false); +} + +#[test] +fn a_completed_or_disconnected_statement_stops_the_watcher_without_an_interrupt() { + for acknowledge in [true, false] { + let (cancel, receiver) = mpsc::channel(); + if acknowledge { + cancel.send(()).unwrap(); + } + drop(cancel); + let mut interrupts = 0; + deadline::watch(Instant::now(), Duration::ZERO, receiver, || interrupts += 1); + assert_eq!(interrupts, 0, "completion acknowledgement={acknowledge}"); + } +} From 0436878b4a6325b245104c239fdcee1f3687d3d3 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 04:30:13 +0800 Subject: [PATCH 137/155] CI requires pinned toolchains for evaluate and formal features --- tools/ci/src/main.rs | 13 +- tools/ci/tests/integration/main.rs | 1 + .../tests/integration/mandatory_toolchains.rs | 132 ++++++++++++++++++ 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 tools/ci/tests/integration/mandatory_toolchains.rs diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index abd3fe81b..b620a5f1c 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -478,7 +478,11 @@ fn run_stage(root: &Path, stage: &str, only: Option<&[String]>, base: &str, boun workspace(root, only)? } "acceptance" => acceptance(root)?, - "evaluate" => measure::evaluate(root)?, + "evaluate" => { + provision_lean(root)?; + provision_wasm(root)?; + measure::evaluate(root)? + } "features" => features(root, only)?, "crate-graph" => { committed_lock(root)?; @@ -658,6 +662,13 @@ fn cli_partition(args: &mut Vec<&str>, ordinary: bool, formal: bool) { /// (`assurance.build.target-dir-per-stage`). fn features(root: &Path, only: Option<&[String]>) -> Result<()> { let featured = featured_packages(root)?; + let formal = featured.iter().filter(|package| package.name == topology::BINARY).any(|package| { + package.runs().iter().any(|(label, _)| only.is_none_or(|parts| parts.iter().any(|part| part.strip_prefix("formal-") == Some(label.as_str())))) + }); + if formal { + provision_lean(root)?; + provision_wasm(root)?; + } if featured.is_empty() { eprintln!("features: no workspace package declares a feature"); } diff --git a/tools/ci/tests/integration/main.rs b/tools/ci/tests/integration/main.rs index b07f4f6fe..e0ae223f2 100644 --- a/tools/ci/tests/integration/main.rs +++ b/tools/ci/tests/integration/main.rs @@ -15,6 +15,7 @@ mod image; mod lean; mod measure; mod mirrors; +mod mandatory_toolchains; mod probe; mod release; mod secrets; diff --git a/tools/ci/tests/integration/mandatory_toolchains.rs b/tools/ci/tests/integration/mandatory_toolchains.rs new file mode 100644 index 000000000..2ef10c5df --- /dev/null +++ b/tools/ci/tests/integration/mandatory_toolchains.rs @@ -0,0 +1,132 @@ +use crate::{manifest, stderr, Repo}; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::process::Command; + +const PIN: &str = "leanprover/lean4:v4.29.1"; + +fn executable(path: &Path, script: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, format!("#!/bin/sh\nset -eu\n{script}")).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); +} + +/// The child command consumes a pinned toolchain and a provisioned target, rather than +/// inheriting either from the host. Mock executables keep provisioning deterministic. +fn run(part: &str, required: bool) { + run_with_package(part, required, Some("contextful-cli")); +} + +fn run_with_package(part: &str, required: bool, package: Option<&str>) { + let r = Repo::init(); + let tools = tempfile::tempdir().unwrap(); + let elan = tools.path().join("elan"); + let bin = tools.path().join("bin"); + let wasm = tools.path().join("wasm-lib"); + r.write("formal/lean-toolchain", &format!("{PIN}\n")); + if let Some(package) = package { + r.write(&format!("crates/{package}/Cargo.toml"), &manifest(package, "")); + r.write(&format!("crates/{package}/tests/integration/main.rs"), "mod measured;\n"); + r.write(&format!("crates/{package}/tests/integration/measured.rs"), "#[test]\nfn emits() {}\n"); + } + r.write("spec/spec.lock.json", "{\"clauses\":[{\"id\":\"assurance.measure.record\"}]}\n"); + r.write("evals/ledger.toml", "[entry.mandatory-proof]\nclause = \"assurance.measure.record\"\nmetric = \"proof.value\"\nkind = \"test\"\ntier = \"gate\"\nmethod = { test = \"contextful_cli::measured::emits\" }\ntarget = { op = \"==\", value = 0 }\n"); + let packages: Vec<_> = package.into_iter().map(|name| serde_json::json!({ + "name": name, "manifest_path": r.root.join(format!("crates/{name}/Cargo.toml")), + "features": {"data-plane": [], "contextful-edge": []}, + "metadata": {"contextful": {"feature-runs": ["data-plane", "contextful-edge"]}} + })).collect(); + let meta = serde_json::json!({"packages": packages}); + r.write("metadata.json", &meta.to_string()); + r.commit("a method requiring pinned Lean and WebAssembly"); + executable(&elan.join("bin/elan"), r#" +case "$1 $2" in + 'toolchain list') if test -f "$ELAN_HOME/installed"; then cat "$ELAN_HOME/installed"; fi ;; + 'toolchain install') test "$3" = "$EXPECTED_PIN"; printf '%s\n' "$3" > "$ELAN_HOME/installed" ;; + *) exit 91 ;; +esac +"#); + executable(&elan.join("bin/lean"), r#" +test "$ELAN_TOOLCHAIN" = "$EXPECTED_PIN" +test "$(cat "$ELAN_HOME/installed")" = "$EXPECTED_PIN" +printf '%s\n' "$ELAN_TOOLCHAIN" +"#); + executable(&bin.join("rustc"), "printf '%s\\n' \"$WASM_LIB\"\n"); + executable(&bin.join("rustup"), r#" +test "$*" = 'target add wasm32-unknown-unknown' +mkdir -p "$WASM_LIB" +touch "$WASM_LIB/libstd.rlib" +"#); + executable(&bin.join("cargo"), r#" +if test "$1" = metadata; then cat metadata.json; exit 0; fi +test "$1" = test +if test "$EXPECT_REQUIRED" = 1; then + test "${CONTEXTFUL_REQUIRE_LEAN:-}" = 1 || { echo 'mandatory Lean flag absent' >&2; exit 92; } + test "${CONTEXTFUL_REQUIRE_WASM:-}" = 1 || { echo 'mandatory Wasm flag absent' >&2; exit 93; } + test "$(lean --version)" = "$EXPECTED_PIN" + test -f "$(rustc --print target-libdir --target wasm32-unknown-unknown)/libstd.rlib" +else + test -z "${CONTEXTFUL_REQUIRE_LEAN:-}" + test -z "${CONTEXTFUL_REQUIRE_WASM:-}" +fi +printf '%s\n' "$*" >> child-commands +if test -n "${CONTEXTFUL_MEASURE_DIR:-}"; then + mkdir -p "$CONTEXTFUL_MEASURE_DIR" + printf '%s\n' '{"id":"mandatory-proof","value":0,"n":1,"seed":7,"run":{"processor":"fixture","nproc":1,"memory_limit":null}}' > "$CONTEXTFUL_MEASURE_DIR/mandatory-proof.json" +fi +"#); + let host_path = std::env::var("PATH").unwrap(); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&host_path))).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_contextful-ci")) + .args(["gate", "--stage", part]) + .current_dir(&r.root) + .env("PATH", path) + .env("ELAN_HOME", &elan) + .env("WASM_LIB", &wasm) + .env("EXPECTED_PIN", PIN) + .env("EXPECT_REQUIRED", if required { "1" } else { "0" }) + .env_remove("ELAN_TOOLCHAIN") + .env_remove("CONTEXTFUL_REQUIRE_LEAN") + .env_remove("CONTEXTFUL_REQUIRE_WASM") + .env_remove("CARGO_TARGET_DIR") + .output().unwrap(); + assert!(output.status.success(), "{part}: {}", stderr(&output)); + let commands = std::fs::read_to_string(r.root.join("child-commands")).unwrap_or_default(); + assert_eq!(!commands.is_empty(), package.is_some(), "{part} selected child tests"); + assert_eq!(elan.join("installed").exists(), required, "{part} toolchain provisioning"); + assert_eq!(wasm.join("libstd.rlib").exists(), required, "{part} target provisioning"); + if part == "evaluate" { + assert!(stderr(&output).contains("measure: mandatory-proof = 0"), "{}", stderr(&output)); + } +} + +#[test] +fn evaluate_requires_pinned_lean_and_wasm_before_collecting_a_record() { run("evaluate", true); } + +#[test] +fn formal_none_requires_pinned_lean_and_wasm() { run("features.formal-none", true); } + +#[test] +fn formal_all_requires_pinned_lean_and_wasm() { run("features.formal-all", true); } + +#[test] +fn formal_data_plane_requires_pinned_lean_and_wasm() { run("features.formal-data-plane", true); } + +#[test] +fn formal_contextful_edge_requires_pinned_lean_and_wasm() { run("features.formal-contextful-edge", true); } + +#[test] +fn whole_features_requires_pinned_lean_and_wasm() { run("features", true); } + +#[test] +fn binary_only_features_preserve_the_narrow_toolchain_cost() { run("features.binary-none", false); } + +#[test] +fn whole_features_without_a_binary_preserve_the_narrow_toolchain_cost() { + run_with_package("features", false, Some("demo")); +} + +#[test] +fn whole_features_without_featured_packages_provision_no_toolchains() { + run_with_package("features", false, None); +} From d46c0a5ba875c6393612e19059d426f85ef33733 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 05:37:25 +0800 Subject: [PATCH 138/155] Specify Windows command descendant lifetimes --- .../tests/fixtures/command_tree.rs | 30 ++++ .../tests/integration/command_windows.rs | 147 ++++++++++++++++++ .../tests/integration/main.rs | 3 + 3 files changed, 180 insertions(+) create mode 100644 crates/contextful-engine/tests/fixtures/command_tree.rs create mode 100644 crates/contextful-engine/tests/integration/command_windows.rs diff --git a/crates/contextful-engine/tests/fixtures/command_tree.rs b/crates/contextful-engine/tests/fixtures/command_tree.rs new file mode 100644 index 000000000..cfe1e9043 --- /dev/null +++ b/crates/contextful-engine/tests/fixtures/command_tree.rs @@ -0,0 +1,30 @@ +//! A finite command-source process tree for native integration tests. +use std::io::Write; +use std::time::{Duration, Instant}; + +fn main() { + let mode = std::env::args().nth(1).expect("fixture mode"); + if mode == "grandchild" { + std::fs::write("grandchild.pid", std::process::id().to_string()).unwrap(); + std::thread::sleep(Duration::from_secs(30)); + return; + } + assert!(matches!(mode.as_str(), "cancel" | "finished")); + let _child = std::process::Command::new(std::env::current_exe().unwrap()) + .arg("grandchild") + .stdin(std::process::Stdio::null()) + .spawn() + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while !std::path::Path::new("grandchild.pid").exists() { + assert!(Instant::now() < deadline, "grandchild readiness"); + std::thread::sleep(Duration::from_millis(5)); + } + std::fs::write("parent.ready", "ready").unwrap(); + if mode == "finished" { + print!("{{\"rows\":[]}}"); + std::io::stdout().flush().unwrap(); + } else { + std::thread::sleep(Duration::from_secs(30)); + } +} diff --git a/crates/contextful-engine/tests/integration/command_windows.rs b/crates/contextful-engine/tests/integration/command_windows.rs new file mode 100644 index 000000000..739e2fe9d --- /dev/null +++ b/crates/contextful-engine/tests/integration/command_windows.rs @@ -0,0 +1,147 @@ +//! Native Windows process-tree lifetime, verified through retained handles. +use contextful_core::run::ports::{PullRequest, Source}; +use contextful_core::run::FailureTag; +use contextful_engine::{cancel::CancelToken, command::CommandSource}; +use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; + +#[link(name = "kernel32")] +unsafe extern "system" { + fn OpenProcess(access: u32, inherit: i32, pid: u32) -> *mut std::ffi::c_void; + fn WaitForSingleObject(handle: *mut std::ffi::c_void, millis: u32) -> u32; + fn TerminateProcess(handle: *mut std::ffi::c_void, code: u32) -> i32; +} + +struct Descendant(OwnedHandle); +impl Descendant { + fn open(pid: u32) -> Self { + // SAFETY: OpenProcess receives plain integers and returns a newly owned + // handle. The fixture remains alive until this test terminates it. + let raw = unsafe { OpenProcess(0x0010_0000 | 0x0001, 0, pid) }; + assert!( + !raw.is_null(), + "open descendant: {}", + std::io::Error::last_os_error() + ); + // SAFETY: successful OpenProcess returns an exclusively owned live handle. + Self(unsafe { OwnedHandle::from_raw_handle(raw) }) + } + fn exited(&self) -> bool { + // SAFETY: the borrowed handle remains owned throughout the wait. + match unsafe { WaitForSingleObject(self.0.as_raw_handle(), 0) } { + 0 => true, + 258 => false, + other => panic!( + "descendant wait {other}: {}", + std::io::Error::last_os_error() + ), + } + } +} +impl Drop for Descendant { + fn drop(&mut self) { + // SAFETY: the retained fixture handle has terminate/synchronize rights; + // no PID lookup or unrelated process is involved in cleanup. + unsafe { + TerminateProcess(self.0.as_raw_handle(), 1); + WaitForSingleObject(self.0.as_raw_handle(), 5_000); + } + } +} + +fn fixture() -> &'static Path { + static EXE: OnceLock<(tempfile::TempDir, PathBuf)> = OnceLock::new(); + EXE.get_or_init(|| { + let dir = tempfile::tempdir().unwrap(); + let executable = dir.path().join("command-tree-fixture.exe"); + let out = std::process::Command::new("rustc") + .arg("--edition=2021") + .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/command_tree.rs")) + .arg("-o") + .arg(&executable) + .output() + .unwrap(); + assert!( + out.status.success(), + "{}", + String::from_utf8_lossy(&out.stderr) + ); + (dir, executable) + }) + .1 + .as_path() +} + +fn source(dir: &Path, mode: &str) -> CommandSource { + CommandSource { + argv: vec![fixture().to_string_lossy().into_owned(), mode.into()], + cwd: dir.into(), + } +} +fn ready(dir: &Path) -> Descendant { + let deadline = Instant::now() + Duration::from_secs(10); + while !dir.join("parent.ready").exists() { + assert!(Instant::now() < deadline, "parent and descendant readiness"); + std::thread::sleep(Duration::from_millis(5)); + } + let pid = std::fs::read_to_string(dir.join("grandchild.pid")) + .unwrap() + .parse() + .unwrap(); + Descendant::open(pid) +} +fn request() -> PullRequest { + PullRequest { + step_label: "windows-tree".into(), + position: None, + idempotency_key: "windows-tree".into(), + } +} + +// spec: run.cancel.child-reaped@8c92882f +#[test] +fn cancellation_returns_only_after_the_descendant_handle_is_signalled() { + let dir = tempfile::tempdir().unwrap(); + let mut src = source(dir.path(), "cancel"); + let token = CancelToken::default(); + let child_token = token.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + tx.send(src.pull(&request(), &child_token)).unwrap(); + }); + let descendant = ready(dir.path()); + assert!(!descendant.exited(), "fixture alive before cancellation"); + token.fire(); + let result = rx + .recv_timeout(Duration::from_secs(15)) + .expect("bounded cancellation"); + worker.join().unwrap(); + assert_eq!(result.unwrap_err().tag, FailureTag::Canceled); + assert!( + descendant.exited(), + "a cancelled pull leaves a live descendant" + ); +} + +#[test] +fn a_finished_parent_reaps_its_pipe_holding_descendant() { + let dir = tempfile::tempdir().unwrap(); + let mut src = source(dir.path(), "finished"); + let (tx, rx) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + tx.send(src.pull(&request(), &CancelToken::default())) + .unwrap(); + }); + let descendant = ready(dir.path()); + let result = rx + .recv_timeout(Duration::from_secs(15)) + .expect("bounded finished pull"); + worker.join().unwrap(); + assert_eq!(result.unwrap(), br#"{"rows":[]}"#); + assert!( + descendant.exited(), + "a finished pull leaves a live descendant" + ); +} diff --git a/crates/contextful-engine/tests/integration/main.rs b/crates/contextful-engine/tests/integration/main.rs index 27673e6f0..7765f41de 100644 --- a/crates/contextful-engine/tests/integration/main.rs +++ b/crates/contextful-engine/tests/integration/main.rs @@ -1,7 +1,10 @@ //! The engine's one integration binary, one module per operation. mod cancel; +#[cfg(unix)] mod command; +#[cfg(windows)] +mod command_windows; mod control; mod coordinate; mod drive; From f1131d7f65c726e906787129cac729927f28420b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 05:38:53 +0800 Subject: [PATCH 139/155] Replay Windows command lifetime tests against the unchanged engine --- .github/workflows/windows.yml | 24 +++++++++++++++++++ .../tests/integration/command_windows.rs | 7 ++++++ 2 files changed, 31 insertions(+) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 53a4b752f..53cab2b4a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -30,3 +30,27 @@ jobs: - run: cargo test --locked --target ${{ matrix.target }} -p contextful-policy --test integration an_audit_append_syncs_its_directories - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_ledger_append_syncs_its_directory - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it + + command-tree-red: + if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} + strategy: + fail-fast: false + matrix: + include: + - runner: windows-2025 + target: x86_64-pc-windows-msvc + - runner: windows-11-arm + target: aarch64-pc-windows-msvc + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + ref: e6c777fd07f74b85ec20f4a2247c8dc58192a580 + fetch-depth: 0 + - run: rustup target add ${{ matrix.target }} + - run: git fetch --no-tags origin ${{ github.event.pull_request.head.sha }} + - run: git restore --source=${{ github.event.pull_request.head.sha }} -- crates/contextful-engine/tests/integration/main.rs crates/contextful-engine/tests/integration/command_windows.rs crates/contextful-engine/tests/fixtures/command_tree.rs + - run: git diff --exit-code e6c777fd07f74b85ec20f4a2247c8dc58192a580 -- crates/contextful-engine/src + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration --no-run + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --list + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --test-threads=2 diff --git a/crates/contextful-engine/tests/integration/command_windows.rs b/crates/contextful-engine/tests/integration/command_windows.rs index 739e2fe9d..6fa712cf6 100644 --- a/crates/contextful-engine/tests/integration/command_windows.rs +++ b/crates/contextful-engine/tests/integration/command_windows.rs @@ -56,8 +56,15 @@ fn fixture() -> &'static Path { EXE.get_or_init(|| { let dir = tempfile::tempdir().unwrap(); let executable = dir.path().join("command-tree-fixture.exe"); + let abi = if cfg!(target_env = "gnu") { + "gnu" + } else { + "msvc" + }; + let target = format!("{}-pc-windows-{abi}", std::env::consts::ARCH); let out = std::process::Command::new("rustc") .arg("--edition=2021") + .args(["--target", &target]) .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/command_tree.rs")) .arg("-o") .arg(&executable) From 0f705e137b12944a8d125a12e11ceded4ae81dfb Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 05:43:19 +0800 Subject: [PATCH 140/155] Verify Windows fixture identity before descendant cleanup --- .github/workflows/windows.yml | 10 ++- .../tests/fixtures/command_tree.rs | 52 ++++++++++- .../tests/integration/command_windows.rs | 88 +++++++++++++++++-- 3 files changed, 138 insertions(+), 12 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 53cab2b4a..d2977e56d 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -9,6 +9,9 @@ permissions: jobs: windows: + timeout-minutes: 30 + env: + RUSTUP_TOOLCHAIN: 1.97.0 if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} strategy: fail-fast: false @@ -21,7 +24,7 @@ jobs: runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - - run: rustup target add ${{ matrix.target }} + - run: rustup toolchain install 1.97.0 --profile minimal --target ${{ matrix.target }} - run: cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration an_open_directory_handle_syncs_its_entries - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration a_renamed_open_file_does_not_identify_its_replacement @@ -32,6 +35,9 @@ jobs: - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it command-tree-red: + timeout-minutes: 30 + env: + RUSTUP_TOOLCHAIN: 1.97.0 if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} strategy: fail-fast: false @@ -47,7 +53,7 @@ jobs: with: ref: e6c777fd07f74b85ec20f4a2247c8dc58192a580 fetch-depth: 0 - - run: rustup target add ${{ matrix.target }} + - run: rustup toolchain install 1.97.0 --profile minimal --target ${{ matrix.target }} - run: git fetch --no-tags origin ${{ github.event.pull_request.head.sha }} - run: git restore --source=${{ github.event.pull_request.head.sha }} -- crates/contextful-engine/tests/integration/main.rs crates/contextful-engine/tests/integration/command_windows.rs crates/contextful-engine/tests/fixtures/command_tree.rs - run: git diff --exit-code e6c777fd07f74b85ec20f4a2247c8dc58192a580 -- crates/contextful-engine/src diff --git a/crates/contextful-engine/tests/fixtures/command_tree.rs b/crates/contextful-engine/tests/fixtures/command_tree.rs index cfe1e9043..a3b414c3a 100644 --- a/crates/contextful-engine/tests/fixtures/command_tree.rs +++ b/crates/contextful-engine/tests/fixtures/command_tree.rs @@ -2,10 +2,60 @@ use std::io::Write; use std::time::{Duration, Instant}; +#[cfg(windows)] +fn creation_identity() -> u64 { + #[repr(C)] + #[derive(Default)] + struct FileTime { + low: u32, + high: u32, + } + #[link(name = "kernel32")] + unsafe extern "system" { + fn GetCurrentProcess() -> *mut std::ffi::c_void; + fn GetProcessTimes( + process: *mut std::ffi::c_void, + creation: *mut FileTime, + exit: *mut FileTime, + kernel: *mut FileTime, + user: *mut FileTime, + ) -> i32; + } + let (mut creation, mut exit, mut kernel, mut user) = ( + FileTime::default(), + FileTime::default(), + FileTime::default(), + FileTime::default(), + ); + // SAFETY: the pseudo-handle denotes this fixture process and all output + // pointers address initialized FILETIME-layout values for the call. + assert_ne!( + unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }, + 0 + ); + (u64::from(creation.high) << 32) | u64::from(creation.low) +} +#[cfg(not(windows))] +fn creation_identity() -> u64 { + 0 +} + fn main() { let mode = std::env::args().nth(1).expect("fixture mode"); if mode == "grandchild" { - std::fs::write("grandchild.pid", std::process::id().to_string()).unwrap(); + std::fs::write( + "grandchild.pid", + format!("{} {}", std::process::id(), creation_identity()), + ) + .unwrap(); std::thread::sleep(Duration::from_secs(30)); return; } diff --git a/crates/contextful-engine/tests/integration/command_windows.rs b/crates/contextful-engine/tests/integration/command_windows.rs index 6fa712cf6..f78798206 100644 --- a/crates/contextful-engine/tests/integration/command_windows.rs +++ b/crates/contextful-engine/tests/integration/command_windows.rs @@ -2,6 +2,7 @@ use contextful_core::run::ports::{PullRequest, Source}; use contextful_core::run::FailureTag; use contextful_engine::{cancel::CancelToken, command::CommandSource}; +use std::os::windows::ffi::OsStringExt; use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; @@ -12,21 +13,90 @@ unsafe extern "system" { fn OpenProcess(access: u32, inherit: i32, pid: u32) -> *mut std::ffi::c_void; fn WaitForSingleObject(handle: *mut std::ffi::c_void, millis: u32) -> u32; fn TerminateProcess(handle: *mut std::ffi::c_void, code: u32) -> i32; + fn QueryFullProcessImageNameW( + handle: *mut std::ffi::c_void, + flags: u32, + buffer: *mut u16, + size: *mut u32, + ) -> i32; + fn GetProcessTimes( + process: *mut std::ffi::c_void, + creation: *mut FileTime, + exit: *mut FileTime, + kernel: *mut FileTime, + user: *mut FileTime, + ) -> i32; +} + +#[repr(C)] +#[derive(Default)] +struct FileTime { + low: u32, + high: u32, } struct Descendant(OwnedHandle); impl Descendant { - fn open(pid: u32) -> Self { + fn open(pid: u32, creation_identity: u64) -> Self { // SAFETY: OpenProcess receives plain integers and returns a newly owned // handle. The fixture remains alive until this test terminates it. - let raw = unsafe { OpenProcess(0x0010_0000 | 0x0001, 0, pid) }; + let raw = unsafe { OpenProcess(0x0010_0000 | 0x1000 | 0x0001, 0, pid) }; assert!( !raw.is_null(), "open descendant: {}", std::io::Error::last_os_error() ); // SAFETY: successful OpenProcess returns an exclusively owned live handle. - Self(unsafe { OwnedHandle::from_raw_handle(raw) }) + let handle = unsafe { OwnedHandle::from_raw_handle(raw) }; + let (mut creation, mut exit, mut kernel, mut user) = ( + FileTime::default(), + FileTime::default(), + FileTime::default(), + FileTime::default(), + ); + // SAFETY: handle stays owned and each FILETIME-layout output is initialized. + assert_ne!( + unsafe { + GetProcessTimes( + handle.as_raw_handle(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }, + 0, + "fixture creation identity query" + ); + assert_eq!( + (u64::from(creation.high) << 32) | u64::from(creation.low), + creation_identity, + "PID reuse differs from fixture creation identity" + ); + let mut image = vec![0u16; 32_768]; + let mut length = image.len() as u32; + // SAFETY: image holds length writable UTF-16 code units and handle stays owned. + assert_ne!( + unsafe { + QueryFullProcessImageNameW( + handle.as_raw_handle(), + 0, + image.as_mut_ptr(), + &mut length, + ) + }, + 0, + "fixture image identity query" + ); + assert!(length as usize <= image.len()); + let image = PathBuf::from(std::ffi::OsString::from_wide(&image[..length as usize])); + assert_eq!( + std::fs::canonicalize(image).unwrap(), + std::fs::canonicalize(fixture()).unwrap(), + "descendant uses the unique owned fixture executable" + ); + // Cleanup-capable ownership starts only after both identity checks pass. + Self(handle) } fn exited(&self) -> bool { // SAFETY: the borrowed handle remains owned throughout the wait. @@ -93,11 +163,12 @@ fn ready(dir: &Path) -> Descendant { assert!(Instant::now() < deadline, "parent and descendant readiness"); std::thread::sleep(Duration::from_millis(5)); } - let pid = std::fs::read_to_string(dir.join("grandchild.pid")) - .unwrap() - .parse() - .unwrap(); - Descendant::open(pid) + let identity = std::fs::read_to_string(dir.join("grandchild.pid")).unwrap(); + let mut fields = identity.split_whitespace(); + let pid = fields.next().unwrap().parse().unwrap(); + let creation = fields.next().unwrap().parse().unwrap(); + assert!(fields.next().is_none()); + Descendant::open(pid, creation) } fn request() -> PullRequest { PullRequest { @@ -107,7 +178,6 @@ fn request() -> PullRequest { } } -// spec: run.cancel.child-reaped@8c92882f #[test] fn cancellation_returns_only_after_the_descendant_handle_is_signalled() { let dir = tempfile::tempdir().unwrap(); From b933640dcd8a091ec6808b064815f7b24b8eae0b Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 05:45:34 +0800 Subject: [PATCH 141/155] Keep the finished-command fixture descendant alive for reaping proof --- crates/contextful-engine/tests/fixtures/command_tree.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/contextful-engine/tests/fixtures/command_tree.rs b/crates/contextful-engine/tests/fixtures/command_tree.rs index a3b414c3a..9112d61cb 100644 --- a/crates/contextful-engine/tests/fixtures/command_tree.rs +++ b/crates/contextful-engine/tests/fixtures/command_tree.rs @@ -60,6 +60,9 @@ fn main() { return; } assert!(matches!(mode.as_str(), "cancel" | "finished")); + // The finished-parent case deliberately leaves the descendant alive; the + // command source and retained-handle test own its bounded cleanup. + #[allow(clippy::zombie_processes)] let _child = std::process::Command::new(std::env::current_exe().unwrap()) .arg("grandchild") .stdin(std::process::Stdio::null()) From 2cbd30b515a5e616258bbeea78dab7fa3c883c20 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 05:50:27 +0800 Subject: [PATCH 142/155] Quote the native Windows test filters in workflow commands --- .github/workflows/windows.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d2977e56d..599a746cb 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -58,5 +58,5 @@ jobs: - run: git restore --source=${{ github.event.pull_request.head.sha }} -- crates/contextful-engine/tests/integration/main.rs crates/contextful-engine/tests/integration/command_windows.rs crates/contextful-engine/tests/fixtures/command_tree.rs - run: git diff --exit-code e6c777fd07f74b85ec20f4a2247c8dc58192a580 -- crates/contextful-engine/src - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration --no-run - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --list - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --test-threads=2 + - run: "cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --list" + - run: "cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --test-threads=2" From 135979ab56db246d3fc6d40eae28a2ea098aff3a Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:03:32 +0800 Subject: [PATCH 143/155] Require unambiguous owned Windows launch threads --- .../tests/integration/command_admission.rs | 30 +++++++++++++++++++ .../tests/integration/main.rs | 1 + 2 files changed, 31 insertions(+) create mode 100644 crates/contextful-engine/tests/integration/command_admission.rs diff --git a/crates/contextful-engine/tests/integration/command_admission.rs b/crates/contextful-engine/tests/integration/command_admission.rs new file mode 100644 index 000000000..316b8fb05 --- /dev/null +++ b/crates/contextful-engine/tests/integration/command_admission.rs @@ -0,0 +1,30 @@ +#[path = "../../src/command/admission.rs"] +mod admission; + +use admission::{resume_count, sole_thread, thread_owner}; + +#[test] +fn suspended_launch_selects_only_its_single_owned_thread() { + assert_eq!(sole_thread(42, [(7, 11), (9, 42), (12, 13)]).unwrap(), 9); +} + +#[test] +fn suspended_launch_refuses_missing_or_ambiguous_owned_threads() { + assert!(sole_thread(42, [(7, 11)]).is_err()); + assert!(sole_thread(42, [(9, 42), (10, 42)]).is_err()); +} + +#[test] +fn retained_thread_handle_requires_the_childs_ownership() { + thread_owner(42, 42).unwrap(); + assert!(thread_owner(42, 43).is_err()); + assert!(thread_owner(42, 0).is_err()); +} + +#[test] +fn launch_resume_requires_exactly_one_suspend_count() { + resume_count(1).unwrap(); + for count in [0, 2, u32::MAX] { + assert!(resume_count(count).is_err()); + } +} diff --git a/crates/contextful-engine/tests/integration/main.rs b/crates/contextful-engine/tests/integration/main.rs index 7765f41de..54ba1fe3c 100644 --- a/crates/contextful-engine/tests/integration/main.rs +++ b/crates/contextful-engine/tests/integration/main.rs @@ -5,6 +5,7 @@ mod cancel; mod command; #[cfg(windows)] mod command_windows; +mod command_admission; mod control; mod coordinate; mod drive; From 2d5d95bda4c082bf979a60dd5b5da68ce9a96bf7 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:08:15 +0800 Subject: [PATCH 144/155] Contain Windows command descendants before launch --- .github/workflows/windows.yml | 28 +-- Cargo.lock | 1 + crates/contextful-engine/Cargo.toml | 3 + crates/contextful-engine/src/command.rs | 39 +++- .../src/command/admission.rs | 38 ++++ .../contextful-engine/src/command/windows.rs | 184 ++++++++++++++++++ 6 files changed, 259 insertions(+), 34 deletions(-) create mode 100644 crates/contextful-engine/src/command/admission.rs create mode 100644 crates/contextful-engine/src/command/windows.rs diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 599a746cb..53f0bc2de 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -33,30 +33,4 @@ jobs: - run: cargo test --locked --target ${{ matrix.target }} -p contextful-policy --test integration an_audit_append_syncs_its_directories - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_ledger_append_syncs_its_directory - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it - - command-tree-red: - timeout-minutes: 30 - env: - RUSTUP_TOOLCHAIN: 1.97.0 - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - strategy: - fail-fast: false - matrix: - include: - - runner: windows-2025 - target: x86_64-pc-windows-msvc - - runner: windows-11-arm - target: aarch64-pc-windows-msvc - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - with: - ref: e6c777fd07f74b85ec20f4a2247c8dc58192a580 - fetch-depth: 0 - - run: rustup toolchain install 1.97.0 --profile minimal --target ${{ matrix.target }} - - run: git fetch --no-tags origin ${{ github.event.pull_request.head.sha }} - - run: git restore --source=${{ github.event.pull_request.head.sha }} -- crates/contextful-engine/tests/integration/main.rs crates/contextful-engine/tests/integration/command_windows.rs crates/contextful-engine/tests/fixtures/command_tree.rs - - run: git diff --exit-code e6c777fd07f74b85ec20f4a2247c8dc58192a580 -- crates/contextful-engine/src - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration --no-run - - run: "cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --list" - - run: "cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command_windows:: -- --test-threads=2" + - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command -- --test-threads=2 diff --git a/Cargo.lock b/Cargo.lock index f267e499f..2d8bb9aeb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,6 +960,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror 2.0.20", + "windows-sys 0.61.2", ] [[package]] diff --git a/crates/contextful-engine/Cargo.toml b/crates/contextful-engine/Cargo.toml index a93db5a72..0c939972a 100644 --- a/crates/contextful-engine/Cargo.toml +++ b/crates/contextful-engine/Cargo.toml @@ -22,6 +22,9 @@ thiserror = "2" [target.'cfg(unix)'.dependencies] libc = "0.2" +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61.2", features = ["Win32_Foundation", "Win32_Security", "Win32_System_JobObjects", "Win32_System_Diagnostics_ToolHelp", "Win32_System_Threading"] } + [dev-dependencies] # Ledger measures record their figures (`assurance.measure.record`). contextful-eval = { path = "../../tools/eval" } diff --git a/crates/contextful-engine/src/command.rs b/crates/contextful-engine/src/command.rs index 33f46f760..42c58d444 100644 --- a/crates/contextful-engine/src/command.rs +++ b/crates/contextful-engine/src/command.rs @@ -1,5 +1,5 @@ //! A source whose pulls a command serves: one child process per pull, in a process -//! group of its own, waited on under the run's cancellation token. +//! group on Unix or a private job on Windows, waited on under the run's cancellation token. //! //! The child reads its request from the environment — `CONTEXTFUL_CURSOR` holds the //! position as JSON, empty from the start; `CONTEXTFUL_IDEMPOTENCY_KEY` and @@ -19,6 +19,7 @@ use std::time::{Duration, Instant}; /// Interval at which a running child is checked for exit and for a stop. const WAIT_TICK: Duration = Duration::from_millis(10); /// How long a signalled group has to exit on `SIGTERM` before it is killed. +#[cfg(not(windows))] const TERM_GRACE: Duration = Duration::from_millis(500); /// How long a killed group has to disappear before the pull fails instead of waiting on. const KILL_CEILING: Duration = Duration::from_secs(5); @@ -30,6 +31,15 @@ pub struct CommandSource { pub cwd: PathBuf, } +#[cfg(windows)] +mod admission; +#[cfg(windows)] +mod windows; +#[cfg(windows)] +type ProcessTree = windows::Job; +#[cfg(not(windows))] +type ProcessTree = u32; + #[derive(Deserialize)] struct Reported { error: ReportedError, @@ -90,7 +100,7 @@ mod group { } /// Where no process groups exist, the child alone is signalled, through its handle. -#[cfg(not(unix))] +#[cfg(not(any(unix, windows)))] mod group { pub const TERM: i32 = 15; pub const KILL: i32 = 9; @@ -105,7 +115,9 @@ mod group { /// Signal the group to terminate, escalate to a kill past the grace period, and return /// once the child is reaped and no process of the group runs. A group still running /// `KILL_CEILING` after the kill fails the pull. -fn reap_group(child: &mut std::process::Child, pgid: u32, step: &str) -> Result<(), Failure> { +#[cfg(not(windows))] +fn reap_group(child: &mut std::process::Child, tree: &ProcessTree, step: &str) -> Result<(), Failure> { + let pgid = *tree; group::signal(pgid, group::TERM); let started = Instant::now(); let mut reaped = false; @@ -132,6 +144,11 @@ fn reap_group(child: &mut std::process::Child, pgid: u32, step: &str) -> Result< } } +#[cfg(windows)] +fn reap_group(child: &mut std::process::Child, job: &ProcessTree, step: &str) -> Result<(), Failure> { + job.reap(child).map_err(|error| Failure::new(FailureTag::Transient, format!("reaping the job of `{step}`: {error}"))) +} + /// Read a pipe on a thread of its own, delivering each chunk over a channel; the channel /// disconnects at end-of-file. fn drain(pipe: Option) -> Receiver> { @@ -187,24 +204,32 @@ impl Source for CommandSource { .stderr(Stdio::piped()); #[cfg(unix)] std::os::unix::process::CommandExt::process_group(&mut cmd, 0); + #[cfg(windows)] + let (mut child, tree) = windows::Job::launch(&mut cmd).map_err(|e| Failure::new(FailureTag::Config, format!("spawning `{program}` in its job: {e}")))?; + #[cfg(not(windows))] let mut child = cmd.spawn().map_err(|e| Failure::new(FailureTag::Config, format!("spawning `{program}`: {e}")))?; - let pgid = child.id(); + #[cfg(not(windows))] + let tree = child.id(); let out = drain(child.stdout.take()); let err = drain(child.stderr.take()); let status = loop { if cancel.requested() { - reap_group(&mut child, pgid, &request.step_label)?; + reap_group(&mut child, &tree, &request.step_label)?; return Err(Failure::canceled(format!("stopped during `{}`; its process group is reaped", request.step_label))); } match child.try_wait() { Ok(Some(status)) => break status, Ok(None) => std::thread::sleep(WAIT_TICK), - Err(e) => return Err(Failure::new(FailureTag::Transient, format!("waiting on `{program}`: {e}"))), + Err(e) => { + #[cfg(windows)] + reap_group(&mut child, &tree, &request.step_label)?; + return Err(Failure::new(FailureTag::Transient, format!("waiting on `{program}`: {e}"))); + } } }; // A process the child left behind in its group is signalled and reaped too, which // also closes any pipe it held open. - reap_group(&mut child, pgid, &request.step_label)?; + reap_group(&mut child, &tree, &request.step_label)?; let stdout = collect(&out, cancel, &request.step_label, "stdout")?; let stderr = collect(&err, cancel, &request.step_label, "stderr")?; if status.success() { diff --git a/crates/contextful-engine/src/command/admission.rs b/crates/contextful-engine/src/command/admission.rs new file mode 100644 index 000000000..bb2f0df66 --- /dev/null +++ b/crates/contextful-engine/src/command/admission.rs @@ -0,0 +1,38 @@ +//! Fail-closed checks for the sole thread of a suspended command process. + +use std::io; + +pub(super) fn sole_thread( + process: u32, + threads: impl IntoIterator, +) -> io::Result { + let mut owned = threads + .into_iter() + .filter_map(|(thread, owner)| (owner == process).then_some(thread)); + match (owned.next(), owned.next()) { + (Some(thread), None) => Ok(thread), + _ => Err(io::Error::other( + "the suspended command requires exactly one owned thread", + )), + } +} + +pub(super) fn thread_owner(process: u32, owner: u32) -> io::Result<()> { + if owner == process && owner != 0 { + Ok(()) + } else { + Err(io::Error::other( + "the retained launch thread belongs to another process", + )) + } +} + +pub(super) fn resume_count(count: u32) -> io::Result<()> { + if count == 1 { + Ok(()) + } else { + Err(io::Error::other( + "the launch thread requires exactly one suspended count", + )) + } +} diff --git a/crates/contextful-engine/src/command/windows.rs b/crates/contextful-engine/src/command/windows.rs new file mode 100644 index 000000000..46d169753 --- /dev/null +++ b/crates/contextful-engine/src/command/windows.rs @@ -0,0 +1,184 @@ +//! A private, non-inheritable job contains a command before its first instruction. + +use super::admission; +use std::io; +use std::mem::size_of; +use std::os::windows::io::{AsHandle, AsRawHandle, FromRawHandle, OwnedHandle}; +use std::os::windows::process::CommandExt; +use std::process::{Child, Command}; +use std::time::Instant; +use windows_sys::Win32::Foundation::{ERROR_NO_MORE_FILES, INVALID_HANDLE_VALUE}; +use windows_sys::Win32::System::Diagnostics::ToolHelp::{ + CreateToolhelp32Snapshot, Thread32First, Thread32Next, TH32CS_SNAPTHREAD, THREADENTRY32, +}; +use windows_sys::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectBasicAccountingInformation, + JobObjectExtendedLimitInformation, QueryInformationJobObject, SetInformationJobObject, + TerminateJobObject, JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, +}; +use windows_sys::Win32::System::Threading::{ + GetProcessIdOfThread, OpenThread, ResumeThread, CREATE_SUSPENDED, + THREAD_QUERY_LIMITED_INFORMATION, THREAD_SUSPEND_RESUME, +}; + +pub(super) struct Job(OwnedHandle); + +impl Job { + pub(super) fn launch(command: &mut Command) -> io::Result<(Child, Self)> { + // SAFETY: null attributes/name create a private, non-inheritable job. + let raw = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; + if raw.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: CreateJobObjectW returned a fresh owned handle, checked above. + let job = Self(unsafe { OwnedHandle::from_raw_handle(raw) }); + let mut limits = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + // Neither breakaway flag is enabled; descendants remain in this job. + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + // SAFETY: the owned job is live; limits has the required C layout and size. + if unsafe { + SetInformationJobObject( + job.0.as_raw_handle(), + JobObjectExtendedLimitInformation, + (&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + size_of::() as u32, + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // The child's first instruction waits until job admission and thread verification. + let mut child = command.creation_flags(CREATE_SUSPENDED).spawn()?; + if let Err(error) = job.admit_and_resume(&child) { + // Admission fails closed: no uncontained command executes, and the owned + // suspended child is killed and waited on before its handles are released. + // A successful reap proves exit even if kill reports an already-ended + // process. A failed reap preserves its error and the job's kill-on-close. + let _ = child.kill(); + job.reap(&mut child)?; + return Err(error); + } + Ok((child, job)) + } + + fn admit_and_resume(&self, child: &Child) -> io::Result<()> { + let thread = launch_thread(child)?; + // SAFETY: both handles remain owned and live; std's child process handle + // carries process admission rights. Assignment precedes the only resume. + if unsafe { + AssignProcessToJobObject(self.0.as_raw_handle(), child.as_handle().as_raw_handle()) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: the retained handle belongs to the suspended child's sole thread, + // checked below; the process already belongs to the private job. + let count = unsafe { ResumeThread(thread.as_raw_handle()) }; + if count == u32::MAX { + return Err(io::Error::last_os_error()); + } + admission::resume_count(count) + } + + pub(super) fn terminate(&self) -> io::Result<()> { + // SAFETY: this owned job is live and contains only this command's descendants. + if unsafe { TerminateJobObject(self.0.as_raw_handle(), 1) } == 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } + + pub(super) fn reap(&self, child: &mut Child) -> io::Result<()> { + self.terminate()?; + let started = Instant::now(); + loop { + let reaped = child.try_wait()?.is_some(); + if self.active()? == 0 && reaped { + return Ok(()); + } + if started.elapsed() >= super::KILL_CEILING { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "the command job still contains active processes after termination", + )); + } + std::thread::sleep(super::WAIT_TICK); + } + } + + pub(super) fn active(&self) -> io::Result { + let mut accounting = JOBOBJECT_BASIC_ACCOUNTING_INFORMATION::default(); + // SAFETY: accounting is initialized, writable, and exactly the requested C + // structure. A failed query remains an error, never a proof of an empty job. + if unsafe { + QueryInformationJobObject( + self.0.as_raw_handle(), + JobObjectBasicAccountingInformation, + (&mut accounting as *mut JOBOBJECT_BASIC_ACCOUNTING_INFORMATION).cast(), + size_of::() as u32, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + Ok(accounting.ActiveProcesses) + } +} + +fn launch_thread(child: &Child) -> io::Result { + // SAFETY: Toolhelp takes integer flags and creates an owned, finite snapshot. + let raw = unsafe { CreateToolhelp32Snapshot(TH32CS_SNAPTHREAD, 0) }; + if raw == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: the snapshot is a fresh valid handle, checked above. + let snapshot = unsafe { OwnedHandle::from_raw_handle(raw) }; + let mut entry = THREADENTRY32 { + dwSize: size_of::() as u32, + ..Default::default() + }; + let mut threads = Vec::new(); + // SAFETY: the live snapshot and initialized C structure are valid for this API. + let mut found = unsafe { Thread32First(snapshot.as_raw_handle(), &mut entry) }; + loop { + if found == 0 { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(ERROR_NO_MORE_FILES as i32) { + return Err(error); + } + break; + } + if entry.th32OwnerProcessID == child.id() { + threads.push((entry.th32ThreadID, entry.th32OwnerProcessID)); + if threads.len() == 2 { + break; // Two owned entries prove ambiguity; no thread is resumed. + } + } + entry.dwSize = size_of::() as u32; + // SAFETY: the same retained snapshot and writable C structure remain valid. + found = unsafe { Thread32Next(snapshot.as_raw_handle(), &mut entry) }; + } + let id = admission::sole_thread(child.id(), threads)?; + // SAFETY: OpenThread acquires a new handle; no inherited handle is requested. + let raw = unsafe { + OpenThread( + THREAD_SUSPEND_RESUME | THREAD_QUERY_LIMITED_INFORMATION, + 0, + id, + ) + }; + if raw.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: OpenThread returned a new owned handle checked above. + let thread = unsafe { OwnedHandle::from_raw_handle(raw) }; + // SAFETY: this thread handle is retained and supports the ownership query. + let owner = unsafe { GetProcessIdOfThread(thread.as_raw_handle()) }; + if owner == 0 { + return Err(io::Error::last_os_error()); + } + admission::thread_owner(child.id(), owner)?; + Ok(thread) +} From 8b67d1737619bd7bf6adc8683270c29090ec99c2 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:09:54 +0800 Subject: [PATCH 145/155] Describe cancellation across command tree implementations --- crates/contextful-engine/src/command.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/contextful-engine/src/command.rs b/crates/contextful-engine/src/command.rs index 42c58d444..736dab559 100644 --- a/crates/contextful-engine/src/command.rs +++ b/crates/contextful-engine/src/command.rs @@ -215,7 +215,7 @@ impl Source for CommandSource { let status = loop { if cancel.requested() { reap_group(&mut child, &tree, &request.step_label)?; - return Err(Failure::canceled(format!("stopped during `{}`; its process group is reaped", request.step_label))); + return Err(Failure::canceled(format!("stopped during `{}`; its command tree is reaped", request.step_label))); } match child.try_wait() { Ok(Some(status)) => break status, From 1487443b41d3c6fe0ce3822b9588a35ca052898f Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:21:23 +0800 Subject: [PATCH 146/155] Require signalled job members before pull completion --- .../tests/integration/command_admission.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/contextful-engine/tests/integration/command_admission.rs b/crates/contextful-engine/tests/integration/command_admission.rs index 316b8fb05..a7f7aea4d 100644 --- a/crates/contextful-engine/tests/integration/command_admission.rs +++ b/crates/contextful-engine/tests/integration/command_admission.rs @@ -1,7 +1,7 @@ #[path = "../../src/command/admission.rs"] mod admission; -use admission::{resume_count, sole_thread, thread_owner}; +use admission::{reaped_members, resume_count, sole_thread, thread_owner}; #[test] fn suspended_launch_selects_only_its_single_owned_thread() { @@ -28,3 +28,11 @@ fn launch_resume_requires_exactly_one_suspend_count() { assert!(resume_count(count).is_err()); } } + +#[test] +fn zero_job_accounting_waits_for_retained_member_handles() { + assert!(!reaped_members(0, true, [true, false])); + assert!(!reaped_members(1, true, [true, true])); + assert!(!reaped_members(0, false, [true, true])); + assert!(reaped_members(0, true, [true, true])); +} From 5c1bded527fc4f964542b64971792188f6aafd42 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:25:00 +0800 Subject: [PATCH 147/155] Wait for verified Windows job member handles to signal --- .../src/command/admission.rs | 8 + .../contextful-engine/src/command/windows.rs | 150 +++++++++++++++++- 2 files changed, 150 insertions(+), 8 deletions(-) diff --git a/crates/contextful-engine/src/command/admission.rs b/crates/contextful-engine/src/command/admission.rs index bb2f0df66..ae1494f15 100644 --- a/crates/contextful-engine/src/command/admission.rs +++ b/crates/contextful-engine/src/command/admission.rs @@ -2,6 +2,14 @@ use std::io; +pub(super) fn reaped_members( + active: u32, + child_reaped: bool, + members: impl IntoIterator, +) -> bool { + active == 0 && child_reaped && members.into_iter().all(|signalled| signalled) +} + pub(super) fn sole_thread( process: u32, threads: impl IntoIterator, diff --git a/crates/contextful-engine/src/command/windows.rs b/crates/contextful-engine/src/command/windows.rs index 46d169753..d7800cf8a 100644 --- a/crates/contextful-engine/src/command/windows.rs +++ b/crates/contextful-engine/src/command/windows.rs @@ -7,18 +7,22 @@ use std::os::windows::io::{AsHandle, AsRawHandle, FromRawHandle, OwnedHandle}; use std::os::windows::process::CommandExt; use std::process::{Child, Command}; use std::time::Instant; -use windows_sys::Win32::Foundation::{ERROR_NO_MORE_FILES, INVALID_HANDLE_VALUE}; +use windows_sys::Win32::Foundation::{ + ERROR_NO_MORE_FILES, INVALID_HANDLE_VALUE, WAIT_OBJECT_0, WAIT_TIMEOUT, +}; use windows_sys::Win32::System::Diagnostics::ToolHelp::{ CreateToolhelp32Snapshot, Thread32First, Thread32Next, TH32CS_SNAPTHREAD, THREADENTRY32, }; use windows_sys::Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JobObjectBasicAccountingInformation, + AssignProcessToJobObject, CreateJobObjectW, IsProcessInJob, + JobObjectBasicAccountingInformation, JobObjectBasicProcessIdList, JobObjectExtendedLimitInformation, QueryInformationJobObject, SetInformationJobObject, - TerminateJobObject, JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, + TerminateJobObject, JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, JOBOBJECT_BASIC_PROCESS_ID_LIST, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, }; use windows_sys::Win32::System::Threading::{ - GetProcessIdOfThread, OpenThread, ResumeThread, CREATE_SUSPENDED, + GetProcessIdOfThread, OpenProcess, OpenThread, ResumeThread, WaitForSingleObject, + CREATE_SUSPENDED, PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, THREAD_QUERY_LIMITED_INFORMATION, THREAD_SUSPEND_RESUME, }; @@ -90,11 +94,49 @@ impl Job { } pub(super) fn reap(&self, child: &mut Child) -> io::Result<()> { - self.terminate()?; let started = Instant::now(); + let before = self.accounting(); + let members = before + .as_ref() + .map_err(|e| io::Error::new(e.kind(), e.to_string())) + .and_then(|info| self.members(info.ActiveProcesses, started)); + // Even a failed membership snapshot closes admission by terminating this + // owned job before the error returns. No partial snapshot proves completion. + self.terminate()?; + let before = before?; + let members = members?; + if members.len() < before.ActiveProcesses as usize { + return Err(io::Error::other( + "job membership changed before termination", + )); + } loop { + if started.elapsed() >= super::KILL_CEILING { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "the command job exceeds its termination deadline", + )); + } let reaped = child.try_wait()?.is_some(); - if self.active()? == 0 && reaped { + let now = self.accounting()?; + if now.TotalProcesses != before.TotalProcesses { + return Err(io::Error::other( + "new job membership lacks retained completion handles", + )); + } + let mut signalled = Vec::with_capacity(members.len()); + for member in &members { + // SAFETY: each retained handle has synchronize rights and verified + // membership in this exact job; no historical PID establishes exit. + signalled.push( + match unsafe { WaitForSingleObject(member.as_raw_handle(), 0) } { + WAIT_OBJECT_0 => true, + WAIT_TIMEOUT => false, + _ => return Err(io::Error::last_os_error()), + }, + ); + } + if admission::reaped_members(now.ActiveProcesses, reaped, signalled) { return Ok(()); } if started.elapsed() >= super::KILL_CEILING { @@ -107,7 +149,7 @@ impl Job { } } - pub(super) fn active(&self) -> io::Result { + fn accounting(&self) -> io::Result { let mut accounting = JOBOBJECT_BASIC_ACCOUNTING_INFORMATION::default(); // SAFETY: accounting is initialized, writable, and exactly the requested C // structure. A failed query remains an error, never a proof of an empty job. @@ -123,7 +165,99 @@ impl Job { { return Err(io::Error::last_os_error()); } - Ok(accounting.ActiveProcesses) + Ok(accounting) + } + + fn members(&self, count: u32, started: Instant) -> io::Result> { + // The kernel reports the capacity. Growing membership refuses completion, + // rather than starting an unbounded resize loop or trusting a historical PID. + let count = (count as usize).max(1); + let header = std::mem::offset_of!(JOBOBJECT_BASIC_PROCESS_ID_LIST, ProcessIdList); + let bytes = header + .checked_add( + count + .checked_mul(size_of::()) + .ok_or_else(|| io::Error::other("job membership size overflow"))?, + ) + .ok_or_else(|| io::Error::other("job membership size overflow"))?; + let bytes_u32 = + u32::try_from(bytes).map_err(|_| io::Error::other("job membership size overflow"))?; + let mut buffer = Vec::::new(); + buffer + .try_reserve_exact(bytes.div_ceil(size_of::())) + .map_err(io::Error::other)?; + buffer.resize(bytes.div_ceil(size_of::()), 0); + let pointer = buffer + .as_mut_ptr() + .cast::(); + // SAFETY: usize storage provides the C structure's alignment; initialized + // capacity holds its eight-byte header and count variable-length PID slots. + if unsafe { + QueryInformationJobObject( + self.0.as_raw_handle(), + JobObjectBasicProcessIdList, + pointer.cast(), + bytes_u32, + std::ptr::null_mut(), + ) + } == 0 + { + return Err(io::Error::last_os_error()); + } + // SAFETY: the successful query initializes the header in this live buffer. + let list = unsafe { &*pointer }; + let populated = list.NumberOfProcessIdsInList as usize; + if populated > count || list.NumberOfAssignedProcesses as usize != populated { + return Err(io::Error::other("job membership snapshot is incomplete")); + } + // SAFETY: the validated count fits the allocation's contiguous PID area; + // the underlying buffer remains live for the whole iteration. + let ids = unsafe { + std::slice::from_raw_parts( + buffer.as_ptr().cast::().add(header).cast::(), + populated, + ) + }; + let mut handles = Vec::new(); + for &id in ids { + if started.elapsed() >= super::KILL_CEILING { + return Err(io::Error::new( + io::ErrorKind::TimedOut, + "job membership capture exceeds its termination deadline", + )); + } + let pid = u32::try_from(id) + .map_err(|_| io::Error::other("job PID exceeds its native representation"))?; + // SAFETY: lookup acquires query/synchronize rights only, never cleanup + // rights. Membership in the exact owned job independently gates use. + let raw = unsafe { + OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION | PROCESS_SYNCHRONIZE, + 0, + pid, + ) + }; + if raw.is_null() { + return Err(io::Error::last_os_error()); + } + // SAFETY: OpenProcess returned a fresh, checked, owned process handle. + let handle = unsafe { OwnedHandle::from_raw_handle(raw) }; + let mut member = 0; + // SAFETY: both retained handles are live; member is writable BOOL storage. + if unsafe { + IsProcessInJob(handle.as_raw_handle(), self.0.as_raw_handle(), &mut member) + } == 0 + { + return Err(io::Error::last_os_error()); + } + if member == 0 { + return Err(io::Error::other( + "the retained process does not belong to the command job", + )); + } + handles.push(handle); + } + Ok(handles) } } From 58c4696072751ee43cc9759994727898c26dd776 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:33:52 +0800 Subject: [PATCH 148/155] Keep Windows containment verification in integration tests --- .github/workflows/windows.yml | 36 ----------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 .github/workflows/windows.yml diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index 53f0bc2de..000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,36 +0,0 @@ -# Native execution verifies directory flushes and file identity on both Windows targets. -name: native-windows - -on: - pull_request: - -permissions: - contents: read - -jobs: - windows: - timeout-minutes: 30 - env: - RUSTUP_TOOLCHAIN: 1.97.0 - if: ${{ github.event.pull_request.head.repo.full_name == github.repository }} - strategy: - fail-fast: false - matrix: - include: - - runner: windows-2025 - target: x86_64-pc-windows-msvc - - runner: windows-11-arm - target: aarch64-pc-windows-msvc - runs-on: ${{ matrix.runner }} - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - - run: rustup toolchain install 1.97.0 --profile minimal --target ${{ matrix.target }} - - run: cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration an_open_directory_handle_syncs_its_entries - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration a_renamed_open_file_does_not_identify_its_replacement - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration open_handles_keep_their_identity_across_rename_and_path_reuse - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-fs --test integration racing_creates_land_exactly_one - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-policy --test integration an_audit_append_syncs_its_directories - - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_ledger_append_syncs_its_directory - - run: cargo test --locked --target ${{ matrix.target }} --no-default-features -p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it - - run: cargo test --locked --target ${{ matrix.target }} -p contextful-engine --test integration command -- --test-threads=2 From 651be10e6e2bf7641ad9ebda1c02518bb0bd5e68 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 06:40:22 +0800 Subject: [PATCH 149/155] Retain native regressions independently of the evidence harness --- tools/ci/tests/integration/stages.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/tools/ci/tests/integration/stages.rs b/tools/ci/tests/integration/stages.rs index c9fdf0ca8..812a7bc72 100644 --- a/tools/ci/tests/integration/stages.rs +++ b/tools/ci/tests/integration/stages.rs @@ -129,28 +129,6 @@ fn predecessors_adds_every_stage_whose_output_a_selected_stage_reads() { } -#[test] -fn native_windows_checks_cover_both_targets_and_directory_sync() { - let workflow = std::fs::read_to_string(crate::repo_root().join(".github/workflows/windows.yml")).unwrap(); - for pair in [ - "runner: windows-2025\n target: x86_64-pc-windows-msvc", - "runner: windows-11-arm\n target: aarch64-pc-windows-msvc", - ] { - assert!(workflow.contains(pair), "the native Windows matrix omits {pair}"); - } - assert!(workflow.contains("cargo check --locked --target ${{ matrix.target }} --no-default-features -p contextful-core -p contextful-fs -p contextful-policy -p contextful-context")); - for command in [ - "-p contextful-fs --test integration an_open_directory_handle_syncs_its_entries", - "-p contextful-fs --test integration a_renamed_open_file_does_not_identify_its_replacement", - "-p contextful-fs --test integration racing_creates_land_exactly_one", - "-p contextful-policy --test integration an_audit_append_syncs_its_directories", - "-p contextful-context --test integration a_ledger_append_syncs_its_directory", - "-p contextful-context --test integration a_renamed_lock_does_not_remove_the_file_that_replaces_it", - ] { - assert!(workflow.contains(command), "the native Windows matrix omits {command}"); - } -} - /// Each stage prints the environment it leaves and its memory limit, peak and event counts, and a failing stage prints its diagnostics before propagating its exit code. // spec: assurance.gate.stage-reports@e77a2384 #[test] From 4e8af02dbe5b28930f16bafee92200877931db86 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 07:44:46 +0800 Subject: [PATCH 150/155] fix(ci): require red tests in changed source packages --- spec/81-engineering.md | 7 +++-- tools/ci/src/main.rs | 14 +++++----- tools/ci/tests/integration/test_first.rs | 35 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/spec/81-engineering.md b/spec/81-engineering.md index 45981cd09..c41270c89 100644 --- a/spec/81-engineering.md +++ b/spec/81-engineering.md @@ -88,7 +88,8 @@ Assertion construction, guard validation, suite placement, test-first and accept - `connector-kit` — The connector authoring toolkit ships a conformance suite — discovery returns valid schemas, an opened table yields a finite stream, a position round-trips — plus recorded-HTTP fixture replay and property tests over position monotonicity. - `test-first` — A change altering Rust source under `crates/` or `tools/` adds or alters a test under a package's `tests/` that fails against the base commit's source; a change without one raises `TestNotFirst`. *A-assurance* -- `test-first-scope` — The test-first stage builds each changed test file's target against the base source with every feature enabled, then runs exactly the tests under that file's top-level module; a target failing to compile there counts as failing. +- `test-first-scope` — For each package held by {{assurance.gate.test-first-parts}}, the test-first stage builds each changed test file's target against the base source with every feature enabled, then runs its top-level module; a target failing to compile there counts as failing. + *A-assurance* - `base-run-bound` — One test-first execution against the base, its build excluded, runs for at most 300 s; a run past the bound is killed with its process group and counts as failing. - `base-unrunnable` — A base invocation whose output reports a full disk, an unloadable manifest or an unfetchable dependency raises `TestFirstBaseUnrunnable` instead of a verdict. *because such a fault fails at base whatever the tests assert, and reading it as red admits an untested change* @@ -167,8 +168,8 @@ Stage order, secrets of record, the crate-graph, row-token, egress and dependenc *A-assurance* - `windows-native` — The pull-request workflow checks core, filesystem, policy and store-write packages and exercises directory sync on native x86 and ARM Windows runners. *because native runners provide the target C toolchain and execute directory flushes that cross-compilation cannot verify* -- `test-first-parts` — Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each changed source or test package checks only its own tests against the base in a separate dispatch. - *because one base build per package fits the sandbox's wall clock more reliably* +- `test-first-parts` — Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each such package checks only its changed tests against the base in a separate dispatch. + *A-assurance* - `workspace-parts` — Remote workspace checks compile the feature-unified workspace without running tests, run differential and remaining CLI tests separately, and run each other non-acceptance package suite in exactly one of four groups. *A-assurance* - `remote-predecessors` — Each dispatched part invokes `contextful-ci gate --predecessors --stage --base `, so no check reads another check's sandbox. diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index b620a5f1c..696d9f1d3 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -1015,11 +1015,6 @@ fn test_first_parts(root: &Path, base: &str) -> Result> { for source in sources_outside_refactors(base)? { packages.insert(changed_package(root, &source)?); } - if !packages.is_empty() { - for test in changed_tests(base)? { - packages.insert(changed_package(root, &test)?); - } - } Ok(std::iter::once("validate".to_string()).chain(packages).collect()) } @@ -1030,14 +1025,19 @@ fn test_first(root: &Path, base: &str, bound: Duration, only: Option<&[String]>) return Ok(()); } let changed = changed_tests(base)?; - let tests: Vec<&str> = changed.iter().map(String::as_str).collect(); + let source_packages: std::collections::BTreeSet = sources.iter().map(|source| changed_package(root, source)).collect::>()?; + let mut tests = Vec::new(); + for test in &changed { + if source_packages.contains(&changed_package(root, test)?) { + tests.push(test.as_str()); + } + } if tests.is_empty() { return Err(refuse( "TestNotFirst", format!("{} source file(s) changed and no test under a package's tests/ did: {}", sources.len(), sources.join(", ")), )); } - let source_packages: std::collections::BTreeSet = sources.iter().map(|source| changed_package(root, source)).collect::>()?; let test_packages: std::collections::BTreeSet = tests.iter().map(|test| changed_package(root, test)).collect::>()?; if only.is_none() || only.is_some_and(|parts| parts == ["validate"]) { if let Some(missing) = source_packages.difference(&test_packages).next() { diff --git a/tools/ci/tests/integration/test_first.rs b/tools/ci/tests/integration/test_first.rs index 5b2f58254..c7729d1c0 100644 --- a/tools/ci/tests/integration/test_first.rs +++ b/tools/ci/tests/integration/test_first.rs @@ -68,6 +68,41 @@ fn a_test_failing_on_the_base_passes() { assert!(o.status.success(), "{}", stderr(&o)); } +#[test] +fn test_only_portability_changes_stay_workspace_checked_without_a_red_obligation() { + let r = Repo::init(); + r.write("crates/other/Cargo.toml", &manifest("other", "")); + r.write("crates/other/src/lib.rs", "pub fn value() -> i32 { 1 }\n"); + r.write("crates/other/tests/integration/main.rs", "mod value;\n"); + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn positive() { assert!(other::value() > 0); }\n"); + r.lock(); + r.commit("second package and workspace lock"); + let base = r.head(); + r.write("crates/demo/src/lib.rs", TRIPLE); + r.write("crates/demo/tests/integration/main.rs", "mod double;\nmod triple;\n"); + r.write("crates/demo/tests/integration/triple.rs", "#[test]\nfn triples() { assert_eq!(demo::triple(2), 6); }\n"); + r.write("crates/other/tests/integration/value.rs", "#[cfg(unix)]\n#[test]\nfn positive() { assert!(other::value() >= 1); }\n"); + r.commit("new source behavior and an unrelated portable test"); + + let listed = r.run_ci(&["stages", "--parts", "--base", &base, "--json"]); + assert!(listed.status.success(), "{}", stderr(&listed)); + let stages: Vec = serde_json::from_slice(&listed.stdout).unwrap(); + assert!(stages.contains(&"test-first.demo".to_string()), "{stages:?}"); + assert!(!stages.contains(&"test-first.other".to_string()), "test-only package has no source behavior to specify: {stages:?}"); + let whole = r.gate(&["--stage", "test-first", "--base", &base]); + assert!(whole.status.success(), "{}", stderr(&whole)); + let workspace = r.gate(&["--stage", "workspace"]); + assert!(workspace.status.success(), "{}", stderr(&workspace)); + + r.write("crates/other/tests/integration/value.rs", "#[test]\nfn positive() { assert_eq!(other::value(), 2); }\n"); + r.commit("a failing test-only change"); + let whole = r.gate(&["--stage", "test-first", "--base", &base]); + assert!(whole.status.success(), "{}", stderr(&whole)); + let workspace = r.gate(&["--stage", "workspace"]); + assert!(!workspace.status.success(), "the workspace skipped the unrelated test-only failure"); + assert!(String::from_utf8_lossy(&workspace.stdout).contains("positive"), "{}", stderr(&workspace)); +} + #[test] fn each_changed_test_package_has_its_own_dispatch_part() { let r = Repo::init(); From 3b4365190ea0d704285179a9b4a1e18c21d40c87 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 08:02:37 +0800 Subject: [PATCH 151/155] docs(assurance): bind red checks to source package revisions --- spec/pins.toml | 6 +- spec/spec.lock.json | 286 ++++++++++++----------- spec/status.md | 6 +- tools/ci/tests/integration/test_first.rs | 6 +- 4 files changed, 154 insertions(+), 150 deletions(-) diff --git a/spec/pins.toml b/spec/pins.toml index e036b7b18..bef6b24c3 100644 --- a/spec/pins.toml +++ b/spec/pins.toml @@ -100,13 +100,13 @@ # its live performed count, and a live count below its floor raises SpecCoverageRegression. [floor] -assurance = 141 +assurance = 145 authority = 207 connector = 229 corpus = 43 disclosure = 63 -read = 178 +read = 183 run = 306 store = 216 -surface = 66 +surface = 85 topology = 38 diff --git a/spec/spec.lock.json b/spec/spec.lock.json index a1dc55713..2316898c3 100644 --- a/spec/spec.lock.json +++ b/spec/spec.lock.json @@ -22115,8 +22115,8 @@ "operation": "test", "subject": "test-first-scope", "kind": "behavior", - "statement": "The test-first stage builds each changed test file's target against the base source with every feature enabled, then runs exactly the tests under that file's top-level module; a target failing to compile there counts as failing.", - "why": "", + "statement": "For each package held by {{assurance.gate.test-first-parts}}, the test-first stage builds each changed test file's target against the base source with every feature enabled, then runs its top-level module; a target failing to compile there counts as failing.", + "why": "A-assurance", "file": "spec/81-engineering.md", "line": 91 }, @@ -22129,7 +22129,7 @@ "statement": "One test-first execution against the base, its build excluded, runs for at most 300 s; a run past the bound is killed with its process group and counts as failing.", "why": "", "file": "spec/81-engineering.md", - "line": 92 + "line": 93 }, { "id": "assurance.test.base-unrunnable", @@ -22140,7 +22140,7 @@ "statement": "A base invocation whose output reports a full disk, an unloadable manifest or an unfetchable dependency raises `TestFirstBaseUnrunnable` instead of a verdict.", "why": "because such a fault fails at base whatever the tests assert, and reading it as red admits an untested change", "file": "spec/81-engineering.md", - "line": 93 + "line": 94 }, { "id": "assurance.test.refactor-trailer", @@ -22151,7 +22151,7 @@ "statement": "A commit carrying the trailer `Test-First: refactor` exempts the source it alters from {{assurance.test.test-first}}; the rest of the range stays held, and the workspace stage alone holds that commit.", "why": "because a behavior-preserving change has no failing test to write, and the existing suite is its specification", "file": "spec/81-engineering.md", - "line": 95 + "line": 96 }, { "id": "assurance.test.acceptance-surface", @@ -22162,7 +22162,7 @@ "statement": "An acceptance test drives a built binary through its command line, MCP or HTTP surface; a workspace package among the acceptance package's dependencies raises `AcceptanceLinksEngine`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 97 + "line": 98 }, { "id": "assurance.build.target-dir-per-stage", @@ -22173,7 +22173,7 @@ "statement": "Each cargo stage builds into a target directory of its own, reclaimed once the stage passes.", "why": "because stages under different feature unification share no artifacts, and peak disk is then one stage", "file": "spec/81-engineering.md", - "line": 120 + "line": 121 }, { "id": "assurance.build.one-engine-build", @@ -22184,7 +22184,7 @@ "statement": "The workspace stage builds every engine-linked package in one cargo invocation over the union of their features, and the store adapter's suites resolved without `read` link no SQL engine, so the features stage compiles one copy.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 122 + "line": 123 }, { "id": "assurance.build.staged-feature-runs", @@ -22195,7 +22195,7 @@ "statement": "A defect appearing under one feature combination alone is reached by a staged run, one command per container.", "why": "", "file": "spec/81-engineering.md", - "line": 124 + "line": 125 }, { "id": "assurance.build.linked-query-functions", @@ -22206,7 +22206,7 @@ "statement": "Columnar file reading and statement serialization link into every build linking the SQL engine, and the read path loads no extension while serving.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 125 + "line": 126 }, { "id": "assurance.build.runtime-extension-load", @@ -22217,7 +22217,7 @@ "statement": "A read path loading an extension into a binary that statically links the SQL engine raises `ExtensionAutoloadRefused`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 127 + "line": 128 }, { "id": "assurance.build.debug-info", @@ -22228,7 +22228,7 @@ "statement": "Development and test profiles carry line-tables-only debug information for workspace code and none for dependencies.", "why": "because every engine-linked test binary embeds the bundled engine, and full debug information multiplies each past a sandbox's disk", "file": "spec/81-engineering.md", - "line": 129 + "line": 130 }, { "id": "assurance.build.release-profile", @@ -22239,7 +22239,7 @@ "statement": "Release builds compile with thin link-time optimization, one codegen unit per crate and symbols stripped, and unwind on panic.", "why": "because the run keeper survives a panicking job only by unwinding to its guard", "file": "spec/81-engineering.md", - "line": 131 + "line": 132 }, { "id": "assurance.build.profile-build", @@ -22250,7 +22250,7 @@ "statement": "The features stage tests each package under every feature set its manifest lists in `feature-runs`: other packages share one check; each binary feature run has separate checks for differential tests and all remaining tests.", "why": "because one cold build of the binary under one feature set fills most of a stage's wall clock, and a listed set reaches what none and all miss", "file": "spec/81-engineering.md", - "line": 133 + "line": 134 }, { "id": "assurance.build.container-image", @@ -22261,7 +22261,7 @@ "statement": "The repository's `Dockerfile` builds one profile, `contextful-full` unless `PROFILE` names another, as a static `linux/amd64` binary, and ships it in a shell-free runtime image as a non-root user over a declared store volume.", "why": "", "file": "spec/81-engineering.md", - "line": 135 + "line": 136 }, { "id": "assurance.build.targets", @@ -22272,7 +22272,7 @@ "statement": "All three profiles cross-compile to `x86_64-unknown-linux-musl` and `aarch64-unknown-linux-musl`; edge and full also build for `aarch64-apple-darwin` and `x86_64-apple-darwin`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 136 + "line": 137 }, { "id": "assurance.build.windows-target", @@ -22283,7 +22283,7 @@ "statement": "Edge and full also build for `x86_64-pc-windows-msvc` and `aarch64-pc-windows-msvc`.", "why": "", "file": "spec/81-engineering.md", - "line": 138 + "line": 139 }, { "id": "assurance.build.wasi-probe", @@ -22294,7 +22294,7 @@ "statement": "No profile ships a `wasm32-wasip2` release; a scheduled-tier ledger entry builds the edge profile for it and holds the compressed artifact to {{assurance.gate.edge-budget}}.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 139 + "line": 140 }, { "id": "assurance.build.release-artifact", @@ -22305,7 +22305,7 @@ "statement": "Each profile ships a release archive with a SHA-256 checksum and an SBOM, a package-manager formula and an independently tagged container image; the bare formula name and the install script resolve to the full profile.", "why": "", "file": "spec/81-engineering.md", - "line": 141 + "line": 142 }, { "id": "assurance.build.release-builder", @@ -22316,7 +22316,7 @@ "statement": "The release command uses `cargo build` by default and `cargo zigbuild` under `--builder zigbuild`, forwarding the selected target and profile features.", "why": "because Cloudflare Linux builds Darwin targets through Zig while the local release command keeps its native build path", "file": "spec/81-engineering.md", - "line": 142, + "line": 143, "scenarios": [ "WHEN a Darwin cell selects `--builder zigbuild`, THEN the release command invokes `cargo zigbuild` for that profile and target." ] @@ -22330,7 +22330,7 @@ "statement": "Each release cell writes a JSON record naming its profile, target, archive, SHA-256 digest and SBOM.", "why": "because distributed release cells need a small authenticated input for formula generation", "file": "spec/81-engineering.md", - "line": 144, + "line": 145, "scenarios": [ "WHEN a release cell packages an archive, THEN its JSON metadata names the archive, digest and SBOM." ] @@ -22344,7 +22344,7 @@ "statement": "The formula command accepts a manifest covering every release matrix cell once, with matching asset names and SHA-256 digests, and writes formulas and SHA256SUMS without local archives.", "why": "because one aggregation sandbox need not download every archive to produce formulas", "file": "spec/81-engineering.md", - "line": 146, + "line": 147, "scenarios": [ "WHEN every cell's metadata is present without local archives, THEN formula generation writes the profile formulae and SHA256SUMS." ] @@ -22358,7 +22358,7 @@ "statement": "Every workspace package under `crates/` or `tools/` declares `license = \"Apache-2.0\"`, inherited from `[workspace.package]`; a package declaring another value or none raises `PackageLicenceMissing`, naming its manifest.", "why": "because cargo-deny, cargo-about and SBOM generators read the manifest field, not the `LICENSE` file, so an unlicensed package fails a consumer's licence check", "file": "spec/81-engineering.md", - "line": 148 + "line": 149 }, { "id": "assurance.build.dependency-allowlist", @@ -22369,7 +22369,7 @@ "statement": "The connector authoring dependency allowlist carries `regex` and `serde_json` at its depth limit; a profile graph reaching `fancy-regex`, `pcre2`, `onig` or `serde_json`'s `unbounded_depth` feature raises `DependencyAllowlistViolation`, naming the profile and path.", "why": "because a backtracking matcher or a parser without a depth limit lets one hostile record pin a core or exhaust the stack", "file": "spec/81-engineering.md", - "line": 150 + "line": 151 }, { "id": "assurance.gate.stage-sequence", @@ -22380,7 +22380,7 @@ "statement": "The gate runs its stages in order — pins, toolchain, schema, test-first, workspace, acceptance, evaluate, features, crate graph, connectors, TypeScript surfaces, formal, budget — and a subset is selectable by name.", "why": "", "file": "spec/81-engineering.md", - "line": 163 + "line": 164 }, { "id": "assurance.gate.stage-subset", @@ -22391,7 +22391,7 @@ "statement": "A selected subset runs in the sequence's order; a stage reading an unselected predecessor's output, with that output absent, raises `StagePredecessorMissing`, naming both stages, before any stage runs.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 164 + "line": 165 }, { "id": "assurance.gate.remote-check", @@ -22402,7 +22402,7 @@ "statement": "The FlareDispatch pull-request webhook dispatches every part from `contextful-ci stages --parts --base `, each as `flare-dispatch/check:` on the head commit.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 166 + "line": 167 }, { "id": "assurance.gate.windows-native", @@ -22413,7 +22413,7 @@ "statement": "The pull-request workflow checks core, filesystem, policy and store-write packages and exercises directory sync on native x86 and ARM Windows runners.", "why": "because native runners provide the target C toolchain and execute directory flushes that cross-compilation cannot verify", "file": "spec/81-engineering.md", - "line": 168 + "line": 169 }, { "id": "assurance.gate.test-first-parts", @@ -22421,10 +22421,10 @@ "operation": "gate", "subject": "test-first-parts", "kind": "behavior", - "statement": "Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each changed source or test package checks only its own tests against the base in a separate dispatch.", - "why": "because one base build per package fits the sandbox's wall clock more reliably", + "statement": "Remote validation requires a changed test in each changed source package under {{assurance.test.test-first}}; each such package checks only its changed tests against the base in a separate dispatch.", + "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 170 + "line": 171 }, { "id": "assurance.gate.workspace-parts", @@ -22435,7 +22435,7 @@ "statement": "Remote workspace checks compile the feature-unified workspace without running tests, run differential and remaining CLI tests separately, and run each other non-acceptance package suite in exactly one of four groups.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 172 + "line": 173 }, { "id": "assurance.gate.remote-predecessors", @@ -22446,7 +22446,7 @@ "statement": "Each dispatched part invokes `contextful-ci gate --predecessors --stage --base `, so no check reads another check's sandbox.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 174 + "line": 175 }, { "id": "assurance.gate.fork-dispatch", @@ -22457,7 +22457,7 @@ "statement": "FlareDispatch dispatches only a pull-request head pushed to this repository; a fork receives none of the required stage checks.", "why": "because a dispatch runs untrusted code in the organization's compute account, and absent required checks fail closed", "file": "spec/81-engineering.md", - "line": 176 + "line": 177 }, { "id": "assurance.gate.stage-reports", @@ -22468,7 +22468,7 @@ "statement": "Each stage prints the environment it leaves and its memory limit, peak and event counts, and a failing stage prints its diagnostics before propagating its exit code.", "why": "because memory exhaustion is silent, and a kill then reads as a number in the log", "file": "spec/81-engineering.md", - "line": 178 + "line": 179 }, { "id": "assurance.gate.pins-stage", @@ -22479,7 +22479,7 @@ "statement": "The pins stage resolves every pinned artifact identity a run depends on before any compilation.", "why": "", "file": "spec/81-engineering.md", - "line": 180 + "line": 181 }, { "id": "assurance.gate.evaluate-stage", @@ -22490,7 +22490,7 @@ "statement": "The evaluate stage runs every gate-tier ledger entry and the native case set in the deterministic tier, and reports the floor and baseline verdicts.", "why": "", "file": "spec/81-engineering.md", - "line": 181 + "line": 182 }, { "id": "assurance.gate.schema-stage", @@ -22501,7 +22501,7 @@ "statement": "The schema stage regenerates each derived artifact into a scratch location, compares it byte for byte against the committed copy, and runs `contextful-spec lint`.", "why": "", "file": "spec/81-engineering.md", - "line": 182 + "line": 183 }, { "id": "assurance.gate.secret-ciphertext", @@ -22512,7 +22512,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file other than `.env.example` holding a value without the `encrypted:` prefix, or a tracked `.env.keys`, raises `SecretPlaintext`, naming file and key.", "why": "because a tracked file reaches every clone, and only dotenvx ciphertext is safe there while its private key stays untracked", "file": "spec/81-engineering.md", - "line": 183 + "line": 184 }, { "id": "assurance.gate.secret-scope", @@ -22523,7 +22523,7 @@ "statement": "In the schema stage, a key other than `DOTENV_PUBLIC_KEY*` in a git-tracked `.env*` file whose block — the assignments above it, up to the first blank line — opens with no comment carrying text raises `SecretScopeMissing`, naming file and key.", "why": "because a provider token cannot report its own grants, so the comment above it is the one record of what it reaches", "file": "spec/81-engineering.md", - "line": 185 + "line": 186 }, { "id": "assurance.gate.crate-graph", @@ -22534,7 +22534,7 @@ "statement": "The crate-graph stage holds run-path crates to reaching read-path crates through the three crossing crates alone; another edge raises `CrateGraphViolation`, naming both crates.", "why": "P5", "file": "spec/81-engineering.md", - "line": 187 + "line": 188 }, { "id": "assurance.gate.locked-resolve", @@ -22545,7 +22545,7 @@ "statement": "The crate-graph stage resolves every graph `--locked` against the committed `Cargo.lock`, so a lock behind its manifests, or one rewritten before the stage starts, fails the stage before any rule runs; the stage never rewrites it.", "why": "because a rule held over a graph the lock file does not record passes a build that resolves another", "file": "spec/81-engineering.md", - "line": 189 + "line": 190 }, { "id": "assurance.gate.row-token", @@ -22556,7 +22556,7 @@ "statement": "Every row-returning function of the store crate takes the enforcement token type, whose constructor is private to the enforcement module; a row path outside a registered relation raises `EnforceUnmediatedPath`, naming the function.", "why": "P5", "file": "spec/81-engineering.md", - "line": 191 + "line": 192 }, { "id": "assurance.gate.dependency-deny", @@ -22567,7 +22567,7 @@ "statement": "The crate-graph stage runs pinned cargo-deny `check bans` over each profile's resolved graph, denying every crate a profile-wide dependency refusal names; a hit raises that refusal's error, naming the profile and the path.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 193 + "line": 194 }, { "id": "assurance.gate.deny-profile-graph", @@ -22578,7 +22578,7 @@ "statement": "A profile's graph is the binary resolved with default features off and the profile's feature on, across every target, excluding development dependencies; the stage prints each profile the binary declares no feature for.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 195 + "line": 196 }, { "id": "assurance.gate.deny-list", @@ -22589,7 +22589,7 @@ "statement": "`deny.toml` holds one `[bans]` entry per crate or feature a profile-wide dependency refusal names, its reason that refusal's clause id; an entry missing or extra raises `DenyListDrift`, naming it.", "why": "because a refusal whose crate the list omits passes every profile graph linking it", "file": "spec/81-engineering.md", - "line": 197 + "line": 198 }, { "id": "assurance.gate.unconfigured-egress", @@ -22600,7 +22600,7 @@ "statement": "An outbound call reachable under the default configuration — usage ping, license check, update probe — raises `UnconfiguredEgress` in the crate-graph stage, naming the call site.", "why": "because some deployments run with no external reach, and a call nobody configured moves data outside every grant, zone and audit record", "file": "spec/81-engineering.md", - "line": 199 + "line": 200 }, { "id": "assurance.gate.interpolated-claim", @@ -22611,7 +22611,7 @@ "statement": "A source lint over the runtime crates finding a subject claim formatted into SQL text raises `EnforceInterpolatedSubjectClaim`, naming the file and line.", "why": "P1", "file": "spec/81-engineering.md", - "line": 201 + "line": 202 }, { "id": "assurance.gate.formal-stage", @@ -22622,7 +22622,7 @@ "statement": "The formal stage runs {{assurance.audit-assumptions.check-command}}, {{assurance.differential-test.command}} and {{assurance.model.protocol-check}}; a non-zero exit from any reds the run.", "why": "P7", "file": "spec/81-engineering.md", - "line": 203 + "line": 204 }, { "id": "assurance.gate.container", @@ -22633,7 +22633,7 @@ "statement": "The gate container carries four ceilings — processor count, 12 GiB of memory, 18 GiB of usable disk, and a 30 min per-stage wall clock — and a run dies on any one.", "why": "", "file": "spec/81-engineering.md", - "line": 205 + "line": 206 }, { "id": "assurance.gate.parallel-jobs", @@ -22644,7 +22644,7 @@ "statement": "Parallel build jobs number the memory ceiling divided by 3 GiB.", "why": "", "file": "spec/81-engineering.md", - "line": 206 + "line": 207 }, { "id": "assurance.gate.free-disk", @@ -22655,7 +22655,7 @@ "statement": "A stage starting with less than 2 GiB of free disk raises `BuildDiskPrecondition` and exits 28 before doing work.", "why": "P7", "file": "spec/81-engineering.md", - "line": 207 + "line": 208 }, { "id": "assurance.gate.build-size", @@ -22666,7 +22666,7 @@ "statement": "The budget stage holds total build size to 12 GiB, printing that size, the free space and the largest artifacts; a larger build raises `BuildSizeCeilingExceeded`, naming the head of that list.", "why": "P7", "file": "spec/81-engineering.md", - "line": 209 + "line": 210 }, { "id": "assurance.gate.edge-budget", @@ -22677,7 +22677,7 @@ "statement": "The edge profile holds to 50 MiB compressed and 60 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 211 + "line": 212 }, { "id": "assurance.gate.full-budget", @@ -22688,7 +22688,7 @@ "statement": "The full profile holds to 150 MiB compressed and 180 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 212 + "line": 213 }, { "id": "assurance.gate.control-budget", @@ -22699,7 +22699,7 @@ "statement": "The control profile holds to 60 MiB compressed and 80 MiB idle resident set.", "why": "", "file": "spec/81-engineering.md", - "line": 213 + "line": 214 }, { "id": "assurance.gate.footprint", @@ -22710,7 +22710,7 @@ "statement": "Per change and per profile, the footprint step builds the static-linked Linux target, compresses it, and holds its size to the profile's budget and its dynamic dependencies to the platform C library.", "why": "P7", "file": "spec/81-engineering.md", - "line": 214 + "line": 215 }, { "id": "assurance.gate.budget-stage", @@ -22721,7 +22721,7 @@ "statement": "The budget stage runs {{assurance.gate.footprint}} over every profile, in one part per profile dispatched as its own check, and the evaluate stage builds no profile.", "why": "because one link-time-optimized release build of the full profile fills most of one stage's wall clock", "file": "spec/81-engineering.md", - "line": 216 + "line": 217 }, { "id": "assurance.gate.footprint-exceeded", @@ -22732,7 +22732,7 @@ "statement": "An artifact over its profile's budget, or carrying a dynamic dependency beyond the platform C library, raises `FootprintBudgetExceeded`, naming the profile.", "why": "P7", "file": "spec/81-engineering.md", - "line": 218 + "line": 219 }, { "id": "assurance.gate.typescript-surfaces", @@ -22743,7 +22743,7 @@ "statement": "The TypeScript surfaces run each declared typecheck, test and build script in one stage; an absent script skips only that script.", "why": "", "file": "spec/81-engineering.md", - "line": 220 + "line": 221 }, { "id": "assurance.gate.native-test-files", @@ -22754,7 +22754,7 @@ "statement": "The surfaces stage runs every regular native `.test.ts` file under a direct TypeScript surface's `test/` or `tests/` tree through Node's TypeScript strip runner, and refuses absent passing tests or pinned names.", "why": "because a package script can omit a pinned test while reporting success", "file": "spec/81-engineering.md", - "line": 221 + "line": 222 }, { "id": "assurance.gate.surface-check-failed", @@ -22765,7 +22765,7 @@ "statement": "A surface whose typecheck, unit tests or framework build fails raises `SurfaceCheckFailed`, naming the surface and the script.", "why": "P7", "file": "spec/81-engineering.md", - "line": 223 + "line": 224 }, { "id": "assurance.evaluate.real-read-path", @@ -22776,7 +22776,7 @@ "statement": "An evaluation run ingests, indexes, retrieves, reads, judges and scores through the surfaces a caller uses, and the harness adds no storage or runtime primitive.", "why": "", "file": "spec/81-engineering.md", - "line": 270 + "line": 271 }, { "id": "assurance.evaluate.through-the-store", @@ -22787,7 +22787,7 @@ "statement": "The corpus loads through the real store, and the retriever under test calls {{read.retrieve.ranked-call}} with the options a caller passes.", "why": "because an unordered full-scan retriever cannot go red when ranking breaks", "file": "spec/81-engineering.md", - "line": 271 + "line": 272 }, { "id": "assurance.evaluate.policy-labels", @@ -22798,7 +22798,7 @@ "statement": "An evaluation corpus carries zone and row-policy labels in its table declarations, and a run reads it through the access-control path under one admitted credential.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 273, + "line": 274, "scenarios": [ "WHEN a regression case names a row another agent owns under `owner = subject.agent`, THEN no leg returns it and the hybrid forbidden-row rate reads 0." ] @@ -22812,7 +22812,7 @@ "statement": "A run over a corpus whose tables declare no zone or row policy, with no `local:` zone passed as `--zone`, raises `EvalCorpusUnlabeled` before any row lands.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 275, + "line": 276, "scenarios": [ "WHEN a corpus's tables declare no policy and the run passes no `--zone`, THEN the run raises `EvalCorpusUnlabeled` and lands no row." ] @@ -22826,7 +22826,7 @@ "statement": "A case carries an id, a corpus path relative to its case file, a question, tags, and an expected block of answer, artifacts, edges, entities, must-cite, must-abstain, must-not-retrieve, time anchor and recency bound.", "why": "", "file": "spec/81-engineering.md", - "line": 277 + "line": 278 }, { "id": "assurance.evaluate.row-reference", @@ -22837,7 +22837,7 @@ "statement": "A case names a row as `
#`, the key joining the row's declared primary-key values with commas, and the runner keys each returned row the same way.", "why": "", "file": "spec/81-engineering.md", - "line": 278 + "line": 279 }, { "id": "assurance.evaluate.regression-case", @@ -22848,7 +22848,7 @@ "statement": "A case tagged `regression` is a regression case, and its must-not-retrieve set scores {{assurance.evaluate.forbidden-row-rate}}.", "why": "", "file": "spec/81-engineering.md", - "line": 279 + "line": 280 }, { "id": "assurance.evaluate.corpus-layout", @@ -22859,7 +22859,7 @@ "statement": "A corpus is a directory holding `contextful.toml`, its table declarations and policy labels, and one `rows/
.jsonl` per table.", "why": "", "file": "spec/81-engineering.md", - "line": 280 + "line": 281 }, { "id": "assurance.evaluate.run-command", @@ -22870,7 +22870,7 @@ "statement": "`contextful eval run --goldens ` loads the case file, lands and folds each referenced corpus into a scratch store at `--landed-at`, the Unix epoch by default, reads every case, and writes the run report.", "why": "because an undated row landed at the epoch predates every question's window, so a replay scores alike on any day", "file": "spec/81-engineering.md", - "line": 281 + "line": 282 }, { "id": "assurance.evaluate.legs", @@ -22881,7 +22881,7 @@ "statement": "Each case calls {{read.retrieve.ranked-call}} three times at limit k: the lexical leg with the question alone, the vector leg with an embedding and an empty query, the hybrid leg with both.", "why": "", "file": "spec/81-engineering.md", - "line": 283 + "line": 284 }, { "id": "assurance.evaluate.stub-embedder", @@ -22892,7 +22892,7 @@ "statement": "The deterministic tier embeds every row and question with a seeded feature-hashing stub embedder; a corpus row or a case carrying its own embedding keeps it.", "why": "", "file": "spec/81-engineering.md", - "line": 284 + "line": 285 }, { "id": "assurance.evaluate.run-verdict", @@ -22903,7 +22903,7 @@ "statement": "A run holds its report to the floors and, given `--baseline`, to that file; either red verdict exits non-zero, and `--update-baseline` applies {{assurance.baseline.raise-only}} on green alone.", "why": "", "file": "spec/81-engineering.md", - "line": 285 + "line": 286 }, { "id": "assurance.evaluate.converter", @@ -22914,7 +22914,7 @@ "statement": "Each benchmark and application ground truth converts once into the case format through its own converter, and the runner reads nothing else.", "why": "", "file": "spec/81-engineering.md", - "line": 286 + "line": 287 }, { "id": "assurance.evaluate.deterministic-tier", @@ -22925,7 +22925,7 @@ "statement": "Recall@k, R-precision, hit-rate@k, reciprocal rank and nDCG@k run with no model call, reported per leg — lexical, vector, hybrid — and per slice.", "why": "", "file": "spec/81-engineering.md", - "line": 287 + "line": 288 }, { "id": "assurance.evaluate.distinct-top-k", @@ -22936,7 +22936,7 @@ "statement": "Membership in the top k is distinct, and a repeated id counts once.", "why": "", "file": "spec/81-engineering.md", - "line": 288 + "line": 289 }, { "id": "assurance.evaluate.absent-truth", @@ -22947,7 +22947,7 @@ "statement": "A ranked metric over a case with no relevant set is NaN and drops out of every aggregate; an empty ranking against a non-empty relevant set scores zero.", "why": "", "file": "spec/81-engineering.md", - "line": 289 + "line": 290 }, { "id": "assurance.evaluate.precision-floor", @@ -22958,7 +22958,7 @@ "statement": "Precision at rank min(k, R), R being the relevant-set size, holds at or above 60 percent.", "why": "because precision@k caps at R/k, below the floor whenever R is small", "file": "spec/81-engineering.md", - "line": 290 + "line": 291 }, { "id": "assurance.evaluate.forbidden-row-rate", @@ -22969,7 +22969,7 @@ "statement": "A row named in a must-not-retrieve set holds at 0 percent of a regression case's returned rows.", "why": "", "file": "spec/81-engineering.md", - "line": 292 + "line": 293 }, { "id": "assurance.evaluate.duplicate-row-rate", @@ -22980,7 +22980,7 @@ "statement": "A repeated table-and-row-key pair holds at 0 percent of the ranking.", "why": "", "file": "spec/81-engineering.md", - "line": 293 + "line": 294 }, { "id": "assurance.evaluate.in-window-rate", @@ -22991,7 +22991,7 @@ "statement": "A case declaring a recency bound holds its in-window rate at or above 95 percent.", "why": "", "file": "spec/81-engineering.md", - "line": 294 + "line": 295 }, { "id": "assurance.evaluate.abstention", @@ -23002,7 +23002,7 @@ "statement": "A must-abstain case returns zero rows.", "why": "", "file": "spec/81-engineering.md", - "line": 295 + "line": 296 }, { "id": "assurance.evaluate.truth-per-surface", @@ -23013,7 +23013,7 @@ "statement": "Expected artifacts score the artifact retriever and expected edges score the edge retriever, reported apart; edge truth with no edge surface configured counts in an unscored tally.", "why": "", "file": "spec/81-engineering.md", - "line": 296 + "line": 297 }, { "id": "assurance.evaluate.judged-dimensions", @@ -23024,7 +23024,7 @@ "statement": "The reading stage adds four judged dimensions: answer accuracy; abstention, as correct-refusal and hallucination-on-unknown rates; citation faithfulness; and temporal correctness against the bitemporal columns.", "why": "", "file": "spec/81-engineering.md", - "line": 297 + "line": 298 }, { "id": "assurance.evaluate.grounded-reader", @@ -23035,7 +23035,7 @@ "statement": "The harness reader answers from retrieved rows alone, and the deterministic tier substitutes a stub reader.", "why": "", "file": "spec/81-engineering.md", - "line": 298 + "line": 299 }, { "id": "assurance.evaluate.model-endpoint", @@ -23046,7 +23046,7 @@ "statement": "The judge and the reader reach a model through {{connector.infer.model-endpoint}}, and the harness holds no credential.", "why": "", "file": "spec/81-engineering.md", - "line": 299 + "line": 300 }, { "id": "assurance.evaluate.judge", @@ -23057,7 +23057,7 @@ "statement": "The judge is one pinned open-weights instruct model, called once per judged item at temperature zero; a judge swap is a full re-baseline.", "why": "because a majority vote at temperature zero samples one mode and adds cost without reducing variance", "file": "spec/81-engineering.md", - "line": 300 + "line": 301 }, { "id": "assurance.evaluate.systems-metrics", @@ -23068,7 +23068,7 @@ "statement": "Tokens per query, latency and cost report beside the quality figures, bucketed by corpus size in tokens relative to the reader's context window.", "why": "", "file": "spec/81-engineering.md", - "line": 302 + "line": 303 }, { "id": "assurance.evaluate.checkpoint", @@ -23079,7 +23079,7 @@ "statement": "A run appends each case's result as JSONL, and a crash re-runs the in-flight case.", "why": "", "file": "spec/81-engineering.md", - "line": 303 + "line": 304 }, { "id": "assurance.evaluate.run-report", @@ -23090,7 +23090,7 @@ "statement": "A run report carries one field per gated metric, the sample count behind each mean, the run block, the per-slice breakdown, and the tally of cases sampled, dropped and unscored.", "why": "", "file": "spec/81-engineering.md", - "line": 304 + "line": 305 }, { "id": "assurance.baseline.file", @@ -23101,7 +23101,7 @@ "statement": "A baseline file carries a reserved run block `{k, tier, model, samples}` and one entry per gated metric: a bare number at the run's dead band, or an object overriding the band.", "why": "", "file": "spec/81-engineering.md", - "line": 315 + "line": 316 }, { "id": "assurance.baseline.default-dead-band", @@ -23112,7 +23112,7 @@ "statement": "A rate-valued entry with no override gates at a 2 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 316 + "line": 317 }, { "id": "assurance.baseline.rank-quality-dead-band", @@ -23123,7 +23123,7 @@ "statement": "The ranked-quality entry gates at a 3 percent dead band.", "why": "", "file": "spec/81-engineering.md", - "line": 317 + "line": 318 }, { "id": "assurance.baseline.band-units", @@ -23134,7 +23134,7 @@ "statement": "A band carries its metric's own units, and a count entry is pinned at zero with no band.", "why": "", "file": "spec/81-engineering.md", - "line": 318 + "line": 319 }, { "id": "assurance.baseline.metric-path", @@ -23145,7 +23145,7 @@ "statement": "An entry names a metric by the report's field path: `retrieval..`, `edge_retrieval..`, `judge.`, `latency_ms`, `n_cases`, or `slices..` before a bare `` or any of these but `latency_ms`.", "why": "", "file": "spec/81-engineering.md", - "line": 319 + "line": 320 }, { "id": "assurance.baseline.sample-count", @@ -23156,7 +23156,7 @@ "statement": "Every mean-valued path takes a `.n` suffix naming its sample count.", "why": "because blanking one case's truth lifts a NaN-filtered mean without moving the case count", "file": "spec/81-engineering.md", - "line": 320 + "line": 321 }, { "id": "assurance.baseline.unresolvable-path", @@ -23167,7 +23167,7 @@ "statement": "A path resolving to nothing, a mean over fewer than 30 cases, a malformed entry or a non-finite band raises `BaselinePathUnresolved` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 322 + "line": 323 }, { "id": "assurance.baseline.run-stamp-drift", @@ -23178,7 +23178,7 @@ "statement": "A run block differing from the run's own configuration raises `BaselineRunStampMismatch` before the suite runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 324 + "line": 325 }, { "id": "assurance.baseline.interval", @@ -23189,7 +23189,7 @@ "statement": "Each judged figure reports a 95 percent percentile-bootstrap interval over cases from 1000 resamples.", "why": "", "file": "spec/81-engineering.md", - "line": 326 + "line": 327 }, { "id": "assurance.baseline.raise-only", @@ -23200,7 +23200,7 @@ "statement": "A baseline update runs after every gate and floor passes, moves each improved entry to its measured value, moves none in the worse direction, and adds no path.", "why": "", "file": "spec/81-engineering.md", - "line": 327 + "line": 328 }, { "id": "assurance.baseline.floors-are-absolute", @@ -23211,7 +23211,7 @@ "statement": "An absolute floor gates independently of every committed value.", "why": "", "file": "spec/81-engineering.md", - "line": 328 + "line": 329 }, { "id": "assurance.baseline.floor-coverage", @@ -23222,7 +23222,7 @@ "statement": "A report lacking a floor's figure on any leg of its `retrieval` surface breaches that floor; only a forbidden-row or in-window rate over no case holds.", "why": "", "file": "spec/81-engineering.md", - "line": 329 + "line": 330 }, { "id": "assurance.baseline.offline", @@ -23233,7 +23233,7 @@ "statement": "The gate is a local command comparing the report against in-tree baselines and floors, reporting both verdicts, with or without a trace store reachable.", "why": "", "file": "spec/81-engineering.md", - "line": 330 + "line": 331 }, { "id": "assurance.baseline.trace-store", @@ -23244,7 +23244,7 @@ "statement": "A trace store records run history and curation staging, and decides no verdict.", "why": "", "file": "spec/81-engineering.md", - "line": 331 + "line": 332 }, { "id": "assurance.baseline.golden-custody", @@ -23255,7 +23255,7 @@ "statement": "Version-controlled JSONL under `evals/cases/` is the canonical golden set, one loader is its only ingestion door, and a reviewer's edit reaches the gate as a change to the tree.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 332 + "line": 333 }, { "id": "assurance.baseline.answer-key-in-the-store", @@ -23266,7 +23266,7 @@ "statement": "An artifact path normalizing under the evaluation directory, reached by a manifest lint or a connector, raises `GoldenSetIngested`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 334 + "line": 335 }, { "id": "assurance.baseline.generated-redaction", @@ -23277,7 +23277,7 @@ "statement": "A golden mined from a live query or promoted from a labeled outcome passes {{authority.redact.write-time}} before commit; a case it cannot redact raises `GoldenRedactionFailed`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 336 + "line": 337 }, { "id": "assurance.baseline.promotion-source", @@ -23288,7 +23288,7 @@ "statement": "Only an adjudicated outcome label promotes into a golden set; a self-rated row does not.", "why": "", "file": "spec/81-engineering.md", - "line": 338 + "line": 339 }, { "id": "assurance.baseline.generators", @@ -23299,7 +23299,7 @@ "statement": "Three generators draft candidate cases from the deployment's store — entity-to-fact-to-edge walks, composed edge chains, and questions about absent entities — and a human approves each before commit.", "why": "", "file": "spec/81-engineering.md", - "line": 339 + "line": 340 }, { "id": "assurance.baseline.held-out", @@ -23310,7 +23310,7 @@ "statement": "Retrieval and synthesis are tuned on no gate input.", "why": "", "file": "spec/81-engineering.md", - "line": 340 + "line": 341 }, { "id": "assurance.baseline.rotation", @@ -23321,7 +23321,7 @@ "statement": "The native set rotates on a cadence, a sample of its truth is reviewed on each rotation, and a comparison across generations resolves through the pinned run stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 341 + "line": 342 }, { "id": "assurance.baseline.native-gate", @@ -23332,7 +23332,7 @@ "statement": "A native benchmark generated from a real corpus is the red or green gate; public benchmark sets run beside it as held-out comparison and decide nothing.", "why": "", "file": "spec/81-engineering.md", - "line": 342 + "line": 343 }, { "id": "assurance.baseline.trace-export", @@ -23343,7 +23343,7 @@ "statement": "A run touching a deployed store with a hosted trace endpoint configured raises `TraceExportOutOfPerimeter`; hosted collection serves runs over public or synthetic fixtures alone.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 343 + "line": 344 }, { "id": "assurance.measure.ledger", @@ -23354,7 +23354,7 @@ "statement": "`evals/ledger.toml` holds one entry per tracked target: an id, an owning clause id, a run-report metric path, a tier, a method naming one integration test, case set or probe, and a threshold.", "why": "", "file": "spec/81-engineering.md", - "line": 352 + "line": 353 }, { "id": "assurance.measure.unresolved-entry", @@ -23365,7 +23365,7 @@ "statement": "An entry whose owning clause, method or metric path resolves to nothing raises `MeasureEntryUnresolved` before any measure runs.", "why": "P7", "file": "spec/81-engineering.md", - "line": 353, + "line": 354, "scenarios": [ "WHEN an entry names `run.journal.entry-keys`, THEN the run raises `MeasureEntryUnresolved` and no test runs." ] @@ -23379,7 +23379,7 @@ "statement": "An entry naming an issue in place of a method reports open and gates nothing, and `evals/ledger.md` carries every entry's computed status.", "why": "", "file": "spec/81-engineering.md", - "line": 355, + "line": 356, "scenarios": [ "WHEN an entry's method is `{ issue = 43 }`, THEN `evals/ledger.md` lists it open and the evaluate stage ignores it." ] @@ -23393,7 +23393,7 @@ "statement": "A gate-tier entry decides the evaluate stage, a trend-tier entry is measured on the default-branch run and decides nothing, and a scheduled-tier entry runs on the scheduled job alone.", "why": "", "file": "spec/81-engineering.md", - "line": 356 + "line": 357 }, { "id": "assurance.measure.count-first", @@ -23404,7 +23404,7 @@ "statement": "A gate-tier entry measures a count, a ratio within one run or a size under a locked resolve; a wall-clock or resident-memory figure is trend-tier.", "why": "because a shared container moves wall-clock figures past any band narrow enough to catch a regression", "file": "spec/81-engineering.md", - "line": 357 + "line": 358 }, { "id": "assurance.measure.record", @@ -23415,7 +23415,7 @@ "statement": "A measure writes one JSON record carrying its entry id, value, sample count, seed and run stamp; a gate-tier method finishing without one raises `MeasureRecordMissing`.", "why": "P7", "file": "spec/81-engineering.md", - "line": 359 + "line": 360 }, { "id": "assurance.measure.seeded", @@ -23426,7 +23426,7 @@ "statement": "Every generated fixture and randomized schedule derives from the record's seed, and replaying that seed reproduces a count-valued entry's value.", "why": "because a failure that cannot replay cannot be fixed", "file": "spec/81-engineering.md", - "line": 361 + "line": 362 }, { "id": "assurance.measure.seed-mismatch", @@ -23437,7 +23437,7 @@ "statement": "A gate or scheduled record whose seed differs from its entry's declared seed raises `MeasureSeedMismatch` and counts as red; a trend mismatch leaves no comparable record.", "why": "because a figure measured under another seed replays nothing the ledger names", "file": "spec/81-engineering.md", - "line": 363 + "line": 364 }, { "id": "assurance.measure.timing-iterations", @@ -23448,7 +23448,7 @@ "statement": "A timed batch reports p50 and p95 over 200 repeats of its operation, each after warm-up.", "why": "", "file": "spec/81-engineering.md", - "line": 365 + "line": 366 }, { "id": "assurance.measure.timing-batches", @@ -23459,7 +23459,7 @@ "statement": "A timed figure is the median of 5 repeats of its timed batch.", "why": "", "file": "spec/81-engineering.md", - "line": 366 + "line": 367 }, { "id": "assurance.measure.trend-direction", @@ -23470,7 +23470,7 @@ "statement": "Each trend-tier ledger entry declares `higher_is_better` or `lower_is_better` for its metric.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 367 + "line": 368 }, { "id": "assurance.measure.trend-baseline", @@ -23481,7 +23481,7 @@ "statement": "A trend figure compares with the newest earlier successful default-branch report for the same entry and seed under {{assurance.measure.runner-stamp}}; absent matching history yields no annotation.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 369, + "line": 370, "scenarios": [ "WHEN a newer failed report follows two successful matches, THEN the trend compares with the newer successful match and the stage passes." ] @@ -23495,7 +23495,7 @@ "statement": "A trend figure more than 25 percent worse than its baseline annotates the run report and fails no stage.", "why": "", "file": "spec/81-engineering.md", - "line": 371, + "line": 372, "scenarios": [ "WHEN a trend p95 moves from 40 ms to 52 ms on a matching runner stamp, THEN the report carries a +30 percent annotation and the stage passes." ] @@ -23509,7 +23509,7 @@ "statement": "The run block carries the runner's processor model, processor count and memory limit, and a trend figure compares only against a baseline with the same stamp.", "why": "", "file": "spec/81-engineering.md", - "line": 372 + "line": 373 }, { "id": "assurance.measure.absolute-threshold", @@ -23520,7 +23520,7 @@ "statement": "A threshold is an absolute figure of this system's own measure, and it tightens only through {{assurance.baseline.raise-only}}.", "why": "", "file": "spec/81-engineering.md", - "line": 373 + "line": 374 }, { "id": "assurance.measure.history", @@ -23531,7 +23531,7 @@ "statement": "A run on the default branch attaches its run report to the measured commit under `refs/notes/measures`, and no verdict reads a note.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 374 + "line": 375 }, { "id": "assurance.measure.scheduled", @@ -23542,7 +23542,7 @@ "statement": "The FlareDispatch nightly run executes every tier against the default-branch head and reports one check per run.", "why": "", "file": "spec/81-engineering.md", - "line": 376 + "line": 377 }, { "id": "assurance.measure.measured-basis", @@ -23553,7 +23553,7 @@ "statement": "A bound with a measured basis names a trend-tier or scheduled-tier ledger entry id as its benchmark.", "why": "", "file": "spec/81-engineering.md", - "line": 377 + "line": 378 }, { "id": "assurance.release.version", @@ -23564,7 +23564,7 @@ "statement": "A release tag is `v0..`: `` counts the milestones computing `closed` in the tagged commit's `spec/status.md`, and `` counts the existing tags `v0..*`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 407, + "line": 408, "scenarios": [ "WHEN milestones 0 to 7 and 11 compute `closed` and `v0.9.0` exists, THEN the computed version is `v0.9.1`." ] @@ -23578,7 +23578,7 @@ "statement": "`contextful-ci tag` creates a signed annotated tag on `HEAD` naming the closed milestones once every refusal of this operation clears, and never moves or replaces an existing tag.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 409 + "line": 410 }, { "id": "assurance.release.dirty-tree", @@ -23589,7 +23589,7 @@ "statement": "A working tree or index differing from `HEAD` raises `TagTreeDirty`.", "why": "because the gate then judges files the tagged commit does not hold", "file": "spec/81-engineering.md", - "line": 411 + "line": 412 }, { "id": "assurance.release.off-branch", @@ -23600,7 +23600,7 @@ "statement": "A `HEAD` unreachable from the default branch, `origin/HEAD` unless `--branch` names another, raises `TagOffDefaultBranch`.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 413 + "line": 414 }, { "id": "assurance.release.version-regressed", @@ -23611,7 +23611,7 @@ "statement": "A computed version below the highest existing `v..` tag raises `TagVersionRegressed`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 415, + "line": 416, "scenarios": [ "WHEN milestone 11 reopens after `v0.9.0`, THEN the computed `v0.8.0` raises `TagVersionRegressed`." ] @@ -23625,7 +23625,7 @@ "statement": "A `[workspace.package]` version in `HEAD`'s `Cargo.toml` that is absent or differs from the computed version raises `TagWorkspaceVersionMismatch`, naming both.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 417 + "line": 418 }, { "id": "assurance.release.gate-failed", @@ -23636,7 +23636,7 @@ "statement": "Once the other refusals clear, every gate stage runs against `HEAD` with `HEAD~1` as base; a failing stage raises `TagGateFailed`, naming the stage's refusal, and no tag is created.", "why": "A-assurance", "file": "spec/81-engineering.md", - "line": 419 + "line": 420 }, { "id": "assurance.release.release-run", @@ -23647,7 +23647,7 @@ "statement": "A `v*` tag at a default-branch commit starts FlareDispatch `contextful-release`, with one `contextful-release-cell` child per {{assurance.build.targets}} matrix cell and one `contextful-release-formula` child for their metadata.", "why": "because distributed builds and formula generation share one tag identity", "file": "spec/81-engineering.md", - "line": 421 + "line": 422 }, { "id": "assurance.release.release-dry-run", @@ -23658,7 +23658,7 @@ "statement": "A manual `contextful-release` dry run builds the release assets and publishes neither a GitHub release nor container images.", "why": "because a packaging rehearsal must not claim a released version", "file": "spec/81-engineering.md", - "line": 423 + "line": 424 } ], "owns": { @@ -25234,6 +25234,10 @@ "assurance.structure-tree.mirror-exemption", "assurance.structure-tree.duplicated-capability" ], + [ + "assurance.test.test-first-scope", + "assurance.gate.test-first-parts" + ], [ "assurance.test.refactor-trailer", "assurance.test.test-first" diff --git a/spec/status.md b/spec/status.md index bc3101951..a591a980c 100644 --- a/spec/status.md +++ b/spec/status.md @@ -5,15 +5,15 @@ An unpinned clause is `committed`; a pinned one is `performed` when its Rust, Le | Contract | Files | Operations | Clauses | Refusals | Limits | Unsettled | Performed | Broken | Floor | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| `assurance` | 2 | 15 | 237 | 59 | 22 | 14 | 145 | 0 | 141 | +| `assurance` | 2 | 15 | 237 | 59 | 22 | 14 | 145 | 0 | 145 | | `authority` | 2 | 16 | 270 | 77 | 27 | 10 | 207 | 0 | 207 | | `connector` | 2 | 14 | 349 | 89 | 21 | 15 | 229 | 0 | 229 | | `corpus` | 1 | 10 | 79 | 37 | 7 | 0 | 43 | 0 | 43 | | `disclosure` | 3 | 16 | 134 | 66 | 13 | 3 | 63 | 0 | 63 | -| `read` | 2 | 15 | 206 | 35 | 22 | 9 | 183 | 0 | 178 | +| `read` | 2 | 15 | 206 | 35 | 22 | 9 | 183 | 0 | 183 | | `run` | 3 | 26 | 406 | 102 | 39 | 28 | 306 | 0 | 306 | | `store` | 1 | 17 | 252 | 59 | 13 | 19 | 216 | 0 | 216 | -| `surface` | 2 | 20 | 123 | 56 | 22 | 0 | 85 | 0 | 66 | +| `surface` | 2 | 20 | 123 | 56 | 22 | 0 | 85 | 0 | 85 | | `topology` | 1 | 6 | 70 | 24 | 5 | 5 | 38 | 0 | 38 | | **total** | 19 | 155 | 2126 | 604 | 191 | 103 | 1515 | 0 | | diff --git a/tools/ci/tests/integration/test_first.rs b/tools/ci/tests/integration/test_first.rs index c7729d1c0..cd64e0ee9 100644 --- a/tools/ci/tests/integration/test_first.rs +++ b/tools/ci/tests/integration/test_first.rs @@ -138,7 +138,7 @@ fn each_changed_test_package_has_its_own_dispatch_part() { assert!(validation.status.success(), "{}", stderr(&validation)); } -// spec: assurance.gate.test-first-parts@0bd45e25 +// spec: assurance.gate.test-first-parts@268f02f4 #[test] fn a_source_package_requires_its_own_changed_test_when_another_package_is_red() { let r = Repo::init(); @@ -210,9 +210,9 @@ fn a_new_package_counts_red_and_leaves_the_base_workspace_loadable() { assert!(!err.contains("failed to load manifest"), "{err}"); } -/// The test-first stage builds each changed test file's target against the base source with every feature enabled, +/// For each changed source package, test-first builds each changed test file's target against the base source with every feature enabled, /// then runs exactly the tests under that file's top-level module; a target failing to compile there counts as failing. -// spec: assurance.test.test-first-scope@38d52656 +// spec: assurance.test.test-first-scope@0ff77a1e #[test] fn only_the_changes_test_modules_run_against_the_base() { let r = Repo::init(); From 376e07214de90e4b51519cb42f34847e19373e3a Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 03:38:33 +0800 Subject: [PATCH 152/155] Read exchange test responses by Content-Length The exchange integration client accepts a complete framed response before a later TCP reset and refuses truncated, ambiguous or oversized frames. Server close and admission semantics remain unchanged. --- .../contextful-cli/tests/integration/serve.rs | 88 +++++++++++++++++-- 1 file changed, 82 insertions(+), 6 deletions(-) diff --git a/crates/contextful-cli/tests/integration/serve.rs b/crates/contextful-cli/tests/integration/serve.rs index 01f436f66..277fdeecd 100644 --- a/crates/contextful-cli/tests/integration/serve.rs +++ b/crates/contextful-cli/tests/integration/serve.rs @@ -14,6 +14,85 @@ use std::sync::atomic::{AtomicU64, Ordering}; const AUD: &str = "contextful://acme-research"; +/// One bounded Content-Length response ends at its body, so a later TCP reset has +/// no bearing on the HTTP answer; a reset or EOF inside the frame remains an error. +fn exchange_response(stream: &mut impl Read) -> std::io::Result { + let invalid = |message| std::io::Error::new(std::io::ErrorKind::InvalidData, message); + let mut raw = Vec::new(); + while !raw.ends_with(b"\r\n\r\n") { + if raw.len() == 8192 { return Err(invalid("the test response head exceeds 8 KiB")); } + let mut byte = [0]; + stream.read_exact(&mut byte)?; + raw.push(byte[0]); + } + let head = std::str::from_utf8(&raw).map_err(|_| invalid("the response head is not UTF-8"))?; + let mut lines = head.split("\r\n"); + let mut status = lines.next().unwrap_or_default().split_whitespace(); + if status.next() != Some("HTTP/1.1") || + !status.next().is_some_and(|code| code.len() == 3 && code.bytes().all(|b| b.is_ascii_digit())) { + return Err(invalid("the response status does not parse")); + } + let mut length = None; + for line in lines.filter(|line| !line.is_empty()) { + let (name, value) = line.split_once(':').ok_or_else(|| invalid("the response header does not parse"))?; + if name.eq_ignore_ascii_case("Transfer-Encoding") { + return Err(invalid("the test response requires Content-Length framing")); + } + if name.eq_ignore_ascii_case("Content-Length") { + let value = value.trim(); + if length.is_some() || value.is_empty() || !value.bytes().all(|b| b.is_ascii_digit()) { + return Err(invalid("the response length is ambiguous or malformed")); + } + length = Some(value.parse::().map_err(|_| invalid("the response length exceeds usize"))?); + } + } + let length = length.ok_or_else(|| invalid("the response has no Content-Length"))?; + if length > 1024 * 1024 { return Err(invalid("the test response body exceeds 1 MiB")); } + let start = raw.len(); + raw.resize(start + length, 0); + stream.read_exact(&mut raw[start..])?; + String::from_utf8(raw).map_err(|_| invalid("the response is not UTF-8")) +} + +struct ResetAfter(std::io::Cursor>); + +impl Read for ResetAfter { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + match self.0.read(out)? { + 0 => Err(std::io::ErrorKind::ConnectionReset.into()), + n => Ok(n), + } + } +} + +#[test] +fn exchange_response_reads_one_complete_frame_before_a_later_reset() { + let wire = b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n\r\n{}"; + assert_eq!(exchange_response(&mut &wire[..]).unwrap(), std::str::from_utf8(wire).unwrap()); + assert_eq!(exchange_response(&mut ResetAfter(std::io::Cursor::new(wire.to_vec()))).unwrap(), std::str::from_utf8(wire).unwrap()); +} + +#[test] +fn exchange_response_refuses_truncated_headers_and_bodies() { + for wire in [b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n".as_slice(), + b"HTTP/1.1 404 Not Found\r\nContent-Length: 2\r\n\r\n{".as_slice()] { + assert!(exchange_response(&mut &wire[..]).is_err(), "EOF inside a frame refuses"); + assert!(exchange_response(&mut ResetAfter(std::io::Cursor::new(wire.to_vec()))).is_err(), "a reset inside a frame refuses"); + } +} + +#[test] +fn exchange_response_refuses_ambiguous_or_unbounded_framing() { + for head in ["not-http\r\nContent-Length: 0", "HTTP/1.1 404 Not Found", + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nContent-Length: 0", + "HTTP/1.1 404 Not Found\r\nContent-Length: -1", + "HTTP/1.1 404 Not Found\r\nContent-Length: 1048577", + "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nTransfer-Encoding: chunked"] { + assert!(exchange_response(&mut format!("{head}\r\n\r\n").as_bytes()).is_err(), "{head}"); + } + assert!(exchange_response(&mut vec![b'x'; 8193].as_slice()).is_err()); +} + fn run(dir: &Path, args: &[&str]) -> Output { Command::new(env!("CARGO_BIN_EXE_contextful")) .args(args) @@ -114,8 +193,7 @@ fn served_exchange_without_policy_returns_the_unconfigured_refusal() { let body = "not-json"; let mut stream = TcpStream::connect(&addr).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).unwrap(); + let raw = exchange_response(&mut stream).unwrap(); let (head, data) = raw.split_once("\r\n\r\n").unwrap(); assert_eq!(head.split(' ').nth(1), Some("404")); let answer: Value = serde_json::from_str(data).unwrap(); @@ -123,8 +201,7 @@ fn served_exchange_without_policy_returns_the_unconfigured_refusal() { let mut stream = TcpStream::connect(&addr).unwrap(); stream.set_read_timeout(Some(std::time::Duration::from_secs(2))).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: 100\r\n\r\n").unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).expect("unconfigured exchange answers before waiting for its body"); + let raw = exchange_response(&mut stream).expect("unconfigured exchange answers before waiting for its body"); assert!(raw.starts_with("HTTP/1.1 404"), "{raw}"); } @@ -149,8 +226,7 @@ fn served_exchange_mints_a_reader_credential_for_the_read_face() { let body = json!({ "jwt": jwt }).to_string(); let mut stream = TcpStream::connect(&addr).unwrap(); write!(stream, "POST /auth/exchange HTTP/1.1\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", body.len()).unwrap(); - let mut raw = String::new(); - stream.read_to_string(&mut raw).unwrap(); + let raw = exchange_response(&mut stream).unwrap(); let (head, data) = raw.split_once("\r\n\r\n").unwrap(); (head.split(' ').nth(1).unwrap().parse::().unwrap(), serde_json::from_str::(data).unwrap()) }; From ef99410420b7c1878e6161960831d360367feb5d Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Wed, 7 Oct 2026 05:29:53 +0800 Subject: [PATCH 153/155] CI fixture repositories carry a local Git identity --- tools/ci/tests/integration/main.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/ci/tests/integration/main.rs b/tools/ci/tests/integration/main.rs index 236d5092b..fa27c2d06 100644 --- a/tools/ci/tests/integration/main.rs +++ b/tools/ci/tests/integration/main.rs @@ -40,6 +40,9 @@ impl Repo { let root = dir.path().to_path_buf(); let r = Repo { _dir: dir, root }; r.git(&["init", "-q", "-b", "main"]); + r.git(&["config", "--local", "user.name", "t"]); + r.git(&["config", "--local", "user.email", "t@example.com"]); + r.git(&["config", "--local", "commit.gpgsign", "false"]); r.write("Cargo.toml", "[workspace]\nresolver = \"2\"\nmembers = [\"crates/*\"]\n"); r.write("crates/demo/Cargo.toml", &manifest("demo", "")); r.write("crates/demo/src/lib.rs", "pub fn double(x: i32) -> i32 {\n x * 2\n}\n"); @@ -64,7 +67,6 @@ impl Repo { pub fn git(&self, args: &[&str]) -> String { let out = Command::new("git") - .args(["-c", "commit.gpgsign=false", "-c", "user.name=t", "-c", "user.email=t@example.com"]) .args(args) .current_dir(&self.root) .output() From 3717295a6260ed68a051fdfb5d4173e87d109363 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 04:22:59 +0800 Subject: [PATCH 154/155] Evaluation requires formal tools and read fixtures bound planning --- .../tests/integration/read/respond.rs | 5 +-- tools/ci/src/main.rs | 6 +++- tools/ci/tests/integration/stages.rs | 35 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/crates/contextful-context/tests/integration/read/respond.rs b/crates/contextful-context/tests/integration/read/respond.rs index cf117ea0c..20c3a1552 100644 --- a/crates/contextful-context/tests/integration/read/respond.rs +++ b/crates/contextful-context/tests/integration/read/respond.rs @@ -14,8 +14,9 @@ fn restriction(r: &Response) -> &Value { } fn expensive_statement() -> String { - let tables = (0..14).map(|i| format!("\"research/notes\" t{i}")).collect::>().join(", "); - format!("SELECT sum(random()) FROM {tables}") + // Four materialized arms execute 96^4 random evaluations under the statement deadline. + let rows = (0..32).map(|_| "SELECT note_id FROM \"research/notes\"").collect::>().join(" UNION ALL "); + format!("WITH work AS MATERIALIZED ({rows}) SELECT sum(random()) FROM work t0, work t1, work t2, work t3") } // spec: read.respond.duration-ceiling@cfa152d7 diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 100f20cca..ba2cbb167 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -469,7 +469,11 @@ fn run_stage(root: &Path, stage: &str, only: Option<&[String]>, base: &str, boun workspace(root, only)? } "acceptance" => acceptance(root)?, - "evaluate" => measure::evaluate(root)?, + "evaluate" => { + provision_lean(root)?; + provision_wasm(root)?; + measure::evaluate(root)? + } "features" => features(root, only)?, "crate-graph" => { committed_lock(root)?; diff --git a/tools/ci/tests/integration/stages.rs b/tools/ci/tests/integration/stages.rs index b6b87b466..812a7bc72 100644 --- a/tools/ci/tests/integration/stages.rs +++ b/tools/ci/tests/integration/stages.rs @@ -289,6 +289,41 @@ fn declared_derived_artifact_is_checked_by_schema_stage() { assert!(err.contains("schema: spec/custom.md differs from its regeneration"), "{err}"); } +#[test] +fn the_evaluate_stage_provisions_required_lean_and_wasm_before_measuring() { + const PIN: &str = "leanprover/lean4:fixture"; + const MEASURED: &str = r###" +#[test] +fn required_tools() { + assert_eq!(std::env::var("CONTEXTFUL_REQUIRE_LEAN").as_deref(), Ok("1")); + assert_eq!(std::env::var("CONTEXTFUL_REQUIRE_WASM").as_deref(), Ok("1")); + assert_eq!(std::env::var("ELAN_TOOLCHAIN").as_deref(), Ok("leanprover/lean4:fixture")); + let lean = std::process::Command::new("lean").arg("--version").output().unwrap(); + assert!(lean.status.success()); + assert_eq!(String::from_utf8(lean.stdout).unwrap().trim(), "fixture Lean"); + let dir = std::path::PathBuf::from(std::env::var_os("CONTEXTFUL_MEASURE_DIR").unwrap()); + std::fs::write(dir.join("required-tools.json"), r##"{"id":"required-tools","value":1,"n":1,"seed":7,"run":{"processor":"t","nproc":1,"memory_limit":null}}"##).unwrap(); +} +"###; + let r = Repo::init(); + r.write("formal/lean-toolchain", &format!("{PIN}\n")); + r.write("spec/spec.lock.json", "{\"clauses\":[{\"id\":\"run.journal.entry-key\"}]}\n"); + r.write("evals/ledger.toml", "[entry.required-tools]\nclause = \"run.journal.entry-key\"\nmetric = \"tools.ready\"\nkind = \"test\"\ntier = \"gate\"\nmethod = { test = \"demo::measured::required_tools\" }\ntarget = { op = \">=\", value = 1 }\nseed = 7\n"); + r.write("crates/demo/tests/integration/main.rs", "mod double;\nmod measured;\n"); + r.write("crates/demo/tests/integration/measured.rs", MEASURED); + r.commit("a measurement requiring Lean and WebAssembly"); + let bin = Bin::new(); + bin.fake("elan", "echo \"elan $*\" >> \"$CALLS\"\n[ \"$*\" = 'toolchain list' ] || exit 1\necho leanprover/lean4:fixture\n"); + bin.fake("lean", "[ \"$ELAN_TOOLCHAIN\" = leanprover/lean4:fixture ] || exit 1\necho 'fixture Lean'\n"); + let elan = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(bin.dir.path(), elan.path().join("bin")).unwrap(); + let o = gate_env(&r, Some(&bin), &[("ELAN_HOME", elan.path().to_str().unwrap())], &["--stage", "evaluate"]); + let err = stderr(&o); + assert!(o.status.success(), "{err}"); + assert!(err.contains("measure: required-tools = 1"), "{err}"); + assert_eq!(bin.calls(), ["elan toolchain list"]); +} + /// The evaluate stage runs every gate-tier ledger entry and the native case set in the deterministic tier, and reports the floor and baseline verdicts. // spec: assurance.gate.evaluate-stage@316d20b1 #[test] From 74b7891551492a406aa765cd3852154a376391dc Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Thu, 8 Oct 2026 23:44:56 +0800 Subject: [PATCH 155/155] Query redacts reader secrets and permits pinned esbuild --- apps/console/pnpm-workspace.yaml | 5 +++++ apps/console/src/live.ts | 28 +++++++++++++++++++++++----- apps/console/test/live.test.ts | 29 +++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 apps/console/pnpm-workspace.yaml diff --git a/apps/console/pnpm-workspace.yaml b/apps/console/pnpm-workspace.yaml new file mode 100644 index 000000000..2f24ed578 --- /dev/null +++ b/apps/console/pnpm-workspace.yaml @@ -0,0 +1,5 @@ +packages: + - . + +allowBuilds: + esbuild@0.25.12: true diff --git a/apps/console/src/live.ts b/apps/console/src/live.ts index 18bab497c..0be43193f 100644 --- a/apps/console/src/live.ts +++ b/apps/console/src/live.ts @@ -1,9 +1,23 @@ import type { StoreEntry } from "../../gateway/src/index.ts"; import type { TurnInput, TurnResult } from "./index.ts"; -import { ConsoleError, createTurn, resolveReaderCredential, type Source, type ToolResult } from "./turn.ts"; +import { ConsoleError, createTurn, resolveReaderCredential, StreamingRedactor, type Source, type ToolResult } from "./turn.ts"; export type LiveOptions = { stores: StoreEntry[]; env: NodeJS.ProcessEnv; fetcher?: typeof fetch }; +export function consoleDenylist(env: NodeJS.ProcessEnv): string[] { + if (!env.CONTEXTFUL_CONSOLE_DENYLIST) return []; + const parsed: unknown = JSON.parse(env.CONTEXTFUL_CONSOLE_DENYLIST); + if (!Array.isArray(parsed) || !parsed.every((entry) => typeof entry === "string")) throw new Error("ConsoleDenylistMalformed"); + return parsed; +} + +export function redactText(value: string, denylist: string[]): string { + const long = denylist.filter((entry) => entry.length > 128); + const exact = long.reduce((text, entry) => text.replaceAll(entry, "[redacted]"), value); + const redactor = new StreamingRedactor(denylist.filter((entry) => entry.length <= 128)); + return redactor.push(exact) + redactor.finish(); +} + function record(value: unknown): value is Record { return value !== null && typeof value === "object" && !Array.isArray(value); } @@ -84,7 +98,7 @@ export async function openReader({ stores, env, fetcher = fetch }: LiveOptions, return message.result.structuredContent; }; - return { store, call }; + return { store, call, credential }; } export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Promise { @@ -92,14 +106,17 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom const modelEndpoint = env.CONTEXTFUL_MODEL_ENDPOINT; const modelId = env.CONTEXTFUL_MODEL_ID; if (!modelEndpoint || !modelId) throw new Error("ConsoleModelUnconfigured"); + const configuredDenylist = consoleDenylist(env); return async (input) => { - const { store, call } = await openReader(options, input); + const { store, call, credential } = await openReader(options, input); + const denylist = [...configuredDenylist, credential, ...(input.operator.assertion ? [input.operator.assertion] : [])]; const description = await call("context.describe", {}); const selected = selectTable(input.question, description.tables); if (!selected) throw new ConsoleError("ConsoleUngroundedAnswer", "No data table matches this question."); const tables = [selected]; let remainingSources = 8; const turn = createTurn({ + denylist: denylist.filter((entry) => entry.length <= 128), tools: tables.map((table) => ({ name: `read:${table}`, pack: "data", kind: "read" as const, table })), tables: tables.map((name) => ({ name, kind: "data" as const })), planner: async ({ previous }) => previous.length ? [] : tables.map((table) => ({ tool: `read:${table}`, arguments: {} })), @@ -112,7 +129,8 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom remainingSources -= sourced.length; return { rows: sourced.map(({ row }) => Object.fromEntries(columns.map((column, index) => [column, row[index]]))), - sources: sourced.map(({ citation }) => citation), + sources: sourced.map(({ citation }) => ({ ...citation, label: redactText(citation.label, denylist), + url: citation.url && !denylist.some((entry) => entry && citation.url!.includes(entry)) ? citation.url : undefined })), }; } }, synthesize: async function* ({ question, results, sources }) { @@ -128,7 +146,7 @@ export function createLiveTurn(options: LiveOptions): (input: TurnInput) => Prom const choices = record(body) && Array.isArray(body.choices) ? body.choices : []; const message = choices.length && record(choices[0]) ? choices[0].message : undefined; if (!record(message) || typeof message.content !== "string") throw new ConsoleError("ConsoleModelUnavailable", "model answer absent", 503); - yield message.content; + yield redactText(message.content, denylist); }, }); const answer = await turn.ask({ question: input.question, packs: ["data"], store: store.id }); diff --git a/apps/console/test/live.test.ts b/apps/console/test/live.test.ts index 5e03361c3..c345c4f45 100644 --- a/apps/console/test/live.test.ts +++ b/apps/console/test/live.test.ts @@ -5,6 +5,35 @@ import { ConsoleError } from "../src/turn.ts"; import { createConsole, issueCognitoSession } from "../src/index.ts"; import type { StoreEntry } from "../../gateway/src/index.ts"; +test("Query redacts long denylisted values and credentials from prose and source URLs", async () => { + const assertion = "assertion-" + "a".repeat(512); + const credential = "credential-" + "c".repeat(512); + const denied = "private-" + "p".repeat(256); + const fetcher: typeof fetch = async (input, init) => { + const url = String(input); + if (url.endsWith("/auth/exchange")) return Response.json({ token: credential }); + if (url.endsWith("/mcp")) { + const body = JSON.parse(String(init?.body)); + const value = body.params.name === "context.describe" ? { tables: [{ table: "filings", kind: "data" }] } : { + columns: ["filing_id", "summary", "source_url"], + rows: [["filing-1", "A filing arrived", `https://example.test/${credential}`]], + }; + return Response.json({ result: { structuredContent: value } }); + } + return Response.json({ choices: [{ message: { content: `A filing arrived [filing-1]. ${assertion} ${credential} ${denied}` } }] }); + }; + const turn = createLiveTurn({ stores: [store], env: { + CONTEXTFUL_MODEL_ENDPOINT: "https://model.example/v1", CONTEXTFUL_MODEL_ID: "fixture", + CONTEXTFUL_CONSOLE_DENYLIST: JSON.stringify([denied]), + }, fetcher }); + const result = await turn({ operator: { subject: "reader", grants: new Set(["query"]), assertion }, + store: store.id, question: "Which filing arrived?" }); + const visible = JSON.stringify(result); + for (const secret of [assertion, credential, denied]) assert.equal(visible.includes(secret), false); + assert.match(result.answer, /A filing arrived \[source-1\]/); + assert.equal((result.sources[0] as { url?: string }).url, undefined); +}); + const store: StoreEntry = { id: "field-notes", label: "Field notes", endpoint: "https://store.example", auth: "exchange", exchangeRoute: "/auth/exchange", credentialName: "FIELD_NOTES_QUERY_TOKEN",