From af134506416e031f8237d7c8979637f2c77f1b3e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:06:45 -0800 Subject: [PATCH 1/7] chore(quest): claim quest/m0/routing-evidence/charts.md Quest-Claim: 0f2f37390dd71883 From c9b6974ba97adbc0a1ea1c10ad45aefd1f3fee06 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:23:39 -0800 Subject: [PATCH 2/7] feat(usage)!: cost and first rounds per writer and reviewer cell, for the routing charts quest usage --json now carries every measure the cost-against-quality charts plot: each implementer's writing cost over the merged PRs it was found on, a local reviewer rollup with tallies and reviewing cost per PR, and first-round findings, which quest review's tally now records. A sub-agent that names no PR's branch takes its parent's PR, and its parent's role there, so a review's sub-agents count as reviewing. The model check carries the chart procedure and the published benchmark table; the charts quest is done. Co-Authored-By: Claude Opus 5.5 --- README.md | 34 +++- quest/m0/routing-evidence/README.md | 11 +- quest/m0/routing-evidence/charts.md | 76 ------- quest/m1/model-check.md | 93 +++++++-- src/cli/main.ts | 9 +- src/cli/review-record.ts | 60 ++++-- src/cli/review.ts | 10 +- src/cli/usage.ts | 305 ++++++++++++++++++++++------ test/review.test.ts | 10 +- test/usage.test.ts | 71 ++++++- 10 files changed, 478 insertions(+), 201 deletions(-) delete mode 100644 quest/m0/routing-evidence/charts.md diff --git a/README.md b/README.md index 4932833a..b42c2903 100644 --- a/README.md +++ b/README.md @@ -1440,9 +1440,11 @@ names the reviewed head and the base's short SHA, then for each reviewer its harness, model, and effort, its Reviewer line ending in `, shadow` for the shadow, the session and round, the score and rationale, its tally, each finding with its location, and the last round's findings with their dispositions. The -tally, as `- Rounds: 3; findings 4 (high 1, medium 3): fixed 3, refuted 1, open 0`, -counts the reviewer's rounds on the branch and each finding it raised once, -by its latest severity, however many rounds keep it open. A finding counts +tally, as `- Rounds: 3; first round 2 (medium 2); findings 4 (high 1, medium 3): fixed 3, refuted 1, open 0`, +counts the reviewer's rounds on the branch, its first round's findings as that +round graded them, and each finding it raised once, +by its latest severity, however many rounds keep it open. A tally begun before +it kept first rounds leaves that part out. A finding counts for the reviewer that raised it when a new size pick's reviewer fixes or refutes it, but one that reviewer keeps open becomes its own, since its round reports it again. `--new` drops the findings it leaves unsettled from the @@ -1504,10 +1506,13 @@ prompt or reply. Since another repository may use the same branch name, the session must also be this repository's, as the repositories token-max resolved for it on its own host name, or, once its checkout is deleted, those resolved there for another session in that exact directory, as a Codex log records -(never a parent's, which a nested clone may not share); otherwise it stays +(never a parent's, which a nested clone may not share). A sub-agent or +spawned session that names no PR's branch, as one in a detached checkout, +takes the PR of the session that spawned it; otherwise it stays unattributed. A session is reviewing when the PR's `## Review` record names -it or Codex marks it a review, implementing when its first request since -`--since` came before the PR opened, and iterating otherwise, so a session +it or Codex marks it a review, its parent's role when both are on the same +PR, so a review's sub-agents count as reviewing, implementing when its first +request since `--since` came before the PR opened, and iterating otherwise, so a session that opened a PR before `--since` and kept working on it reads as iterating. Each PR shows its `## Models` size and implementers, and as `unread:` each Implementer line it cannot read; each reviewer the `quest review` record names, with its latest round and score and its tally of findings across every round, marked `shadow` for the shadow, where a merged PR whose last round left only low @@ -1534,15 +1539,26 @@ a bot resolving its own thread or a later review leaving it out, and as unanswered otherwise. Rollups follow: spend by size, role, harness, model, and effort; accuracy -by size and implementer, which sums the tallies but the shadow's, so it counts +by size and implementer, which sums the tallies but the shadow's, and their +first rounds over the PRs whose every such tally records one, so it counts every finding the routed reviewers caught before merge, while the shadow's findings measure an effort step rather than the writer. Since a branch's tallies span every PR from it, only the PR whose record holds its latest tally, by the most rounds and then the newest, counts them, and a record written before tallies counts as unreviewed. An `Implementer:` line's own size, as -`(iterate, XS)`, counts over the PR's; and, over the merged PRs, each online +`(iterate, XS)`, counts over the PR's. Each implementer also carries its +writing sessions' cost on the merged PRs they were found on, so cost per merged +PR divides by those PRs, not the ones no host reported: a PR's spend on a +model goes to its line of that model, of the same effort where both name one, +and spend on a model no line names, such as a sub-agent on a smaller model, +to the line that took the most. Local reviewers roll up the same way, by the +PR's size and each reviewer's harness, model, effort, and shadow mark: PRs +reviewed, tallies, first rounds, and reviewing cost per PR. Over the merged +PRs come each online reviewer's coverage, final heads reviewed, findings per PR reviewed, and -answers. Sessions on no PR here, from any +answers. A hand-back counts only while its draft is open, so a merged PR +never shows one. +Sessions on no PR here, from any repository or agent token-max reads, total on an `Unattributed` line. `--pr` and `--quest` narrow the report to one PR or one quest's branch, and `--json` prints it as one object. Without `token-max` on `PATH`, it says how to install diff --git a/quest/m0/routing-evidence/README.md b/quest/m0/routing-evidence/README.md index 650d210f..80e1fb41 100644 --- a/quest/m0/routing-evidence/README.md +++ b/quest/m0/routing-evidence/README.md @@ -58,9 +58,14 @@ Readout, the agent's choices for the implementer to revisit: chooses to. - Carry the tallies into [the model check](/quest/m1/model-check.md)'s baseline. -- Publish the arms' cost and quality as - [the cost-against-quality charts](/quest/m0/routing-evidence/charts.md) quest describes, - and link them with the recommendations. +- Chart the arms' cost against quality as + [the model check](/quest/m1/model-check.md)'s chart procedure describes, + one point per arm, and link the charts with the recommendations. Its + `quest usage --json` rows give each arm's cost per merged PR and + first-round findings, and each shadow pair's cost and findings per PR. + Only tallies begun after + [moq-dev/quest#721](https://github.com/moq-dev/quest/pull/721) merged + record a first round, so count first rounds only over those. Baseline from the 2026-11-11 audit, at API list price: diff --git a/quest/m0/routing-evidence/charts.md b/quest/m0/routing-evidence/charts.md deleted file mode 100644 index 0f60984f..00000000 --- a/quest/m0/routing-evidence/charts.md +++ /dev/null @@ -1,76 +0,0 @@ -# [M] Chart routing's cost and quality for every model and effort - -## Goal - -Each [model check](/quest/m1/model-check.md) run, and this epic's -readout, publishes charts of cost against quality for every writer and -reviewer model and effort `quest` runs, and links them in its PR, so a -routing change is read off the cost/quality frontier rather than tables: - -- Writers: cost per merged PR against each quality measure the epic's - readout compares for the writer arms (first-round findings, review - rounds, hand-backs, and later `## Caused by` escapes), one point per - size, model, and effort, with cost and quality as separate axes. -- Local reviewers: cost per review against findings raised and the share - fixed. -- Online reviewers, which run on flat plans and have no cost per review: - coverage against findings per PR and acceptance, which `quest usage` - reports since [moq-dev/quest#642](https://github.com/moq-dev/quest/pull/642), - on a chart of their own. -- Published benchmarks: one chart per benchmark suite, score against its - own cost per task, for the same models and efforts. Benchmark and PR - metrics differ, so compare where each cell sits on each chart's - frontier rather than its values; a cell on the frontier for the - benchmarks but not for our PRs, or the reverse, stands out. - -Non-goal: a live dashboard. No chart code ships in `quest`. - -## Plan - -Decided 2026-11-11 (user, /quest-plan in a /quest-run session, when picking -[moq-dev/quest#642](https://github.com/moq-dev/quest/pull/642) to merge): - -- Goal? ✅ Charts for the model check and the readout, the two places - routing changes (rejected: a live dashboard; benchmarks only) -- Which data? ✅ Our measured points from `quest usage --json` with published - benchmark points beside them (rejected: ours only; benchmarks only) -- Where do they live? ✅ An artifact per run: the agent running the check or - the readout renders a private claude.ai page from `quest usage --json` and - the benchmark table, and links it in its PR, so the core stays - independent of a harness. The page is for the maintainer, who picks - routing; the PR body also carries the plotted numbers as tables, so a - reviewer who cannot open the page can check them (rejected: a `quest usage --html` report, which - adds chart code and tests to the CLI; token-max, which knows no PR - quality signals) -- Milestone? ✅ m0, in this epic (rejected: m1) - -The benchmarks behind the XS writer change to Sonnet 5.5 high -([moq-dev/quest#646](https://github.com/moq-dev/quest/pull/646)) and the XS -arm of the [writer A/B](/quest/m0/routing-evidence/writer-ab.md) -([moq-dev/quest#644](https://github.com/moq-dev/quest/pull/644)), both -decided on 2026-11-11, for the first table: - -- Anthropic's - [cost guide](https://platform.claude.com/docs/en/about-claude/models/optimizing-for-cost-and-intelligence), - SWE-bench Pro subset: Opus 5.5 low 87.4% and medium 92.8%, Sonnet 5.5 - high 87.6% with medium about 7 points lower, at about $0.11 per solved - task for Opus low and Sonnet high. -- [Artificial Analysis](https://artificialanalysis.ai/models/comparisons/claude-sonnet-5-5-high-vs-claude-opus-5-5-low), - Terminal-Bench 4.0 and cost per task: Opus 5.5 low 31.3%, $0.98; Sonnet - 5.5 high 43.9%, $1.64; Sonnet 5.5 medium 29.8%, $0.78; Haiku 5.5 xhigh - 29.3%, $0.25. - -Agent's choices, for the implementer to revisit: - -- Keep the benchmark points as a table in the model check's Plan, where its - existing benchmark notes already live, with each row's source, effort, and - date; each run refreshes it. A row without a stated effort stays out. -- Write the chart procedure into the model check's Plan and the epic's - readout: which `quest usage --json` fields feed each axis, how cells with - too few PRs are marked rather than dropped, and that each chart names its - date range and PR counts. -- `quest usage --json` reports each reviewer's latest round and its tally, - which sums findings across rounds. If the readout compares first-round - findings on their own, record them beside the tally. -- Make the first charts from the data on main when this lands, link them in - this quest's PR, and say which cells are still too thin. diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index aa234994..7f2df82c 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -48,6 +48,74 @@ checkpoints and outages to count by hand: maintainer with its tally, as a decision in the PR, rather than changing it unasked. Replacing a stale model id with the same vendor's newer one, below, needs no decision, as this check always did. +- Chart cost against quality for every writer and reviewer cell, as decided + 2026-11-11 (user, /quest-plan, moq-dev/quest#642's follow-ups), so a + routing change is read off the frontier rather than tables. The agent + running the check makes the charts itself, from the saved + `quest usage --since --json` (with `--ssh` for each host that runs + agents) and the benchmark table below. The charts are an artifact of the + run, and no chart code goes into `quest`. It links or embeds them in its + PR, and puts every plotted number in the PR body as tables, so a + reviewer who cannot open the charts can check them. Each + chart names its date range, the command, and each point's PR count. + Points too thin for the run's variance estimate stay on the chart, + hollow and labelled with their count, never dropped. Cells whose + `Implementer:` effort is `unknown` stay separate points. The charts: + - Writers, one point per `accuracy` row (size, model, effort): cost per + merged PR, `spent.cost / spent.pulls`, against each quality measure + on its own chart. The measures are first-round findings per PR + (`firstRounds.findings` over `firstRounds.pulls`), review rounds per + reviewed PR (`tally.rounds / reviewed`), hand-backs per PR + (`handedBack / pulls`, which counts only drafts still open), and later + `## Caused by` escapes per merged PR (`fixes / merged`). + - Local reviewers, one point per `reviewers` row (PR size, harness, + model, effort, shadow): cost per PR reviewed, `spent.cost / + spent.pulls`, against findings per reviewed PR (`tally.findings` over + `reviewed`) and the share fixed, `tally.fixed` over the findings + settled one way or another (fixed, refuted, or settled). + - Online reviewers, which run on flat plans and so have no cost per + review, on a chart of their own: one point per `online` row, coverage + (`reviewed / merged`) against findings per reviewed PR, labelled with + its acceptance (`accepted / findings`). + - Published benchmarks, one chart per suite in the table below: score + against that suite's cost per task, for the same models and efforts, + with the frontier drawn. Benchmark and PR metrics differ, so compare + where each cell sits on each chart's frontier, not its values; flag a + cell on the frontier in one and not the other. +- Published benchmarks, refreshed each run: rows with a stated effort, each + with its source and the date it was read. Cost is per task in the suite's + own terms; `~` marks a value derived from the source's prose. + + | Suite | Model | Effort | Score | $ per task | Read | + |---|---|---|---|---|---| + | [mypriors](https://mypriors.com/adamholter) 10-benchmark index, subscription cost | Haiku 5.5 | high | 30 | 0.0034 | 2026-11-09 | + | mypriors | Haiku 5.5 | xhigh | 34.6 | 0.0053 | 2026-11-09 | + | mypriors | Sonnet 5.5 | medium | 30.7 | 0.012 | 2026-11-09 | + | mypriors | Opus 5.5 | low | 34.7 | 0.015 | 2026-11-09 | + | mypriors | Opus 5.5 | medium | 48 | 0.037 | 2026-11-09 | + | mypriors | Opus 5.5 | high | 51.7 | 0.052 | 2026-11-09 | + | mypriors | Opus 5.5 | xhigh | 57.3 | 0.092 | 2026-11-09 | + | mypriors | gpt-6.1-sol | low | 34.8 | 0.0094 | 2026-11-09 | + | mypriors | gpt-6.1-sol | medium | 46.3 | 0.021 | 2026-11-09 | + | mypriors | gpt-6.1-sol | high | 50.3 | 0.034 | 2026-11-09 | + | mypriors | gpt-6.1-sol | xhigh | 52.5 | 0.045 | 2026-11-09 | + | mypriors | Fable 5.2 | medium | 56 | 3.8 | 2026-11-09 | + | [Anthropic's cost guide](https://platform.claude.com/docs/en/about-claude/models/optimizing-for-cost-and-intelligence), SWE-bench Pro 478-problem subset, % solved | Sonnet 5.5 | medium | ~80.6 | ~0.057 | 2026-11-13 | + | Anthropic, SWE-bench Pro | Sonnet 5.5 | high | 87.6 | ~0.096 | 2026-11-13 | + | Anthropic, SWE-bench Pro | Opus 5.5 | low | 87.4 | 0.11 | 2026-11-13 | + | Anthropic, SWE-bench Pro | Opus 5.5 | medium | 92.8 | 0.21 | 2026-11-13 | + | Anthropic, SWE-bench Pro | Opus 5.5 | high | 95.3 | 0.29 | 2026-11-13 | + | Anthropic, SWE-bench Pro | Opus 5.5 | xhigh | 96.7 | 0.74 | 2026-11-13 | + | [Artificial Analysis](https://artificialanalysis.ai/models/comparisons/claude-sonnet-5-5-high-vs-claude-opus-5-5-low), Terminal-Bench 4.0, % | Haiku 5.5 | xhigh | 29.3 | 0.25 | 2026-11-11 | + | Artificial Analysis, Terminal-Bench 4.0 | Sonnet 5.5 | medium | 29.8 | 0.78 | 2026-11-11 | + | Artificial Analysis, Terminal-Bench 4.0 | Sonnet 5.5 | high | 43.9 | 1.64 | 2026-11-13 | + | Artificial Analysis, Terminal-Bench 4.0 | Opus 5.5 | low | 31.3 | 0.98 | 2026-11-13 | + + Anthropic gives Sonnet high's cost per solved task ($0.11), Sonnet + medium as about 7 points lower at about 59% of high's cost per task, and + calls its figures directional. Its guide also has rows on internal + suites without a cost per task, which stay out. The `max` rows stay out + too, since `quest agent` and `quest review` refuse `max`. - Read [routing evidence](/quest/m0/routing-evidence/README.md)'s readout and baseline first: its shadow reviews and writer A/B measure the cells @@ -80,25 +148,16 @@ checkpoints and outages to count by hand: effort (6 of the 20 lines said `unknown`). - Changed after that run rather than on tallies (maintainer, 2026-11-09): XS writes on Haiku xhigh instead of high. Tally that cell fresh. The - source was [mypriors.com/adamholter](https://mypriors.com/adamholter)'s - score against subscription cost per task, a 10-benchmark index read - off its chart (score, then $ per task), which this check compares each - cell with again: - - Haiku 5.5: high 30, $0.0034; xhigh 34.6, $0.0053. - - Sonnet 5.5: medium 30.7, $0.012; below the frontier at every effort, - so rejected for XS then, though Anthropic's + source was the mypriors rows of the benchmark table above: + - Sonnet 5.5 sat below the frontier at every effort, so it was rejected + for XS then, though Anthropic's [model guide](https://claude.dev/blog/which-claude-model-should-you-build-with/) favors it for scoped coding. - - Opus 5.5: low 34.7, $0.015; medium 48, $0.037; high 51.7, $0.052; - xhigh 57.3, $0.092; max 58.9, $0.15. - - gpt-6.1-sol: low 34.8, $0.0094; medium 46.3, $0.021; high 50.3, - $0.034; xhigh 52.5, $0.045. - - Fable 5.2: medium 56, $3.8; max 64.6, $26.5. Rejected for XL, which - stays on Opus xhigh, on the frontier; zero-data-retention - organizations may also be refused Fable. - - S looks lopsided: Opus low scores well below its Sol medium fallback - (34.7 against 46.3) for $0.006 less a task. Check its tallies before - moving it. + - Fable 5.2 was rejected for XL, which stays on Opus xhigh, on the + frontier; zero-data-retention organizations may also be refused + Fable. + - S looked lopsided: Opus low scored well below its Sol medium fallback + (34.7 against 46.3) for $0.006 less a task. - Changed again (maintainer, 2026-11-11): XS writes on Sonnet high instead of Haiku xhigh, since two Haiku XS agents in a row on [moq-dev/quest#643](https://github.com/moq-dev/quest/pull/643) died diff --git a/src/cli/main.ts b/src/cli/main.ts index 1aad738a..4801d148 100644 --- a/src/cli/main.ts +++ b/src/cli/main.ts @@ -1111,13 +1111,13 @@ export function main(argv: string[], out: Output): number | Promise { "--ssh host, as token-max sessions exports and prices them, the cost an estimate at each model's API list " + "price; its sub-agents are sessions of their own. Each Claude Code or Codex session belongs to the PR whose " + "head branch it last names, when a repository token-max resolved for it on its own host, or for another " + - "session in the same directory there, is this one; no prompt or reply is read out. Its role is " + - "reviewing when the PR's ## Review record names it or Codex marks it a review, implementing when its first " + + "session in the same directory there, is this one, else to the PR of the session that spawned it; no prompt or reply is read out. Its role is " + + "reviewing when the PR's ## Review record names it or Codex marks it a review, its parent's on the same PR, implementing when its first " + "request since --since came before the PR opened, and iterating otherwise. A session found on several hosts counts once, at its " + "costliest copy, and the report says how many copies it folded from which hosts; a session with a request " + "token-max could not price or a line it could not read is listed, its cost a lower bound. A host that fails fails the report. " + "Each PR also shows its ## Models size and implementers, and as unread each Implementer line it cannot read, " + - "each reviewer the ## Review record names, with its latest round and score and its tally of findings across every round, marked when it is the shadow, " + + "each reviewer the ## Review record names, with its latest round and score and its tally of findings across every round and in its first, marked when it is the shadow, " + "where a merged PR whose last round left only low findings open counts them settled, since the author fixes or refutes those before merge, " + "the later merged PRs whose ## Caused by bullets open with its link, and each online reviewer's record from GitHub: whether it " + "reviewed the last head, and its findings accepted, declined, unclassified, or unanswered. An online reviewer " + @@ -1125,7 +1125,8 @@ export function main(argv: string[], out: Output): number | Promise { "review thread it opens or a [P] line in its review body, and its first answer's first word takes or " + "declines it. Rollups follow: spend by size, role, harness, model, and " + "effort, accuracy by size and implementer, summing each branch's tallies but the shadow's once, where an Implementer line's own size, as (iterate, XS), wins " + - "over the PR's, and each online reviewer's coverage and findings over the merged PRs. Without token-max on PATH, it says how to install it and reports the PRs' records alone. " + + "over the PR's, with its writing cost over the merged PRs it was found on, local reviewers by size, harness, model, effort, and shadow, with their " + + "tallies and reviewing cost per PR, and each online reviewer's coverage and findings over the merged PRs. Without token-max on PATH, it says how to install it and reports the PRs' records alone. " + "--json prints the same report as one object.\n\n" + "Uses GH_TOKEN, GITHUB_TOKEN, or gh auth token. Exits 0 when done, 2 for invalid arguments, and 1 for " + "other failures.", diff --git a/src/cli/review-record.ts b/src/cli/review-record.ts index d24c3c02..18d95754 100644 --- a/src/cli/review-record.ts +++ b/src/cli/review-record.ts @@ -34,6 +34,8 @@ const reviewerLine = (seat: Pick): string { - const total = SEVERITIES.reduce((sum, severity) => sum + counts.findings[severity], 0); - const severities = SEVERITIES.filter((severity) => counts.findings[severity] > 0) - .map((severity) => `${severity} ${counts.findings[severity]}`) +/** Findings as their total, then each severity found, such as `3 (high 1, low 2)`. */ +export function severityText(findings: Tally["findings"]): string { + const total = SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0); + const severities = SEVERITIES.filter((severity) => findings[severity] > 0) + .map((severity) => `${severity} ${findings[severity]}`) .join(", "); - return `findings ${total}${severities && ` (${severities})`}: fixed ${counts.fixed}, refuted ${counts.refuted}${counts.settled > 0 ? `, settled ${counts.settled}` : ""}, open ${counts.open}`; + return `${total}${severities && ` (${severities})`}`; +} + +/** The counts as the record's `Rounds:` line ends, after the rounds. */ +export function findingsText(counts: Omit): string { + return `${counts.first === null ? "" : `first round ${severityText(counts.first)}; `}findings ${severityText(counts.findings)}: fixed ${counts.fixed}, refuted ${counts.refuted}${counts.settled > 0 ? `, settled ${counts.settled}` : ""}, open ${counts.open}`; } /** A tally's counts, where a finding neither fixed nor refuted is open. */ -const counted = (tally: Tally): Counts => ({ +const counted = ({ first, ...tally }: Tally): Counts => ({ ...tally, + first: first ?? null, settled: 0, open: SEVERITIES.reduce((sum, severity) => sum + tally.findings[severity], 0) - tally.fixed - tally.refuted, }); @@ -183,27 +191,35 @@ function parseCounts(text: string): Counts | null { const line = /^- Rounds: (.*?)\r?$/m.exec(text); if (!line) return null; const found = - /^(\d+); findings (\d+)(?: \(([^)]*)\))?: fixed (\d+), refuted (\d+)(?:, settled (\d+))?, open (\d+)$/.exec( + /^(\d+); (?:first round (\d+)(?: \(([^)]*)\))?; )?findings (\d+)(?: \(([^)]*)\))?: fixed (\d+), refuted (\d+)(?:, settled (\d+))?, open (\d+)$/.exec( line[1]!, ); - if (!found) throw new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); - const findings = { high: 0, medium: 0, low: 0 }; - for (const part of found[3]?.split(", ") ?? []) { - const [severity, count] = part.split(" "); - if (!SEVERITIES.includes(severity as keyof typeof findings) || !/^\d+$/.test(count ?? "")) - throw new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); - findings[severity as keyof typeof findings] = Number(count); - } + const malformed = () => new Error(`the ## Review record's Rounds line is malformed: ${line[1]}`); + if (!found) throw malformed(); + // Each severity's count, checked against the total the line states. + const severities = (total: string, list: string | undefined) => { + const findings = { high: 0, medium: 0, low: 0 }; + for (const part of list?.split(", ") ?? []) { + const [severity, count] = part.split(" "); + if (!SEVERITIES.includes(severity as keyof typeof findings) || !/^\d+$/.test(count ?? "")) + throw malformed(); + findings[severity as keyof typeof findings] = Number(count); + } + if (SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0) !== Number(total)) + throw new Error(`the ## Review record's Rounds line does not add up: ${line[1]}`); + return findings; + }; + const findings = severities(found[4]!, found[5]); const counts = { rounds: Number(found[1]), + first: found[2] === undefined ? null : severities(found[2], found[3]), findings, - fixed: Number(found[4]), - refuted: Number(found[5]), - settled: Number(found[6] ?? 0), - open: Number(found[7]), + fixed: Number(found[6]), + refuted: Number(found[7]), + settled: Number(found[8] ?? 0), + open: Number(found[9]), }; - const total = SEVERITIES.reduce((sum, severity) => sum + findings[severity], 0); - if (total !== Number(found[2]) || counts.fixed + counts.refuted + counts.settled + counts.open !== total) + if (counts.fixed + counts.refuted + counts.settled + counts.open !== Number(found[4])) throw new Error(`the ## Review record's Rounds line does not add up: ${line[1]}`); return counts; } diff --git a/src/cli/review.ts b/src/cli/review.ts index 11535613..1b0ce2a4 100644 --- a/src/cli/review.ts +++ b/src/cli/review.ts @@ -313,6 +313,8 @@ export interface Tally { * reviewer keeps open becomes that reviewer's, since its round reports it again unlinked to the earlier one. */ findings: Record<(typeof SEVERITIES)[number], number>; + /** Its first round's findings by severity, which later rounds leave as they were; absent on a tally begun before. */ + first?: Record<(typeof SEVERITIES)[number], number>; fixed: number; refuted: number; } @@ -1261,7 +1263,9 @@ export function parseState(value: unknown): State { typeof tally.rounds === "number" && typeof tally.fixed === "number" && typeof tally.refuted === "number" && - SEVERITIES.every((severity) => typeof tally.findings?.[severity] === "number"), + SEVERITIES.every((severity) => typeof tally.findings?.[severity] === "number") && + (tally.first === undefined || + SEVERITIES.every((severity) => typeof tally.first?.[severity] === "number")), ) && state.sessions.every( (session) => @@ -1596,6 +1600,10 @@ export function tallied( own.session = round.session; own.rounds++; for (const finding of round.findings) own.findings[finding.severity]++; + if (own.rounds === 1) { + own.first = { high: 0, medium: 0, low: 0 }; + for (const finding of round.findings) own.first[finding.severity]++; + } by.set(seatKey(round), own); } const run = rounds.map(({ round }) => seatKey(round)); diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 379b2b51..b4ed0033 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -12,7 +12,7 @@ import { MERGER } from "./land-watch-fix"; import { githubRepository, requireRemote } from "./github"; import { type GitHub, type ObjectValue, type Repository, link } from "../github/client"; import { type OnlineReview, readOnlineReviews } from "../github/online-review"; -import { type Counts, type ReviewSummary, findingsText, parseReview } from "./review-record"; +import { type Counts, type ReviewSummary, findingsText, parseReview, severityText } from "./review-record"; import { SEVERITIES } from "./review"; /** A session's tokens, and their cost at the model's API list price, which subscriptions do not bill. */ @@ -36,6 +36,8 @@ export interface Session { harness: string; /** The harness's own ID, as `quest review` records it; null when the transcript names none. */ id: string | null; + /** The ID of the session that spawned it, as a sub-agent's or a Codex review's parent; null for a top session. */ + parent: string | null; review: boolean; cwd: string | null; /** The GitHub repositories, as lowercase `owner/repo`, its host resolved for it; null when it could not. */ @@ -98,6 +100,15 @@ export interface SpendRow extends Tokens { sessions: number; } +/** Spend at list price over the PRs it was found on, which leave out PRs whose sessions no host reported. */ +export type Spend = { pulls: number } & Tokens; + +/** First rounds summed over the reviewed PRs whose every counted tally records one. */ +export interface FirstRounds { + pulls: number; + findings: Counts["findings"]; +} + export interface AccuracyRow { /** The implementer's own size, as its line ends with it, else the PR's. */ size: Size | null; @@ -113,13 +124,31 @@ export interface AccuracyRow { scored: number; score: number; /** Every reviewer's findings but the shadow's across every round on the reviewed PRs, and as `rounds`, each PR's most rounds. */ - tally: Counts; + tally: Omit; + firstRounds: FirstRounds; + /** Its writing sessions' spend on its merged PRs, so `spent.cost / spent.pulls` is its cost per merged PR. */ + spent: Spend; /** The land fixer's model-run fixes, leaving out the watcher's own base merges. */ landFixes: number; /** Later PRs naming these PRs under `## Caused by`. */ fixes: number; } +/** One local reviewer seat over the PRs reported, by the PR's size, since review efforts follow it. */ +export interface ReviewerRow { + size: Size | null; + harness: string; + model: string; + effort: string; + shadow: boolean; + /** PRs whose record tallies this reviewer, each branch counted once, and its findings across every round. */ + reviewed: number; + tally: Omit; + firstRounds: FirstRounds; + /** Its reviewing sessions' spend, so `spent.cost / spent.pulls` is its cost per PR reviewed. */ + spent: Spend; +} + /** One online reviewer over the merged PRs reported. */ export interface OnlineRow { reviewer: string; @@ -143,6 +172,7 @@ export interface Usage { pulls: PullRow[]; spend: SpendRow[]; accuracy: AccuracyRow[]; + reviewers: ReviewerRow[]; online: OnlineRow[]; /** The sessions on no PR reported here, from every repository and agent token-max reads; null when narrowed. */ unattributed: ({ sessions: number } & Tokens) | null; @@ -223,6 +253,7 @@ export function parseSessions(text: string): Session[] { host: optional(row.host, "a host"), harness: word(row.harness, "a harness"), id: optional(row.session, "a session ID"), + parent: optional(row.parent, "a parent"), review: flag(row.review, "a review mark"), cwd: optional(row.cwd, "a directory"), // A repository that is not on GitHub can head no PR here, so only GitHub's count. @@ -430,11 +461,12 @@ function owners(sessions: Session[]): (session: Session) => string[] | null { /** * Attribute each session to the PR whose head branch it last named, among those still open when the session - * reached the branch and headed from a repository its host resolved for it, and give it a role: reviewing when a - * PR's review record names it or Codex marks it a review, implementing when its first request in the window came - * before its PR opened, and iterating otherwise. `evidence` holds every exported copy, folded ones included, since a - * dropped copy may be the only one naming its directory's repository. Returns the sessions it gave a PR in `rows`, - * and those on no PR here. + * reached the branch and headed from a repository its host resolved for it, else to its parent's PR, as a sub-agent + * in a detached checkout. Give it a role: reviewing when a PR's review record names it or Codex marks it a review, + * its parent's when both share the PR, as a review's sub-agents do, else implementing when its first request in the + * window came before its PR opened, and iterating otherwise. `evidence` holds every exported copy, folded ones + * included, since a dropped copy may be the only one naming its directory's repository. Returns the sessions it gave + * a PR in `rows`, and those on no PR here. */ function attribute( sessions: Session[], @@ -446,15 +478,12 @@ function attribute( const byBranch = new Map(); for (const each of [...pulls].sort((a, b) => a.created - b.created)) byBranch.set(each.branch, [...(byBranch.get(each.branch) ?? []), each]); - const attributed: Session[] = []; - const unattributed: Session[] = []; const owned = owners(evidence); - for (const session of sessions) { + const own = (session: Session): Pull | undefined => { // Another repository may use the same branch name, so a session counts only where its repository is known. const known = owned(session); - let found: Pull | undefined; + if (known === null) return undefined; for (const seen of [...session.branches].sort((a, b) => b.last - a.last)) { - if (known === null) break; // A branch can head a new PR after an earlier one closed: take the one open when the session reached it. const open = (byBranch.get(seen.name) ?? []).filter( (each) => @@ -464,26 +493,52 @@ function attribute( ); // A checkout with remotes for both a fork and its upstream cannot tell which one's branch it was on. if (new Set(open.map((each) => each.repository)).size > 1) continue; - found = open[0]; - if (found) break; + if (open[0]) return open[0]; } + return undefined; + }; + const id = (harness: string, session: string | null) => JSON.stringify([harness, session]); + const byId = new Map(sessions.filter((each) => each.id !== null).map((each) => [id(each.harness, each.id), each])); + type Placed = { pull: Pull; role: Role } | null; + const placed = new Map(); + const place = (session: Session, seen: Set): Placed => { + if (placed.has(session)) return placed.get(session)!; + const parent = session.parent === null ? undefined : byId.get(id(session.harness, session.parent)); + // A parent chain that loops back is cut there, so its sessions place by their own branches. + const above = parent === undefined || seen.has(parent) ? null : place(parent, new Set([...seen, session])); + const pull = own(session) ?? above?.pull; + let result: Placed = null; + if (pull) + result = { + pull, + role: + session.review || (session.id !== null && reviewers.has(session.id)) + ? "reviewing" + : above && above.pull === pull + ? above.role + : session.start < pull.created + ? "implementing" + : "iterating", + }; + placed.set(session, result); + return result; + }; + const attributed: Session[] = []; + const unattributed: Session[] = []; + for (const session of sessions) { + const found = place(session, new Set()); if (!found) { unattributed.push(session); continue; } - const row = rows.get(found.number); + const row = rows.get(found.pull.number); if (!row) continue; attributed.push(session); row.sessions.push({ host: session.host, harness: session.harness, session: session.id, - role: - session.review || (session.id !== null && reviewers.has(session.id)) - ? "reviewing" - : session.start < found.created - ? "implementing" - : "iterating", + role: found.role, models: session.models, }); } @@ -521,18 +576,80 @@ function spendRows(pulls: PullRow[]): SpendRow[] { ); } -function accuracyRows(pulls: PullRow[]): AccuracyRow[] { - const rows = new Map(); - // A branch's saved tallies span every PR from it, so only one PR that has both a tally and an implementer - // to count it for counts them: the one whose record holds the latest tally, as the most rounds show, since - // each round rewrites only its own PR's record; the newest breaks a tie. A deleted fork has no repository, - // so its PR stands alone. +const noCounts = (): Omit => ({ + rounds: 0, + findings: { high: 0, medium: 0, low: 0 }, + fixed: 0, + refuted: 0, + settled: 0, + open: 0, +}); +const noFirst = (): FirstRounds => ({ pulls: 0, findings: { high: 0, medium: 0, low: 0 } }); +const noSpend = (): Spend => ({ pulls: 0, ...zero() }); + +/** Add one PR's `tallies` of a cell to its counts, its rounds as their most, and to its first rounds when all have one. */ +function count(to: { tally: Omit; firstRounds: FirstRounds }, tallies: Counts[]) { + to.tally.rounds += Math.max(...tallies.map((tally) => tally.rounds)); + for (const tally of tallies) { + for (const severity of SEVERITIES) to.tally.findings[severity] += tally.findings[severity]; + to.tally.fixed += tally.fixed; + to.tally.refuted += tally.refuted; + to.tally.settled += tally.settled; + to.tally.open += tally.open; + } + if (tallies.every((tally) => tally.first !== null)) { + to.firstRounds.pulls++; + for (const tally of tallies) + for (const severity of SEVERITIES) to.firstRounds.findings[severity] += tally.first![severity]; + } +} + +/** + * A PR's spend split among `seats`, as its record names them: each model's spend to the seat of that model, of its + * effort where both name one, and spend of a model no seat names, as a sub-agent on a smaller model, to the seat + * that took the most, the PR's main one. Returns each seat's share, by index, and nothing without a seat. + */ +function share(spent: Spent[], seats: { model: string; effort: string }[]): Map { + const shares = new Map(); + if (seats.length === 0) return shares; + const into = (seat: number, from: Tokens) => { + const to = shares.get(seat) ?? zero(); + add(to, from); + shares.set(seat, to); + }; + const rest: Spent[] = []; + for (const each of spent) { + const same = seats.flatMap((seat, i) => (seat.model === each.model ? [i] : [])); + const effort = each.effort ?? "unknown"; + const seat = + same.find((i) => seats[i]!.effort === effort) ?? + same.find((i) => seats[i]!.effort === "unknown" || effort === "unknown") ?? + same[0]; + if (seat === undefined) rest.push(each); + else into(seat, each); + } + if (rest.length > 0) { + const main = [...shares].reduce<[number, Tokens] | undefined>( + (best, entry) => (best === undefined || entry[1].cost > best[1].cost ? entry : best), + undefined, + ); + for (const each of rest) into(main?.[0] ?? 0, each); + } + return shares; +} + +/** + * A branch's saved tallies span every PR from it, so only one of its PRs that `counts` counts them: the one whose + * record holds the latest tally, as the most rounds show, since each round rewrites only its own PR's record; the + * newest breaks a tie. A deleted fork has no repository, so its PR stands alone. + */ +function tallied(pulls: PullRow[], counts: (pull: PullRow) => boolean): Set { const branch = (each: PullRow) => JSON.stringify([each.repository ?? each.number, each.branch]); const rounds = (each: PullRow) => each.review.reduce((sum, review) => sum + (review.shadow ? 0 : (review.tally?.rounds ?? 0)), 0); const latest = new Map(); for (const each of pulls) { - if (each.implementers.length === 0 || each.review.every((review) => review.tally === null)) continue; + if (!counts(each) || each.review.every((review) => review.tally === null)) continue; const other = latest.get(branch(each)); if ( other === undefined || @@ -541,15 +658,25 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { ) latest.set(branch(each), each); } + return new Set(latest.values()); +} + +function accuracyRows(pulls: PullRow[]): AccuracyRow[] { + const rows = new Map(); + const latest = tallied(pulls, (each) => each.implementers.length > 0); for (const each of pulls) { // A session that pushed twice is one implementer of the PR, whether or not each line names its size. - const distinct = new Map( - each.implementers.map((i) => { - const at = { ...i, size: i.size ?? each.size }; - return [JSON.stringify(at), at]; - }), - ); - for (const implementer of distinct.values()) { + const distinct = [ + ...new Map( + each.implementers.map((i) => { + const at = { ...i, size: i.size ?? each.size }; + return [JSON.stringify(at), at]; + }), + ).values(), + ]; + const writing = each.sessions.filter((session) => session.role !== "reviewing").flatMap((s) => s.models); + const shares = each.state === "merged" ? share(writing, distinct) : new Map(); + for (const [i, implementer] of distinct.entries()) { const size = implementer.size; const id = JSON.stringify([size, implementer.harness, implementer.model, implementer.effort]); let row = rows.get(id); @@ -565,16 +692,11 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { reviewed: 0, scored: 0, score: 0, - tally: { - rounds: 0, - findings: { high: 0, medium: 0, low: 0 }, - fixed: 0, - refuted: 0, - settled: 0, - open: 0, - }, + tally: noCounts(), + firstRounds: noFirst(), landFixes: 0, fixes: 0, + spent: noSpend(), }; rows.set(id, row); } @@ -583,26 +705,26 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { if (each.handedBack) row.handedBack++; row.fixes += each.fixedBy.length; row.landFixes += each.landFixes.filter((by) => by !== MERGER).length; - // The shadow's findings measure an effort step, not the writer, so they stay out of the writer's row. - const tallied = - latest.get(branch(each)) === each - ? each.review.filter((review) => review.tally !== null && !review.shadow) - : []; - if (tallied.length > 0) { - row.reviewed++; - row.tally.rounds += Math.max(...tallied.map((review) => review.tally!.rounds)); + const spent = shares.get(i); + if (spent) { + row.spent.pulls++; + add(row.spent, spent); } - for (const { score, tally } of tallied) { + // The shadow's findings measure an effort step, not the writer, so they stay out of the writer's row. + const counted = latest.has(each) + ? each.review.filter((review) => review.tally !== null && !review.shadow) + : []; + if (counted.length === 0) continue; + row.reviewed++; + for (const { score } of counted) if (score !== null) { row.scored++; row.score += score; } - for (const severity of SEVERITIES) row.tally.findings[severity] += tally!.findings[severity]; - row.tally.fixed += tally!.fixed; - row.tally.refuted += tally!.refuted; - row.tally.settled += tally!.settled; - row.tally.open += tally!.open; - } + count( + row, + counted.map((review) => review.tally!), + ); } } return [...rows.values()].sort((a, b) => @@ -610,6 +732,50 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { ); } +function reviewerRows(pulls: PullRow[]): ReviewerRow[] { + const rows = new Map(); + const latest = tallied(pulls, () => true); + const row = (each: PullRow, review: ReviewSummary) => { + const id = JSON.stringify([each.size, review.harness, review.model, review.effort, review.shadow]); + let found = rows.get(id); + if (!found) { + found = { + size: each.size, + harness: review.harness, + model: review.model, + effort: review.effort, + shadow: review.shadow, + reviewed: 0, + tally: noCounts(), + firstRounds: noFirst(), + spent: noSpend(), + }; + rows.set(id, found); + } + return found; + }; + for (const each of pulls) { + const reviewing = each.sessions.filter((session) => session.role === "reviewing").flatMap((s) => s.models); + for (const [i, spent] of share(reviewing, each.review)) { + const to = row(each, each.review[i]!).spent; + to.pulls++; + add(to, spent); + } + if (!latest.has(each)) continue; + for (const review of each.review.filter((review) => review.tally !== null)) { + const to = row(each, review); + to.reviewed++; + count(to, [review.tally!]); + } + } + return [...rows.values()].sort((a, b) => + order( + [a.size, a.harness, a.model, a.effort, String(a.shadow)], + [b.size, b.harness, b.model, b.effort, String(b.shadow)], + ), + ); +} + function onlineRows(pulls: PullRow[]): OnlineRow[] { const merged = pulls.filter((each) => each.state === "merged"); const rows = new Map(); @@ -707,6 +873,7 @@ export async function usage(root: string, options: UsageOptions, deps: UsageDepe pulls: rows, spend: spendRows(rows), accuracy: accuracyRows(rows), + reviewers: reviewerRows(rows), online: onlineRows(rows), unattributed, folded: { copies: folded.length, hosts: [...new Set(folded.map((session) => session.host))] }, @@ -730,6 +897,11 @@ const compact = (n: number) => const tokens = (t: Tokens) => `in ${compact(t.input)}, cache ${compact(t.cacheWrite)} written ${compact(t.cacheRead)} read, out ${compact(t.output)}, ~$${t.cost.toFixed(2)}${t.complete ? "" : " or more"}`; const mean = (sum: number, count: number) => (count ? (sum / count).toFixed(1) : "-"); +const firsts = (f: FirstRounds) => `first rounds ${f.pulls}: ${severityText(f.findings)}`; +const per = (s: Spend, what: string) => + s.pulls + ? `~$${(s.cost / s.pulls).toFixed(2)}${s.complete ? "" : " or more"} per ${what} over ${s.pulls}` + : `no ${what} priced`; const percent = (part: number, whole: number) => (whole ? `${Math.round((100 * part) / whole)}%` : "-"); const answers = (r: Omit) => `${r.findings} finding${r.findings === 1 ? "" : "s"}: ${r.accepted} accepted, ${r.declined} declined, ${r.unclassified} unclassified, ${r.unanswered} unanswered`; @@ -805,7 +977,22 @@ export function usageLines(report: Usage): string[] { a.harness, a.model, a.effort, - `${a.pulls} PRs, ${a.merged} merged, ${a.handedBack} handed back, ${a.reviewed} reviewed: score ${mean(a.score, a.scored)}, rounds ${mean(a.tally.rounds, a.reviewed)}, ${findingsText(a.tally)}; ${a.landFixes} land fixes, ${a.fixes} later fixes`, + `${a.pulls} PRs, ${a.merged} merged, ${a.handedBack} handed back, ${a.reviewed} reviewed: score ${mean(a.score, a.scored)}, rounds ${mean(a.tally.rounds, a.reviewed)}, ${findingsText({ ...a.tally, first: null })}, ${firsts(a.firstRounds)}; ${a.landFixes} land fixes, ${a.fixes} later fixes; ${per(a.spent, "merged PR")}`, + ]), + ), + ); + if (report.reviewers.length) + lines.push( + "", + "Local reviewers by PR size:", + ...table( + report.reviewers.map((r) => [ + " ", + r.size ?? "-", + r.harness, + r.model, + `${r.effort}${r.shadow ? ", shadow" : ""}`, + `${r.reviewed} reviewed: rounds ${mean(r.tally.rounds, r.reviewed)}, ${findingsText({ ...r.tally, first: null })}, ${firsts(r.firstRounds)}; ${per(r.spent, "PR reviewed")}`, ]), ), ); diff --git a/test/review.test.ts b/test/review.test.ts index 29f97256..701ca81c 100644 --- a/test/review.test.ts +++ b/test/review.test.ts @@ -1315,7 +1315,7 @@ fakes("each round rewrites the Review section of the branch's open PR, keeping t "- Reviewer: Codex, gpt-6.1-sol, medium effort", "- Session: `thread-1`, round 1", "- Score: 8/10, `Lands once the off-by-one is fixed.`", - "- Rounds: 1; findings 1 (high 1): fixed 0, refuted 0, open 1", + "- Rounds: 1; first round 1 (high 1); findings 1 (high 1): fixed 0, refuted 0, open 1", "- Findings, open until the next round settles them:", " - High, `src/lib.ts:2`: `Off by one`", "", @@ -1323,7 +1323,7 @@ fakes("each round rewrites the Review section of the branch's open PR, keeping t "- Reviewer: Codex, gpt-6.1-sol, low effort, shadow", "- Session: `thread-1`, round 1", "- Score: 8/10, `Lands once the off-by-one is fixed.`", - "- Rounds: 1; findings 1 (high 1): fixed 0, refuted 0, open 1", + "- Rounds: 1; first round 1 (high 1); findings 1 (high 1): fixed 0, refuted 0, open 1", "- Findings, open until the next round settles them:", " - High, `src/lib.ts:2`: `Off by one`", "", @@ -1356,7 +1356,9 @@ fakes("each round rewrites the Review section of the branch's open PR, keeping t expect(record).toContain("- Session: `thread-1`, round 2\n"); expect(record).toContain("- Score: 9/10, `Fixes #3.`\n"); // The refuted finding counts beside the new one, across both rounds. - expect(record).toContain("- Rounds: 2; findings 2 (high 2): fixed 0, refuted 1, open 1\n"); + expect(record).toContain( + "- Rounds: 2; first round 1 (high 1); findings 2 (high 2): fixed 0, refuted 1, open 1\n", + ); expect(record).toContain( ` - High, \`src/lib.ts\`: \`\`Ping @someone about \`${base.slice(0, 12)}\` in #12\`\`\n`, ); @@ -2050,6 +2052,8 @@ test("a tally counts each finding once, by who raised it, through every round", session: "s", rounds: 2, findings: { high: 2, medium: 0, low: 1 }, + // The first round's findings stay as it reported them, however later rounds settle or regrade them. + first: { high: 1, medium: 1, low: 1 }, fixed: 1, refuted: 1, }, diff --git a/test/usage.test.ts b/test/usage.test.ts index cef90c77..13247fe6 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -19,6 +19,8 @@ const GONE = "83a08f0e-6ef1-4b41-ad0c-58b37fc993f8"; const BOTH = "5d2c7f3e-8a41-4b6e-9f0d-2c6a1e7b4d93"; const REMOTE = "a1c2e3f4-0000-4000-8000-000000000001"; const ITERATE = "11111111-1111-1111-1111-111111111111"; +const HELPER = "33333333-3333-3333-3333-333333333333"; +const CHECKER = "44444444-4444-4444-4444-444444444444"; // Checkouts since deleted: directories the test created and removed, so no host path of the same name is read. const removed = () => { @@ -76,6 +78,7 @@ describe("parseSessions", () => { host: "box", harness: "codex", id: THREAD, + parent: null, review: false, cwd: WORK, repositories: ["acme/proj"], @@ -244,7 +247,7 @@ describe("PR records", () => { refuted, }); const tallies = [ - tally("codex", 3, { high: 1, medium: 1, low: 2 }, 2, 1), + { ...tally("codex", 3, { high: 1, medium: 1, low: 2 }, 2, 1), first: { high: 1, medium: 0, low: 1 } }, { ...tally("codex", 1, { high: 0, medium: 0, low: 0 }, 0, 0), shadow: true as const }, // A left-out reviewer still names its session, so `quest usage` counts that session as reviewing. { ...tally("claude", 2, { high: 2, medium: 0, low: 0 }, 1, 0), session: CLAUDE }, @@ -252,7 +255,9 @@ describe("PR records", () => { test("reads back each reviewer's round and tally quest review records, after the land fixer's lines", () => { const body = `Summary.\n\n## Review\n\nmechanical fix by codex, gpt-6.1-sol, medium effort: \`merged\` (head ${"d".repeat(40)})\n\n${reviewRecord([round, second], tallies)}\n\n## Models\n\n- Size: L\n`; - expect(body).toContain("\n- Rounds: 3; findings 4 (high 1, medium 1, low 2): fixed 2, refuted 1, open 1\n"); + expect(body).toContain( + "\n- Rounds: 3; first round 2 (high 1, low 1); findings 4 (high 1, medium 1, low 2): fixed 2, refuted 1, open 1\n", + ); expect(parseReview(body)).toEqual([ { harness: "Codex", @@ -264,6 +269,7 @@ describe("PR records", () => { score: 7, tally: { rounds: 3, + first: { high: 1, medium: 0, low: 1 }, findings: { high: 1, medium: 1, low: 2 }, fixed: 2, refuted: 1, @@ -282,6 +288,7 @@ describe("PR records", () => { score: 10, tally: { rounds: 1, + first: null, findings: { high: 0, medium: 0, low: 0 }, fixed: 0, refuted: 0, @@ -301,6 +308,7 @@ describe("PR records", () => { score: null, tally: { rounds: 2, + first: null, findings: { high: 2, medium: 0, low: 0 }, fixed: 1, refuted: 0, @@ -328,8 +336,18 @@ describe("PR records", () => { const settled = "1; findings 2 (low 2): fixed 1, refuted 0, settled 1, open 0"; expect( parseReview(`## Review\n\n${record.replace(/^- Rounds: .*$/m, `- Rounds: ${settled}`)}\n`)[0]!.tally, - ).toEqual({ rounds: 1, findings: { high: 0, medium: 0, low: 2 }, fixed: 1, refuted: 0, settled: 1, open: 0 }); + ).toEqual({ + rounds: 1, + first: null, + findings: { high: 0, medium: 0, low: 2 }, + fixed: 1, + refuted: 0, + settled: 1, + open: 0, + }); for (const line of [ + "2; first round 1 (high 2); findings 2 (high 2): fixed 2, refuted 0, open 0", + "2; first round 1 (severe 1); findings 1 (high 1): fixed 1, refuted 0, open 0", "1; findings 1 (high 1): fixed 0, refuted 0, open 0", "1; findings 2 (high 1): fixed 1, refuted 0, open 1", "1; findings 2 (low 2): fixed 1, refuted 0, settled 1, open 1", @@ -435,6 +453,14 @@ describe("usage", () => { }), models: [{ ...exported(GONE, "claude-opus-5-5", "high", 8).models[0], complete: false }], }, + // The implementer's sub-agent on a smaller model in a detached checkout, which takes its parent's PR and + // role though it started after the PR opened, and the reviewer's sub-agent, which reviews with it. + exported(HELPER, "claude-haiku-5-5", "low", 0.5, { + parent: CLAUDE, + branches: [{ name: "HEAD", first: "2026-11-06T12:30:00Z", last: "2026-11-06T12:31:00Z" }], + start: "2026-11-06T12:30:00Z", + }), + exported(CHECKER, "gpt-6.1-sol", "high", 0.25, { parent: THREAD, start: "2026-11-06T12:10:00Z" }), // Another host's session in a directory of the same path: this host's sessions there tell nothing of it. exported(REMOTE, "claude-opus-5-5", "high", 4, { host: "box", repositories: null }), // A checkout with remotes for both the fork and its upstream, on a branch each has a PR from. @@ -465,7 +491,7 @@ describe("usage", () => { "- Reviewer: Codex, gpt-6.1-sol, high effort", `- Session: \`${THREAD}\`, round 2`, "- Score: 9/10, `Fine.`", - "- Rounds: 2; findings 2 (high 1, low 1): fixed 1, refuted 1, open 0", + "- Rounds: 2; first round 1 (high 1); findings 2 (high 1, low 1): fixed 1, refuted 1, open 0", "- Findings: none", "", // A harness quest review does not run, as a hand-written record may name, reads as any other. @@ -663,6 +689,8 @@ describe("usage", () => { [null, "codex", THREAD, "reviewing", "high"], [null, "codex", ITERATE, "iterating", "high"], [null, "claude", OWNED, "iterating", "unknown"], + [null, "claude", HELPER, "implementing", "low"], + [null, "codex", CHECKER, "reviewing", "high"], ]); // #12 names #10 too, but closed without merging. expect(ten.fixedBy).toEqual([11]); @@ -687,10 +715,11 @@ describe("usage", () => { complete: false, }); expect(report.spend.map((s) => [s.size, s.role, s.harness, s.cost])).toEqual([ + ["L", "implementing", "claude", 0.5], ["L", "implementing", "claude", 5], ["L", "iterating", "claude", 7], ["L", "iterating", "codex", 2], - ["L", "reviewing", "codex", 1], + ["L", "reviewing", "codex", 1.25], ]); const tally = { rounds: 2, @@ -718,6 +747,29 @@ describe("usage", () => { ["S", "gpt-6.1-sol", 1, 1, 17, tally, 1], ["L", "claude-opus-5-5", 7, 5, 85, five, 1], ]); + // #10's writing spend: the iterate's Sol session to its own line though the efforts differ, the Opus + // session of unknown effort to the Opus line, and the Haiku sub-agent, which no line names, to the PR's main + // writer. Its other reviewers record no first round, so no PR counts toward the writers' first rounds. + expect(report.accuracy.map((a) => [a.spent.pulls, a.spent.cost, a.firstRounds.pulls])).toEqual([ + [1, 2, 0], + [1, 12.5, 0], + ]); + // The routed Sol reviewer over #10, #5, #6, and #3, each branch's latest record, with its first rounds; its + // session and sub-agent price it on #10. + const sol = report.reviewers.find((r) => r.size === "L" && r.model === "gpt-6.1-sol" && !r.shadow)!; + expect([sol.reviewed, sol.tally.rounds, sol.firstRounds, sol.spent.pulls, sol.spent.cost]).toEqual([ + 4, + 9, + { pulls: 4, findings: { high: 4, medium: 0, low: 0 } }, + 1, + 1.25, + ]); + expect(report.reviewers.filter((r) => r.size === "L").map((r) => [r.model, r.effort, r.shadow])).toEqual([ + ["claude-opus-5-5", "high", false], + ["gpt-6.1-sol", "high", false], + ["gpt-6.1-sol", "medium", true], + ["gemini-3-pro", "high", false], + ]); expect(ten.online).toEqual([ { reviewer: "rabbit[bot]", @@ -747,7 +799,12 @@ describe("usage", () => { expect(text).toContain( " review: Claude Code, claude-opus-5-5, high effort; not in the latest run; rounds 1, findings 1 (medium 1): fixed 1, refuted 0, open 0\n", ); - expect(text).toContain("rounds 2.0, findings 3 (high 1, medium 1, low 1): fixed 2, refuted 1, open 0;"); + expect(text).toContain( + "rounds 2.0, findings 3 (high 1, medium 1, low 1): fixed 2, refuted 1, open 0, first rounds 0: 0; 0 land fixes, 1 later fixes; ~$2.00 per merged PR over 1", + ); + expect(text).toContain( + " L Codex gpt-6.1-sol high 4 reviewed: rounds 2.3, findings 8 (high 4, medium 1, low 3): fixed 4, refuted 2, settled 1, open 1, first rounds 4: 4 (high 4); ~$1.25 per PR reviewed over 1\n", + ); expect(ten.review.map((r) => [r.effort, r.shadow])).toEqual([ ["high", false], ["high", false], @@ -767,7 +824,7 @@ describe("usage", () => { ); expect(text).toContain("fixed later by: #11\n"); expect(text).toContain( - " implementing local claude claude-opus-5-5 high in 10, cache 20 written 30 read, out 40, ~$5.00\n", + " implementing local claude claude-opus-5-5 high in 10, cache 20 written 30 read, out 40, ~$5.00\n", ); expect(text).toContain( "Unattributed: 5 sessions on no PR here, any repository or agent: in 50, cache 100 written 150 read, out 200, ~$30.00 or more\n", From 048485eeca4926de1d4321ad08503e24741f75fb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:52:42 -0800 Subject: [PATCH 3/7] quest: Mermaid charts in the PR body; plan counting hand-backs on merged PRs Co-Authored-By: Claude Opus 5.5 --- quest/m0/routing-evidence/handbacks.md | 37 ++++++++++++++++++++++++++ quest/m1/model-check.md | 17 ++++++++---- 2 files changed, 49 insertions(+), 5 deletions(-) create mode 100644 quest/m0/routing-evidence/handbacks.md diff --git a/quest/m0/routing-evidence/handbacks.md b/quest/m0/routing-evidence/handbacks.md new file mode 100644 index 00000000..f5b454a5 --- /dev/null +++ b/quest/m0/routing-evidence/handbacks.md @@ -0,0 +1,37 @@ +# [S] Count hand-backs on merged PRs + +## Goal + +`quest usage` counts a PR as handed back when any version of its body had a +`## Waiting` (or `## Decisions needed`) section while it was a draft, merged +PRs included, so the writer A/B's and the model check's hand-back measure +sees every PR rather than only drafts still open. + +## Plan + +Decided 2026-11-14 (user, /quest-run, from +[moq-dev/quest#721](https://github.com/moq-dev/quest/pull/721)'s +follow-ups): add this quest rather than drop the measure. + +Today `handedBack` in `src/cli/usage.ts` reads only the current body of an +open draft. `quest ready` and hand edits remove `## Waiting` before merge, +so every merged PR reads 0. + +Recommended (agent's choice): read each PR's body history from GitHub +(GraphQL `userContentEdits`, whose entries carry each earlier body) where +`quest usage` already reads the PR, and count a hand-back when an earlier +body had a hand-back section, using `handedBack` from `src/core/claim.ts`. +It works on PRs merged before this lands and needs no change to the skills +or `quest ready`. The alternative, having `quest ready` record hand-backs +in `## Models`, counts only PRs readied after it lands and trusts every +writer to keep the record. Batch the history reads with the existing PR +query so `--since` over a week stays within the GraphQL rate limit. + +Count each PR once, however many times it was handed back, and report the +count in the `accuracy` and per-PR rows as today. + +## Related + +- [Writer A/B](/quest/m0/routing-evidence/writer-ab.md) - compares + hand-backs per arm +- [Model check](/quest/m1/model-check.md) - charts hand-backs per PR diff --git a/quest/m1/model-check.md b/quest/m1/model-check.md index 7f2df82c..711525e7 100644 --- a/quest/m1/model-check.md +++ b/quest/m1/model-check.md @@ -54,19 +54,26 @@ checkpoints and outages to count by hand: running the check makes the charts itself, from the saved `quest usage --since --json` (with `--ssh` for each host that runs agents) and the benchmark table below. The charts are an artifact of the - run, and no chart code goes into `quest`. It links or embeds them in its - PR, and puts every plotted number in the PR body as tables, so a - reviewer who cannot open the charts can check them. Each + run, and no chart code goes into `quest`. It draws them as Mermaid + charts in its PR body, which GitHub renders and any harness can write + (user, 2026-11-14, /quest-run on + [moq-dev/quest#721](https://github.com/moq-dev/quest/pull/721); rejected: + images in a secret gist, a claude.ai page made by hand). Mermaid's + quadrant charts scale both axes to 0-1, so each title gives the scales + and each point's label carries its PR count. The body also puts every + plotted number in tables, so a reviewer can check the charts. Each chart names its date range, the command, and each point's PR count. Points too thin for the run's variance estimate stay on the chart, - hollow and labelled with their count, never dropped. Cells whose + marked thin in their label, never dropped. Cells whose `Implementer:` effort is `unknown` stay separate points. The charts: - Writers, one point per `accuracy` row (size, model, effort): cost per merged PR, `spent.cost / spent.pulls`, against each quality measure on its own chart. The measures are first-round findings per PR (`firstRounds.findings` over `firstRounds.pulls`), review rounds per reviewed PR (`tally.rounds / reviewed`), hand-backs per PR - (`handedBack / pulls`, which counts only drafts still open), and later + (`handedBack / pulls`, which counts only drafts still open until + [hand-backs on merged PRs](/quest/m0/routing-evidence/handbacks.md) + lands), and later `## Caused by` escapes per merged PR (`fixes / merged`). - Local reviewers, one point per `reviewers` row (PR size, harness, model, effort, shadow): cost per PR reviewed, `spent.cost / From 193102970b6dcecebc3b4da5ec56b26e079fea03 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 16:58:13 -0800 Subject: [PATCH 4/7] fix(usage): tell writer and iterate seats of one model apart by role, split true ties Co-Authored-By: Claude Opus 5.5 --- src/cli/usage.ts | 42 +++++++++++++++++++++++++++++--------- test/usage.test.ts | 51 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 11 deletions(-) diff --git a/src/cli/usage.ts b/src/cli/usage.ts index b4ed0033..15944c08 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -605,11 +605,17 @@ function count(to: { tally: Omit; firstRounds: FirstRounds }, t } /** - * A PR's spend split among `seats`, as its record names them: each model's spend to the seat of that model, of its + * A PR's spend split among `seats`, as its record names them: each model's spend to the seats of that model, of its * effort where both name one, and spend of a model no seat names, as a sub-agent on a smaller model, to the seat - * that took the most, the PR's main one. Returns each seat's share, by index, and nothing without a seat. + * that took the most, the PR's main one. Spend that fits several seats goes to the first, the PR's writer, when an + * implementing session spent it, and to the others when an iterating one did; what still fits several, such as a + * routed and a shadow reviewer at one effort, is split evenly among them rather than given to one. Returns each + * seat's share, by index, and nothing without a seat. */ -function share(spent: Spent[], seats: { model: string; effort: string }[]): Map { +export function share( + spent: (Spent & { role?: Role })[], + seats: { model: string; effort: string }[], +): Map { const shares = new Map(); if (seats.length === 0) return shares; const into = (seat: number, from: Tokens) => { @@ -621,12 +627,14 @@ function share(spent: Spent[], seats: { model: string; effort: string }[]): Map< for (const each of spent) { const same = seats.flatMap((seat, i) => (seat.model === each.model ? [i] : [])); const effort = each.effort ?? "unknown"; - const seat = - same.find((i) => seats[i]!.effort === effort) ?? - same.find((i) => seats[i]!.effort === "unknown" || effort === "unknown") ?? - same[0]; - if (seat === undefined) rest.push(each); - else into(seat, each); + const exact = same.filter((i) => seats[i]!.effort === effort); + const loose = same.filter((i) => seats[i]!.effort === "unknown" || effort === "unknown"); + let fits = exact.length > 0 ? exact : loose.length > 0 ? loose : same; + if (fits.length > 1 && each.role === "implementing" && fits.includes(0)) fits = [0]; + else if (fits.length > 1 && each.role === "iterating" && fits.some((i) => i !== 0)) + fits = fits.filter((i) => i !== 0); + if (fits.length === 0) rest.push(each); + else for (const seat of fits) into(seat, part(each, fits.length)); } if (rest.length > 0) { const main = [...shares].reduce<[number, Tokens] | undefined>( @@ -638,6 +646,18 @@ function share(spent: Spent[], seats: { model: string; effort: string }[]): Map< return shares; } +/** One `n`th of `tokens`. */ +function part(tokens: Tokens, n: number): Tokens { + return { + input: tokens.input / n, + cacheWrite: tokens.cacheWrite / n, + cacheRead: tokens.cacheRead / n, + output: tokens.output / n, + cost: tokens.cost / n, + complete: tokens.complete, + }; +} + /** * A branch's saved tallies span every PR from it, so only one of its PRs that `counts` counts them: the one whose * record holds the latest tally, as the most rounds show, since each round rewrites only its own PR's record; the @@ -674,7 +694,9 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { }), ).values(), ]; - const writing = each.sessions.filter((session) => session.role !== "reviewing").flatMap((s) => s.models); + const writing = each.sessions + .filter((session) => session.role !== "reviewing") + .flatMap((session) => session.models.map((spent) => ({ ...spent, role: session.role }))); const shares = each.state === "merged" ? share(writing, distinct) : new Map(); for (const [i, implementer] of distinct.entries()) { const size = implementer.size; diff --git a/test/usage.test.ts b/test/usage.test.ts index 13247fe6..0e917582 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -8,7 +8,7 @@ import { implementer } from "../src/cli/models-record"; import { fixes } from "../src/cli/land-fixer"; import type { Round, Tally } from "../src/cli/review"; import { parseReview, reviewRecord } from "../src/cli/review-record"; -import { type Session, causes, fold, parseSessions, usage, usageLines } from "../src/cli/usage"; +import { type Session, causes, fold, parseSessions, share, usage, usageLines } from "../src/cli/usage"; import { tempDir } from "./fixture"; const THREAD = "01a1b8e8-8fca-7793-99d1-062f41f8a7b0"; @@ -384,6 +384,55 @@ describe("PR records", () => { }); }); +describe("share", () => { + const spend = (model: string, effort: string, cost: number, role?: "implementing" | "iterating") => ({ + model, + effort, + input: 0, + cacheWrite: 0, + cacheRead: 0, + output: 0, + cost, + complete: true, + ...(role ? { role } : {}), + }); + const costs = (shares: Map) => [...shares].map(([i, t]) => [i, t.cost]); + + test("tells seats of one model and effort apart by role, and splits what it cannot", () => { + // An M writer on Opus medium and an S iterate on Opus medium: each session goes to its own seat. + const seats = [ + { model: "claude-opus-5-5", effort: "medium" }, + { model: "claude-opus-5-5", effort: "medium" }, + ]; + expect( + costs( + share( + [ + spend("claude-opus-5-5", "medium", 4, "implementing"), + spend("claude-opus-5-5", "medium", 2, "iterating"), + ], + seats, + ), + ), + ).toEqual([ + [0, 4], + [1, 2], + ]); + // A routed and a shadow reviewer at one effort: their spend is split evenly. + expect( + costs( + share( + [spend("gpt-6.1-sol", "low", 1)], + seats.map(() => ({ model: "gpt-6.1-sol", effort: "low" })), + ), + ), + ).toEqual([ + [0, 0.5], + [1, 0.5], + ]); + }); +}); + describe("usage", () => { const json = (value: unknown) => new Response(JSON.stringify(value), { status: 200 }); const pr = (number: number, branch: string, body: string, created: string, extra: object = {}) => ({ From 5bba01fbcb5d4b853e3aa844937b473b1d0d9108 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 17:00:32 -0800 Subject: [PATCH 5/7] fix(usage): give a reviewer the sessions its record names, and report split spend Co-Authored-By: Claude Opus 5.5 --- README.md | 9 +++++-- src/cli/usage.ts | 59 +++++++++++++++++++++++++++++++++------------- test/usage.test.ts | 22 ++++++++--------- 3 files changed, 60 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index b42c2903..1dbb20c6 100644 --- a/README.md +++ b/README.md @@ -1551,9 +1551,14 @@ writing sessions' cost on the merged PRs they were found on, so cost per merged PR divides by those PRs, not the ones no host reported: a PR's spend on a model goes to its line of that model, of the same effort where both name one, and spend on a model no line names, such as a sub-agent on a smaller model, -to the line that took the most. Local reviewers roll up the same way, by the +to the line that took the most. Spend that fits several lines goes to the +first line when an implementing session spent it and to the later ones when an +iterating one did. Local reviewers roll up the same way, by the PR's size and each reviewer's harness, model, effort, and shadow mark: PRs -reviewed, tallies, first rounds, and reviewing cost per PR. Over the merged +reviewed, tallies, first rounds, and reviewing cost per PR, where a session +the record names, or its sub-agent, goes to that reviewer. Spend still left +fitting several, such as a routed and a shadow reviewer at one effort whose +sessions no record names, is split evenly among them and reported as `split`. Over the merged PRs come each online reviewer's coverage, final heads reviewed, findings per PR reviewed, and answers. A hand-back counts only while its draft is open, so a merged PR diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 15944c08..8520f61d 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -61,6 +61,8 @@ export interface SessionRow { host: string | null; harness: string; session: string | null; + /** The session that spawned it, as a sub-agent's. */ + parent: string | null; role: Role; models: Spent[]; } @@ -101,7 +103,11 @@ export interface SpendRow extends Tokens { } /** Spend at list price over the PRs it was found on, which leave out PRs whose sessions no host reported. */ -export type Spend = { pulls: number } & Tokens; +export type Spend = { + pulls: number; + /** The part of `cost` split evenly with other seats of its model and effort that nothing told apart. */ + split: number; +} & Tokens; /** First rounds summed over the reviewed PRs whose every counted tally records one. */ export interface FirstRounds { @@ -538,6 +544,7 @@ function attribute( host: session.host, harness: session.harness, session: session.id, + parent: session.parent, role: found.role, models: session.models, }); @@ -585,7 +592,7 @@ const noCounts = (): Omit => ({ open: 0, }); const noFirst = (): FirstRounds => ({ pulls: 0, findings: { high: 0, medium: 0, low: 0 } }); -const noSpend = (): Spend => ({ pulls: 0, ...zero() }); +const noSpend = (): Spend => ({ pulls: 0, split: 0, ...zero() }); /** Add one PR's `tallies` of a cell to its counts, its rounds as their most, and to its first rounds when all have one. */ function count(to: { tally: Omit; firstRounds: FirstRounds }, tallies: Counts[]) { @@ -605,26 +612,31 @@ function count(to: { tally: Omit; firstRounds: FirstRounds }, t } /** - * A PR's spend split among `seats`, as its record names them: each model's spend to the seats of that model, of its - * effort where both name one, and spend of a model no seat names, as a sub-agent on a smaller model, to the seat - * that took the most, the PR's main one. Spend that fits several seats goes to the first, the PR's writer, when an - * implementing session spent it, and to the others when an iterating one did; what still fits several, such as a - * routed and a shadow reviewer at one effort, is split evenly among them rather than given to one. Returns each - * seat's share, by index, and nothing without a seat. + * A PR's spend split among `seats`, as its record names them: spend whose `seat` is known, as a reviewer's session its + * record names, to that seat; each model's other spend to the seats of that model, of its effort where both name one; + * and spend of a model no seat names, as a sub-agent on a smaller model, to the seat that took the most, the PR's + * main one. Spend that fits several seats goes to the first, the PR's writer, when an implementing session spent it, + * and to the others when an iterating one did; what still fits several is split evenly among them and counted as + * `split`, rather than given to one. Returns each seat's share, by index, and nothing without a seat. */ export function share( - spent: (Spent & { role?: Role })[], + spent: (Spent & { role?: Role; seat?: number })[], seats: { model: string; effort: string }[], -): Map { - const shares = new Map(); +): Map { + const shares = new Map(); if (seats.length === 0) return shares; - const into = (seat: number, from: Tokens) => { - const to = shares.get(seat) ?? zero(); + const into = (seat: number, from: Tokens, split = false) => { + const to = shares.get(seat) ?? { split: 0, ...zero() }; add(to, from); + if (split) to.split += from.cost; shares.set(seat, to); }; const rest: Spent[] = []; for (const each of spent) { + if (each.seat !== undefined) { + into(each.seat, each); + continue; + } const same = seats.flatMap((seat, i) => (seat.model === each.model ? [i] : [])); const effort = each.effort ?? "unknown"; const exact = same.filter((i) => seats[i]!.effort === effort); @@ -634,7 +646,7 @@ export function share( else if (fits.length > 1 && each.role === "iterating" && fits.some((i) => i !== 0)) fits = fits.filter((i) => i !== 0); if (fits.length === 0) rest.push(each); - else for (const seat of fits) into(seat, part(each, fits.length)); + else for (const seat of fits) into(seat, part(each, fits.length), fits.length > 1); } if (rest.length > 0) { const main = [...shares].reduce<[number, Tokens] | undefined>( @@ -697,7 +709,8 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const writing = each.sessions .filter((session) => session.role !== "reviewing") .flatMap((session) => session.models.map((spent) => ({ ...spent, role: session.role }))); - const shares = each.state === "merged" ? share(writing, distinct) : new Map(); + const shares = + each.state === "merged" ? share(writing, distinct) : new Map(); for (const [i, implementer] of distinct.entries()) { const size = implementer.size; const id = JSON.stringify([size, implementer.harness, implementer.model, implementer.effort]); @@ -730,6 +743,7 @@ function accuracyRows(pulls: PullRow[]): AccuracyRow[] { const spent = shares.get(i); if (spent) { row.spent.pulls++; + row.spent.split += spent.split; add(row.spent, spent); } // The shadow's findings measure an effort step, not the writer, so they stay out of the writer's row. @@ -777,10 +791,23 @@ function reviewerRows(pulls: PullRow[]): ReviewerRow[] { return found; }; for (const each of pulls) { - const reviewing = each.sessions.filter((session) => session.role === "reviewing").flatMap((s) => s.models); + // A session its record names, or that one's sub-agent, is that reviewer's, so a routed and a shadow reviewer + // at one effort are told apart. + const named = (session: SessionRow) => { + const at = each.review.findIndex( + (review) => + review.session !== null && + (review.session === session.session || review.session === session.parent), + ); + return at < 0 ? undefined : at; + }; + const reviewing = each.sessions + .filter((session) => session.role === "reviewing") + .flatMap((session) => session.models.map((spent) => ({ ...spent, seat: named(session) }))); for (const [i, spent] of share(reviewing, each.review)) { const to = row(each, each.review[i]!).spent; to.pulls++; + to.split += spent.split; add(to, spent); } if (!latest.has(each)) continue; diff --git a/test/usage.test.ts b/test/usage.test.ts index 0e917582..a69e0f05 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -396,7 +396,8 @@ describe("share", () => { complete: true, ...(role ? { role } : {}), }); - const costs = (shares: Map) => [...shares].map(([i, t]) => [i, t.cost]); + const costs = (shares: Map) => + [...shares].map(([i, t]) => [i, t.cost, t.split]); test("tells seats of one model and effort apart by role, and splits what it cannot", () => { // An M writer on Opus medium and an S iterate on Opus medium: each session goes to its own seat. @@ -415,20 +416,17 @@ describe("share", () => { ), ), ).toEqual([ - [0, 4], - [1, 2], + [0, 4, 0], + [1, 2, 0], ]); - // A routed and a shadow reviewer at one effort: their spend is split evenly. + // A routed and a shadow reviewer at one effort: a session the record names goes to its seat, and the rest is + // split evenly and counted as split. + const reviewers = seats.map(() => ({ model: "gpt-6.1-sol", effort: "low" })); expect( - costs( - share( - [spend("gpt-6.1-sol", "low", 1)], - seats.map(() => ({ model: "gpt-6.1-sol", effort: "low" })), - ), - ), + costs(share([{ ...spend("gpt-6.1-sol", "low", 8), seat: 0 }, spend("gpt-6.1-sol", "low", 2)], reviewers)), ).toEqual([ - [0, 0.5], - [1, 0.5], + [0, 9, 1], + [1, 1, 1], ]); }); }); From 04392afd84c954ac56f4d3a39a42a26a40281a09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 17:07:35 -0800 Subject: [PATCH 6/7] fix(usage): show split spend in the text rollups Co-Authored-By: Claude Opus 5.5 --- src/cli/usage.ts | 2 +- test/usage.test.ts | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 8520f61d..b973c98d 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -949,7 +949,7 @@ const mean = (sum: number, count: number) => (count ? (sum / count).toFixed(1) : const firsts = (f: FirstRounds) => `first rounds ${f.pulls}: ${severityText(f.findings)}`; const per = (s: Spend, what: string) => s.pulls - ? `~$${(s.cost / s.pulls).toFixed(2)}${s.complete ? "" : " or more"} per ${what} over ${s.pulls}` + ? `~$${(s.cost / s.pulls).toFixed(2)}${s.complete ? "" : " or more"} per ${what} over ${s.pulls}${s.split ? `, ~$${s.split.toFixed(2)} of it split with other seats` : ""}` : `no ${what} priced`; const percent = (part: number, whole: number) => (whole ? `${Math.round((100 * part) / whole)}%` : "-"); const answers = (r: Omit) => diff --git a/test/usage.test.ts b/test/usage.test.ts index a69e0f05..ba7083dd 100644 --- a/test/usage.test.ts +++ b/test/usage.test.ts @@ -852,6 +852,12 @@ describe("usage", () => { expect(text).toContain( " L Codex gpt-6.1-sol high 4 reviewed: rounds 2.3, findings 8 (high 4, medium 1, low 3): fixed 4, refuted 2, settled 1, open 1, first rounds 4: 4 (high 4); ~$1.25 per PR reviewed over 1\n", ); + // Spend split among seats nothing told apart is shown, so the text report does not hide it. + sol.spent.split = 0.5; + expect(usageLines(report).join("\n")).toContain( + "~$1.25 per PR reviewed over 1, ~$0.50 of it split with other seats\n", + ); + sol.spent.split = 0; expect(ten.review.map((r) => [r.effort, r.shadow])).toEqual([ ["high", false], ["high", false], From a44a1e542f902206042aaae5f846addd9eef4d9c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 13 Nov 2026 17:57:38 -0800 Subject: [PATCH 7/7] Move tallied's doc comment back above it Co-Authored-By: Claude Opus 5.5 --- src/cli/usage.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/cli/usage.ts b/src/cli/usage.ts index 91261e3b..c54a38b9 100644 --- a/src/cli/usage.ts +++ b/src/cli/usage.ts @@ -705,11 +705,6 @@ function part(tokens: Tokens, n: number): Tokens { }; } -/** - * A branch's saved tallies span every PR from it, so only one of its PRs that `counts` counts them: the one whose - * record holds the latest tally, as the most rounds show, since each round rewrites only its own PR's record; the - * newest breaks a tie. A deleted fork has no repository, so its PR stands alone. - */ const noOutcomes = (): Omit => ({ pulls: 0, merged: 0, @@ -745,6 +740,11 @@ function outcome(row: ReturnType, each: PullRow, latest: Set< ); } +/** + * A branch's saved tallies span every PR from it, so only one of its PRs that `counts` counts them: the one whose + * record holds the latest tally, as the most rounds show, since each round rewrites only its own PR's record; the + * newest breaks a tie. A deleted fork has no repository, so its PR stands alone. + */ function tallied(pulls: PullRow[], counts: (pull: PullRow) => boolean): Set { const branch = (each: PullRow) => JSON.stringify([each.repository ?? each.number, each.branch]); const rounds = (each: PullRow) =>