diff --git a/cloudflare/GITHUB.md b/cloudflare/GITHUB.md index af7f49e6..2e7deb4a 100644 --- a/cloudflare/GITHUB.md +++ b/cloudflare/GITHUB.md @@ -130,6 +130,19 @@ project; a paused project's running jobs can still preserve their work on their branches. Pull requests use `maintainer_can_modify: false`, because an upstream-scoped token cannot grant edits on the fork branch. +A planning pull request's body ends with a decision section the service +writes from the job's answered `decision` questions, after those of a failed +job whose pull request a retry continues (`src/jobs/trail.ts`): each round's +questions, every option with ✅ on each choice, and the +maintainer's own words. Permission requests and the service's `account` and +`publication` questions stay out, since they can name runners and accounts. +Prompts and answers are untrusted, so each appears only as a code span with its +line breaks and bidirectional controls removed: GitHub renders no Markdown or +HTML there and links no mention, issue, cross-repository or commit reference, +URL, or closing keyword. Every delivery renders the section again between +`` markers, after dropping any the agent wrote. Rounds +past GitHub's 65,536-character body limit are counted rather than cut. + GitHub keeps pull request status, checks, and reviews. `mergePullRequest` merges only an exact head a maintainer explicitly chose, with the first method the repository allows: squash, then a merge commit, then rebase. GitHub's diff --git a/cloudflare/README.md b/cloudflare/README.md index f60ad2c2..1ba0b55d 100644 --- a/cloudflare/README.md +++ b/cloudflare/README.md @@ -102,7 +102,10 @@ attempts it is running, and its free job slots. It also sends `checkpoint`, whenever jobs or connections change. A status that lists an assigned attempt acknowledges it; until then the assignment holds one of the runner's slots and the hub resends it, and cancelling a -job whose assignment was never acknowledged stops it at once. +job whose assignment was never acknowledged stops it at once. An `assign` +carries the job and its `instructions`, which the runner gives the agent +alongside the workflow's skill, or `null`. A planning job's instructions say +kix writes its pull request's decision record, so the agent does not. Routing gives a job to the connected runner with a free slot and the owner's most preferred usable account. A pin, the job's trust, or a paused session's diff --git a/cloudflare/src/jobs/hub.ts b/cloudflare/src/jobs/hub.ts index b29e9cd7..89ac72e9 100644 --- a/cloudflare/src/jobs/hub.ts +++ b/cloudflare/src/jobs/hub.ts @@ -4,6 +4,7 @@ import * as output from "./output"; import { MessageError, type RunnerMessage, parse } from "./protocol"; import * as store from "./store"; import { JobError } from "./store"; +import { instructions } from "./trail"; /** The authenticated runner the Worker forwards a connection for. */ export const RUNNER_HEADER = "X-Quest-Runner"; @@ -120,7 +121,10 @@ export class RunnerHub extends DurableObject { if (reported !== null) slots.set(runner, reported); } const { assign, cancel, wake } = await store.dispatch(this.env.DB, owner, slots, Date.now()); - for (const { runner, job } of assign) connections.get(runner)?.send(JSON.stringify({ type: "assign", job })); + for (const { runner, job } of assign) { + const message = { type: "assign", job, instructions: instructions(job.workflow) }; + connections.get(runner)?.send(JSON.stringify(message)); + } for (const { runner, job, attempt } of cancel) connections.get(runner)?.send(JSON.stringify({ type: "cancel", job, attempt })); if (wake === null) await this.ctx.storage.deleteAlarm(); diff --git a/cloudflare/src/jobs/publish.ts b/cloudflare/src/jobs/publish.ts index 56391d42..9b4ffba5 100644 --- a/cloudflare/src/jobs/publish.ts +++ b/cloudflare/src/jobs/publish.ts @@ -7,6 +7,7 @@ import { type Branch, type Change, GitHubService, PublicationBlocked, type GitHu import { id, text } from "./protocol"; import * as store from "./store"; import { JobError, type Runner, type Workflow } from "./store"; +import { withDecisions } from "./trail"; /** Bounds one request's GitHub writes: inline text costs no extra request, other files one each. */ export const LIMITS = { changes: 500, bytes: 16 * 1024 * 1024, request: 24 * 1024 * 1024 }; @@ -196,7 +197,12 @@ export async function publish(env: Env, runner: Runner, input: Publication): Pro const current = await store.publishing(db, runner, input.job, input.attempt); // Planning PRs always await maintainer review; so does work that was asked to stop. const draft = input.pull.draft || current.workflow === "plan" || current.cancel === 1; - const pull = await github.deliver(upstream, current.actor, current, { ...input.pull, draft }); + // A planning pull request's decision record comes from the answers kix recorded, not the agent. + const body = + current.workflow === "plan" + ? withDecisions(input.pull.body, await store.decisions(db, job.id)) + : input.pull.body; + const pull = await github.deliver(upstream, current.actor, current, { ...input.pull, body, draft }); await store.delivered(db, job.id, pull.number); return { head, pull }; } diff --git a/cloudflare/src/jobs/store.ts b/cloudflare/src/jobs/store.ts index e5d7e24a..2f7ca0e2 100644 --- a/cloudflare/src/jobs/store.ts +++ b/cloudflare/src/jobs/store.ts @@ -537,6 +537,27 @@ export async function settle(db: D1Database, job: string, commit: string, landed .run(); } +/** + * The answered agent questions behind the job's pull request, oldest first, + * for its decision record: the job's own, and those of any earlier job on its + * target that delivered the same pull request, which a retry continues. + */ +export async function decisions(db: D1Database, job: string) { + const { results } = await db + .prepare( + `SELECT q.prompt,q.answer FROM jobs c + JOIN jobs j ON j.id=c.id OR (j.repository=c.repository AND j.target=c.target AND j.pull=c.pull) + JOIN job_questions q ON q.job=j.id + WHERE c.id=? AND q.kind='agent' AND q.answer IS NOT NULL ORDER BY q.askedAt,q.rowid`, + ) + .bind(job) + .all<{ prompt: string; answer: string }>(); + return results.map((row) => ({ + prompt: JSON.parse(row.prompt) as unknown, + answer: JSON.parse(row.answer) as unknown, + })); +} + /** Remember the pull request a job delivered, so later deliveries update it. */ export async function delivered(db: D1Database, job: string, pull: number) { await db.prepare("UPDATE jobs SET pull=? WHERE id=?").bind(pull, job).run(); diff --git a/cloudflare/src/jobs/trail.ts b/cloudflare/src/jobs/trail.ts new file mode 100644 index 00000000..cffe5d78 --- /dev/null +++ b/cloudflare/src/jobs/trail.ts @@ -0,0 +1,100 @@ +// A planning pull request's paper trail. The service renders the decision +// section from the job's recorded questions and answers, so it cannot be +// paraphrased or drift from what the maintainer chose. Prompts and answers are +// untrusted text: every piece appears only inside a code span, where GitHub +// renders no Markdown or HTML and links no mention, issue or commit reference, +// URL, or closing keyword. +import { type Answer, type Prompt, parseAnswer, parsePrompt } from "./prompt"; +import type { Workflow } from "./store"; + +/** Brackets the service's section, which replaces any copy in the agent's body. */ +export const START = ""; +export const END = ""; +/** GitHub's limit on a pull request body, in characters. */ +export const BODY_LIMIT = 65_536; + +/** What a job's agent is told beyond the workflow's own skill, or null. */ +export function instructions(workflow: Workflow): string | null { + if (workflow !== "plan") return null; + return "kix adds this pull request's decision record from the maintainer's recorded answers. Do not write the interview's questions, options, or answers into the pull request body; summarize the proposed quests instead. Still record each settled decision and its reason in the quest's Plan."; +} + +/** + * Untrusted text as one inline code span. Line breaks and control characters + * become spaces, so the span cannot end its paragraph or start a line; + * bidirectional controls are dropped so the text reads as stored. The fence is + * longer than any backtick run inside, so the text cannot close it. + */ +export function code(value: string): string { + const flat = value + .replace(/[\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, "") + .replace(/[\x00-\x1f\x7f-\x9f\u2028\u2029]/g, " "); + const longest = Math.max(0, ...(flat.match(/`+/g) ?? []).map((run) => run.length)); + const fence = "`".repeat(longest + 1); + // One space on each side is stripped when rendering, so a leading or trailing backtick stays literal. + return `${fence} ${flat} ${fence}`; +} + +/** One question's record: the prompt, every option with ✅ on each choice, and any free text. */ +function question(ask: Prompt["questions"][number], reply: Answer["answers"][number]): string { + const lines = [`- ${ask.header ? `${code(ask.header)} ` : ""}${code(ask.question)}`]; + for (const option of ask.options) + lines.push(` - ${reply.choices.includes(option.label) ? "✅" : "⬜"} ${code(option.label)}`); + if (reply.text) lines.push(` - ✍️ ${code(reply.text)}`); + return lines.join("\n"); +} + +/** A stored question's prompt and answer, when it is an answered decision. */ +function decision(stored: { prompt: unknown; answer: unknown }): { prompt: Prompt; answer: Answer } | null { + try { + const prompt = parsePrompt(stored.prompt); + if (prompt.kind !== "decision") return null; + return { prompt, answer: parseAnswer(prompt, stored.answer) }; + } catch { + // Not a valid round, so there is nothing faithful to render. + return null; + } +} + +/** + * The decision section for a job's answered agent questions, oldest first, in + * at most `budget` characters. Rounds past the budget are counted, not cut. + * Null when there is nothing to record. + */ +export function render(questions: { prompt: unknown; answer: unknown }[], budget: number): string | null { + const rounds = questions.map(decision).filter((round) => round !== null); + if (!rounds.length) return null; + const head = `${START}\n## Decisions\n\nRecorded by kix from the maintainer's answers.`; + const blocks = rounds.map( + ({ prompt, answer }, i) => + `### Round ${i + 1}\n\n${prompt.questions.map((ask, j) => question(ask, answer.answers[j]!)).join("\n")}`, + ); + const omitted = (n: number) => `_${n} more ${n === 1 ? "round is" : "rounds are"} in the job's record on kix._`; + let section = head; + for (const [i, block] of blocks.entries()) { + const rest = blocks.length - i - 1; + const tail = rest ? `\n\n${omitted(rest)}` : ""; + if (`${section}\n\n${block}${tail}\n${END}`.length > budget) { + section += `\n\n${omitted(blocks.length - i)}`; + break; + } + section += `\n\n${block}`; + } + section += `\n${END}`; + return section.length > budget ? null : section; +} + +/** + * The agent's pull request body with the service's decision section: any + * section the agent wrote is removed, and the rendered one is appended. A + * section ends only at an end marker on its own line, which untrusted text + * inside a code span can never start, or else at the end of the body. + */ +export function withDecisions(body: string, questions: { prompt: unknown; answer: unknown }[]): string { + const own = body + .replace(/[\s\S]*?(?:^$|(?![\s\S]))/gm, "") + .replaceAll(END, "") + .trimEnd(); + const section = render(questions, BODY_LIMIT - own.length - 2); + return section ? (own ? `${own}\n\n${section}` : section) : own; +} diff --git a/cloudflare/test/jobs.test.ts b/cloudflare/test/jobs.test.ts index 80c72ecd..9a999421 100644 --- a/cloudflare/test/jobs.test.ts +++ b/cloudflare/test/jobs.test.ts @@ -585,7 +585,20 @@ describe("runner connections", () => { // Resuming sends no runner event, so the resume itself dispatches. await project(10, "owner"); await resume(env, 10); - expect(await live.next()).toMatchObject({ type: "assign", job: { id, attempt: 1 } }); + expect(await live.next()).toMatchObject({ type: "assign", job: { id, attempt: 1 }, instructions: null }); + }); + + it("tells a planning job's agent that kix writes its decision record", async () => { + const machine = await runner("owner", "machine"); + const live = await connect(machine.token); + const id = await job(10, "owner", { workflow: "plan", target: "plans/x", branch: "plan/x" }, Date.now()); + live.send(status(1, "main")); + expect(await live.next()).toEqual({ type: "reply", id: 1, attempts: [] }); + expect(await live.next()).toMatchObject({ + type: "assign", + job: { id, workflow: "plan" }, + instructions: expect.stringContaining("Do not write the interview's questions, options, or answers"), + }); }); it("keeps a job across a replaced connection and reassigns it after a disconnect", async () => { diff --git a/cloudflare/test/publish.test.ts b/cloudflare/test/publish.test.ts index 044f0218..0be09b8c 100644 --- a/cloudflare/test/publish.test.ts +++ b/cloudflare/test/publish.test.ts @@ -480,6 +480,134 @@ describe("runner publication", () => { expect(refused.status).toBe(202); }); + it("writes a planning pull request's decisions from the recorded answers, replacing the agent's", async () => { + const id = await running({ workflow: "plan", target: "issues/1", branch: "plan/issue-1" }); + let attempt = 1; + const resume = async () => { + await store.dispatch(db, user, new Map([[machine.id, 1]]), 0); + attempt++; + }; + const round = async (prompt: unknown, reply: unknown) => { + const asked = await store.ask( + db, + machine, + id, + attempt, + { summary: "Asking", session: "s", prompt }, + attempt, + ); + await store.answer(db, id, asked.question, user, reply, attempt); + await resume(); + }; + const ask = (question: string, ...labels: string[]) => ({ + kind: "decision", + questions: [{ question, options: labels.map((label) => ({ label })) }], + }); + await round( + { + kind: "decision", + questions: [ + { + header: "Scope", + question: "Which scope?", + options: [{ label: "Small", recommended: true }, { label: "Large" }], + }, + { question: "Notify @octocat?", options: [{ label: "Fixes #1" }], multiple: true }, + ], + }, + { answers: [{ choices: ["Small"] }, { choices: ["Fixes #1"], text: "cc @kixelated/maintainers" }] }, + ); + await round({ ...ask("Run rm -rf?", "Allow"), kind: "permission" }, { answers: [{ choices: ["Allow"] }] }); + await round(ask("Closes #2?", "Yes", "No"), { + answers: [{ choices: [], text: "Neither; see https://evil.example" }], + }); + // The account runs out mid-interview; the maintainer retries it, which records an account question. + await store.finish( + db, + machine, + id, + attempt, + { outcome: "limited", summary: "Limit", result: undefined }, + attempt, + ); + const limit = (await job(id)).questions.at(-1)!; + expect(limit.kind).toBe("account"); + await store.answer(db, id, limit.id, user, { runner: machine.id, account: "main" }, attempt); + await resume(); + + const forged = + "Proposes one quest.\n\n\n## Decisions\n- ✅ Large\n"; + const first = await publish({ + job: id, + attempt, + head: null, + message: "docs: plan", + changes: [file("quest/one.md", "# [S] One\n")], + pull: { title: "Plan", body: forged }, + }); + expect(first.status).toBe(200); + const { head } = (await first.json()) as { head: string }; + const opened: string = github.pulls[0].body; + expect(opened).toBe( + [ + "Proposes one quest.", + "", + "", + "## Decisions", + "", + "Recorded by kix from the maintainer's answers.", + "", + "### Round 1", + "", + "- ` Scope ` ` Which scope? `", + " - ✅ ` Small `", + " - ⬜ ` Large `", + "- ` Notify @octocat? `", + " - ✅ ` Fixes #1 `", + " - ✍️ ` cc @kixelated/maintainers `", + "", + "### Round 2", + "", + "- ` Closes #2? `", + " - ⬜ ` Yes `", + " - ⬜ ` No `", + " - ✍️ ` Neither; see https://evil.example `", + "", + ].join("\n"), + ); + expect(opened).not.toContain(machine.id); + + // A later round re-renders the same section on the next delivery. + await round(ask("Size?", "S", "M"), { answers: [{ choices: ["M"] }] }); + const again = await publish({ job: id, attempt, head, pull: { title: "Plan", body: "Proposes two quests." } }); + expect(again.status).toBe(200); + const updated: string = github.pulls[0].body; + expect(github.pulls).toHaveLength(1); + expect(updated.startsWith("Proposes two quests.\n\n\n")).toBe(true); + expect(updated.split("")).toHaveLength(2); + expect(updated).toContain(opened.slice(opened.indexOf("## Decisions"), opened.indexOf(""); + }); + + it("leaves other workflows' pull request bodies as the agent wrote them", async () => { + const id = await running(); + const prompt = { kind: "decision", questions: [{ question: "Q?", options: [{ label: "A" }] }] }; + const asked = await store.ask(db, machine, id, 1, { summary: "?", session: "s", prompt }, 1); + await store.answer(db, id, asked.question, user, { answers: [{ choices: ["A"] }] }, 1); + await store.dispatch(db, user, new Map([[machine.id, 1]]), 0); + const body = "Summary left alone"; + const response = await publish({ + job: id, + attempt: 2, + head: null, + message: "feat: first", + changes: [file("src/first.ts", "export {};\n")], + pull: { title: "First", body }, + }); + expect(response.status).toBe(200); + expect(github.pulls[0].body).toBe(body); + }); + it("recovers a branch update whose outcome was lost, without overwriting or duplicating", async () => { const id = await running(); github.loseMove = false; @@ -666,6 +794,32 @@ describe("runner publication", () => { expect(github.pulls).toHaveLength(1); }); + it("keeps a failed planning job's decisions in the pull request its retry continues", async () => { + const target = { workflow: "plan" as const, target: "issues/1", branch: "plan/issue-1" }; + const decide = async (id: string, question: string, choice: string) => { + const prompt = { kind: "decision", questions: [{ question, options: [{ label: choice }] }] }; + const asked = await store.ask(db, machine, id, 1, { summary: "?", session: "s", prompt }, 1); + await store.answer(db, id, asked.question, user, { answers: [{ choices: [choice] }] }, 1); + await store.dispatch(db, user, new Map([[machine.id, 1]]), 0); + }; + const failed = await running(target); + await decide(failed, "First?", "A"); + const pull = { title: "Plan", body: "Plan" }; + const changes = [file("quest/a.md", "# A\n")]; + const first = await publish({ job: failed, attempt: 2, head: null, message: "m", changes, pull }); + const { head } = (await first.json()) as { head: string }; + await store.finish(db, machine, failed, 2, { outcome: "failed", summary: "Failed", result: undefined }, 2); + + const retry = await running(target, 1000); + await decide(retry, "Second?", "B"); + const response = await publish({ job: retry, attempt: 2, head, pull }); + expect(response.status).toBe(200); + expect(github.pulls).toHaveLength(1); + const body: string = github.pulls[0].body; + expect(body).toContain("### Round 1\n\n- ` First? `\n - ✅ ` A `"); + expect(body).toContain("### Round 2\n\n- ` Second? `\n - ✅ ` B `"); + }); + it("starts a retried job fresh when its failed job published nothing", async () => { await fail(await running()); const id = await running({}, 1000); diff --git a/cloudflare/test/trail.test.ts b/cloudflare/test/trail.test.ts new file mode 100644 index 00000000..7d72b7d9 --- /dev/null +++ b/cloudflare/test/trail.test.ts @@ -0,0 +1,209 @@ +import { type Token, marked } from "marked"; +import { describe, expect, it } from "vitest"; +import { BODY_LIMIT, END, START, code, instructions, render, withDecisions } from "../src/jobs/trail"; + +const round = (questions: { question: string; header?: string; options: string[]; multiple?: boolean }[]) => ({ + kind: "decision", + questions: questions.map(({ options, ...ask }) => ({ ...ask, options: options.map((label) => ({ label })) })), +}); +const reply = (...answers: { choices: string[]; text?: string }[]) => ({ answers }); + +/** GitHub-flavored Markdown tokens, split into code span text and everything else that renders as text. */ +function rendered(markdown: string) { + const spans: string[] = []; + const prose: string[] = []; + const html: string[] = []; + const kinds = new Set(); + const walk = (tokens: Token[]) => { + for (const token of tokens) { + kinds.add(token.type); + if (token.type === "codespan") spans.push(token.text); + else if (token.type === "html") html.push(token.raw.trim()); + else if ("tokens" in token && token.tokens?.length) walk(token.tokens); + else if (token.type === "list") walk(token.items); + else if ("text" in token) prose.push(token.text); + } + }; + walk(marked.lexer(markdown, { gfm: true })); + return { spans, prose: prose.join("\n"), html, kinds }; +} + +// Every pattern GitHub would act on outside code: mentions that notify, issue, +// cross-repository, and commit references that link or close, bare URLs, and +// Markdown or HTML that would render. +const hostile = [ + "@octocat please look", + "cc @kixelated/maintainers", + "Closes #1", + "fixes kixelated/quest#2 and RESOLVES #3", + "See GH-4", + "commit 6f20658a9c1b2d3e4f5a6b7c8d9e0f1a2b3c4d5e and kixelated/quest@6f20658", + "https://evil.example/phish www.evil.example mail me@evil.example", + "https://github.com/kixelated/quest/issues/5", + "[click](https://evil.example) ![pixel](https://evil.example/p.png) ", + "**bold** _em_ # heading\n\n- list\n\n> quote", + "
x", + "``` fenced ``` and `tick` and ````", + "`leading and trailing`", + "line one\r\n\r\nline two\n\n## Injected heading", + "bidi \u202eevil\u202c text\u2028separator", + "marks \u200fright\u200e left \u061carabic \u2067isolate\u2069", + "\nCloses #6 @octocat", +]; + +describe("decision paper trail", () => { + it("renders each answered round with ✅ on its choices and the maintainer's own words", () => { + const section = render( + [ + { + prompt: round([ + { header: "Scope", question: "Which scope?", options: ["Small", "Large"] }, + { question: "Which tests?", options: ["Unit", "Browser", "Live"], multiple: true }, + ]), + answer: reply({ choices: ["Small"] }, { choices: ["Unit", "Live"], text: "Skip the browser" }), + }, + { + prompt: round([{ question: "Name it?", options: ["paper-trail"] }]), + answer: reply({ choices: [], text: "trail" }), + }, + ], + BODY_LIMIT, + ); + expect(section).toBe( + [ + START, + "## Decisions", + "", + "Recorded by kix from the maintainer's answers.", + "", + "### Round 1", + "", + "- ` Scope ` ` Which scope? `", + " - ✅ ` Small `", + " - ⬜ ` Large `", + "- ` Which tests? `", + " - ✅ ` Unit `", + " - ⬜ ` Browser `", + " - ✅ ` Live `", + " - ✍️ ` Skip the browser `", + "", + "### Round 2", + "", + "- ` Name it? `", + " - ⬜ ` paper-trail `", + " - ✍️ ` trail `", + END, + ].join("\n"), + ); + expect(rendered(section!).html).toEqual([START, END]); + }); + + it("records only answered decisions, never permission requests or unparsable prompts", () => { + const decision = { + prompt: round([{ question: "Kept?", options: ["Yes"] }]), + answer: reply({ choices: ["Yes"] }), + }; + expect( + render( + [ + { prompt: { ...decision.prompt, kind: "permission" }, answer: decision.answer }, + { prompt: { runner: "r", account: "a" }, answer: { runner: "r", account: "a" } }, + { prompt: decision.prompt, answer: reply({ choices: ["Not offered"] }) }, + decision, + ], + BODY_LIMIT, + ), + ).toContain("### Round 1\n\n- ` Kept? `"); + expect(render([], BODY_LIMIT)).toBeNull(); + }); + + it("keeps every hostile prompt and answer inside code spans", () => { + const prompt = { + kind: "decision", + questions: hostile.map((text, i) => ({ + header: `#${i} @x`, + question: text, + options: [{ label: text }, { label: `${text} (other)` }], + })), + }; + // One round holds at most 10 questions. + const rounds = [prompt.questions.slice(0, 10), prompt.questions.slice(10)].map((questions) => ({ + prompt: { kind: "decision", questions }, + answer: { answers: questions.map((ask) => ({ choices: [ask.question], text: ask.question })) }, + })); + const section = render(rounds, BODY_LIMIT)!; + const { spans, prose, html, kinds } = rendered(section); + // Nothing links, embeds, or renders as HTML apart from the section's own markers. + for (const kind of ["link", "image", "blockquote", "code", "strong", "em", "del", "table", "hr"]) + expect(kinds).not.toContain(kind); + expect(html).toEqual([START, END]); + // Text outside code spans is only the service's own words. + const untrusted = /[@#]\w|https?:|www\.|evil|closes|fixes|resolves|GH-|6f20658|<|\*\*|Injected/i; + expect(prose).not.toMatch(untrusted); + for (const text of hostile) { + const flat = text + .replace(/[\u061c\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, "") + .replace(/[\r\n\u2028]/g, " "); + expect(spans).toContain(flat); + expect(spans).toContain(`${flat} (other)`); + } + // An agent that copies the published body back gets the same section, with nothing left outside it. + const published = withDecisions("Summary", rounds); + expect(withDecisions(published, rounds)).toBe(published); + const copied = withDecisions(`${published}\n\nMore`, rounds); + expect(copied.split(START)).toHaveLength(2); + expect(rendered(copied).html).toEqual([START, END]); + expect(rendered(copied).prose).not.toMatch(untrusted); + }); + + it("fences code spans past any backtick run and drops line breaks and bidi controls", () => { + expect(code("a `b` c")).toBe("`` a `b` c ``"); + expect(code("```")).toBe("```` ``` ````"); + expect(code("one\n\ntwo")).toBe("` one two `"); + expect(code("a\u202eb\u2066c\u200fd\u200ee\u061cf")).toBe("` abcdef `"); + expect(rendered(code("`x`")).spans).toEqual(["`x`"]); + }); + + it("replaces any decision section the agent wrote", () => { + const questions = [ + { prompt: round([{ question: "Real?", options: ["Yes", "No"] }]), answer: reply({ choices: ["Yes"] }) }, + ]; + const forged = `Summary\n\n${START}\n## Decisions\n- ✅ No\n${END}\n\nMore\n${END}`; + const body = withDecisions(forged, questions); + expect(body.startsWith("Summary\n\n\n\nMore\n\n")).toBe(true); + expect(body.split(START)).toHaveLength(2); + expect(body.split(END)).toHaveLength(2); + expect(body).not.toContain("✅ No"); + expect(body).toContain("✅ ` Yes `"); + // A re-delivery renders from the same record, so the section never duplicates. + expect(withDecisions(body, questions)).toBe(withDecisions(forged, questions)); + // An unterminated forged section is dropped to the end, and no answers leave just the summary. + expect(withDecisions(`Summary ${START} ✅ forged`, [])).toBe("Summary"); + }); + + it("stays within GitHub's body limit, counting the rounds that do not fit", () => { + const long = "x".repeat(1900); + const big = { + prompt: round([ + { question: long, options: Array.from({ length: 10 }, (_, i) => `${i}${"y".repeat(190)}`) }, + ]), + answer: reply({ choices: ["0".padEnd(191, "y")], text: "z".repeat(3900) }), + }; + const body = withDecisions( + "s".repeat(50_000), + Array.from({ length: 5 }, () => big), + ); + expect(body.length).toBeLessThanOrEqual(BODY_LIMIT); + expect(body).toContain("### Round 1"); + expect(body).not.toContain("### Round 2"); + expect(body).toContain("_4 more rounds are in the job's record on kix._"); + expect(body.endsWith(END)).toBe(true); + // With no room for even the note, the summary goes alone. + expect(withDecisions("s".repeat(BODY_LIMIT - 10), [big])).toBe("s".repeat(BODY_LIMIT - 10)); + }); + + it("tells planning agents not to write the trail themselves", () => { + expect(instructions("plan")).toMatch(/Do not write the interview's questions, options, or answers/); + for (const workflow of ["start", "iterate", "bootstrap"] as const) expect(instructions(workflow)).toBeNull(); + }); +}); diff --git a/quest/m0/kix/README.md b/quest/m0/kix/README.md index b2061a20..4c8ccef2 100644 --- a/quest/m0/kix/README.md +++ b/quest/m0/kix/README.md @@ -102,7 +102,6 @@ operator-paid-hosting decisions: - [Job sandbox](/quest/m0/kix/sandbox.md) - OS-sandboxed commands, with untrusted PR devShells built in a VM on Linux runners - [opencode adapter](/quest/m0/kix/opencode.md) - OpenCode Go, ChatGPT plans, and API-key providers through opencode - [Merge jobs](/quest/m0/kix/merge-job.md) - a merge pick fixes conflicts, CI, and reviews on a runner, then lands the PR -- [Decision paper trail](/quest/m0/kix/paper-trail.md) - planning PRs carry the recorded questions and answers, rendered by the service - [Agent activity](/quest/m0/kix/activity.md) - agent cards and map highlights built from checkpoint summaries - [Web planning](/quest/m0/kix/planning.md) - issue-to-quest interviews produce draft planning PRs - [Web execution](/quest/m0/kix/execution.md) - spawn and iterate through the UI, ending in contribution PRs diff --git a/quest/m0/kix/paper-trail.md b/quest/m0/kix/paper-trail.md deleted file mode 100644 index 4c43e049..00000000 --- a/quest/m0/kix/paper-trail.md +++ /dev/null @@ -1,40 +0,0 @@ -# [S] Render planning decisions into PRs from recorded answers - -## Goal - -A planning PR published by kix carries a decision section that the service -writes from the job's recorded questions and answers, with ✅ on each chosen -option and any free-text answer. The agent no longer writes its own paper -trail, so the record cannot be paraphrased or drift from what the maintainer -actually chose. - -## Plan - -Decided 2026-10-19 as a follow-up of bot publication (#145), which takes the -paper trail from the agent's quest files and PR body: - -- The service renders the section from the job's stored questions and - answers (the decision inbox's prompt contract) when it creates or updates - the planning PR. Include only answered `agent` questions whose prompt is a - `decision`, not a `permission` request. Operational question kinds such as - `account` and `publication` stay out, since they can name runners and - accounts in a public PR. -- Prompts and answers are untrusted text. Escaping Markdown is not enough: - also neutralize GitHub's own semantics, so no `@` mention notifies anyone, - no `#N`, `owner/repo#N`, or commit reference autolinks, no bare URL links, - and no closing keyword (`Closes #N`, `Fixes`, `Resolves`) links or closes - an issue. For example, render all prompt text in code spans or break each - pattern with a zero-width character. -- Replace, don't append: the agent's PR body keeps its summary, and the - service owns the decision section. Re-rendering on later deliveries - updates the same section in place rather than duplicating it. -- Quest files still record each settled decision and its reason in their - Plan, as the guide requires; this changes only the PR's paper trail. -- Tell agents in kix planning jobs, through the job's own instructions - rather than the shared skills, not to write the trail. Local agents running - the skills without kix keep writing their own. -- Test multi-round interviews, free-text answers, a re-delivery that - updates the section, and a planning job that hit an account limit, whose - account question must not appear. Cover hostile prompts with Markdown, - `@user` and `@org/team` mentions, bare URLs, issue and cross-repository - references, and closing keywords. diff --git a/quest/m0/kix/planning.md b/quest/m0/kix/planning.md index 55f2d44b..d88e947c 100644 --- a/quest/m0/kix/planning.md +++ b/quest/m0/kix/planning.md @@ -38,8 +38,9 @@ Earlier guidance: - Keep the current frontier of questions, prior answers, and draft proposal across reconnects. Consequential choices wait for answers; time passing and selected defaults are not approval. -- The bot creates/updates a draft quest PR with settled decisions and the - interview paper trail. Merging through GitHub makes the resulting ready +- The bot creates/updates a draft quest PR with settled decisions; the + service renders the interview paper trail from the recorded answers + ([publication](/cloudflare/GITHUB.md#publication)). Merging through GitHub makes the resulting ready quests eligible; a draft planning branch cannot start implementation. - Reuse the same prompt flow when an implementation worker needs a decision. Record the settled answer in its plan where relevant without adding a diff --git a/quest/m0/kix/runner.md b/quest/m0/kix/runner.md index 0d3c6462..448d5ac7 100644 --- a/quest/m0/kix/runner.md +++ b/quest/m0/kix/runner.md @@ -71,7 +71,8 @@ replacing the hosted Agent SDK runtime: says is no longer this runner's or whose `checkpoint` or `question` reply says `cancel`. Acknowledge an `assign` by sending a status that lists it; the hub resends it until then, so treat a repeated assignment - as one. Reconnect with backoff; running jobs move after two minutes + as one. Give the agent an assignment's `instructions`, when present, + alongside the workflow's skill. Reconnect with backoff; running jobs move after two minutes away. Each account's opaque ID must stay stable across restarts so its preference rank and paused sessions follow it. A CLI sign-in flow and a control to reorder account preference remain here.