From de0dfbbe4372c76ce146088438836e64a7c5d4ff Mon Sep 17 00:00:00 2001 From: wz-gsa Date: Tue, 13 Oct 2026 21:56:13 +0000 Subject: [PATCH 1/2] feat(usai-catalog): select default models in the catalog pipeline Add a harness-neutral defaults block to the catalog so role selection is derived once from live model data instead of hardcoded in a kit-specific writer. Resolve primary, small, and compaction from priority candidates, fail closed when no candidate resolves, and reject dangling or unsafe default references during validation. Add an OpenCode defaults emitter and a config patcher that reproduces both shipped configs byte-for-byte, with scoped replacements that cannot escape their target keys. Closes GSA-TTS/agentic-coding-patterns#365 Co-authored-by: OpenCode Agent --- integrations/providers/usai/catalog.json | 5 + .../providers/usai/catalog.schema.json | 38 ++++++- .../providers/usai/emitters/opencode.mjs | 99 +++++++++++++++++++ .../providers/usai/scripts/build-catalog.mjs | 71 ++++++++++++- .../usai/tests/catalog-check.test.mjs | 8 +- .../usai/tests/catalog-validation.test.mjs | 45 ++++++++- .../providers/usai/tests/emitters.test.mjs | 94 +++++++++++++++++- 7 files changed, 351 insertions(+), 9 deletions(-) diff --git a/integrations/providers/usai/catalog.json b/integrations/providers/usai/catalog.json index b5852f75..2e8eb700 100644 --- a/integrations/providers/usai/catalog.json +++ b/integrations/providers/usai/catalog.json @@ -36,6 +36,11 @@ "usaiBackend": "amazon-bedrock" } ], + "defaults": { + "primary": "claude-sonnet-5", + "small": "claude_4_5_haiku", + "compaction": "gpt-5.6-terra" + }, "models": [ { "id": "claude_4_5_haiku", diff --git a/integrations/providers/usai/catalog.schema.json b/integrations/providers/usai/catalog.schema.json index ce5ef08e..18d9c21b 100644 --- a/integrations/providers/usai/catalog.schema.json +++ b/integrations/providers/usai/catalog.schema.json @@ -5,7 +5,7 @@ "description": "Harness-neutral catalog of USAi gateway models. Generated from live feeds by build-catalog.mjs; consumed by per-harness emitters. Do not hand-edit the generated catalog.json.", "type": "object", "additionalProperties": false, - "required": ["schemaVersion", "gateway", "vendors", "models"], + "required": ["schemaVersion", "gateway", "vendors", "defaults", "models"], "properties": { "schemaVersion": { "type": "string", @@ -50,6 +50,42 @@ } } }, + "defaults": { + "type": "object", + "description": "Recommended default models for standard agent roles. Every value must reference a valid model id in models[].", + "additionalProperties": false, + "required": ["primary", "small"], + "properties": { + "primary": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Primary interactive coding and instruction-following model." + }, + "small": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Fast, lower-cost model for lightweight sub-tasks and completions." + }, + "compaction": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Cost-effective model for context compaction and conversation summarization." + }, + "reasoning": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "High-capacity reasoning and architectural planning model." + } + } + }, "vendors": { "type": "array", "description": "Model vendors (Anthropic, OpenAI, Google, ...), for grouping and display order.", diff --git a/integrations/providers/usai/emitters/opencode.mjs b/integrations/providers/usai/emitters/opencode.mjs index 751296cb..3a68604a 100644 --- a/integrations/providers/usai/emitters/opencode.mjs +++ b/integrations/providers/usai/emitters/opencode.mjs @@ -221,4 +221,103 @@ export function emitOpenCodeBlockFromCatalog(catalog) { return lines.join("\n") } +/** + * Map neutral catalog defaults to OpenCode configuration values. + * Values are prefixed with "usai/" per OpenCode provider model namespace convention. + * + * @param {object} catalog parsed catalog.json + * @returns {object} OpenCode formatted defaults { model, small_model, compaction } + */ +export function emitOpenCodeDefaults(catalog) { + if (!catalog || typeof catalog !== "object") { + throw new TypeError("emitOpenCodeDefaults: catalog must be an object") + } + if (!catalog.defaults || typeof catalog.defaults !== "object" || Array.isArray(catalog.defaults)) { + throw new TypeError("emitOpenCodeDefaults: catalog.defaults must be an object") + } + const { primary, small, compaction } = catalog.defaults + if (typeof primary !== "string" || primary.length === 0) { + throw new TypeError("emitOpenCodeDefaults: defaults.primary must be a non-empty string") + } + if (typeof small !== "string" || small.length === 0) { + throw new TypeError("emitOpenCodeDefaults: defaults.small must be a non-empty string") + } + if (compaction !== undefined && (typeof compaction !== "string" || compaction.length === 0)) { + throw new TypeError("emitOpenCodeDefaults: defaults.compaction must be a non-empty string when present") + } + + // Ensure chosen default IDs do not contain characters that could break out of JSON strings + // or trigger unexpected template expansions + const SAFE_ID_RE = /^[A-Za-z0-9_.:-]+$/ + if (!SAFE_ID_RE.test(primary)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.primary contains invalid characters: "${primary}"`) + } + if (!SAFE_ID_RE.test(small)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.small contains invalid characters: "${small}"`) + } + if (compaction && !SAFE_ID_RE.test(compaction)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.compaction contains invalid characters: "${compaction}"`) + } + + return { + model: `usai/${primary}`, + small_model: `usai/${small}`, + compaction: compaction ? `usai/${compaction}` : null, + } +} + +/** + * Patch an existing opencode.jsonc template text with both the models block + * and the default model selections from the catalog. + * + * @param {string} templateText raw opencode.jsonc text + * @param {object} catalog parsed catalog.json + * @returns {string} updated opencode.jsonc text + */ +export function patchOpenCodeConfig(templateText, catalog) { + if (typeof templateText !== "string") { + throw new TypeError("patchOpenCodeConfig: templateText must be a string") + } + const defaults = emitOpenCodeDefaults(catalog) + const modelsBlock = emitOpenCodeBlockFromCatalog(catalog) + + // 1. Replace generated models block + const beginIdx = templateText.indexOf(BEGIN_MARKER) + const endIdx = templateText.indexOf(END_MARKER) + if (beginIdx === -1 || endIdx === -1 || endIdx <= beginIdx) { + throw new Error("patchOpenCodeConfig: BEGIN/END GENERATED USAI MODELS markers not found in template") + } + + const beforeMarker = templateText.slice(0, beginIdx) + const afterMarker = templateText.slice(endIdx + END_MARKER.length) + + let updated = `${beforeMarker}${modelsBlock.trim()}${afterMarker}` + + // 2. Replace top-level model and small_model + // Anchor to line start to avoid matching comments, and use function replacements to avoid `$` issues + const modelPattern = /^([ \t]*"model"[ \t]*:[ \t]*")([^"]*)(")/m + if (!modelPattern.test(updated)) { + throw new Error("patchOpenCodeConfig: Could not find top-level 'model' in template") + } + updated = updated.replace(modelPattern, (_m, p1, _p2, p3) => `${p1}${defaults.model}${p3}`) + + const smallModelPattern = /^([ \t]*"small_model"[ \t]*:[ \t]*")([^"]*)(")/m + if (!smallModelPattern.test(updated)) { + throw new Error("patchOpenCodeConfig: Could not find top-level 'small_model' in template") + } + updated = updated.replace(smallModelPattern, (_m, p1, _p2, p3) => `${p1}${defaults.small_model}${p3}`) + + // 3. Replace agent.compaction.model if present in defaults and template + // Scoped strictly within "agent": { ... "compaction": { ... "model": "..." } } + // to avoid matching across closing braces + if (defaults.compaction) { + const compactionPattern = /("agent"\s*:\s*\{[^{}]*?"compaction"\s*:\s*\{[^{}]*?"model"\s*:\s*")([^"]*)(")/ + if (compactionPattern.test(updated)) { + updated = updated.replace(compactionPattern, (_m, p1, _p2, p3) => `${p1}${defaults.compaction}${p3}`) + } + } + + return updated +} + export default emitOpenCodeBlockFromCatalog diff --git a/integrations/providers/usai/scripts/build-catalog.mjs b/integrations/providers/usai/scripts/build-catalog.mjs index 5133d7ff..c96d3751 100644 --- a/integrations/providers/usai/scripts/build-catalog.mjs +++ b/integrations/providers/usai/scripts/build-catalog.mjs @@ -755,6 +755,31 @@ function validateCatalog(catalog, schema) { } } + // defaults + assert(catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults), "defaults must be an object", errors) + if (catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults)) { + const defaultsAllowed = new Set(["primary", "small", "compaction", "reasoning"]) + for (const k of Object.keys(catalog.defaults)) { + assert(defaultsAllowed.has(k), `defaults: disallowed key "${k}"`, errors) + } + for (const r of ["primary", "small"]) { + assert(r in catalog.defaults, `defaults: missing required key "${r}"`, errors) + } + const modelIds = new Set((catalog.models || []).map((m) => m.id)) + const defaultsProperties = schema.properties.defaults.properties + for (const [role, id] of Object.entries(catalog.defaults)) { + if (defaultsProperties[role]) { + validateStringFromSchema(id, defaultsProperties[role], `defaults.${role}`, errors) + } else { + assert(typeof id === "string", `defaults.${role} must be string`, errors) + } + if (typeof id === "string") { + assert(!id.includes("/"), `defaults.${role} "${id}" must not include a provider prefix`, errors) + assert(modelIds.has(id), `defaults.${role} references unknown model "${id}" (not in models[])`, errors) + } + } + } + // models assert(Array.isArray(catalog.models), "models must be an array", errors) if (Array.isArray(catalog.models)) { @@ -784,14 +809,48 @@ function validateCatalog(catalog, schema) { return errors } +// ----------------------------------------------------------------------------- +// Default role candidates in priority order. +// Roles represent standard agentic execution tiers: +// primary: flagship instruction-following and multi-step reasoning (OpenCode `model`) +// small: fast, cost-effective lightweight completions (OpenCode `small_model`) +// compaction: context compaction and conversation summarization (OpenCode `agent.compaction.model`) +// ----------------------------------------------------------------------------- +const DEFAULT_ROLE_PRIORITY = { + primary: ["claude-sonnet-5", "claude_4_5_sonnet", "claude-opus-5", "claude_4_5_opus"], + small: ["claude_4_5_haiku", "claude-3-5-haiku"], + compaction: ["gpt-5.6-terra", "gpt_5_5_default_v2", "gpt-5.4-latest-guardrails-defaultv2"], +} + +function resolveDefaultRole(models, priorityList, existingValue, roleName) { + const modelIds = new Set(models.map((m) => m.id)) + for (const candidate of priorityList) { + if (modelIds.has(candidate)) return candidate + } + if (existingValue && modelIds.has(existingValue)) return existingValue + throw new Error( + `Cannot resolve default for role "${roleName}": none of the priority candidates ` + + `([${priorityList.join(", ")}]) or fallback "${existingValue}" exist in the catalog.`, + ) +} + +function resolveCatalogDefaults(models, existingDefaults = {}) { + return { + primary: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.primary, existingDefaults.primary, "primary"), + small: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.small, existingDefaults.small, "small"), + compaction: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.compaction, existingDefaults.compaction, "compaction"), + } +} + // ----------------------------------------------------------------------------- // Deterministic serialization: keys in a fixed order, 2-space indent, trailing // newline. Order is stable so the committed catalog.json is reproducible. // ----------------------------------------------------------------------------- const KEY_ORDER = { - root: ["schemaVersion", "generatedBy", "sources", "gateway", "vendors", "models"], + root: ["schemaVersion", "generatedBy", "sources", "gateway", "vendors", "defaults", "models"], gateway: ["host", "baseUrl", "api", "apiKeyEnv"], vendor: ["key", "label", "order", "usaiBackend"], + defaults: ["primary", "small", "compaction", "reasoning"], model: ["id", "vendor", "name", "contextWindow", "maxOutputTokens", "cost", "costAbove200kContext", "reasoning", "inputModalities"], cost: ["input", "output", "cacheRead", "cacheWrite"], } @@ -808,6 +867,7 @@ function orderCatalog(catalog) { const out = orderObject(catalog, KEY_ORDER.root) if (out.gateway) out.gateway = orderObject(out.gateway, KEY_ORDER.gateway) if (Array.isArray(out.vendors)) out.vendors = out.vendors.map((v) => orderObject(v, KEY_ORDER.vendor)) + if (out.defaults) out.defaults = orderObject(out.defaults, KEY_ORDER.defaults) if (Array.isArray(out.models)) { out.models = out.models.map((m) => { const om = orderObject(m, KEY_ORDER.model) @@ -877,6 +937,7 @@ async function buildCatalog(args) { else throw new Error("models.dev source has unexpected shape (expected provider-keyed object)") } const { models, vendorOrder } = shapeFromFeeds(usaiList, modelsDevCatalog) + const defaults = resolveCatalogDefaults(models) catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, @@ -886,6 +947,7 @@ async function buildCatalog(args) { }, gateway: buildGateway(), vendors: buildVendors(vendorOrder), + defaults, models, } } else { @@ -898,6 +960,7 @@ async function buildCatalog(args) { // Preserve vendor first-appearance order from the block. const vendorOrder = [] for (const m of models) if (!vendorOrder.includes(m.vendor)) vendorOrder.push(m.vendor) + const defaults = resolveCatalogDefaults(models) catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, @@ -906,6 +969,7 @@ async function buildCatalog(args) { }, gateway: buildGateway(), vendors: buildVendors(vendorOrder), + defaults, models, } } @@ -945,6 +1009,9 @@ function describeDrift(existing, fresh) { if (JSON.stringify(existing?.vendors) !== JSON.stringify(fresh?.vendors)) { lines.push("vendors block changed") } + if (JSON.stringify(existing?.defaults) !== JSON.stringify(fresh?.defaults)) { + lines.push("defaults block changed") + } const byId = (list) => new Map((list || []).map((m) => [m.id, m])) const before = byId(existing?.models) @@ -1003,6 +1070,8 @@ export { serialize, substantive, describeDrift, + resolveCatalogDefaults, + DEFAULT_ROLE_PRIORITY, parseBootstrapModels, bootstrapModelsFromBlock, shapeFromFeeds, diff --git a/integrations/providers/usai/tests/catalog-check.test.mjs b/integrations/providers/usai/tests/catalog-check.test.mjs index 0c105593..6dce647a 100644 --- a/integrations/providers/usai/tests/catalog-check.test.mjs +++ b/integrations/providers/usai/tests/catalog-check.test.mjs @@ -35,7 +35,7 @@ test("substantive() strips provenance and keeps rendered content", () => { const catalog = loadCatalog() const stripped = substantive(catalog) assert.equal("sources" in stripped, false, "sources must be excluded") - for (const key of ["schemaVersion", "generatedBy", "gateway", "vendors", "models"]) { + for (const key of ["schemaVersion", "generatedBy", "gateway", "vendors", "defaults", "models"]) { assert.ok(key in stripped, `${key} must be retained`) } }) @@ -108,6 +108,12 @@ test("gateway and vendor changes ARE drift", () => { assert.deepEqual(describeDrift(substantive(before), substantive(vendorsChanged)), [ "vendors block changed", ]) + + const defaultsChanged = loadCatalog() + defaultsChanged.defaults = { ...defaultsChanged.defaults, primary: "claude-opus-5" } + assert.deepEqual(describeDrift(substantive(before), substantive(defaultsChanged)), [ + "defaults block changed", + ]) }) test("schemaVersion drift is reported with both values", () => { diff --git a/integrations/providers/usai/tests/catalog-validation.test.mjs b/integrations/providers/usai/tests/catalog-validation.test.mjs index 89546ace..f9d91825 100644 --- a/integrations/providers/usai/tests/catalog-validation.test.mjs +++ b/integrations/providers/usai/tests/catalog-validation.test.mjs @@ -8,9 +8,21 @@ const catalog = JSON.parse(readFileSync(new URL("../catalog.json", import.meta.u const schema = JSON.parse(readFileSync(new URL("../catalog.schema.json", import.meta.url), "utf8")) function withFirstModel(overrides) { + // Use models[1] (claude_4_5_sonnet) when modifying the first model, + // or update defaults so defaults don't point to an overridden ID. + const m0 = catalog.models[0] return { ...catalog, - models: [{ ...catalog.models[0], ...overrides }, ...catalog.models.slice(1)], + models: [{ ...m0, ...overrides }, ...catalog.models.slice(1)], + } +} + +function withNonDefaultModel(overrides) { + // models[1] is claude_4_5_sonnet which is not currently defaults.primary, small, or compaction + const m1 = catalog.models[1] + return { + ...catalog, + models: [catalog.models[0], { ...m1, ...overrides }, ...catalog.models.slice(2)], } } @@ -19,7 +31,7 @@ test("the committed catalog satisfies the hand-rolled validator", () => { }) test("catalog validation rejects model ids that can escape generated config", () => { - assert.deepEqual(validateCatalog(withFirstModel({ id: 'provider/"quoted"\\model:v1' }), schema), []) + assert.deepEqual(validateCatalog(withNonDefaultModel({ id: 'provider/"quoted"\\model:v1' }), schema), []) }) test("catalog validation rejects control characters in display names", () => { @@ -28,17 +40,17 @@ test("catalog validation rejects control characters in display names", () => { test("catalog validation rejects control characters in ids", () => { for (const id of ["line\nbreak", "carriage\rreturn", "nul\u0000byte", "delete\u007fbyte"]) { - assert.match(validateCatalog(withFirstModel({ id }), schema).join("\n"), /id does not match/) + assert.match(validateCatalog(withNonDefaultModel({ id }), schema).join("\n"), /id does not match/) } }) test("catalog validation bounds feed-controlled string lengths", () => { - assert.match(validateCatalog(withFirstModel({ id: "a".repeat(129) }), schema).join("\n"), /id exceeds 128/) + assert.match(validateCatalog(withNonDefaultModel({ id: "a".repeat(129) }), schema).join("\n"), /id exceeds 128/) assert.match(validateCatalog(withFirstModel({ name: "a".repeat(257) }), schema).join("\n"), /name exceeds 256/) }) test("catalog validation allows provider-qualified ids and counts Unicode code points", () => { - assert.deepEqual(validateCatalog(withFirstModel({ id: "provider/model:v1" }), schema), []) + assert.deepEqual(validateCatalog(withNonDefaultModel({ id: "provider/model:v1" }), schema), []) assert.deepEqual(validateCatalog(withFirstModel({ name: "😀".repeat(128) }), schema), []) }) @@ -73,3 +85,26 @@ test("catalog validation rejects non-positive and fractional limits", () => { assert.match(validateCatalog(withFirstModel({ maxOutputTokens: value }), schema).join("\n"), /positive integer/) } }) + +test("catalog validation enforces defaults structure and referential integrity", () => { + assert.match( + validateCatalog({ ...catalog, defaults: null }, schema).join("\n"), + /defaults must be an object/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, extraRole: "claude-sonnet-5" } }, schema).join("\n"), + /disallowed key "extraRole"/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { small: "claude_4_5_haiku" } }, schema).join("\n"), + /missing required key "primary"/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, primary: "usai/claude-sonnet-5" } }, schema).join("\n"), + /must not include a provider prefix/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, primary: "nonexistent-model" } }, schema).join("\n"), + /references unknown model "nonexistent-model"/, + ) +}) diff --git a/integrations/providers/usai/tests/emitters.test.mjs b/integrations/providers/usai/tests/emitters.test.mjs index 59d47626..6b06ac73 100644 --- a/integrations/providers/usai/tests/emitters.test.mjs +++ b/integrations/providers/usai/tests/emitters.test.mjs @@ -20,7 +20,11 @@ import path from "node:path" import { fileURLToPath } from "node:url" import { test } from "node:test" -import { emitOpenCodeBlockFromCatalog } from "../emitters/opencode.mjs" +import { + emitOpenCodeBlockFromCatalog, + emitOpenCodeDefaults, + patchOpenCodeConfig, +} from "../emitters/opencode.mjs" const HERE = path.dirname(fileURLToPath(import.meta.url)) const USAI_DIR = path.resolve(HERE, "..") @@ -193,3 +197,91 @@ test("duplicate ids and multiline vendor labels are refused", () => { duplicateVendor.vendors.push({ ...duplicateVendor.vendors[0] }) assert.throws(() => emitOpenCodeBlockFromCatalog(duplicateVendor), /duplicate vendor key/) }) + +for (const { label, file } of SHIPPED_CONFIGS) { + test(`patchOpenCodeConfig reproduces the ${label} shipped opencode.jsonc byte-for-byte`, () => { + const originalText = readFileSync(file, "utf8") + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig(originalText, catalog) + + if (patched !== originalText) { + // eslint-disable-next-line no-console + console.error(`[${label} patchOpenCodeConfig mismatch]\n${firstMismatchReport(originalText, patched)}`) + } + assert.strictEqual( + patched, + originalText, + `patchOpenCodeConfig does not reproduce ${label} opencode.jsonc byte-for-byte`, + ) + }) +} + +test("emitOpenCodeDefaults maps neutral catalog defaults to usai/ prefixed roles", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const defaults = emitOpenCodeDefaults(catalog) + assert.deepEqual(defaults, { + model: "usai/claude-sonnet-5", + small_model: "usai/claude_4_5_haiku", + compaction: "usai/gpt-5.6-terra", + }) +}) + +test("emitOpenCodeDefaults rejects malformed defaults or invalid characters", () => { + assert.throws(() => emitOpenCodeDefaults(null), /must be an object/) + assert.throws(() => emitOpenCodeDefaults({ defaults: null }), /must be an object/) + assert.throws(() => emitOpenCodeDefaults({ defaults: { small: "haiku" } }), /defaults.primary must be/) + assert.throws(() => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "" } }), /defaults.small must be/) + for (const bad of ['sonnet", "permission": {}', "model\\escape", "model{env:VAR}", "model}"]) { + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: bad, small: "haiku" } }), + /invalid characters/, + ) + } +}) + +test("patchOpenCodeConfig safely handles $ in model ids without string replacement expansion", () => { + const originalText = readFileSync(SHIPPED_CONFIGS[0].file, "utf8") + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + // Add a test model and set as default to check replacement function + const testCat = { + ...catalog, + models: [ + { + id: "model_with_dollar", + vendor: "anthropic", + name: "Dollar Model", + contextWindow: 1000, + maxOutputTokens: 100, + }, + ...catalog.models, + ], + defaults: { + primary: "model_with_dollar", + small: "claude_4_5_haiku", + }, + } + const patched = patchOpenCodeConfig(originalText, testCat) + assert.ok(patched.includes('"model": "usai/model_with_dollar"')) +}) + +test("patchOpenCodeConfig does not cross object braces if agent.compaction is missing", () => { + // A template without agent.compaction.model + const template = ` +{ + "provider": { "usai": { "models": { +// BEGIN GENERATED USAI MODELS +// END GENERATED USAI MODELS + } } }, + "model": "usai/old-model", + "small_model": "usai/old-small", + "compaction": { "auto": true }, + "instructions": "important instructions" +} +` + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig(template, catalog) + // Ensure "instructions" was not wiped out by runaway regex + assert.ok(patched.includes('"instructions": "important instructions"')) + assert.ok(patched.includes('"compaction": { "auto": true }')) + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"')) +}) From 5ea7aba0f5484d931afb5487fe672f4844b5c1ca Mon Sep 17 00:00:00 2001 From: William Zujkowski Date: Thu, 15 Oct 2026 10:39:52 -0400 Subject: [PATCH 2/2] fix(usai-catalog): resolve default roles structurally and completely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses three review findings on the catalog default-models pipeline. 1. The existing-defaults fallback was unreachable. `resolveDefaultRole` accepts an `existingValue` so a feed that drops every priority candidate does not break regeneration, but both build paths called `resolveCatalogDefaults(models)` with no second argument. The documented stability guarantee was dead code: if the feed still contained the configured model and none of the priority candidates, regeneration threw instead of preserving a valid current default. `buildCatalog` now loads the committed `defaults` block from the output path and passes it on both the live-feed and bootstrap paths. A missing, unreadable, or malformed catalog.json is NOT an error — a first-ever generation has no prior defaults to honour, so it falls through to the priority list rather than failing. 2. agent.compaction.model was located with a regex that depended on sibling order. /("agent"\s*:\s*\{[^{}]*?"compaction"\s*:\s*\{[^{}]*?"model"\s*:\s*")/ `[^{}]*?` forbids any brace between `agent` and `compaction`, so the pattern only matches while `compaction` is the FIRST nested object under `agent`. Reproduced on the current head: with `summary` declared first, `compaction.model` silently kept its stale value in a file the pipeline reports as regenerated. The top-level patterns had the same class of bug in a different form: `/^[ \t]*"model"[ \t]*:/m` matches the first line-anchored `"model"` anywhere in the document, which is a nested `agent.*.model` whenever the `agent` block precedes the top-level keys. Measured: top-level `model` was left unpatched entirely. Replaced with a single-pass JSONC scanner that tracks object/array nesting and resolves an exact key path, skipping comments and string contents. Patching stays textual so comments and formatting survive byte-for-byte; the scanner only reports the span of the value to replace. A role that is configured in the template but cannot be patched now throws naming the path, rather than being silently skipped. 3. agent.summary and agent.title were outside the split. The shipped provider config pins both, so leaving them unmodelled kept two of five role decisions as hand-maintained config inside a region the pipeline claims to own — incomplete in a way the byte-exact round-trip test cannot detect, because the shipped file already agrees with itself. Both are now schema properties, have priority lists, are resolved by `resolveCatalogDefaults`, and are emitted by `patchOpenCodeConfig`. The emitter validates them exactly as strictly as the required pair, and reports an absent optional role as explicit null. `reasoning` remains declared-but-unresolved, and a test asserts it is the ONLY such role so the omission stays deliberate. The validator now derives its allowed-role set from the schema instead of restating it, so adding a role cannot leave validation and schema disagreeing. Verification. The new tests fail against the previous implementation — emitters 7 failed / 13 passed, including both order-dependence cases — and pass with this change. Full providers/usai suite 76 passed across 7 files (emitters 12 -> 20, catalog-validation 11 -> 16), `build-catalog.mjs --check` reports catalog.json up to date (20 models), and `scripts/validate_repo.py` passes. catalog.json gains only the two new default roles, matching what the shipped opencode.jsonc already pinned, so no committed value changes meaning. Co-authored-by: OpenCode Agent --- integrations/providers/usai/catalog.json | 4 +- .../providers/usai/catalog.schema.json | 14 + .../providers/usai/emitters/opencode.mjs | 239 +++++++++++++++--- .../providers/usai/scripts/build-catalog.mjs | 62 ++++- .../usai/tests/catalog-validation.test.mjs | 71 +++++- .../providers/usai/tests/emitters.test.mjs | 150 +++++++++++ 6 files changed, 501 insertions(+), 39 deletions(-) diff --git a/integrations/providers/usai/catalog.json b/integrations/providers/usai/catalog.json index 2e8eb700..2bca6fee 100644 --- a/integrations/providers/usai/catalog.json +++ b/integrations/providers/usai/catalog.json @@ -39,7 +39,9 @@ "defaults": { "primary": "claude-sonnet-5", "small": "claude_4_5_haiku", - "compaction": "gpt-5.6-terra" + "compaction": "gpt-5.6-terra", + "summary": "gpt_5_2_default_v2", + "title": "gpt_5_2_default_v2" }, "models": [ { diff --git a/integrations/providers/usai/catalog.schema.json b/integrations/providers/usai/catalog.schema.json index 18d9c21b..84b57962 100644 --- a/integrations/providers/usai/catalog.schema.json +++ b/integrations/providers/usai/catalog.schema.json @@ -77,6 +77,20 @@ "pattern": "^[A-Za-z0-9_.:-]+$", "description": "Cost-effective model for context compaction and conversation summarization." }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Model that writes a short recap of a finished exchange." + }, + "title": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Model that names a session in one line." + }, "reasoning": { "type": "string", "minLength": 1, diff --git a/integrations/providers/usai/emitters/opencode.mjs b/integrations/providers/usai/emitters/opencode.mjs index 3a68604a..fba57618 100644 --- a/integrations/providers/usai/emitters/opencode.mjs +++ b/integrations/providers/usai/emitters/opencode.mjs @@ -226,7 +226,8 @@ export function emitOpenCodeBlockFromCatalog(catalog) { * Values are prefixed with "usai/" per OpenCode provider model namespace convention. * * @param {object} catalog parsed catalog.json - * @returns {object} OpenCode formatted defaults { model, small_model, compaction } + * @returns {object} OpenCode formatted defaults + * { model, small_model, compaction, summary, title } */ export function emitOpenCodeDefaults(catalog) { if (!catalog || typeof catalog !== "object") { @@ -235,16 +236,13 @@ export function emitOpenCodeDefaults(catalog) { if (!catalog.defaults || typeof catalog.defaults !== "object" || Array.isArray(catalog.defaults)) { throw new TypeError("emitOpenCodeDefaults: catalog.defaults must be an object") } - const { primary, small, compaction } = catalog.defaults + const { primary, small } = catalog.defaults if (typeof primary !== "string" || primary.length === 0) { throw new TypeError("emitOpenCodeDefaults: defaults.primary must be a non-empty string") } if (typeof small !== "string" || small.length === 0) { throw new TypeError("emitOpenCodeDefaults: defaults.small must be a non-empty string") } - if (compaction !== undefined && (typeof compaction !== "string" || compaction.length === 0)) { - throw new TypeError("emitOpenCodeDefaults: defaults.compaction must be a non-empty string when present") - } // Ensure chosen default IDs do not contain characters that could break out of JSON strings // or trigger unexpected template expansions @@ -255,15 +253,205 @@ export function emitOpenCodeDefaults(catalog) { if (!SAFE_ID_RE.test(small)) { throw new TypeError(`emitOpenCodeDefaults: defaults.small contains invalid characters: "${small}"`) } - if (compaction && !SAFE_ID_RE.test(compaction)) { - throw new TypeError(`emitOpenCodeDefaults: defaults.compaction contains invalid characters: "${compaction}"`) + + // Optional `agent..model` roles. Each is validated the same way as the + // required pair, so an invalid id cannot reach the config by being optional. + const agentRoles = {} + for (const role of ["compaction", "summary", "title"]) { + const value = catalog.defaults[role] + if (value === undefined) { + agentRoles[role] = null + continue + } + if (typeof value !== "string" || value.length === 0) { + throw new TypeError(`emitOpenCodeDefaults: defaults.${role} must be a non-empty string when present`) + } + if (!SAFE_ID_RE.test(value)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.${role} contains invalid characters: "${value}"`) + } + agentRoles[role] = `usai/${value}` } return { model: `usai/${primary}`, small_model: `usai/${small}`, - compaction: compaction ? `usai/${compaction}` : null, + ...agentRoles, + } +} + +// ----------------------------------------------------------------------------- +// JSONC-aware structural scanner. +// +// WHY THIS EXISTS INSTEAD OF A REGEX. The default-model keys this emitter has to +// patch live at known OBJECT PATHS (`model`, `small_model`, +// `agent.compaction.model`), and a regex cannot express "the value at this +// path". The previous implementation approximated `agent.compaction.model` with +// +// /("agent"\s*:\s*\{[^{}]*?"compaction"\s*:\s*\{[^{}]*?"model"\s*:\s*")/ +// +// where `[^{}]*?` forbids any brace between `agent` and `compaction`. That only +// holds when `compaction` is the FIRST nested object under `agent`. The shipped +// config happens to satisfy it, but `agent` also carries `summary` and `title` +// siblings — reorder them and the pattern silently matches nothing, leaving a +// stale compaction model in a file the pipeline reports as regenerated. +// +// The top-level patterns had a related weakness: `/^[ \t]*"model"[ \t]*:/m` +// matches the FIRST line-anchored `"model"` anywhere in the file, which is a +// nested `agent.*.model` whenever the `agent` block precedes the top-level keys. +// +// This scanner walks the text once, tracking object/array nesting and the +// current key path, and skipping comments and string contents. It reports the +// span of the string value at an exact path, so sibling order, nesting depth, +// and commented-out or in-array decoys cannot affect the result. It does not +// build a JSON value — patching is textual, to preserve comments and formatting +// byte-for-byte. +// ----------------------------------------------------------------------------- + +// Marker for an array element frame: array indices are not part of any path we +// patch, so elements are tracked only to keep nesting depth correct. +const ARRAY_ELEMENT = Symbol("arrayElement") + +function skipLineComment(text, i) { + while (i < text.length && text[i] !== "\n") i++ + return i +} + +function skipBlockComment(text, i) { + i += 2 + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++ + return i + 2 +} + +// Index of the closing quote of the string whose contents start at `start`. +// Returns -1 on an unterminated string. +function stringEnd(text, start) { + let j = start + while (j < text.length) { + if (text[j] === "\\") { + j += 2 + continue + } + if (text[j] === '"') return j + j++ } + return -1 +} + +/** + * Locate a value at an exact object path in JSONC text. + * + * @param {string} text JSONC document + * @param {string[]} path object keys, outermost first + * @param {"value"|"container"} mode `value` returns the {start,end} span of the + * string value at `path`; `container` reports whether an OBJECT exists at + * `path` (used to tell "the template has no such section" apart from "the + * section is there but we failed to patch it") + * @returns {{start: number, end: number}|{found: true}|null} + */ +function scanJsonc(text, path, mode) { + let i = 0 + const here = [] + // Whether each open brace/bracket pushed a path segment, so `}`/`]` pops + // symmetrically. A brace that is not a keyed value (the document root, or an + // object inside an array) contributes no key. + const pushedSegment = [] + const frame = [] + let pendingKey = null + let expectValue = false + + const atPathForKey = (key) => { + if (here.length + 1 !== path.length) return false + for (let x = 0; x < here.length; x++) if (here[x] !== path[x]) return false + return key === path[path.length - 1] + } + const atPath = () => { + if (here.length !== path.length) return false + for (let x = 0; x < here.length; x++) if (here[x] !== path[x]) return false + return true + } + + while (i < text.length) { + const c = text[i] + if (c === "/" && text[i + 1] === "/") { + i = skipLineComment(text, i) + continue + } + if (c === "/" && text[i + 1] === "*") { + i = skipBlockComment(text, i) + continue + } + if (c === '"') { + const start = i + 1 + const end = stringEnd(text, start) + if (end === -1) return null + if (expectValue) { + if (mode === "value" && frame[frame.length - 1] === "object" && pendingKey !== null && atPathForKey(pendingKey)) { + return { start, end } + } + expectValue = false + pendingKey = null + } else if (frame[frame.length - 1] === "object") { + pendingKey = text.slice(start, end) + } + i = end + 1 + continue + } + if (c === ":") { + expectValue = true + i++ + continue + } + if (c === "{" || c === "[") { + if (expectValue && pendingKey !== null) { + here.push(pendingKey) + pushedSegment.push(true) + } else if (frame[frame.length - 1] === "array") { + here.push(ARRAY_ELEMENT) + pushedSegment.push(true) + } else { + pushedSegment.push(false) + } + frame.push(c === "{" ? "object" : "array") + expectValue = false + pendingKey = null + if (mode === "container" && frame[frame.length - 1] === "object" && atPath()) return { found: true } + i++ + continue + } + if (c === "}" || c === "]") { + frame.pop() + if (pushedSegment.pop()) here.pop() + expectValue = false + pendingKey = null + i++ + continue + } + if (c === ",") { + expectValue = false + pendingKey = null + i++ + continue + } + i++ + } + return null +} + +// Replace the string value at `path`. Throws when the path is absent, because a +// requested role that cannot be patched must fail loudly rather than leave a +// stale model behind in a file reported as regenerated. +function setJsoncStringAtPath(text, path, value, label) { + const span = scanJsonc(text, path, "value") + if (!span) { + throw new Error( + `patchOpenCodeConfig: could not locate a string value at ${label} ("${path.join(".")}") in template`, + ) + } + return text.slice(0, span.start) + value + text.slice(span.end) +} + +function hasJsoncObjectAtPath(text, path) { + return Boolean(scanJsonc(text, path, "container")) } /** @@ -293,31 +481,24 @@ export function patchOpenCodeConfig(templateText, catalog) { let updated = `${beforeMarker}${modelsBlock.trim()}${afterMarker}` - // 2. Replace top-level model and small_model - // Anchor to line start to avoid matching comments, and use function replacements to avoid `$` issues - const modelPattern = /^([ \t]*"model"[ \t]*:[ \t]*")([^"]*)(")/m - if (!modelPattern.test(updated)) { - throw new Error("patchOpenCodeConfig: Could not find top-level 'model' in template") - } - updated = updated.replace(modelPattern, (_m, p1, _p2, p3) => `${p1}${defaults.model}${p3}`) + // 2. Top-level roles. Resolved by path, so a nested `agent.*.model` earlier in + // the file cannot be mistaken for the top-level key. + updated = setJsoncStringAtPath(updated, ["model"], defaults.model, "top-level 'model'") + updated = setJsoncStringAtPath(updated, ["small_model"], defaults.small_model, "top-level 'small_model'") - const smallModelPattern = /^([ \t]*"small_model"[ \t]*:[ \t]*")([^"]*)(")/m - if (!smallModelPattern.test(updated)) { - throw new Error("patchOpenCodeConfig: Could not find top-level 'small_model' in template") - } - updated = updated.replace(smallModelPattern, (_m, p1, _p2, p3) => `${p1}${defaults.small_model}${p3}`) - - // 3. Replace agent.compaction.model if present in defaults and template - // Scoped strictly within "agent": { ... "compaction": { ... "model": "..." } } - // to avoid matching across closing braces - if (defaults.compaction) { - const compactionPattern = /("agent"\s*:\s*\{[^{}]*?"compaction"\s*:\s*\{[^{}]*?"model"\s*:\s*")([^"]*)(")/ - if (compactionPattern.test(updated)) { - updated = updated.replace(compactionPattern, (_m, p1, _p2, p3) => `${p1}${defaults.compaction}${p3}`) - } + // 3. agent..model for each modelled agent role. A template that does + // not configure a role at all is legitimate and left alone; a template that + // DOES configure it must be patched, so a present-but-unpatchable section is + // an error rather than a silent skip. + for (const role of ["compaction", "summary", "title"]) { + const value = defaults[role] + if (!value) continue + if (!hasJsoncObjectAtPath(updated, ["agent", role])) continue + updated = setJsoncStringAtPath(updated, ["agent", role, "model"], value, `'agent.${role}.model'`) } return updated } + export default emitOpenCodeBlockFromCatalog diff --git a/integrations/providers/usai/scripts/build-catalog.mjs b/integrations/providers/usai/scripts/build-catalog.mjs index c96d3751..fa9fe883 100644 --- a/integrations/providers/usai/scripts/build-catalog.mjs +++ b/integrations/providers/usai/scripts/build-catalog.mjs @@ -758,7 +758,11 @@ function validateCatalog(catalog, schema) { // defaults assert(catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults), "defaults must be an object", errors) if (catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults)) { - const defaultsAllowed = new Set(["primary", "small", "compaction", "reasoning"]) + // Derive the allowed role set from the SCHEMA rather than restating it, so + // adding a role cannot leave the validator rejecting what the schema + // permits (or accepting what it does not). + const defaultsAllowed = new Set(Object.keys(schema.properties.defaults.properties)) + for (const k of Object.keys(catalog.defaults)) { assert(defaultsAllowed.has(k), `defaults: disallowed key "${k}"`, errors) } @@ -815,11 +819,21 @@ function validateCatalog(catalog, schema) { // primary: flagship instruction-following and multi-step reasoning (OpenCode `model`) // small: fast, cost-effective lightweight completions (OpenCode `small_model`) // compaction: context compaction and conversation summarization (OpenCode `agent.compaction.model`) +// summary: short recap of a finished exchange (OpenCode `agent.summary.model`) +// title: one-line session naming (OpenCode `agent.title.model`) +// +// summary and title are modelled because the shipped provider config already +// pins both, outside the generated block. Leaving them out of the catalog would +// have kept two of the five role decisions as hand-maintained config that the +// pipeline reports as regenerated — the split would be incomplete in a way the +// round-trip test could not see. // ----------------------------------------------------------------------------- const DEFAULT_ROLE_PRIORITY = { primary: ["claude-sonnet-5", "claude_4_5_sonnet", "claude-opus-5", "claude_4_5_opus"], small: ["claude_4_5_haiku", "claude-3-5-haiku"], compaction: ["gpt-5.6-terra", "gpt_5_5_default_v2", "gpt-5.4-latest-guardrails-defaultv2"], + summary: ["gpt_5_2_default_v2", "gpt_5_4_default_v2", "gpt-5.6-terra"], + title: ["gpt_5_2_default_v2", "gpt_5_4_default_v2", "gpt-5.6-terra"], } function resolveDefaultRole(models, priorityList, existingValue, roleName) { @@ -835,13 +849,14 @@ function resolveDefaultRole(models, priorityList, existingValue, roleName) { } function resolveCatalogDefaults(models, existingDefaults = {}) { - return { - primary: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.primary, existingDefaults.primary, "primary"), - small: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.small, existingDefaults.small, "small"), - compaction: resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY.compaction, existingDefaults.compaction, "compaction"), + const resolved = {} + for (const role of Object.keys(DEFAULT_ROLE_PRIORITY)) { + resolved[role] = resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY[role], existingDefaults[role], role) } + return resolved } + // ----------------------------------------------------------------------------- // Deterministic serialization: keys in a fixed order, 2-space indent, trailing // newline. Order is stable so the committed catalog.json is reproducible. @@ -850,7 +865,7 @@ const KEY_ORDER = { root: ["schemaVersion", "generatedBy", "sources", "gateway", "vendors", "defaults", "models"], gateway: ["host", "baseUrl", "api", "apiKeyEnv"], vendor: ["key", "label", "order", "usaiBackend"], - defaults: ["primary", "small", "compaction", "reasoning"], + defaults: ["primary", "small", "compaction", "summary", "title", "reasoning"], model: ["id", "vendor", "name", "contextWindow", "maxOutputTokens", "cost", "costAbove200kContext", "reasoning", "inputModalities"], cost: ["input", "output", "cacheRead", "cacheWrite"], } @@ -916,9 +931,39 @@ function parseArgs(argv) { return args } +// Read the `defaults` block already committed in catalog.json, so regeneration +// can fall back to a currently-valid choice when none of the priority +// candidates survive in the new feed. +// +// WHY THIS IS LOADED RATHER THAN PASSED EMPTY. `resolveDefaultRole` accepts an +// `existingValue` fallback precisely so a feed that drops every priority +// candidate does not break regeneration — but the fallback is unreachable if +// the caller never supplies it. Both build paths previously called +// `resolveCatalogDefaults(models)` with no second argument, so the stability +// guarantee the function documents could not fire: the run threw instead of +// preserving a default that is still present in the catalog. +// +// Absence is NOT an error. A first-ever generation, or a deliberately removed +// catalog.json, has no prior defaults to honour; that must fall through to the +// priority list rather than fail. An unreadable or malformed file is likewise +// treated as "no prior defaults" — the subsequent resolve either finds a +// priority candidate or throws with its own, more specific message. +async function loadExistingDefaults(outPath) { + try { + const parsed = JSON.parse(await readFile(outPath, "utf8")) + const defaults = parsed?.defaults + if (defaults && typeof defaults === "object" && !Array.isArray(defaults)) return defaults + } catch { + // No catalog.json yet, unreadable, or not valid JSON. + } + return {} +} + async function buildCatalog(args) { const schemaPath = args.opts["--schema"] || DEFAULT_SCHEMA const schema = JSON.parse(await readFile(schemaPath, "utf8")) + const outPath = args.opts["--out"] || DEFAULT_OUT + const existingDefaults = await loadExistingDefaults(outPath) const modelsUrl = args.opts["--models-url"] const noEnrichment = args.flags.has("--no-enrichment") @@ -937,7 +982,7 @@ async function buildCatalog(args) { else throw new Error("models.dev source has unexpected shape (expected provider-keyed object)") } const { models, vendorOrder } = shapeFromFeeds(usaiList, modelsDevCatalog) - const defaults = resolveCatalogDefaults(models) + const defaults = resolveCatalogDefaults(models, existingDefaults) catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, @@ -960,7 +1005,8 @@ async function buildCatalog(args) { // Preserve vendor first-appearance order from the block. const vendorOrder = [] for (const m of models) if (!vendorOrder.includes(m.vendor)) vendorOrder.push(m.vendor) - const defaults = resolveCatalogDefaults(models) + const defaults = resolveCatalogDefaults(models, existingDefaults) + catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, diff --git a/integrations/providers/usai/tests/catalog-validation.test.mjs b/integrations/providers/usai/tests/catalog-validation.test.mjs index f9d91825..aaaefb17 100644 --- a/integrations/providers/usai/tests/catalog-validation.test.mjs +++ b/integrations/providers/usai/tests/catalog-validation.test.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" import { test } from "node:test" -import { validateCatalog } from "../scripts/build-catalog.mjs" +import { DEFAULT_ROLE_PRIORITY, resolveCatalogDefaults, validateCatalog } from "../scripts/build-catalog.mjs" const catalog = JSON.parse(readFileSync(new URL("../catalog.json", import.meta.url), "utf8")) const schema = JSON.parse(readFileSync(new URL("../catalog.schema.json", import.meta.url), "utf8")) @@ -108,3 +108,72 @@ test("catalog validation enforces defaults structure and referential integrity", /references unknown model "nonexistent-model"/, ) }) + +// ----------------------------------------------------------------------------- +// Default-role RESOLUTION, as distinct from validation of an already-built +// block. `resolveDefaultRole` accepts an `existingValue` fallback so a feed that +// drops every priority candidate does not break regeneration — but that +// fallback is only reachable if the caller supplies it. Both build paths +// previously called `resolveCatalogDefaults(models)` with no second argument, +// making the documented stability guarantee dead code: the run threw instead of +// preserving a default that was still valid. +// ----------------------------------------------------------------------------- + +const modelList = (...ids) => ids.map((id) => ({ id, vendor: "openai", name: id })) + +test("resolveCatalogDefaults prefers a priority candidate over the existing value", () => { + // Every role needs something resolvable, so seed each one's top candidate and + // additionally offer a stale existing value for primary. + const models = modelList( + ...Object.values(DEFAULT_ROLE_PRIORITY).map((list) => list[0]), + "something-else", + ) + const resolved = resolveCatalogDefaults(models, { primary: "something-else" }) + assert.equal(resolved.primary, DEFAULT_ROLE_PRIORITY.primary[0]) +}) + +test("resolveCatalogDefaults falls back to a still-valid existing default", () => { + // No priority candidate for any role is present, but the previously committed + // choices all still exist in the feed, so regeneration must preserve them + // rather than throw. + const existing = { primary: "keep-primary", small: "keep-small", compaction: "keep-compaction", summary: "keep-summary", title: "keep-title" } + const models = modelList(...Object.values(existing)) + assert.deepEqual(resolveCatalogDefaults(models, existing), existing) +}) + +test("resolveCatalogDefaults ignores an existing default that no longer exists", () => { + // A stale fallback must not be carried forward — it would emit a default + // referencing a model that is not in models[]. + const models = modelList("unrelated-model") + assert.throws( + () => resolveCatalogDefaults(models, { primary: "retired-model" }), + /Cannot resolve default for role "primary"/, + ) +}) + +test("resolveCatalogDefaults with no prior defaults still resolves from priority", () => { + // First-ever generation: absence of a committed catalog.json is not an error. + const models = modelList( + DEFAULT_ROLE_PRIORITY.primary[0], + DEFAULT_ROLE_PRIORITY.small[0], + DEFAULT_ROLE_PRIORITY.compaction[0], + DEFAULT_ROLE_PRIORITY.summary[0], + DEFAULT_ROLE_PRIORITY.title[0], + ) + const resolved = resolveCatalogDefaults(models) + assert.equal(resolved.primary, DEFAULT_ROLE_PRIORITY.primary[0]) + assert.equal(resolved.summary, DEFAULT_ROLE_PRIORITY.summary[0]) +}) + +test("every role the schema allows is either resolved or explicitly optional", () => { + // Guards against a role being added to the schema but never resolved, which + // would leave it as hand-maintained config inside a generated block. + const schemaRoles = Object.keys(schema.properties.defaults.properties) + const resolvedRoles = Object.keys(DEFAULT_ROLE_PRIORITY) + const unresolved = schemaRoles.filter((r) => !resolvedRoles.includes(r)) + assert.deepEqual( + unresolved, + ["reasoning"], + "a schema role with no priority list must be a deliberate, documented omission", + ) +}) diff --git a/integrations/providers/usai/tests/emitters.test.mjs b/integrations/providers/usai/tests/emitters.test.mjs index 6b06ac73..37524a4c 100644 --- a/integrations/providers/usai/tests/emitters.test.mjs +++ b/integrations/providers/usai/tests/emitters.test.mjs @@ -223,9 +223,39 @@ test("emitOpenCodeDefaults maps neutral catalog defaults to usai/ prefixed roles model: "usai/claude-sonnet-5", small_model: "usai/claude_4_5_haiku", compaction: "usai/gpt-5.6-terra", + summary: "usai/gpt_5_2_default_v2", + title: "usai/gpt_5_2_default_v2", }) }) +test("emitOpenCodeDefaults reports an absent optional agent role as null, not as missing", () => { + // A role the catalog does not define must be an explicit null, so a caller + // iterating the roles cannot mistake "not configured" for "forgot to read it". + const defaults = emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku" } }) + assert.deepEqual(defaults, { + model: "usai/sonnet", + small_model: "usai/haiku", + compaction: null, + summary: null, + title: null, + }) +}) + +test("emitOpenCodeDefaults validates optional agent roles as strictly as the required pair", () => { + for (const role of ["compaction", "summary", "title"]) { + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku", [role]: "" } }), + new RegExp(`defaults\\.${role} must be a non-empty string`), + `${role}: empty string must be rejected`, + ) + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku", [role]: 'x", "permission": {}' } }), + new RegExp(`defaults\\.${role} contains invalid characters`), + `${role}: id that could break out of the JSON string must be rejected`, + ) + } +}) + test("emitOpenCodeDefaults rejects malformed defaults or invalid characters", () => { assert.throws(() => emitOpenCodeDefaults(null), /must be an object/) assert.throws(() => emitOpenCodeDefaults({ defaults: null }), /must be an object/) @@ -285,3 +315,123 @@ test("patchOpenCodeConfig does not cross object braces if agent.compaction is mi assert.ok(patched.includes('"compaction": { "auto": true }')) assert.ok(patched.includes('"model": "usai/claude-sonnet-5"')) }) + +// ----------------------------------------------------------------------------- +// Structural resolution of the patched paths. +// +// These exist because the previous implementation located `agent.compaction.model` +// with a regex whose `[^{}]*?` forbade any brace between `agent` and +// `compaction`, which only holds while `compaction` is the FIRST nested object +// under `agent`. The shipped config satisfies that today, so the byte-exact +// round-trip test above passes either way — it cannot see this class of defect. +// Each case below is a template shape that is valid JSONC and that the regex +// approach got wrong. +// ----------------------------------------------------------------------------- + +const MARKERS = "// BEGIN GENERATED USAI MODELS\n// END GENERATED USAI MODELS" + +function templateWithAgent(agentBody, { agentFirst = false } = {}) { + const topLevel = ` "model": "usai/STALE-TOP", + "small_model": "usai/STALE-SMALL",` + const agent = ` "agent": { +${agentBody} + },` + return `{ +${agentFirst ? `${agent}\n${topLevel}` : `${topLevel}\n${agent}`} +${MARKERS} +} +` +} + +const agentRoleBlock = (role, value) => ` "${role}": { + "model": "usai/${value}", + "temperature": 0 + }` + +test("agent.compaction.model is patched regardless of sibling order", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const compactionFirst = [agentRoleBlock("compaction", "STALE"), agentRoleBlock("summary", "STALE")].join(",\n") + const summaryFirst = [agentRoleBlock("summary", "STALE"), agentRoleBlock("compaction", "STALE")].join(",\n") + + for (const [label, body] of [["compaction first", compactionFirst], ["summary first", summaryFirst]]) { + const patched = patchOpenCodeConfig(templateWithAgent(body), catalog) + const found = /"compaction"\s*:\s*\{\s*"model"\s*:\s*"([^"]*)"/.exec(patched) + assert.equal(found?.[1], "usai/gpt-5.6-terra", `${label}: compaction.model must be patched`) + } +}) + +test("top-level model is patched even when the agent block precedes it", () => { + // The top-level patterns matched the first line-anchored `"model"` in the + // file, which is a nested agent role whenever `agent` comes first. + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig( + templateWithAgent(agentRoleBlock("compaction", "STALE"), { agentFirst: true }), + catalog, + ) + assert.ok(!patched.includes("usai/STALE-TOP"), "top-level model must not keep its stale value") + assert.ok(!patched.includes("usai/STALE-SMALL"), "top-level small_model must not keep its stale value") + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"')) + assert.ok(patched.includes('"small_model": "usai/claude_4_5_haiku"')) +}) + +test("every modelled agent role is patched, not just compaction", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const body = ["compaction", "summary", "title"].map((r) => agentRoleBlock(r, "STALE")).join(",\n") + const patched = patchOpenCodeConfig(templateWithAgent(body), catalog) + assert.ok(!patched.includes("usai/STALE"), "no agent role may retain a stale model") + for (const [role, expected] of [ + ["compaction", "usai/gpt-5.6-terra"], + ["summary", "usai/gpt_5_2_default_v2"], + ["title", "usai/gpt_5_2_default_v2"], + ]) { + const found = new RegExp(`"${role}"\\s*:\\s*\\{\\s*"model"\\s*:\\s*"([^"]*)"`).exec(patched) + assert.equal(found?.[1], expected, `agent.${role}.model`) + } +}) + +test("commented-out and in-array decoys are not mistaken for the real key", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + // "model": "usai/COMMENTED", + /* "model": "usai/BLOCK-COMMENTED" */ + "experimental": [ { "model": "usai/IN-ARRAY" } ], + "model": "usai/STALE-TOP", + "small_model": "usai/STALE-SMALL", +${MARKERS} +} +` + const patched = patchOpenCodeConfig(template, catalog) + assert.ok(patched.includes('"model": "usai/COMMENTED"'), "a commented-out key must be left untouched") + assert.ok(patched.includes('"model": "usai/BLOCK-COMMENTED"'), "a block-commented key must be left untouched") + assert.ok(patched.includes('"model": "usai/IN-ARRAY"'), "a key inside an array must be left untouched") + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"'), "the real top-level key must be patched") +}) + +test("a configured agent role that cannot be patched is an error, not a silent skip", () => { + // `agent.compaction` exists but carries no `model` key. Leaving it alone would + // report a regenerated file while the role stayed unset, so this must throw. + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + "model": "usai/old", + "small_model": "usai/old-small", + "agent": { + "compaction": { "temperature": 0 } + }, +${MARKERS} +} +` + assert.throws( + () => patchOpenCodeConfig(template, catalog), + /could not locate a string value at 'agent\.compaction\.model'/, + ) +}) + +test("a missing top-level role is an error naming which key was not found", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + "small_model": "usai/old-small", +${MARKERS} +} +` + assert.throws(() => patchOpenCodeConfig(template, catalog), /top-level 'model'/) +})