diff --git a/integrations/providers/usai/catalog.json b/integrations/providers/usai/catalog.json index b5852f7..2bca6fe 100644 --- a/integrations/providers/usai/catalog.json +++ b/integrations/providers/usai/catalog.json @@ -36,6 +36,13 @@ "usaiBackend": "amazon-bedrock" } ], + "defaults": { + "primary": "claude-sonnet-5", + "small": "claude_4_5_haiku", + "compaction": "gpt-5.6-terra", + "summary": "gpt_5_2_default_v2", + "title": "gpt_5_2_default_v2" + }, "models": [ { "id": "claude_4_5_haiku", diff --git a/integrations/providers/usai/catalog.schema.json b/integrations/providers/usai/catalog.schema.json index ce5ef08..84b5796 100644 --- a/integrations/providers/usai/catalog.schema.json +++ b/integrations/providers/usai/catalog.schema.json @@ -5,7 +5,7 @@ "description": "Harness-neutral catalog of USAi gateway models. Generated from live feeds by build-catalog.mjs; consumed by per-harness emitters. Do not hand-edit the generated catalog.json.", "type": "object", "additionalProperties": false, - "required": ["schemaVersion", "gateway", "vendors", "models"], + "required": ["schemaVersion", "gateway", "vendors", "defaults", "models"], "properties": { "schemaVersion": { "type": "string", @@ -50,6 +50,56 @@ } } }, + "defaults": { + "type": "object", + "description": "Recommended default models for standard agent roles. Every value must reference a valid model id in models[].", + "additionalProperties": false, + "required": ["primary", "small"], + "properties": { + "primary": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Primary interactive coding and instruction-following model." + }, + "small": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Fast, lower-cost model for lightweight sub-tasks and completions." + }, + "compaction": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Cost-effective model for context compaction and conversation summarization." + }, + "summary": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Model that writes a short recap of a finished exchange." + }, + "title": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "Model that names a session in one line." + }, + "reasoning": { + "type": "string", + "minLength": 1, + "maxLength": 128, + "pattern": "^[A-Za-z0-9_.:-]+$", + "description": "High-capacity reasoning and architectural planning model." + } + } + }, "vendors": { "type": "array", "description": "Model vendors (Anthropic, OpenAI, Google, ...), for grouping and display order.", diff --git a/integrations/providers/usai/emitters/opencode.mjs b/integrations/providers/usai/emitters/opencode.mjs index 751296c..fba5761 100644 --- a/integrations/providers/usai/emitters/opencode.mjs +++ b/integrations/providers/usai/emitters/opencode.mjs @@ -221,4 +221,284 @@ export function emitOpenCodeBlockFromCatalog(catalog) { return lines.join("\n") } +/** + * Map neutral catalog defaults to OpenCode configuration values. + * Values are prefixed with "usai/" per OpenCode provider model namespace convention. + * + * @param {object} catalog parsed catalog.json + * @returns {object} OpenCode formatted defaults + * { model, small_model, compaction, summary, title } + */ +export function emitOpenCodeDefaults(catalog) { + if (!catalog || typeof catalog !== "object") { + throw new TypeError("emitOpenCodeDefaults: catalog must be an object") + } + if (!catalog.defaults || typeof catalog.defaults !== "object" || Array.isArray(catalog.defaults)) { + throw new TypeError("emitOpenCodeDefaults: catalog.defaults must be an object") + } + const { primary, small } = catalog.defaults + if (typeof primary !== "string" || primary.length === 0) { + throw new TypeError("emitOpenCodeDefaults: defaults.primary must be a non-empty string") + } + if (typeof small !== "string" || small.length === 0) { + throw new TypeError("emitOpenCodeDefaults: defaults.small must be a non-empty string") + } + + // Ensure chosen default IDs do not contain characters that could break out of JSON strings + // or trigger unexpected template expansions + const SAFE_ID_RE = /^[A-Za-z0-9_.:-]+$/ + if (!SAFE_ID_RE.test(primary)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.primary contains invalid characters: "${primary}"`) + } + if (!SAFE_ID_RE.test(small)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.small contains invalid characters: "${small}"`) + } + + // Optional `agent..model` roles. Each is validated the same way as the + // required pair, so an invalid id cannot reach the config by being optional. + const agentRoles = {} + for (const role of ["compaction", "summary", "title"]) { + const value = catalog.defaults[role] + if (value === undefined) { + agentRoles[role] = null + continue + } + if (typeof value !== "string" || value.length === 0) { + throw new TypeError(`emitOpenCodeDefaults: defaults.${role} must be a non-empty string when present`) + } + if (!SAFE_ID_RE.test(value)) { + throw new TypeError(`emitOpenCodeDefaults: defaults.${role} contains invalid characters: "${value}"`) + } + agentRoles[role] = `usai/${value}` + } + + return { + model: `usai/${primary}`, + small_model: `usai/${small}`, + ...agentRoles, + } +} + +// ----------------------------------------------------------------------------- +// JSONC-aware structural scanner. +// +// WHY THIS EXISTS INSTEAD OF A REGEX. The default-model keys this emitter has to +// patch live at known OBJECT PATHS (`model`, `small_model`, +// `agent.compaction.model`), and a regex cannot express "the value at this +// path". The previous implementation approximated `agent.compaction.model` with +// +// /("agent"\s*:\s*\{[^{}]*?"compaction"\s*:\s*\{[^{}]*?"model"\s*:\s*")/ +// +// where `[^{}]*?` forbids any brace between `agent` and `compaction`. That only +// holds when `compaction` is the FIRST nested object under `agent`. The shipped +// config happens to satisfy it, but `agent` also carries `summary` and `title` +// siblings — reorder them and the pattern silently matches nothing, leaving a +// stale compaction model in a file the pipeline reports as regenerated. +// +// The top-level patterns had a related weakness: `/^[ \t]*"model"[ \t]*:/m` +// matches the FIRST line-anchored `"model"` anywhere in the file, which is a +// nested `agent.*.model` whenever the `agent` block precedes the top-level keys. +// +// This scanner walks the text once, tracking object/array nesting and the +// current key path, and skipping comments and string contents. It reports the +// span of the string value at an exact path, so sibling order, nesting depth, +// and commented-out or in-array decoys cannot affect the result. It does not +// build a JSON value — patching is textual, to preserve comments and formatting +// byte-for-byte. +// ----------------------------------------------------------------------------- + +// Marker for an array element frame: array indices are not part of any path we +// patch, so elements are tracked only to keep nesting depth correct. +const ARRAY_ELEMENT = Symbol("arrayElement") + +function skipLineComment(text, i) { + while (i < text.length && text[i] !== "\n") i++ + return i +} + +function skipBlockComment(text, i) { + i += 2 + while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) i++ + return i + 2 +} + +// Index of the closing quote of the string whose contents start at `start`. +// Returns -1 on an unterminated string. +function stringEnd(text, start) { + let j = start + while (j < text.length) { + if (text[j] === "\\") { + j += 2 + continue + } + if (text[j] === '"') return j + j++ + } + return -1 +} + +/** + * Locate a value at an exact object path in JSONC text. + * + * @param {string} text JSONC document + * @param {string[]} path object keys, outermost first + * @param {"value"|"container"} mode `value` returns the {start,end} span of the + * string value at `path`; `container` reports whether an OBJECT exists at + * `path` (used to tell "the template has no such section" apart from "the + * section is there but we failed to patch it") + * @returns {{start: number, end: number}|{found: true}|null} + */ +function scanJsonc(text, path, mode) { + let i = 0 + const here = [] + // Whether each open brace/bracket pushed a path segment, so `}`/`]` pops + // symmetrically. A brace that is not a keyed value (the document root, or an + // object inside an array) contributes no key. + const pushedSegment = [] + const frame = [] + let pendingKey = null + let expectValue = false + + const atPathForKey = (key) => { + if (here.length + 1 !== path.length) return false + for (let x = 0; x < here.length; x++) if (here[x] !== path[x]) return false + return key === path[path.length - 1] + } + const atPath = () => { + if (here.length !== path.length) return false + for (let x = 0; x < here.length; x++) if (here[x] !== path[x]) return false + return true + } + + while (i < text.length) { + const c = text[i] + if (c === "/" && text[i + 1] === "/") { + i = skipLineComment(text, i) + continue + } + if (c === "/" && text[i + 1] === "*") { + i = skipBlockComment(text, i) + continue + } + if (c === '"') { + const start = i + 1 + const end = stringEnd(text, start) + if (end === -1) return null + if (expectValue) { + if (mode === "value" && frame[frame.length - 1] === "object" && pendingKey !== null && atPathForKey(pendingKey)) { + return { start, end } + } + expectValue = false + pendingKey = null + } else if (frame[frame.length - 1] === "object") { + pendingKey = text.slice(start, end) + } + i = end + 1 + continue + } + if (c === ":") { + expectValue = true + i++ + continue + } + if (c === "{" || c === "[") { + if (expectValue && pendingKey !== null) { + here.push(pendingKey) + pushedSegment.push(true) + } else if (frame[frame.length - 1] === "array") { + here.push(ARRAY_ELEMENT) + pushedSegment.push(true) + } else { + pushedSegment.push(false) + } + frame.push(c === "{" ? "object" : "array") + expectValue = false + pendingKey = null + if (mode === "container" && frame[frame.length - 1] === "object" && atPath()) return { found: true } + i++ + continue + } + if (c === "}" || c === "]") { + frame.pop() + if (pushedSegment.pop()) here.pop() + expectValue = false + pendingKey = null + i++ + continue + } + if (c === ",") { + expectValue = false + pendingKey = null + i++ + continue + } + i++ + } + return null +} + +// Replace the string value at `path`. Throws when the path is absent, because a +// requested role that cannot be patched must fail loudly rather than leave a +// stale model behind in a file reported as regenerated. +function setJsoncStringAtPath(text, path, value, label) { + const span = scanJsonc(text, path, "value") + if (!span) { + throw new Error( + `patchOpenCodeConfig: could not locate a string value at ${label} ("${path.join(".")}") in template`, + ) + } + return text.slice(0, span.start) + value + text.slice(span.end) +} + +function hasJsoncObjectAtPath(text, path) { + return Boolean(scanJsonc(text, path, "container")) +} + +/** + * Patch an existing opencode.jsonc template text with both the models block + * and the default model selections from the catalog. + * + * @param {string} templateText raw opencode.jsonc text + * @param {object} catalog parsed catalog.json + * @returns {string} updated opencode.jsonc text + */ +export function patchOpenCodeConfig(templateText, catalog) { + if (typeof templateText !== "string") { + throw new TypeError("patchOpenCodeConfig: templateText must be a string") + } + const defaults = emitOpenCodeDefaults(catalog) + const modelsBlock = emitOpenCodeBlockFromCatalog(catalog) + + // 1. Replace generated models block + const beginIdx = templateText.indexOf(BEGIN_MARKER) + const endIdx = templateText.indexOf(END_MARKER) + if (beginIdx === -1 || endIdx === -1 || endIdx <= beginIdx) { + throw new Error("patchOpenCodeConfig: BEGIN/END GENERATED USAI MODELS markers not found in template") + } + + const beforeMarker = templateText.slice(0, beginIdx) + const afterMarker = templateText.slice(endIdx + END_MARKER.length) + + let updated = `${beforeMarker}${modelsBlock.trim()}${afterMarker}` + + // 2. Top-level roles. Resolved by path, so a nested `agent.*.model` earlier in + // the file cannot be mistaken for the top-level key. + updated = setJsoncStringAtPath(updated, ["model"], defaults.model, "top-level 'model'") + updated = setJsoncStringAtPath(updated, ["small_model"], defaults.small_model, "top-level 'small_model'") + + // 3. agent..model for each modelled agent role. A template that does + // not configure a role at all is legitimate and left alone; a template that + // DOES configure it must be patched, so a present-but-unpatchable section is + // an error rather than a silent skip. + for (const role of ["compaction", "summary", "title"]) { + const value = defaults[role] + if (!value) continue + if (!hasJsoncObjectAtPath(updated, ["agent", role])) continue + updated = setJsoncStringAtPath(updated, ["agent", role, "model"], value, `'agent.${role}.model'`) + } + + return updated +} + + export default emitOpenCodeBlockFromCatalog diff --git a/integrations/providers/usai/scripts/build-catalog.mjs b/integrations/providers/usai/scripts/build-catalog.mjs index 5133d7f..fa9fe88 100644 --- a/integrations/providers/usai/scripts/build-catalog.mjs +++ b/integrations/providers/usai/scripts/build-catalog.mjs @@ -755,6 +755,35 @@ function validateCatalog(catalog, schema) { } } + // defaults + assert(catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults), "defaults must be an object", errors) + if (catalog.defaults && typeof catalog.defaults === "object" && !Array.isArray(catalog.defaults)) { + // Derive the allowed role set from the SCHEMA rather than restating it, so + // adding a role cannot leave the validator rejecting what the schema + // permits (or accepting what it does not). + const defaultsAllowed = new Set(Object.keys(schema.properties.defaults.properties)) + + for (const k of Object.keys(catalog.defaults)) { + assert(defaultsAllowed.has(k), `defaults: disallowed key "${k}"`, errors) + } + for (const r of ["primary", "small"]) { + assert(r in catalog.defaults, `defaults: missing required key "${r}"`, errors) + } + const modelIds = new Set((catalog.models || []).map((m) => m.id)) + const defaultsProperties = schema.properties.defaults.properties + for (const [role, id] of Object.entries(catalog.defaults)) { + if (defaultsProperties[role]) { + validateStringFromSchema(id, defaultsProperties[role], `defaults.${role}`, errors) + } else { + assert(typeof id === "string", `defaults.${role} must be string`, errors) + } + if (typeof id === "string") { + assert(!id.includes("/"), `defaults.${role} "${id}" must not include a provider prefix`, errors) + assert(modelIds.has(id), `defaults.${role} references unknown model "${id}" (not in models[])`, errors) + } + } + } + // models assert(Array.isArray(catalog.models), "models must be an array", errors) if (Array.isArray(catalog.models)) { @@ -784,14 +813,59 @@ function validateCatalog(catalog, schema) { return errors } +// ----------------------------------------------------------------------------- +// Default role candidates in priority order. +// Roles represent standard agentic execution tiers: +// primary: flagship instruction-following and multi-step reasoning (OpenCode `model`) +// small: fast, cost-effective lightweight completions (OpenCode `small_model`) +// compaction: context compaction and conversation summarization (OpenCode `agent.compaction.model`) +// summary: short recap of a finished exchange (OpenCode `agent.summary.model`) +// title: one-line session naming (OpenCode `agent.title.model`) +// +// summary and title are modelled because the shipped provider config already +// pins both, outside the generated block. Leaving them out of the catalog would +// have kept two of the five role decisions as hand-maintained config that the +// pipeline reports as regenerated — the split would be incomplete in a way the +// round-trip test could not see. +// ----------------------------------------------------------------------------- +const DEFAULT_ROLE_PRIORITY = { + primary: ["claude-sonnet-5", "claude_4_5_sonnet", "claude-opus-5", "claude_4_5_opus"], + small: ["claude_4_5_haiku", "claude-3-5-haiku"], + compaction: ["gpt-5.6-terra", "gpt_5_5_default_v2", "gpt-5.4-latest-guardrails-defaultv2"], + summary: ["gpt_5_2_default_v2", "gpt_5_4_default_v2", "gpt-5.6-terra"], + title: ["gpt_5_2_default_v2", "gpt_5_4_default_v2", "gpt-5.6-terra"], +} + +function resolveDefaultRole(models, priorityList, existingValue, roleName) { + const modelIds = new Set(models.map((m) => m.id)) + for (const candidate of priorityList) { + if (modelIds.has(candidate)) return candidate + } + if (existingValue && modelIds.has(existingValue)) return existingValue + throw new Error( + `Cannot resolve default for role "${roleName}": none of the priority candidates ` + + `([${priorityList.join(", ")}]) or fallback "${existingValue}" exist in the catalog.`, + ) +} + +function resolveCatalogDefaults(models, existingDefaults = {}) { + const resolved = {} + for (const role of Object.keys(DEFAULT_ROLE_PRIORITY)) { + resolved[role] = resolveDefaultRole(models, DEFAULT_ROLE_PRIORITY[role], existingDefaults[role], role) + } + return resolved +} + + // ----------------------------------------------------------------------------- // Deterministic serialization: keys in a fixed order, 2-space indent, trailing // newline. Order is stable so the committed catalog.json is reproducible. // ----------------------------------------------------------------------------- const KEY_ORDER = { - root: ["schemaVersion", "generatedBy", "sources", "gateway", "vendors", "models"], + root: ["schemaVersion", "generatedBy", "sources", "gateway", "vendors", "defaults", "models"], gateway: ["host", "baseUrl", "api", "apiKeyEnv"], vendor: ["key", "label", "order", "usaiBackend"], + defaults: ["primary", "small", "compaction", "summary", "title", "reasoning"], model: ["id", "vendor", "name", "contextWindow", "maxOutputTokens", "cost", "costAbove200kContext", "reasoning", "inputModalities"], cost: ["input", "output", "cacheRead", "cacheWrite"], } @@ -808,6 +882,7 @@ function orderCatalog(catalog) { const out = orderObject(catalog, KEY_ORDER.root) if (out.gateway) out.gateway = orderObject(out.gateway, KEY_ORDER.gateway) if (Array.isArray(out.vendors)) out.vendors = out.vendors.map((v) => orderObject(v, KEY_ORDER.vendor)) + if (out.defaults) out.defaults = orderObject(out.defaults, KEY_ORDER.defaults) if (Array.isArray(out.models)) { out.models = out.models.map((m) => { const om = orderObject(m, KEY_ORDER.model) @@ -856,9 +931,39 @@ function parseArgs(argv) { return args } +// Read the `defaults` block already committed in catalog.json, so regeneration +// can fall back to a currently-valid choice when none of the priority +// candidates survive in the new feed. +// +// WHY THIS IS LOADED RATHER THAN PASSED EMPTY. `resolveDefaultRole` accepts an +// `existingValue` fallback precisely so a feed that drops every priority +// candidate does not break regeneration — but the fallback is unreachable if +// the caller never supplies it. Both build paths previously called +// `resolveCatalogDefaults(models)` with no second argument, so the stability +// guarantee the function documents could not fire: the run threw instead of +// preserving a default that is still present in the catalog. +// +// Absence is NOT an error. A first-ever generation, or a deliberately removed +// catalog.json, has no prior defaults to honour; that must fall through to the +// priority list rather than fail. An unreadable or malformed file is likewise +// treated as "no prior defaults" — the subsequent resolve either finds a +// priority candidate or throws with its own, more specific message. +async function loadExistingDefaults(outPath) { + try { + const parsed = JSON.parse(await readFile(outPath, "utf8")) + const defaults = parsed?.defaults + if (defaults && typeof defaults === "object" && !Array.isArray(defaults)) return defaults + } catch { + // No catalog.json yet, unreadable, or not valid JSON. + } + return {} +} + async function buildCatalog(args) { const schemaPath = args.opts["--schema"] || DEFAULT_SCHEMA const schema = JSON.parse(await readFile(schemaPath, "utf8")) + const outPath = args.opts["--out"] || DEFAULT_OUT + const existingDefaults = await loadExistingDefaults(outPath) const modelsUrl = args.opts["--models-url"] const noEnrichment = args.flags.has("--no-enrichment") @@ -877,6 +982,7 @@ async function buildCatalog(args) { else throw new Error("models.dev source has unexpected shape (expected provider-keyed object)") } const { models, vendorOrder } = shapeFromFeeds(usaiList, modelsDevCatalog) + const defaults = resolveCatalogDefaults(models, existingDefaults) catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, @@ -886,6 +992,7 @@ async function buildCatalog(args) { }, gateway: buildGateway(), vendors: buildVendors(vendorOrder), + defaults, models, } } else { @@ -898,6 +1005,8 @@ async function buildCatalog(args) { // Preserve vendor first-appearance order from the block. const vendorOrder = [] for (const m of models) if (!vendorOrder.includes(m.vendor)) vendorOrder.push(m.vendor) + const defaults = resolveCatalogDefaults(models, existingDefaults) + catalog = { schemaVersion: SCHEMA_VERSION, generatedBy: GENERATED_MARKER, @@ -906,6 +1015,7 @@ async function buildCatalog(args) { }, gateway: buildGateway(), vendors: buildVendors(vendorOrder), + defaults, models, } } @@ -945,6 +1055,9 @@ function describeDrift(existing, fresh) { if (JSON.stringify(existing?.vendors) !== JSON.stringify(fresh?.vendors)) { lines.push("vendors block changed") } + if (JSON.stringify(existing?.defaults) !== JSON.stringify(fresh?.defaults)) { + lines.push("defaults block changed") + } const byId = (list) => new Map((list || []).map((m) => [m.id, m])) const before = byId(existing?.models) @@ -1003,6 +1116,8 @@ export { serialize, substantive, describeDrift, + resolveCatalogDefaults, + DEFAULT_ROLE_PRIORITY, parseBootstrapModels, bootstrapModelsFromBlock, shapeFromFeeds, diff --git a/integrations/providers/usai/tests/catalog-check.test.mjs b/integrations/providers/usai/tests/catalog-check.test.mjs index 0c10559..6dce647 100644 --- a/integrations/providers/usai/tests/catalog-check.test.mjs +++ b/integrations/providers/usai/tests/catalog-check.test.mjs @@ -35,7 +35,7 @@ test("substantive() strips provenance and keeps rendered content", () => { const catalog = loadCatalog() const stripped = substantive(catalog) assert.equal("sources" in stripped, false, "sources must be excluded") - for (const key of ["schemaVersion", "generatedBy", "gateway", "vendors", "models"]) { + for (const key of ["schemaVersion", "generatedBy", "gateway", "vendors", "defaults", "models"]) { assert.ok(key in stripped, `${key} must be retained`) } }) @@ -108,6 +108,12 @@ test("gateway and vendor changes ARE drift", () => { assert.deepEqual(describeDrift(substantive(before), substantive(vendorsChanged)), [ "vendors block changed", ]) + + const defaultsChanged = loadCatalog() + defaultsChanged.defaults = { ...defaultsChanged.defaults, primary: "claude-opus-5" } + assert.deepEqual(describeDrift(substantive(before), substantive(defaultsChanged)), [ + "defaults block changed", + ]) }) test("schemaVersion drift is reported with both values", () => { diff --git a/integrations/providers/usai/tests/catalog-validation.test.mjs b/integrations/providers/usai/tests/catalog-validation.test.mjs index 89546ac..aaaefb1 100644 --- a/integrations/providers/usai/tests/catalog-validation.test.mjs +++ b/integrations/providers/usai/tests/catalog-validation.test.mjs @@ -2,15 +2,27 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" import { test } from "node:test" -import { validateCatalog } from "../scripts/build-catalog.mjs" +import { DEFAULT_ROLE_PRIORITY, resolveCatalogDefaults, validateCatalog } from "../scripts/build-catalog.mjs" const catalog = JSON.parse(readFileSync(new URL("../catalog.json", import.meta.url), "utf8")) const schema = JSON.parse(readFileSync(new URL("../catalog.schema.json", import.meta.url), "utf8")) function withFirstModel(overrides) { + // Use models[1] (claude_4_5_sonnet) when modifying the first model, + // or update defaults so defaults don't point to an overridden ID. + const m0 = catalog.models[0] return { ...catalog, - models: [{ ...catalog.models[0], ...overrides }, ...catalog.models.slice(1)], + models: [{ ...m0, ...overrides }, ...catalog.models.slice(1)], + } +} + +function withNonDefaultModel(overrides) { + // models[1] is claude_4_5_sonnet which is not currently defaults.primary, small, or compaction + const m1 = catalog.models[1] + return { + ...catalog, + models: [catalog.models[0], { ...m1, ...overrides }, ...catalog.models.slice(2)], } } @@ -19,7 +31,7 @@ test("the committed catalog satisfies the hand-rolled validator", () => { }) test("catalog validation rejects model ids that can escape generated config", () => { - assert.deepEqual(validateCatalog(withFirstModel({ id: 'provider/"quoted"\\model:v1' }), schema), []) + assert.deepEqual(validateCatalog(withNonDefaultModel({ id: 'provider/"quoted"\\model:v1' }), schema), []) }) test("catalog validation rejects control characters in display names", () => { @@ -28,17 +40,17 @@ test("catalog validation rejects control characters in display names", () => { test("catalog validation rejects control characters in ids", () => { for (const id of ["line\nbreak", "carriage\rreturn", "nul\u0000byte", "delete\u007fbyte"]) { - assert.match(validateCatalog(withFirstModel({ id }), schema).join("\n"), /id does not match/) + assert.match(validateCatalog(withNonDefaultModel({ id }), schema).join("\n"), /id does not match/) } }) test("catalog validation bounds feed-controlled string lengths", () => { - assert.match(validateCatalog(withFirstModel({ id: "a".repeat(129) }), schema).join("\n"), /id exceeds 128/) + assert.match(validateCatalog(withNonDefaultModel({ id: "a".repeat(129) }), schema).join("\n"), /id exceeds 128/) assert.match(validateCatalog(withFirstModel({ name: "a".repeat(257) }), schema).join("\n"), /name exceeds 256/) }) test("catalog validation allows provider-qualified ids and counts Unicode code points", () => { - assert.deepEqual(validateCatalog(withFirstModel({ id: "provider/model:v1" }), schema), []) + assert.deepEqual(validateCatalog(withNonDefaultModel({ id: "provider/model:v1" }), schema), []) assert.deepEqual(validateCatalog(withFirstModel({ name: "😀".repeat(128) }), schema), []) }) @@ -73,3 +85,95 @@ test("catalog validation rejects non-positive and fractional limits", () => { assert.match(validateCatalog(withFirstModel({ maxOutputTokens: value }), schema).join("\n"), /positive integer/) } }) + +test("catalog validation enforces defaults structure and referential integrity", () => { + assert.match( + validateCatalog({ ...catalog, defaults: null }, schema).join("\n"), + /defaults must be an object/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, extraRole: "claude-sonnet-5" } }, schema).join("\n"), + /disallowed key "extraRole"/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { small: "claude_4_5_haiku" } }, schema).join("\n"), + /missing required key "primary"/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, primary: "usai/claude-sonnet-5" } }, schema).join("\n"), + /must not include a provider prefix/, + ) + assert.match( + validateCatalog({ ...catalog, defaults: { ...catalog.defaults, primary: "nonexistent-model" } }, schema).join("\n"), + /references unknown model "nonexistent-model"/, + ) +}) + +// ----------------------------------------------------------------------------- +// Default-role RESOLUTION, as distinct from validation of an already-built +// block. `resolveDefaultRole` accepts an `existingValue` fallback so a feed that +// drops every priority candidate does not break regeneration — but that +// fallback is only reachable if the caller supplies it. Both build paths +// previously called `resolveCatalogDefaults(models)` with no second argument, +// making the documented stability guarantee dead code: the run threw instead of +// preserving a default that was still valid. +// ----------------------------------------------------------------------------- + +const modelList = (...ids) => ids.map((id) => ({ id, vendor: "openai", name: id })) + +test("resolveCatalogDefaults prefers a priority candidate over the existing value", () => { + // Every role needs something resolvable, so seed each one's top candidate and + // additionally offer a stale existing value for primary. + const models = modelList( + ...Object.values(DEFAULT_ROLE_PRIORITY).map((list) => list[0]), + "something-else", + ) + const resolved = resolveCatalogDefaults(models, { primary: "something-else" }) + assert.equal(resolved.primary, DEFAULT_ROLE_PRIORITY.primary[0]) +}) + +test("resolveCatalogDefaults falls back to a still-valid existing default", () => { + // No priority candidate for any role is present, but the previously committed + // choices all still exist in the feed, so regeneration must preserve them + // rather than throw. + const existing = { primary: "keep-primary", small: "keep-small", compaction: "keep-compaction", summary: "keep-summary", title: "keep-title" } + const models = modelList(...Object.values(existing)) + assert.deepEqual(resolveCatalogDefaults(models, existing), existing) +}) + +test("resolveCatalogDefaults ignores an existing default that no longer exists", () => { + // A stale fallback must not be carried forward — it would emit a default + // referencing a model that is not in models[]. + const models = modelList("unrelated-model") + assert.throws( + () => resolveCatalogDefaults(models, { primary: "retired-model" }), + /Cannot resolve default for role "primary"/, + ) +}) + +test("resolveCatalogDefaults with no prior defaults still resolves from priority", () => { + // First-ever generation: absence of a committed catalog.json is not an error. + const models = modelList( + DEFAULT_ROLE_PRIORITY.primary[0], + DEFAULT_ROLE_PRIORITY.small[0], + DEFAULT_ROLE_PRIORITY.compaction[0], + DEFAULT_ROLE_PRIORITY.summary[0], + DEFAULT_ROLE_PRIORITY.title[0], + ) + const resolved = resolveCatalogDefaults(models) + assert.equal(resolved.primary, DEFAULT_ROLE_PRIORITY.primary[0]) + assert.equal(resolved.summary, DEFAULT_ROLE_PRIORITY.summary[0]) +}) + +test("every role the schema allows is either resolved or explicitly optional", () => { + // Guards against a role being added to the schema but never resolved, which + // would leave it as hand-maintained config inside a generated block. + const schemaRoles = Object.keys(schema.properties.defaults.properties) + const resolvedRoles = Object.keys(DEFAULT_ROLE_PRIORITY) + const unresolved = schemaRoles.filter((r) => !resolvedRoles.includes(r)) + assert.deepEqual( + unresolved, + ["reasoning"], + "a schema role with no priority list must be a deliberate, documented omission", + ) +}) diff --git a/integrations/providers/usai/tests/emitters.test.mjs b/integrations/providers/usai/tests/emitters.test.mjs index 59d4762..37524a4 100644 --- a/integrations/providers/usai/tests/emitters.test.mjs +++ b/integrations/providers/usai/tests/emitters.test.mjs @@ -20,7 +20,11 @@ import path from "node:path" import { fileURLToPath } from "node:url" import { test } from "node:test" -import { emitOpenCodeBlockFromCatalog } from "../emitters/opencode.mjs" +import { + emitOpenCodeBlockFromCatalog, + emitOpenCodeDefaults, + patchOpenCodeConfig, +} from "../emitters/opencode.mjs" const HERE = path.dirname(fileURLToPath(import.meta.url)) const USAI_DIR = path.resolve(HERE, "..") @@ -193,3 +197,241 @@ test("duplicate ids and multiline vendor labels are refused", () => { duplicateVendor.vendors.push({ ...duplicateVendor.vendors[0] }) assert.throws(() => emitOpenCodeBlockFromCatalog(duplicateVendor), /duplicate vendor key/) }) + +for (const { label, file } of SHIPPED_CONFIGS) { + test(`patchOpenCodeConfig reproduces the ${label} shipped opencode.jsonc byte-for-byte`, () => { + const originalText = readFileSync(file, "utf8") + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig(originalText, catalog) + + if (patched !== originalText) { + // eslint-disable-next-line no-console + console.error(`[${label} patchOpenCodeConfig mismatch]\n${firstMismatchReport(originalText, patched)}`) + } + assert.strictEqual( + patched, + originalText, + `patchOpenCodeConfig does not reproduce ${label} opencode.jsonc byte-for-byte`, + ) + }) +} + +test("emitOpenCodeDefaults maps neutral catalog defaults to usai/ prefixed roles", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const defaults = emitOpenCodeDefaults(catalog) + assert.deepEqual(defaults, { + model: "usai/claude-sonnet-5", + small_model: "usai/claude_4_5_haiku", + compaction: "usai/gpt-5.6-terra", + summary: "usai/gpt_5_2_default_v2", + title: "usai/gpt_5_2_default_v2", + }) +}) + +test("emitOpenCodeDefaults reports an absent optional agent role as null, not as missing", () => { + // A role the catalog does not define must be an explicit null, so a caller + // iterating the roles cannot mistake "not configured" for "forgot to read it". + const defaults = emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku" } }) + assert.deepEqual(defaults, { + model: "usai/sonnet", + small_model: "usai/haiku", + compaction: null, + summary: null, + title: null, + }) +}) + +test("emitOpenCodeDefaults validates optional agent roles as strictly as the required pair", () => { + for (const role of ["compaction", "summary", "title"]) { + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku", [role]: "" } }), + new RegExp(`defaults\\.${role} must be a non-empty string`), + `${role}: empty string must be rejected`, + ) + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "haiku", [role]: 'x", "permission": {}' } }), + new RegExp(`defaults\\.${role} contains invalid characters`), + `${role}: id that could break out of the JSON string must be rejected`, + ) + } +}) + +test("emitOpenCodeDefaults rejects malformed defaults or invalid characters", () => { + assert.throws(() => emitOpenCodeDefaults(null), /must be an object/) + assert.throws(() => emitOpenCodeDefaults({ defaults: null }), /must be an object/) + assert.throws(() => emitOpenCodeDefaults({ defaults: { small: "haiku" } }), /defaults.primary must be/) + assert.throws(() => emitOpenCodeDefaults({ defaults: { primary: "sonnet", small: "" } }), /defaults.small must be/) + for (const bad of ['sonnet", "permission": {}', "model\\escape", "model{env:VAR}", "model}"]) { + assert.throws( + () => emitOpenCodeDefaults({ defaults: { primary: bad, small: "haiku" } }), + /invalid characters/, + ) + } +}) + +test("patchOpenCodeConfig safely handles $ in model ids without string replacement expansion", () => { + const originalText = readFileSync(SHIPPED_CONFIGS[0].file, "utf8") + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + // Add a test model and set as default to check replacement function + const testCat = { + ...catalog, + models: [ + { + id: "model_with_dollar", + vendor: "anthropic", + name: "Dollar Model", + contextWindow: 1000, + maxOutputTokens: 100, + }, + ...catalog.models, + ], + defaults: { + primary: "model_with_dollar", + small: "claude_4_5_haiku", + }, + } + const patched = patchOpenCodeConfig(originalText, testCat) + assert.ok(patched.includes('"model": "usai/model_with_dollar"')) +}) + +test("patchOpenCodeConfig does not cross object braces if agent.compaction is missing", () => { + // A template without agent.compaction.model + const template = ` +{ + "provider": { "usai": { "models": { +// BEGIN GENERATED USAI MODELS +// END GENERATED USAI MODELS + } } }, + "model": "usai/old-model", + "small_model": "usai/old-small", + "compaction": { "auto": true }, + "instructions": "important instructions" +} +` + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig(template, catalog) + // Ensure "instructions" was not wiped out by runaway regex + assert.ok(patched.includes('"instructions": "important instructions"')) + assert.ok(patched.includes('"compaction": { "auto": true }')) + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"')) +}) + +// ----------------------------------------------------------------------------- +// Structural resolution of the patched paths. +// +// These exist because the previous implementation located `agent.compaction.model` +// with a regex whose `[^{}]*?` forbade any brace between `agent` and +// `compaction`, which only holds while `compaction` is the FIRST nested object +// under `agent`. The shipped config satisfies that today, so the byte-exact +// round-trip test above passes either way — it cannot see this class of defect. +// Each case below is a template shape that is valid JSONC and that the regex +// approach got wrong. +// ----------------------------------------------------------------------------- + +const MARKERS = "// BEGIN GENERATED USAI MODELS\n// END GENERATED USAI MODELS" + +function templateWithAgent(agentBody, { agentFirst = false } = {}) { + const topLevel = ` "model": "usai/STALE-TOP", + "small_model": "usai/STALE-SMALL",` + const agent = ` "agent": { +${agentBody} + },` + return `{ +${agentFirst ? `${agent}\n${topLevel}` : `${topLevel}\n${agent}`} +${MARKERS} +} +` +} + +const agentRoleBlock = (role, value) => ` "${role}": { + "model": "usai/${value}", + "temperature": 0 + }` + +test("agent.compaction.model is patched regardless of sibling order", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const compactionFirst = [agentRoleBlock("compaction", "STALE"), agentRoleBlock("summary", "STALE")].join(",\n") + const summaryFirst = [agentRoleBlock("summary", "STALE"), agentRoleBlock("compaction", "STALE")].join(",\n") + + for (const [label, body] of [["compaction first", compactionFirst], ["summary first", summaryFirst]]) { + const patched = patchOpenCodeConfig(templateWithAgent(body), catalog) + const found = /"compaction"\s*:\s*\{\s*"model"\s*:\s*"([^"]*)"/.exec(patched) + assert.equal(found?.[1], "usai/gpt-5.6-terra", `${label}: compaction.model must be patched`) + } +}) + +test("top-level model is patched even when the agent block precedes it", () => { + // The top-level patterns matched the first line-anchored `"model"` in the + // file, which is a nested agent role whenever `agent` comes first. + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const patched = patchOpenCodeConfig( + templateWithAgent(agentRoleBlock("compaction", "STALE"), { agentFirst: true }), + catalog, + ) + assert.ok(!patched.includes("usai/STALE-TOP"), "top-level model must not keep its stale value") + assert.ok(!patched.includes("usai/STALE-SMALL"), "top-level small_model must not keep its stale value") + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"')) + assert.ok(patched.includes('"small_model": "usai/claude_4_5_haiku"')) +}) + +test("every modelled agent role is patched, not just compaction", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const body = ["compaction", "summary", "title"].map((r) => agentRoleBlock(r, "STALE")).join(",\n") + const patched = patchOpenCodeConfig(templateWithAgent(body), catalog) + assert.ok(!patched.includes("usai/STALE"), "no agent role may retain a stale model") + for (const [role, expected] of [ + ["compaction", "usai/gpt-5.6-terra"], + ["summary", "usai/gpt_5_2_default_v2"], + ["title", "usai/gpt_5_2_default_v2"], + ]) { + const found = new RegExp(`"${role}"\\s*:\\s*\\{\\s*"model"\\s*:\\s*"([^"]*)"`).exec(patched) + assert.equal(found?.[1], expected, `agent.${role}.model`) + } +}) + +test("commented-out and in-array decoys are not mistaken for the real key", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + // "model": "usai/COMMENTED", + /* "model": "usai/BLOCK-COMMENTED" */ + "experimental": [ { "model": "usai/IN-ARRAY" } ], + "model": "usai/STALE-TOP", + "small_model": "usai/STALE-SMALL", +${MARKERS} +} +` + const patched = patchOpenCodeConfig(template, catalog) + assert.ok(patched.includes('"model": "usai/COMMENTED"'), "a commented-out key must be left untouched") + assert.ok(patched.includes('"model": "usai/BLOCK-COMMENTED"'), "a block-commented key must be left untouched") + assert.ok(patched.includes('"model": "usai/IN-ARRAY"'), "a key inside an array must be left untouched") + assert.ok(patched.includes('"model": "usai/claude-sonnet-5"'), "the real top-level key must be patched") +}) + +test("a configured agent role that cannot be patched is an error, not a silent skip", () => { + // `agent.compaction` exists but carries no `model` key. Leaving it alone would + // report a regenerated file while the role stayed unset, so this must throw. + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + "model": "usai/old", + "small_model": "usai/old-small", + "agent": { + "compaction": { "temperature": 0 } + }, +${MARKERS} +} +` + assert.throws( + () => patchOpenCodeConfig(template, catalog), + /could not locate a string value at 'agent\.compaction\.model'/, + ) +}) + +test("a missing top-level role is an error naming which key was not found", () => { + const catalog = JSON.parse(readFileSync(CATALOG_JSON, "utf8")) + const template = `{ + "small_model": "usai/old-small", +${MARKERS} +} +` + assert.throws(() => patchOpenCodeConfig(template, catalog), /top-level 'model'/) +})