From c6546a36dcd9b43467cebaf2f11b005e33e68c5b Mon Sep 17 00:00:00 2001 From: Aiden Cline Date: Mon, 12 Oct 2026 23:36:06 -0500 Subject: [PATCH] fix(aimlapi): stop issue floods and remove sunset models --- packages/core/src/sync/providers/aimlapi.ts | 83 ++++++++++++++----- packages/core/test/aimlapi.test.ts | 58 ++++++++++++- .../models/openai/gpt-5.1-codex-mini.toml | 14 ---- .../aimlapi/models/openai/gpt-5.1-codex.toml | 13 --- .../aimlapi/models/openai/gpt-5.2-codex.toml | 13 --- 5 files changed, 116 insertions(+), 65 deletions(-) delete mode 100644 providers/aimlapi/models/openai/gpt-5.1-codex-mini.toml delete mode 100644 providers/aimlapi/models/openai/gpt-5.1-codex.toml delete mode 100644 providers/aimlapi/models/openai/gpt-5.2-codex.toml diff --git a/packages/core/src/sync/providers/aimlapi.ts b/packages/core/src/sync/providers/aimlapi.ts index 5c8a2b80628..1f7572c1775 100755 --- a/packages/core/src/sync/providers/aimlapi.ts +++ b/packages/core/src/sync/providers/aimlapi.ts @@ -14,6 +14,10 @@ import { // The public catalog needs no key, and `include` is what turns on the pricing // and modality blocks this sync depends on. const API_ENDPOINT = "https://api.aimlapi.com/v1/models?include=pricing,modalities"; +// The catalog and docs can retain sunset models. Match both canonical IDs and +// aliases against the host's lifecycle feed before updating or reporting them. +// https://docs.aimlapi.com/api-references/service-endpoints/model-deprecations +const DEPRECATIONS_ENDPOINT = "https://api.aimlapi.com/v1/models/deprecations"; // Per-model request schema. It is the only place the API states which reasoning // controls a model actually accepts, so reasoning_options is read from here @@ -532,12 +536,23 @@ export const AimlapiModel = z .nullish(), /** Attached by fetchModels; not part of the upstream payload. */ reasoningEffort: z.array(z.string()).nullish(), + deprecated: z.boolean().optional(), }) .passthrough(); +const Deprecation = z.object({ + id: z.string(), + aliases: z.array(z.string()), + status: z.enum(["deprecated", "superseded", "withdrawn"]), + shutdown_at: z.string().regex(/^\d{4}-\d{2}-\d{2}$/).nullable(), +}); + +const DeprecationsResponse = z.object({ data: z.array(Deprecation) }); + export const AimlapiResponse = z .object({ data: z.array(AimlapiModel).min(1), + deprecations: z.array(Deprecation).optional(), }) .passthrough(); @@ -1122,17 +1137,16 @@ function wireHeader(id: string, reasoningOptions: Array<{ type: "effort"; values * The remote id to report for a chat candidate, or `undefined` for records * sync intentionally ignores: uncallable rows, dashed alias duplicates, and * anything that is not a text-output chat model. `sourceID` reports every - * candidate skip in the sync report. `missingModelID` is narrower — it routes - * a skip to the missing-model issue flow only when an issue can fix it: + * candidate skip in the sync report. `missingModelID` identifies unresolved + * metadata so the runner preserves existing cards instead of deleting them: * - * - No lab entry to point `base_model` at: the issue flow authors it. + * - No lab entry to point `base_model` at. * - A card already on disk that sync can no longer state (its reasoning - * control stopped resolving): preserved, and the issue asks for research. + * control stopped resolving). * - * A new model whose lab exists but whose thinking control cannot be stated - * on this host is deliberately not issued: no entry an issue could write - * would make the control exist. It stays unlisted until the host offers one - * that resolves, and appears here only as a report notice. + * Automatic issues are disabled for all these skips while the backlog is + * audited. Missing evidence about a thinking control is not proof that no + * working control exists; it requires research rather than an invented []. */ function candidateID(model: AimlapiModel): string | undefined { if (BLACKLISTED.has(model.id)) return undefined; @@ -1148,6 +1162,24 @@ function issueModelID(model: AimlapiModel): string | undefined { return existsSync(local) ? model.id : undefined; } +function activeModels(raw: unknown) { + const parsed = AimlapiResponse.parse(raw); + const retired = new Map((parsed.deprecations ?? []).flatMap((entry) => + [entry.id, ...entry.aliases].map((id) => [id, entry] as const), + )); + const today = new Date().toISOString().slice(0, 10); + const models = parsed.data.flatMap((model) => { + const entry = retired.get(model.id); + if (entry?.status === "withdrawn") return []; + if (entry?.status === "deprecated" && entry.shutdown_at !== null && entry.shutdown_at <= today) return []; + // Superseded IDs still resolve; only an actual sunset removes them. + return [{ ...model, deprecated: entry?.status === "deprecated" }]; + }); + indexMediaOutputs(models); + indexAliasDuplicates(models); + return models; +} + export const aimlapi = { id: "aimlapi", name: "AI/ML API", @@ -1156,11 +1188,12 @@ export const aimlapi = { // limits, reasoning controls, and removal once the host stops serving a // model (`deleteMissing`). It never creates a model: the catalog cannot // fully populate a new entry (no lab metadata, capabilities, or dates), so - // every remote-only model goes through the missing-model issue flow - // (`skipCreates`), where the lab entry and provider card are authored and - // reviewed. Later syncs then keep that file updated. + // Remote-only models remain report-only until the existing upstream backlog + // has been audited. skipCreates also admits every skipped remote ID to the + // runner's issue pipeline, independently of missingModelID; opt out here to + // prevent another batch of issues and issue-fixer PRs. skipCreates: true, - trackMissingModels: true, + trackMissingModels: false, deleteMissing: true, authoritativeHeaders: true, sourceID(model) { @@ -1172,28 +1205,31 @@ export const aimlapi = { skippedNotice(ids) { if (ids.length === 0) return []; return [ - `${ids.length} AI/ML API chat models were not written by sync. Models with no lab entry to point \`base_model\` at are routed to the missing-model issue flow for authoring; models whose thinking control cannot be stated on this host stay unlisted until the host offers one that resolves, and are listed here only. Existing files for skipped ids are preserved.`, + `${ids.length} AI/ML API chat models were not written by sync. Automatic missing-model issues are disabled pending a catalog audit; skipped IDs are report-only. Existing files for skipped ids are preserved.`, `Skipped remote IDs: ${ids.map((id) => `\`${id}\``).join(", ")}`, ]; }, async fetchModels() { - const response = await fetch(API_ENDPOINT); + const [response, deprecations] = await Promise.all([ + fetch(API_ENDPOINT), + fetch(DEPRECATIONS_ENDPOINT), + ]); if (!response.ok) { throw new Error(`AI/ML API request failed: ${response.status} ${response.statusText}`); } + if (!deprecations.ok) { + throw new Error(`AI/ML API deprecations request failed: ${deprecations.status} ${deprecations.statusText}`); + } const raw = await response.json(); const parsed = AimlapiResponse.parse(raw); - indexMediaOutputs(parsed.data); - indexAliasDuplicates(parsed.data); - await attachReasoningEffort(parsed.data); - return parsed; + const retired = DeprecationsResponse.parse(await deprecations.json()); + const models = activeModels({ ...parsed, deprecations: retired.data }); + await attachReasoningEffort(models); + return { ...parsed, data: models, deprecations: retired.data }; }, parseModels(raw) { - const models = AimlapiResponse.parse(raw).data; - // Replays parse a cached payload without going through fetchModels. - indexMediaOutputs(models); - indexAliasDuplicates(models); - return models; + // Replays apply the same lifecycle and cross-surface filters as live fetches. + return activeModels(raw); }, translateModel(model, context) { if (BLACKLISTED.has(model.id)) return undefined; @@ -1273,6 +1309,7 @@ export const aimlapi = { name: STALE_HOST_ROW.has(model.id) || STALE_HOST_NAME.has(model.id) ? undefined : (info.name ?? undefined), reasoning_options: reasoningOptions, + status: model.deprecated ? "deprecated" : undefined, interleaved: INTERLEAVED_FIELD[measuredAs(model.id)] ? { field: INTERLEAVED_FIELD[measuredAs(model.id)]! } : undefined, diff --git a/packages/core/test/aimlapi.test.ts b/packages/core/test/aimlapi.test.ts index 1013274da0e..a9fde8a447f 100644 --- a/packages/core/test/aimlapi.test.ts +++ b/packages/core/test/aimlapi.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { syncProvider } from "../src/sync/index.js"; import { aimlapi, labEffortValues, labLadder, resolveLadder } from "../src/sync/providers/aimlapi.js"; import { providerDirForMetadataLab } from "../src/sync/providers/openrouter.js"; @@ -123,12 +127,62 @@ test("a wire alias resolves through the measurements of the id it answers as", ( // Sync owns updates and removals for models already on disk, but never // creates: a new model goes through the missing-model issue flow, where its // lab entry and provider card are authored and reviewed. -test("sync updates and removes existing models but routes creates to issues", () => { +test("sync updates and removes existing models without opening backlog issues", () => { expect(aimlapi.skipCreates).toBe(true); - expect(aimlapi.trackMissingModels).toBe(true); + expect(aimlapi.trackMissingModels).toBe(false); expect(aimlapi.deleteMissing).toBe(true); }); +test("lifecycle feed removes withdrawn and expired IDs, including aliases, but retains serving models", () => { + const models = aimlapi.parseModels({ + data: ["withdrawn-alias", "expired", "future-sunset", "superseded-alias", "active"].map((id) => ({ + id, + type: "openai/chat-completions", + })), + deprecations: [ + { id: "withdrawn", aliases: ["withdrawn-alias"], status: "withdrawn", shutdown_at: null }, + { id: "expired", aliases: [], status: "deprecated", shutdown_at: "2000-01-01" }, + { id: "future-sunset", aliases: [], status: "deprecated", shutdown_at: "2999-01-01" }, + { id: "superseded", aliases: ["superseded-alias"], status: "superseded", shutdown_at: null }, + ], + }); + expect(models.map((model) => model.id)).toEqual(["future-sunset", "superseded-alias", "active"]); + expect(models[0]?.deprecated).toBe(true); + expect(models[1]?.deprecated).toBe(false); + expect(models[2]?.deprecated).toBe(false); +}); + +test("the actual sync runner never admits aimlapi skips to issues, even with openIssues enabled", async () => { + const dir = await mkdtemp(path.join(tmpdir(), "sync-aimlapi-issues-")); + try { + const result = await syncProvider({ + ...aimlapi, + modelsDir: dir, + async fetchModels() { + return { + data: [ + { id: "acme/unmapped", type: "openai/chat-completions" }, + { + id: "openai/gpt-4", + type: "openai/chat-completions", + pricing: { units: [ + { name: "token", content: "text", origin: "provided", price: 1, per: 1_000_000 }, + { name: "token", content: "text", origin: "generated", price: 2, per: 1_000_000 }, + ] }, + }, + ], + }; + }, + }, { dryRun: true, openIssues: true }); + expect(result.created).toBe(0); + expect(result.notices.some((notice) => notice.includes("Would open GitHub issue"))).toBe(false); + expect(result.notices.some((notice) => notice.includes("acme/unmapped"))).toBe(true); + expect(result.notices.some((notice) => notice.includes("openai/gpt-4"))).toBe(true); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("only chat candidates are reported for the missing-model issue flow", () => { const models = aimlapi.parseModels({ data: [ diff --git a/providers/aimlapi/models/openai/gpt-5.1-codex-mini.toml b/providers/aimlapi/models/openai/gpt-5.1-codex-mini.toml deleted file mode 100644 index dccf0944ca6..00000000000 --- a/providers/aimlapi/models/openai/gpt-5.1-codex-mini.toml +++ /dev/null @@ -1,14 +0,0 @@ -base_model = "openai/gpt-5.1-codex-mini" -name = "GPT-5.1 Codex Mini" - -[interleaved] -field = "reasoning_details" - -[[reasoning_options]] -type = "effort" -values = ["low", "medium", "high"] - -[cost] -input = 0.34385 -output = 2.7508 -cache_read = 0.041262 diff --git a/providers/aimlapi/models/openai/gpt-5.1-codex.toml b/providers/aimlapi/models/openai/gpt-5.1-codex.toml deleted file mode 100644 index 33b2d94cbe8..00000000000 --- a/providers/aimlapi/models/openai/gpt-5.1-codex.toml +++ /dev/null @@ -1,13 +0,0 @@ -base_model = "openai/gpt-5.1-codex" - -[interleaved] -field = "reasoning_details" - -[[reasoning_options]] -type = "effort" -values = ["low", "medium", "high"] - -[cost] -input = 3.4385 -output = 27.508 -cache_read = 0.34385 diff --git a/providers/aimlapi/models/openai/gpt-5.2-codex.toml b/providers/aimlapi/models/openai/gpt-5.2-codex.toml deleted file mode 100644 index 5b16e1f8b2c..00000000000 --- a/providers/aimlapi/models/openai/gpt-5.2-codex.toml +++ /dev/null @@ -1,13 +0,0 @@ -base_model = "openai/gpt-5.2-codex" - -[interleaved] -field = "reasoning_details" - -[[reasoning_options]] -type = "effort" -values = ["low", "medium", "high"] - -[cost] -input = 4.8139 -output = 38.5112 -cache_read = 0.48139