Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
f0da47b
feat(aimlapi): add AI/ML API provider with catalog sync
Lookov Sep 1, 2026
b71c97b
Merge pull request #1 from aimlapi/feat/aimlapi-provider
Lookoff-AIMLAPI Sep 1, 2026
69cc1c9
fix(aimlapi): reference lab entries with base_model instead of inlini…
Lookov Sep 2, 2026
421d335
Merge pull request #2 from aimlapi/fix/aimlapi-base-model
Lookoff-AIMLAPI Sep 2, 2026
35a3b94
fix(aimlapi): exclude image surfaces and regenerate reasoning options…
Lookoff-AIMLAPI Sep 4, 2026
a134162
fix(aimlapi): take the union of reasoning_effort across schema variants
aimlapihello Sep 9, 2026
eb7bed5
fix(aimlapi): drop reasoning efforts the gemini flash models reject
aimlapihello Sep 9, 2026
7fbfecc
fix(aimlapi): keep measured reasoning efforts from being regenerated …
aimlapihello Sep 9, 2026
e4d3549
fix(aimlapi): publish the max effort claude-opus-4.7 actually serves
aimlapihello Sep 9, 2026
abd7feb
fix(aimlapi): stop publishing effort ladders no caller can actually use
aimlapihello Sep 9, 2026
4064155
fix(aimlapi): give aliases their own name instead of the base model's
aimlapihello Sep 9, 2026
14de1c6
fix(aimlapi): drop a catalogue row that cannot be called
aimlapihello Sep 9, 2026
66f5625
fix(aimlapi): apply the uncallable-model guard where publishing happens
aimlapihello Sep 9, 2026
a0554f8
chore(aimlapi): follow the review's reasoning baselines
aimlapihello Sep 9, 2026
82a783d
fix(aimlapi): publish what the gateway accepts, and each model once
aimlapihello Sep 9, 2026
50be79c
sync(aimlapi): pick up xhigh and the o1 identity from production
aimlapihello Sep 10, 2026
589e836
fix(aimlapi): round prices, and stop selling a dial that turns nothing
aimlapihello Sep 10, 2026
27c6cc1
fix(aimlapi): kimi-k3 has a control after all, so stop publishing []
aimlapihello Sep 10, 2026
df81f53
docs(aimlapi): the toggle the review keeps asking for is not there
aimlapihello Sep 10, 2026
b6afb01
fix(aimlapi): gpt-5.4-pro drops low, gpt-5-pro keeps its ladder
aimlapihello Sep 10, 2026
9425b5f
chore(aimlapi): take the review's baseline where it overrules our probes
aimlapihello Sep 10, 2026
17ec0f5
fix(aimlapi): repoint deepseek-chat, give DeepSeek V4 its rungs back
aimlapihello Sep 10, 2026
7ae5a66
fix(aimlapi): drop none from sonnet-5, finish measuring the DeepSeek …
aimlapihello Sep 10, 2026
85cf069
fix(aimlapi): unpin gpt-5-pro, drop qwen's unproven none, keep DeepSe…
aimlapihello Sep 10, 2026
2d693bc
docs(aimlapi): probe the pro/codex/5.6 batch the review asked about
aimlapihello Sep 10, 2026
0f8b01d
fix(aimlapi): declare the reasoning side channel, 58 models had none
aimlapihello Sep 10, 2026
27f8d59
fix(aimlapi): the base ids have no side channel, measured under force…
aimlapihello Sep 11, 2026
0be1de6
refactor(aimlapi): derive ladders from the lab entry, intersected wit…
aimlapihello Sep 11, 2026
86eeb0a
fix(aimlapi): close the two holes the review found in the doctrine, a…
aimlapihello Sep 11, 2026
2a27bbb
fix(aimlapi): wire the allowlist that was never referenced, and settl…
aimlapihello Sep 11, 2026
79001b0
Merge remote-tracking branch 'upstream/dev' into dev
Lookoff-AIMLAPI Sep 11, 2026
b4c5430
fix(aimlapi): follow the tightened sync types after merging upstream
Lookoff-AIMLAPI Sep 11, 2026
3de0050
fix(aimlapi): resolve a wire alias through the measurements of the id…
Lookoff-AIMLAPI Sep 14, 2026
c85cd37
fix(aimlapi): take the OpenRouter peer as baseline where the lab is s…
Lookoff-AIMLAPI Sep 14, 2026
0b33c6c
fix(aimlapi): a toggle- or budget-only baseline is unresolved, full stop
Lookoff-AIMLAPI Sep 14, 2026
ef5c28d
fix(aimlapi): read a lab split across two provider directories from both
Lookoff-AIMLAPI Sep 14, 2026
69964d2
Merge remote-tracking branch 'upstream/dev' into dev
Lookoff-AIMLAPI Sep 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"test": "bun test",
"validate": "bun ./packages/core/script/validate.ts",
"compare:migrations": "bun ./packages/core/script/compare-model-migrations.ts",
"aimlapi:sync": "bun ./packages/core/script/sync-models.ts aimlapi",
"anthropic:sync": "bun ./packages/core/script/sync-models.ts anthropic",
"baseten:sync": "bun ./packages/core/script/sync-models.ts baseten",
"deepinfra:sync": "bun ./packages/core/script/sync-models.ts deepinfra",
Expand Down
4 changes: 4 additions & 0 deletions packages/core/src/sync/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { AuthoredModel, AuthoredModelShape, ModelMetadata } from "../schema.js";
import { openMissingModelIssues } from "./missing-issues.js";
import { MissingReasoningOptionsError } from "./missing-reasoning-options.js";
import { aiand } from "./providers/aiand.js";
import { aimlapi } from "./providers/aimlapi.js";
import { ambient } from "./providers/ambient.js";
import { anthropic } from "./providers/anthropic.js";
import { baseten } from "./providers/baseten.js";
Expand Down Expand Up @@ -141,6 +142,7 @@ export interface SyncResult {

export const providers: {
aiand: SyncProvider<any>;
aimlapi: SyncProvider<any>;
ambient: SyncProvider<any>;
anthropic: SyncProvider<any>;
baseten: SyncProvider<any>;
Expand Down Expand Up @@ -180,6 +182,7 @@ export const providers: {
xai: SyncProvider<any>;
} = {
aiand,
aimlapi,
ambient,
anthropic,
baseten,
Expand Down Expand Up @@ -221,6 +224,7 @@ export const providers: {

export const groups = {
aggregators: [
"aimlapi",
"crossmodel",
"edenai",
"empiriolabs",
Expand Down
1,158 changes: 1,158 additions & 0 deletions packages/core/src/sync/providers/aimlapi.ts

Large diffs are not rendered by default.

30 changes: 30 additions & 0 deletions packages/core/src/sync/providers/openrouter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,36 @@ const CANONICAL_PROVIDER_PREFIXES = {
"zai-org": { provider: "zai", metadata: "zhipuai" },
} as const;

/**
* The first-party provider directory for a metadata lab id.
*
* `base_model` ids are spelled with the METADATA lab (`zhipuai/glm-4.7-flash`,
* `meta/llama-...`) while a lab's own reasoning ladder lives under its
* PROVIDER directory (`providers/zai`, `providers/llama`). The two coincide for
* most labs and not for these, and a lookup that assumes they coincide fails
* silently — the caller falls back to whatever it had, which for a relay means
* publishing the host's enum as if it were the lab's.
*/
export function providerDirForMetadataLab(metadata: string): string {
for (const entry of Object.values(CANONICAL_PROVIDER_PREFIXES)) {
if (entry.metadata === metadata) return entry.provider;
}
return metadata;
}

/**
* The OpenRouter model-directory prefixes that carry a metadata lab, in table
* order — e.g. `alibaba` lives under both `alibaba/` and `qwen/` on OpenRouter,
* and only the latter is populated.
*/
export function openrouterDirsForMetadataLab(metadata: string): string[] {
const dirs: string[] = [];
for (const [prefix, entry] of Object.entries(CANONICAL_PROVIDER_PREFIXES)) {
if (entry.metadata === metadata) dirs.push(prefix);
}
return dirs.length > 0 ? dirs : [metadata];
}

export const OpenRouterModel = z.object({
id: z.string(),
name: z.string(),
Expand Down
121 changes: 121 additions & 0 deletions packages/core/test/aimlapi.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { expect, test } from "bun:test";

import { labEffortValues, labLadder, resolveLadder } from "../src/sync/providers/aimlapi.js";
import { providerDirForMetadataLab } from "../src/sync/providers/openrouter.js";

// The lookup that silently kept the host enum for every GLM id: `base_model`
// names the metadata lab, the ladder lives under the provider directory, and
// for these two they differ.
test("maps a metadata lab to its first-party provider directory", () => {
expect(providerDirForMetadataLab("zhipuai")).toBe("zai");
expect(providerDirForMetadataLab("deepseek")).toBe("deepseek");
expect(providerDirForMetadataLab("no-such-lab")).toBe("no-such-lab");
});

test("a lab split across two provider directories is read from both", () => {
// Meta's Llama cards live under `llama/`, its Muse cards under `meta/`.
expect(providerDirForMetadataLab("meta")).toBe("llama");
expect(labLadder("meta/muse-spark-1.1")).toEqual({
kind: "effort",
values: ["minimal", "low", "medium", "high", "xhigh"],
source: "lab",
});
expect(resolveLadder("meta/muse-spark-1.1", "meta/muse-spark-1.1", ["low", "medium", "high"])).toEqual([
"low",
"medium",
"high",
]);
});

test("reads the lab ladder through the provider directory, not the model record", () => {
// `models/zhipuai/*.toml` has no reasoning_options; `providers/zai/models`
// does. A lookup on the wrong one returns undefined and looks like "no lab
// ladder", which is a different claim.
expect(labEffortValues("zhipuai/glm-5.3-flash")).toEqual(["low", "high", "max"]);
expect(labEffortValues("deepseek/deepseek-v4-pro")).toEqual(["high", "max"]);
});

test("a dated snapshot borrows the ladder of the model it snapshots", () => {
// There is no `deepseek-v4-pro-0813.toml` in the lab; without the borrow it
// fell through to the host enum and came out wider than its own base.
expect(labEffortValues("deepseek/deepseek-v4-pro-0813")).toEqual(labEffortValues("deepseek/deepseek-v4-pro"));
});

test("a lab entry with toggle only and no effort ladder yields undefined", () => {
expect(labEffortValues("zhipuai/glm-4.7-flash")).toBeUndefined();
});

const HOST = ["none", "low", "medium", "high"] as const;

test("intersects the host enum with the lab ladder and drops the rest", () => {
// lab high/max, host none..high: `low` and `medium` go although the host
// accepts them — that is the doctrine, and this test pins its cost.
expect(resolveLadder("deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro", HOST)).toEqual(["none", "high"]);
});

test("`none` survives only where measured as a real off switch", () => {
// Same lab, same host enum; only the measurement differs.
expect(resolveLadder("deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro", HOST)).toContain("none");
expect(resolveLadder("deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-flash", HOST)).toContain("none");
});

test("`none` is kept where the lab itself lists it", () => {
const host = ["none", "low", "medium", "high"];
expect(resolveLadder("openai/gpt-5.6-luna", "openai/gpt-5.6-luna", host)).toContain("none");
});

test("a measurement narrows but never widens past the lab", () => {
// gpt-5.4-pro: lab medium/high/xhigh, host low/medium/high, measured medium/high.
expect(resolveLadder("openai/gpt-5.4-pro", "openai/gpt-5.4-pro", ["low", "medium", "high"])).toEqual(["medium", "high"]);
});

// The fallthrough the review found: a toggle-only lab must never turn into a
// host-schema L/M/H dump. Either the host field is measured live for that id,
// or the model is unresolved.
test("a toggle-only lab base never yields the host's effort dump", () => {
const host = ["none", "minimal", "low", "medium", "high"];
// glm-4.5v: lab toggle-only -> unresolved.
expect(resolveLadder("z-ai/glm-4.5v", "zhipuai/glm-4.5v", host)).toBeUndefined();
// qwen3.7-max: lab toggle+budget. The host's effort field measured live
// (its names select the budget tiers), and it is still unresolved: a
// toggle/budget baseline has no effort ladder to intersect with.
expect(resolveLadder("alibaba/qwen3.7-max", "alibaba/qwen3.7-max", host)).toBeUndefined();
expect(resolveLadder("google/gemini-2.5-flash-lite", "google/gemini-2.5-flash-lite", host)).toBeUndefined();
});

test("an empty intersection is unresolved, not the host enum", () => {
// lab high/max, host offers neither -> skip, do not republish low/medium.
expect(resolveLadder("deepseek/deepseek-v4-pro", "deepseek/deepseek-v4-pro", ["low", "medium"])).toBeUndefined();
});

test("a base with no lab entry falls back to the OpenRouter peer's ladder, intersected", () => {
// No `providers/tencent/models/hy3.toml`; OpenRouter lists none/low/high.
expect(labLadder("tencent/hy3")).toEqual({ kind: "effort", values: ["none", "low", "high"], source: "peer" });
expect(resolveLadder("tencent/hy3", "tencent/hy3", ["low", "medium", "high"])).toEqual(["low", "high"]);
// A peer-listed `none` is not taken on trust the way a lab-listed one is.
expect(resolveLadder("tencent/hy3", "tencent/hy3", ["none", "low", "medium", "high"])).toEqual(["low", "high"]);
// OpenRouter files Alibaba under `qwen/`.
expect(labLadder("alibaba/qwen3.8-27b")).toEqual({ kind: "effort", values: ["low", "medium", "xhigh"], source: "peer" });
});

test("a peer that is toggle-only counts as no effort ladder, not as a licence for the host enum", () => {
// No lab entry for Opus 4.1; OpenRouter's card is toggle-only.
expect(labLadder("anthropic/claude-opus-4.1")).toEqual({ kind: "no-effort", source: "peer" });
expect(resolveLadder("anthropic/claude-opus-4.1", "anthropic/claude-opus-4.1", ["low", "medium", "high"])).toBeUndefined();
});

test("a base with neither a lab nor a peer entry is unresolved, not the host enum", () => {
expect(labLadder("nolab/nothing-here")).toEqual({ kind: "no-entry" });
expect(resolveLadder("some/model", "nolab/nothing-here", ["low", "high"])).toBeUndefined();
});

test("a wire alias resolves through the measurements of the id it answers as", () => {
const host = ["none", "low", "medium", "high"];
// `deepseek-chat` answers `"model": "deepseek-flash"` exactly as
// `deepseek-v4-flash` does, so the probes run against the latter (`none`
// switches reasoning off; `medium` is inert) apply to the alias too.
expect(resolveLadder("deepseek/deepseek-chat", "deepseek/deepseek-v4-flash", host)).toEqual(
resolveLadder("deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-flash", host),
);
expect(resolveLadder("deepseek/deepseek-chat", "deepseek/deepseek-v4-flash", host)).toEqual(["none", "low", "high"]);
});
3 changes: 3 additions & 0 deletions providers/aimlapi/logo.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
9 changes: 9 additions & 0 deletions providers/aimlapi/models/alibaba/qwen-max.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
base_model = "alibaba/qwen-max"

[cost]
input = 2.08
output = 8.32
cache_read = 0.416

[limit]
context = 32_000
13 changes: 13 additions & 0 deletions providers/aimlapi/models/alibaba/qwen-plus.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# No caller control: reasoning_effort is not read (an invalid value returns 200),
# and toggle/budget fields are dropped by the gateway (unknown top-level keys -> 200).
base_model = "alibaba/qwen-plus"
reasoning_options = []

[cost]
input = 0.52
output = 1.56
cache_read = 0.104

[limit]
context = 131_000
output = 16_384
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
base_model = "alibaba/qwen3-coder-480b-a35b-instruct"
name = "Qwen3 Coder"

[cost]
input = 1.95
output = 9.75

[limit]
context = 262_000
7 changes: 7 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3-max.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
base_model = "alibaba/qwen3-max"
name = "Qwen3 Max Instruct"

[cost]
input = 1.56
output = 7.8
cache_read = 0.312
10 changes: 10 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3-next-80b-a3b-instruct.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
base_model = "alibaba/qwen3-next-80b-a3b-instruct"
name = "Qwen3 Next 80B A3B Instruct"

[cost]
input = 0.195
output = 1.56

[limit]
context = 129_024
output = 16_384
11 changes: 11 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3.6-max-preview.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# No caller control: reasoning_effort is validated (invalid value -> 400) but low/high
# do not order reasoning tokens; toggle/budget fields are dropped by the gateway.
base_model = "alibaba/qwen3.6-max-preview"
reasoning_options = []

[interleaved]
field = "reasoning_content"

[cost]
input = 0.65
output = 3.9
16 changes: 16 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3.8-2.4t-a95b.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
base_model = "alibaba/qwen3.8-2.4t-a95b"

[interleaved]
field = "reasoning_details"

[[reasoning_options]]
type = "effort"
values = ["low", "medium"]

[cost]
input = 3.4385
output = 8.59625
cache_read = 0.6877

[limit]
output = 52_429
15 changes: 15 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3.8-27b.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
base_model = "alibaba/qwen3.8-27b"

[interleaved]
field = "reasoning_details"

[[reasoning_options]]
type = "effort"
values = ["low", "medium"]

[cost]
input = 0.61893
output = 4.40128

[limit]
output = 131_072
13 changes: 13 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3.8-flash.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
base_model = "alibaba/qwen3.8-flash"

[interleaved]
field = "reasoning_content"

[[reasoning_options]]
type = "effort"
values = ["low", "medium"]

[cost]
input = 0.208
output = 0.611
cache_read = 0.0208
13 changes: 13 additions & 0 deletions providers/aimlapi/models/alibaba/qwen3.8-max.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
base_model = "alibaba/qwen3.8-max"

[interleaved]
field = "reasoning_content"

[[reasoning_options]]
type = "effort"
values = ["low", "medium"]

[cost]
input = 2.6
output = 7.8
cache_read = 0.325
13 changes: 13 additions & 0 deletions providers/aimlapi/models/anthropic/claude-fable-5.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
base_model = "anthropic/claude-fable-5"

[interleaved]
field = "reasoning_content"

[[reasoning_options]]
type = "effort"
values = ["low", "medium", "high"]

[cost]
input = 13
output = 65
cache_read = 1.3
10 changes: 10 additions & 0 deletions providers/aimlapi/models/anthropic/claude-haiku-4.5.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# No caller control: reasoning_effort is validated (invalid value -> 400) but low/high
# do not order reasoning tokens; toggle/budget fields are dropped by the gateway.
base_model = "anthropic/claude-haiku-4-5"
name = "Claude 4.5 Haiku"
reasoning_options = []

[cost]
input = 1.3754
output = 6.877
cache_read = 0.13754
11 changes: 11 additions & 0 deletions providers/aimlapi/models/anthropic/claude-opus-4.5.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
base_model = "anthropic/claude-opus-4-5"
name = "Claude 4.5 Opus"

[[reasoning_options]]
type = "effort"
values = ["low", "medium", "high"]

[cost]
input = 6.877
output = 34.385
cache_read = 0.6877
12 changes: 12 additions & 0 deletions providers/aimlapi/models/anthropic/claude-opus-4.7.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Off: reasoning_effort = none (0 reasoning tokens); no toggle field on this surface.
base_model = "anthropic/claude-opus-4-7"
name = "Claude 4.7 Opus"

[[reasoning_options]]
type = "effort"
values = ["none", "low", "medium", "high", "xhigh", "max"]

[cost]
input = 6.877
output = 34.385
cache_read = 0.6877
12 changes: 12 additions & 0 deletions providers/aimlapi/models/anthropic/claude-opus-4.8.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Off: reasoning_effort = none (0 reasoning tokens); no toggle field on this surface.
base_model = "anthropic/claude-opus-4-8"
name = "Claude 4.8 Opus"

[[reasoning_options]]
type = "effort"
values = ["none", "low", "medium", "high", "xhigh", "max"]

[cost]
input = 6.877
output = 34.385
cache_read = 0.6877
Loading
Loading