diff --git a/README.md b/README.md index b914017f6..aed03c4d8 100644 --- a/README.md +++ b/README.md @@ -415,6 +415,10 @@ Common commands include `/start`, `/help`, `/pair`, `/new`, `/sessions`, `/use`, Requesty Icon
Requesty + + LiteLLM Icon
+ LiteLLM + Opper Icon
Opper diff --git a/src/main/provider/defaults.ts b/src/main/provider/defaults.ts index f13773993..aed300382 100644 --- a/src/main/provider/defaults.ts +++ b/src/main/provider/defaults.ts @@ -580,6 +580,20 @@ const PROVIDER_DEFAULTS: LLM_PROVIDER_BASE[] = [ models: 'https://www.requesty.ai/models' } }, + { + id: 'litellm', + name: 'LiteLLM', + apiType: 'openai-completions', + apiKey: '', + baseUrl: 'http://localhost:4000/v1', + enable: false, + websites: { + official: 'https://www.litellm.ai/', + apiKey: 'https://docs.litellm.ai/docs/proxy/virtual_keys', + docs: 'https://docs.litellm.ai/docs/simple_proxy', + models: 'https://docs.litellm.ai/docs/providers' + } + }, { id: 'opper', name: 'Opper', diff --git a/src/main/provider/providerImportService.ts b/src/main/provider/providerImportService.ts index 64ebb6358..856c58a5e 100644 --- a/src/main/provider/providerImportService.ts +++ b/src/main/provider/providerImportService.ts @@ -125,6 +125,7 @@ const PROVIDER_ID_ALIASES: Record = { ppinfra: 'ppio', openrouter: 'openrouter', requesty: 'requesty', + litellm: 'litellm', demonroute: 'demonroute', orcarouter: 'orcarouter', synthorai: 'synthorai', diff --git a/src/main/provider/providerRegistry.ts b/src/main/provider/providerRegistry.ts index 2ad76a69d..5051881d8 100644 --- a/src/main/provider/providerRegistry.ts +++ b/src/main/provider/providerRegistry.ts @@ -24,6 +24,7 @@ export type AiSdkModelSourceStrategy = | 'new-api' | 'openrouter' | 'requesty' + | 'litellm' | 'demonroute' | 'ppio' | 'groq' @@ -34,6 +35,7 @@ export type AiSdkModelSourceStrategy = export type AiSdkKeyStatusStrategy = | 'none' + | 'litellm' | 'openrouter' | 'deepseek' | 'ppio' @@ -594,6 +596,15 @@ const PROVIDER_ID_REGISTRY = new Map([ credentialStrategy: 'api-key' }) ], + [ + 'litellm', + // Self-hosted proxy: the key is optional (master or virtual key only when auth is on). + createDefinition({ + ...OPENAI_BASE, + modelSource: 'litellm', + keyStatusStrategy: 'litellm' + }) + ], [ 'demonroute', createDefinition({ diff --git a/src/main/provider/providers/aiSdkProvider.ts b/src/main/provider/providers/aiSdkProvider.ts index b94c789b8..046e21891 100644 --- a/src/main/provider/providers/aiSdkProvider.ts +++ b/src/main/provider/providers/aiSdkProvider.ts @@ -187,6 +187,24 @@ function toModelRecordArray(payload: unknown): Array> { return [] } +// LiteLLM `model_info.mode` -> the DeepChat model type whose route the proxy also serves. +// `responses` aliases are served on chat completions too. +const LITELLM_MODE_TYPES: Record = { + chat: ModelType.Chat, + responses: ModelType.Chat, + embedding: ModelType.Embedding, + image_generation: ModelType.ImageGeneration, + video_generation: ModelType.VideoGeneration, + audio_speech: ModelType.TTS +} + +// LiteLLM `litellm_provider` -> model group, e.g. `vertex_ai-language-models` -> `vertex_ai` and +// `text-completion-codestral` -> `codestral`. +function toLiteLLMProviderGroup(value: unknown): string { + if (typeof value !== 'string' || !value) return 'default' + return value.replace(/^text-completion-/i, '').split('-')[0] || 'default' +} + function toPositiveFiniteNumber(value: unknown): number | undefined { return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : undefined } @@ -1741,6 +1759,8 @@ export class AiSdkProvider extends BaseLLMProvider { return this.fetchOpenAiDerivedModels(strategy) case 'requesty': return this.fetchRequestyModels() + case 'litellm': + return this.fetchLiteLLMModels() case 'demonroute': return this.fetchDemonRouteModels() case 'bedrock': @@ -1978,6 +1998,91 @@ export class AiSdkProvider extends BaseLLMProvider { return models } + /** LiteLLM management routes (`/model/info`, `/key/info`) live at the proxy root, not `/v1`. */ + private getLiteLLMProxyRoot(): string { + return (this.provider.baseUrl || '').replace(/\/+$/, '').replace(/\/v1$/i, '') + } + + /** + * LiteLLM model names are aliases chosen by whoever runs the proxy, so the shared model catalog + * cannot describe them. The proxy's `/model/info` carries the routed model's mode, limits and + * capabilities per alias; `/models` (ids only) is the fallback when the proxy returns no model + * records or the key cannot read them (401, 403, 404). Other errors are surfaced. + */ + private async fetchLiteLLMModels(): Promise { + const timeout = this.getModelFetchTimeout() + const infoUrl = `${this.getLiteLLMProxyRoot()}/model/info` + let records: Array> + try { + records = toModelRecordArray( + await this.requestProviderJson(infoUrl, { method: 'GET' }, timeout) + ) + } catch (error) { + if (!(error instanceof ProviderHttpError) || ![401, 403, 404].includes(error.status ?? 0)) { + throw error + } + console.warn('LiteLLM model info unavailable, falling back to /models:', error.status) + records = [] + } + + // Single-model CLI proxies return a data object with a wildcard name; /models has callable IDs. + if (records.length === 0) { + const listed = await this.fetchDefaultOpenAIModels({ timeout }) + return listed.filter((model) => !model.id.includes('*')) + } + + const models = new Map() + for (const record of records) { + const modelId = typeof record.model_name === 'string' ? record.model_name : '' + const info = + record.model_info && typeof record.model_info === 'object' + ? (record.model_info as Record) + : {} + // Modes without a DeepChat route (rerank, audio_transcription, moderation, ...) are skipped, + // and so are wildcard routes (`*`, `openai/*`), which are patterns rather than callable names. + const type = LITELLM_MODE_TYPES[typeof info.mode === 'string' ? info.mode : 'chat'] + if (!modelId || modelId.includes('*') || !type) { + continue + } + const contextLength = toPositiveFiniteNumber(info.max_input_tokens) + const maxTokens = toPositiveFiniteNumber(info.max_output_tokens) + const existing = models.get(modelId) + if (existing) { + // Deployments of one alias are load balanced, so a request can land on any of them: keep + // the smallest limits and only the capabilities every deployment reports. + if (contextLength !== undefined) { + existing.contextLength = Math.min(existing.contextLength ?? contextLength, contextLength) + } + if (maxTokens !== undefined) { + existing.maxTokens = Math.min(existing.maxTokens ?? maxTokens, maxTokens) + } + existing.vision = existing.vision === true && info.supports_vision === true + existing.functionCall = + existing.functionCall === true && info.supports_function_calling === true + existing.reasoning = existing.reasoning === true && info.supports_reasoning === true + continue + } + models.set(modelId, { + id: modelId, + name: modelId, + group: toLiteLLMProviderGroup(info.litellm_provider), + providerId: this.provider.id, + isCustom: false, + type, + ...(contextLength !== undefined ? { contextLength } : {}), + ...(maxTokens !== undefined ? { maxTokens } : {}), + ...(typeof info.supports_vision === 'boolean' ? { vision: info.supports_vision } : {}), + ...(typeof info.supports_function_calling === 'boolean' + ? { functionCall: info.supports_function_calling } + : {}), + ...(typeof info.supports_reasoning === 'boolean' + ? { reasoning: info.supports_reasoning } + : {}) + }) + } + return [...models.values()] + } + /** * DemonRoute exposes a standard OpenAI-compatible model catalog at `{baseUrl}/models`. * The catalog request is authenticated and doubles as the key check. @@ -2250,6 +2355,43 @@ export class AiSdkProvider extends BaseLLMProvider { } return keyStatus } + case 'litellm': { + // Spend and budget exist for virtual keys only; a keyless proxy or the master key has none. + if (!this.provider.apiKey) return null + const { signal, dispose } = this.createModelRequestSignal({ + timeout: this.getModelFetchTimeout() + }) + try { + const response = await this.fetchProvider(`${this.getLiteLLMProxyRoot()}/key/info`, { + method: 'GET', + headers: { Authorization: `Bearer ${this.provider.apiKey}` }, + signal + }) + if (response.status === 404) return null + if (!response.ok) { + const errorText = await response.text() + throw new Error( + `LiteLLM key check failed: ${response.status} ${response.statusText} - ${errorText}` + ) + } + const payload = (await response.json()) as { + info?: { spend?: number | null; max_budget?: number | null } + } + // Per-request spend is often below a cent; keep enough digits to show it. + const usd = (value: number) => `$${value.toFixed(Math.abs(value) < 0.01 ? 6 : 2)}` + const spend = Number(payload.info?.spend ?? 0) + const keyStatus: KeyStatus = { usage: usd(spend) } + const budget = payload.info?.max_budget + if (typeof budget === 'number') { + const remaining = budget - spend + keyStatus.limit_remaining = usd(remaining) + keyStatus.remainNum = remaining + } + return keyStatus + } finally { + dispose() + } + } case 'deepseek': { const response = await this.fetchProvider('https://api.deepseek.com/user/balance', { method: 'GET', diff --git a/src/renderer/settings/components/ProviderApiConfig.vue b/src/renderer/settings/components/ProviderApiConfig.vue index 818bf7b1e..fe696103c 100644 --- a/src/renderer/settings/components/ProviderApiConfig.vue +++ b/src/renderer/settings/components/ProviderApiConfig.vue @@ -487,7 +487,7 @@ watch( keyStatusError.value = '' if ( !storedApiKey || - !['ppio', 'openrouter', 'deepseek', '302ai', 'cherryin'].includes(providerId) + !['ppio', 'openrouter', 'deepseek', '302ai', 'cherryin', 'litellm'].includes(providerId) ) return try { diff --git a/src/renderer/src/assets/llm-icons/litellm.svg b/src/renderer/src/assets/llm-icons/litellm.svg new file mode 100644 index 000000000..82cbe3eeb --- /dev/null +++ b/src/renderer/src/assets/llm-icons/litellm.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/renderer/src/components/icons/modelIconRegistry.ts b/src/renderer/src/components/icons/modelIconRegistry.ts index 92ab979bc..bad7dcf9d 100644 --- a/src/renderer/src/components/icons/modelIconRegistry.ts +++ b/src/renderer/src/components/icons/modelIconRegistry.ts @@ -50,6 +50,7 @@ import kimiColorIcon from '@/assets/llm-icons/kimi-color.svg?url' import moonshotColorIcon from '@/assets/llm-icons/moonshot.svg?url' import openrouterColorIcon from '@/assets/llm-icons/openrouter.svg?url' import requestyColorIcon from '@/assets/llm-icons/requesty.png?url' +import litellmColorIcon from '@/assets/llm-icons/litellm.svg?url' import opperColorIcon from '@/assets/llm-icons/opper.svg?url' import demonrouteColorIcon from '@/assets/llm-icons/demonroute.svg?url' import orcarouterColorIcon from '@/assets/llm-icons/orcarouter.svg?url' @@ -189,6 +190,7 @@ export const modelIcons = { moonshot: moonshotColorIcon, openrouter: openrouterColorIcon, requesty: requestyColorIcon, + litellm: litellmColorIcon, opper: opperColorIcon, demonroute: demonrouteColorIcon, orcarouter: orcarouterColorIcon, diff --git a/test/main/provider/basicApiKeyProviders.test.ts b/test/main/provider/basicApiKeyProviders.test.ts index f8e9f96e7..b047afce3 100644 --- a/test/main/provider/basicApiKeyProviders.test.ts +++ b/test/main/provider/basicApiKeyProviders.test.ts @@ -483,6 +483,287 @@ describe('basic API-key provider registrations', () => { expect(mockRunAiSdkGenerateText).not.toHaveBeenCalled() }) + it('reads LiteLLM aliases with their capabilities from the proxy model info', async () => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + expect(defaults).toMatchObject({ baseUrl: 'http://localhost:4000/v1', apiKey: '' }) + const deployment = { + model_name: 'smart', + model_info: { + mode: 'chat', + litellm_provider: 'vertex_ai-language-models', + max_input_tokens: 1048576, + max_output_tokens: 65535, + supports_vision: true, + supports_function_calling: true, + supports_reasoning: true + } + } + const fetchMock = vi.fn().mockImplementation(async () => + Response.json({ + data: [ + deployment, + deployment, + { model_name: 'fast', model_info: { mode: 'chat', litellm_provider: 'openai' } }, + { model_name: 'embed', model_info: { mode: 'embedding', litellm_provider: 'openai' } }, + { + model_name: 'draw', + model_info: { mode: 'image_generation', litellm_provider: 'openai' } + }, + { model_name: 'speak', model_info: { mode: 'audio_speech', litellm_provider: 'openai' } }, + { + model_name: 'clip', + model_info: { mode: 'video_generation', litellm_provider: 'openai' } + }, + { model_name: 'rank', model_info: { mode: 'rerank', litellm_provider: 'cohere' } }, + { model_name: '*', model_info: { mode: 'chat', litellm_provider: 'openai' } }, + { model_name: 'openai/*', model_info: { mode: 'chat', litellm_provider: 'openai' } }, + { + model_name: 'code', + model_info: { mode: 'chat', litellm_provider: 'text-completion-codestral' } + }, + { + model_name: 'pool', + model_info: { + mode: 'chat', + litellm_provider: 'openai', + max_input_tokens: 1047576, + max_output_tokens: 32768, + supports_vision: true, + supports_function_calling: true + } + }, + { + model_name: 'pool', + model_info: { + mode: 'chat', + litellm_provider: 'azure', + max_input_tokens: 128000, + max_output_tokens: 16384, + supports_vision: false, + supports_function_calling: true + } + } + ] + }) + ) + global.fetch = fetchMock as unknown as typeof fetch + + const provider = new AiSdkProvider({ ...defaults }, createProviderSettings()) + expect(resolveAiSdkProviderDefinition(defaults)).toMatchObject({ + runtimeKind: 'openai-compatible', + modelSource: 'litellm', + checkStrategy: 'fetch-models', + embeddingStrategy: 'openai' + }) + expect(resolveAiSdkProviderDefinition(defaults).credentialStrategy).toBeUndefined() + + await expect(provider.fetchModels({ suppressErrors: false })).resolves.toEqual([ + expect.objectContaining({ + id: 'smart', + group: 'vertex_ai', + providerId: 'litellm', + type: 'chat', + contextLength: 1048576, + maxTokens: 65535, + vision: true, + functionCall: true, + reasoning: true + }), + expect.objectContaining({ id: 'fast', group: 'openai', type: 'chat' }), + expect.objectContaining({ id: 'embed', group: 'openai', type: 'embedding' }), + expect.objectContaining({ id: 'draw', group: 'openai', type: 'imageGeneration' }), + expect.objectContaining({ id: 'speak', group: 'openai', type: 'tts' }), + expect.objectContaining({ id: 'clip', group: 'openai', type: 'videoGeneration' }), + expect.objectContaining({ id: 'code', group: 'codestral', type: 'chat' }), + expect.objectContaining({ + id: 'pool', + group: 'openai', + contextLength: 128000, + maxTokens: 16384, + vision: false, + functionCall: true + }) + ]) + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:4000/model/info', + expect.objectContaining({ method: 'GET' }) + ) + await expect(provider.check()).resolves.toEqual({ isOk: true, errorMsg: null }) + }) + + it.each([false, true])( + 'keeps incomplete LiteLLM deployment capabilities disabled (reversed: %s)', + async (reversed) => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + const deployments = [ + { + model_name: 'pool', + model_info: { + mode: 'chat', + supports_vision: true, + supports_function_calling: true, + supports_reasoning: true + } + }, + { model_name: 'pool', model_info: { mode: 'chat' } } + ] + if (reversed) deployments.reverse() + global.fetch = vi.fn().mockImplementation(async () => Response.json({ data: deployments })) + + const provider = new AiSdkProvider({ ...defaults }, createProviderSettings()) + await expect(provider.fetchModels({ suppressErrors: false })).resolves.toEqual([ + expect.objectContaining({ + id: 'pool', + vision: false, + functionCall: false, + reasoning: false + }) + ]) + } + ) + + it('reports LiteLLM virtual key spend and remaining budget', async () => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + const fetchMock = vi + .fn() + .mockImplementation(async (url: string) => + url.endsWith('/key/info') && !url.includes('/v1/') + ? Response.json({ key: 'sk-virtual', info: { spend: 1.25, max_budget: 5 } }) + : new Response('Not Found', { status: 404 }) + ) + global.fetch = fetchMock as unknown as typeof fetch + + const provider = new AiSdkProvider( + { ...defaults, apiKey: 'sk-virtual' }, + createProviderSettings() + ) + + await expect(provider.getKeyStatus()).resolves.toEqual({ + usage: '$1.25', + limit_remaining: '$3.75', + remainNum: 3.75 + }) + expect(fetchMock).toHaveBeenCalledWith( + 'http://localhost:4000/key/info', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer sk-virtual' }) + }) + ) + }) + + it('reports no LiteLLM key status for the master key or a keyless proxy', async () => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + const fetchMock = vi + .fn() + .mockImplementation(async () => + Response.json({ error: { message: 'Key not found in database' } }, { status: 404 }) + ) + global.fetch = fetchMock as unknown as typeof fetch + + const keyless = new AiSdkProvider({ ...defaults }, createProviderSettings()) + await expect(keyless.getKeyStatus()).resolves.toBeNull() + expect(fetchMock).not.toHaveBeenCalled() + + const master = new AiSdkProvider({ ...defaults, apiKey: 'sk-master' }, createProviderSettings()) + await expect(master.getKeyStatus()).resolves.toBeNull() + }) + + it('times out stalled LiteLLM key status requests', async () => { + vi.useFakeTimers() + try { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + global.fetch = vi.fn().mockImplementation( + (_url: string, init?: RequestInit) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener('abort', () => reject(init.signal?.reason), { + once: true + }) + }) + ) + const provider = new AiSdkProvider( + { ...defaults, apiKey: 'sk-virtual' }, + createProviderSettings() + ) + const result = expect(provider.getKeyStatus()).rejects.toMatchObject({ + name: 'AbortError', + code: 'provider_request_timeout' + }) + + await vi.advanceTimersByTimeAsync(12_000) + await result + expect(vi.getTimerCount()).toBe(0) + } finally { + vi.useRealTimers() + } + }) + + it.each([401, 403, 404, 'single-model'] as const)( + 'falls back to LiteLLM /models when model info returns %s', + async (status) => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + const fetchMock = vi.fn().mockImplementation(async (url: string) => { + if (url.endsWith('/model/info')) { + if (status === 'single-model') { + return Response.json({ + data: { + model_name: '*', + litellm_params: { model: 'gpt-4.1-mini' }, + model_info: { mode: 'chat' } + } + }) + } + return new Response('Unavailable', { status }) + } + return Response.json({ + object: 'list', + data: [ + { id: 'gpt-4.1-mini', object: 'model' }, + { id: 'openai/*', object: 'model' } + ] + }) + }) + global.fetch = fetchMock as unknown as typeof fetch + vi.spyOn(console, 'warn').mockImplementation(() => {}) + + const provider = new AiSdkProvider( + { ...defaults, apiKey: 'sk-virtual' }, + createProviderSettings() + ) + + await expect(provider.fetchModels({ suppressErrors: false })).resolves.toEqual([ + expect.objectContaining({ id: 'gpt-4.1-mini', group: 'default', providerId: 'litellm' }) + ]) + expect(fetchMock).toHaveBeenLastCalledWith( + 'http://localhost:4000/v1/models', + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer sk-virtual' }) + }) + ) + } + ) + + it('surfaces LiteLLM model info errors other than a hidden route', async () => { + const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'litellm')! + const fetchMock = vi + .fn() + .mockImplementation(async (url: string) => + url.endsWith('/model/info') + ? new Response('Internal Server Error', { status: 500 }) + : Response.json({ object: 'list', data: [{ id: 'gpt-4.1-mini', object: 'model' }] }) + ) + global.fetch = fetchMock as unknown as typeof fetch + + const provider = new AiSdkProvider( + { ...defaults, apiKey: 'sk-virtual' }, + createProviderSettings() + ) + + await expect(provider.fetchModels({ suppressErrors: false })).rejects.toThrow( + 'Internal Server Error' + ) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + it('lists DemonRoute models from the authenticated OpenAI-compatible catalog', async () => { const defaults = DEFAULT_PROVIDERS.find((provider) => provider.id === 'demonroute')! expect(defaults).toBeDefined() diff --git a/test/main/provider/defaultProviders.test.ts b/test/main/provider/defaultProviders.test.ts index 5b7f95783..b604b1175 100644 --- a/test/main/provider/defaultProviders.test.ts +++ b/test/main/provider/defaultProviders.test.ts @@ -96,6 +96,25 @@ describe('DEFAULT_PROVIDERS', () => { ) }) + it('includes LiteLLM as a disabled built-in OpenAI-compatible provider', () => { + expect(DEFAULT_PROVIDERS).toContainEqual( + expect.objectContaining({ + id: 'litellm', + name: 'LiteLLM', + apiType: 'openai-completions', + baseUrl: 'http://localhost:4000/v1', + enable: false, + websites: expect.objectContaining({ + official: 'https://www.litellm.ai/', + apiKey: 'https://docs.litellm.ai/docs/proxy/virtual_keys', + docs: 'https://docs.litellm.ai/docs/simple_proxy', + models: 'https://docs.litellm.ai/docs/providers', + defaultBaseUrl: 'http://localhost:4000/v1' + }) + }) + ) + }) + it('includes Opper as a disabled built-in OpenAI-compatible provider', () => { expect(DEFAULT_PROVIDERS).toContainEqual( expect.objectContaining({