diff --git a/core/llm/autodetect.ts b/core/llm/autodetect.ts index c8511554b8b..105feb04a6f 100644 --- a/core/llm/autodetect.ts +++ b/core/llm/autodetect.ts @@ -65,6 +65,7 @@ const PROVIDER_HANDLES_TEMPLATING: string[] = [ "relace", "openrouter", "clawrouter", + "litellm", "deepseek", "xAI", "minimax", @@ -123,6 +124,7 @@ const PROVIDER_SUPPORTS_IMAGES: string[] = [ "sagemaker", "openrouter", "clawrouter", + "litellm", "venice", "sambanova", "vertexai", diff --git a/core/llm/llms/LiteLLM.ts b/core/llm/llms/LiteLLM.ts new file mode 100644 index 00000000000..3b8166b250b --- /dev/null +++ b/core/llm/llms/LiteLLM.ts @@ -0,0 +1,33 @@ +import { LLMOptions } from "../../index.js"; + +import OpenAI from "./OpenAI.js"; + +/** + * LiteLLM LLM Provider + * + * LiteLLM is an open-source AI gateway that exposes 100+ LLM providers + * (OpenAI, Anthropic, Azure, Bedrock, Gemini, and more) behind a single + * OpenAI-compatible API. It is commonly self-hosted as a proxy so teams can + * centralize API keys, spend limits, fallbacks, and routing. + * + * Because the proxy is OpenAI-compatible, this provider extends the OpenAI + * adapter and defaults to the standard local LiteLLM proxy endpoint. Point + * `apiBase` at a remote proxy to use a hosted deployment. + * + * @see https://docs.litellm.ai/docs/simple_proxy + */ +class LiteLLM extends OpenAI { + static providerName = "litellm"; + + // A LiteLLM proxy can route to reasoning models (DeepSeek, o-series, etc.). + protected supportsReasoningField = true; + protected supportsReasoningDetailsField = true; + + static defaultOptions: Partial = { + apiBase: "http://localhost:4000/v1/", + model: "gpt-4o-mini", + useLegacyCompletionsEndpoint: false, + }; +} + +export default LiteLLM; diff --git a/core/llm/llms/LiteLLM.vitest.ts b/core/llm/llms/LiteLLM.vitest.ts new file mode 100644 index 00000000000..d4b863760fb --- /dev/null +++ b/core/llm/llms/LiteLLM.vitest.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; + +import LiteLLM from "./LiteLLM"; + +describe("LiteLLM", () => { + it("should have correct provider name", () => { + expect(LiteLLM.providerName).toBe("litellm"); + }); + + it("should default to the local LiteLLM proxy endpoint", () => { + expect(LiteLLM.defaultOptions.apiBase).toBe("http://localhost:4000/v1/"); + expect(LiteLLM.defaultOptions.useLegacyCompletionsEndpoint).toBe(false); + }); + + it("should support reasoning fields", () => { + const litellm = new LiteLLM({ model: "gpt-4o-mini" }); + + expect(litellm["supportsReasoningField"]).toBe(true); + expect(litellm["supportsReasoningDetailsField"]).toBe(true); + }); + + it("should honor a custom apiBase (remote proxy)", () => { + const litellm = new LiteLLM({ + model: "claude-3-5-sonnet", + apiBase: "https://litellm.example.com/v1/", + }); + + expect(litellm.apiBase).toBe("https://litellm.example.com/v1/"); + }); + + it("should route arbitrary proxy model names", () => { + const models = ["gpt-4o", "claude-3-5-sonnet", "gemini-2.5-flash"]; + for (const model of models) { + const litellm = new LiteLLM({ model }); + expect(litellm.model).toBe(model); + } + }); +}); diff --git a/core/llm/llms/index.ts b/core/llm/llms/index.ts index 4978f0617f2..f4b7142fcb2 100644 --- a/core/llm/llms/index.ts +++ b/core/llm/llms/index.ts @@ -35,6 +35,7 @@ import LlamaCpp from "./LlamaCpp"; import Llamafile from "./Llamafile"; import LlamaStack from "./LlamaStack"; import Lemonade from "./Lemonade"; +import LiteLLM from "./LiteLLM"; import LMStudio from "./LMStudio"; import Mistral from "./Mistral"; import Mimo from "./Mimo"; @@ -91,6 +92,7 @@ export const LLMClasses = [ OpenAI, OVHcloud, Lemonade, + LiteLLM, LMStudio, Mistral, Mimo, diff --git a/core/llm/toolSupport.test.ts b/core/llm/toolSupport.test.ts index be14a76d619..60c158aee2e 100644 --- a/core/llm/toolSupport.test.ts +++ b/core/llm/toolSupport.test.ts @@ -380,6 +380,22 @@ describe("PROVIDER_TOOL_SUPPORT", () => { }); }); + describe("litellm", () => { + const supportsFn = PROVIDER_TOOL_SUPPORT["litellm"]; + + it("should return true for tool-supporting models", () => { + expect(supportsFn("gpt-4o")).toBe(true); + expect(supportsFn("claude-3-sonnet")).toBe(true); + expect(supportsFn("gemini-pro")).toBe(true); + expect(supportsFn("deepseek-chat")).toBe(true); + }); + + it("should return false for non-tool-supporting patterns", () => { + expect(supportsFn("random-model")).toBe(false); + expect(supportsFn("")).toBe(false); + }); + }); + describe("edge cases", () => { it("should handle empty model names", () => { expect(PROVIDER_TOOL_SUPPORT["anthropic"]("")).toBe(false); diff --git a/core/llm/toolSupport.ts b/core/llm/toolSupport.ts index 3f65473233c..0b50604b00a 100644 --- a/core/llm/toolSupport.ts +++ b/core/llm/toolSupport.ts @@ -412,6 +412,35 @@ export const PROVIDER_TOOL_SUPPORT: Record boolean> = !!lower.match(/\bo[1-9]\b/) ); }, + litellm: (model) => { + // LiteLLM is a gateway that routes to many providers, so we check common + // tool-supporting model name patterns (the model name is the proxy alias). + const lower = model.toLowerCase(); + + const toolSupportingPatterns = [ + "claude", + "sonnet", + "opus", + "haiku", + "gemini", + "command-r", + "mistral", + "mixtral", + "llama-3.1", + "llama-3.2", + "llama-3.3", + "llama-4", + "qwen3", + "qwen-2.5", + "deepseek", + ]; + + return ( + toolSupportingPatterns.some((pattern) => lower.includes(pattern)) || + !!lower.match(/gpt-[4-9]/) || + !!lower.match(/\bo[1-9]\b/) + ); + }, zAI: (model) => { const lower = model.toLowerCase(); return !!lower.match(/^glm-[4-9]/); diff --git a/docs/customize/model-providers/more/litellm.mdx b/docs/customize/model-providers/more/litellm.mdx new file mode 100644 index 00000000000..5e65c0f07f6 --- /dev/null +++ b/docs/customize/model-providers/more/litellm.mdx @@ -0,0 +1,76 @@ +--- +title: "How to Configure LiteLLM with Continue" +sidebarTitle: "LiteLLM" +--- + + + [LiteLLM](https://docs.litellm.ai/docs/simple_proxy) is an open-source AI gateway that exposes 100+ LLM providers (OpenAI, Anthropic, Azure, Bedrock, Gemini, and more) behind a single OpenAI-compatible API. Run it as a self-hosted proxy to centralize API keys, spend limits, fallbacks, and routing. + + + + Get started with [LiteLLM on GitHub](https://github.com/BerriAI/litellm). + + +## Installation + +The LiteLLM proxy runs locally and provides an OpenAI-compatible API: + +```bash +pip install "litellm[proxy]" +litellm --model gpt-4o-mini +``` + +This starts the proxy at `http://localhost:4000`. Add your providers and model aliases in a [LiteLLM config](https://docs.litellm.ai/docs/proxy/configs). + +## Configuration + +Use the model name (alias) your proxy routes, and set `apiKey` to your LiteLLM virtual/master key. + + + + ```yaml title="config.yaml" + name: My Config + version: 0.0.1 + schema: v1 + + models: + - name: LiteLLM GPT-4o mini + provider: litellm + model: gpt-4o-mini + apiKey: + apiBase: http://localhost:4000/v1/ + ``` + + + ```json title="config.json" + { + "models": [ + { + "title": "LiteLLM GPT-4o mini", + "provider": "litellm", + "model": "gpt-4o-mini", + "apiKey": "", + "apiBase": "http://localhost:4000/v1/" + } + ] + } + ``` + + + +`apiBase` defaults to `http://localhost:4000/v1/`; point it at a remote proxy to use a hosted deployment. Because LiteLLM is OpenAI-compatible, any model your proxy exposes (for example `claude-3-5-sonnet`, `gemini-2.5-flash`, or a self-hosted model) can be used by setting `model` to its LiteLLM alias. + +## Auto-discover models + +Instead of listing each model, set `model: AUTODETECT` and Continue fetches the model list from your proxy's `/v1/models` endpoint, adding every model your LiteLLM proxy serves: + +```yaml title="config.yaml" +models: + - name: LiteLLM + provider: litellm + model: AUTODETECT + apiKey: + apiBase: http://localhost:4000/v1/ +``` + +This is also what the **LiteLLM** option in the "Add Model" UI configures — provide the proxy URL and key, and the available models are discovered for you. diff --git a/docs/customize/model-providers/overview.mdx b/docs/customize/model-providers/overview.mdx index 7ba030dcb4d..46211a9c4eb 100644 --- a/docs/customize/model-providers/overview.mdx +++ b/docs/customize/model-providers/overview.mdx @@ -35,6 +35,7 @@ Beyond the top-level providers, Continue supports many other options: | [DeepInfra](/customize/model-providers/more/deepinfra) | Hosting for various open source models | | [OpenRouter](/customize/model-providers/top-level/openrouter) | Gateway to multiple model providers | | [ClawRouter](/customize/model-providers/more/clawrouter) | Open-source LLM router with automatic cost-optimized model selection | +| [LiteLLM](/customize/model-providers/more/litellm) | Open-source AI gateway exposing 100+ providers via an OpenAI-compatible proxy | | [Tetrate Agent Router Service](/customize/model-providers/top-level/tetrate_agent_router_service) | Gateway with intelligent routing across multiple model providers | | [Cohere](/customize/model-providers/more/cohere) | Models specialized for semantic search and text generation | | [NVIDIA](/customize/model-providers/more/nvidia) | GPU-accelerated model hosting | diff --git a/docs/docs.json b/docs/docs.json index b7a1d83f13a..461181b35c3 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -110,6 +110,7 @@ "customize/model-providers/more/deepseek", "customize/model-providers/more/deepinfra", "customize/model-providers/more/groq", + "customize/model-providers/more/litellm", "customize/model-providers/more/llamacpp", "customize/model-providers/more/llamastack", "customize/model-providers/more/mimo", diff --git a/gui/src/pages/AddNewModel/configs/providers.ts b/gui/src/pages/AddNewModel/configs/providers.ts index 9e2aba08c5c..bfeffe53b97 100644 --- a/gui/src/pages/AddNewModel/configs/providers.ts +++ b/gui/src/pages/AddNewModel/configs/providers.ts @@ -224,6 +224,36 @@ export const providers: Partial> = { ], }, + litellm: { + title: "LiteLLM", + provider: "litellm", + description: + "Self-hosted OpenAI-compatible gateway to 100+ providers; auto-discovers the models your proxy serves.", + longDescription: + "[LiteLLM](https://docs.litellm.ai/docs/simple_proxy) is a self-hosted proxy that exposes 100+ LLM providers behind a single OpenAI-compatible API. Run the proxy, then point Continue at its URL with your key — Continue auto-discovers the models your proxy serves via `/v1/models`.", + tags: [ModelProviderTags.RequiresApiKey], + collectInputFor: [ + { + inputType: "text", + key: "apiKey", + label: "API Key", + placeholder: "Enter your LiteLLM virtual/master key", + required: true, + }, + { ...apiBaseInput, defaultValue: "http://localhost:4000/v1" }, + ...completionParamsInputsConfigs, + ], + packages: [ + { + ...models.AUTODETECT, + params: { + ...models.AUTODETECT.params, + title: "LiteLLM", + }, + }, + ], + }, + moonshot: { title: "Moonshot", provider: "moonshot",