diff --git a/core/llm/llms/Nebius.ts b/core/llm/llms/Nebius.ts index e346e4ca8cb..2a502a55163 100644 --- a/core/llm/llms/Nebius.ts +++ b/core/llm/llms/Nebius.ts @@ -5,34 +5,9 @@ import OpenAI from "./OpenAI"; class Nebius extends OpenAI { static providerName = "nebius"; static defaultOptions: Partial = { - apiBase: "https://api.studio.nebius.ai/v1/", + apiBase: "https://api.tokenfactory.nebius.com/v1/", useLegacyCompletionsEndpoint: false, }; - - private static MODEL_IDS: { [name: string]: string } = { - "deepseek/deepseek_v3": "deepseek-ai/DeepSeek-V3", - "deepseek/deepseek-r1": "deepseek-ai/DeepSeek-R1", - "qwen2.5-coder-32b": "Qwen/Qwen2.5-Coder-32B-Instruct-fast", - "Qwen/QwQ-32B-Preview": "Qwen/QwQ-32B-Preview", - "llama3.1-70b-nemotron": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF-fast", - "llama3.1-8b": "meta-llama/Meta-Llama-3.1-70B-Instruct-fast", - "llama3.1-70b": "meta-llama/Meta-Llama-3.1-70B-Instruct-fast", - "llama3.1-405b": "meta-llama/Meta-Llama-3.1-405B-Instruct", - "mistral-nemo": "mistralai/Mistral-Nemo-Instruct-2407-fast", - "mistral-8x7b": "mistralai/Mixtral-8x7B-Instruct-v0.1-fast", - "mistral-8x22b": "mistralai/Mixtral-8x22B-Instruct-v0.1-fast", - "qwen-coder2.5-7b": "Qwen/Qwen2.5-Coder-7B-Instruct-fast", - "deepseek-2-lite": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct-fast", - "phi-3-mini": "microsoft/Phi-3-mini-4k-instruct-fast", - "phi-3-medium": "microsoft/Phi-3-medium-128k-instruct-fast", - "gemma2-2b-it": "google/gemma-2-2b-it-fast", - "gemma2-9b-it": "google/gemma-2-9b-it-fast", - "olmo-7b": "allenai/OLMo-7B-Instruct-hf", - }; - - protected _convertModelName(model: string) { - return Nebius.MODEL_IDS[model] || this.model; - } } export default Nebius; diff --git a/core/llm/llms/OpenAI-compatible.vitest.ts b/core/llm/llms/OpenAI-compatible.vitest.ts index 402fb7e7585..fd0b637087b 100644 --- a/core/llm/llms/OpenAI-compatible.vitest.ts +++ b/core/llm/llms/OpenAI-compatible.vitest.ts @@ -450,7 +450,8 @@ createOpenAISubclassTests(LlamaStack, { createOpenAISubclassTests(Nebius, { providerName: "nebius", - defaultApiBase: "https://api.studio.nebius.ai/v1/", + defaultApiBase: "https://api.tokenfactory.nebius.com/v1/", + testTools: true, }); createOpenAISubclassTests(OVHcloud, { diff --git a/core/llm/llms/test-utils/openai-test-utils.ts b/core/llm/llms/test-utils/openai-test-utils.ts index 4c9c3fbdb59..7527fa5b31e 100644 --- a/core/llm/llms/test-utils/openai-test-utils.ts +++ b/core/llm/llms/test-utils/openai-test-utils.ts @@ -141,6 +141,7 @@ export interface OpenAISubclassConfig { customEmbeddingsHeaders?: { [key: string]: string }; customEmbeddingsBody?: any; customBodyOptions?: any; + testTools?: boolean; } function getExpectedUrl( @@ -252,6 +253,58 @@ export const createOpenAISubclassTests = ( }); }); + if (config.testTools) { + test("chat should send tools to the provider", async () => { + const provider = new ProviderClass({ + apiKey: "test-api-key", + model: "catalog/model-id", + apiBase: config.defaultApiBase || "https://api.openai.com/v1/", + }); + const tools = [ + { + type: "function" as const, + function: { + name: "read_file", + description: "Read a file", + parameters: { + type: "object", + properties: { path: { type: "string" } }, + required: ["path"], + }, + }, + }, + ]; + + await runLlmTest({ + llm: provider, + methodToTest: "chat", + params: [ + [{ role: "user", content: "read README.md" }], + new AbortController().signal, + { tools }, + ], + expectedRequest: { + url: getExpectedUrl(config, "chat/completions"), + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: "Bearer test-api-key", + "api-key": "test-api-key", + }, + body: { + model: "catalog/model-id", + messages: [{ role: "user", content: "read README.md" }], + stream: true, + max_tokens: 4096, + tools, + ...config.customBodyOptions, + }, + }, + mockStream: [{ choices: [{ delta: { content: "" } }] }], + }); + }); + } + test("streamComplete should send a valid request", async () => { const provider = new ProviderClass({ apiKey: "test-api-key", diff --git a/docs/customize/model-providers/more/nebius.mdx b/docs/customize/model-providers/more/nebius.mdx index a678c551246..fb2161f4767 100644 --- a/docs/customize/model-providers/more/nebius.mdx +++ b/docs/customize/model-providers/more/nebius.mdx @@ -1,75 +1,44 @@ --- -title: "Nebius" -description: "Configure Nebius AI Studio with Continue to access their language model offerings, including DeepSeek R1 for chat and BAAI embeddings models" +title: "Nebius Token Factory" +description: "Configure Nebius Token Factory's OpenAI-compatible API with Continue" --- -You can get an API key from the [Nebius AI Studio API keys page](https://studio.nebius.ai/settings/api-keys) +[Nebius Token Factory](https://tokenfactory.nebius.com) provides an OpenAI-compatible inference API. Create an API key in Token Factory, then choose a model ID from its live catalog. -## Available Models +## Choose a model -Available models can be found on the [Nebius AI Studio models page](https://studio.nebius.ai/models/text2text) +Token Factory's catalog changes independently of Continue. Copy a current model ID from the [Models page](https://tokenfactory.nebius.com/models) or use the authenticated [`GET /v1/models` endpoint](https://docs.tokenfactory.nebius.com/api-reference/examples/list-of-models). Continue also fetches the catalog after you enter an API key in the model setup form. -## Chat Model +## Chat model - - - ```yaml title="config.yaml" - name: My Config - version: 0.0.1 - schema: v1 +```yaml title="config.yaml" +name: My Config +version: 0.0.1 +schema: v1 - models: - - name: DeepSeek R1 - provider: nebius - model: deepseek-ai/DeepSeek-R1 - apiKey: - ``` - - - ```json title="config.json" - { - "models": [ - { - "title": "DeepSeek R1", - "provider": "nebius", - "model": "deepseek-ai/DeepSeek-R1", - "apiKey": "" - } - ] - } - ``` - - +models: + - name: Nebius model + provider: nebius + model: + apiKey: +``` -## Embeddings Model +The provider uses the Chat Completions endpoint by default. Tool calling and other capabilities depend on the selected model; check that model's catalog metadata before enabling it for an agent role. -Available models can be found on the [Nebius AI Studio embeddings page](https://studio.nebius.ai/models/embedding) +## Embeddings model - - - ```yaml title="config.yaml" - name: My Config - version: 0.0.1 - schema: v1 +```yaml title="config.yaml" +name: My Config +version: 0.0.1 +schema: v1 - models: - - name: BAAI Embedder - provider: nebius - model: BAAI/bge-en-icl - apiKey: - roles: - - embed - ``` - - - ```json title="config.json" - { - "embeddingsProvider": { - "provider": "nebius", - "model": "BAAI/bge-en-icl", - "apiKey": "" - } - } - ``` - - +models: + - name: Nebius embedder + provider: nebius + model: + apiKey: + roles: + - embed +``` + +See the [Token Factory API documentation](https://docs.tokenfactory.nebius.com/api-reference/introduction) for authentication and API details. diff --git a/docs/customize/model-providers/overview.mdx b/docs/customize/model-providers/overview.mdx index 7ba030dcb4d..219bb379372 100644 --- a/docs/customize/model-providers/overview.mdx +++ b/docs/customize/model-providers/overview.mdx @@ -57,7 +57,7 @@ Beyond the top-level providers, Continue supports many other options: | [SambaNova](/customize/model-providers/more/SambaNova) | Enterprise AI platform | | [Watson x](/customize/model-providers/more/watsonx) | IBM's enterprise AI platform | | [Sagemaker](/customize/model-providers/more/sagemaker) | AWS machine learning platform | -| [Nebius](/customize/model-providers/more/nebius) | Cloud-based machine learning platform | +| [Nebius](/customize/model-providers/more/nebius) | OpenAI-compatible model inference API | ## How to Choose a Model Provider diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index fb3f4c61362..a06b9f67e56 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -275,7 +275,7 @@ "### vLLM\nvLLM is a highly performant way of hosting LLMs for a team. To get started, follow their [quickstart](https://docs.vllm.ai/en/latest/getting_started/quickstart.html) to set up your server.", "### Cerebras\nCerebras Inference uses specialized silicon to provides superfast inference. To get started, get your API keys from [their console](https://cloud.cerebras.ai/).", "### Ask Sage\nAsk Sage is an agnostic hosted service that provides language models. To get started with Ask Sage:\n1. Obtain an API key from your account. For more information, visit [Ask Sage](https://docs.asksage.ai/).\n2. Paste the API key below.\n3. Select a model preset.\n> [Reference](https://docs.asksage.ai/)", - "### Nebius\nNebius AI platform is a cloud service for machine learning model inference and training. To get started, obtain an API key from the [Nebius console](https://console.nebius.ai/).", + "### Nebius Token Factory\nNebius Token Factory provides an OpenAI-compatible model inference API. To get started, obtain an API key from [Token Factory](https://tokenfactory.nebius.com/).", "### Vertex AI\nVertex AI provides access to Google's foundation models and ML tools. To get started, enable the [Vertex AI API](https://console.cloud.google.com/marketplace/product/google/aiplatform.googleapis.com) and set up [Google Application Default Credentials](https://cloud.google.com/docs/authentication/provide-credentials-adc).", "### xAI offers a world class developer tool set to build scalable applications powered by Grok. To get started, obtain an API key from [the x Console](https://console.x.ai/), and see the [docs](https://docs.x.ai/docs/)", "### Secure AI management software that helps enterprises adopt and manage AI across their workforce. To get started, obtain an API key from [the Kindo console](https://app.kindo.ai/settings/api), and see the [website](https://app.kindo.ai//)", diff --git a/gui/src/pages/AddNewModel/configs/providers.ts b/gui/src/pages/AddNewModel/configs/providers.ts index 9e2aba08c5c..e3fa578d435 100644 --- a/gui/src/pages/AddNewModel/configs/providers.ts +++ b/gui/src/pages/AddNewModel/configs/providers.ts @@ -1183,11 +1183,11 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo apiKeyUrl: "https://chat.asksage.ai/", }, nebius: { - title: "Nebius AI Studio", + title: "Nebius Token Factory", provider: "nebius", refPage: "nebius", - description: "Use the Nebius API to run open-source models", - longDescription: `Nebius AI Studio is a cheap hosted service with $100 trial. To get started with Nebius AI Studio:\n1. Obtain an API key from [here](https://studio.nebius.ai)\n2. Paste below\n3. Select a model preset`, + description: "Use the Nebius Token Factory inference API", + longDescription: `Nebius Token Factory provides an OpenAI-compatible inference API. To get started:\n1. Create an API key in [Token Factory](https://tokenfactory.nebius.com)\n2. Paste it below\n3. Refresh and select a model from the live catalog`, params: { apiKey: "", }, @@ -1196,7 +1196,7 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo inputType: "text", key: "apiKey", label: "API Key", - placeholder: "Enter your Nebius AI Studio API key", + placeholder: "Enter your Nebius Token Factory API key", required: true, }, ...completionParamsInputsConfigs, @@ -1204,26 +1204,20 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo icon: "nebius.png", tags: [ModelProviderTags.RequiresApiKey, ModelProviderTags.OpenSource], packages: [ - models.deepseekR1Chat, - models.deepseekV3Chat, - models.QwenQwQ_32b_preview, - models.Qwen25Coder_32b, - models.llama318bChat, - models.llama3170bChat, - models.llama31405bChat, - models.llama3170bNemotron, - models.mistral8x7b, - models.mistral8x22b, - models.mistralNemo, - models.phi3mini, - models.phi3medium, - models.gemma2_2b, - models.gemma2_9b, - models.Qwen2Coder, - models.deepseekCoder2Lite, - models.olmo7b, - ], - apiKeyUrl: "https://studio.nebius.ai/settings/api-keys", + { + title: "gpt-oss-120b", + description: + "A current Token Factory model. Refresh after entering an API key to load the live catalog.", + params: { + title: "gpt-oss-120b", + model: "openai/gpt-oss-120b", + }, + providerOptions: ["nebius"], + icon: "openai.png", + isOpenSource: true, + }, + ], + apiKeyUrl: "https://tokenfactory.nebius.com", }, siliconflow: { title: "SiliconFlow", diff --git a/packages/openai-adapters/src/index.ts b/packages/openai-adapters/src/index.ts index 52fb2d33a0c..9ab8914038d 100644 --- a/packages/openai-adapters/src/index.ts +++ b/packages/openai-adapters/src/index.ts @@ -171,7 +171,10 @@ export function constructLlmApi(config: LLMConfig): BaseLlmApi | undefined { case "novita": return openAICompatible("https://api.novita.ai/v3/openai", config); case "nebius": - return openAICompatible("https://api.studio.nebius.ai/v1/", config); + return openAICompatible( + "https://api.tokenfactory.nebius.com/v1/", + config, + ); case "function-network": return openAICompatible("https://api.function.network/v1/", config); case "tensorix": diff --git a/packages/openai-adapters/src/test/main.test.ts b/packages/openai-adapters/src/test/main.test.ts index 8a10c8b727f..6721c202e87 100644 --- a/packages/openai-adapters/src/test/main.test.ts +++ b/packages/openai-adapters/src/test/main.test.ts @@ -216,6 +216,27 @@ if (process.env.IGNORE_API_KEY_TESTS === "true") { } describe("Configuration", () => { + it("should configure Nebius with the Token Factory API by default", () => { + const nebius = constructLlmApi({ + provider: "nebius", + apiKey: "test-api-key", + }); + + expect((nebius as OpenAIApi).openai.baseURL).toBe( + "https://api.tokenfactory.nebius.com/v1/", + ); + expect((nebius as OpenAIApi).openai.apiKey).toBe("test-api-key"); + + const customNebius = constructLlmApi({ + provider: "nebius", + apiKey: "test-api-key", + apiBase: "https://custom.example/v1/", + }); + expect((customNebius as OpenAIApi).openai.baseURL).toBe( + "https://custom.example/v1/", + ); + }); + it("should configure DeepSeek OpenAI client with correct apiBase and apiKey", () => { const deepseek = constructLlmApi({ provider: "deepseek",