From 7eb23210203d57ec4fd62cc7176f7e83d0f6fd5d Mon Sep 17 00:00:00 2001 From: np Date: Sun, 17 May 2026 10:23:51 +0200 Subject: [PATCH] feat(providers): add "local/" routing prefix for OpenAI-compatible local servers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A common use case is running an OpenAI-compatible inference server locally — Ollama, LM Studio, vLLM, llama.cpp — and routing to it through the OpenAI provider client. The existing "openai/" prefix would work, but mentally conflates "local Ollama / vLLM" with "the OpenAI API itself", and the more obvious model id formats actively misroute: - "qwen/qwen3:14b" or "qwen3-coder" → metadata_for_model maps the "qwen" family to DashScope (DASHSCOPE_API_KEY, dashscope.aliyuncs.com). A user serving Qwen3 locally on Ollama gets routed to Alibaba's cloud instead of their local server. - "kimi/kimi-k2.5" → same DashScope routing. - "grok-..." → routes to xAI. None of these can express "this is a local copy of that family, route as OpenAI-compatible to my local server" without forcibly setting OPENAI_BASE_URL plus the "openai/" prefix — which is unintuitive. The "local/" prefix solves it: operators express intent ("this is a local inference server, route as OpenAI client") without naming collision. Routing semantics are identical to "openai/": OpenAi provider, OPENAI_API_KEY auth env (typically an unused placeholder for local servers), OPENAI_BASE_URL. Two places updated: - metadata_for_model: recognise "local/" alongside "openai/" - wire_model_for_base_url: strip "local/" prefix on the wire No change to strip_routing_prefix (#[allow(dead_code)], tests only) and no change to the OpenAI gateway slug preservation logic (used by OpenRouter and similar gateways with non-default OPENAI_BASE_URL) — that behaviour for "openai/" stays exactly as upstream. --- rust/crates/api/src/providers/mod.rs | 5 ++++- rust/crates/api/src/providers/openai_compat.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/rust/crates/api/src/providers/mod.rs b/rust/crates/api/src/providers/mod.rs index af1776fefb..2df23980cf 100644 --- a/rust/crates/api/src/providers/mod.rs +++ b/rust/crates/api/src/providers/mod.rs @@ -254,7 +254,10 @@ pub fn metadata_for_model(model: &str) -> Option { // route to the correct provider regardless of which auth env vars are set. // Without this, detect_provider_kind falls through to the auth-sniffer // order and misroutes to Anthropic if ANTHROPIC_API_KEY is present. - if canonical.starts_with("openai/") || canonical.starts_with("gpt-") { + if canonical.starts_with("openai/") + || canonical.starts_with("local/") + || canonical.starts_with("gpt-") + { return Some(ProviderMetadata { provider: ProviderKind::OpenAi, auth_env: "OPENAI_API_KEY", diff --git a/rust/crates/api/src/providers/openai_compat.rs b/rust/crates/api/src/providers/openai_compat.rs index 1d53f120b4..21f6dc244a 100644 --- a/rust/crates/api/src/providers/openai_compat.rs +++ b/rust/crates/api/src/providers/openai_compat.rs @@ -939,7 +939,7 @@ fn wire_model_for_base_url<'a>( return Cow::Borrowed(&model[pos + 1..]); } - if matches!(lowered_prefix.as_str(), "xai" | "grok" | "qwen" | "kimi") { + if matches!(lowered_prefix.as_str(), "local" | "xai" | "grok" | "qwen" | "kimi") { return Cow::Borrowed(&model[pos + 1..]); }