Skip to content

feat: add automatic model routing #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
SDU-W-ZhangLab wants to merge 1 commit into from
+438 −63
Closed

feat: add automatic model routing #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions crates/agent-gui/src/i18n/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,9 @@ export const translations: Record<Locale, Record<string, string>> = {
"chat.retryConfirmTitle": "重试该回复？",
"chat.retryConfirmDescription": "将以原始提问重新发送，该回复及之后的对话内容将被删除。",
"chat.selectModel": "选择模型",
"chat.autoModel": "Auto · 自动选模型",
"chat.autoModelHint": "根据当前任务自动选择模型，并在同一会话中接力",
"chat.autoModelConfigureHint": "先在供应商设置中为模型配置 Auto 路由档位",
"chat.searchModel": "搜索模型...",
"chat.noModelFound": "未找到匹配的模型",
"chat.expandProvider": "展开该提供商的模型",
Expand Down Expand Up @@ -1228,6 +1231,12 @@ export const translations: Record<Locale, Record<string, string>> = {
"settings.refreshModels": "刷新模型列表",
"settings.addModel": "手动添加模型",
"settings.modelSettings": "模型参数",
"settings.autoRoutingTier": "Auto 路由档位",
"settings.autoRoutingTierHint": "Auto 会按任务复杂度选择档位；关闭的模型不会参与路由。",
"settings.autoRoutingTier.off": "不参与",
"settings.autoRoutingTier.fast": "Fast · 快速",
"settings.autoRoutingTier.balanced": "Balanced · 均衡",
"settings.autoRoutingTier.reasoning": "Reasoning · 深度推理",
"settings.modelName": "模型名称",
"settings.contextWindow": "Context Window",
"settings.maxOutputToken": "Max Output Token",
Expand Down Expand Up @@ -1943,6 +1952,9 @@ export const translations: Record<Locale, Record<string, string>> = {
"chat.retryConfirmDescription":
"The original prompt will be sent again. This reply and everything after it will be removed.",
"chat.selectModel": "Select Model",
"chat.autoModel": "Auto · Choose for me",
"chat.autoModelHint": "Choose a model for each task while preserving the conversation context",
"chat.autoModelConfigureHint": "Assign Auto routing tiers to models in provider settings first",
"chat.searchModel": "Search models...",
"chat.noModelFound": "No matching models found",
"chat.expandProvider": "Expand provider models",
Expand Down Expand Up @@ -3041,6 +3053,13 @@ export const translations: Record<Locale, Record<string, string>> = {
"settings.refreshModels": "Refresh model list",
"settings.addModel": "Add model manually",
"settings.modelSettings": "Model settings",
"settings.autoRoutingTier": "Auto routing tier",
"settings.autoRoutingTierHint":
"Auto selects a tier from task complexity. Disabled models are never routed.",
"settings.autoRoutingTier.off": "Not routed",
"settings.autoRoutingTier.fast": "Fast",
"settings.autoRoutingTier.balanced": "Balanced",
"settings.autoRoutingTier.reasoning": "Reasoning",
"settings.modelName": "Model name",
"settings.contextWindow": "Context Window",
"settings.maxOutputToken": "Max Output Token",
Expand Down
36 changes: 36 additions & 0 deletions crates/agent-gui/src/lib/settings/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,34 @@ export type SelectedModel = {
model: string;
};

export const AUTO_ROUTER_PROVIDER_ID = "__liveagent_auto__";
export const AUTO_ROUTER_MODEL_ID = "auto";
export type AutoModelSelection = SelectedModel & {
customProviderId: typeof AUTO_ROUTER_PROVIDER_ID;
model: typeof AUTO_ROUTER_MODEL_ID;
};

export const AUTO_ROUTER_SELECTION: AutoModelSelection = {
customProviderId: AUTO_ROUTER_PROVIDER_ID,
model: AUTO_ROUTER_MODEL_ID,
};

export function isAutoModelSelection(
selectedModel: SelectedModel | undefined,
): selectedModel is AutoModelSelection {
return (
selectedModel?.customProviderId === AUTO_ROUTER_PROVIDER_ID &&
selectedModel.model === AUTO_ROUTER_MODEL_ID
);
}

export type AutoRoutingTier = "fast" | "balanced" | "reasoning";

export type ProviderModelConfig = {
id: string;
contextWindow: number;
maxOutputToken: number;
autoRoutingTier?: AutoRoutingTier;
};

export type ChatRuntimeControls = {
Expand Down Expand Up @@ -1010,13 +1034,20 @@ export function normalizeProviderModelConfig(
if (!id) return null;

const defaults = getProviderModelDefaults(providerId, id);
const autoRoutingTier =
obj.autoRoutingTier === "fast" ||
obj.autoRoutingTier === "balanced" ||
obj.autoRoutingTier === "reasoning"
? obj.autoRoutingTier
: undefined;
return {
id,
contextWindow: normalizePositiveInteger(obj.contextWindow, defaults.contextWindow),
maxOutputToken: normalizePositiveInteger(
obj.maxOutputToken ?? obj.maxTokens,
defaults.maxOutputToken,
),
...(autoRoutingTier ? { autoRoutingTier } : {}),
};
}

Expand Down Expand Up @@ -1491,10 +1522,14 @@ export function computeNextMemoryOrganizerRunAt(
export function normalizeSelectedModelForProviders(
selectedModel: SelectedModel | undefined,
customProviders: CustomProvider[],
options?: { allowAuto?: boolean },
): SelectedModel | undefined {
if (!selectedModel) {
return undefined;
}
if (options?.allowAuto && isAutoModelSelection(selectedModel)) {
return AUTO_ROUTER_SELECTION;
}

const provider = customProviders.find((item) => item.id === selectedModel.customProviderId);
if (!provider) {
Expand Down Expand Up @@ -1829,6 +1864,7 @@ export function normalizeSettings(input?: Partial<AppSettings> | null): AppSetti
const selectedModel = normalizeSelectedModelForProviders(
normalizeSelectedModel(obj.selectedModel),
customProviders,
{ allowAuto: true },
);

return {
Expand Down
123 changes: 65 additions & 58 deletions crates/agent-gui/src/pages/ChatPage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ import {
getSshProjectHostIds,
isAgentDevMode,
isAgentExecutionMode,
isAutoModelSelection,
isRightDockSingletonTabOpen,
normalizeChatRuntimeControls,
normalizeChatRuntimeControlsForProvider,
Expand Down Expand Up @@ -2720,7 +2721,9 @@ export function ChatPage(props: ChatPageProps) {
getDefaultNewConversationWorkdir: () =>
isAgentMode ? activeWorkspaceProjectPath || undefined : undefined,
resolveConversationSelectedModel: (json) =>
normalizeSelectedModelForProviders(parseSelectedModelJson(json), settings.customProviders),
normalizeSelectedModelForProviders(parseSelectedModelJson(json), settings.customProviders, {
allowAuto: true,
}),
setCurrentConversationId,
setErrorMessage,
setHydratingConversationId,
Expand Down Expand Up @@ -3275,6 +3278,7 @@ export function ChatPage(props: ChatPageProps) {
selectedModel: normalizeSelectedModelForProviders(
parseSelectedModelJson(record.selectedModelJson),
settings.customProviders,
{ allowAuto: true },
),
});
const historySummary: ChatHistorySummary = {
Expand Down Expand Up @@ -3601,13 +3605,71 @@ export function ChatPage(props: ChatPageProps) {
}));
}

const textOverride =
typeof overrides?.textOverride === "string" ? overrides.textOverride : null;
const hasTextOverride = textOverride !== null;
const composerDraft = hasTextOverride
? null
: (overrides?.composerDraftOverride ?? composerRef.current?.getDraft() ?? null);
let text = hasTextOverride
? textOverride.trim()
: composerDraft
? (effectiveIsAgentMode && composerDraft.largePastes.length > 0
? composerDraft.textWithoutLargePastes
: buildTextFromComposerDraft(composerDraft)
).trim()
: "";
let uploadedFiles = overrides?.uploadedFilesOverride ?? pendingUploadedFiles;

if (
effectiveIsAgentMode &&
composerDraft &&
composerDraft.largePastes.length > 0 &&
!hasTextOverride
) {
isImportingPastedTextRef.current = true;
setIsImportingPastedText(true);
try {
const imported = await importPastedTextsAsFiles(
effectiveWorkdir,
composerDraft.largePastes,
);
text = buildTextFromComposerDraft(composerDraft, imported.fileByPasteId).trim();
uploadedFiles = mergePendingUploadedFiles(uploadedFiles, imported.files);
} catch (error) {
const message = asErrorMessage(error, "大段粘贴内容导入工作区失败");
setConversationErrorState(message);
setErrorMessage(message);
gatewayBridgeEvents.emitError(message, conversationId);
gatewayBridgeEvents.close();
return false;
} finally {
isImportingPastedTextRef.current = false;
setIsImportingPastedText(false);
}
}

const userMessage = createUserMessageWithUploads(text, uploadedFiles, Date.now());
if (!userMessage) {
if (gatewayBridgeRequest) {
const message = "Message is required.";
gatewayBridgeEvents.emitError(message, conversationId);
gatewayBridgeEvents.close();
}
return false;
}
const pendingUserMessage = userMessage;
const content =
typeof pendingUserMessage.content === "string" ? pendingUserMessage.content : "";

let effectiveSelectedModel: EffectiveChatModelSelection;
try {
effectiveSelectedModel = resolveEffectiveChatModelSelection({
settings,
conversationSelectedModel:
conversationRuntimeCacheRef.current.get(conversationId)?.selectedModel,
gatewaySelectedModel: gatewayBridgeRequest?.selectedModelOverride,
routingInput: { text, hasAttachments: uploadedFiles.length > 0 },
});
} catch (error) {
const message = asErrorMessage(error, "当前模型配置不可用，请重新选择后重试。");
Expand Down Expand Up @@ -3664,63 +3726,6 @@ export function ChatPage(props: ChatPageProps) {
providerConfig.modelConfig,
);

const textOverride =
typeof overrides?.textOverride === "string" ? overrides.textOverride : null;
const hasTextOverride = textOverride !== null;
const composerDraft = hasTextOverride
? null
: (overrides?.composerDraftOverride ?? composerRef.current?.getDraft() ?? null);
let text = hasTextOverride
? textOverride.trim()
: composerDraft
? (effectiveIsAgentMode && composerDraft.largePastes.length > 0
? composerDraft.textWithoutLargePastes
: buildTextFromComposerDraft(composerDraft)
).trim()
: "";
let uploadedFiles = overrides?.uploadedFilesOverride ?? pendingUploadedFiles;

if (
effectiveIsAgentMode &&
composerDraft &&
composerDraft.largePastes.length > 0 &&
!hasTextOverride
) {
isImportingPastedTextRef.current = true;
setIsImportingPastedText(true);
try {
const imported = await importPastedTextsAsFiles(
effectiveWorkdir,
composerDraft.largePastes,
);
text = buildTextFromComposerDraft(composerDraft, imported.fileByPasteId).trim();
uploadedFiles = mergePendingUploadedFiles(uploadedFiles, imported.files);
} catch (error) {
const message = asErrorMessage(error, "大段粘贴内容导入工作区失败");
setConversationErrorState(message);
setErrorMessage(message);
gatewayBridgeEvents.emitError(message, conversationId);
gatewayBridgeEvents.close();
return false;
} finally {
isImportingPastedTextRef.current = false;
setIsImportingPastedText(false);
}
}

const userMessage = createUserMessageWithUploads(text, uploadedFiles, Date.now());
if (!userMessage) {
if (gatewayBridgeRequest) {
const message = "Message is required.";
gatewayBridgeEvents.emitError(message, conversationId);
gatewayBridgeEvents.close();
}
return false;
}
const pendingUserMessage = userMessage;
const content =
typeof pendingUserMessage.content === "string" ? pendingUserMessage.content : "";

const titleSourceText = text || uploadedFiles.map((file) => file.fileName).join(", ");

const sessionId = runtimeEntry.sessionId;
Expand Down Expand Up @@ -4892,6 +4897,7 @@ export function ChatPage(props: ChatPageProps) {

const currentModelLabel = (() => {
if (!activeSelectedModel) return t("chat.selectModel");
if (isAutoModelSelection(activeSelectedModel)) return t("chat.autoModel");
const opt = modelOptions.find((o) => o.value === selectedValue);
if (opt) return `${opt.providerName} / ${opt.model}`;
return activeSelectedModel.model;
Expand Down Expand Up @@ -4943,6 +4949,7 @@ export function ChatPage(props: ChatPageProps) {
const parsed = normalizeSelectedModelForProviders(
parseSelectedModelJson(displayedConversationPersistedModelJson),
settings.customProviders,
{ allowAuto: true },
);
if (!parsed) return;
const entry = conversationRuntimeCacheRef.current.get(currentConversationId);
Expand Down
Loading