From 0fcf72411484c3494d3394f5aa550c39c9a11be3 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sat, 25 Jul 2026 23:28:21 +0200 Subject: [PATCH 1/8] fix: adapt to breaking `llama.cpp` changes --- llama/addon/AddonModel.cpp | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/llama/addon/AddonModel.cpp b/llama/addon/AddonModel.cpp index bd447e11..356a30e3 100644 --- a/llama/addon/AddonModel.cpp +++ b/llama/addon/AddonModel.cpp @@ -331,16 +331,14 @@ AddonModel::AddonModel(const Napi::CallbackInfo& info) : model_params.vocab_only = options.Get("vocabOnly").As().Value(); } - if (options.Has("useMmap")) { - model_params.use_mmap = options.Get("useMmap").As().Value(); - } - - if (options.Has("useDirectIo")) { - model_params.use_direct_io = options.Get("useDirectIo").As().Value(); - } - - if (options.Has("useMlock")) { - model_params.use_mlock = options.Get("useMlock").As().Value(); + if (options.Has("useMlock") && options.Get("useMlock").As().Value()) { + model_params.load_mode = LLAMA_LOAD_MODE_MLOCK; + } else if (options.Has("useDirectIo") && options.Get("useDirectIo").As().Value()) { + model_params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; + } else if (options.Has("useMmap") && options.Get("useMmap").As().Value()) { + model_params.load_mode = LLAMA_LOAD_MODE_MMAP; + } else { + model_params.load_mode = LLAMA_LOAD_MODE_NONE; } if (options.Has("checkTensors")) { @@ -440,8 +438,7 @@ AddonModel::AddonModel(const Napi::CallbackInfo& info) : } if (model_params.no_alloc) { - model_params.use_mlock = false; - model_params.use_mmap = false; + model_params.load_mode = LLAMA_LOAD_MODE_NONE; } } From f3e4ad60e5c0a4b958522cdab1ce4983fd6598dc Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sat, 25 Jul 2026 23:51:42 +0200 Subject: [PATCH 2/8] fix: map GgmlType configs directly from native code --- llama/addon/addon.cpp | 56 +++++++++++++++++++ src/bindings/AddonTypes.ts | 1 + .../commands/InspectEstimateCommand.ts | 4 +- .../inspect/commands/InspectMeasureCommand.ts | 8 +-- src/cli/utils/resolveCommandGgufPath.ts | 6 +- src/evaluator/LlamaContext/LlamaContext.ts | 4 +- src/evaluator/LlamaModel/LlamaModel.ts | 16 +++--- src/gguf/types/GgufTensorInfoTypes.ts | 12 +++- 8 files changed, 85 insertions(+), 22 deletions(-) diff --git a/llama/addon/addon.cpp b/llama/addon/addon.cpp index 51347210..f8ce97af 100644 --- a/llama/addon/addon.cpp +++ b/llama/addon/addon.cpp @@ -1,6 +1,8 @@ #include +#include #include #include +#include #include "AddonContext.h" #include "AddonGgufMetadata.h" @@ -22,6 +24,20 @@ std::mutex backendMutex; bool backendInitialized = false; bool backendDisposed = false; +static bool compareWithUpperString(std::string_view source, std::string_view target) { + if (source.size() != target.size()) { + return false; + } + + for (std::size_t i = 0; i < source.size(); i++) { + if (static_cast(source[i]) != std::toupper(static_cast(target[i]))) { + return false; + } + } + + return true; +} + Napi::Value systemInfo(const Napi::CallbackInfo& info) { return Napi::String::From(info.Env(), llama_print_system_info()); } @@ -94,6 +110,45 @@ Napi::Value addonGetGgmlGraphOverheadCustom(const Napi::CallbackInfo& info) { return Napi::Number::New(info.Env(), graphOverhead); } +Napi::Value addonGetGgmlType(const Napi::CallbackInfo& info) { + if (info.Length() < 1) { + return info.Env().Undefined(); + } + + const auto typeParam = info[0]; + if (typeParam.IsNumber()) { + const auto typeParamValue = typeParam.As().Int32Value(); + if (typeParamValue < 0 || typeParamValue >= GGML_TYPE_COUNT) { + return info.Env().Undefined(); + } + + if (ggml_type_size(static_cast(typeParamValue)) == 0) { + return info.Env().Undefined(); + } + + return Napi::Number::New(info.Env(), typeParamValue); + } else if (typeParam.IsString()) { + const auto typeParamValue = typeParam.As().Utf8Value(); + + for (int i = 0; i < GGML_TYPE_COUNT; i++) { + if (ggml_type_size(static_cast(i)) == 0) { + continue; + } + + const auto typeName = ggml_type_name(static_cast(i)); + if (typeName == nullptr) { + continue; + } + + if (compareWithUpperString(typeParamValue, typeName)) { + return Napi::Number::New(info.Env(), i); + } + } + } + + return info.Env().Undefined(); +} + Napi::Value addonGetConsts(const Napi::CallbackInfo& info) { Napi::Object consts = Napi::Object::New(info.Env()); consts.Set("ggmlMaxDims", Napi::Number::New(info.Env(), GGML_MAX_DIMS)); @@ -301,6 +356,7 @@ Napi::Object registerCallback(Napi::Env env, Napi::Object exports) { Napi::PropertyDescriptor::Function("getBlockSizeForGgmlType", addonGetBlockSizeForGgmlType), Napi::PropertyDescriptor::Function("getTypeSizeForGgmlType", addonGetTypeSizeForGgmlType), Napi::PropertyDescriptor::Function("getGgmlGraphOverheadCustom", addonGetGgmlGraphOverheadCustom), + Napi::PropertyDescriptor::Function("getGgmlType", addonGetGgmlType), Napi::PropertyDescriptor::Function("getConsts", addonGetConsts), Napi::PropertyDescriptor::Function("setLogger", setLogger), Napi::PropertyDescriptor::Function("setLoggerLogLevel", setLoggerLogLevel), diff --git a/src/bindings/AddonTypes.ts b/src/bindings/AddonTypes.ts index 63a879ed..9c4911be 100644 --- a/src/bindings/AddonTypes.ts +++ b/src/bindings/AddonTypes.ts @@ -72,6 +72,7 @@ export type BindingModule = { getBlockSizeForGgmlType(ggmlType: number): number | undefined, getTypeSizeForGgmlType(ggmlType: number): number | undefined, getGgmlGraphOverheadCustom(size: number, grads: boolean): number, + getGgmlType(ggmlType: string | number): number | undefined, getConsts(): { ggmlMaxDims: number, ggmlTypeF16Size: number, diff --git a/src/cli/commands/inspect/commands/InspectEstimateCommand.ts b/src/cli/commands/inspect/commands/InspectEstimateCommand.ts index 8f152a02..c6b735fd 100644 --- a/src/cli/commands/inspect/commands/InspectEstimateCommand.ts +++ b/src/cli/commands/inspect/commands/InspectEstimateCommand.ts @@ -347,10 +347,10 @@ export const InspectEstimateCommand: CommandModule const resolvedKvCacheKeyType = kvCacheKeyType === "currentQuant" ? ggufInsights.dominantTensorType ?? GgmlType.F16 - : resolveGgmlTypeOption(kvCacheKeyType) ?? GgmlType.F16; + : resolveGgmlTypeOption(kvCacheKeyType, llama) ?? GgmlType.F16; const resolvedKvCacheValueType = kvCacheValueType === "currentQuant" ? ggufInsights.dominantTensorType ?? GgmlType.F16 - : resolveGgmlTypeOption(kvCacheValueType) ?? GgmlType.F16; + : resolveGgmlTypeOption(kvCacheValueType, llama) ?? GgmlType.F16; if (resolvedKvCacheKeyType != GgmlType.F16 || resolvedKvCacheValueType != GgmlType.F16) console.info(`${chalk.yellow("KV cache:")} ${GgmlType[resolvedKvCacheKeyType] + " " + GgmlType[resolvedKvCacheValueType]}`); @@ -609,7 +609,7 @@ function renderDiffPercentageWithColors(percentage: number, { } = {}): string { if (nanIsZero && Number.isNaN(percentage)) percentage = 0; - + const percentageText = percentage.toFixed(2).padStart(5, "0") + "%"; const absPercentage = Math.abs(percentage); @@ -916,7 +916,7 @@ async function runTestWorkerLogic() { batchSize, failedCreationRemedy: false }); - + if (evaluateText != null && evaluateText != "") { const sequence = context.getSequence(); await sequence.evaluateWithoutGeneratingNewTokens(model.tokenize(evaluateText)); diff --git a/src/cli/utils/resolveCommandGgufPath.ts b/src/cli/utils/resolveCommandGgufPath.ts index cd50ddca..4a6e25a9 100644 --- a/src/cli/utils/resolveCommandGgufPath.ts +++ b/src/cli/utils/resolveCommandGgufPath.ts @@ -32,10 +32,10 @@ export async function resolveCommandGgufPath(ggufPath: string | undefined, llama useMmap, kvCacheKeyType: kvCacheKeyType === "currentQuant" ? "currentQuant" - : resolveGgmlTypeOption(kvCacheKeyType), + : resolveGgmlTypeOption(kvCacheKeyType, llama), kvCacheValueType: kvCacheValueType === "currentQuant" ? "currentQuant" - : resolveGgmlTypeOption(kvCacheValueType) + : resolveGgmlTypeOption(kvCacheValueType, llama) }); const resolvedModelDestination = resolveModelDestination(ggufPath); @@ -134,7 +134,7 @@ export async function resolveCommandGgufPath(ggufPath: string | undefined, llama fileStats.map(async ({stats, info}) => { if (stats == null) return; - + if (stats.size !== info.totalSize) await fs.remove(info.filePath); }) diff --git a/src/evaluator/LlamaContext/LlamaContext.ts b/src/evaluator/LlamaContext/LlamaContext.ts index cde97350..9039b693 100644 --- a/src/evaluator/LlamaContext/LlamaContext.ts +++ b/src/evaluator/LlamaContext/LlamaContext.ts @@ -901,10 +901,10 @@ export class LlamaContext { : Boolean(flashAttentionOption); const kvCacheKeyType = options.experimentalKvCacheKeyType === "currentQuant" ? _model.fileInsights.dominantTensorType ?? _model.defaultContextKvCacheKeyType - : resolveGgmlTypeOption(options.experimentalKvCacheKeyType) ?? _model.defaultContextKvCacheKeyType; + : resolveGgmlTypeOption(options.experimentalKvCacheKeyType, _model._llama) ?? _model.defaultContextKvCacheKeyType; const kvCacheValueType = options.experimentalKvCacheValueType === "currentQuant" ? _model.fileInsights.dominantTensorType ?? _model.defaultContextKvCacheValueType - : resolveGgmlTypeOption(options.experimentalKvCacheValueType) ?? _model.defaultContextKvCacheValueType; + : resolveGgmlTypeOption(options.experimentalKvCacheValueType, _model._llama) ?? _model.defaultContextKvCacheValueType; const swaFullCache = options.swaFullCache ?? _model.defaultContextSwaFullCache; const loraOptions = typeof options.lora === "string" ? {adapters: [{filePath: options.lora}]} satisfies LlamaContextOptions["lora"] diff --git a/src/evaluator/LlamaModel/LlamaModel.ts b/src/evaluator/LlamaModel/LlamaModel.ts index 32fae3ae..c53f0968 100644 --- a/src/evaluator/LlamaModel/LlamaModel.ts +++ b/src/evaluator/LlamaModel/LlamaModel.ts @@ -73,7 +73,7 @@ export type LlamaModelOptions = { * * When using mmap, you might notice a delay the first time you actually use the model, * which is caused by the OS itself loading the model into memory. - * + * * When this option is set to `"auto"`, mmap may be disabled in scenarios where doing so allows more layers to be offloaded to the GPU. * * Defaults to `"auto"` if the current system supports it. @@ -392,7 +392,7 @@ export class LlamaModel { /** * Whether the model is loaded using mmap (memory-mapped file) or not. - * + * * When Direct I/O (setting the `useDirectIo` option to `true`) is used it'll override mmap and this value may be out of sync * with the actual usage of mmap for the loading of this model instance. */ @@ -810,11 +810,11 @@ export class LlamaModel { const resolvedDefaultContextSwaFullCache = modelOptions.defaultContextSwaFullCache ?? defaultContextSwaFullCache; const resolvedDefaultContextKvCacheKeyType = experimentalDefaultContextKvCacheKeyType === "currentQuant" ? ggufInsights.dominantTensorType ?? GgmlType.F16 - : resolveGgmlTypeOption(experimentalDefaultContextKvCacheKeyType) ?? GgmlType.F16; + : resolveGgmlTypeOption(experimentalDefaultContextKvCacheKeyType, _llama) ?? GgmlType.F16; const resolvedDefaultContextKvCacheValueType = experimentalDefaultContextKvCacheValueType === "currentQuant" ? ggufInsights.dominantTensorType ?? GgmlType.F16 - : resolveGgmlTypeOption(experimentalDefaultContextKvCacheValueType) ?? GgmlType.F16; - + : resolveGgmlTypeOption(experimentalDefaultContextKvCacheValueType, _llama) ?? GgmlType.F16; + let gpuLayers: number; let resolvedUseMmap: boolean; let resourceRequirementsEstimation: GgufInsightsResourceRequirements; @@ -830,7 +830,7 @@ export class LlamaModel { figuringGpuLayersValueLoadPercentage.percentagePerStepModelMemorySize ) ); - + const layersResolutionStartTime = Date.now(); const layersResolution = await ggufInsights.configurationResolver.resolveModelGpuLayersV2(modelOptions.gpuLayers, { ignoreMemorySafetyChecks: modelOptions.ignoreMemorySafetyChecks, @@ -855,7 +855,7 @@ export class LlamaModel { modelOptions.onLoadProgress?.(layersResolutionLoadedPercentage); }, - + _simulatorSession: simulatorSession }); const layersResolutionEndTime = Date.now(); @@ -872,7 +872,7 @@ export class LlamaModel { resourceRequirementsEstimation = await ggufInsights.estimateModelResourceRequirementsV2({ gpuLayers, useMmap: resolvedUseMmap, - + _simulatorSession: simulatorSession }); } finally { diff --git a/src/gguf/types/GgufTensorInfoTypes.ts b/src/gguf/types/GgufTensorInfoTypes.ts index 42b038a5..a8ef9b6b 100644 --- a/src/gguf/types/GgufTensorInfoTypes.ts +++ b/src/gguf/types/GgufTensorInfoTypes.ts @@ -1,3 +1,5 @@ +import type {Llama} from "../../bindings/Llama.js"; + export type GgufTensorInfo = { readonly name: string, readonly dimensions: readonly (number | bigint)[], @@ -62,14 +64,18 @@ export enum GgmlType { IQ4_NL_8_8 = 38, MXFP4 = 39, // MXFP4 (1 block) NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) - Q1_0 = 41 + Q1_0 = 41, + Q2_0 = 42 } -export function resolveGgmlTypeOption(option?: keyof typeof GgmlType | GgmlType) { +export function resolveGgmlTypeOption(option?: keyof typeof GgmlType | GgmlType, llama?: Llama) { if (option == null) return undefined; - if (typeof option === "number" && Object.hasOwn(GgmlType, option)) + const llamaGgmlType = llama?._bindings.getGgmlType(option) as GgmlType | undefined; + if (llamaGgmlType != null) + return llamaGgmlType; + else if (typeof option === "number" && Object.hasOwn(GgmlType, option)) return option as GgmlType; else if (typeof option === "string" && Object.hasOwn(GgmlType, option)) return GgmlType[option as keyof typeof GgmlType]; From b05987285dd8f6ab8a0d4b10a35c7a1979c0ace1 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Sat, 25 Jul 2026 23:58:04 +0200 Subject: [PATCH 3/8] feat: expose the model's architecture directly on the model instance --- src/chatWrappers/QwenChatWrapper.ts | 2 +- src/chatWrappers/utils/resolveChatWrapper.ts | 28 ++++++++++++-------- src/evaluator/LlamaModel/LlamaModel.ts | 4 +++ src/types.ts | 4 ++- src/utils/LruCache.ts | 2 +- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/chatWrappers/QwenChatWrapper.ts b/src/chatWrappers/QwenChatWrapper.ts index cd34057c..2b05ec60 100644 --- a/src/chatWrappers/QwenChatWrapper.ts +++ b/src/chatWrappers/QwenChatWrapper.ts @@ -354,7 +354,7 @@ export class QwenChatWrapper extends ChatWrapper { /** @internal */ public static override _checkModelCompatibility(options: ChatWrapperCheckModelCompatibilityParams): boolean { - const architecture = options.fileInfo?.metadata.general.architecture; + const architecture = options.architecture; return ( architecture == null || architecture === GgufArchitectureType.qwen2 || diff --git a/src/chatWrappers/utils/resolveChatWrapper.ts b/src/chatWrappers/utils/resolveChatWrapper.ts index b1faa330..8bb1143c 100644 --- a/src/chatWrappers/utils/resolveChatWrapper.ts +++ b/src/chatWrappers/utils/resolveChatWrapper.ts @@ -24,6 +24,7 @@ import {SeedChatWrapper} from "../SeedChatWrapper.js"; import {isJinjaTemplateEquivalentToSpecializedChatWrapper} from "./isJinjaTemplateEquivalentToSpecializedChatWrapper.js"; import {getModelLinageNames} from "./getModelLinageNames.js"; import type {GgufFileInfo} from "../../gguf/types/GgufFileInfoTypes.js"; +import type {GgufArchitectureType} from "../../gguf/types/GgufMetadataTypes.js"; export const specializedChatWrapperTypeNames = Object.freeze([ @@ -88,6 +89,7 @@ export type ResolveChatWrapperOptions = { type?: "auto" | SpecializedChatWrapperTypeName | TemplateChatWrapperTypeName, bosString?: string | null, + architecture?: GgufArchitectureType, filename?: string, fileInfo?: GgufFileInfo, tokenizer?: Tokenizer, @@ -182,6 +184,7 @@ export type ResolveChatWrapperWithModelOptions = { * * const chatWrapper = resolveChatWrapper({ * bosString: model.tokens.bosString, + * architecture: model.architecture, * filename: model.filename, * fileInfo: model.fileInfo, * tokenizer: model.tokenizer @@ -199,6 +202,7 @@ export function resolveChatWrapper( ...(modelOptions ?? {}), customWrapperSettings: modelOptions?.customWrapperSettings as ResolveChatWrapperOptions["customWrapperSettings"], bosString: options.tokens.bosString, + architecture: options.fileInfo?.metadata?.general?.architecture, filename: options.filename, fileInfo: options.fileInfo, tokenizer: options.tokenizer @@ -207,6 +211,7 @@ export function resolveChatWrapper( const { type = "auto", bosString, + architecture: archOption, filename, fileInfo, tokenizer, @@ -216,6 +221,8 @@ export function resolveChatWrapper( noJinja = false } = options; + const architecture = archOption ?? fileInfo?.metadata?.general?.architecture; + function createSpecializedChatWrapper( specializedChatWrapper: T, defaultSettings: ConstructorParameters[0] = {} @@ -293,7 +300,8 @@ export function resolveChatWrapper( const isCompatible = Wrapper._checkModelCompatibility({ tokenizer, - fileInfo + fileInfo, + architecture }); if (!isCompatible) @@ -359,9 +367,9 @@ export function resolveChatWrapper( } for (const modelNames of getModelLinageNames(fileInfo?.metadata)) { - if (includesText(modelNames, ["llama 3.2", "llama-3.2", "llama3.2"]) && Llama3_2LightweightChatWrapper._checkModelCompatibility({tokenizer, fileInfo})) + if (includesText(modelNames, ["llama 3.2", "llama-3.2", "llama3.2"]) && Llama3_2LightweightChatWrapper._checkModelCompatibility({tokenizer, fileInfo, architecture})) return createSpecializedChatWrapper(Llama3_2LightweightChatWrapper); - else if (includesText(modelNames, ["llama 3.1", "llama-3.1", "llama3.1"]) && Llama3_1ChatWrapper._checkModelCompatibility({tokenizer, fileInfo})) + else if (includesText(modelNames, ["llama 3.1", "llama-3.1", "llama3.1"]) && Llama3_1ChatWrapper._checkModelCompatibility({tokenizer, fileInfo, architecture})) return createSpecializedChatWrapper(Llama3_1ChatWrapper); else if (includesText(modelNames, ["llama 3", "llama-3", "llama3"])) return createSpecializedChatWrapper(Llama3ChatWrapper); @@ -395,7 +403,7 @@ export function resolveChatWrapper( addSpaceBeforeEos: modelJinjaTemplate.includes("' ' + eos_token") }); else if (modelJinjaTemplate.includes("<|start_header_id|>") && modelJinjaTemplate.includes("<|end_header_id|>")) { - if (Llama3_1ChatWrapper._checkModelCompatibility({tokenizer, fileInfo})) + if (Llama3_1ChatWrapper._checkModelCompatibility({tokenizer, fileInfo, architecture})) return createSpecializedChatWrapper(Llama3_1ChatWrapper); else return createSpecializedChatWrapper(Llama3ChatWrapper); @@ -455,16 +463,14 @@ export function resolveChatWrapper( } } - if (fileInfo != null) { - const arch = fileInfo.metadata.general?.architecture; - - if (arch === "llama") + if (architecture != null) { + if (architecture === "llama") return createSpecializedChatWrapper(GeneralChatWrapper); - else if (arch === "falcon") + else if (architecture === "falcon") return createSpecializedChatWrapper(FalconChatWrapper); - else if (arch === "gemma" || arch === "gemma2") + else if (architecture === "gemma" || architecture === "gemma2") return createSpecializedChatWrapper(GemmaChatWrapper); - else if (arch === "gemma4") + else if (architecture === "gemma4") return createSpecializedChatWrapper(Gemma4ChatWrapper); } diff --git a/src/evaluator/LlamaModel/LlamaModel.ts b/src/evaluator/LlamaModel/LlamaModel.ts index c53f0968..a15d939b 100644 --- a/src/evaluator/LlamaModel/LlamaModel.ts +++ b/src/evaluator/LlamaModel/LlamaModel.ts @@ -382,6 +382,10 @@ export class LlamaModel { return this._fileInsights; } + public get architecture(): GgufArchitectureType { + return this._fileInfo.metadata?.general?.architecture ?? GgufArchitectureType.unknown; + } + /** * Number of layers offloaded to the GPU. * If GPU support is disabled, this will always be `0`. diff --git a/src/types.ts b/src/types.ts index 630da6c1..7e3a4d9d 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,7 @@ import {GbnfJsonSchema, GbnfJsonSchemaToType} from "./utils/gbnfJson/types.js"; import {LlamaText, BuiltinSpecialTokenValue, LlamaTextJSON} from "./utils/LlamaText.js"; import type {GgufFileInfo} from "./gguf/types/GgufFileInfoTypes.js"; +import type {GgufArchitectureType} from "./gguf/types/GgufMetadataTypes.js"; export type Token = number & { __token: never @@ -135,7 +136,8 @@ export type ChatWrapperGenerateContextStateOptions = { export type ChatWrapperCheckModelCompatibilityParams = { tokenizer?: Tokenizer, - fileInfo?: GgufFileInfo + fileInfo?: GgufFileInfo, + architecture?: GgufArchitectureType }; export type ChatWrapperGeneratedContextState = diff --git a/src/utils/LruCache.ts b/src/utils/LruCache.ts index 7f44cd17..894f7024 100644 --- a/src/utils/LruCache.ts +++ b/src/utils/LruCache.ts @@ -57,6 +57,6 @@ export class LruCache { } public delete(key: Key) { - this._cache.delete(key); + return this._cache.delete(key); } } From ca674fd8dccb8aa3bda7f5fa062fc3f78cf3f1ca Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 4 Aug 2026 07:02:53 +0200 Subject: [PATCH 4/8] feat: improve thought segment syntax extraction --- .../generic/JinjaTemplateChatWrapper.ts | 404 +++++++++++------- ...ctFunctionCallSettingsFromJinjaTemplate.ts | 2 +- ...entSettingsFromTokenizerAndChatTemplate.ts | 370 +++++++++++++++- src/evaluator/LlamaChat/LlamaChat.ts | 40 ++ src/types.ts | 1 + src/utils/OpenAIFormat.ts | 3 +- 6 files changed, 659 insertions(+), 161 deletions(-) diff --git a/src/chatWrappers/generic/JinjaTemplateChatWrapper.ts b/src/chatWrappers/generic/JinjaTemplateChatWrapper.ts index 7ad8d990..39570a91 100644 --- a/src/chatWrappers/generic/JinjaTemplateChatWrapper.ts +++ b/src/chatWrappers/generic/JinjaTemplateChatWrapper.ts @@ -2,7 +2,7 @@ import {Template} from "@huggingface/jinja"; import {splitText} from "lifecycle-utils"; import { ChatHistoryItem, ChatModelFunctions, ChatUserMessage, ChatWrapperGenerateContextStateOptions, ChatWrapperGeneratedContextState, - ChatWrapperSettings, Tokenizer + ChatWrapperSettings, isChatModelResponseSegment, Tokenizer } from "../../types.js"; import {SpecialToken, LlamaText, SpecialTokensText} from "../../utils/LlamaText.js"; import {ChatWrapper} from "../../ChatWrapper.js"; @@ -27,6 +27,15 @@ import {extractSegmentSettingsFromTokenizerAndChatTemplate} from "./utils/extrac export type JinjaTemplateChatWrapperOptions = { template: string, + /** + * Whether to enable reasoning in the Jinja template. + * + * When set to `null`, the thinking setting will be omitted from the Jinja template, which would cause its default setting to be used. + * + * Defaults to `true`. + */ + reasoning?: boolean | null, + /** * Defaults to `"assistant"`. */ @@ -97,6 +106,15 @@ export type JinjaTemplateChatWrapperOptions = { */ segments?: TemplateChatWrapperSegmentsOptions, + /** + * Whether to keep only the chain of thought from the last model response. + * + * When `false`, all the chain of thoughts from the model responses will be kept in the context state. + * + * The default setting is extracted from the Jinja template, and the extraction fails, defaults to `false`. + */ + keepOnlyLastThought?: boolean, + /** * Pass a model's tokenizer to attempt to detect common tokens used for chat formatting from it. * @@ -154,12 +172,14 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { public override readonly settings: ChatWrapperSettings; public readonly template: string; + public readonly reasoning: boolean | null; public readonly modelRoleName: string; public readonly userRoleName: string; public readonly systemRoleName: string; public readonly convertUnsupportedSystemMessagesToUserMessages?: JinjaTemplateChatWrapperOptionsConvertMessageFormat; public readonly joinAdjacentMessagesOfTheSameType: boolean; public readonly trimLeadingWhitespaceInResponses: boolean; + public readonly keepOnlyLastThought: boolean; public readonly additionalRenderParameters?: Record; /** @internal */ private readonly _jinjaTemplate: Template; @@ -178,6 +198,7 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { const { template, + reasoning = true, modelRoleName = "assistant", userRoleName = "user", systemRoleName = "system", @@ -185,6 +206,7 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { functionCallMessageTemplate = "auto", joinAdjacentMessagesOfTheSameType = true, trimLeadingWhitespaceInResponses = true, + keepOnlyLastThought, additionalRenderParameters, segments, tokenizer, @@ -196,6 +218,7 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { throw new Error("template cannot be null"); this.template = template; + this.reasoning = reasoning; this.modelRoleName = modelRoleName; this.userRoleName = userRoleName; this.systemRoleName = systemRoleName; @@ -218,11 +241,7 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { const {supportsSystemMessages, needsToEndJinjaMessagesWithUserMessage} = this._runSanityTest(); this.settings = { ...this.settings, - supportsSystemMessages, - segments: { - ...this.settings.segments, - ...extractSegmentSettingsFromTokenizerAndChatTemplate(this.template, tokenizer) - } + supportsSystemMessages }; if (needsToEndJinjaMessagesWithUserMessage) @@ -233,138 +252,137 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { ? undefined : functionCallMessageTemplate ); - if (functionCallSettings == null && functionCallMessageTemplate !== "noJinja") { - try { - const renderTemplate: ExtractFunctionCallSettingsRenderTemplate = ({ - chatHistory, functions, additionalParams, stringifyFunctionParams, stringifyFunctionResults, - combineModelMessageAndToolCalls, squashModelTextResponses = true, setFunctionNameInResponse - }) => { - const render = ( - convertSystemMessagesToUserMessagesFormat: - JinjaTemplateChatWrapperOptionsConvertMessageFormat["format"] | undefined, - wipeFunctionCallIds: boolean | "align", - setFunctionNameInResponse?: string | boolean - ) => { - let inputChatHistory = chatHistory; - if (this._wrapFunctionParamsInsideMapKey != null) - inputChatHistory = inputChatHistory.map((item) => { - if (item.type !== "model") - return item; + const renderTemplate: ExtractFunctionCallSettingsRenderTemplate = ({ + chatHistory, functions, additionalParams, stringifyFunctionParams, stringifyFunctionResults, + combineModelMessageAndToolCalls, squashModelTextResponses = true, setFunctionNameInResponse + }) => { + const render = ( + convertSystemMessagesToUserMessagesFormat: JinjaTemplateChatWrapperOptionsConvertMessageFormat["format"] | undefined, + wipeFunctionCallIds: boolean | "align", + setFunctionNameInResponse?: string | boolean + ) => { + let inputChatHistory = chatHistory; + if (this._wrapFunctionParamsInsideMapKey != null) + inputChatHistory = inputChatHistory.map((item) => { + if (item.type !== "model") + return item; + + return { + ...item, + response: item.response.map((response) => { + if (typeof response === "string" || response.type !== "functionCall") + return response; return { - ...item, - response: item.response.map((response) => { - if (typeof response === "string" || response.type !== "functionCall") - return response; - - return { - ...response, - params: {[this._wrapFunctionParamsInsideMapKey!]: response.params} - }; - }) + ...response, + params: {[this._wrapFunctionParamsInsideMapKey!]: response.params} }; - }); - - const {messages: intermediateMessages, tools} = fromChatHistoryToIntermediateOpenAiMessages({ - chatHistory: this._transformChatHistory(inputChatHistory, { - convertSystemMessagesToUserMessagesFormat, - joinAdjacentMessagesOfTheSameType: !squashModelTextResponses - ? false - : undefined - }).transformedHistory, - chatWrapperSettings: this.settings, - useRawValues: false, - functions, - stringifyFunctionParams, - stringifyFunctionResults, - combineModelMessageAndToolCalls, - squashModelTextResponses, - setFunctionNameInResponse - }); - - const messages = fromIntermediateToCompleteOpenAiMessages(intermediateMessages) - .map((item) => { - if (!wipeFunctionCallIds) - return item; - - if (item.role === "assistant" && item["tool_calls"] != null && item["tool_calls"].length > 0) { - for (const toolCall of item["tool_calls"]) { - if (wipeFunctionCallIds === "align") - toolCall.id = "fc_1_0001"; - else - delete (toolCall as {id?: string}).id; - } - } else if (item.role === "tool") { - if (wipeFunctionCallIds === "align") - item["tool_call_id"] = "fc_1_0001"; - else - delete (item as {"tool_call_id"?: string})["tool_call_id"]; - } - - return item; - }); - - const lastJinjaItem = messages.at(-1); - let eraseRenderedJinjaFromId: string | undefined; - if (this._endJinjaMessagesWithUserMessage && lastJinjaItem?.role === this.modelRoleName && - typeof lastJinjaItem.content === "string" && - lastJinjaItem.content.length > 0 && - ( - (lastJinjaItem as OpenAiChatAssistantMessage)["tool_calls"] == null || - (lastJinjaItem as OpenAiChatAssistantMessage)["tool_calls"]?.length === 0 - ) - ) { - eraseRenderedJinjaFromId = lastJinjaItem.content; - messages.push({ - role: this.userRoleName, - content: idsGenerator.generateId() - } as OpenAiChatMessage); - } + }) + }; + }); - let res = this._jinjaTemplate.render({ - ...( - this.additionalRenderParameters == null - ? {} - : structuredClone(this.additionalRenderParameters) - ), - ...additionalParams, - messages, - ...removeUndefinedFields({tools}) - }); - - if (eraseRenderedJinjaFromId != null) { - const eraseIndex = res.lastIndexOf(eraseRenderedJinjaFromId); - if (eraseIndex >= 0) - res = res.slice(0, eraseIndex + eraseRenderedJinjaFromId.length); + const {messages: intermediateMessages, tools} = fromChatHistoryToIntermediateOpenAiMessages({ + chatHistory: this._transformChatHistory(inputChatHistory, { + convertSystemMessagesToUserMessagesFormat, + joinAdjacentMessagesOfTheSameType: !squashModelTextResponses + ? false + : undefined + }).transformedHistory, + chatWrapperSettings: this.settings, + useRawValues: false, + functions, + stringifyFunctionParams, + stringifyFunctionResults, + combineModelMessageAndToolCalls, + squashModelTextResponses, + setFunctionNameInResponse + }); + + const messages = fromIntermediateToCompleteOpenAiMessages(intermediateMessages) + .map((item) => { + if (!wipeFunctionCallIds) + return item; + + if (item.role === "assistant" && item["tool_calls"] != null && item["tool_calls"].length > 0) { + for (const toolCall of item["tool_calls"]) { + if (wipeFunctionCallIds === "align") + toolCall.id = "fc_1_0001"; + else + delete (toolCall as {id?: string}).id; + } + } else if (item.role === "tool") { + if (wipeFunctionCallIds === "align") + item["tool_call_id"] = "fc_1_0001"; + else + delete (item as {"tool_call_id"?: string})["tool_call_id"]; } - // attempt to remove the ID pattern from the output - if (wipeFunctionCallIds === "align") - res = res - .replaceAll(/,\s*"(tool_call_id|call_id|id)":\s*"fc_1_0001"/g, "") - .replaceAll(/"(tool_call_id|call_id|id)":\s*"fc_1_0001"\s*,/g, ""); + return item; + }); - return res; - }; + const lastJinjaItem = messages.at(-1); + let eraseRenderedJinjaFromId: string | undefined; + if (this._endJinjaMessagesWithUserMessage && lastJinjaItem?.role === this.modelRoleName && + typeof lastJinjaItem.content === "string" && + lastJinjaItem.content.length > 0 && + ( + (lastJinjaItem as OpenAiChatAssistantMessage)["tool_calls"] == null || + (lastJinjaItem as OpenAiChatAssistantMessage)["tool_calls"]?.length === 0 + ) + ) { + eraseRenderedJinjaFromId = lastJinjaItem.content; + messages.push({ + role: this.userRoleName, + content: idsGenerator.generateId() + } as OpenAiChatMessage); + } - return tryMatrix({ - convertSystemMessagesToUserMessagesFormat: - getConvertUnsupportedSystemMessagesToUserMessagesTryOptions( - this.convertUnsupportedSystemMessagesToUserMessages - ), - wipeFunctionCallIds: [true, "align", false], - setFunctionNameInResponse: setFunctionNameInResponse == null - ? [false] - : [false, setFunctionNameInResponse] - }, ({convertSystemMessagesToUserMessagesFormat, wipeFunctionCallIds, setFunctionNameInResponse}) => { - return render(convertSystemMessagesToUserMessagesFormat, wipeFunctionCallIds, setFunctionNameInResponse); - }); - }; - const idsGenerator = new UniqueIdGenerator( - this.template + this.modelRoleName + this.userRoleName + this.systemRoleName + - (this.convertUnsupportedSystemMessagesToUserMessages?.format ?? "") - ); + let res = this._jinjaTemplate.render({ + ...( + this.additionalRenderParameters == null + ? {} + : structuredClone(this.additionalRenderParameters) + ), + ...additionalParams, + messages, + ...removeUndefinedFields({tools}) + }); + + if (eraseRenderedJinjaFromId != null) { + const eraseIndex = res.lastIndexOf(eraseRenderedJinjaFromId); + if (eraseIndex >= 0) + res = res.slice(0, eraseIndex + eraseRenderedJinjaFromId.length); + } + + // attempt to remove the ID pattern from the output + if (wipeFunctionCallIds === "align") + res = res + .replaceAll(/,\s*"(tool_call_id|call_id|id)":\s*"fc_1_0001"/g, "") + .replaceAll(/"(tool_call_id|call_id|id)":\s*"fc_1_0001"\s*,/g, ""); + + return res; + }; + return tryMatrix({ + convertSystemMessagesToUserMessagesFormat: + getConvertUnsupportedSystemMessagesToUserMessagesTryOptions( + this.convertUnsupportedSystemMessagesToUserMessages + ), + wipeFunctionCallIds: [true, "align", false], + setFunctionNameInResponse: setFunctionNameInResponse == null + ? [false] + : [false, setFunctionNameInResponse] + }, ({convertSystemMessagesToUserMessagesFormat, wipeFunctionCallIds, setFunctionNameInResponse}) => { + return render(convertSystemMessagesToUserMessagesFormat, wipeFunctionCallIds, setFunctionNameInResponse); + }); + }; + const idsGenerator = new UniqueIdGenerator( + this.template + this.modelRoleName + this.userRoleName + this.systemRoleName + + (this.convertUnsupportedSystemMessagesToUserMessages?.format ?? "") + ); + + if (functionCallSettings == null && functionCallMessageTemplate !== "noJinja") { + try { this._wrapFunctionParamsInsideMapKey = detectNeedToWrapFunctionArgumentsWithMap({idsGenerator, renderTemplate}); const extractedSettings = extractFunctionCallSettingsFromJinjaTemplate({ idsGenerator, @@ -386,10 +404,31 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { throw new Error("failed to extract function call settings from the Jinja template"); } + const extractedSegmentSettings = extractSegmentSettingsFromTokenizerAndChatTemplate({ + chatTemplate: this.template, + tokenizer, + renderRawJinjaTemplate: (params: Record) => { + return this._jinjaTemplate.render({ + ...( + this.additionalRenderParameters == null + ? {} + : structuredClone(this.additionalRenderParameters) + ), + ...params + }); + }, + idsGenerator, + enableReasoning: this.reasoning + }); this.settings = { ...this.settings, - functions: functionCallSettings ?? ChatWrapper.defaultSettings.functions + functions: functionCallSettings ?? ChatWrapper.defaultSettings.functions, + segments: { + ...this.settings.segments, + ...extractedSegmentSettings.settings + } }; + this.keepOnlyLastThought = keepOnlyLastThought ?? extractedSegmentSettings.keepOnlyLastThought ?? false; } /** @@ -565,7 +604,9 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { } = this._transformChatHistory(history, {convertSystemMessagesToUserMessagesFormat, availableFunctions, documentFunctionParams}); const generateMessagesWithEmbeddedTools = (chatHistory: readonly ChatHistoryItem[]) => ({ - messages: chatHistory.map((item): IntermediateOpenAiMessage => { + messages: chatHistory.map((item, index): IntermediateOpenAiMessage => { + const isLastItem = index === chatHistory.length - 1; + if (item.type === "system") return { role: "system", @@ -579,7 +620,13 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { else if (item.type === "model") return { role: "assistant", - content: this.generateModelResponseText(item.response) + content: this.generateModelResponseText( + (!this.keepOnlyLastThought || isLastItem) + ? item.response + : item.response.filter((response) => ( + !isChatModelResponseSegment(response) || response.segmentType !== "thought") + ) + ) }; void (item satisfies never); @@ -648,6 +695,7 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { } as const; const idToContent = new Map(); const modelMessageIds = new Set(); + const lastModelMessageIds = new Set(); const messageIds = new Set(); for (const intermediateMessage of intermediateMessages) { @@ -669,8 +717,11 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { content: id } as OpenAiChatMessage); - if (intermediateMessage.role === "assistant" || intermediateMessage.role === "tool") + if (intermediateMessage.role === "assistant" || intermediateMessage.role === "tool") { modelMessageIds.add(id); + lastModelMessageIds.add(id); + } else if (intermediateMessage.role === "user") + lastModelMessageIds.clear(); } const bosTokenId = idsGenerator.generateId(); @@ -713,6 +764,11 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { "bos_token": bosTokenId, "eos_token": eosTokenId, "eot_token": eotTokenId, + ...( + this.reasoning == null + ? {} + : {"enable_thinking": this.reasoning} + ), ...options }) )); @@ -781,21 +837,59 @@ export class JinjaTemplateChatWrapper extends ChatWrapper { const {splitJinjaParts, stopGenerationJinjaParts} = renderJinjaAndSplitIntoParts(); const messageIdsLeftToProcess = new Set(messageIds); - const contextText = LlamaText( - splitJinjaParts.map((part) => { - if (typeof part === "string") - return new SpecialTokensText(part); // things that are not message content can be tokenized with special tokens - - const message = idToContent.get(part.separator); - - if (message == null) - throw new Error(`Message with id "${part.separator}" not found`); + const thoughSegmentPrefix = getLlamaTextOnlyText(this.settings.segments?.thought?.prefix); + const thoughSegmentSuffix = getLlamaTextOnlyText(this.settings.segments?.thought?.suffix); + let inLastModelResponseSection: boolean | null = ( + thoughSegmentPrefix == null || + thoughSegmentSuffix == null || + this.settings.segments?.thought?.openOnResponseStart !== true + ) + ? null + : false; + const llamaTextContent: Array = []; + for (let i = 0; i < splitJinjaParts.length; i++) { + const part = splitJinjaParts[i]!; + + if (typeof part === "string") { + // things that are not message content can be tokenized with special tokens + llamaTextContent.push(new SpecialTokensText(part)); + continue; + } - messageIdsLeftToProcess.delete(part.separator); + const message = idToContent.get(part.separator); + + if (message == null) + throw new Error(`Message with id "${part.separator}" not found`); + + messageIdsLeftToProcess.delete(part.separator); + + if (inLastModelResponseSection === false && lastModelMessageIds.has(part.separator)) + inLastModelResponseSection = true; + + // remove empty thinking blocks added by the template if the last model response is supposed to always have a thought + // segment rendered by the chat wrapper + if (inLastModelResponseSection === true) { + const lastPart = llamaTextContent.at(-1); + if (lastPart instanceof SpecialTokensText && thoughSegmentPrefix != null && thoughSegmentSuffix != null) { + const thoughPrefixIndex = lastPart.value.indexOf(thoughSegmentPrefix); + const thoughSuffixIndex = thoughPrefixIndex >= 0 + ? lastPart.value.indexOf(thoughSegmentSuffix, thoughPrefixIndex + thoughSegmentPrefix.length) + : -1; + + if (thoughPrefixIndex >= 0 && thoughSuffixIndex >= 0) { + const thoughtText = lastPart.value.slice(thoughPrefixIndex + thoughSegmentPrefix.length, thoughSuffixIndex); + if (thoughtText.trim() === "") + llamaTextContent[llamaTextContent.length - 1] = new SpecialTokensText( + lastPart.value.slice(0, thoughPrefixIndex) + + lastPart.value.slice(thoughSuffixIndex + thoughSegmentSuffix.length) + ); + } + } + } - return message; - }) - ); + llamaTextContent.push(message); + } + const contextText = LlamaText(llamaTextContent); if (messageIdsLeftToProcess.size !== 0) throw new Error("Some input messages are not present in the generated Jinja template output"); @@ -919,6 +1013,24 @@ function getConvertUnsupportedSystemMessagesToUserMessagesTryOptions( return [undefined, convertUnsupportedSystemMessagesToUserMessages.format]; } +function getLlamaTextOnlyText(llamaText: LlamaText | string | undefined): string | undefined { + if (llamaText == null || typeof llamaText === "string") + return llamaText; + + const texts: string[] = []; + + for (const value of llamaText.values) { + if (typeof value === "string") + texts.push(value); + else if (value instanceof SpecialTokensText) + texts.push(value.value); + else + return undefined; + } + + return texts.join(""); +} + const chatHistoriesForSanityTest: ChatHistoryItem[][] = [ [{ type: "system", diff --git a/src/chatWrappers/generic/utils/extractFunctionCallSettingsFromJinjaTemplate.ts b/src/chatWrappers/generic/utils/extractFunctionCallSettingsFromJinjaTemplate.ts index 1d18e631..3fc1e035 100644 --- a/src/chatWrappers/generic/utils/extractFunctionCallSettingsFromJinjaTemplate.ts +++ b/src/chatWrappers/generic/utils/extractFunctionCallSettingsFromJinjaTemplate.ts @@ -426,7 +426,7 @@ export function extractFunctionCallSettingsFromJinjaTemplate({ const callPrefixLength = findCommonEndLength(modelMessage1ToFunc1Name.text, func1ParamsToFunc2Name.text); const callPrefixText = func1ParamsToFunc2Name.text.slice(func1ParamsToFunc2Name.text.length - callPrefixLength); const parallelismCallPrefix = modelMessage1ToFunc1Name.text.slice(0, modelMessage1ToFunc1Name.text.length - callPrefixLength); - + const callSuffixAndParallelismBetweenCallsText = func1ParamsToFunc2Name.text.slice( 0, func1ParamsToFunc2Name.text.length - callPrefixLength diff --git a/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts b/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts index f4c6a949..e0c93059 100644 --- a/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts +++ b/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts @@ -1,11 +1,35 @@ import {ChatWrapperSettings, Tokenizer} from "../../../types.js"; import {LlamaText, SpecialTokensText} from "../../../utils/LlamaText.js"; +import {tryMatrix} from "../../../utils/optionsMatrix.js"; import {removeUndefinedFields} from "../../../utils/removeNullFields.js"; +import {OpenAiChatMessage} from "../../../utils/OpenAIFormat.js"; +import {UniqueIdGenerator} from "./UniqueIdGenerator.js"; -export function extractSegmentSettingsFromTokenizerAndChatTemplate( - chatTemplate: string | undefined, tokenizer?: Tokenizer -): ChatWrapperSettings["segments"] { - function tryMatchPrefixSuffixPair(tryMatchGroups: [prefix: string, suffix: string][]) { +const knownThinkingSegmentControls = new Map([ + ["", ""], // DeepSeek, QwQ + ["", ""], // EXAONE Deep + ["[THINK]", "[/THINK]"], // Mistral + ["<|START_THINKING|>", "<|END_THINKING|>"], // Command R7B + ["<|begin_of_thought|>", "<|end_of_thought|>"] // JoyAI +]); + +export function extractSegmentSettingsFromTokenizerAndChatTemplate({ + chatTemplate, + tokenizer, + renderRawJinjaTemplate, + idsGenerator, + enableReasoning +}: { + chatTemplate: string | undefined, + tokenizer: Tokenizer | undefined, + renderRawJinjaTemplate(params: Record): string, + idsGenerator: UniqueIdGenerator, + enableReasoning: boolean | null +}): { + settings: ChatWrapperSettings["segments"], + keepOnlyLastThought?: boolean +} { + function tryMatchPrefixSuffixPair(tryMatchGroups: Iterable<[prefix: string, suffix: string]>) { if (chatTemplate != null) { for (const [prefix, suffix] of tryMatchGroups) { if ( @@ -74,15 +98,335 @@ export function extractSegmentSettingsFromTokenizerAndChatTemplate( return undefined; } - return removeUndefinedFields({ - thought: tryMatchPrefixSuffixPair([ - ["", ""], // DeepSeek, QwQ - ["", ""], // EXAONE Deep - ["[THINK]", "[/THINK]"], // Mistral - ["<|START_THINKING|>", "<|END_THINKING|>"], // Command R7B - ["<|begin_of_thought|>", "<|end_of_thought|>"] // JoyAI - ]) - }); + function extractThoughtSettingsFromRendering(): ( + { + thoughtSegment: Exclude["thought"] | undefined, + keepPastReasoning: boolean | undefined + } + ) { + if (chatTemplate == null) + return { + thoughtSegment: undefined, + keepPastReasoning: undefined + }; + + const systemMessage = idsGenerator.generateId(); + const userMessage1 = idsGenerator.generateId(); + const userMessage2 = idsGenerator.generateId(); + const modelResponse1 = idsGenerator.generateId(); + const modelResponse2 = idsGenerator.generateId(); + const modelResponse3 = idsGenerator.generateId(); + const modelReasoning2 = idsGenerator.generateId(); + const modelReasoning3 = idsGenerator.generateId(); + + const bosTokenId = idsGenerator.generateId(); + const eosTokenId = idsGenerator.generateId(); + const eotTokenId = idsGenerator.generateId(); + + const renderTemplate = (messages: OpenAiChatMessage[], params?: Record) => tryMatrix({ + skipSystemPrompt: [false, true] + }, ({skipSystemPrompt}) => { + let messageToRender = messages; + if (skipSystemPrompt) + messageToRender = messageToRender.slice(1); + + return renderRawJinjaTemplate({ + messages: messageToRender, + "bos_token": bosTokenId, + "eos_token": eosTokenId, + "eot_token": eotTokenId, + ...params + }); + }); + + const baseMessages: OpenAiChatMessage[] = [{ + role: "system", + content: systemMessage + }, { + role: "user", + content: userMessage1 + }]; + const longBaseMessages: OpenAiChatMessage[] = [...baseMessages, { + role: "assistant", + content: modelResponse1 + }, { + role: "user", + content: userMessage2 + }]; + + const messagesWithModelResponse: OpenAiChatMessage[] = [...baseMessages, { + role: "assistant", + content: modelResponse2 + }]; + const messagesWithModelResponseLongBase: OpenAiChatMessage[] = [...longBaseMessages, { + role: "assistant", + content: modelResponse2 + }]; + + const messagesWithModelReasoning: OpenAiChatMessage[] = [...baseMessages, { + role: "assistant", + content: modelResponse2, + "reasoning_content": modelReasoning2 + }]; + const messagesWithModelReasoningLongBase: OpenAiChatMessage[] = [...longBaseMessages, { + role: "assistant", + content: modelResponse2, + "reasoning_content": modelReasoning2 + }]; + + function extractControls() { + const {responseOnly, withReasoning} = tryMatrix({ + enableThinking: [true, null], + variation: ["simple", "separateReasoning", "nullContent", "reasoningFirst", "reasoningFirstNullMessage"] + }, ({enableThinking, variation}) => { + const thinkingParam = enableThinking === true + ? {"enable_thinking": true} + : {}; + + if (variation === "simple") + return { + responseOnly: renderTemplate(messagesWithModelResponseLongBase, thinkingParam), + withReasoning: { + long: renderTemplate(messagesWithModelReasoningLongBase, thinkingParam), + short: renderTemplate(messagesWithModelReasoning, thinkingParam) + } + }; + else if (variation === "separateReasoning" || variation === "nullContent") + return { + responseOnly: renderTemplate([...messagesWithModelResponseLongBase, { + role: "assistant", + content: "" + }], thinkingParam), + withReasoning: { + long: renderTemplate([...messagesWithModelResponseLongBase, { + role: "assistant", + ...(variation === "nullContent" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning2 + }], thinkingParam), + short: renderTemplate([...messagesWithModelResponse, { + role: "assistant", + ...(variation === "nullContent" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning2 + }], thinkingParam) + } + }; + else if (variation === "reasoningFirst" || variation === "reasoningFirstNullMessage") + return { + responseOnly: renderTemplate(messagesWithModelResponseLongBase, thinkingParam), + withReasoning: { + long: renderTemplate([...longBaseMessages, { + role: "assistant", + ...(variation === "reasoningFirstNullMessage" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning2 + }, { + role: "assistant", + content: modelResponse2 + }], thinkingParam), + short: renderTemplate([...baseMessages, { + role: "assistant", + ...(variation === "reasoningFirstNullMessage" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning2 + }, { + role: "assistant", + content: modelResponse2 + }], thinkingParam) + } + }; + + void (variation satisfies never); + throw new Error(`Unsupported variation: ${variation}`); + }); + + let reasoningSectionStartPrefix: string | undefined = undefined; + let reasoningSectionEndPrefix: string | undefined = undefined; + + if (responseOnly === withReasoning.long) + return undefined; + + const modelResponseIndex = responseOnly.indexOf(modelResponse1); + if (modelResponseIndex < 0) + return undefined; + + const modelResponsePrefix = responseOnly.slice(0, modelResponseIndex); + const withReasoningPrefixContent = withReasoning.short.slice(0, modelResponseIndex); + + if (modelResponsePrefix !== withReasoningPrefixContent) + return undefined; + + const reasoningSectionStartIndex = modelResponseIndex; + const reasoningContentStartIndex = withReasoning.short.indexOf(modelReasoning2, reasoningSectionStartIndex); + if (reasoningContentStartIndex < 0) + return undefined; + + reasoningSectionStartPrefix = withReasoning.short.slice(modelResponseIndex, reasoningContentStartIndex); + + const reasoningContentEndIndex = reasoningContentStartIndex + modelReasoning2.length; + const modelResponseStartIndex = withReasoning.short.indexOf(modelResponse2, reasoningContentEndIndex); + if (modelResponseStartIndex < 0) + return undefined; + + reasoningSectionEndPrefix = withReasoning.short.slice(reasoningContentEndIndex, modelResponseStartIndex); + + return { + prefix: reasoningSectionStartPrefix, + suffix: reasoningSectionEndPrefix + }; + } + + function shouldKeepPastThinking() { + const renderedOutput = tryMatrix({ + enableThinking: [true, null], + variation: ["simple", "reasoningFirst", "reasoningFirstNullMessage"] + }, ({enableThinking, variation}) => { + const thinkingParam = enableThinking === true + ? {"enable_thinking": true} + : {}; + + if (variation === "simple") + return renderTemplate([...messagesWithModelReasoning, { + role: "user", + content: userMessage2 + }, { + role: "assistant", + content: modelResponse3, + "reasoning_content": modelReasoning3 + }], thinkingParam); + else if (variation === "reasoningFirst" || variation === "reasoningFirstNullMessage") + return renderTemplate([...baseMessages, { + role: "assistant", + ...(variation === "reasoningFirstNullMessage" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning2 + }, { + role: "assistant", + content: modelResponse2 + }, { + role: "user", + content: userMessage2 + }, { + role: "assistant", + ...(variation === "reasoningFirstNullMessage" ? {} : { + content: "" + }), + "reasoning_content": modelReasoning3 + }, { + role: "assistant", + content: modelResponse3 + }], thinkingParam); + + void (variation satisfies never); + throw new Error(`Unsupported variation: ${variation}`); + }); + + return renderedOutput.includes(modelReasoning2) && renderedOutput.includes(modelReasoning3); + } + + function shouldOpenThinkingSegmentOnModelResponseStart(reasoningSectionPrefix: string, reasoningSectionSuffix: string) { + if (!enableReasoning) + return false; + + const {responseOnly, withGenerationPrompt} = tryMatrix({ + enableThinking: enableReasoning + ? [true, null] + : [null] + }, ({enableThinking}) => { + const thinkingParam = (enableThinking === true || enableThinking === false) + ? {"enable_thinking": enableThinking} + : {}; + + return { + responseOnly: renderTemplate(baseMessages, thinkingParam), + withGenerationPrompt: renderTemplate(baseMessages, { + ...thinkingParam, + "add_generation_prompt": true + }) + }; + }); + + if (responseOnly === withGenerationPrompt) + return false; + + const userMessage1Index = responseOnly.indexOf(userMessage1); + if (userMessage1Index < 0) + return false; + + const withReasoningUserMessage1Index = withGenerationPrompt.indexOf(userMessage1); + if (withReasoningUserMessage1Index < 0) + return false; + + return ( + responseOnly.indexOf(reasoningSectionPrefix, userMessage1Index) < 0 && + withGenerationPrompt.indexOf(reasoningSectionPrefix, withReasoningUserMessage1Index) >= 0 && + withGenerationPrompt.indexOf(reasoningSectionSuffix, withReasoningUserMessage1Index) < 0 + ); + } + + let controls: ReturnType; + let keepPastReasoning: boolean | undefined = undefined; + let openOnResponseStart: boolean | undefined = undefined; + try { + controls = extractControls(); + + if (controls == null || controls.prefix.trim() === "") + return { + thoughtSegment: undefined, + keepPastReasoning: undefined + }; + } catch (err) { + return { + thoughtSegment: undefined, + keepPastReasoning: undefined + }; + } + + try { + keepPastReasoning = shouldKeepPastThinking(); + } catch (err) { + // do nothing + } + + try { + openOnResponseStart = controls != null && shouldOpenThinkingSegmentOnModelResponseStart(controls.prefix, controls.suffix); + } catch (err) { + // do nothing + } + + const thoughtSuffix = controls.suffix.trim() === "" + ? ( + knownThinkingSegmentControls.get(controls.prefix) ?? + knownThinkingSegmentControls.get(controls.prefix.trim()) + ) + : controls.suffix; + + return { + thoughtSegment: { + prefix: LlamaText(new SpecialTokensText(controls.prefix)), + suffix: thoughtSuffix != null + ? LlamaText(new SpecialTokensText(thoughtSuffix)) + : undefined, + openOnResponseStart + }, + keepPastReasoning + }; + } + + const extractedFromRendering = extractThoughtSettingsFromRendering(); + + return { + settings: removeUndefinedFields({ + thought: extractedFromRendering.thoughtSegment ?? tryMatchPrefixSuffixPair(knownThinkingSegmentControls) + }), + keepOnlyLastThought: !extractedFromRendering.keepPastReasoning + }; } function hasAll(text: string, matches: string[]) { diff --git a/src/evaluator/LlamaChat/LlamaChat.ts b/src/evaluator/LlamaChat/LlamaChat.ts index affbb324..285aa32d 100644 --- a/src/evaluator/LlamaChat/LlamaChat.ts +++ b/src/evaluator/LlamaChat/LlamaChat.ts @@ -691,6 +691,7 @@ export class LlamaChat { let tookInitialCheckpoint = false; generateResponseState.ensureLastHistoryItemIsModel(); + generateResponseState.openThoughtSegmentOnModelResponseStartIfNeeded(); generateResponseState.ensureReopenedThoughtSegmentAfterFunctionCallsIfNeeded(); const loadContextWindow = async (avoidReloadingHistory: boolean = false) => { @@ -2018,6 +2019,45 @@ class GenerateResponseState 1) + return; + else if (lastModelResponseItem.response.length === 1 && lastModelResponseItem.response[0] !== "") + return; + + const currentResponseSegmentsStack = SegmentHandler.getStackFromModelResponse(lastModelResponseItem.response); + if (currentResponseSegmentsStack.includes("thought")) + return; + + if (this.abortOnNonText) + // we won't force-open a though segment if we are aborting on non-text, + // as it would never allow a textual generation even if the model would choose it otherwise + return; + else { + this.resolvedHistory[this.resolvedHistory.length - 1] = { + ...lastModelResponseItem, + response: [ + ...lastModelResponseItem.response, + { + type: "segment", + segmentType: "thought", + text: "", + ended: false, + startTime: new Date().toISOString() + } + ] + }; + this.segmentHandler.openSegment("thought"); + } + } + public ensureReopenedThoughtSegmentAfterFunctionCallsIfNeeded() { if (this.chatWrapper.settings.segments?.thought?.reopenAfterFunctionCalls !== true) return; diff --git a/src/types.ts b/src/types.ts index 7e3a4d9d..7034a5b6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -112,6 +112,7 @@ export type ChatWrapperSettings = { /** Chain of Thought text segment */ readonly thought?: ChatWrapperSettingsSegment & { + openOnResponseStart?: boolean, reopenAfterFunctionCalls?: boolean }, diff --git a/src/utils/OpenAIFormat.ts b/src/utils/OpenAIFormat.ts index c1a839ab..c3d7426c 100644 --- a/src/utils/OpenAIFormat.ts +++ b/src/utils/OpenAIFormat.ts @@ -608,7 +608,8 @@ export type OpenAiChatAssistantMessage = { name: string, arguments: string } - }> + }>, + "reasoning_content"?: string }; export type OpenAiChatToolMessage = { role: "tool", From d5900d1bfa746b58996215e3830d2a80a0c24ff9 Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 4 Aug 2026 07:04:16 +0200 Subject: [PATCH 5/8] chore: add missing GGUF metadata types --- src/gguf/types/GgufMetadataTypes.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/gguf/types/GgufMetadataTypes.ts b/src/gguf/types/GgufMetadataTypes.ts index 3d53e764..0b887110 100644 --- a/src/gguf/types/GgufMetadataTypes.ts +++ b/src/gguf/types/GgufMetadataTypes.ts @@ -238,6 +238,11 @@ export type GgufMetadataGeneral Date: Tue, 4 Aug 2026 07:16:21 +0200 Subject: [PATCH 6/8] test: segment syntax extraction --- .../generic/JinjaTemplateChatWrapper.test.ts | 210 +++++++++++++++++- 1 file changed, 203 insertions(+), 7 deletions(-) diff --git a/test/standalone/chatWrappers/generic/JinjaTemplateChatWrapper.test.ts b/test/standalone/chatWrappers/generic/JinjaTemplateChatWrapper.test.ts index 0658e6af..83d5b88a 100644 --- a/test/standalone/chatWrappers/generic/JinjaTemplateChatWrapper.test.ts +++ b/test/standalone/chatWrappers/generic/JinjaTemplateChatWrapper.test.ts @@ -2,7 +2,7 @@ import {describe, expect, test} from "vitest"; import {Template} from "@huggingface/jinja"; import {ChatHistoryItem, ChatModelFunctions, JinjaTemplateChatWrapper} from "../../../../src/index.js"; import {defaultChatSystemPrompt} from "../../../../src/config.js"; -import {LlamaText} from "../../../../src/utils/LlamaText.js"; +import {LlamaText, SpecialTokensText} from "../../../../src/utils/LlamaText.js"; import {fromChatHistoryToIntermediateOpenAiMessages, fromIntermediateToCompleteOpenAiMessages} from "../../../../src/utils/OpenAIFormat.js"; import {removeUndefinedFields} from "../../../../src/utils/removeNullFields.js"; @@ -200,6 +200,167 @@ const llama3_1ChatJinjaTemplate = ` {%- endif %} `.slice(1, -1); +const qwen3_6Template = ` +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- set num_sys = 0 %} +{%- set merged_system = '' %} +{%- if messages[0].role == 'system' or messages[0].role == 'developer' %} + {%- set first = render_content(messages[0].content, false, true)|trim %} + {%- if messages|length > 1 and (messages[1].role == 'system' or messages[1].role == 'developer') %} + {%- set second = render_content(messages[1].content, false, true)|trim %} + {%- set merged_system = first + '\n' + second %} + {%- set num_sys = 2 %} + {%- else %} + {%- set merged_system = first %} + {%- set num_sys = 1 %} + {%- endif %} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if merged_system %} + {{- '\n\n' + merged_system }} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if merged_system %} + {{- '<|im_start|>system\n' + merged_system + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if loop.index0 >= num_sys and message.role != "system" and message.role != "developer" %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {%- if (preserve_thinking is defined and preserve_thinking is true) or (loop.index0 > ns.last_query_index) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is mapping %} + {%- for args_name in tool_call.arguments %} + {%- set args_value = tool_call.arguments[args_name] %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if enable_thinking is defined and enable_thinking is false %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n' }} + {%- endif %} +{%- endif %} +{#- Unsloth fixes - developer role, tool calling #} +`.slice(1, -1); + describe("JinjaTemplateChatWrapper", () => { const template1 = "{{ bos_token }}" + @@ -804,10 +965,10 @@ describe("JinjaTemplateChatWrapper", () => { function func8(params: { // The main message message: string, - + // The feeling feeling: "good" | "bad", - + // The number of words. // For example, 6 words: number @@ -883,10 +1044,10 @@ describe("JinjaTemplateChatWrapper", () => { function func8(params: { // The main message message: string, - + // The feeling feeling: "good" | "bad", - + // The number of words. // For example, 6 words: number @@ -961,10 +1122,10 @@ describe("JinjaTemplateChatWrapper", () => { function func8(params: { // The main message message: string, - + // The feeling feeling: "good" | "bad", - + // The number of words. // For example, 6 words: number @@ -1534,6 +1695,41 @@ describe("JinjaTemplateChatWrapper", () => { }); }); + describe("thought segment extraction", () => { + test("Qwen 3.6 resolves thinking segment settings properly", {timeout: 1000 * 60 * 60 * 2}, async () => { + const chatWrapper = new JinjaTemplateChatWrapper({ + template: qwen3_6Template + }); + expect(chatWrapper.keepOnlyLastThought).to.be.eql(true); + expect(chatWrapper.settings.segments?.thought).to.eql({ + openOnResponseStart: true, + prefix: LlamaText(new SpecialTokensText("\n")), + suffix: LlamaText(new SpecialTokensText("\n\n\n")) + }); + }); + + test("basic template resolves thinking segment settings properly", {timeout: 1000 * 60 * 60 * 2}, async () => { + const template = ( + "{%- for m in messages %}" + + "{{- '<|im_start|>' + m.role + '\\n' }}" + + "{%- if m.reasoning_content %}" + + "{{- '\\n' + m.reasoning_content + '\\n\\n' }}" + + "{%- endif %}" + + "{{- m.content + '<|im_end|>\\n' }}" + + "{%- endfor %}" + + "{%- if add_generation_prompt %}" + + "{{- '<|im_start|>assistant\\n\\n' }}" + + "{%- endif %}" + ); + const chatWrapper = new JinjaTemplateChatWrapper({template}); + expect(chatWrapper.settings.segments?.thought).to.eql({ + openOnResponseStart: true, + prefix: LlamaText(new SpecialTokensText("\n")), + suffix: LlamaText(new SpecialTokensText("\n\n")) + }); + }); + }); + test("Fails when messages are not present in the render output", () => { try { new JinjaTemplateChatWrapper({ From e8d88adeec583f74cb6f07db5149b04cafb108af Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 4 Aug 2026 07:26:17 +0200 Subject: [PATCH 7/8] fix: only discard empty reasoning block sections for not indicating a thinking segment opening in a generation prompt --- ...entSettingsFromTokenizerAndChatTemplate.ts | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts b/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts index e0c93059..3e564522 100644 --- a/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts +++ b/src/chatWrappers/generic/utils/extractSegmentSettingsFromTokenizerAndChatTemplate.ts @@ -363,11 +363,25 @@ export function extractSegmentSettingsFromTokenizerAndChatTemplate({ if (withReasoningUserMessage1Index < 0) return false; - return ( - responseOnly.indexOf(reasoningSectionPrefix, userMessage1Index) < 0 && - withGenerationPrompt.indexOf(reasoningSectionPrefix, withReasoningUserMessage1Index) >= 0 && - withGenerationPrompt.indexOf(reasoningSectionSuffix, withReasoningUserMessage1Index) < 0 - ); + if (responseOnly.indexOf(reasoningSectionPrefix, userMessage1Index) >= 0) + return false; + + const reasoningSectionPrefixIndex = withGenerationPrompt.indexOf(reasoningSectionPrefix, withReasoningUserMessage1Index); + if (reasoningSectionPrefixIndex < 0) + return false; + + const reasoningSectionSuffixIndex = withGenerationPrompt.indexOf(reasoningSectionSuffix, reasoningSectionPrefixIndex); + if (reasoningSectionSuffixIndex >= 0) { + const reasoningSectionContent = withGenerationPrompt.slice( + reasoningSectionPrefixIndex + reasoningSectionPrefix.length, + reasoningSectionSuffixIndex + ); + + if (reasoningSectionContent.trim() === "") + return false; + } + + return true; } let controls: ReturnType; From 20b17ea2e6030526909e4ab36007d880b939051b Mon Sep 17 00:00:00 2001 From: "Gilad S." Date: Tue, 4 Aug 2026 08:05:07 +0200 Subject: [PATCH 8/8] fix: redundant whitespace --- src/utils/getTypeScriptTypeStringForGbnfJsonSchema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/getTypeScriptTypeStringForGbnfJsonSchema.ts b/src/utils/getTypeScriptTypeStringForGbnfJsonSchema.ts index 5cb0c0c5..c4b6fff3 100644 --- a/src/utils/getTypeScriptTypeStringForGbnfJsonSchema.ts +++ b/src/utils/getTypeScriptTypeStringForGbnfJsonSchema.ts @@ -143,7 +143,7 @@ function _getTypeScriptTypeStringForGbnfJsonSchema( "\n ", valueTypes .map((value) => value.split("\n").join("\n ")) - .join(",\n ") + .join(",\n") .trimStart(), "\n" ].join("")