From c22dc0d7fead22da787873a80f144cefcc67b159 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:29:26 +0200 Subject: [PATCH 01/43] build: add dependency and update CMake configuration for LLM extension --- packages/react-native-executorch/android/CMakeLists.txt | 2 ++ packages/react-native-executorch/package.json | 3 +++ yarn.lock | 8 ++++++++ 3 files changed, 13 insertions(+) diff --git a/packages/react-native-executorch/android/CMakeLists.txt b/packages/react-native-executorch/android/CMakeLists.txt index 0212e869f8..3030946c35 100644 --- a/packages/react-native-executorch/android/CMakeLists.txt +++ b/packages/react-native-executorch/android/CMakeLists.txt @@ -27,6 +27,7 @@ file(GLOB CORE_SOURCES ${CPP_DIR}/core/*.cpp) file(GLOB MATH_SOURCES ${CPP_DIR}/extensions/math/*.cpp) file(GLOB NLP_SOURCES ${CPP_DIR}/extensions/nlp/*.cpp) file(GLOB SPEECH_SOURCES ${CPP_DIR}/extensions/speech/*.cpp) +file(GLOB LLM_SOURCES ${CPP_DIR}/extensions/llm/*.cpp) file(GLOB OPENCV_SOURCES ${CPP_DIR}/extensions/cv/*.cpp) set(RNE_SOURCES @@ -35,6 +36,7 @@ set(RNE_SOURCES ${MATH_SOURCES} ${NLP_SOURCES} ${SPEECH_SOURCES} + ${LLM_SOURCES} cpp-adapter.cpp ) diff --git a/packages/react-native-executorch/package.json b/packages/react-native-executorch/package.json index 2eb9dc91cb..c15604ccea 100644 --- a/packages/react-native-executorch/package.json +++ b/packages/react-native-executorch/package.json @@ -83,6 +83,9 @@ "publishConfig": { "registry": "https://registry.npmjs.org/" }, + "dependencies": { + "@huggingface/jinja": "^0.5.9" + }, "peerDependencies": { "react": "*", "react-native": "*", diff --git a/yarn.lock b/yarn.lock index 875b8db572..bab478be6d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3130,6 +3130,13 @@ __metadata: languageName: node linkType: hard +"@huggingface/jinja@npm:^0.5.9": + version: 0.5.9 + resolution: "@huggingface/jinja@npm:0.5.9" + checksum: 10/8147f05df29b609ebb923f9e294f485897c5b5b92cddb1e41d8469b2992def242d9b536b8e9ca450a24a88fae0df6764986ab1248033f34edd4f5eb9dc2c4d78 + languageName: node + linkType: hard + "@humanwhocodes/config-array@npm:^0.13.0": version: 0.13.0 resolution: "@humanwhocodes/config-array@npm:0.13.0" @@ -12398,6 +12405,7 @@ __metadata: resolution: "react-native-executorch@workspace:packages/react-native-executorch" dependencies: "@babel/core": "npm:^7.25.1" + "@huggingface/jinja": "npm:^0.5.9" "@react-native/babel-preset": "npm:0.83.6" "@react-native/metro-config": "npm:^0.86.0" "@types/react": "npm:^19.1.12" From 008107ba76b7ccff109466e5b0b6f69e4d682e22 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:31:41 +0200 Subject: [PATCH 02/43] feat: implement native JSI wrapper for TextLLMRunner --- .../cpp/RnExecutorch.cpp | 2 + .../cpp/extensions/llm/install.cpp | 14 ++ .../cpp/extensions/llm/install.h | 7 + .../cpp/extensions/llm/llm_runner.cpp | 234 ++++++++++++++++++ .../cpp/extensions/llm/llm_runner.h | 29 +++ 5 files changed, 286 insertions(+) create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/install.h create mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp create mode 100644 packages/react-native-executorch/cpp/extensions/llm/llm_runner.h diff --git a/packages/react-native-executorch/cpp/RnExecutorch.cpp b/packages/react-native-executorch/cpp/RnExecutorch.cpp index df1bcb9d4d..98fb8c97ff 100644 --- a/packages/react-native-executorch/cpp/RnExecutorch.cpp +++ b/packages/react-native-executorch/cpp/RnExecutorch.cpp @@ -1,6 +1,7 @@ #include "RnExecutorch.h" #include "core/install.h" +#include "extensions/llm/install.h" #include "extensions/math/install.h" #include "extensions/nlp/install.h" #include "extensions/speech/install.h" @@ -22,6 +23,7 @@ void install(jsi::Runtime &jsiRuntime) { rnexecutorch::extensions::math::install(jsiRuntime, module); rnexecutorch::extensions::nlp::install(jsiRuntime, module); rnexecutorch::extensions::speech::install(jsiRuntime, module); + rnexecutorch::extensions::llm::install(jsiRuntime, module); jsiRuntime.global().setProperty(jsiRuntime, "__rnexecutorch_jsi__", std::move(module)); } diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.cpp b/packages/react-native-executorch/cpp/extensions/llm/install.cpp new file mode 100644 index 0000000000..f30a2bd80a --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.cpp @@ -0,0 +1,14 @@ +#include "install.h" +#include "llm_runner.h" + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; + +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module) { + jsi::Object llmModule = jsi::Object(rt); + + install_createLLMRunner(rt, llmModule); + + module.setProperty(rt, "llm", llmModule); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/install.h b/packages/react-native-executorch/cpp/extensions/llm/install.h new file mode 100644 index 0000000000..89d3de5e6b --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/install.h @@ -0,0 +1,7 @@ +#pragma once + +#include + +namespace rnexecutorch::extensions::llm { +void install(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp new file mode 100644 index 0000000000..e9e07c5775 --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -0,0 +1,234 @@ +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include "llm_runner.h" +#include "core/conversions.h" +#include +#include +#include +#include + +namespace rnexecutorch::extensions::llm { +namespace jsi = facebook::jsi; +namespace conversions = rnexecutorch::core::conversions; + +namespace { +jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { + jsi::Object obj(rt); + obj.setProperty(rt, "numPromptTokens", static_cast(stats.num_prompt_tokens)); + obj.setProperty(rt, "numGeneratedTokens", static_cast(stats.num_generated_tokens)); + obj.setProperty(rt, "firstTokenMs", static_cast(stats.first_token_ms)); + obj.setProperty(rt, "inferenceStartMs", static_cast(stats.inference_start_ms)); + obj.setProperty(rt, "inferenceEndMs", static_cast(stats.inference_end_ms)); + obj.setProperty(rt, "modelLoadStartMs", static_cast(stats.model_load_start_ms)); + obj.setProperty(rt, "modelLoadEndMs", static_cast(stats.model_load_end_ms)); + return obj; +} +} // namespace + +LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, + const std::string &tokenizerPath) + : modelPath_(modelPath), + tokenizerPath_(tokenizerPath) { + auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); + if (!tokenizer) { + throw std::runtime_error("LLMRunner: Failed to load runner tokenizer at path: " + tokenizerPath); + } + + runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); + if (!runner_) { + throw std::runtime_error("LLMRunner: Failed to create text llm runner"); + } + + auto loadError = runner_->load(); + if (loadError != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(loadError); + throw std::runtime_error("LLMRunner: Failed to load model: " + errorMsg); + } +} + +jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { + auto nameStr = name.utf8(rt); + + if (nameStr == "modelPath") { + return jsi::String::createFromUtf8(rt, modelPath_); + } + + if (nameStr == "tokenizerPath") { + return jsi::String::createFromUtf8(rt, tokenizerPath_); + } + + if (nameStr == "generate") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw jsi::JSError(rt, "LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); + } + + std::string prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); + + executorch::extension::llm::GenerationConfig config; + if (count > 1 && !args[1].isUndefined() && !args[1].isNull()) { + auto configObj = conversions::asType(rt, "LLMRunner.generate: config", args[1]); + if (auto echoOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "echo")) { + config.echo = *echoOpt; + } + if (auto ignoreEosOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "ignoreEos")) { + config.ignore_eos = *ignoreEosOpt; + } + if (auto maxNewTokensOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "maxNewTokens")) { + config.max_new_tokens = *maxNewTokensOpt; + } + if (auto tempOpt = conversions::getOptionalProperty(rt, "LLMRunner.generate: config", configObj, "temperature")) { + config.temperature = *tempOpt; + } + } + + std::function tokenCallback; + if (count > 2 && !args[2].isUndefined() && !args[2].isNull()) { + auto tokenFn = std::make_shared(conversions::asType(rt, "LLMRunner.generate: onToken", args[2])); + tokenCallback = [&rt, tokenFn](const std::string &token) { + tokenFn->call(rt, jsi::String::createFromUtf8(rt, token)); + }; + } + + auto finalStats = std::make_shared(); + auto statsCallback = [finalStats](const executorch::extension::llm::Stats &stats) { + finalStats->num_prompt_tokens = stats.num_prompt_tokens; + finalStats->num_generated_tokens = stats.num_generated_tokens; + finalStats->first_token_ms = stats.first_token_ms; + finalStats->inference_start_ms = stats.inference_start_ms; + finalStats->inference_end_ms = stats.inference_end_ms; + finalStats->model_load_start_ms = stats.model_load_start_ms; + finalStats->model_load_end_ms = stats.model_load_end_ms; + finalStats->aggregate_sampling_time_ms = stats.aggregate_sampling_time_ms; + }; + + // Hold the lock for the whole call so dispose() cannot free the + // runner mid-generation (dispose blocks on this lock until we + // return). try_to_lock: only one prefill/generate may run at a + // time, so fail fast instead of queuing. stop() is lock-free and + // can still interrupt us. + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw jsi::JSError(rt, "LLMRunner.generate: Runner is already in use"); + } + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.generate: Runner has been disposed"); + } + auto error = self->runner_->generate(prompt, config, tokenCallback, statsCallback); + + if (error != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(error); + throw jsi::JSError(rt, "LLMRunner.generate: Failed to generate: " + errorMsg); + } + + return statsToJSI(rt, *finalStats); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, fnBody); + } + + if (nameStr == "prefill") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count < 1) { + throw jsi::JSError(rt, "LLMRunner.prefill: Usage: prefill(prompt)"); + } + + std::string prompt = conversions::asType(rt, "LLMRunner.prefill: prompt", args[0]); + + // Lock held for the whole call, same as generate(). + std::unique_lock lock(self->mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + throw jsi::JSError(rt, "LLMRunner.prefill: Runner is already in use"); + } + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.prefill: Runner has been disposed"); + } + auto result = self->runner_->prefill({executorch::extension::llm::make_text_input(prompt)}); + if (result.error() != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(result.error()); + throw jsi::JSError(rt, "LLMRunner.prefill: Failed: " + errorMsg); + } + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, fnBody); + } + + if (nameStr == "stop") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t /*count*/) -> jsi::Value { + // Intentionally no mutex here: stop() is designed to be called + // concurrently to interrupt an in-progress generate(). Taking the + // lock would block until generate() finishes, defeating the point. + // runner_ is only cleared by dispose() on this same (JS) thread, + // so reading it lock-free here is safe. + if (!self->runner_) { + throw jsi::JSError(rt, "LLMRunner.stop: Runner has been disposed"); + } + self->runner_->stop(); + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, fnBody); + } + + if (nameStr == "dispose") { + auto self = shared_from_this(); + auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { + if (count != 0) { + throw jsi::JSError(rt, "dispose: Usage: dispose()"); + } + + // Signal stop before locking so any in-progress generate() exits + // quickly; we then block on the lock until it returns and clear + // the runner, which frees the model. Idempotent: a second + // dispose() finds a null runner_ and is a no-op. + if (self->runner_) { + self->runner_->stop(); + } + + std::unique_lock lock(self->mutex_); + self->runner_ = nullptr; + + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); + } + + return jsi::Value::undefined(); +} + +std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime &rt) { + std::vector properties; + properties.push_back(jsi::PropNameID::forAscii(rt, "modelPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "tokenizerPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "prefill")); + properties.push_back(jsi::PropNameID::forAscii(rt, "generate")); + properties.push_back(jsi::PropNameID::forAscii(rt, "stop")); + properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); + return properties; +} + +void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { + const auto *name = "createLLMRunner"; + auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { + if (count != 2) { + throw jsi::JSError(rt, "createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath)"); + } + + auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); + auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); + + try { + auto runnerInstance = std::make_shared(modelPath, tokenizerPath); + return jsi::Object::createFromHostObject(rt, runnerInstance); + } catch (const std::exception &e) { + throw jsi::JSError(rt, std::format("createLLMRunner: {}", e.what())); + } + }; + + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, fnBody)); +} +} // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h new file mode 100644 index 0000000000..b5205d02bf --- /dev/null +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -0,0 +1,29 @@ +#pragma once + +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include +#include +#include +#include +#include + +namespace rnexecutorch::extensions::llm { +class LLMRunnerHostObject : public facebook::jsi::HostObject, public std::enable_shared_from_this { +public: + LLMRunnerHostObject(const std::string &modelPath, const std::string &tokenizerPath); + + facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; + std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; + +private: + std::unique_ptr runner_; + std::mutex mutex_; + std::string modelPath_; + std::string tokenizerPath_; +}; + +void install_createLLMRunner(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); +} // namespace rnexecutorch::extensions::llm From e4c1ab6ef4dff51fbb4f4e2ad3c6ce8811fd59d7 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 13 Jul 2026 14:38:59 +0200 Subject: [PATCH 03/43] feat: implement TypeScript wrapper and hook for LLM Chat Session --- .../src/extensions/llm/index.ts | 3 + .../src/extensions/llm/jinja.ts | 37 ++ .../src/extensions/llm/llmRunner.ts | 73 ++++ .../extensions/llm/tasks/llmChatSession.ts | 162 +++++++++ .../src/extensions/llm/tokenizerConfig.ts | 39 +++ .../src/hooks/useLLMChatSession.ts | 111 ++++++ packages/react-native-executorch/src/index.ts | 3 + .../react-native-executorch/src/models.ts | 320 +++++++++++++++++- 8 files changed, 743 insertions(+), 5 deletions(-) create mode 100644 packages/react-native-executorch/src/extensions/llm/index.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/jinja.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/llmRunner.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts create mode 100644 packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts create mode 100644 packages/react-native-executorch/src/hooks/useLLMChatSession.ts diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts new file mode 100644 index 0000000000..5d5ce8f45e --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -0,0 +1,3 @@ +export * from './llmRunner'; +export * from './jinja'; +export * from './tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/jinja.ts b/packages/react-native-executorch/src/extensions/llm/jinja.ts new file mode 100644 index 0000000000..615ccf3f4c --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/jinja.ts @@ -0,0 +1,37 @@ +import { Template } from '@huggingface/jinja'; + +import type { ChatFormatter } from './tasks/llmChatSession'; + +/** Configuration options for the Jinja chat formatter. */ +export type JinjaFormatterOptions = { + readonly bosToken?: string; + readonly extraContext?: Record; +}; + +/** + * Creates a chat formatter function that renders messages using a Jinja template. + * @param chatTemplate The Jinja template string (e.g. from tokenizer_config.json). + * @param options Jinja formatter options. + * @returns A ChatFormatter function. + */ +export function createJinjaChatFormatter( + chatTemplate: string, + options: JinjaFormatterOptions = {} +): ChatFormatter { + const { bosToken = '', extraContext } = options; + const template = new Template(chatTemplate); + + return (message, { isFirst }) => { + const isGenerationPrompt = message.role === 'assistant' && message.content === ''; + return template.render({ + // Only the first prefill of a conversation should carry the BOS token; + // later turns append to the model's existing KV cache. + // eslint-disable-next-line camelcase + bos_token: isFirst ? bosToken : '', + // eslint-disable-next-line camelcase + add_generation_prompt: isGenerationPrompt, + messages: isGenerationPrompt ? [] : [{ role: message.role, content: message.content }], + ...extraContext, + }); + }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts new file mode 100644 index 0000000000..b8a9d68bf5 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -0,0 +1,73 @@ +import { rnexecutorchJsi } from '../../native/bridge'; + +declare const llmRunnerBrand: unique symbol; + +/** Configuration options for LLM text generation. */ +export type GenerationConfig = { + readonly echo?: boolean; + readonly ignoreEos?: boolean; + readonly maxNewTokens?: number; + readonly temperature?: number; +}; + +/** Execution and performance statistics for a generation call. */ +export type GenerationStats = { + readonly numPromptTokens: number; + readonly numGeneratedTokens: number; + readonly firstTokenMs: number; + readonly inferenceStartMs: number; + readonly inferenceEndMs: number; + readonly modelLoadStartMs: number; + readonly modelLoadEndMs: number; +}; + +/** Handle to a native ExecuTorch text LLM runner. */ +export type LLMRunner = { + /** Path to the local model file. */ + readonly modelPath: string; + + /** Path to the local tokenizer configuration file. */ + readonly tokenizerPath: string; + + /** Disposes the native LLM runner and releases the loaded model memory. */ + dispose(): void; + + /** + * Prefills the runner with a prompt to build up the KV cache. + * @param prompt The prefill text prompt. + */ + prefill(prompt: string): void; + + /** Interrupts and stops any active generation call on this runner. */ + stop(): void; + + /** + * Generates text continuation from a prompt. + * @param prompt The text prompt to generate continuation for. + * @param config Generation configuration options. + * @param onToken Callback function triggered whenever a new token is generated. + * @returns Generation performance statistics. + */ + generate( + prompt: string, + config?: GenerationConfig, + onToken?: (token: string) => void + ): GenerationStats; + + /** + * Prevents plain JS objects from being cast as LLMRunners. + * @internal + */ + readonly [llmRunnerBrand]: never; +}; + +/** + * Creates a native ExecuTorch Text LLM runner instance. + * @param modelPath Path to the local .pte model file. + * @param tokenizerPath Path to the local tokenizer configuration file (e.g. tokenizer.json). + * @returns A native LLMRunner instance. + */ +export function createLLMRunner(modelPath: string, tokenizerPath: string): LLMRunner { + 'worklet'; + return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath) as LLMRunner; +} diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts new file mode 100644 index 0000000000..b32c1068e6 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -0,0 +1,162 @@ +import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; + +import { wrapAsync } from '../../../core/runtime'; +import { + createLLMRunner, + type LLMRunner, + type GenerationConfig, + type GenerationStats, +} from '../llmRunner'; +import { createJinjaChatFormatter } from '../jinja'; +import type { TokenizerChatConfig } from '../tokenizerConfig'; + +export type { GenerationConfig, GenerationStats }; + +/** Message interface for chat history and inputs. */ +export type ChatMessage = { + readonly role: 'system' | 'user' | 'assistant'; + readonly content: string; +}; + +/** Interface for converting a ChatMessage history turn into raw prompt text. */ +export type ChatFormatter = ( + message: ChatMessage, + options: { readonly isFirst: boolean } +) => string; + +/** Model path configuration for LLM chat. */ +export type LLMModel = { + readonly modelPath: string; + readonly tokenizerPath: string; + readonly tokenizerConfigPath: string; +}; + +/** Custom generation and state options for an LLM chat session. */ +export type LLMChatSessionOptions = { + readonly initialMessages?: readonly ChatMessage[]; + readonly generationConfig?: GenerationConfig; + readonly stopTokens?: readonly string[]; +}; + +/** Config package passed to instantiate an LLM chat session. */ +export type LLMChatSessionConfig = { + readonly model: Omit & { tokenizerConfig: TokenizerChatConfig }; + readonly options?: LLMChatSessionOptions; +}; + +/** Return wrapper holding generated response text and performance stats. */ +export type GenerationResult = { + readonly response: string; + readonly stats: GenerationStats; +}; + +/** Orchestrator interface for active LLM chat sessions. */ +export type LLMChatSession = { + dispose(): void; + sendMessage( + message: string, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise; + getHistory(): readonly ChatMessage[]; + stop(): void; +}; + +type SessionState = { + history: ChatMessage[]; +}; + +function generateChatTurn( + nativeRunner: LLMRunner, + prompt: string, + options: { + readonly genConfig: GenerationConfig; + readonly stopTokens: readonly string[]; + readonly onToken?: (token: string) => void; + } +): GenerationResult { + 'worklet'; + const { genConfig, stopTokens, onToken } = options; + + let response = ''; + + const callback = (token: string) => { + if (stopTokens.includes(token)) return; + response += token; + if (onToken) scheduleOnRN(onToken, token); + }; + + const stats = nativeRunner.generate(prompt, genConfig, callback); + + return { response, stats }; +} + +/** + * Instantiates an LLM chat session using background thread execution. + * @param config Chat session configuration and settings. + * @param runtime The worklet runtime thread to run native generation on. + * @returns A Promise resolving to an LLMChatSession instance. + */ +export async function createLLMChatSession( + config: LLMChatSessionConfig, + runtime?: WorkletRuntime +): Promise { + const { model, options } = config; + const { modelPath, tokenizerPath, tokenizerConfig } = model; + + const initialMessages = options?.initialMessages ?? []; + const defaultGenerationConfig = options?.generationConfig; + + const { chatTemplate, bosToken, eosToken } = tokenizerConfig; + + const format = createJinjaChatFormatter(chatTemplate, { bosToken }); + const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; + + const state: SessionState = { history: [] }; + const nativeRunner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath); + const prefill = wrapAsync(nativeRunner.prefill, runtime); + + for (const msg of initialMessages) { + const fmtMsg = format(msg, { isFirst: state.history.length === 0 }); + if (fmtMsg.length > 0) { + await prefill(fmtMsg); + } + state.history.push(msg); + } + + const stop = () => nativeRunner.stop(); + const dispose = () => nativeRunner.dispose(); + const runGeneration = wrapAsync(generateChatTurn, runtime); + + const sendMessage = async ( + message: string, + onToken?: (token: string) => void, + genConfig?: GenerationConfig + ): Promise => { + const userMsg: ChatMessage = { role: 'user', content: message }; + const assistantHeader: ChatMessage = { role: 'assistant', content: '' }; + + const fmtUserMsg = format(userMsg, { isFirst: state.history.length === 0 }); + const fmtAssistantHeader = format(assistantHeader, { isFirst: false }); + + state.history.push(userMsg); + + const prompt = fmtUserMsg + fmtAssistantHeader; + const { response, stats } = await runGeneration(nativeRunner, prompt, { + genConfig: { ...defaultGenerationConfig, ...genConfig }, + stopTokens, + onToken, + }); + + state.history.push({ role: 'assistant', content: response }); + + return { response, stats }; + }; + + return { + stop, + dispose, + sendMessage, + getHistory: () => state.history, + }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts new file mode 100644 index 0000000000..829e2b20bc --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts @@ -0,0 +1,39 @@ +/** Model chat template configuration resolved from tokenizer config file. */ +export type TokenizerChatConfig = { + readonly chatTemplate: string; + readonly bosToken?: string; + readonly eosToken?: string; +}; + +function resolveToken(token: unknown): string | undefined { + if (typeof token === 'string') return token; + if (token && typeof token === 'object' && typeof (token as any).content === 'string') { + return (token as any).content; + } + return undefined; +} + +/** + * Parses raw JSON configuration from `tokenizer_config.json` into a normalized format. + * @param config Raw JSON object from tokenizer_config.json. + * @returns A parsed TokenizerChatConfig object. + */ +export function parseTokenizerConfig(config: any): TokenizerChatConfig { + let chatTemplate = config?.chat_template; + + // Some models ship multiple named templates as `[{ name, template }]`. + if (Array.isArray(chatTemplate)) { + const entry = chatTemplate.find((t) => t?.name === 'default') ?? chatTemplate[0]; + chatTemplate = entry?.template; + } + + if (typeof chatTemplate !== 'string') { + throw new Error('tokenizer_config.json does not contain a string `chat_template`'); + } + + return { + chatTemplate, + bosToken: resolveToken(config?.bos_token), + eosToken: resolveToken(config?.eos_token), + }; +} diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts new file mode 100644 index 0000000000..8a9fcce64e --- /dev/null +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -0,0 +1,111 @@ +import { useEffect, useState } from 'react'; +import RNFS from 'react-native-fs'; + +import { useModel } from './useModel'; +import { useResourceDownload } from './useResourceDownload'; +import { + createLLMChatSession, + type LLMModel, + type LLMChatSessionOptions, + type LLMChatSessionConfig, +} from '../extensions/llm/tasks/llmChatSession'; +import { parseTokenizerConfig, type TokenizerChatConfig } from '../extensions/llm/tokenizerConfig'; + +/** + * Custom React hook to resolve and parse the tokenizer chat template config. + * @category Hooks + * @param source Remote URL or local path to the tokenizer config. + * @param options Config options. + * @returns Object containing parsed config, downloadProgress, and any download/parsing error. + */ +export function useTokenizerConfig(source: string, options?: { preventLoad?: boolean }) { + const { localPath, downloadProgress, downloadError } = useResourceDownload( + source, + options?.preventLoad + ); + const [config, setConfig] = useState(null); + const [error, setError] = useState(null); + + useEffect(() => { + setConfig(null); + setError(null); + if (!localPath) return; + + let isMounted = true; + RNFS.readFile(localPath, 'utf8') + .then((text) => { + if (isMounted) setConfig(parseTokenizerConfig(JSON.parse(text))); + }) + .catch((e) => { + if (isMounted) setError(e instanceof Error ? e : new Error(String(e))); + }); + return () => { + isMounted = false; + }; + }, [localPath]); + + return { config, downloadProgress, error: downloadError || error }; +} + +/** + * React hook to manage downloading, caching, loading, and interacting with an LLM Chat Session model. + * @category Hooks + * @param model Configuration defining model, tokenizer, and tokenizer template paths. + * @param options Chat session options and preventLoad flag. + * @returns Object containing chat session state, sendMessage function, stop function, and errors. + */ +export function useLLMChatSession( + model: LLMModel, + options?: LLMChatSessionOptions & { preventLoad?: boolean } +) { + const { + localPath: localModelPath, + downloadProgress: modelProgress, + downloadError: modelError, + } = useResourceDownload(model.modelPath, options?.preventLoad); + + const { localPath: localTokenizerPath, downloadError: tokenizerError } = useResourceDownload( + model.tokenizerPath, + options?.preventLoad + ); + + const { config: tokenizerConfig, error: configError } = useTokenizerConfig( + model.tokenizerConfigPath, + { preventLoad: options?.preventLoad } + ); + + const downloadProgress = modelProgress; + const downloadError = modelError || tokenizerError || configError; + + const sessionOptions = options + ? { + initialMessages: options.initialMessages, + generationConfig: options.generationConfig, + stopTokens: options.stopTokens, + } + : undefined; + + let sessionConfig: LLMChatSessionConfig | null = null; + if (localModelPath && localTokenizerPath && tokenizerConfig) + sessionConfig = { + model: { modelPath: localModelPath, tokenizerPath: localTokenizerPath, tokenizerConfig }, + options: sessionOptions, + }; + + const { model: session, error: loadError } = useModel(createLLMChatSession, sessionConfig, [ + localModelPath, + localTokenizerPath, + tokenizerConfig, + ]); + + return { + isReady: !!session, + downloadProgress, + error: downloadError || loadError, + localModelPath, + localTokenizerPath, + sendMessage: session?.sendMessage, + getHistory: session?.getHistory, + stop: session?.stop, + }; +} diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index 0865295145..a16855f25f 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -6,6 +6,7 @@ export * from './hooks/useInstanceSegmenter'; export * from './hooks/useKeypointDetector'; export * from './hooks/useObjectDetector'; export * from './hooks/useTokenizer'; +export * from './hooks/useLLMChatSession'; export * from './hooks/useTextEmbedder'; export * from './hooks/usePrivacyFilter'; export * from './hooks/useImageEmbedder'; @@ -38,6 +39,7 @@ export * from './extensions/nlp/tasks/privacyFilter'; export * from './extensions/speech/tasks/fsmnVoiceActivityDetection'; export * from './extensions/speech/tasks/whisperSpeechToText'; export * from './extensions/speech/tasks/supertonicTextToSpeech'; +export * from './extensions/llm/tasks/llmChatSession'; // Core primitives — for library builders and power users export * from './core/error'; @@ -52,6 +54,7 @@ export * as math from './extensions/math'; export * as cv from './extensions/cv'; export * as nlp from './extensions/nlp'; export * as speech from './extensions/speech'; +export * as llm from './extensions/llm'; // Utils export * from './utils'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index fe9155015e..62f2db4d8d 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -14,6 +14,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; +import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -816,6 +817,213 @@ const PRIVACY_FILTER_NEMOTRON_MLX_INT8: PrivacyFilterModel>>>>>> 3599f2c3a (feat: implement TypeScript wrapper and hook for LLM Chat Session) textEmbeddings: { /** * Compact 384-dimensional sentence transformer mapping text to a dense From ff2678552da0fd1b8733f749e62c64c8d1978367 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 13:56:16 +0200 Subject: [PATCH 04/43] refactor(llm): update LLM C++ errors, task pipeline, and useLLMChatSession hook to current rne-rewrite conventions --- .../cpp/extensions/llm/llm_runner.cpp | 52 ++++----- .../extensions/llm/tasks/llmChatSession.ts | 21 ++-- .../src/extensions/llm/tokenizerConfig.ts | 7 +- .../src/hooks/useLLMChatSession.ts | 107 +++--------------- .../react-native-executorch/src/models.ts | 1 - 5 files changed, 59 insertions(+), 129 deletions(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index e9e07c5775..c7baf3c2a4 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -4,6 +4,7 @@ #include "llm_runner.h" #include "core/conversions.h" +#include "core/error.h" #include #include #include @@ -12,6 +13,7 @@ namespace rnexecutorch::extensions::llm { namespace jsi = facebook::jsi; namespace conversions = rnexecutorch::core::conversions; +namespace error = rnexecutorch::core::error; namespace { jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { @@ -33,18 +35,18 @@ LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, tokenizerPath_(tokenizerPath) { auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); if (!tokenizer) { - throw std::runtime_error("LLMRunner: Failed to load runner tokenizer at path: " + tokenizerPath); + throw error::LoadFailed("LLMRunner: Failed to load runner tokenizer at path: " + tokenizerPath); } runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); if (!runner_) { - throw std::runtime_error("LLMRunner: Failed to create text llm runner"); + throw error::LoadFailed("LLMRunner: Failed to create text llm runner"); } auto loadError = runner_->load(); if (loadError != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(loadError); - throw std::runtime_error("LLMRunner: Failed to load model: " + errorMsg); + throw error::LoadFailed("LLMRunner: Failed to load model: " + errorMsg, loadError); } } @@ -63,7 +65,7 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam auto self = shared_from_this(); auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count < 1) { - throw jsi::JSError(rt, "LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); + throw error::InvalidArgument("LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); } std::string prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); @@ -112,28 +114,28 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam // can still interrupt us. std::unique_lock lock(self->mutex_, std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, "LLMRunner.generate: Runner is already in use"); + throw error::ResourceBusy("LLMRunner.generate: Runner is already in use"); } if (!self->runner_) { - throw jsi::JSError(rt, "LLMRunner.generate: Runner has been disposed"); + throw error::ResourceDisposed("LLMRunner.generate: Runner has been disposed"); } auto error = self->runner_->generate(prompt, config, tokenCallback, statsCallback); if (error != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(error); - throw jsi::JSError(rt, "LLMRunner.generate: Failed to generate: " + errorMsg); + throw error::ExecutionFailed("LLMRunner.generate: Failed to generate: " + errorMsg, error); } return statsToJSI(rt, *finalStats); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, fnBody); + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "generate"), 1, error::guarded(fnBody)); } if (nameStr == "prefill") { auto self = shared_from_this(); auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count < 1) { - throw jsi::JSError(rt, "LLMRunner.prefill: Usage: prefill(prompt)"); + throw error::InvalidArgument("LLMRunner.prefill: Usage: prefill(prompt)"); } std::string prompt = conversions::asType(rt, "LLMRunner.prefill: prompt", args[0]); @@ -141,44 +143,44 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam // Lock held for the whole call, same as generate(). std::unique_lock lock(self->mutex_, std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, "LLMRunner.prefill: Runner is already in use"); + throw error::ResourceBusy("LLMRunner.prefill: Runner is already in use"); } if (!self->runner_) { - throw jsi::JSError(rt, "LLMRunner.prefill: Runner has been disposed"); + throw error::ResourceDisposed("LLMRunner.prefill: Runner has been disposed"); } auto result = self->runner_->prefill({executorch::extension::llm::make_text_input(prompt)}); if (result.error() != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(result.error()); - throw jsi::JSError(rt, "LLMRunner.prefill: Failed: " + errorMsg); + throw error::ExecutionFailed("LLMRunner.prefill: Failed: " + errorMsg, result.error()); } return jsi::Value::undefined(); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, fnBody); + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "prefill"), 1, error::guarded(fnBody)); } if (nameStr == "stop") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t /*count*/) -> jsi::Value { + auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t /*count*/) -> jsi::Value { // Intentionally no mutex here: stop() is designed to be called // concurrently to interrupt an in-progress generate(). Taking the // lock would block until generate() finishes, defeating the point. // runner_ is only cleared by dispose() on this same (JS) thread, // so reading it lock-free here is safe. if (!self->runner_) { - throw jsi::JSError(rt, "LLMRunner.stop: Runner has been disposed"); + throw error::ResourceDisposed("LLMRunner.stop: Runner has been disposed"); } self->runner_->stop(); return jsi::Value::undefined(); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, fnBody); + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "stop"), 0, error::guarded(fnBody)); } if (nameStr == "dispose") { auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { + auto fnBody = [self](jsi::Runtime & /*rt*/, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { if (count != 0) { - throw jsi::JSError(rt, "dispose: Usage: dispose()"); + throw error::InvalidArgument("dispose: Usage: dispose()"); } // Signal stop before locking so any in-progress generate() exits @@ -194,7 +196,7 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam return jsi::Value::undefined(); }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, fnBody); + return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "dispose"), 0, error::guarded(fnBody)); } return jsi::Value::undefined(); @@ -215,20 +217,16 @@ void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { const auto *name = "createLLMRunner"; auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count != 2) { - throw jsi::JSError(rt, "createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath)"); + throw error::InvalidArgument("createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath)"); } auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); - try { - auto runnerInstance = std::make_shared(modelPath, tokenizerPath); - return jsi::Object::createFromHostObject(rt, runnerInstance); - } catch (const std::exception &e) { - throw jsi::JSError(rt, std::format("createLLMRunner: {}", e.what())); - } + auto runnerInstance = std::make_shared(modelPath, tokenizerPath); + return jsi::Object::createFromHostObject(rt, runnerInstance); }; - module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, fnBody)); + module.setProperty(rt, name, jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, error::guarded(fnBody))); } } // namespace rnexecutorch::extensions::llm diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index b32c1068e6..03a00dc998 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -1,4 +1,5 @@ import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; +import RNBlobUtil from 'react-native-blob-util'; import { wrapAsync } from '../../../core/runtime'; import { @@ -8,7 +9,7 @@ import { type GenerationStats, } from '../llmRunner'; import { createJinjaChatFormatter } from '../jinja'; -import type { TokenizerChatConfig } from '../tokenizerConfig'; +import { parseTokenizerConfig } from '../tokenizerConfig'; export type { GenerationConfig, GenerationStats }; @@ -38,12 +39,6 @@ export type LLMChatSessionOptions = { readonly stopTokens?: readonly string[]; }; -/** Config package passed to instantiate an LLM chat session. */ -export type LLMChatSessionConfig = { - readonly model: Omit & { tokenizerConfig: TokenizerChatConfig }; - readonly options?: LLMChatSessionOptions; -}; - /** Return wrapper holding generated response text and performance stats. */ export type GenerationResult = { readonly response: string; @@ -93,20 +88,24 @@ function generateChatTurn( /** * Instantiates an LLM chat session using background thread execution. - * @param config Chat session configuration and settings. + * @param config Model configuration containing model, tokenizer, and tokenizer config paths. + * @param options Custom generation and state options. * @param runtime The worklet runtime thread to run native generation on. * @returns A Promise resolving to an LLMChatSession instance. */ export async function createLLMChatSession( - config: LLMChatSessionConfig, + config: LLMModel, + options?: LLMChatSessionOptions, runtime?: WorkletRuntime ): Promise { - const { model, options } = config; - const { modelPath, tokenizerPath, tokenizerConfig } = model; + const { modelPath, tokenizerPath, tokenizerConfigPath } = config; const initialMessages = options?.initialMessages ?? []; const defaultGenerationConfig = options?.generationConfig; + // Read and parse tokenizer_config.json + const configStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); + const tokenizerConfig = parseTokenizerConfig(JSON.parse(configStr)); const { chatTemplate, bosToken, eosToken } = tokenizerConfig; const format = createJinjaChatFormatter(chatTemplate, { bosToken }); diff --git a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts index 829e2b20bc..e06b3113db 100644 --- a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts +++ b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts @@ -1,3 +1,5 @@ +import { RnExecuTorchError } from '../../core/error'; + /** Model chat template configuration resolved from tokenizer config file. */ export type TokenizerChatConfig = { readonly chatTemplate: string; @@ -28,7 +30,10 @@ export function parseTokenizerConfig(config: any): TokenizerChatConfig { } if (typeof chatTemplate !== 'string') { - throw new Error('tokenizer_config.json does not contain a string `chat_template`'); + throw RnExecuTorchError( + 'LOAD_FAILED', + 'tokenizer_config.json does not contain a string `chat_template`' + ); } return { diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts index 8a9fcce64e..5d3bde5176 100644 --- a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -1,109 +1,38 @@ -import { useEffect, useState } from 'react'; -import RNFS from 'react-native-fs'; - import { useModel } from './useModel'; -import { useResourceDownload } from './useResourceDownload'; +import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; import { createLLMChatSession, type LLMModel, type LLMChatSessionOptions, - type LLMChatSessionConfig, } from '../extensions/llm/tasks/llmChatSession'; -import { parseTokenizerConfig, type TokenizerChatConfig } from '../extensions/llm/tokenizerConfig'; /** - * Custom React hook to resolve and parse the tokenizer chat template config. + * React hook to load and run an LLM chat session model. + * + * This hook manages downloading (if they are remote URLs) and loading the `.pte` model + * file, `tokenizer.json`, and `tokenizer_config.json`, tracking download progress and errors, + * and cleaning up native memory when the component unmounts or configuration changes. * @category Hooks - * @param source Remote URL or local path to the tokenizer config. - * @param options Config options. - * @returns Object containing parsed config, downloadProgress, and any download/parsing error. - */ -export function useTokenizerConfig(source: string, options?: { preventLoad?: boolean }) { - const { localPath, downloadProgress, downloadError } = useResourceDownload( - source, - options?.preventLoad - ); - const [config, setConfig] = useState(null); - const [error, setError] = useState(null); - - useEffect(() => { - setConfig(null); - setError(null); - if (!localPath) return; - - let isMounted = true; - RNFS.readFile(localPath, 'utf8') - .then((text) => { - if (isMounted) setConfig(parseTokenizerConfig(JSON.parse(text))); - }) - .catch((e) => { - if (isMounted) setError(e instanceof Error ? e : new Error(String(e))); - }); - return () => { - isMounted = false; - }; - }, [localPath]); - - return { config, downloadProgress, error: downloadError || error }; -} - -/** - * React hook to manage downloading, caching, loading, and interacting with an LLM Chat Session model. - * @category Hooks - * @param model Configuration defining model, tokenizer, and tokenizer template paths. - * @param options Chat session options and preventLoad flag. - * @returns Object containing chat session state, sendMessage function, stop function, and errors. + * @param config The LLM model configuration. + * @param options Chat session options and load/caching options. See {@link ResourceOptions}. + * @returns An object containing the session's loading state, error, download progress, + * and chat functions. */ export function useLLMChatSession( - model: LLMModel, - options?: LLMChatSessionOptions & { preventLoad?: boolean } + config: LLMModel, + options?: LLMChatSessionOptions & ResourceOptions ) { - const { - localPath: localModelPath, - downloadProgress: modelProgress, - downloadError: modelError, - } = useResourceDownload(model.modelPath, options?.preventLoad); - - const { localPath: localTokenizerPath, downloadError: tokenizerError } = useResourceDownload( - model.tokenizerPath, - options?.preventLoad - ); - - const { config: tokenizerConfig, error: configError } = useTokenizerConfig( - model.tokenizerConfigPath, - { preventLoad: options?.preventLoad } + const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); + const { model: session, error } = useModel( + (res) => createLLMChatSession(res, options), + resource ?? null ); - const downloadProgress = modelProgress; - const downloadError = modelError || tokenizerError || configError; - - const sessionOptions = options - ? { - initialMessages: options.initialMessages, - generationConfig: options.generationConfig, - stopTokens: options.stopTokens, - } - : undefined; - - let sessionConfig: LLMChatSessionConfig | null = null; - if (localModelPath && localTokenizerPath && tokenizerConfig) - sessionConfig = { - model: { modelPath: localModelPath, tokenizerPath: localTokenizerPath, tokenizerConfig }, - options: sessionOptions, - }; - - const { model: session, error: loadError } = useModel(createLLMChatSession, sessionConfig, [ - localModelPath, - localTokenizerPath, - tokenizerConfig, - ]); - return { isReady: !!session, + error: downloadError || error, downloadProgress, - error: downloadError || loadError, - localModelPath, - localTokenizerPath, + resource, sendMessage: session?.sendMessage, getHistory: session?.getHistory, stop: session?.stop, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 62f2db4d8d..231fb8e732 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1594,7 +1594,6 @@ export const models = { }, }, }, ->>>>>>> 3599f2c3a (feat: implement TypeScript wrapper and hook for LLM Chat Session) textEmbeddings: { /** * Compact 384-dimensional sentence transformer mapping text to a dense From 30382147f8e1aad4f6951960bf94852494d32664 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 13:59:10 +0200 Subject: [PATCH 05/43] fix(llm): use concise preprocessor directive in llm_runner --- .../react-native-executorch/cpp/extensions/llm/llm_runner.cpp | 2 +- .../react-native-executorch/cpp/extensions/llm/llm_runner.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index c7baf3c2a4..236dcbcd82 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -1,4 +1,4 @@ -#if defined(__clang__) +#ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index b5205d02bf..867ba6a511 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -1,6 +1,6 @@ #pragma once -#if defined(__clang__) +#ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif From d90d2fbc384d2960e14755dc62275e21976a15a5 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 14:34:44 +0200 Subject: [PATCH 06/43] style(models): prefix LLM size sub-keys with P for valid unquoted identifiers --- .../react-native-executorch/src/models.ts | 53 +++++++++++++------ 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 231fb8e732..41e83fdb01 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -832,6 +832,11 @@ const LFM2_5_1_2B_XNNPACK_FP16: LLMModel = { tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, }; +const LFM2_5_1_2B_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/1_2b/mlx/lfm_2_5_1_2b_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/1_2b/tokenizer_config.json`, +}; const LFM2_5_350M_XNNPACK_8DA4W: LLMModel = { modelPath: `${LFM2_5_BASE_URL}/350m/xnnpack/lfm_2_5_350m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, @@ -842,6 +847,11 @@ const LFM2_5_350M_XNNPACK_FP16: LLMModel = { tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; +const LFM2_5_350M_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/350m/mlx/lfm_2_5_350m_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; @@ -1490,65 +1500,72 @@ export const models = { }, llm: { LFM2_5: { - '1_2B': { + ...LFM2_5_1_2B_XNNPACK_8DA4W, + P1_2B: { ...LFM2_5_1_2B_XNNPACK_8DA4W, XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, + MLX_INT4: LFM2_5_1_2B_MLX_INT4, }, - '350M': { + P350M: { ...LFM2_5_350M_XNNPACK_8DA4W, XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, + MLX_INT4: LFM2_5_350M_MLX_INT4, }, }, BIELIK_V3: { - '1_5B': { + ...BIELIK_V3_1_5B_XNNPACK_8DA4W, + P1_5B: { ...BIELIK_V3_1_5B_XNNPACK_8DA4W, XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, }, }, LLAMA3_2: { - '1B': { + ...LLAMA3_2_1B_SPINQUANT, + P1B: { ...LLAMA3_2_1B_SPINQUANT, XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_1B_BF16, }, - '3B': { + P3B: { ...LLAMA3_2_3B_SPINQUANT, XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, XNNPACK_BF16: LLAMA3_2_3B_BF16, }, }, SMOLLM2: { - '135M': { + ...SMOLLM2_1_7B_8DA4W, + P135M: { ...SMOLLM2_135M_8DA4W, XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, XNNPACK_BF16: SMOLLM2_135M_BF16, }, - '360M': { + P360M: { ...SMOLLM2_360M_8DA4W, XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, XNNPACK_BF16: SMOLLM2_360M_BF16, }, - '1_7B': { + P1_7B: { ...SMOLLM2_1_7B_8DA4W, XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, XNNPACK_BF16: SMOLLM2_1_7B_BF16, }, }, HAMMER2_1: { - '0_5B': { + ...HAMMER2_1_1_5B_XNNPACK_8DA4W, + P0_5B: { ...HAMMER2_1_0_5B_XNNPACK_8DA4W, XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, }, - '1_5B': { + P1_5B: { ...HAMMER2_1_1_5B_XNNPACK_8DA4W, XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, }, - '3B': { + P3B: { ...HAMMER2_1_3B_XNNPACK_8DA4W, XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, @@ -1560,34 +1577,36 @@ export const models = { XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, }, QWEN2_5: { - '0_5B': { + ...QWEN2_5_1_5B_XNNPACK_8DA4W, + P0_5B: { ...QWEN2_5_0_5B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, }, - '1_5B': { + P1_5B: { ...QWEN2_5_1_5B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, }, - '3B': { + P3B: { ...QWEN2_5_3B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, }, }, QWEN3: { - '0_6B': { + ...QWEN3_1_7B_XNNPACK_8DA4W, + P0_6B: { ...QWEN3_0_6B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, }, - '1_7B': { + P1_7B: { ...QWEN3_1_7B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, }, - '4B': { + P4B: { ...QWEN3_4B_XNNPACK_8DA4W, XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, From 1439616ea6e1b5326a82818aa15fc9aff90b9ee7 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 14:38:22 +0200 Subject: [PATCH 07/43] docs(llm): document LLMRunnerHostObject in llm_runner.h --- .../cpp/extensions/llm/llm_runner.h | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index 867ba6a511..66ffe22a97 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -11,17 +11,36 @@ #include namespace rnexecutorch::extensions::llm { -class LLMRunnerHostObject : public facebook::jsi::HostObject, public std::enable_shared_from_this { +/** + * JSI HostObject wrapping an ExecuTorch Text LLM runner instance + * (`executorch::extension::llm::TextLLMRunner`). + * + * Exposes methods to JavaScript for prefilling prompt context, generating token + * stream continuations, interrupting generation, and releasing native model memory. + */ +class LLMRunnerHostObject : public facebook::jsi::HostObject, + public std::enable_shared_from_this { public: + /** + * Constructs an LLMRunnerHostObject by loading an ExecuTorch model binary and tokenizer. + * + * @param modelPath Absolute file system path to the `.pte` LLM model binary. + * @param tokenizerPath Absolute file system path to the local tokenizer configuration file (e.g. `tokenizer.json`). + * @throws core::error::RnExecuTorchException with code LoadFailed if loading the model or tokenizer fails. + */ LLMRunnerHostObject(const std::string &modelPath, const std::string &tokenizerPath); facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; private: + /** Owning pointer to the underlying ExecuTorch TextLLMRunner instance. */ std::unique_ptr runner_; + /** Mutex guarding concurrent access to prefill and generation operations. */ std::mutex mutex_; + /** File system path to the loaded `.pte` model binary. */ std::string modelPath_; + /** File system path to the loaded tokenizer configuration file. */ std::string tokenizerPath_; }; From 259755db95468f61ac7904cdd58356a0f9e5fd4b Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 14:42:32 +0200 Subject: [PATCH 08/43] docs(llm): explain ET_EXPERIMENTAL macro expansion for deprecation warning pragma --- .../react-native-executorch/cpp/extensions/llm/llm_runner.cpp | 4 ++++ .../react-native-executorch/cpp/extensions/llm/llm_runner.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index 236dcbcd82..53a6905542 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -1,3 +1,7 @@ +// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, load_tokenizer, +// create_text_llm_runner) with `ET_EXPERIMENTAL`, which expands to `[[deprecated("...")]]`. +// We suppress -Wdeprecated-declarations so Clang does not fail builds on ExecuTorch's +// experimental API tags. #ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index 66ffe22a97..a3d4d9ec25 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -1,5 +1,9 @@ #pragma once +// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, etc.) +// with `ET_EXPERIMENTAL`, which expands to `[[deprecated("This API is experimental...")]]`. +// We suppress -Wdeprecated-declarations so Clang does not fail builds on ExecuTorch's +// experimental API tags. #ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif From d10b22d6af3e2ee372082e1ad6d726ae02682be2 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 12 Aug 2026 14:50:07 +0200 Subject: [PATCH 09/43] docs(models): add wrapped JSDoc comments for LLM category and model families --- .../react-native-executorch/src/models.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 41e83fdb01..a764aeb83a 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1498,7 +1498,16 @@ export const models = { /** WordPiece tokenizer URL for the `all-MiniLM-L6-v2` embedding model. */ ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER, }, + /** + * Generative Large Language Models (LLMs) for instruction following, + * chat, text generation, and reasoning. + */ llm: { + /** + * Liquid AI LFM 2.5 generative text model family (350M to 1.2B + * parameters) optimized for high-efficiency on-device chat and text + * generation. + */ LFM2_5: { ...LFM2_5_1_2B_XNNPACK_8DA4W, P1_2B: { @@ -1514,6 +1523,9 @@ export const models = { MLX_INT4: LFM2_5_350M_MLX_INT4, }, }, + /** + * Bielik v3 1.5B Polish and English instruction-tuned language model. + */ BIELIK_V3: { ...BIELIK_V3_1_5B_XNNPACK_8DA4W, P1_5B: { @@ -1522,6 +1534,10 @@ export const models = { XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, }, }, + /** + * Meta Llama 3.2 lightweight multilingual text generation model + * family (1B and 3B parameters). + */ LLAMA3_2: { ...LLAMA3_2_1B_SPINQUANT, P1B: { @@ -1535,6 +1551,10 @@ export const models = { XNNPACK_BF16: LLAMA3_2_3B_BF16, }, }, + /** + * Hugging Face SmolLM2 ultra-compact language model family (135M to + * 1.7B parameters). + */ SMOLLM2: { ...SMOLLM2_1_7B_8DA4W, P135M: { @@ -1553,6 +1573,10 @@ export const models = { XNNPACK_BF16: SMOLLM2_1_7B_BF16, }, }, + /** + * Hammer 2.1 function-calling and agentic tool-use language model + * family. + */ HAMMER2_1: { ...HAMMER2_1_1_5B_XNNPACK_8DA4W, P0_5B: { @@ -1571,11 +1595,19 @@ export const models = { XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, }, }, + /** + * Microsoft Phi-4 Mini 3.8B parameter lightweight reasoning language + * model. + */ PHI4_MINI: { ...PHI4_MINI_XNNPACK_8DA4W, XNNPACK_8DA4W: PHI4_MINI_XNNPACK_8DA4W, XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, }, + /** + * Alibaba Qwen 2.5 multilingual instruction-tuned language model + * family. + */ QWEN2_5: { ...QWEN2_5_1_5B_XNNPACK_8DA4W, P0_5B: { @@ -1594,6 +1626,10 @@ export const models = { XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, }, }, + /** + * Alibaba Qwen 3 high-performance multilingual text generation model + * family. + */ QWEN3: { ...QWEN3_1_7B_XNNPACK_8DA4W, P0_6B: { From f25191e8393ccf50d7714c651cff254837d027c2 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 02:39:32 +0200 Subject: [PATCH 10/43] feat(llm): add multimodal input support to LLM runner --- apps/nlp/app/llm/index.tsx | 447 ++++++++++++++++++ .../cpp/extensions/llm/llm_runner.cpp | 150 ++++-- .../cpp/extensions/llm/llm_runner.h | 30 +- .../src/extensions/llm/llmRunner.ts | 47 +- 4 files changed, 623 insertions(+), 51 deletions(-) create mode 100644 apps/nlp/app/llm/index.tsx diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx new file mode 100644 index 0000000000..96a656da61 --- /dev/null +++ b/apps/nlp/app/llm/index.tsx @@ -0,0 +1,447 @@ +import React, { useEffect, useRef, useState, type ComponentRef } from 'react'; +import { + View, + Text, + TextInput, + TouchableOpacity, + ScrollView, + StyleSheet, + ActivityIndicator, + KeyboardAvoidingView, + Platform, +} from 'react-native'; +import { + useLLMChatSession, + models, + type ChatMessage, + type GenerationStats, +} from 'react-native-executorch'; +import ScreenWrapper from '../../components/ScreenWrapper'; +import { NestedModelPicker, findPath } from '../../components/ModelPicker'; + +const SYSTEM_PROMPT = + "You are a pirate. You must start every response with 'Ahoy matey!' and speak like a pirate."; +const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; +const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; + +type Turn = { role: 'user' | 'assistant'; content: string; stats?: GenerationStats }; + +function formatStats(stats: GenerationStats): string { + const decodeMs = stats.inferenceEndMs - stats.firstTokenMs; + const tokensPerSec = (stats.numGeneratedTokens / decodeMs) * 1000; + const totalMs = stats.inferenceEndMs - stats.inferenceStartMs; + const ttftMs = stats.firstTokenMs - stats.inferenceStartMs; + return ( + `gen ${stats.numGeneratedTokens} tokens · ` + + `${tokensPerSec.toFixed(1)} tok/s · ` + + `${ttftMs.toFixed(0)}ms ttft · ` + + `${(totalMs / 1000).toFixed(2)}s` + ); +} + +function getFirstLeafModel(node: any): any { + if (!node || typeof node !== 'object') return null; + for (const key of Object.keys(node)) { + if (typeof node[key] === 'object' && node[key] !== null) { + const leaf = getFirstLeafModel(node[key]); + if (leaf) return leaf; + } + } + if (typeof node.modelPath === 'string') return node; + return null; +} + +function LLMContent() { + const [selectedModel, setSelectedModel] = useState(getFirstLeafModel(models.llm)); + const [activeModel, setActiveModel] = useState(null); + const [forceDownload, setForceDownload] = useState(false); + + const { isReady, downloadProgress, error, sendMessage, stop } = useLLMChatSession( + activeModel || selectedModel, + { + initialMessages: INITIAL_MESSAGES, + generationConfig: GENERATION_CONFIG, + preventLoad: !activeModel, + forceDownload, + } + ); + + const [input, setInput] = useState(''); + const [turns, setTurns] = useState([]); + const [streamingResponse, setStreamingResponse] = useState(null); + + const scrollRef = useRef>(null); + const isGenerating = streamingResponse !== null; + + const selectedModelName = findPath(models.llm, selectedModel)?.join(' ') || 'Selected Model'; + + // Reset chat turns when model changes + useEffect(() => { + setTurns([]); + setStreamingResponse(null); + setInput(''); + }, [activeModel]); + + // Reset forceDownload when model finishes loading and is ready + useEffect(() => { + if (isReady) setForceDownload(false); + }, [isReady]); + + const handleLoadModel = (force = false) => { + setForceDownload(force); + setActiveModel(selectedModel); + }; + + const handleSend = async () => { + const message = input.trim(); + if (!message || !sendMessage || isGenerating) return; + + setInput(''); + setStreamingResponse(''); + setTurns((prev) => [...prev, { role: 'user', content: message }]); + + try { + const { response, stats } = await sendMessage(message, (token) => { + setStreamingResponse((prev) => (prev !== null ? prev + token : token)); + }); + setTurns((prev) => [...prev, { role: 'assistant', content: response, stats }]); + } catch (e: any) { + setTurns((prev) => [...prev, { role: 'assistant', content: `[Error] ${e?.message}` }]); + } finally { + setStreamingResponse(null); + } + }; + + const renderContent = () => { + if (!activeModel) { + return ( + + No model loaded + + {selectedModelName} is selected. Click below to load it and start chatting. + + handleLoadModel(false)}> + Load Model + + + ); + } + + if (activeModel !== selectedModel) { + return ( + + Switch to {selectedModelName}? + + Switching models will unload the current model and reset the chat session. + + handleLoadModel(false)}> + Load New Model + + setSelectedModel(activeModel)} + > + Keep Current Model + + + ); + } + + if (error) { + return ( + + Failed to load model + {error.message} + handleLoadModel(false)}> + Retry Loading + + handleLoadModel(true)}> + Force Redownload + + + ); + } + + if (!isReady) { + return ( + + + + {downloadProgress < 100 + ? `Downloading model… ${downloadProgress.toFixed(0)}%` + : 'Loading model into memory…'} + + {activeModel.modelPath} + + ); + } + + return ( + <> + scrollRef.current?.scrollToEnd({ animated: false })} + > + {turns.length === 0 && streamingResponse === null && ( + Ask the on-device model anything to get started. + )} + {turns.map((turn, idx) => ( + + + + {turn.content || '…'} + + + {turn.stats && ( + + {formatStats(turn.stats)} + + )} + + ))} + {streamingResponse !== null && ( + + + {streamingResponse || '…'} + + + )} + + + + + {isGenerating ? ( + stop?.()} + > + Stop + + ) : ( + + Send + + )} + + + ); + }; + + return ( + + {/* Model Selector Header */} + + + + { + setForceDownload(false); + setSelectedModel(m); + }} + /> + + handleLoadModel(true)} + > + Redownload + + + + + {/* Screen Content */} + {renderContent()} + + ); +} + +export default function LLMScreen() { + return ( + + + + ); +} + +const styles = StyleSheet.create({ + container: { flex: 1, backgroundColor: '#f8f9fa' }, + header: { + paddingHorizontal: 16, + paddingTop: 12, + paddingBottom: 4, + backgroundColor: '#fff', + borderBottomWidth: 1, + borderBottomColor: '#e9ecef', + }, + content: { + flex: 1, + }, + centered: { flex: 1, justifyContent: 'center', alignItems: 'center', padding: 24 }, + loadingText: { marginTop: 16, fontSize: 15, color: '#495057', fontWeight: '600' }, + loadingSub: { marginTop: 4, fontSize: 13, color: '#868e96', textAlign: 'center' }, + errorTitle: { fontSize: 16, fontWeight: '700', color: '#e03131', marginBottom: 8 }, + errorBody: { fontSize: 13, color: '#868e96', textAlign: 'center' }, + messages: { flex: 1 }, + messagesContent: { padding: 16, paddingBottom: 8 }, + placeholder: { textAlign: 'center', color: '#adb5bd', marginTop: 40, fontSize: 14 }, + turn: { marginBottom: 12 }, + bubble: { + maxWidth: '85%', + borderRadius: 16, + paddingHorizontal: 14, + paddingVertical: 10, + }, + statsLine: { + alignSelf: 'flex-start', + marginTop: 5, + marginLeft: 4, + fontSize: 11, + color: '#adb5bd', + // cspell:disable-next-line + fontVariant: ['tabular-nums'], + }, + userBubble: { alignSelf: 'flex-end', backgroundColor: '#0070f3' }, + assistantBubble: { + alignSelf: 'flex-start', + backgroundColor: '#fff', + borderWidth: 1, + borderColor: '#e9ecef', + }, + userText: { color: '#fff', fontSize: 15, lineHeight: 21 }, + assistantText: { color: '#212529', fontSize: 15, lineHeight: 21 }, + inputRow: { + flexDirection: 'row', + alignItems: 'flex-end', + padding: 12, + gap: 8, + borderTopWidth: 1, + borderTopColor: '#e9ecef', + backgroundColor: '#fff', + }, + input: { + flex: 1, + backgroundColor: '#f1f3f5', + borderRadius: 20, + paddingHorizontal: 16, + paddingVertical: 10, + fontSize: 15, + color: '#212529', + maxHeight: 120, + }, + sendButton: { + backgroundColor: '#0070f3', + borderRadius: 20, + paddingHorizontal: 18, + height: 44, + justifyContent: 'center', + alignItems: 'center', + }, + sendButtonDisabled: { backgroundColor: '#a3cdff' }, + stopButton: { backgroundColor: '#e03131' }, + sendButtonText: { color: '#fff', fontSize: 15, fontWeight: '600' }, + infoTitle: { + fontSize: 16, + fontWeight: '700', + color: '#212529', + marginBottom: 8, + textAlign: 'center', + }, + infoBody: { + fontSize: 14, + color: '#495057', + textAlign: 'center', + marginBottom: 20, + lineHeight: 20, + }, + loadButton: { + backgroundColor: '#0070f3', + borderRadius: 20, + paddingHorizontal: 24, + paddingVertical: 12, + justifyContent: 'center', + alignItems: 'center', + shadowColor: '#000', + shadowOffset: { width: 0, height: 2 }, + shadowOpacity: 0.1, + shadowRadius: 4, + elevation: 2, + }, + loadButtonText: { + color: '#fff', + fontSize: 15, + fontWeight: '600', + }, + cancelButton: { + marginTop: 12, + paddingVertical: 8, + }, + cancelButtonText: { + color: '#6c757d', + fontSize: 14, + fontWeight: '500', + }, + secondaryButton: { + marginTop: 12, + backgroundColor: '#f1f3f5', + borderRadius: 20, + paddingHorizontal: 24, + paddingVertical: 12, + justifyContent: 'center', + alignItems: 'center', + }, + secondaryButtonText: { + color: '#495057', + fontSize: 15, + fontWeight: '600', + }, + headerRow: { + flexDirection: 'row', + alignItems: 'center', + gap: 8, + }, + pickerContainer: { + flex: 1, + }, + redownloadHeaderButton: { + paddingHorizontal: 12, + paddingVertical: 6, + backgroundColor: '#f1f3f5', + borderRadius: 12, + }, + redownloadHeaderText: { + color: '#495057', + fontSize: 13, + fontWeight: '500', + }, +}); diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index 53a6905542..f3a5b5b260 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -1,23 +1,34 @@ -// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, load_tokenizer, -// create_text_llm_runner) with `ET_EXPERIMENTAL`, which expands to `[[deprecated("...")]]`. -// We suppress -Wdeprecated-declarations so Clang does not fail builds on ExecuTorch's -// experimental API tags. +// Upstream ExecuTorch annotates experimental LLM APIs (MultimodalRunner, Stats, +// load_tokenizer, create_multimodal_runner) with `ET_EXPERIMENTAL`, which +// expands to `[[deprecated("...")]]`. We suppress -Wdeprecated-declarations so +// Clang does not fail builds on ExecuTorch's experimental API tags. #ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif #include "llm_runner.h" -#include "core/conversions.h" -#include "core/error.h" -#include + +#include +#include +#include + #include #include +#include #include +#include + +#include "core/conversions.h" +#include "core/error.h" +#include "core/tensor_helpers.h" namespace rnexecutorch::extensions::llm { namespace jsi = facebook::jsi; -namespace conversions = rnexecutorch::core::conversions; namespace error = rnexecutorch::core::error; +namespace tensor = rnexecutorch::core::tensor; +namespace conversions = rnexecutorch::core::conversions; + +using rnexecutorch::core::types::DType; namespace { jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats &stats) { @@ -31,26 +42,93 @@ jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats obj.setProperty(rt, "modelLoadEndMs", static_cast(stats.model_load_end_ms)); return obj; } + +std::vector parseMultimodalPromptArray( + jsi::Runtime &rt, + const std::string &ctx, + const jsi::Value &value, + const std::vector &supportedModalities) { + + auto arr = conversions::asType(rt, std::format("{}: prompt array", ctx), value); + size_t len = arr.length(rt); + + std::vector inputs; + inputs.reserve(len); + + for (size_t i = 0; i < len; ++i) { + auto elem = arr.getValueAtIndex(rt, i); + if (elem.isString()) { + inputs.emplace_back(elem.asString(rt).utf8(rt)); + } else if (elem.isObject()) { + auto mediaObj = elem.asObject(rt); + std::string itemCtx = std::format("{}[{}]", ctx, i); + auto kind = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "kind"); + + if (std::ranges::find(supportedModalities, kind) == supportedModalities.end()) { + throw error::InvalidArgument(std::format("{}: Modality '{}' is not supported " + "by this runner instance", + itemCtx, kind)); + } + + if (kind == "image") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "image"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"C", "H", "W"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto C = shape[0]; + const auto H = shape[1]; + const auto W = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Image(std::move(data), W, H, C)); + } else if (kind == "audio") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "audio"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"batch", "n_bins", "n_frames"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto batchSize = shape[0]; + const auto nBins = shape[1]; + const auto nFrames = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Audio(std::move(data), batchSize, nBins, nFrames)); + } else { + throw error::InvalidArgument(std::format("{}: Unsupported media kind '{}'", itemCtx, kind)); + } + } else { + throw error::InvalidArgument(std::format("{}: Prompt array elements must be strings or media objects", ctx)); + } + } + + return inputs; +} } // namespace LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, - const std::string &tokenizerPath) + const std::string &tokenizerPath, + const std::vector &modalities) : modelPath_(modelPath), - tokenizerPath_(tokenizerPath) { + tokenizerPath_(tokenizerPath), + modalities_(modalities) { + auto tokenizer = executorch::extension::llm::load_tokenizer(tokenizerPath); if (!tokenizer) { - throw error::LoadFailed("LLMRunner: Failed to load runner tokenizer at path: " + tokenizerPath); + throw error::LoadFailed(std::format("LLMRunner: Failed to load runner tokenizer at path: {}", tokenizerPath)); } - runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); + runner_ = executorch::extension::llm::create_multimodal_runner(modelPath, std::move(tokenizer)); if (!runner_) { - throw error::LoadFailed("LLMRunner: Failed to create text llm runner"); + throw error::LoadFailed("LLMRunner: Failed to create llm runner"); } auto loadError = runner_->load(); if (loadError != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(loadError); - throw error::LoadFailed("LLMRunner: Failed to load model: " + errorMsg, loadError); + throw error::LoadFailed(std::format("LLMRunner: Failed to load model: {}", errorMsg), loadError); } } @@ -65,6 +143,10 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam return jsi::String::createFromUtf8(rt, tokenizerPath_); } + if (nameStr == "modalities") { + return conversions::toJsiArray(rt, modalities_); + } + if (nameStr == "generate") { auto self = shared_from_this(); auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { @@ -72,8 +154,6 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam throw error::InvalidArgument("LLMRunner.generate: Usage: generate(prompt, config?, onToken?)"); } - std::string prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); - executorch::extension::llm::GenerationConfig config; if (count > 1 && !args[1].isUndefined() && !args[1].isNull()) { auto configObj = conversions::asType(rt, "LLMRunner.generate: config", args[1]); @@ -123,11 +203,19 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam if (!self->runner_) { throw error::ResourceDisposed("LLMRunner.generate: Runner has been disposed"); } - auto error = self->runner_->generate(prompt, config, tokenCallback, statsCallback); - if (error != executorch::runtime::Error::Ok) { - std::string errorMsg = executorch::runtime::to_string(error); - throw error::ExecutionFailed("LLMRunner.generate: Failed to generate: " + errorMsg, error); + auto genError = executorch::runtime::Error::Ok; + if (args[0].isString()) { + std::string prompt = args[0].asString(rt).utf8(rt); + genError = self->runner_->generate(prompt, config, tokenCallback, statsCallback); + } else { + auto inputs = parseMultimodalPromptArray(rt, "LLMRunner.generate", args[0], self->modalities_); + genError = self->runner_->generate(inputs, config, tokenCallback, statsCallback); + } + + if (genError != executorch::runtime::Error::Ok) { + std::string errorMsg = executorch::runtime::to_string(genError); + throw error::ExecutionFailed(std::format("LLMRunner.generate: Failed to generate: {}", errorMsg), genError); } return statsToJSI(rt, *finalStats); @@ -142,8 +230,6 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam throw error::InvalidArgument("LLMRunner.prefill: Usage: prefill(prompt)"); } - std::string prompt = conversions::asType(rt, "LLMRunner.prefill: prompt", args[0]); - // Lock held for the whole call, same as generate(). std::unique_lock lock(self->mutex_, std::try_to_lock); if (!lock.owns_lock()) { @@ -152,10 +238,14 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam if (!self->runner_) { throw error::ResourceDisposed("LLMRunner.prefill: Runner has been disposed"); } - auto result = self->runner_->prefill({executorch::extension::llm::make_text_input(prompt)}); + + auto result = args[0].isString() + ? self->runner_->prefill(args[0].asString(rt).utf8(rt)) + : self->runner_->prefill(parseMultimodalPromptArray(rt, "LLMRunner.prefill", args[0], self->modalities_)); + if (result.error() != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(result.error()); - throw error::ExecutionFailed("LLMRunner.prefill: Failed: " + errorMsg, result.error()); + throw error::ExecutionFailed(std::format("LLMRunner.prefill: Failed: {}", errorMsg), result.error()); } return jsi::Value::undefined(); @@ -210,6 +300,7 @@ std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime std::vector properties; properties.push_back(jsi::PropNameID::forAscii(rt, "modelPath")); properties.push_back(jsi::PropNameID::forAscii(rt, "tokenizerPath")); + properties.push_back(jsi::PropNameID::forAscii(rt, "modalities")); properties.push_back(jsi::PropNameID::forAscii(rt, "prefill")); properties.push_back(jsi::PropNameID::forAscii(rt, "generate")); properties.push_back(jsi::PropNameID::forAscii(rt, "stop")); @@ -220,14 +311,19 @@ std::vector LLMRunnerHostObject::getPropertyNames(jsi::Runtime void install_createLLMRunner(jsi::Runtime &rt, jsi::Object &module) { const auto *name = "createLLMRunner"; auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - if (count != 2) { - throw error::InvalidArgument("createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath)"); + if (count < 2 || count > 3) { + throw error::InvalidArgument("createLLMRunner: Usage: createLLMRunner(modelPath, tokenizerPath, modalities?)"); } auto modelPath = conversions::asType(rt, "createLLMRunner: modelPath", args[0]); auto tokenizerPath = conversions::asType(rt, "createLLMRunner: tokenizerPath", args[1]); - auto runnerInstance = std::make_shared(modelPath, tokenizerPath); + std::vector modalities; + if (count > 2 && !args[2].isNull() && !args[2].isUndefined()) { + modalities = conversions::asVector(rt, "createLLMRunner: modalities", args[2]); + } + + auto runnerInstance = std::make_shared(modelPath, tokenizerPath, std::move(modalities)); return jsi::Object::createFromHostObject(rt, runnerInstance); }; diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index a3d4d9ec25..8208de7ae5 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -1,23 +1,26 @@ #pragma once -// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, etc.) -// with `ET_EXPERIMENTAL`, which expands to `[[deprecated("This API is experimental...")]]`. -// We suppress -Wdeprecated-declarations so Clang does not fail builds on ExecuTorch's -// experimental API tags. +// Upstream ExecuTorch annotates experimental LLM APIs (TextLLMRunner, Stats, +// etc.) with `ET_EXPERIMENTAL`, which expands to `[[deprecated("This API is +// experimental...")]]`. We suppress -Wdeprecated-declarations so Clang does not +// fail builds on ExecuTorch's experimental API tags. #ifdef __clang__ #pragma clang diagnostic ignored "-Wdeprecated-declarations" #endif -#include -#include #include #include #include +#include + +#include + +#include namespace rnexecutorch::extensions::llm { /** - * JSI HostObject wrapping an ExecuTorch Text LLM runner instance - * (`executorch::extension::llm::TextLLMRunner`). + * JSI HostObject wrapping an ExecuTorch Multimodal LLM runner instance + * (`executorch::extension::llm::MultimodalRunner`). * * Exposes methods to JavaScript for prefilling prompt context, generating token * stream continuations, interrupting generation, and releasing native model memory. @@ -30,22 +33,27 @@ class LLMRunnerHostObject : public facebook::jsi::HostObject, * * @param modelPath Absolute file system path to the `.pte` LLM model binary. * @param tokenizerPath Absolute file system path to the local tokenizer configuration file (e.g. `tokenizer.json`). + * @param modalities Vector of supported input modality names (e.g. `{"image"}`). * @throws core::error::RnExecuTorchException with code LoadFailed if loading the model or tokenizer fails. */ - LLMRunnerHostObject(const std::string &modelPath, const std::string &tokenizerPath); + LLMRunnerHostObject(const std::string &modelPath, + const std::string &tokenizerPath, + const std::vector &modalities); facebook::jsi::Value get(facebook::jsi::Runtime &rt, const facebook::jsi::PropNameID &name) override; std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; private: - /** Owning pointer to the underlying ExecuTorch TextLLMRunner instance. */ - std::unique_ptr runner_; + /** Owning pointer to the underlying ExecuTorch MultimodalRunner instance. */ + std::unique_ptr runner_; /** Mutex guarding concurrent access to prefill and generation operations. */ std::mutex mutex_; /** File system path to the loaded `.pte` model binary. */ std::string modelPath_; /** File system path to the loaded tokenizer configuration file. */ std::string tokenizerPath_; + /** List of supported non-text input modalities. */ + std::vector modalities_; }; void install_createLLMRunner(facebook::jsi::Runtime &rt, facebook::jsi::Object &module); diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index b8a9d68bf5..c419e63e34 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -1,3 +1,4 @@ +import type { Tensor } from '../../core/tensor'; import { rnexecutorchJsi } from '../../native/bridge'; declare const llmRunnerBrand: unique symbol; @@ -21,35 +22,49 @@ export type GenerationStats = { readonly modelLoadEndMs: number; }; -/** Handle to a native ExecuTorch text LLM runner. */ -export type LLMRunner = { +/** Supported non-text media input objects (e.g. images, audio tensors). */ +export type MediaType = + | { readonly kind: 'image'; readonly image: Tensor } + | { readonly kind: 'audio'; readonly audio: Tensor }; + +/** Supported input modality kinds. */ +export type Modality = MediaType['kind']; + +/** Text or interleaved multimodal prompt input for an LLM runner. */ +export type Prompt = + | string + | readonly (string | Extract)[]; + +/** Handle to a native ExecuTorch LLM runner. */ +export type LLMRunner = { /** Path to the local model file. */ readonly modelPath: string; - /** Path to the local tokenizer configuration file. */ readonly tokenizerPath: string; + /** List of supported non-text input modalities for this runner (e.g. 'image', 'audio'). */ + readonly modalities: readonly M[]; /** Disposes the native LLM runner and releases the loaded model memory. */ dispose(): void; + /** Interrupts and stops any active generation call on this runner. */ + stop(): void; + /** * Prefills the runner with a prompt to build up the KV cache. - * @param prompt The prefill text prompt. + * @param prompt The prefill text or multimodal prompt. */ - prefill(prompt: string): void; - - /** Interrupts and stops any active generation call on this runner. */ - stop(): void; + prefill(prompt: Prompt): void; /** * Generates text continuation from a prompt. - * @param prompt The text prompt to generate continuation for. + * @param prompt The text or multimodal prompt to generate continuation for. * @param config Generation configuration options. * @param onToken Callback function triggered whenever a new token is generated. * @returns Generation performance statistics. */ generate( - prompt: string, + prompt: Prompt, config?: GenerationConfig, onToken?: (token: string) => void ): GenerationStats; @@ -62,12 +77,18 @@ export type LLMRunner = { }; /** - * Creates a native ExecuTorch Text LLM runner instance. + * Creates a native ExecuTorch LLM runner instance. * @param modelPath Path to the local .pte model file. * @param tokenizerPath Path to the local tokenizer configuration file (e.g. tokenizer.json). + * @param modalities List of supported input non-text modalities (e.g. `['image']`). + * Defaults to text-only. * @returns A native LLMRunner instance. */ -export function createLLMRunner(modelPath: string, tokenizerPath: string): LLMRunner { +export function createLLMRunner( + modelPath: string, + tokenizerPath: string, + modalities?: Ms +): LLMRunner { 'worklet'; - return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath) as LLMRunner; + return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath, modalities ?? []); } From bb5b202f42fc5b8224cb27f9072a9961f32b38bc Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 03:18:23 +0200 Subject: [PATCH 11/43] fix(llm): support TextLLMRunner and MultimodalRunner via IRunner dispatch --- .../cpp/extensions/llm/llm_runner.cpp | 111 ++++++++++-------- .../cpp/extensions/llm/llm_runner.h | 10 +- .../extensions/llm/tasks/llmChatSession.ts | 14 ++- .../react-native-executorch/src/models.ts | 42 +++++++ 4 files changed, 117 insertions(+), 60 deletions(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index f3a5b5b260..ef09c07204 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "core/conversions.h" @@ -43,13 +44,17 @@ jsi::Object statsToJSI(jsi::Runtime &rt, const executorch::extension::llm::Stats return obj; } -std::vector parseMultimodalPromptArray( +std::vector parsePrompt( jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, const std::vector &supportedModalities) { - auto arr = conversions::asType(rt, std::format("{}: prompt array", ctx), value); + if (value.isString()) { + return {executorch::extension::llm::MultimodalInput(conversions::asType(rt, ctx, value))}; + } + + auto arr = conversions::asType(rt, ctx, value); size_t len = arr.length(rt); std::vector inputs; @@ -57,50 +62,49 @@ std::vector parseMultimodalPromptAr for (size_t i = 0; i < len; ++i) { auto elem = arr.getValueAtIndex(rt, i); + std::string itemCtx = std::format("{}[{}]", ctx, i); + if (elem.isString()) { - inputs.emplace_back(elem.asString(rt).utf8(rt)); - } else if (elem.isObject()) { - auto mediaObj = elem.asObject(rt); - std::string itemCtx = std::format("{}[{}]", ctx, i); - auto kind = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "kind"); - - if (std::ranges::find(supportedModalities, kind) == supportedModalities.end()) { - throw error::InvalidArgument(std::format("{}: Modality '{}' is not supported " - "by this runner instance", - itemCtx, kind)); - } + inputs.emplace_back(conversions::asType(rt, itemCtx, elem)); + continue; + } + auto mediaObj = conversions::asType(rt, itemCtx, elem); + auto kind = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "kind"); - if (kind == "image") { - auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "image"); - auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"C", "H", "W"}); - auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); - - const auto &shape = tensorHost->shape_; - const auto C = shape[0]; - const auto H = shape[1]; - const auto W = shape[2]; - - std::vector data(tensorHost->numel_); - std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); - inputs.emplace_back(executorch::extension::llm::Image(std::move(data), W, H, C)); - } else if (kind == "audio") { - auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "audio"); - auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"batch", "n_bins", "n_frames"}); - auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); - - const auto &shape = tensorHost->shape_; - const auto batchSize = shape[0]; - const auto nBins = shape[1]; - const auto nFrames = shape[2]; - - std::vector data(tensorHost->numel_); - std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); - inputs.emplace_back(executorch::extension::llm::Audio(std::move(data), batchSize, nBins, nFrames)); - } else { - throw error::InvalidArgument(std::format("{}: Unsupported media kind '{}'", itemCtx, kind)); - } + if (std::ranges::find(supportedModalities, kind) == supportedModalities.end()) { + throw error::InvalidArgument(std::format("{}: Modality '{}' is not supported " + "by this runner instance", + itemCtx, kind)); + } + + if (kind == "image") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "image"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"C", "H", "W"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto C = shape[0]; + const auto H = shape[1]; + const auto W = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Image(std::move(data), W, H, C)); + } else if (kind == "audio") { + auto tensorJs = conversions::getRequiredProperty(rt, itemCtx, mediaObj, "audio"); + auto tensorHost = tensor::fromJs(rt, itemCtx, tensorJs, DType::float32, {"batch", "n_bins", "n_frames"}); + auto tensorLock = tensor::tryLockShared(rt, itemCtx, tensorHost); + + const auto &shape = tensorHost->shape_; + const auto batchSize = shape[0]; + const auto nBins = shape[1]; + const auto nFrames = shape[2]; + + std::vector data(tensorHost->numel_); + std::memcpy(data.data(), tensorHost->data_.get(), tensorHost->numel_ * sizeof(float)); + inputs.emplace_back(executorch::extension::llm::Audio(std::move(data), batchSize, nBins, nFrames)); } else { - throw error::InvalidArgument(std::format("{}: Prompt array elements must be strings or media objects", ctx)); + throw error::InvalidArgument(std::format("{}: Unsupported media kind '{}'", itemCtx, kind)); } } @@ -120,7 +124,12 @@ LLMRunnerHostObject::LLMRunnerHostObject(const std::string &modelPath, throw error::LoadFailed(std::format("LLMRunner: Failed to load runner tokenizer at path: {}", tokenizerPath)); } - runner_ = executorch::extension::llm::create_multimodal_runner(modelPath, std::move(tokenizer)); + if (modalities_.empty()) { + runner_ = executorch::extension::llm::create_text_llm_runner(modelPath, std::move(tokenizer)); + } else { + runner_ = executorch::extension::llm::create_multimodal_runner(modelPath, std::move(tokenizer)); + } + if (!runner_) { throw error::LoadFailed("LLMRunner: Failed to create llm runner"); } @@ -205,12 +214,13 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam } auto genError = executorch::runtime::Error::Ok; - if (args[0].isString()) { - std::string prompt = args[0].asString(rt).utf8(rt); + if (self->modalities_.empty()) { + auto prompt = conversions::asType(rt, "LLMRunner.generate: prompt", args[0]); genError = self->runner_->generate(prompt, config, tokenCallback, statsCallback); } else { - auto inputs = parseMultimodalPromptArray(rt, "LLMRunner.generate", args[0], self->modalities_); - genError = self->runner_->generate(inputs, config, tokenCallback, statsCallback); + auto inputs = parsePrompt(rt, "LLMRunner.generate", args[0], self->modalities_); + auto *multimodalRunner = dynamic_cast(self->runner_.get()); + genError = multimodalRunner->generate(inputs, config, tokenCallback, statsCallback); } if (genError != executorch::runtime::Error::Ok) { @@ -239,9 +249,8 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam throw error::ResourceDisposed("LLMRunner.prefill: Runner has been disposed"); } - auto result = args[0].isString() - ? self->runner_->prefill(args[0].asString(rt).utf8(rt)) - : self->runner_->prefill(parseMultimodalPromptArray(rt, "LLMRunner.prefill", args[0], self->modalities_)); + auto inputs = parsePrompt(rt, "LLMRunner.prefill", args[0], self->modalities_); + auto result = self->runner_->prefill(inputs); if (result.error() != executorch::runtime::Error::Ok) { std::string errorMsg = executorch::runtime::to_string(result.error()); diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index 8208de7ae5..da4dcbb42c 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -15,12 +15,12 @@ #include -#include +#include namespace rnexecutorch::extensions::llm { /** - * JSI HostObject wrapping an ExecuTorch Multimodal LLM runner instance - * (`executorch::extension::llm::MultimodalRunner`). + * JSI HostObject wrapping an ExecuTorch LLM runner instance + * (`executorch::extension::llm::IRunner`). * * Exposes methods to JavaScript for prefilling prompt context, generating token * stream continuations, interrupting generation, and releasing native model memory. @@ -44,8 +44,8 @@ class LLMRunnerHostObject : public facebook::jsi::HostObject, std::vector getPropertyNames(facebook::jsi::Runtime &rt) override; private: - /** Owning pointer to the underlying ExecuTorch MultimodalRunner instance. */ - std::unique_ptr runner_; + /** Owning pointer to the underlying ExecuTorch IRunner instance. */ + std::unique_ptr runner_; /** Mutex guarding concurrent access to prefill and generation operations. */ std::mutex mutex_; /** File system path to the loaded `.pte` model binary. */ diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 03a00dc998..236ed4fcf9 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -7,11 +7,12 @@ import { type LLMRunner, type GenerationConfig, type GenerationStats, + type Modality, } from '../llmRunner'; import { createJinjaChatFormatter } from '../jinja'; import { parseTokenizerConfig } from '../tokenizerConfig'; -export type { GenerationConfig, GenerationStats }; +export type { GenerationConfig, GenerationStats, Modality }; /** Message interface for chat history and inputs. */ export type ChatMessage = { @@ -30,6 +31,7 @@ export type LLMModel = { readonly modelPath: string; readonly tokenizerPath: string; readonly tokenizerConfigPath: string; + readonly modalities?: readonly Modality[]; }; /** Custom generation and state options for an LLM chat session. */ @@ -62,7 +64,7 @@ type SessionState = { }; function generateChatTurn( - nativeRunner: LLMRunner, + nativeRunner: LLMRunner, prompt: string, options: { readonly genConfig: GenerationConfig; @@ -98,7 +100,7 @@ export async function createLLMChatSession( options?: LLMChatSessionOptions, runtime?: WorkletRuntime ): Promise { - const { modelPath, tokenizerPath, tokenizerConfigPath } = config; + const { modelPath, tokenizerPath, tokenizerConfigPath, modalities } = config; const initialMessages = options?.initialMessages ?? []; const defaultGenerationConfig = options?.generationConfig; @@ -112,7 +114,11 @@ export async function createLLMChatSession( const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; const state: SessionState = { history: [] }; - const nativeRunner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath); + const nativeRunner = await wrapAsync(createLLMRunner, runtime)( + modelPath, + tokenizerPath, + modalities + ); const prefill = wrapAsync(nativeRunner.prefill, runtime); for (const msg of initialMessages) { diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index a764aeb83a..c7d6ff80b7 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -852,6 +852,18 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; +const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, + modalities: ['image'], +}; +const LFM2_5_VL_450M_MLX_INT4: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, + modalities: ['image'], +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; @@ -1001,6 +1013,19 @@ const QWEN2_5_3B_XNNPACK_BF16: LLMModel = { tokenizerConfigPath: `${QWEN2_5_BASE_URL}/tokenizer_config.json`, }; +const GEMMA4_BASE_URL = `${BASE_URL}-gemma-4/${VERSION_TAG}`; + +const GEMMA4_E2B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/xnnpack/gemma_4_e2b_xnnpack_8da4w.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; +const GEMMA4_E2B_MLX_INT4: LLMModel = { + modelPath: `${GEMMA4_BASE_URL}/e2b/mlx/gemma4_e2b_mlx_int4.pte`, + tokenizerPath: `${GEMMA4_BASE_URL}/e2b/tokenizer.json`, + tokenizerConfigPath: `${GEMMA4_BASE_URL}/e2b/tokenizer_config.json`, +}; + const QWEN3_BASE_URL = `${BASE_URL}-qwen-3/${VERSION_TAG}`; const QWEN3_0_6B_XNNPACK_8DA4W: LLMModel = { @@ -1522,6 +1547,11 @@ export const models = { XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, MLX_INT4: LFM2_5_350M_MLX_INT4, }, + VL_450M: { + ...LFM2_5_VL_450M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, + MLX_INT4: LFM2_5_VL_450M_MLX_INT4, + }, }, /** * Bielik v3 1.5B Polish and English instruction-tuned language model. @@ -1648,6 +1678,18 @@ export const models = { XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, }, }, + /** + * Google Gemma 4 lightweight generative language model family + * optimized for on-device use. + */ + GEMMA4: { + ...GEMMA4_E2B_XNNPACK_8DA4W, + E2B: { + ...GEMMA4_E2B_XNNPACK_8DA4W, + XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + MLX_INT4: GEMMA4_E2B_MLX_INT4, + }, + }, }, textEmbeddings: { /** From 88affbebd7f08b3189bd7ccafcec3ff47836223e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 03:19:59 +0200 Subject: [PATCH 12/43] feat(apps/nlp): update LLM example app for multimodal testing --- .cspell-wordlist.txt | 3 + apps/nlp/app/index.tsx | 3 + apps/nlp/app/llm/index.tsx | 4 +- apps/nlp/components/ModelPicker.tsx | 197 ++++++++++++++++++++++++++++ apps/nlp/package.json | 6 + 5 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 apps/nlp/components/ModelPicker.tsx diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 964521ab13..33558db948 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -1,4 +1,6 @@ multimodal +subvariant +Subvariant swmansion executorch RNET @@ -314,3 +316,4 @@ Partitioner denoised ttfa TTFA +agentic \ No newline at end of file diff --git a/apps/nlp/app/index.tsx b/apps/nlp/app/index.tsx index 2e0e024076..4020c1956a 100644 --- a/apps/nlp/app/index.tsx +++ b/apps/nlp/app/index.tsx @@ -20,6 +20,9 @@ export default function Home() { router.navigate('privacy-filter/')}> Privacy Filter + router.navigate('llm/')}> + LLM + ); diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 96a656da61..c28739f1cc 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -52,7 +52,9 @@ function getFirstLeafModel(node: any): any { } function LLMContent() { - const [selectedModel, setSelectedModel] = useState(getFirstLeafModel(models.llm)); + const [selectedModel, setSelectedModel] = useState( + models.llm.LFM2_5.VL_450M ?? getFirstLeafModel(models.llm) + ); const [activeModel, setActiveModel] = useState(null); const [forceDownload, setForceDownload] = useState(false); diff --git a/apps/nlp/components/ModelPicker.tsx b/apps/nlp/components/ModelPicker.tsx new file mode 100644 index 0000000000..fa02662f12 --- /dev/null +++ b/apps/nlp/components/ModelPicker.tsx @@ -0,0 +1,197 @@ +import React from 'react'; +import { View, Text, ScrollView, Pressable, StyleSheet } from 'react-native'; + +export type ModelOption = { + label: string; + value: any; + labels?: any; +}; + +interface ModelPickerProps { + label: string; + options: ModelOption[]; + selectedValue: any; + onValueChange: (value: any) => void; +} + +export function ModelPicker({ label, options, selectedValue, onValueChange }: ModelPickerProps) { + return ( + + {label} + + {options.map((option, index) => { + const isSelected = option.value === selectedValue; + return ( + onValueChange(option.value)} + > + + {option.label} + + + ); + })} + + + ); +} + +const styles = StyleSheet.create({ + container: { + width: '100%', + marginBottom: 16, + paddingHorizontal: 4, + }, + label: { + fontSize: 12, + fontWeight: '700', + color: '#666', + textTransform: 'uppercase', + marginBottom: 8, + letterSpacing: 0.5, + }, + scrollContainer: { + flexDirection: 'row', + gap: 8, + }, + chip: { + backgroundColor: '#e0e0e0', + paddingHorizontal: 14, + paddingVertical: 8, + borderRadius: 20, + }, + activeChip: { + backgroundColor: '#000', + }, + chipText: { + fontSize: 13, + color: '#333', + fontWeight: '500', + }, + activeChipText: { + color: '#fff', + fontWeight: '600', + }, +}); + +function hasModelConfig(obj: any): boolean { + if (!obj || typeof obj !== 'object') return false; + if (typeof obj.modelPath === 'string') return true; + for (const key of Object.keys(obj)) { + if (hasModelConfig(obj[key])) return true; + } + return false; +} + +function getSubOptions(node: any): { key: string; value: any }[] { + if (!node || typeof node !== 'object') return []; + return Object.keys(node) + .filter((key) => hasModelConfig(node[key])) + .map((key) => ({ key, value: node[key] })); +} + +export function findPath(node: any, target: any, currentPath: string[] = []): string[] | null { + if (!node || typeof node !== 'object') return null; + + // Search child nodes first to match the deepest leaf node, preventing early returns on parent container objects + for (const key of Object.keys(node)) { + if (hasModelConfig(node[key])) { + const found = findPath(node[key], target, [...currentPath, key]); + if (found) return found; + } + } + + if (node === target) return currentPath; + if (target && typeof node.modelPath === 'string' && node.modelPath === target.modelPath) { + return currentPath; + } + + return null; +} + +function getDefaultPath(node: any, currentPath: string[] = []): string[] { + const subOptions = getSubOptions(node); + if (subOptions.length === 0) return currentPath; + const first = subOptions[0]!; + return getDefaultPath(first.value, [...currentPath, first.key]); +} + +function getValueAtPath(registry: any, path: string[]): any { + let current = registry; + for (const key of path) { + if (current && typeof current === 'object') { + current = current[key]; + } else { + return null; + } + } + return current; +} + +export interface NestedModelPickerProps { + labelPrefix?: string; + registry: any; + selectedValue: any; + onValueChange: (value: any) => void; +} + +export function NestedModelPicker({ + labelPrefix = '', + registry, + selectedValue, + onValueChange, +}: NestedModelPickerProps) { + const path = findPath(registry, selectedValue) || getDefaultPath(registry); + const pickers: React.ReactNode[] = []; + let currentNode = registry; + + for (let i = 0; i <= path.length; i++) { + const subOptions = getSubOptions(currentNode); + if (subOptions.length === 0) break; + + const selectedKey = path[i]; + const options = subOptions.map((opt) => ({ + label: opt.key, + value: opt.key, + })); + + const label = + i === 0 + ? `${labelPrefix ? labelPrefix + ' ' : ''}Family` + : i === 1 + ? `${labelPrefix ? labelPrefix + ' ' : ''}Variant` + : `${labelPrefix ? labelPrefix + ' ' : ''}Subvariant`; + + const levelIndex = i; + pickers.push( + { + const newPath = [...path.slice(0, levelIndex), newKey]; + const newNode = getValueAtPath(registry, newPath); + const leafPath = [...newPath, ...getDefaultPath(newNode)]; + const leafValue = getValueAtPath(registry, leafPath); + onValueChange(leafValue); + }} + /> + ); + + const nextKey = selectedKey || subOptions[0]?.key; + if (nextKey && currentNode[nextKey]) { + currentNode = currentNode[nextKey]; + } else { + break; + } + } + + return {pickers}; +} diff --git a/apps/nlp/package.json b/apps/nlp/package.json index d35e45a6af..55797c4c37 100644 --- a/apps/nlp/package.json +++ b/apps/nlp/package.json @@ -3,6 +3,12 @@ "version": "1.0.0", "main": "expo-router/entry", "react-native-executorch": { + "backends": [ + "xnnpack", + "coreml", + "mlx", + "vulkan" + ], "features": [ "tokenizer", "textEmbeddings", From ecd28e4cac653c78b43a6ae82872e8d7f7482071 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 03:32:55 +0200 Subject: [PATCH 13/43] docs: formatting --- .../cpp/extensions/llm/llm_runner.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h index da4dcbb42c..74e0dbf5b0 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.h @@ -23,18 +23,24 @@ namespace rnexecutorch::extensions::llm { * (`executorch::extension::llm::IRunner`). * * Exposes methods to JavaScript for prefilling prompt context, generating token - * stream continuations, interrupting generation, and releasing native model memory. + * stream continuations, interrupting generation, and releasing native model + * memory. */ class LLMRunnerHostObject : public facebook::jsi::HostObject, public std::enable_shared_from_this { public: /** - * Constructs an LLMRunnerHostObject by loading an ExecuTorch model binary and tokenizer. + * Constructs an LLMRunnerHostObject by loading an ExecuTorch model binary + * and tokenizer. * - * @param modelPath Absolute file system path to the `.pte` LLM model binary. - * @param tokenizerPath Absolute file system path to the local tokenizer configuration file (e.g. `tokenizer.json`). - * @param modalities Vector of supported input modality names (e.g. `{"image"}`). - * @throws core::error::RnExecuTorchException with code LoadFailed if loading the model or tokenizer fails. + * @param modelPath Absolute file system path to the `.pte` LLM model + * binary. + * @param tokenizerPath Absolute file system path to the local tokenizer + * configuration file (e.g. `tokenizer.json`). + * @param modalities Vector of supported input modality names (e.g. + * `{"image"}`). + * @throws core::error::RnExecuTorchException with code LoadFailed if + * loading the model or tokenizer fails. */ LLMRunnerHostObject(const std::string &modelPath, const std::string &tokenizerPath, From dd6a11d76523b70772e3084694a523402a0b2f47 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 04:04:24 +0200 Subject: [PATCH 14/43] fix(llm): use dynamic_cast and null check for MultimodalRunner --- .../react-native-executorch/cpp/extensions/llm/llm_runner.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index ef09c07204..e549d27ffa 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -220,6 +220,9 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam } else { auto inputs = parsePrompt(rt, "LLMRunner.generate", args[0], self->modalities_); auto *multimodalRunner = dynamic_cast(self->runner_.get()); + if (!multimodalRunner) { + throw error::InvalidArgument("LLMRunner.generate: Runner instance is not a multimodal model"); + } genError = multimodalRunner->generate(inputs, config, tokenCallback, statsCallback); } From 422623608f7a8532116e381ed61826b08251b5fa Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 14:06:27 +0200 Subject: [PATCH 15/43] fix --- .../react-native-executorch/cpp/extensions/llm/llm_runner.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index e549d27ffa..b564d652a3 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -220,7 +220,7 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam } else { auto inputs = parsePrompt(rt, "LLMRunner.generate", args[0], self->modalities_); auto *multimodalRunner = dynamic_cast(self->runner_.get()); - if (!multimodalRunner) { + if (multimodalRunner == nullptr) { throw error::InvalidArgument("LLMRunner.generate: Runner instance is not a multimodal model"); } genError = multimodalRunner->generate(inputs, config, tokenCallback, statsCallback); From 4b008a01865847077d30cb72b259b0f4dfb3ef25 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 16:45:37 +0200 Subject: [PATCH 16/43] update models --- .../react-native-executorch/src/extensions/llm/llmRunner.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index c419e63e34..fdcd1503fb 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -23,17 +23,17 @@ export type GenerationStats = { }; /** Supported non-text media input objects (e.g. images, audio tensors). */ -export type MediaType = +export type MediaInput = | { readonly kind: 'image'; readonly image: Tensor } | { readonly kind: 'audio'; readonly audio: Tensor }; /** Supported input modality kinds. */ -export type Modality = MediaType['kind']; +export type Modality = MediaInput['kind']; /** Text or interleaved multimodal prompt input for an LLM runner. */ export type Prompt = | string - | readonly (string | Extract)[]; + | readonly (string | Extract)[]; /** Handle to a native ExecuTorch LLM runner. */ export type LLMRunner = { From b3d17823dd2cdd1dd46c9d0d737f71b091804656 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 16:52:42 +0200 Subject: [PATCH 17/43] tmp fetcher --- .../src/fetcher/fetcher.ts | 471 ++++-------------- 1 file changed, 104 insertions(+), 367 deletions(-) diff --git a/packages/react-native-executorch/src/fetcher/fetcher.ts b/packages/react-native-executorch/src/fetcher/fetcher.ts index 8bbc9f963a..8af28c4108 100644 --- a/packages/react-native-executorch/src/fetcher/fetcher.ts +++ b/packages/react-native-executorch/src/fetcher/fetcher.ts @@ -1,20 +1,11 @@ /* eslint-disable no-bitwise */ import { Platform } from 'react-native'; import RNBlobUtil from 'react-native-blob-util'; -import * as telemetry from './telemetry'; import { RnExecuTorchError } from '../core/error'; const IS_ANDROID = Platform.OS === 'android'; - -// Persistent, per-app directory where downloaded model assets are cached. -// iOS: internal DocumentDir (not CacheDir) so the OS won't evict large models -// between runs and force a costly re-download. -// Android: the app-private EXTERNAL files dir (getExternalFilesDir), so the -// system DownloadManager can write there and same-volume moves stay cheap -// even for multi-GB files. Falls back to DocumentDir if unmounted. -const ANDROID_DIRECTORY = RNBlobUtil.fs.dirs.SDCardDir || RNBlobUtil.fs.dirs.DocumentDir; -const RNE_DIRECTORY = IS_ANDROID - ? `${ANDROID_DIRECTORY}/react-native-executorch` +const BASE_DIR = IS_ANDROID + ? `${RNBlobUtil.fs.dirs.SDCardDir || RNBlobUtil.fs.dirs.DocumentDir}/react-native-executorch` : `${RNBlobUtil.fs.dirs.DocumentDir}/react-native-executorch`; /** @@ -24,10 +15,7 @@ const RNE_DIRECTORY = IS_ANDROID export interface DownloadOptions { /** Called with overall progress in `[0, 1]` as bytes arrive. */ onProgress?: (progress: number) => void; - /** - * Aborts the download. On iOS the bytes fetched so far are kept on disk so a - * later {@link download} of the same source resumes instead of restarting. - */ + /** Aborts the download. */ signal?: AbortSignal; /** * Re-downloads every remote source even when it is already cached, replacing @@ -37,353 +25,92 @@ export interface DownloadOptions { forceDownload?: boolean; } -// djb2 — cheap, dependency-free hash used to derive a stable cache key per URL. const djb2 = (s: string): number => { let h = 5381; - for (let i = 0; i < s.length; i++) { - h = (((h << 5) + h) ^ s.charCodeAt(i)) >>> 0; - } + for (let i = 0; i < s.length; i++) h = (((h << 5) + h) ^ s.charCodeAt(i)) >>> 0; return h; }; const isRemote = (source: string): boolean => /^https?:\/\//.test(source); function cachePathFor(url: string): string { - const urlWithoutQuery = url.split('?')[0]!; - const basename = urlWithoutQuery.split('/').pop() || 'model'; - // Hash guards against basename collisions across different repos/versions. - return `${RNE_DIRECTORY}/${djb2(urlWithoutQuery)}_${basename}`; -} - -async function fileSize(path: string): Promise { - try { - const stat = await RNBlobUtil.fs.stat(path); - return Number(stat.size) || 0; - } catch { - return 0; - } + const cleanUrl = url.split('?')[0]!; + const filename = cleanUrl.split('/').pop() || 'model'; + return `${BASE_DIR}/${djb2(cleanUrl)}_${filename}`; } -// Best-effort remote content length via a HEAD request; 0 when unknown. -async function remoteSize(url: string): Promise { +async function isDownloaded(destPath: string): Promise { try { - const res = await fetch(url, { method: 'HEAD' }); - const len = res.headers.get('content-length'); - return len ? Number(len) : 0; + const stat = await RNBlobUtil.fs.stat(destPath); + return Number(stat.size) > 0; } catch { - return 0; + return false; } } -// Raised when a download is cancelled through its `signal`. Internal: callers -// match on the DOWNLOAD_ABORTED code via isRnExecuTorchError. -const abortError = () => RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); - -type OnBytes = (received: number, total: number) => void; - -interface DownloadUrlCallbacks { - // Reports absolute received/total bytes for this file, including any bytes - // already present from a previous partial download. - onBytes?: OnBytes; - signal?: AbortSignal; - forceDownload?: boolean; -} - -// A download that is currently running, shared by every caller that asked for -// the same URL while it was in flight. -interface InFlightDownload { - promise: Promise; - // Progress is fanned out to every joined caller. - listeners: Set; - // Drives the underlying request; aborted only once every caller has left. - controller: AbortController; - callers: number; -} - -const inFlight = new Map(); - -// Downloads a single remote file into the cache, dispatching to the -// platform-appropriate backend. Returns the local path. -// -// Concurrent calls for the same URL share one request: without this, both would -// miss the cache check, double-count the download in telemetry, and write the -// same temporary file — on Android the second one's opening `unlink` would -// delete the first one's partially downloaded data. -async function downloadUrl(url: string, cb: DownloadUrlCallbacks): Promise { - if (cb.signal?.aborted) throw abortError(); - - const dest = cachePathFor(url); - - if (cb.forceDownload) { - // Drop the cached copy so the checks below fall through to a real fetch. - // A download already in flight hasn't written `dest` yet, so joining it - // still yields freshly fetched bytes. - await RNBlobUtil.fs.unlink(dest).catch(() => {}); - } else if (await RNBlobUtil.fs.exists(dest)) { - // Cache hit — nothing to download. - const size = await fileSize(dest); - cb.onBytes?.(size, size); - return dest; - } - - // Skip an entry whose last caller just left: it is already unwinding, so - // joining it would hand this caller someone else's cancellation. - const existing = inFlight.get(url); - if (existing && !existing.controller.signal.aborted) return joinDownload(existing, cb); - - const entry: InFlightDownload = { - promise: Promise.resolve(''), - listeners: new Set(), - controller: new AbortController(), - callers: 0, - }; - entry.promise = startDownload(url, dest, entry).finally(() => { - inFlight.delete(url); - }); - inFlight.set(url, entry); - - return joinDownload(entry, cb); -} - -// Runs the actual request, fanning progress out to everyone who joined. -async function startDownload(url: string, dest: string, entry: InFlightDownload): Promise { - // Count this actual (non-cached) fetch once, no matter how many callers share it. - telemetry.triggerHuggingFaceDownloadCounter(url); - telemetry.triggerDownloadEvent(url); - - await RNBlobUtil.fs.mkdir(RNE_DIRECTORY).catch(() => {}); - - const cb: DownloadUrlCallbacks = { - signal: entry.controller.signal, - onBytes: (received, total) => { - for (const listener of entry.listeners) listener(received, total); - }, - }; - - return IS_ANDROID - ? downloadUrlViaAndroidDownloadManager(url, dest, cb) - : downloadUrlViaIosStream(url, dest, cb); -} - -// Attaches one caller to a shared download. The caller's own signal only -// detaches it — the request itself is cancelled once the last caller leaves. -function joinDownload(entry: InFlightDownload, cb: DownloadUrlCallbacks): Promise { - entry.callers += 1; - if (cb.onBytes) entry.listeners.add(cb.onBytes); - - return new Promise((resolve, reject) => { - const onAbort = () => { - if (cb.onBytes) entry.listeners.delete(cb.onBytes); - entry.callers -= 1; - if (entry.callers === 0) entry.controller.abort(); - reject(abortError()); - }; - cb.signal?.addEventListener('abort', onAbort); - - entry.promise.then(resolve, reject).finally(() => { - cb.signal?.removeEventListener('abort', onAbort); - }); - }); -} - -// Android backend: the system DownloadManager streams to app-private external -// storage. Unlike blob-util's in-process reader it handles files larger than -// 2 GB, keeps downloading while the app is in the background or killed, and -// resumes across transient network drops on its own — so no manual Range logic. -async function downloadUrlViaAndroidDownloadManager( +// Download a single file URL directly to disk +async function downloadFile( url: string, - dest: string, - cb: DownloadUrlCallbacks + destPath: string, + onBytes?: (recv: number, tot: number) => void, + signal?: AbortSignal ): Promise { - const tmp = `${dest}.downloading`; - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); + if (signal?.aborted) throw RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); - if (cb.signal?.aborted) throw abortError(); - - const task = RNBlobUtil.config({ - addAndroidDownloads: { - useDownloadManager: true, - path: tmp, - notification: false, - mediaScannable: false, - mime: 'application/octet-stream', - }, - }).fetch('GET', url); + await RNBlobUtil.fs.mkdir(BASE_DIR).catch(() => {}); + const tmpPath = `${destPath}.tmp`; + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); + const task = RNBlobUtil.config({ path: tmpPath, fileCache: true }).fetch('GET', url); const onAbort = () => task.cancel(); - cb.signal?.addEventListener('abort', onAbort); + signal?.addEventListener('abort', onAbort); - // DownloadManager reports total as -1 until the size is known; forward the - // received byte count regardless so byte-weighted progress still advances. - task.progress({ count: 100 }, (received, total) => { + task.progress({ count: 50 }, (received, total) => { const recv = Number(received); const tot = Number(total); - cb.onBytes?.(recv, tot > 0 ? tot : recv); + if (recv > 0) onBytes?.(recv, tot > 0 ? tot : 0); }); - try { - await task; - } catch (e) { - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); - throw cb.signal?.aborted ? abortError() : e; - } finally { - cb.signal?.removeEventListener('abort', onAbort); - } - - // DownloadManager doesn't surface an HTTP status; an empty file means failure. - const size = await fileSize(tmp); - if (size <= 0) { - await RNBlobUtil.fs.unlink(tmp).catch(() => {}); - throw RnExecuTorchError('DOWNLOAD_FAILED', `Download of ${url} failed (empty response).`); - } - await RNBlobUtil.fs.mv(tmp, dest); - return dest; -} - -// iOS backend: blob-util streams via the iOS URL session straight to disk. -// Interrupted downloads resume from a `.partial` file via an HTTP Range request. -// `canResume` is set to `false` on an internal retry to avoid recursing forever -// if partial-file assembly ever fails. -async function downloadUrlViaIosStream( - url: string, - dest: string, - cb: DownloadUrlCallbacks, - canResume = true -): Promise { - const part = `${dest}.partial`; - if (!canResume) await RNBlobUtil.fs.unlink(part).catch(() => {}); - const offset = canResume ? await fileSize(part) : 0; - - // Resumed byte ranges land in a separate chunk file that we append onto the - // partial; fresh downloads stream straight into the partial. - const target = offset > 0 ? `${dest}.chunk` : part; - await RNBlobUtil.fs.unlink(target).catch(() => {}); // clear any stale chunk - - const headers: Record = {}; - if (offset > 0) headers.Range = `bytes=${offset}-`; - - if (cb.signal?.aborted) throw abortError(); - - const task = RNBlobUtil.config({ path: target, fileCache: true }).fetch('GET', url, headers); - const onAbort = () => task.cancel(); - cb.signal?.addEventListener('abort', onAbort); - - task.progress({ count: 20 }, (received, total) => { - const recv = Number(received); - const tot = Number(total); - cb.onBytes?.(offset + recv, offset + tot); - }); - - let status: number; try { const res = await task; - status = res.info().status; - } catch (e) { - // Network drop / cancel. Keep the fresh partial for a future resume, but - // discard a resumed chunk — its offset assumptions may not hold. - if (offset > 0) await RNBlobUtil.fs.unlink(target).catch(() => {}); - throw cb.signal?.aborted ? abortError() : e; - } finally { - cb.signal?.removeEventListener('abort', onAbort); - } - - try { - if (status === 416) { - // Range not satisfiable — the partial already holds the whole file. - await RNBlobUtil.fs.unlink(target).catch(() => {}); - } else if (status >= 400) { - await RNBlobUtil.fs.unlink(target).catch(() => {}); + if (res.info().status >= 400 || !(await isDownloaded(tmpPath))) { + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); throw RnExecuTorchError( 'DOWNLOAD_FAILED', - `Download of ${url} failed with HTTP status ${status}.` + `Download of ${url} failed with status ${res.info().status}.` ); - } else if (offset > 0) { - if (status === 206) { - // Server honored the range: append the new bytes onto the partial. - await RNBlobUtil.fs.appendFile(part, `file://${target}`, 'uri'); - await RNBlobUtil.fs.unlink(target).catch(() => {}); - } else { - // Server ignored the range (200) and re-sent the whole file: replace. - await RNBlobUtil.fs.unlink(part).catch(() => {}); - await RNBlobUtil.fs.mv(target, part); - } } - await RNBlobUtil.fs.mv(part, dest); - return dest; - } catch (assemblyErr) { - // A `4xx` we deliberately threw must propagate as-is. - if (status >= 400 && status !== 416) throw assemblyErr; - // Assembling the partial failed (e.g. append unsupported). Wipe and retry - // once as a plain full download so correctness never depends on resume. - await RNBlobUtil.fs.unlink(part).catch(() => {}); - await RNBlobUtil.fs.unlink(target).catch(() => {}); - if (canResume) return downloadUrlViaIosStream(url, dest, cb, false); - throw assemblyErr; + await RNBlobUtil.fs.mv(tmpPath, destPath); + return destPath; + } catch (err) { + await RNBlobUtil.fs.unlink(tmpPath).catch(() => {}); + if (signal?.aborted) throw RnExecuTorchError('DOWNLOAD_ABORTED', 'The download was aborted.'); + throw err; + } finally { + signal?.removeEventListener('abort', onAbort); } } -// Plain data containers we recurse into. Anything else (numbers, functions, -// typed arrays, class instances) is left alone. -function isPlainObject(value: unknown): value is Record { - if (typeof value !== 'object' || value === null) return false; - const proto = Object.getPrototypeOf(value); - return proto === Object.prototype || proto === null; -} - -/** - * Walks a value and collects every distinct remote URL sitting on a string - * leaf. Deduplicating here means a URL repeated across fields is fetched once. - * @param node The value to walk. - * @returns The set of remote URLs referenced by `node`. - */ -function collectRemoteSources(node: unknown): Set { - const out = new Set(); - - const visit = (current: unknown): void => { - if (typeof current === 'string') { - if (isRemote(current)) out.add(current); - } else if (Array.isArray(current)) { - for (const item of current) visit(item); - } else if (isPlainObject(current)) { - for (const value of Object.values(current)) visit(value); - } - }; - - visit(node); +// Collect all remote URLs inside a string, array, or nested config object +function collectUrls(node: unknown, out = new Set()): Set { + if (typeof node === 'string') { + if (isRemote(node)) out.add(node); + } else if (Array.isArray(node)) { + for (const item of node) collectUrls(item, out); + } else if (typeof node === 'object' && node !== null) { + for (const val of Object.values(node)) collectUrls(val, out); + } return out; } -/** - * Rebuilds a value with every resolved URL replaced by its local path. - * Branches containing no resolved URL keep their original reference, so an - * untouched config comes back as-is and stays stable across React renders. - * @typeParam T The shape of the value being rewritten. - * @param node The value to rewrite. - * @param resolved Map of remote URL to downloaded local path. - * @returns `node` with resolved URLs swapped for local paths. - */ -function substituteRemoteSources(node: T, resolved: ReadonlyMap): T { - if (typeof node === 'string') { - return (resolved.get(node) ?? node) as T; - } - if (Array.isArray(node)) { - let changed = false; - const next = node.map((item) => { - const mapped = substituteRemoteSources(item, resolved); - changed ||= mapped !== item; - return mapped; - }); - return (changed ? next : node) as T; - } - if (isPlainObject(node)) { - let changed = false; +// Replace remote URLs in config object with local downloaded file paths +function replaceUrls(node: T, resolved: Map): T { + if (typeof node === 'string') return (resolved.get(node) ?? node) as T; + if (Array.isArray(node)) return node.map((item) => replaceUrls(item, resolved)) as T; + if (typeof node === 'object' && node !== null) { const next: Record = {}; - for (const [key, value] of Object.entries(node)) { - const mapped = substituteRemoteSources(value, resolved); - changed ||= mapped !== value; - next[key] = mapped; - } - return (changed ? next : node) as T; + for (const [k, v] of Object.entries(node)) next[k] = replaceUrls(v, resolved); + return next as T; } return node; } @@ -392,23 +119,6 @@ function substituteRemoteSources(node: T, resolved: ReadonlyMap` factory: - * - * ```ts - * const model = await download(models.classification.EFFICIENTNET_V2_S.XNNPACK_FP32); - * const { classify, dispose } = await createClassifier(model); - * ``` - * - * Downloads go to a persistent cache: on Android via the system DownloadManager - * (handles multi-GB files and continues in the background), on iOS via a - * streaming request that resumes an interrupted download from where it stopped. - * When a config references several files, overall progress is weighted by their - * byte sizes so a large model isn't reported the same as a tiny tokenizer. * @category Utils * @typeParam T The shape of the value being resolved. * @param source A URL, a local path, or any nested object/array holding them. @@ -416,44 +126,71 @@ function substituteRemoteSources(node: T, resolved: ReadonlyMap(source: T, options: DownloadOptions = {}): Promise { - const urls = [...collectRemoteSources(source)]; - - // Nothing to fetch — every source is already a local path. + const urls = [...collectUrls(source)]; if (urls.length === 0) { options.onProgress?.(1); return source; } - // Weight overall progress by real byte sizes so a 1 GB model isn't treated - // like a 1 MB tokenizer. When any HEAD fails we fall back to equal weighting. - const sizes = await Promise.all(urls.map(remoteSize)); - const haveAllSizes = sizes.every((size) => size > 0); - const total = haveAllSizes ? sizes.reduce((a, b) => a + b, 0) : urls.length; - + const resolved = new Map(); const received = new Array(urls.length).fill(0); - const report = () => { + const totals = new Array(urls.length).fill(0); + + // Fast-path: pre-fill cached files + for (let i = 0; i < urls.length; i++) { + const url = urls[i]!; + const destPath = cachePathFor(url); + if (!options.forceDownload && (await isDownloaded(destPath))) { + const stat = await RNBlobUtil.fs.stat(destPath); + const sz = Number(stat.size); + received[i] = sz; + totals[i] = sz; + resolved.set(url, destPath); + } + } + + // If all files are cached, return immediately with 100% progress + if (resolved.size === urls.length) { + options.onProgress?.(1); + return replaceUrls(source, resolved); + } + + const updateProgress = () => { if (!options.onProgress) return; - const sum = received.reduce((a, b) => a + b, 0); - options.onProgress(total > 0 ? Math.min(sum / total, 1) : 0); + const sumReceived = received.reduce((a, b) => a + b, 0); + const sumTotals = totals.reduce((a, b) => a + b, 0); + options.onProgress(sumTotals > 0 ? Math.min(sumReceived / sumTotals, 1) : 0); }; - const resolved = new Map(); + // Initial progress update for any pre-cached files in bundle + updateProgress(); + + // Download uncached files concurrently await Promise.all( - urls.map((url, i) => - // Telemetry fires inside downloadUrl, only for genuine (non-cached) fetches. - downloadUrl(url, { - signal: options.signal, - forceDownload: options.forceDownload, - onBytes: (recv, tot) => { - received[i] = haveAllSizes ? recv : tot > 0 ? recv / tot : 0; - report(); + urls.map(async (url, i) => { + if (resolved.has(url)) return; + + const destPath = cachePathFor(url); + const downloadedPath = await downloadFile( + url, + destPath, + (recv, tot) => { + received[i] = recv; + if (tot > 0) totals[i] = tot; + updateProgress(); }, - }).then((path) => { - resolved.set(url, path); - }) - ) + options.signal + ); + + const stat = await RNBlobUtil.fs.stat(downloadedPath); + const finalSize = Number(stat.size); + received[i] = finalSize; + totals[i] = finalSize; + resolved.set(url, downloadedPath); + updateProgress(); + }) ); options.onProgress?.(1); - return substituteRemoteSources(source, resolved); + return replaceUrls(source, resolved); } From 59335d67de812497b225ace7cee99d0cf2cc896c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 16:53:07 +0200 Subject: [PATCH 18/43] update models --- packages/react-native-executorch/src/models.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index c7d6ff80b7..7b30250573 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -864,6 +864,12 @@ const LFM2_5_VL_450M_MLX_INT4: LLMModel = { tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], }; +const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { + modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, + tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, + tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, + modalities: ['image'], +}; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; @@ -1552,6 +1558,10 @@ export const models = { XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, MLX_INT4: LFM2_5_VL_450M_MLX_INT4, }, + VL_1_6B: { + ...LFM2_5_VL_1_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, + }, }, /** * Bielik v3 1.5B Polish and English instruction-tuned language model. From f602ab4956c2bad42655b42ae81983350eeebe5d Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 17:07:08 +0200 Subject: [PATCH 19/43] update --- .../react-native-executorch/src/extensions/llm/llmRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index fdcd1503fb..d32d347e39 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -27,7 +27,7 @@ export type MediaInput = | { readonly kind: 'image'; readonly image: Tensor } | { readonly kind: 'audio'; readonly audio: Tensor }; -/** Supported input modality kinds. */ +/** Supported non-text input modality kinds. */ export type Modality = MediaInput['kind']; /** Text or interleaved multimodal prompt input for an LLM runner. */ From c77a089947fa2d2e7ccb892c5af1d2deb46ac194 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 21:55:13 +0200 Subject: [PATCH 20/43] initial working multimodal chat --- apps/nlp/app/llm/index.tsx | 192 ++++++++++++++++-- apps/nlp/package.json | 3 + .../src/extensions/llm/chatRenderer.ts | 153 ++++++++++++++ .../src/extensions/llm/index.ts | 2 +- .../src/extensions/llm/jinja.ts | 37 ---- .../extensions/llm/tasks/llmChatSession.ts | 144 +++++++------ .../src/hooks/useLLMChatSession.ts | 7 +- packages/react-native-executorch/src/index.ts | 1 + .../react-native-executorch/src/models.ts | 39 +++- yarn.lock | 3 + 10 files changed, 448 insertions(+), 133 deletions(-) create mode 100644 packages/react-native-executorch/src/extensions/llm/chatRenderer.ts delete mode 100644 packages/react-native-executorch/src/extensions/llm/jinja.ts diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index c28739f1cc..79a7b85e08 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -9,22 +9,31 @@ import { ActivityIndicator, KeyboardAvoidingView, Platform, + Alert, + Image as RNImage, } from 'react-native'; +import * as ImagePicker from 'expo-image-picker'; +import { Skia } from '@shopify/react-native-skia'; import { useLLMChatSession, models, type ChatMessage, type GenerationStats, + type ImageBuffer, } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { NestedModelPicker, findPath } from '../../components/ModelPicker'; -const SYSTEM_PROMPT = - "You are a pirate. You must start every response with 'Ahoy matey!' and speak like a pirate."; +const SYSTEM_PROMPT = 'You are a helpful multimodal assistant by Liquid AI.'; const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; -type Turn = { role: 'user' | 'assistant'; content: string; stats?: GenerationStats }; +type Turn = { + role: 'user' | 'assistant'; + content: string; + imageUri?: string; + stats?: GenerationStats; +}; function formatStats(stats: GenerationStats): string { const decodeMs = stats.inferenceEndMs - stats.firstTokenMs; @@ -94,21 +103,88 @@ function LLMContent() { setActiveModel(selectedModel); }; + const [attachedImage, setAttachedImage] = useState<{ + uri: string; + buffer: ImageBuffer; + name: string; + } | null>(null); + + const handlePickGalleryImage = async () => { + try { + const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync(); + if (!permissionResult.granted) { + Alert.alert('Permission Required', 'Permission to access photo gallery is required!'); + return; + } + + const pickerResult = await ImagePicker.launchImageLibraryAsync({ + mediaTypes: ['images'], + allowsEditing: false, + quality: 1, + base64: true, + }); + + if (pickerResult.canceled || !pickerResult.assets[0]?.base64) return; + const asset = pickerResult.assets[0]; + + const skData = Skia.Data.fromBase64(asset.base64!); + const skImage = Skia.Image.MakeImageFromEncoded(skData); + if (!skImage) { + Alert.alert('Error', 'Failed to decode selected image.'); + return; + } + + const pixels = skImage.readPixels(); + if (!pixels || !(pixels instanceof Uint8Array)) { + Alert.alert('Error', 'Failed to read image pixel data.'); + return; + } + + const buffer: ImageBuffer = { + data: pixels, + width: skImage.width(), + height: skImage.height(), + format: 'rgba', + layout: 'hwc', + }; + + setAttachedImage({ + uri: asset.uri, + buffer, + name: asset.fileName || `Photo (${skImage.width()}×${skImage.height()})`, + }); + } catch (err: any) { + Alert.alert('Error', err?.message || 'Failed to select gallery image'); + } + }; + const handleSend = async () => { - const message = input.trim(); - if (!message || !sendMessage || isGenerating) return; + const textMessage = input.trim(); + if ((!textMessage && !attachedImage) || !sendMessage || isGenerating) return; + const currentImage = attachedImage; + setAttachedImage(null); setInput(''); setStreamingResponse(''); - setTurns((prev) => [...prev, { role: 'user', content: message }]); + + const turnUserMessage = textMessage || '[Image Attached]'; + setTurns((prev) => [ + ...prev, + { role: 'user', content: turnUserMessage, imageUri: currentImage?.uri }, + ]); try { - const { response, stats } = await sendMessage(message, (token) => { + const payload = currentImage + ? ([ + { kind: 'image' as const, image: currentImage.buffer }, + textMessage || 'What is in this image?', + ] as const) + : textMessage; + + const { response, stats } = await sendMessage(payload as any, (token) => { setStreamingResponse((prev) => (prev !== null ? prev + token : token)); }); - setTurns((prev) => [...prev, { role: 'assistant', content: response, stats }]); - } catch (e: any) { - setTurns((prev) => [...prev, { role: 'assistant', content: `[Error] ${e?.message}` }]); + setTurns((prev) => [...prev, { role: 'assistant', content: response as string, stats }]); } finally { setStreamingResponse(null); } @@ -197,6 +273,9 @@ function LLMContent() { turn.role === 'user' ? styles.userBubble : styles.assistantBubble, ]} > + {turn.imageUri && ( + + )} {turn.content || '…'} @@ -222,10 +301,35 @@ function LLMContent() { )} + {attachedImage && ( + + + + + {attachedImage.name} + + Image Attached + + setAttachedImage(null)} + > + ✕ + + + )} + + + 🖼 Gallery + ) : ( Send @@ -446,4 +553,63 @@ const styles = StyleSheet.create({ fontSize: 13, fontWeight: '500', }, + galleryButton: { + paddingHorizontal: 12, + height: 44, + backgroundColor: '#e9ecef', + borderRadius: 20, + justifyContent: 'center', + alignItems: 'center', + }, + galleryButtonText: { + fontSize: 13, + fontWeight: '600', + color: '#495057', + }, + attachmentBar: { + flexDirection: 'row', + alignItems: 'center', + paddingHorizontal: 16, + paddingVertical: 8, + backgroundColor: '#fff', + borderTopWidth: 1, + borderTopColor: '#e9ecef', + gap: 12, + }, + attachmentPreview: { + width: 44, + height: 44, + borderRadius: 8, + backgroundColor: '#e9ecef', + }, + attachmentInfo: { + flex: 1, + }, + attachmentName: { + fontSize: 14, + fontWeight: '600', + color: '#212529', + }, + attachmentSub: { + fontSize: 12, + color: '#868e96', + }, + removeAttachmentButton: { + paddingHorizontal: 10, + paddingVertical: 4, + borderRadius: 12, + backgroundColor: '#f1f3f5', + }, + removeAttachmentText: { + fontSize: 14, + fontWeight: '700', + color: '#868e96', + }, + turnThumbnail: { + width: 160, + height: 120, + borderRadius: 8, + marginBottom: 8, + backgroundColor: '#e9ecef', + }, }); diff --git a/apps/nlp/package.json b/apps/nlp/package.json index 55797c4c37..50394f8c65 100644 --- a/apps/nlp/package.json +++ b/apps/nlp/package.json @@ -27,9 +27,12 @@ "dependencies": { "@react-navigation/drawer": "^7.9.4", "@react-navigation/native": "^7.2.2", + "@shopify/react-native-skia": "2.6.2", "expo": "~56.0.9", "expo-build-properties": "~56.0.17", "expo-constants": "~56.0.17", + "expo-image-manipulator": "~56.0.18", + "expo-image-picker": "~56.0.18", "expo-linking": "~56.0.13", "expo-router": "~56.2.9", "react": "19.2.3", diff --git a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts new file mode 100644 index 0000000000..5c148955b8 --- /dev/null +++ b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts @@ -0,0 +1,153 @@ +import { Template } from '@huggingface/jinja'; + +import { tensor, type Tensor } from '../../core/tensor'; +import { RnExecuTorchError } from '../../core/error'; +import type { ImageBuffer } from '../cv'; +import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/tasks/preprocessing'; +import type { Modality, Prompt, MediaInput } from './llmRunner'; + +/** High-level media payload types for chat messages. */ +export type ChatMediaInput = + | { readonly kind: 'image'; readonly image: ImageBuffer } + | { readonly kind: 'audio'; readonly audio: Float32Array }; + +/** Interleaved multimodal content for high-level ChatMessages. */ +export type ChatMessageContent = + | string + | readonly (string | Extract)[]; + +/** Message interface for chat history and inputs. */ +export type ChatMessage = { + readonly role: 'system' | 'user' | 'assistant'; + readonly content: ChatMessageContent; +}; + +export type LLMMediaPreprocessorConfig = { + image?: { + readonly token: { readonly start: string; readonly end: string }; + readonly preprocessorOpts: ImagePreprocessorOptions; + readonly targetShape: readonly [number, number, number]; + }; + audio?: { + readonly token: { readonly start: string; readonly end: string }; + }; +}; + +/** Options for instantiating a ChatRenderer. */ +export type ChatRendererConfig = { + readonly chatTemplate: string; + readonly bosToken?: string; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; +}; + +/** Turn rendering options for ChatRenderer. */ +export type RenderOpts = { + readonly isFirst?: boolean; + readonly addGenPrompt?: boolean; +}; + +/** + * Handles Jinja template rendering and media tensor preprocessing for chat turns. + * @param config Chat renderer configuration options including template and preprocessor settings. + * @returns A ChatRenderer object with render and dispose methods. + */ +export function createChatRenderer(config: ChatRendererConfig) { + const { chatTemplate, preprocessorConfig, bosToken = '' } = config; + const template = new Template(chatTemplate); + + let imgPreprocessor: ReturnType | undefined; + let imgShape: [number, number, number] | undefined; + + if (preprocessorConfig !== undefined && preprocessorConfig.image !== undefined) { + imgShape = [...preprocessorConfig.image.targetShape]; + imgPreprocessor = createImagePreprocessor(preprocessorConfig.image.preprocessorOpts, imgShape); + } + + const tensors: Tensor[] = []; + + const dispose = () => { + tensors.forEach((t) => t.dispose()); + imgPreprocessor?.dispose(); + }; + + const render = (message: ChatMessage, renderOpts?: RenderOpts): Prompt => { + const isFirst = renderOpts?.isFirst ?? false; + const addGenPrompt = renderOpts?.addGenPrompt ?? false; + + if (typeof message.content === 'string') { + /* eslint-disable camelcase */ + return template.render({ + bos_token: isFirst ? bosToken : '', + add_generation_prompt: addGenPrompt, + messages: [{ role: message.role, content: message.content }], + }); + /* eslint-enable */ + } + + const mediaInputs: MediaInput[] = []; + let syntheticContent = ''; + + for (const item of message.content) { + if (typeof item === 'string') { + syntheticContent += item; + continue; + } + + if (!preprocessorConfig || !(item.kind in preprocessorConfig)) { + throw RnExecuTorchError('INVALID_ARGUMENT', `Modality '${item.kind}' not supported`); + } + + if (item.kind === 'image' && 'image' in item) { + if (!preprocessorConfig?.image || !imgPreprocessor || !imgShape) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + 'Received image input but no image preprocessorConfig was provided.' + ); + } + const tokenStart = preprocessorConfig.image.token.start; + const tokenEnd = preprocessorConfig.image.token.end; + const index = mediaInputs.length; + + syntheticContent += `${tokenStart}\uFFFC__ET_MEDIA_${index}__${tokenEnd}`; + + const tImage = tensor('float32', imgShape); + tensors.push(tImage); + imgPreprocessor.process(item.image).copyTo(tImage); + + mediaInputs.push({ kind: 'image', image: tImage }); + } + + if (item.kind === 'audio' && 'audio' in item) { + throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio input not yet supported'); + } + } + + /* eslint-disable camelcase */ + const renderedContent = template.render({ + bos_token: isFirst ? bosToken : '', + add_generation_prompt: addGenPrompt, + messages: [{ role: message.role, content: syntheticContent }], + }); + /* eslint-enable */ + + const regex = /\uFFFC__ET_MEDIA_(\d+)__/g; + const prompt: (string | MediaInput)[] = []; + + let match: RegExpExecArray | null; + let lastIndex = 0; + while ((match = regex.exec(renderedContent)) !== null) { + const textChunk = renderedContent.slice(lastIndex, match.index); + if (textChunk.length > 0) prompt.push(textChunk); + + prompt.push(mediaInputs[parseInt(match[1]!, 10)]!); + lastIndex = regex.lastIndex; + } + + const tail = renderedContent.slice(lastIndex); + if (tail.length > 0) prompt.push(tail); + + return prompt as Prompt; + }; + + return { dispose, render }; +} diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts index 5d5ce8f45e..37967c90cc 100644 --- a/packages/react-native-executorch/src/extensions/llm/index.ts +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -1,3 +1,3 @@ export * from './llmRunner'; -export * from './jinja'; +export * from './chatRenderer'; export * from './tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/jinja.ts b/packages/react-native-executorch/src/extensions/llm/jinja.ts deleted file mode 100644 index 615ccf3f4c..0000000000 --- a/packages/react-native-executorch/src/extensions/llm/jinja.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Template } from '@huggingface/jinja'; - -import type { ChatFormatter } from './tasks/llmChatSession'; - -/** Configuration options for the Jinja chat formatter. */ -export type JinjaFormatterOptions = { - readonly bosToken?: string; - readonly extraContext?: Record; -}; - -/** - * Creates a chat formatter function that renders messages using a Jinja template. - * @param chatTemplate The Jinja template string (e.g. from tokenizer_config.json). - * @param options Jinja formatter options. - * @returns A ChatFormatter function. - */ -export function createJinjaChatFormatter( - chatTemplate: string, - options: JinjaFormatterOptions = {} -): ChatFormatter { - const { bosToken = '', extraContext } = options; - const template = new Template(chatTemplate); - - return (message, { isFirst }) => { - const isGenerationPrompt = message.role === 'assistant' && message.content === ''; - return template.render({ - // Only the first prefill of a conversation should carry the BOS token; - // later turns append to the model's existing KV cache. - // eslint-disable-next-line camelcase - bos_token: isFirst ? bosToken : '', - // eslint-disable-next-line camelcase - add_generation_prompt: isGenerationPrompt, - messages: isGenerationPrompt ? [] : [{ role: message.role, content: message.content }], - ...extraContext, - }); - }; -} diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 236ed4fcf9..f657cc9913 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -8,70 +8,70 @@ import { type GenerationConfig, type GenerationStats, type Modality, + type Prompt, } from '../llmRunner'; -import { createJinjaChatFormatter } from '../jinja'; import { parseTokenizerConfig } from '../tokenizerConfig'; - -export type { GenerationConfig, GenerationStats, Modality }; - -/** Message interface for chat history and inputs. */ -export type ChatMessage = { - readonly role: 'system' | 'user' | 'assistant'; - readonly content: string; +import { + createChatRenderer, + type ChatMediaInput, + type ChatMessageContent, + type ChatMessage, + type LLMMediaPreprocessorConfig, + type ChatRendererConfig, + type RenderOpts, +} from '../chatRenderer'; + +export type { + GenerationConfig, + GenerationStats, + Modality, + ChatMediaInput, + ChatMessageContent, + ChatMessage, + LLMMediaPreprocessorConfig, + ChatRendererConfig as ChatRendererOpts, + RenderOpts, }; -/** Interface for converting a ChatMessage history turn into raw prompt text. */ -export type ChatFormatter = ( - message: ChatMessage, - options: { readonly isFirst: boolean } -) => string; - -/** Model path configuration for LLM chat. */ -export type LLMModel = { +export type LLMModel = { readonly modelPath: string; readonly tokenizerPath: string; readonly tokenizerConfigPath: string; - readonly modalities?: readonly Modality[]; + readonly modalities?: readonly M[]; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; -/** Custom generation and state options for an LLM chat session. */ -export type LLMChatSessionOptions = { - readonly initialMessages?: readonly ChatMessage[]; +export type LLMChatSessionOptions = { + readonly initialMessages?: readonly ChatMessage[]; readonly generationConfig?: GenerationConfig; readonly stopTokens?: readonly string[]; }; -/** Return wrapper holding generated response text and performance stats. */ -export type GenerationResult = { - readonly response: string; +export type LLMGenerationResult = { + readonly response: ChatMessageContent; readonly stats: GenerationStats; }; -/** Orchestrator interface for active LLM chat sessions. */ -export type LLMChatSession = { +export type LLMChatSession = { dispose(): void; sendMessage( - message: string, + message: ChatMessageContent, onToken?: (token: string) => void, genConfig?: GenerationConfig - ): Promise; - getHistory(): readonly ChatMessage[]; + ): Promise>; + getHistory(): readonly ChatMessage[]; stop(): void; }; -type SessionState = { - history: ChatMessage[]; -}; - -function generateChatTurn( - nativeRunner: LLMRunner, - prompt: string, +function generateChatTurnWorklet( + runner: LLMRunner, + prompt: Prompt, options: { readonly genConfig: GenerationConfig; readonly stopTokens: readonly string[]; readonly onToken?: (token: string) => void; } -): GenerationResult { +): LLMGenerationResult { 'worklet'; const { genConfig, stopTokens, onToken } = options; @@ -83,7 +83,7 @@ function generateChatTurn( if (onToken) scheduleOnRN(onToken, token); }; - const stats = nativeRunner.generate(prompt, genConfig, callback); + const stats = runner.generate(prompt, genConfig, callback); return { response, stats }; } @@ -95,65 +95,57 @@ function generateChatTurn( * @param runtime The worklet runtime thread to run native generation on. * @returns A Promise resolving to an LLMChatSession instance. */ -export async function createLLMChatSession( - config: LLMModel, - options?: LLMChatSessionOptions, +export async function createLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions, runtime?: WorkletRuntime -): Promise { - const { modelPath, tokenizerPath, tokenizerConfigPath, modalities } = config; +): Promise> { + const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = config; const initialMessages = options?.initialMessages ?? []; const defaultGenerationConfig = options?.generationConfig; // Read and parse tokenizer_config.json - const configStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); - const tokenizerConfig = parseTokenizerConfig(JSON.parse(configStr)); + const tokenizerConfigStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); + const tokenizerConfig = parseTokenizerConfig(JSON.parse(tokenizerConfigStr)); const { chatTemplate, bosToken, eosToken } = tokenizerConfig; - const format = createJinjaChatFormatter(chatTemplate, { bosToken }); + // Prepare messages' renderer + const renderer = createChatRenderer({ chatTemplate, bosToken, preprocessorConfig }); const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; - const state: SessionState = { history: [] }; - const nativeRunner = await wrapAsync(createLLMRunner, runtime)( - modelPath, - tokenizerPath, - modalities - ); - const prefill = wrapAsync(nativeRunner.prefill, runtime); + // Prepare runner + const history: ChatMessage[] = []; + const runner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath, modalities); + const prefill = wrapAsync(runner.prefill, runtime); + // Prefill initial messages for (const msg of initialMessages) { - const fmtMsg = format(msg, { isFirst: state.history.length === 0 }); - if (fmtMsg.length > 0) { - await prefill(fmtMsg); - } - state.history.push(msg); + await prefill(renderer.render(msg, { isFirst: history.length === 0 })); + history.push(msg); } - const stop = () => nativeRunner.stop(); - const dispose = () => nativeRunner.dispose(); - const runGeneration = wrapAsync(generateChatTurn, runtime); + const stop = () => runner.stop(); + const dispose = () => { + runner.dispose(); + renderer.dispose(); + }; + const generateChatTurn = wrapAsync(generateChatTurnWorklet, runtime); const sendMessage = async ( - message: string, + message: ChatMessageContent, onToken?: (token: string) => void, genConfig?: GenerationConfig - ): Promise => { - const userMsg: ChatMessage = { role: 'user', content: message }; - const assistantHeader: ChatMessage = { role: 'assistant', content: '' }; - - const fmtUserMsg = format(userMsg, { isFirst: state.history.length === 0 }); - const fmtAssistantHeader = format(assistantHeader, { isFirst: false }); + ): Promise> => { + const userMsg = { role: 'user' as const, content: message }; + const prompt = renderer.render(userMsg, { isFirst: history.length === 0, addGenPrompt: true }); - state.history.push(userMsg); + history.push(userMsg); - const prompt = fmtUserMsg + fmtAssistantHeader; - const { response, stats } = await runGeneration(nativeRunner, prompt, { - genConfig: { ...defaultGenerationConfig, ...genConfig }, - stopTokens, - onToken, - }); + const opts = { genConfig: { ...defaultGenerationConfig, ...genConfig }, stopTokens, onToken }; + const { response, stats } = await generateChatTurn(runner, prompt, opts); - state.history.push({ role: 'assistant', content: response }); + history.push({ role: 'assistant', content: response }); return { response, stats }; }; @@ -162,6 +154,6 @@ export async function createLLMChatSession( stop, dispose, sendMessage, - getHistory: () => state.history, + getHistory: () => history, }; } diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts index 5d3bde5176..a1eaec5f0c 100644 --- a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -2,6 +2,7 @@ import { useModel } from './useModel'; import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; import { createLLMChatSession, + type Modality, type LLMModel, type LLMChatSessionOptions, } from '../extensions/llm/tasks/llmChatSession'; @@ -18,9 +19,9 @@ import { * @returns An object containing the session's loading state, error, download progress, * and chat functions. */ -export function useLLMChatSession( - config: LLMModel, - options?: LLMChatSessionOptions & ResourceOptions +export function useLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions & ResourceOptions ) { const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); const { model: session, error } = useModel( diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index a16855f25f..cbd35a1b70 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -40,6 +40,7 @@ export * from './extensions/speech/tasks/fsmnVoiceActivityDetection'; export * from './extensions/speech/tasks/whisperSpeechToText'; export * from './extensions/speech/tasks/supertonicTextToSpeech'; export * from './extensions/llm/tasks/llmChatSession'; +export type { ImageBuffer, ImageFormat } from './extensions/cv/image'; // Core primitives — for library builders and power users export * from './core/error'; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 7b30250573..552c77bc76 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -852,23 +852,56 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; -const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { +const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], + preprocessorConfig: { + image: { + token: { start: '<|image_start|>', end: '<|image_end|>' }, + preprocessorOpts: { + resizeMode: 'letterbox', + interpolation: 'linear', + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + targetShape: [3, 512, 512], + }, + }, }; -const LFM2_5_VL_450M_MLX_INT4: LLMModel = { +const LFM2_5_VL_450M_MLX_INT4: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], + preprocessorConfig: { + image: { + token: { start: '<|image_start|>', end: '<|image_end|>' }, + preprocessorOpts: { + resizeMode: 'letterbox', + interpolation: 'linear', + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + targetShape: [3, 512, 512], + }, + }, }; -const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { +const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], + preprocessorConfig: { + image: { + token: { start: '<|image_start|>', end: '<|image_end|>' }, + preprocessorOpts: { + resizeMode: 'letterbox', + interpolation: 'linear', + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + targetShape: [3, 512, 512], + }, + }, }; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; diff --git a/yarn.lock b/yarn.lock index bab478be6d..e45de3eec1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11419,11 +11419,14 @@ __metadata: "@react-native/metro-config": "npm:^0.86.0" "@react-navigation/drawer": "npm:^7.9.4" "@react-navigation/native": "npm:^7.2.2" + "@shopify/react-native-skia": "npm:2.6.2" "@types/react": "npm:~19.2.0" babel-preset-expo: "npm:~56.0.14" expo: "npm:~56.0.9" expo-build-properties: "npm:~56.0.17" expo-constants: "npm:~56.0.17" + expo-image-manipulator: "npm:~56.0.18" + expo-image-picker: "npm:~56.0.18" expo-linking: "npm:~56.0.13" expo-router: "npm:~56.2.9" react: "npm:19.2.3" From a4f787cfb343464da9cef6925ab952f680a68262 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 22:02:29 +0200 Subject: [PATCH 21/43] refactor --- .../react-native-executorch/src/models.ts | 48 ++++++------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 552c77bc76..b1f4dc8e86 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -852,56 +852,38 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerPath: `${LFM2_5_BASE_URL}/350m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; + +const LFM2_5_VL_PREPROCESSOR_CONFIG = { + image: { + token: { start: '<|image_start|>', end: '<|image_end|>' }, + targetShape: [3, 512, 512] as const, + preprocessorOpts: { + resizeMode: 'letterbox' as const, + interpolation: 'linear' as const, + normalizeOpts: { alpha: 1.0, beta: 0.0 }, + }, + }, +}; const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], - preprocessorConfig: { - image: { - token: { start: '<|image_start|>', end: '<|image_end|>' }, - preprocessorOpts: { - resizeMode: 'letterbox', - interpolation: 'linear', - normalizeOpts: { alpha: 1.0, beta: 0.0 }, - }, - targetShape: [3, 512, 512], - }, - }, + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_450M_MLX_INT4: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], - preprocessorConfig: { - image: { - token: { start: '<|image_start|>', end: '<|image_end|>' }, - preprocessorOpts: { - resizeMode: 'letterbox', - interpolation: 'linear', - normalizeOpts: { alpha: 1.0, beta: 0.0 }, - }, - targetShape: [3, 512, 512], - }, - }, + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel<'image'> = { modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, modalities: ['image'], - preprocessorConfig: { - image: { - token: { start: '<|image_start|>', end: '<|image_end|>' }, - preprocessorOpts: { - resizeMode: 'letterbox', - interpolation: 'linear', - normalizeOpts: { alpha: 1.0, beta: 0.0 }, - }, - targetShape: [3, 512, 512], - }, - }, + preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; const BIELIK_V3_1_5B_BASE_URL = `${BASE_URL}-bielik-v3.0/${VERSION_TAG}`; From 261e27449b3e68ca2cf34eb2af8027d802e89cba Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 22:08:08 +0200 Subject: [PATCH 22/43] style --- packages/react-native-executorch/src/models.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index b1f4dc8e86..1e0cd93640 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1544,6 +1544,7 @@ export const models = { /** WordPiece tokenizer URL for the `all-MiniLM-L6-v2` embedding model. */ ALL_MINILM_L6_V2: ALL_MINILM_L6_V2_TOKENIZER, }, + /** * Generative Large Language Models (LLMs) for instruction following, * chat, text generation, and reasoning. From 2c6bcf6149ff1949aabc9be02f86e08b77b2fc7f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 22:13:53 +0200 Subject: [PATCH 23/43] fix --- apps/nlp/app/llm/index.tsx | 6 +++--- packages/react-native-executorch/src/index.ts | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 79a7b85e08..0eb1c7cfc2 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -19,7 +19,7 @@ import { models, type ChatMessage, type GenerationStats, - type ImageBuffer, + cv, } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { NestedModelPicker, findPath } from '../../components/ModelPicker'; @@ -105,7 +105,7 @@ function LLMContent() { const [attachedImage, setAttachedImage] = useState<{ uri: string; - buffer: ImageBuffer; + buffer: cv.ImageBuffer; name: string; } | null>(null); @@ -140,7 +140,7 @@ function LLMContent() { return; } - const buffer: ImageBuffer = { + const buffer: cv.ImageBuffer = { data: pixels, width: skImage.width(), height: skImage.height(), diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index cbd35a1b70..d7cb910926 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -33,14 +33,13 @@ export * from './extensions/cv/tasks/keypointDetection'; export * from './extensions/cv/tasks/objectDetection'; export * from './extensions/cv/tasks/imageEmbedding'; export * from './extensions/cv/tasks/sdxsTextToImage'; +export * from './extensions/llm/tasks/llmChatSession'; export * from './extensions/nlp/tasks/tokenization'; export * from './extensions/nlp/tasks/textEmbedding'; export * from './extensions/nlp/tasks/privacyFilter'; export * from './extensions/speech/tasks/fsmnVoiceActivityDetection'; export * from './extensions/speech/tasks/whisperSpeechToText'; export * from './extensions/speech/tasks/supertonicTextToSpeech'; -export * from './extensions/llm/tasks/llmChatSession'; -export type { ImageBuffer, ImageFormat } from './extensions/cv/image'; // Core primitives — for library builders and power users export * from './core/error'; @@ -53,9 +52,9 @@ export * as schema from './core/schema'; export * as math from './extensions/math'; export * as cv from './extensions/cv'; +export * as llm from './extensions/llm'; export * as nlp from './extensions/nlp'; export * as speech from './extensions/speech'; -export * as llm from './extensions/llm'; // Utils export * from './utils'; From 82cc575e72abef7fdbdfce0c645144fbcf13f297 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 22:26:26 +0200 Subject: [PATCH 24/43] refactor(llm): extract ChatRenderer, flatten LLM model registry and update model JSDocs --- apps/nlp/app/llm/index.tsx | 2 +- .../extensions/llm/tasks/llmChatSession.ts | 5 +- .../react-native-executorch/src/models.ts | 283 ++++++++++-------- 3 files changed, 160 insertions(+), 130 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 0eb1c7cfc2..b532bda736 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -62,7 +62,7 @@ function getFirstLeafModel(node: any): any { function LLMContent() { const [selectedModel, setSelectedModel] = useState( - models.llm.LFM2_5.VL_450M ?? getFirstLeafModel(models.llm) + models.llm.LFM2_5_VL_450M ?? getFirstLeafModel(models.llm) ); const [activeModel, setActiveModel] = useState(null); const [forceDownload, setForceDownload] = useState(false); diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index f657cc9913..5a5dbc5598 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -53,14 +53,14 @@ export type LLMGenerationResult = { }; export type LLMChatSession = { + stop(): void; dispose(): void; + getHistory(): readonly ChatMessage[]; sendMessage( message: ChatMessageContent, onToken?: (token: string) => void, genConfig?: GenerationConfig ): Promise>; - getHistory(): readonly ChatMessage[]; - stop(): void; }; function generateChatTurnWorklet( @@ -130,6 +130,7 @@ export async function createLLMChatSession( runner.dispose(); renderer.dispose(); }; + const generateChatTurn = wrapAsync(generateChatTurnWorklet, runtime); const sendMessage = async ( diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 1e0cd93640..61795f81c0 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -1551,109 +1551,126 @@ export const models = { */ llm: { /** - * Liquid AI LFM 2.5 generative text model family (350M to 1.2B - * parameters) optimized for high-efficiency on-device chat and text - * generation. + * Liquid AI LFM 2.5 1.2B general-purpose text model. Excellent for complex + * on-device reasoning, instruction following, and fast multi-turn chat. */ - LFM2_5: { + LFM2_5_1_2B: { ...LFM2_5_1_2B_XNNPACK_8DA4W, - P1_2B: { - ...LFM2_5_1_2B_XNNPACK_8DA4W, - XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, - XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, - MLX_INT4: LFM2_5_1_2B_MLX_INT4, - }, - P350M: { - ...LFM2_5_350M_XNNPACK_8DA4W, - XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, - XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, - MLX_INT4: LFM2_5_350M_MLX_INT4, - }, - VL_450M: { - ...LFM2_5_VL_450M_XNNPACK_8DA4W, - XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, - MLX_INT4: LFM2_5_VL_450M_MLX_INT4, - }, - VL_1_6B: { - ...LFM2_5_VL_1_6B_XNNPACK_8DA4W, - XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, - }, + XNNPACK_8DA4W: LFM2_5_1_2B_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_1_2B_XNNPACK_FP16, + MLX_INT4: LFM2_5_1_2B_MLX_INT4, }, /** - * Bielik v3 1.5B Polish and English instruction-tuned language model. + * Liquid AI LFM 2.5 350M ultra-compact text model. Best for low-latency text + * completion, quick responses, and resource-constrained devices. */ - BIELIK_V3: { + LFM2_5_350M: { + ...LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_350M_XNNPACK_8DA4W, + XNNPACK_FP16: LFM2_5_350M_XNNPACK_FP16, + MLX_INT4: LFM2_5_350M_MLX_INT4, + }, + /** + * Liquid AI LFM 2.5 450M vision-language model. Optimized for real-time + * visual QA, image description, and low-latency multimodal chat. + */ + LFM2_5_VL_450M: { + ...LFM2_5_VL_450M_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_450M_XNNPACK_8DA4W, + MLX_INT4: LFM2_5_VL_450M_MLX_INT4, + }, + /** + * Liquid AI LFM 2.5 1.6B vision-language model. Higher quality visual + * understanding, detailed image analysis, and complex multimodal tasks. + */ + LFM2_5_VL_1_6B: { + ...LFM2_5_VL_1_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: LFM2_5_VL_1_6B_XNNPACK_8DA4W, + }, + /** + * Bielik v3 1.5B Polish & English language model. Fine-tuned specifically for + * native Polish fluency, grammar, and bilingual translation. + */ + BIELIK_V3_1_5B: { ...BIELIK_V3_1_5B_XNNPACK_8DA4W, - P1_5B: { - ...BIELIK_V3_1_5B_XNNPACK_8DA4W, - XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, - XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, - }, + XNNPACK_8DA4W: BIELIK_V3_1_5B_XNNPACK_8DA4W, + XNNPACK_FP16: BIELIK_V3_1_5B_XNNPACK_FP16, }, /** - * Meta Llama 3.2 lightweight multilingual text generation model - * family (1B and 3B parameters). + * Meta Llama 3.2 1B multilingual text model. Ideal for lightweight mobile + * chat, summary generation, and multilingual prompt processing. */ - LLAMA3_2: { + LLAMA3_2_1B: { ...LLAMA3_2_1B_SPINQUANT, - P1B: { - ...LLAMA3_2_1B_SPINQUANT, - XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, - XNNPACK_BF16: LLAMA3_2_1B_BF16, - }, - P3B: { - ...LLAMA3_2_3B_SPINQUANT, - XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, - XNNPACK_BF16: LLAMA3_2_3B_BF16, - }, + XNNPACK_SPINQUANT: LLAMA3_2_1B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_1B_BF16, + }, + /** + * Meta Llama 3.2 3B multilingual text model. Strong instruction following, + * detailed content creation, and high-precision text reasoning. + */ + LLAMA3_2_3B: { + ...LLAMA3_2_3B_SPINQUANT, + XNNPACK_SPINQUANT: LLAMA3_2_3B_SPINQUANT, + XNNPACK_BF16: LLAMA3_2_3B_BF16, }, /** - * Hugging Face SmolLM2 ultra-compact language model family (135M to - * 1.7B parameters). + * Hugging Face SmolLM2 135M sub-parameter model. Best for micro-footprint + * background tasks, simple text tagging, and instant autocomplete. */ - SMOLLM2: { + SMOLLM2_135M: { + ...SMOLLM2_135M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, + XNNPACK_BF16: SMOLLM2_135M_BF16, + }, + /** + * Hugging Face SmolLM2 360M compact model. Balanced speed and intelligence + * for lightweight conversational assistants. + */ + SMOLLM2_360M: { + ...SMOLLM2_360M_8DA4W, + XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, + XNNPACK_BF16: SMOLLM2_360M_BF16, + }, + /** + * Hugging Face SmolLM2 1.7B language model. Powerful general-purpose text + * generation, creative writing, and general knowledge Q&A. + */ + SMOLLM2_1_7B: { ...SMOLLM2_1_7B_8DA4W, - P135M: { - ...SMOLLM2_135M_8DA4W, - XNNPACK_8DA4W: SMOLLM2_135M_8DA4W, - XNNPACK_BF16: SMOLLM2_135M_BF16, - }, - P360M: { - ...SMOLLM2_360M_8DA4W, - XNNPACK_8DA4W: SMOLLM2_360M_8DA4W, - XNNPACK_BF16: SMOLLM2_360M_BF16, - }, - P1_7B: { - ...SMOLLM2_1_7B_8DA4W, - XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, - XNNPACK_BF16: SMOLLM2_1_7B_BF16, - }, + XNNPACK_8DA4W: SMOLLM2_1_7B_8DA4W, + XNNPACK_BF16: SMOLLM2_1_7B_BF16, }, /** - * Hammer 2.1 function-calling and agentic tool-use language model - * family. + * Hammer 2.1 0.5B function-calling model. Specialized for lightweight agentic + * tool calling, JSON extraction, and structured output parsing. */ - HAMMER2_1: { + HAMMER2_1_0_5B: { + ...HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, + }, + /** + * Hammer 2.1 1.5B function-calling model. Optimized for multi-tool agentic + * workflows, API function calling, and structured JSON schemas. + */ + HAMMER2_1_1_5B: { ...HAMMER2_1_1_5B_XNNPACK_8DA4W, - P0_5B: { - ...HAMMER2_1_0_5B_XNNPACK_8DA4W, - XNNPACK_8DA4W: HAMMER2_1_0_5B_XNNPACK_8DA4W, - XNNPACK_BF16: HAMMER2_1_0_5B_XNNPACK_BF16, - }, - P1_5B: { - ...HAMMER2_1_1_5B_XNNPACK_8DA4W, - XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, - XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, - }, - P3B: { - ...HAMMER2_1_3B_XNNPACK_8DA4W, - XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, - XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, - }, + XNNPACK_8DA4W: HAMMER2_1_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_1_5B_XNNPACK_BF16, }, /** - * Microsoft Phi-4 Mini 3.8B parameter lightweight reasoning language - * model. + * Hammer 2.1 3B function-calling model. High-capacity agentic reasoning, + * complex multi-step tool execution, and robust schema compliance. + */ + HAMMER2_1_3B: { + ...HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: HAMMER2_1_3B_XNNPACK_8DA4W, + XNNPACK_BF16: HAMMER2_1_3B_XNNPACK_BF16, + }, + /** + * Microsoft Phi-4 Mini 3.8B reasoning model. Exceptional for math problem + * solving, logical reasoning, code synthesis, and analytical tasks. */ PHI4_MINI: { ...PHI4_MINI_XNNPACK_8DA4W, @@ -1661,62 +1678,74 @@ export const models = { XNNPACK_BF16: PHI4_MINI_XNNPACK_BF16, }, /** - * Alibaba Qwen 2.5 multilingual instruction-tuned language model - * family. + * Alibaba Qwen 2.5 0.5B multilingual model. Extremely efficient for fast + * multi-language translation and basic conversational chat. */ - QWEN2_5: { + QWEN2_5_0_5B: { + ...QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 2.5 1.5B multilingual model. Great for balanced multilingual + * chat, text summarization, and cross-lingual understanding. + */ + QWEN2_5_1_5B: { ...QWEN2_5_1_5B_XNNPACK_8DA4W, - P0_5B: { - ...QWEN2_5_0_5B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN2_5_0_5B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN2_5_0_5B_XNNPACK_BF16, - }, - P1_5B: { - ...QWEN2_5_1_5B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, - }, - P3B: { - ...QWEN2_5_3B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, - }, + XNNPACK_8DA4W: QWEN2_5_1_5B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_1_5B_XNNPACK_BF16, }, /** - * Alibaba Qwen 3 high-performance multilingual text generation model - * family. + * Alibaba Qwen 2.5 3B multilingual model. High capability across 29+ + * languages for complex translation, long-form writing, and Q&A. */ - QWEN3: { + QWEN2_5_3B: { + ...QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN2_5_3B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN2_5_3B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 3 0.6B next-gen text model. Low-latency multilingual model + * for fast turn-taking and concise response generation. + */ + QWEN3_0_6B: { + ...QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, + }, + /** + * Alibaba Qwen 3 1.7B next-gen text model. Versatile multilingual assistant + * for high-quality instruction following and knowledge retrieval. + */ + QWEN3_1_7B: { ...QWEN3_1_7B_XNNPACK_8DA4W, - P0_6B: { - ...QWEN3_0_6B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN3_0_6B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN3_0_6B_XNNPACK_BF16, - }, - P1_7B: { - ...QWEN3_1_7B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, - }, - P4B: { - ...QWEN3_4B_XNNPACK_8DA4W, - XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, - XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, - }, + XNNPACK_8DA4W: QWEN3_1_7B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_1_7B_XNNPACK_BF16, }, /** - * Google Gemma 4 lightweight generative language model family - * optimized for on-device use. + * Alibaba Qwen 3 4B high-capacity text model. Top-tier multilingual + * reasoning, technical content generation, and multi-turn dialogue. */ - GEMMA4: { + QWEN3_4B: { + ...QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_8DA4W: QWEN3_4B_XNNPACK_8DA4W, + XNNPACK_BF16: QWEN3_4B_XNNPACK_BF16, + }, + /** + * Google Gemma 4 E2B generative text model. Built on Google's Gemini tech + * for high-fidelity instruction following and mobile assistance. + */ + GEMMA4_E2B: { ...GEMMA4_E2B_XNNPACK_8DA4W, - E2B: { - ...GEMMA4_E2B_XNNPACK_8DA4W, - XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, - MLX_INT4: GEMMA4_E2B_MLX_INT4, - }, + XNNPACK_8DA4W: GEMMA4_E2B_XNNPACK_8DA4W, + MLX_INT4: GEMMA4_E2B_MLX_INT4, }, }, + + /** + * Text embedding models mapping sentences and documents into dense vector + * representations for semantic search and RAG. + */ textEmbeddings: { /** * Compact 384-dimensional sentence transformer mapping text to a dense From 4e7eec5f62eb415b2533a35504d0f0b2d65223a1 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 22:39:34 +0200 Subject: [PATCH 25/43] fix --- .../src/extensions/llm/tasks/llmChatSession.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 5a5dbc5598..067f978147 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -29,7 +29,7 @@ export type { ChatMessageContent, ChatMessage, LLMMediaPreprocessorConfig, - ChatRendererConfig as ChatRendererOpts, + ChatRendererConfig, RenderOpts, }; From 59980ce5aa97176a2c56e4baad9bd3a303db8f15 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 23:15:33 +0200 Subject: [PATCH 26/43] refactor(llm): simplify app UI, refine modality typing with NoInfer and ChatMediaInputMap --- apps/nlp/app/llm/index.tsx | 606 +++++------------- apps/nlp/utils.ts | 75 +++ .../src/extensions/llm/chatRenderer.ts | 15 +- .../extensions/llm/tasks/llmChatSession.ts | 8 +- .../src/hooks/useLLMChatSession.ts | 4 +- .../react-native-executorch/src/models.ts | 4 +- 6 files changed, 260 insertions(+), 452 deletions(-) create mode 100644 apps/nlp/utils.ts diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index b532bda736..fe8d385c16 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef, useState, type ComponentRef } from 'react'; +import React, { useRef, useState, type ComponentRef } from 'react'; import { View, Text, @@ -12,7 +12,6 @@ import { Alert, Image as RNImage, } from 'react-native'; -import * as ImagePicker from 'expo-image-picker'; import { Skia } from '@shopify/react-native-skia'; import { useLLMChatSession, @@ -22,8 +21,9 @@ import { cv, } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; -import { NestedModelPicker, findPath } from '../../components/ModelPicker'; +import { getImage, skImageToBuffer } from '../../utils'; +const MODEL = models.llm.LFM2_5_VL_450M; const SYSTEM_PROMPT = 'You are a helpful multimodal assistant by Liquid AI.'; const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; @@ -41,41 +41,18 @@ function formatStats(stats: GenerationStats): string { const totalMs = stats.inferenceEndMs - stats.inferenceStartMs; const ttftMs = stats.firstTokenMs - stats.inferenceStartMs; return ( - `gen ${stats.numGeneratedTokens} tokens · ` + + `${stats.numGeneratedTokens} tokens · ` + `${tokensPerSec.toFixed(1)} tok/s · ` + `${ttftMs.toFixed(0)}ms ttft · ` + `${(totalMs / 1000).toFixed(2)}s` ); } -function getFirstLeafModel(node: any): any { - if (!node || typeof node !== 'object') return null; - for (const key of Object.keys(node)) { - if (typeof node[key] === 'object' && node[key] !== null) { - const leaf = getFirstLeafModel(node[key]); - if (leaf) return leaf; - } - } - if (typeof node.modelPath === 'string') return node; - return null; -} - function LLMContent() { - const [selectedModel, setSelectedModel] = useState( - models.llm.LFM2_5_VL_450M ?? getFirstLeafModel(models.llm) - ); - const [activeModel, setActiveModel] = useState(null); - const [forceDownload, setForceDownload] = useState(false); - - const { isReady, downloadProgress, error, sendMessage, stop } = useLLMChatSession( - activeModel || selectedModel, - { - initialMessages: INITIAL_MESSAGES, - generationConfig: GENERATION_CONFIG, - preventLoad: !activeModel, - forceDownload, - } - ); + const { isReady, downloadProgress, error, sendMessage, stop } = useLLMChatSession(MODEL, { + initialMessages: INITIAL_MESSAGES, + generationConfig: GENERATION_CONFIG, + }); const [input, setInput] = useState(''); const [turns, setTurns] = useState([]); @@ -84,75 +61,24 @@ function LLMContent() { const scrollRef = useRef>(null); const isGenerating = streamingResponse !== null; - const selectedModelName = findPath(models.llm, selectedModel)?.join(' ') || 'Selected Model'; - - // Reset chat turns when model changes - useEffect(() => { - setTurns([]); - setStreamingResponse(null); - setInput(''); - }, [activeModel]); - - // Reset forceDownload when model finishes loading and is ready - useEffect(() => { - if (isReady) setForceDownload(false); - }, [isReady]); - - const handleLoadModel = (force = false) => { - setForceDownload(force); - setActiveModel(selectedModel); - }; - const [attachedImage, setAttachedImage] = useState<{ uri: string; - buffer: cv.ImageBuffer; name: string; + buffer: cv.ImageBuffer; } | null>(null); const handlePickGalleryImage = async () => { try { - const permissionResult = await ImagePicker.requestMediaLibraryPermissionsAsync(); - if (!permissionResult.granted) { - Alert.alert('Permission Required', 'Permission to access photo gallery is required!'); - return; - } + const uri = await getImage(false); + if (!uri) return; - const pickerResult = await ImagePicker.launchImageLibraryAsync({ - mediaTypes: ['images'], - allowsEditing: false, - quality: 1, - base64: true, - }); - - if (pickerResult.canceled || !pickerResult.assets[0]?.base64) return; - const asset = pickerResult.assets[0]; - - const skData = Skia.Data.fromBase64(asset.base64!); + const skData = await Skia.Data.fromURI(uri); + if (!skData) throw new Error('Failed to read image file'); const skImage = Skia.Image.MakeImageFromEncoded(skData); - if (!skImage) { - Alert.alert('Error', 'Failed to decode selected image.'); - return; - } - - const pixels = skImage.readPixels(); - if (!pixels || !(pixels instanceof Uint8Array)) { - Alert.alert('Error', 'Failed to read image pixel data.'); - return; - } + if (!skImage) throw new Error('Failed to decode image'); - const buffer: cv.ImageBuffer = { - data: pixels, - width: skImage.width(), - height: skImage.height(), - format: 'rgba', - layout: 'hwc', - }; - - setAttachedImage({ - uri: asset.uri, - buffer, - name: asset.fileName || `Photo (${skImage.width()}×${skImage.height()})`, - }); + const buffer = skImageToBuffer(skImage); + setAttachedImage({ uri, buffer, name: `Photo (${skImage.width()}x${skImage.height()})` }); } catch (err: any) { Alert.alert('Error', err?.message || 'Failed to select gallery image'); } @@ -167,21 +93,17 @@ function LLMContent() { setInput(''); setStreamingResponse(''); - const turnUserMessage = textMessage || '[Image Attached]'; setTurns((prev) => [ ...prev, - { role: 'user', content: turnUserMessage, imageUri: currentImage?.uri }, + { role: 'user', content: textMessage, imageUri: currentImage?.uri }, ]); try { - const payload = currentImage - ? ([ - { kind: 'image' as const, image: currentImage.buffer }, - textMessage || 'What is in this image?', - ] as const) - : textMessage; + const payload = []; + if (currentImage) payload.push({ kind: 'image' as const, image: currentImage.buffer }); + payload.push(textMessage || 'What is in this image?'); - const { response, stats } = await sendMessage(payload as any, (token) => { + const { response, stats } = await sendMessage(payload, (token) => { setStreamingResponse((prev) => (prev !== null ? prev + token : token)); }); setTurns((prev) => [...prev, { role: 'assistant', content: response as string, stats }]); @@ -190,175 +112,27 @@ function LLMContent() { } }; - const renderContent = () => { - if (!activeModel) { - return ( - - No model loaded - - {selectedModelName} is selected. Click below to load it and start chatting. - - handleLoadModel(false)}> - Load Model - - - ); - } - - if (activeModel !== selectedModel) { - return ( - - Switch to {selectedModelName}? - - Switching models will unload the current model and reset the chat session. - - handleLoadModel(false)}> - Load New Model - - setSelectedModel(activeModel)} - > - Keep Current Model - - - ); - } - - if (error) { - return ( - - Failed to load model - {error.message} - handleLoadModel(false)}> - Retry Loading - - handleLoadModel(true)}> - Force Redownload - - - ); - } - - if (!isReady) { - return ( - - - - {downloadProgress < 100 - ? `Downloading model… ${downloadProgress.toFixed(0)}%` - : 'Loading model into memory…'} - - {activeModel.modelPath} - - ); - } - + if (error) { return ( - <> - scrollRef.current?.scrollToEnd({ animated: false })} - > - {turns.length === 0 && streamingResponse === null && ( - Ask the on-device model anything to get started. - )} - {turns.map((turn, idx) => ( - - - {turn.imageUri && ( - - )} - - {turn.content || '…'} - - - {turn.stats && ( - - {formatStats(turn.stats)} - - )} - - ))} - {streamingResponse !== null && ( - - - {streamingResponse || '…'} - - - )} - - - {attachedImage && ( - - - - - {attachedImage.name} - - Image Attached - - setAttachedImage(null)} - > - ✕ - - - )} + + Failed to load model + {error.message} + + ); + } - - - 🖼 Gallery - - - {isGenerating ? ( - stop?.()} - > - Stop - - ) : ( - - Send - - )} - - + if (!isReady) { + return ( + + + + {downloadProgress < 100 + ? `Downloading model… ${downloadProgress.toFixed(0)}%` + : 'Loading model into memory…'} + + ); - }; + } return ( - {/* Model Selector Header */} - - - - { - setForceDownload(false); - setSelectedModel(m); - }} - /> + scrollRef.current?.scrollToEnd({ animated: false })} + > + {turns.length === 0 && streamingResponse === null && ( + Ask LFM 2.5 VL anything or attach an image. + )} + {turns.map((turn, idx) => ( + + + {turn.imageUri && ( + + )} + + {turn.content || '…'} + + + {turn.stats && {formatStats(turn.stats)}} + + ))} + {streamingResponse !== null && ( + + + {streamingResponse || '…'} + + + )} + + + {attachedImage && ( + + + + + {attachedImage.name} + handleLoadModel(true)} + style={styles.removeAttachmentButton} + onPress={() => setAttachedImage(null)} > - Redownload + ✕ - + )} - {/* Screen Content */} - {renderContent()} + + + 📷 + + + {isGenerating ? ( + stop?.()}> + Stop + + ) : ( + + Send + + )} + ); } @@ -405,20 +241,8 @@ export default function LLMScreen() { const styles = StyleSheet.create({ container: { flex: 1, backgroundColor: '#f8f9fa' }, - header: { - paddingHorizontal: 16, - paddingTop: 12, - paddingBottom: 4, - backgroundColor: '#fff', - borderBottomWidth: 1, - borderBottomColor: '#e9ecef', - }, - content: { - flex: 1, - }, centered: { flex: 1, justifyContent: 'center', alignItems: 'center', padding: 24 }, loadingText: { marginTop: 16, fontSize: 15, color: '#495057', fontWeight: '600' }, - loadingSub: { marginTop: 4, fontSize: 13, color: '#868e96', textAlign: 'center' }, errorTitle: { fontSize: 16, fontWeight: '700', color: '#e03131', marginBottom: 8 }, errorBody: { fontSize: 13, color: '#868e96', textAlign: 'center' }, messages: { flex: 1 }, @@ -431,15 +255,6 @@ const styles = StyleSheet.create({ paddingHorizontal: 14, paddingVertical: 10, }, - statsLine: { - alignSelf: 'flex-start', - marginTop: 5, - marginLeft: 4, - fontSize: 11, - color: '#adb5bd', - // cspell:disable-next-line - fontVariant: ['tabular-nums'], - }, userBubble: { alignSelf: 'flex-end', backgroundColor: '#0070f3' }, assistantBubble: { alignSelf: 'flex-start', @@ -449,6 +264,45 @@ const styles = StyleSheet.create({ }, userText: { color: '#fff', fontSize: 15, lineHeight: 21 }, assistantText: { color: '#212529', fontSize: 15, lineHeight: 21 }, + turnThumbnail: { + width: 160, + height: 120, + borderRadius: 8, + marginBottom: 8, + backgroundColor: '#e9ecef', + }, + statsLine: { + alignSelf: 'flex-start', + marginTop: 4, + marginLeft: 4, + fontSize: 11, + color: '#adb5bd', + }, + attachmentBar: { + flexDirection: 'row', + alignItems: 'center', + paddingHorizontal: 16, + paddingVertical: 8, + backgroundColor: '#fff', + borderTopWidth: 1, + borderTopColor: '#e9ecef', + gap: 12, + }, + attachmentPreview: { + width: 40, + height: 40, + borderRadius: 6, + backgroundColor: '#e9ecef', + }, + attachmentInfo: { flex: 1 }, + attachmentName: { fontSize: 13, fontWeight: '500', color: '#212529' }, + removeAttachmentButton: { + paddingHorizontal: 8, + paddingVertical: 4, + borderRadius: 12, + backgroundColor: '#f1f3f5', + }, + removeAttachmentText: { fontSize: 13, fontWeight: '700', color: '#868e96' }, inputRow: { flexDirection: 'row', alignItems: 'flex-end', @@ -458,6 +312,15 @@ const styles = StyleSheet.create({ borderTopColor: '#e9ecef', backgroundColor: '#fff', }, + galleryButton: { + width: 44, + height: 44, + backgroundColor: '#f1f3f5', + borderRadius: 22, + justifyContent: 'center', + alignItems: 'center', + }, + galleryButtonText: { fontSize: 18 }, input: { flex: 1, backgroundColor: '#f1f3f5', @@ -479,137 +342,4 @@ const styles = StyleSheet.create({ sendButtonDisabled: { backgroundColor: '#a3cdff' }, stopButton: { backgroundColor: '#e03131' }, sendButtonText: { color: '#fff', fontSize: 15, fontWeight: '600' }, - infoTitle: { - fontSize: 16, - fontWeight: '700', - color: '#212529', - marginBottom: 8, - textAlign: 'center', - }, - infoBody: { - fontSize: 14, - color: '#495057', - textAlign: 'center', - marginBottom: 20, - lineHeight: 20, - }, - loadButton: { - backgroundColor: '#0070f3', - borderRadius: 20, - paddingHorizontal: 24, - paddingVertical: 12, - justifyContent: 'center', - alignItems: 'center', - shadowColor: '#000', - shadowOffset: { width: 0, height: 2 }, - shadowOpacity: 0.1, - shadowRadius: 4, - elevation: 2, - }, - loadButtonText: { - color: '#fff', - fontSize: 15, - fontWeight: '600', - }, - cancelButton: { - marginTop: 12, - paddingVertical: 8, - }, - cancelButtonText: { - color: '#6c757d', - fontSize: 14, - fontWeight: '500', - }, - secondaryButton: { - marginTop: 12, - backgroundColor: '#f1f3f5', - borderRadius: 20, - paddingHorizontal: 24, - paddingVertical: 12, - justifyContent: 'center', - alignItems: 'center', - }, - secondaryButtonText: { - color: '#495057', - fontSize: 15, - fontWeight: '600', - }, - headerRow: { - flexDirection: 'row', - alignItems: 'center', - gap: 8, - }, - pickerContainer: { - flex: 1, - }, - redownloadHeaderButton: { - paddingHorizontal: 12, - paddingVertical: 6, - backgroundColor: '#f1f3f5', - borderRadius: 12, - }, - redownloadHeaderText: { - color: '#495057', - fontSize: 13, - fontWeight: '500', - }, - galleryButton: { - paddingHorizontal: 12, - height: 44, - backgroundColor: '#e9ecef', - borderRadius: 20, - justifyContent: 'center', - alignItems: 'center', - }, - galleryButtonText: { - fontSize: 13, - fontWeight: '600', - color: '#495057', - }, - attachmentBar: { - flexDirection: 'row', - alignItems: 'center', - paddingHorizontal: 16, - paddingVertical: 8, - backgroundColor: '#fff', - borderTopWidth: 1, - borderTopColor: '#e9ecef', - gap: 12, - }, - attachmentPreview: { - width: 44, - height: 44, - borderRadius: 8, - backgroundColor: '#e9ecef', - }, - attachmentInfo: { - flex: 1, - }, - attachmentName: { - fontSize: 14, - fontWeight: '600', - color: '#212529', - }, - attachmentSub: { - fontSize: 12, - color: '#868e96', - }, - removeAttachmentButton: { - paddingHorizontal: 10, - paddingVertical: 4, - borderRadius: 12, - backgroundColor: '#f1f3f5', - }, - removeAttachmentText: { - fontSize: 14, - fontWeight: '700', - color: '#868e96', - }, - turnThumbnail: { - width: 160, - height: 120, - borderRadius: 8, - marginBottom: 8, - backgroundColor: '#e9ecef', - }, }); diff --git a/apps/nlp/utils.ts b/apps/nlp/utils.ts new file mode 100644 index 0000000000..a8271596c4 --- /dev/null +++ b/apps/nlp/utils.ts @@ -0,0 +1,75 @@ +import { Alert } from 'react-native'; +import * as ImagePicker from 'expo-image-picker'; +import { ImageManipulator, SaveFormat } from 'expo-image-manipulator'; +import type { SkImage } from '@shopify/react-native-skia'; +import type { cv } from 'react-native-executorch'; + +/** + * Converts a Skia image into the raw RGBA/HWC image buffer that + * react-native-executorch vision tasks accept. Throws if the pixel data cannot + * be read. + * @param image - The Skia image to read pixels from. + * @returns The RGBA/HWC image buffer with its `data`, `width`, `height`, + * `format`, and `layout`. + */ +export const skImageToBuffer = (image: SkImage): cv.ImageBuffer => { + const pixels = image.readPixels(); + if (!pixels) { + throw new Error('Failed to read pixels from image'); + } + if (!(pixels instanceof Uint8Array)) { + throw new Error('Expected Uint8Array from readPixels'); + } + return { + data: pixels, + width: image.width(), + height: image.height(), + format: 'rgba' as const, + layout: 'hwc' as const, + }; +}; + +export const getImage = async ( + useCamera: boolean, + targetWidth = 800 +): Promise => { + const permissionResult = useCamera + ? await ImagePicker.requestCameraPermissionsAsync() + : await ImagePicker.requestMediaLibraryPermissionsAsync(); + + if (!permissionResult.granted) { + Alert.alert( + 'Permission Required', + useCamera + ? 'Permission to access camera is required!' + : 'Permission to access camera roll is required!' + ); + return; + } + + const options: ImagePicker.ImagePickerOptions = { + mediaTypes: ['images'], + allowsEditing: false, + quality: 1, + }; + + const pickerResult = useCamera + ? await ImagePicker.launchCameraAsync(options) + : await ImagePicker.launchImageLibraryAsync(options); + + if (pickerResult.canceled || !pickerResult.assets[0]) { + return; + } + + const asset = pickerResult.assets[0]; + let imageRef = await ImageManipulator.manipulate(asset.uri).renderAsync(); + + if (imageRef.width > targetWidth) { + imageRef = await ImageManipulator.manipulate(asset.uri) + .resize({ width: targetWidth }) + .renderAsync(); + } + + const result = await imageRef.saveAsync({ format: SaveFormat.PNG }); + return result.uri; +}; diff --git a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts index 5c148955b8..443c7284b9 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts @@ -7,14 +7,17 @@ import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/ta import type { Modality, Prompt, MediaInput } from './llmRunner'; /** High-level media payload types for chat messages. */ -export type ChatMediaInput = - | { readonly kind: 'image'; readonly image: ImageBuffer } - | { readonly kind: 'audio'; readonly audio: Float32Array }; +export type ChatMediaInputMap = { + image: { readonly kind: 'image'; readonly image: ImageBuffer }; + audio: { readonly kind: 'audio'; readonly audio: unknown }; +}; + +export type ChatMediaInput = ChatMediaInputMap[M]; /** Interleaved multimodal content for high-level ChatMessages. */ export type ChatMessageContent = | string - | readonly (string | Extract)[]; + | readonly (string | ChatMediaInput)[]; /** Message interface for chat history and inputs. */ export type ChatMessage = { @@ -22,7 +25,7 @@ export type ChatMessage = { readonly content: ChatMessageContent; }; -export type LLMMediaPreprocessorConfig = { +export type MediaPreprocessorConfig = { image?: { readonly token: { readonly start: string; readonly end: string }; readonly preprocessorOpts: ImagePreprocessorOptions; @@ -37,7 +40,7 @@ export type LLMMediaPreprocessorConfig = { export type ChatRendererConfig = { readonly chatTemplate: string; readonly bosToken?: string; - readonly preprocessorConfig?: LLMMediaPreprocessorConfig; + readonly preprocessorConfig?: MediaPreprocessorConfig; }; /** Turn rendering options for ChatRenderer. */ diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 067f978147..d98a30cbd0 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -16,7 +16,7 @@ import { type ChatMediaInput, type ChatMessageContent, type ChatMessage, - type LLMMediaPreprocessorConfig, + type MediaPreprocessorConfig, type ChatRendererConfig, type RenderOpts, } from '../chatRenderer'; @@ -28,7 +28,7 @@ export type { ChatMediaInput, ChatMessageContent, ChatMessage, - LLMMediaPreprocessorConfig, + MediaPreprocessorConfig, ChatRendererConfig, RenderOpts, }; @@ -38,7 +38,7 @@ export type LLMModel = { readonly tokenizerPath: string; readonly tokenizerConfigPath: string; readonly modalities?: readonly M[]; - readonly preprocessorConfig?: LLMMediaPreprocessorConfig; + readonly preprocessorConfig?: MediaPreprocessorConfig; }; export type LLMChatSessionOptions = { @@ -97,7 +97,7 @@ function generateChatTurnWorklet( */ export async function createLLMChatSession( config: LLMModel, - options?: LLMChatSessionOptions, + options?: LLMChatSessionOptions>, runtime?: WorkletRuntime ): Promise> { const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = config; diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts index a1eaec5f0c..b67dae10a3 100644 --- a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -21,11 +21,11 @@ import { */ export function useLLMChatSession( config: LLMModel, - options?: LLMChatSessionOptions & ResourceOptions + options?: LLMChatSessionOptions> & ResourceOptions ) { const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); const { model: session, error } = useModel( - (res) => createLLMChatSession(res, options), + (res) => createLLMChatSession(res, options), resource ?? null ); diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 61795f81c0..91a75f65ba 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -14,7 +14,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; -import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; +import type { LLMModel, MediaPreprocessorConfig } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -853,7 +853,7 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; -const LFM2_5_VL_PREPROCESSOR_CONFIG = { +const LFM2_5_VL_PREPROCESSOR_CONFIG: MediaPreprocessorConfig = { image: { token: { start: '<|image_start|>', end: '<|image_end|>' }, targetShape: [3, 512, 512] as const, From e70ab9c2fc2de2c3f86ffc1f2169814eab7d022c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 13 Aug 2026 23:41:41 +0200 Subject: [PATCH 27/43] refactor(llm): prune internal type exports and enforce LLM-prefixed preprocessor types --- .../src/extensions/llm/chatRenderer.ts | 73 ++++++++++++++----- .../src/extensions/llm/llmRunner.ts | 35 +++++---- .../extensions/llm/tasks/llmChatSession.ts | 14 ++-- .../react-native-executorch/src/models.ts | 4 +- 4 files changed, 85 insertions(+), 41 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts index 443c7284b9..cb94577189 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts @@ -6,41 +6,76 @@ import type { ImageBuffer } from '../cv'; import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/tasks/preprocessing'; import type { Modality, Prompt, MediaInput } from './llmRunner'; -/** High-level media payload types for chat messages. */ +/** Map of supported non-text media input modalities to high-level payloads. */ export type ChatMediaInputMap = { + /** Image payload containing an uncompressed RGBA ImageBuffer. */ image: { readonly kind: 'image'; readonly image: ImageBuffer }; + /** Audio payload containing an uncompressed audio sample array. */ audio: { readonly kind: 'audio'; readonly audio: unknown }; }; +/** Extracts high-level media payloads for allowed modalities `M`. */ export type ChatMediaInput = ChatMediaInputMap[M]; -/** Interleaved multimodal content for high-level ChatMessages. */ -export type ChatMessageContent = - | string - | readonly (string | ChatMediaInput)[]; +/** + * Interleaved text and media content for a chat turn. + * Restricts strictly to a single text string when `M` is `never`. + */ +export type ChatMessageContent = [M] extends [never] + ? string + : string | readonly (string | ChatMediaInput)[]; -/** Message interface for chat history and inputs. */ +/** Conversation turn representing system, user, or assistant messages. */ export type ChatMessage = { + /** Conversation role ('system', 'user', or 'assistant'). */ readonly role: 'system' | 'user' | 'assistant'; + /** Message content string or interleaved text and media payloads. */ readonly content: ChatMessageContent; }; -export type MediaPreprocessorConfig = { - image?: { - readonly token: { readonly start: string; readonly end: string }; - readonly preprocessorOpts: ImagePreprocessorOptions; - readonly targetShape: readonly [number, number, number]; - }; - audio?: { - readonly token: { readonly start: string; readonly end: string }; - }; +/** Sentinel token delimiters framing media placeholders in chat templates. */ +export type LLMMediaTokenConfig = { + /** Opening token delimiter (e.g. `<|image_start|>`). */ + readonly start: string; + /** Closing token delimiter (e.g. `<|image_end|>`). */ + readonly end: string; }; +/** Image preprocessing and sentinel token config for vision-language LLMs. */ +export type LLMImagePreprocessorConfig = { + /** Sentinel token delimiters inserted into Jinja prompts. */ + readonly token: LLMMediaTokenConfig; + /** Image preprocessing options (normalization, resize mode, interpolation). */ + readonly preprocessorOpts: ImagePreprocessorOptions; + /** Fixed target shape expected by native LLM `[C, H, W]`. */ + readonly targetShape: readonly [number, number, number]; +}; + +/** Audio preprocessing and sentinel token config for audio-language LLMs. */ +export type LLMAudioPreprocessorConfig = { + /** Sentinel token delimiters inserted into Jinja prompts. */ + readonly token: LLMMediaTokenConfig; +}; + +/** Map of modality keys to their respective LLM preprocessor configs. */ +export type LLMMediaPreprocessorConfigMap = { + image: LLMImagePreprocessorConfig; + audio: LLMAudioPreprocessorConfig; +}; + +/** + * Preprocessor configuration for enabled modalities `M`. + * Enabled modalities in `M` are REQUIRED; un-declared modalities are FORBIDDEN. + */ +// prettier-ignore +export type LLMMediaPreprocessorConfig = + Pick & { [K in Exclude]?: never }; + /** Options for instantiating a ChatRenderer. */ export type ChatRendererConfig = { readonly chatTemplate: string; readonly bosToken?: string; - readonly preprocessorConfig?: MediaPreprocessorConfig; + readonly preprocessorConfig?: Partial; }; /** Turn rendering options for ChatRenderer. */ @@ -51,8 +86,8 @@ export type RenderOpts = { /** * Handles Jinja template rendering and media tensor preprocessing for chat turns. - * @param config Chat renderer configuration options including template and preprocessor settings. - * @returns A ChatRenderer object with render and dispose methods. + * @param config Renderer configuration including template and preprocessor settings. + * @returns Object with render and dispose methods. */ export function createChatRenderer(config: ChatRendererConfig) { const { chatTemplate, preprocessorConfig, bosToken = '' } = config; @@ -149,7 +184,7 @@ export function createChatRenderer(config: ChatRende const tail = renderedContent.slice(lastIndex); if (tail.length > 0) prompt.push(tail); - return prompt as Prompt; + return prompt as unknown as Prompt; }; return { dispose, render }; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index d32d347e39..3b6012c348 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -22,18 +22,27 @@ export type GenerationStats = { readonly modelLoadEndMs: number; }; -/** Supported non-text media input objects (e.g. images, audio tensors). */ -export type MediaInput = - | { readonly kind: 'image'; readonly image: Tensor } - | { readonly kind: 'audio'; readonly audio: Tensor }; +/** Map of supported non-text media input modalities to native tensor payloads. */ +export type MediaInputMap = { + /** Image modality input holding a preprocessed float32 image tensor. */ + image: { readonly kind: 'image'; readonly image: Tensor }; + /** Audio modality input holding a preprocessed float32 audio waveform tensor. */ + audio: { readonly kind: 'audio'; readonly audio: Tensor }; +}; + +/** Supported non-text input modality keys (e.g. `'image'`, `'audio'`). */ +export type Modality = keyof MediaInputMap; -/** Supported non-text input modality kinds. */ -export type Modality = MediaInput['kind']; +/** Extracts native media input tensor objects for allowed modalities `M`. */ +export type MediaInput = MediaInputMap[M]; -/** Text or interleaved multimodal prompt input for an LLM runner. */ -export type Prompt = - | string - | readonly (string | Extract)[]; +/** + * Text or interleaved multimodal prompt input for a low-level LLM runner. + * Restricts strictly to a single text string when `M` is `never`. + */ +export type Prompt = [M] extends [never] + ? string + : string | readonly (string | MediaInput)[]; /** Handle to a native ExecuTorch LLM runner. */ export type LLMRunner = { @@ -84,11 +93,11 @@ export type LLMRunner = { * Defaults to text-only. * @returns A native LLMRunner instance. */ -export function createLLMRunner( +export function createLLMRunner( modelPath: string, tokenizerPath: string, - modalities?: Ms -): LLMRunner { + modalities?: readonly M[] +): LLMRunner { 'worklet'; return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath, modalities ?? []); } diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index d98a30cbd0..75f1550874 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -16,9 +16,9 @@ import { type ChatMediaInput, type ChatMessageContent, type ChatMessage, - type MediaPreprocessorConfig, - type ChatRendererConfig, - type RenderOpts, + type LLMImagePreprocessorConfig, + type LLMAudioPreprocessorConfig, + type LLMMediaPreprocessorConfig, } from '../chatRenderer'; export type { @@ -28,9 +28,9 @@ export type { ChatMediaInput, ChatMessageContent, ChatMessage, - MediaPreprocessorConfig, - ChatRendererConfig, - RenderOpts, + LLMImagePreprocessorConfig, + LLMAudioPreprocessorConfig, + LLMMediaPreprocessorConfig, }; export type LLMModel = { @@ -38,7 +38,7 @@ export type LLMModel = { readonly tokenizerPath: string; readonly tokenizerConfigPath: string; readonly modalities?: readonly M[]; - readonly preprocessorConfig?: MediaPreprocessorConfig; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; export type LLMChatSessionOptions = { diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 91a75f65ba..74ba894fee 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -14,7 +14,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; -import type { LLMModel, MediaPreprocessorConfig } from './extensions/llm/tasks/llmChatSession'; +import type { LLMModel, LLMMediaPreprocessorConfig } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -853,7 +853,7 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; -const LFM2_5_VL_PREPROCESSOR_CONFIG: MediaPreprocessorConfig = { +const LFM2_5_VL_PREPROCESSOR_CONFIG: LLMMediaPreprocessorConfig<'image'> = { image: { token: { start: '<|image_start|>', end: '<|image_end|>' }, targetShape: [3, 512, 512] as const, From a1e0dd0929a6e2e258ef8d9ecfbee146ad1d62a6 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:10:27 +0200 Subject: [PATCH 28/43] refactor(llm): simplify LLM types and add modalities validation to ChatRenderer --- .../src/extensions/llm/chatRenderer.ts | 61 +++++++------------ .../src/extensions/llm/llmRunner.ts | 39 +++++------- .../extensions/llm/tasks/llmChatSession.ts | 58 ++++++++++-------- .../src/hooks/useLLMChatSession.ts | 9 ++- .../react-native-executorch/src/models.ts | 8 +-- 5 files changed, 77 insertions(+), 98 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts index cb94577189..d312ca5e2c 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts @@ -6,31 +6,20 @@ import type { ImageBuffer } from '../cv'; import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/tasks/preprocessing'; import type { Modality, Prompt, MediaInput } from './llmRunner'; -/** Map of supported non-text media input modalities to high-level payloads. */ -export type ChatMediaInputMap = { - /** Image payload containing an uncompressed RGBA ImageBuffer. */ - image: { readonly kind: 'image'; readonly image: ImageBuffer }; - /** Audio payload containing an uncompressed audio sample array. */ - audio: { readonly kind: 'audio'; readonly audio: unknown }; -}; - -/** Extracts high-level media payloads for allowed modalities `M`. */ -export type ChatMediaInput = ChatMediaInputMap[M]; +/** High-level media payload input for chat turns. */ +export type ChatMediaInput = + | { readonly kind: 'image'; readonly image: ImageBuffer } + | { readonly kind: 'audio'; readonly audio: unknown }; -/** - * Interleaved text and media content for a chat turn. - * Restricts strictly to a single text string when `M` is `never`. - */ -export type ChatMessageContent = [M] extends [never] - ? string - : string | readonly (string | ChatMediaInput)[]; +/** Interleaved text and media content for a chat turn. */ +export type ChatMessageContent = string | readonly (string | ChatMediaInput)[]; /** Conversation turn representing system, user, or assistant messages. */ -export type ChatMessage = { +export type ChatMessage = { /** Conversation role ('system', 'user', or 'assistant'). */ readonly role: 'system' | 'user' | 'assistant'; /** Message content string or interleaved text and media payloads. */ - readonly content: ChatMessageContent; + readonly content: ChatMessageContent; }; /** Sentinel token delimiters framing media placeholders in chat templates. */ @@ -57,25 +46,18 @@ export type LLMAudioPreprocessorConfig = { readonly token: LLMMediaTokenConfig; }; -/** Map of modality keys to their respective LLM preprocessor configs. */ -export type LLMMediaPreprocessorConfigMap = { - image: LLMImagePreprocessorConfig; - audio: LLMAudioPreprocessorConfig; +/** Preprocessor configuration for media modalities. */ +export type LLMMediaPreprocessorConfig = { + readonly image?: LLMImagePreprocessorConfig; + readonly audio?: LLMAudioPreprocessorConfig; }; -/** - * Preprocessor configuration for enabled modalities `M`. - * Enabled modalities in `M` are REQUIRED; un-declared modalities are FORBIDDEN. - */ -// prettier-ignore -export type LLMMediaPreprocessorConfig = - Pick & { [K in Exclude]?: never }; - /** Options for instantiating a ChatRenderer. */ export type ChatRendererConfig = { readonly chatTemplate: string; readonly bosToken?: string; - readonly preprocessorConfig?: Partial; + readonly modalities?: readonly Modality[]; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; /** Turn rendering options for ChatRenderer. */ @@ -89,8 +71,8 @@ export type RenderOpts = { * @param config Renderer configuration including template and preprocessor settings. * @returns Object with render and dispose methods. */ -export function createChatRenderer(config: ChatRendererConfig) { - const { chatTemplate, preprocessorConfig, bosToken = '' } = config; +export function createChatRenderer(config: ChatRendererConfig) { + const { chatTemplate, modalities, preprocessorConfig, bosToken = '' } = config; const template = new Template(chatTemplate); let imgPreprocessor: ReturnType | undefined; @@ -108,7 +90,7 @@ export function createChatRenderer(config: ChatRende imgPreprocessor?.dispose(); }; - const render = (message: ChatMessage, renderOpts?: RenderOpts): Prompt => { + const render = (message: ChatMessage, renderOpts?: RenderOpts): Prompt => { const isFirst = renderOpts?.isFirst ?? false; const addGenPrompt = renderOpts?.addGenPrompt ?? false; @@ -131,8 +113,11 @@ export function createChatRenderer(config: ChatRende continue; } - if (!preprocessorConfig || !(item.kind in preprocessorConfig)) { - throw RnExecuTorchError('INVALID_ARGUMENT', `Modality '${item.kind}' not supported`); + if (modalities !== undefined && !modalities.includes(item.kind)) { + throw RnExecuTorchError( + 'INVALID_ARGUMENT', + `Modality '${item.kind}' is not supported by this model instance.` + ); } if (item.kind === 'image' && 'image' in item) { @@ -184,7 +169,7 @@ export function createChatRenderer(config: ChatRende const tail = renderedContent.slice(lastIndex); if (tail.length > 0) prompt.push(tail); - return prompt as unknown as Prompt; + return prompt as unknown as Prompt; }; return { dispose, render }; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 3b6012c348..1d0d584975 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -22,36 +22,25 @@ export type GenerationStats = { readonly modelLoadEndMs: number; }; -/** Map of supported non-text media input modalities to native tensor payloads. */ -export type MediaInputMap = { - /** Image modality input holding a preprocessed float32 image tensor. */ - image: { readonly kind: 'image'; readonly image: Tensor }; - /** Audio modality input holding a preprocessed float32 audio waveform tensor. */ - audio: { readonly kind: 'audio'; readonly audio: Tensor }; -}; - /** Supported non-text input modality keys (e.g. `'image'`, `'audio'`). */ -export type Modality = keyof MediaInputMap; +export type Modality = 'image' | 'audio'; -/** Extracts native media input tensor objects for allowed modalities `M`. */ -export type MediaInput = MediaInputMap[M]; +/** Low-level non-text media input tensor payloads. */ +export type MediaInput = + | { readonly kind: 'image'; readonly image: Tensor } + | { readonly kind: 'audio'; readonly audio: Tensor }; -/** - * Text or interleaved multimodal prompt input for a low-level LLM runner. - * Restricts strictly to a single text string when `M` is `never`. - */ -export type Prompt = [M] extends [never] - ? string - : string | readonly (string | MediaInput)[]; +/** Text or interleaved multimodal prompt input for a low-level LLM runner. */ +export type Prompt = string | readonly (string | MediaInput)[]; /** Handle to a native ExecuTorch LLM runner. */ -export type LLMRunner = { +export type LLMRunner = { /** Path to the local model file. */ readonly modelPath: string; /** Path to the local tokenizer configuration file. */ readonly tokenizerPath: string; /** List of supported non-text input modalities for this runner (e.g. 'image', 'audio'). */ - readonly modalities: readonly M[]; + readonly modalities: readonly Modality[]; /** Disposes the native LLM runner and releases the loaded model memory. */ dispose(): void; @@ -63,7 +52,7 @@ export type LLMRunner = { * Prefills the runner with a prompt to build up the KV cache. * @param prompt The prefill text or multimodal prompt. */ - prefill(prompt: Prompt): void; + prefill(prompt: Prompt): void; /** * Generates text continuation from a prompt. @@ -73,7 +62,7 @@ export type LLMRunner = { * @returns Generation performance statistics. */ generate( - prompt: Prompt, + prompt: Prompt, config?: GenerationConfig, onToken?: (token: string) => void ): GenerationStats; @@ -93,11 +82,11 @@ export type LLMRunner = { * Defaults to text-only. * @returns A native LLMRunner instance. */ -export function createLLMRunner( +export function createLLMRunner( modelPath: string, tokenizerPath: string, - modalities?: readonly M[] -): LLMRunner { + modalities?: readonly Modality[] +): LLMRunner { 'worklet'; return rnexecutorchJsi.llm.createLLMRunner(modelPath, tokenizerPath, modalities ?? []); } diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 75f1550874..3830dd97d2 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -33,45 +33,45 @@ export type { LLMMediaPreprocessorConfig, }; -export type LLMModel = { +export type LLMModel = { readonly modelPath: string; readonly tokenizerPath: string; readonly tokenizerConfigPath: string; - readonly modalities?: readonly M[]; - readonly preprocessorConfig?: LLMMediaPreprocessorConfig; + readonly modalities?: readonly Modality[]; + readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; -export type LLMChatSessionOptions = { - readonly initialMessages?: readonly ChatMessage[]; +export type LLMChatSessionOptions = { + readonly initialMessages?: readonly ChatMessage[]; readonly generationConfig?: GenerationConfig; readonly stopTokens?: readonly string[]; }; -export type LLMGenerationResult = { - readonly response: ChatMessageContent; +export type LLMGenerationResult = { + readonly response: ChatMessageContent; readonly stats: GenerationStats; }; -export type LLMChatSession = { +export type LLMChatSession = { stop(): void; dispose(): void; - getHistory(): readonly ChatMessage[]; + getHistory(): readonly ChatMessage[]; sendMessage( - message: ChatMessageContent, + message: ChatMessageContent | ChatMessage, onToken?: (token: string) => void, genConfig?: GenerationConfig - ): Promise>; + ): Promise; }; -function generateChatTurnWorklet( - runner: LLMRunner, - prompt: Prompt, +function generateChatTurnWorklet( + runner: LLMRunner, + prompt: Prompt, options: { readonly genConfig: GenerationConfig; readonly stopTokens: readonly string[]; readonly onToken?: (token: string) => void; } -): LLMGenerationResult { +): LLMGenerationResult { 'worklet'; const { genConfig, stopTokens, onToken } = options; @@ -95,11 +95,11 @@ function generateChatTurnWorklet( * @param runtime The worklet runtime thread to run native generation on. * @returns A Promise resolving to an LLMChatSession instance. */ -export async function createLLMChatSession( - config: LLMModel, - options?: LLMChatSessionOptions>, +export async function createLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions, runtime?: WorkletRuntime -): Promise> { +): Promise { const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = config; const initialMessages = options?.initialMessages ?? []; @@ -111,11 +111,11 @@ export async function createLLMChatSession( const { chatTemplate, bosToken, eosToken } = tokenizerConfig; // Prepare messages' renderer - const renderer = createChatRenderer({ chatTemplate, bosToken, preprocessorConfig }); + const renderer = createChatRenderer({ chatTemplate, bosToken, modalities, preprocessorConfig }); const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; // Prepare runner - const history: ChatMessage[] = []; + const history: ChatMessage[] = []; const runner = await wrapAsync(createLLMRunner, runtime)(modelPath, tokenizerPath, modalities); const prefill = wrapAsync(runner.prefill, runtime); @@ -134,14 +134,20 @@ export async function createLLMChatSession( const generateChatTurn = wrapAsync(generateChatTurnWorklet, runtime); const sendMessage = async ( - message: ChatMessageContent, + message: ChatMessageContent | ChatMessage, onToken?: (token: string) => void, genConfig?: GenerationConfig - ): Promise> => { - const userMsg = { role: 'user' as const, content: message }; - const prompt = renderer.render(userMsg, { isFirst: history.length === 0, addGenPrompt: true }); + ): Promise => { + let msg: ChatMessage; + if (typeof message === 'object' && 'role' in message) { + msg = message; + } else { + msg = { role: 'user', content: message }; + } - history.push(userMsg); + const prompt = renderer.render(msg, { isFirst: history.length === 0, addGenPrompt: true }); + + history.push(msg); const opts = { genConfig: { ...defaultGenerationConfig, ...genConfig }, stopTokens, onToken }; const { response, stats } = await generateChatTurn(runner, prompt, opts); diff --git a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts index b67dae10a3..5d3bde5176 100644 --- a/packages/react-native-executorch/src/hooks/useLLMChatSession.ts +++ b/packages/react-native-executorch/src/hooks/useLLMChatSession.ts @@ -2,7 +2,6 @@ import { useModel } from './useModel'; import { useResourceDownload, type ResourceOptions } from './useResourceDownload'; import { createLLMChatSession, - type Modality, type LLMModel, type LLMChatSessionOptions, } from '../extensions/llm/tasks/llmChatSession'; @@ -19,13 +18,13 @@ import { * @returns An object containing the session's loading state, error, download progress, * and chat functions. */ -export function useLLMChatSession( - config: LLMModel, - options?: LLMChatSessionOptions> & ResourceOptions +export function useLLMChatSession( + config: LLMModel, + options?: LLMChatSessionOptions & ResourceOptions ) { const { resource, downloadProgress, downloadError } = useResourceDownload(config, options); const { model: session, error } = useModel( - (res) => createLLMChatSession(res, options), + (res) => createLLMChatSession(res, options), resource ?? null ); diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 74ba894fee..05b8fe9d42 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -853,7 +853,7 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; -const LFM2_5_VL_PREPROCESSOR_CONFIG: LLMMediaPreprocessorConfig<'image'> = { +const LFM2_5_VL_PREPROCESSOR_CONFIG: LLMMediaPreprocessorConfig = { image: { token: { start: '<|image_start|>', end: '<|image_end|>' }, targetShape: [3, 512, 512] as const, @@ -864,21 +864,21 @@ const LFM2_5_VL_PREPROCESSOR_CONFIG: LLMMediaPreprocessorConfig<'image'> = { }, }, }; -const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel<'image'> = { +const LFM2_5_VL_450M_XNNPACK_8DA4W: LLMModel = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/xnnpack/lfm_2_5_vl_450m_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -const LFM2_5_VL_450M_MLX_INT4: LLMModel<'image'> = { +const LFM2_5_VL_450M_MLX_INT4: LLMModel = { modelPath: `${LFM2_5_BASE_URL}/vl_450m/mlx/lfm_2_5_vl_450m_mlx_int4.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_450m/tokenizer_config.json`, modalities: ['image'], preprocessorConfig: LFM2_5_VL_PREPROCESSOR_CONFIG, }; -const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel<'image'> = { +const LFM2_5_VL_1_6B_XNNPACK_8DA4W: LLMModel = { modelPath: `${LFM2_5_BASE_URL}/vl_1_6b/xnnpack/lfm_2_5_vl_1_6b_xnnpack_8da4w.pte`, tokenizerPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer.json`, tokenizerConfigPath: `${LFM2_5_BASE_URL}/vl_1_6b/tokenizer_config.json`, From bf740518f5ca35af497e13fb8b75d79fb50ae85a Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:20:22 +0200 Subject: [PATCH 29/43] refactor(llm): update ChatProcessOptions attribute names and default values --- apps/nlp/app/llm/index.tsx | 14 +++++-- .../{chatRenderer.ts => chatPreprocessor.ts} | 42 +++++++++++-------- .../src/extensions/llm/index.ts | 2 +- .../extensions/llm/tasks/llmChatSession.ts | 27 ++++++++---- 4 files changed, 55 insertions(+), 30 deletions(-) rename packages/react-native-executorch/src/extensions/llm/{chatRenderer.ts => chatPreprocessor.ts} (80%) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index fe8d385c16..dddb21af7a 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -23,7 +23,7 @@ import { import ScreenWrapper from '../../components/ScreenWrapper'; import { getImage, skImageToBuffer } from '../../utils'; -const MODEL = models.llm.LFM2_5_VL_450M; +const MODEL = models.llm.GEMMA4_E2B; const SYSTEM_PROMPT = 'You are a helpful multimodal assistant by Liquid AI.'; const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; @@ -99,9 +99,15 @@ function LLMContent() { ]); try { - const payload = []; - if (currentImage) payload.push({ kind: 'image' as const, image: currentImage.buffer }); - payload.push(textMessage || 'What is in this image?'); + let payload; + if (currentImage) { + payload = [ + { kind: 'image' as const, image: currentImage.buffer }, + textMessage || 'What is in this image?', + ]; + } else { + payload = textMessage; + } const { response, stats } = await sendMessage(payload, (token) => { setStreamingResponse((prev) => (prev !== null ? prev + token : token)); diff --git a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts similarity index 80% rename from packages/react-native-executorch/src/extensions/llm/chatRenderer.ts rename to packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index d312ca5e2c..d4a31cf164 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatRenderer.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -52,26 +52,28 @@ export type LLMMediaPreprocessorConfig = { readonly audio?: LLMAudioPreprocessorConfig; }; -/** Options for instantiating a ChatRenderer. */ -export type ChatRendererConfig = { +/** Options for instantiating a ChatPreprocessor. */ +export type ChatPreprocessorConfig = { readonly chatTemplate: string; readonly bosToken?: string; readonly modalities?: readonly Modality[]; readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; -/** Turn rendering options for ChatRenderer. */ -export type RenderOpts = { - readonly isFirst?: boolean; - readonly addGenPrompt?: boolean; +/** Turn preprocessing options for ChatPreprocessor. */ +export type ChatProcessOptions = { + /** Whether this is the initial turn in the conversation (prepends BOS token if true). Defaults to `false`. */ + readonly isFirstTurn?: boolean; + /** Whether to append the assistant generation prompt (e.g. `<|im_start|>assistant\n`). Defaults to `true`. */ + readonly addGenerationPrompt?: boolean; }; /** - * Handles Jinja template rendering and media tensor preprocessing for chat turns. - * @param config Renderer configuration including template and preprocessor settings. - * @returns Object with render and dispose methods. + * Handles Jinja template formatting and media tensor preprocessing for chat turns. + * @param config Preprocessor configuration including template and preprocessor settings. + * @returns Object with process and dispose methods. */ -export function createChatRenderer(config: ChatRendererConfig) { +export function createChatPreprocessor(config: ChatPreprocessorConfig) { const { chatTemplate, modalities, preprocessorConfig, bosToken = '' } = config; const template = new Template(chatTemplate); @@ -90,15 +92,15 @@ export function createChatRenderer(config: ChatRendererConfig) { imgPreprocessor?.dispose(); }; - const render = (message: ChatMessage, renderOpts?: RenderOpts): Prompt => { - const isFirst = renderOpts?.isFirst ?? false; - const addGenPrompt = renderOpts?.addGenPrompt ?? false; + const process = (message: ChatMessage, opts?: ChatProcessOptions): Prompt => { + const isFirstTurn = opts?.isFirstTurn ?? false; + const addGenerationPrompt = opts?.addGenerationPrompt ?? true; if (typeof message.content === 'string') { /* eslint-disable camelcase */ return template.render({ - bos_token: isFirst ? bosToken : '', - add_generation_prompt: addGenPrompt, + bos_token: isFirstTurn ? bosToken : '', + add_generation_prompt: addGenerationPrompt, messages: [{ role: message.role, content: message.content }], }); /* eslint-enable */ @@ -120,6 +122,10 @@ export function createChatRenderer(config: ChatRendererConfig) { ); } + if (!preprocessorConfig || !(item.kind in preprocessorConfig)) { + throw RnExecuTorchError('INVALID_ARGUMENT', `Modality '${item.kind}' not supported`); + } + if (item.kind === 'image' && 'image' in item) { if (!preprocessorConfig?.image || !imgPreprocessor || !imgShape) { throw RnExecuTorchError( @@ -147,8 +153,8 @@ export function createChatRenderer(config: ChatRendererConfig) { /* eslint-disable camelcase */ const renderedContent = template.render({ - bos_token: isFirst ? bosToken : '', - add_generation_prompt: addGenPrompt, + bos_token: isFirstTurn ? bosToken : '', + add_generation_prompt: addGenerationPrompt, messages: [{ role: message.role, content: syntheticContent }], }); /* eslint-enable */ @@ -172,5 +178,5 @@ export function createChatRenderer(config: ChatRendererConfig) { return prompt as unknown as Prompt; }; - return { dispose, render }; + return { dispose, process }; } diff --git a/packages/react-native-executorch/src/extensions/llm/index.ts b/packages/react-native-executorch/src/extensions/llm/index.ts index 37967c90cc..8b338b7712 100644 --- a/packages/react-native-executorch/src/extensions/llm/index.ts +++ b/packages/react-native-executorch/src/extensions/llm/index.ts @@ -1,3 +1,3 @@ export * from './llmRunner'; -export * from './chatRenderer'; +export * from './chatPreprocessor'; export * from './tokenizerConfig'; diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 3830dd97d2..8df888f3e7 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -12,14 +12,14 @@ import { } from '../llmRunner'; import { parseTokenizerConfig } from '../tokenizerConfig'; import { - createChatRenderer, + createChatPreprocessor, type ChatMediaInput, type ChatMessageContent, type ChatMessage, type LLMImagePreprocessorConfig, type LLMAudioPreprocessorConfig, type LLMMediaPreprocessorConfig, -} from '../chatRenderer'; +} from '../chatPreprocessor'; export type { GenerationConfig, @@ -110,8 +110,13 @@ export async function createLLMChatSession( const tokenizerConfig = parseTokenizerConfig(JSON.parse(tokenizerConfigStr)); const { chatTemplate, bosToken, eosToken } = tokenizerConfig; - // Prepare messages' renderer - const renderer = createChatRenderer({ chatTemplate, bosToken, modalities, preprocessorConfig }); + // Prepare chat preprocessor + const chatPreprocessor = createChatPreprocessor({ + chatTemplate, + bosToken, + modalities, + preprocessorConfig, + }); const stopTokens = [...(options?.stopTokens ?? []), ...(eosToken ? [eosToken] : [])]; // Prepare runner @@ -121,14 +126,19 @@ export async function createLLMChatSession( // Prefill initial messages for (const msg of initialMessages) { - await prefill(renderer.render(msg, { isFirst: history.length === 0 })); + await prefill( + chatPreprocessor.process(msg, { + isFirstTurn: history.length === 0, + addGenerationPrompt: false, + }) + ); history.push(msg); } const stop = () => runner.stop(); const dispose = () => { runner.dispose(); - renderer.dispose(); + chatPreprocessor.dispose(); }; const generateChatTurn = wrapAsync(generateChatTurnWorklet, runtime); @@ -145,7 +155,10 @@ export async function createLLMChatSession( msg = { role: 'user', content: message }; } - const prompt = renderer.render(msg, { isFirst: history.length === 0, addGenPrompt: true }); + const prompt = chatPreprocessor.process(msg, { + isFirstTurn: history.length === 0, + addGenerationPrompt: true, + }); history.push(msg); From 0dff43d3bc119ac8fea4843e8e4d5960e9d8fcc7 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:25:35 +0200 Subject: [PATCH 30/43] docs(llm): format method JSDoc blocks with multi-line comments and newlines --- .../src/extensions/llm/chatPreprocessor.ts | 10 ++++- .../extensions/llm/tasks/llmChatSession.ts | 45 +++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index d4a31cf164..7f8585855a 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -62,9 +62,15 @@ export type ChatPreprocessorConfig = { /** Turn preprocessing options for ChatPreprocessor. */ export type ChatProcessOptions = { - /** Whether this is the initial turn in the conversation (prepends BOS token if true). Defaults to `false`. */ + /** + * Whether this is the initial turn in the conversation (prepends BOS token if + * true). Defaults to `false`. + */ readonly isFirstTurn?: boolean; - /** Whether to append the assistant generation prompt (e.g. `<|im_start|>assistant\n`). Defaults to `true`. */ + /** + * Whether to append the assistant generation prompt (e.g. + * `<|im_start|>assistant\n`). Defaults to `true`. + */ readonly addGenerationPrompt?: boolean; }; diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 8df888f3e7..e5ba572305 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -33,29 +33,74 @@ export type { LLMMediaPreprocessorConfig, }; +/** + * Model configuration required to instantiate an LLM chat session. + * @category Types + */ export type LLMModel = { + /** Local path or remote URL of the `.pte` model file. */ readonly modelPath: string; + /** Local path or remote URL of the `tokenizer.json` file. */ readonly tokenizerPath: string; + /** Local path or remote URL of the `tokenizer_config.json` file. */ readonly tokenizerConfigPath: string; + /** Supported input non-text modalities (e.g. `['image']`). */ readonly modalities?: readonly Modality[]; + /** Media preprocessor configuration. */ readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; +/** + * Options for configuring an LLM chat session. + * @category Types + */ export type LLMChatSessionOptions = { + /** Initial conversation history to prefill into the model KV cache. */ readonly initialMessages?: readonly ChatMessage[]; + /** Default generation configuration options. */ readonly generationConfig?: GenerationConfig; + /** Additional stop tokens that interrupt token generation. */ readonly stopTokens?: readonly string[]; }; +/** + * Generation result returned by an LLM chat turn. + * @category Types + */ export type LLMGenerationResult = { + /** The generated assistant response text. */ readonly response: ChatMessageContent; + /** Generation performance statistics. */ readonly stats: GenerationStats; }; +/** + * Handle to an active LLM chat session. + * @category Types + */ export type LLMChatSession = { + /** + * Interrupts and stops any active token generation call. + */ stop(): void; + + /** + * Releases native model memory and preprocessor resources. + */ dispose(): void; + + /** + * Returns the read-only conversation message history. + */ getHistory(): readonly ChatMessage[]; + + /** + * Sends a user message or chat turn to the model and generates a response. + * @param message Message string, media payload array, or ChatMessage object. + * @param onToken Callback fired on the RN thread for each decoded token. + * @param genConfig Generation options overriding session defaults. + * @returns A promise resolving to the response and generation stats. + */ sendMessage( message: ChatMessageContent | ChatMessage, onToken?: (token: string) => void, From a06b8048c5753fa4da46364a034d3b1a341d1a23 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:33:41 +0200 Subject: [PATCH 31/43] docs(llm): add missing @category JSDoc tags to all exported LLM types and APIs --- .../src/extensions/llm/chatPreprocessor.ts | 47 +++++++++++++--- .../src/extensions/llm/llmRunner.ts | 56 +++++++++++++++---- .../extensions/llm/tasks/llmChatSession.ts | 1 + 3 files changed, 84 insertions(+), 20 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index 7f8585855a..cc6fb786a2 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -6,15 +6,24 @@ import type { ImageBuffer } from '../cv'; import { createImagePreprocessor, type ImagePreprocessorOptions } from '../cv/tasks/preprocessing'; import type { Modality, Prompt, MediaInput } from './llmRunner'; -/** High-level media payload input for chat turns. */ +/** + * High-level media payload input for chat turns. + * @category Types + */ export type ChatMediaInput = | { readonly kind: 'image'; readonly image: ImageBuffer } | { readonly kind: 'audio'; readonly audio: unknown }; -/** Interleaved text and media content for a chat turn. */ +/** + * Interleaved text and media content for a chat turn. + * @category Types + */ export type ChatMessageContent = string | readonly (string | ChatMediaInput)[]; -/** Conversation turn representing system, user, or assistant messages. */ +/** + * Conversation turn representing system, user, or assistant messages. + * @category Types + */ export type ChatMessage = { /** Conversation role ('system', 'user', or 'assistant'). */ readonly role: 'system' | 'user' | 'assistant'; @@ -22,7 +31,10 @@ export type ChatMessage = { readonly content: ChatMessageContent; }; -/** Sentinel token delimiters framing media placeholders in chat templates. */ +/** + * Sentinel token delimiters framing media placeholders in chat templates. + * @category Types + */ export type LLMMediaTokenConfig = { /** Opening token delimiter (e.g. `<|image_start|>`). */ readonly start: string; @@ -30,7 +42,10 @@ export type LLMMediaTokenConfig = { readonly end: string; }; -/** Image preprocessing and sentinel token config for vision-language LLMs. */ +/** + * Image preprocessing and sentinel token config for vision-language LLMs. + * @category Types + */ export type LLMImagePreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ readonly token: LLMMediaTokenConfig; @@ -40,19 +55,28 @@ export type LLMImagePreprocessorConfig = { readonly targetShape: readonly [number, number, number]; }; -/** Audio preprocessing and sentinel token config for audio-language LLMs. */ +/** + * Audio preprocessing and sentinel token config for audio-language LLMs. + * @category Types + */ export type LLMAudioPreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ readonly token: LLMMediaTokenConfig; }; -/** Preprocessor configuration for media modalities. */ +/** + * Preprocessor configuration for media modalities. + * @category Types + */ export type LLMMediaPreprocessorConfig = { readonly image?: LLMImagePreprocessorConfig; readonly audio?: LLMAudioPreprocessorConfig; }; -/** Options for instantiating a ChatPreprocessor. */ +/** + * Options for instantiating a ChatPreprocessor. + * @category Types + */ export type ChatPreprocessorConfig = { readonly chatTemplate: string; readonly bosToken?: string; @@ -60,7 +84,10 @@ export type ChatPreprocessorConfig = { readonly preprocessorConfig?: LLMMediaPreprocessorConfig; }; -/** Turn preprocessing options for ChatPreprocessor. */ +/** + * Turn preprocessing options for ChatPreprocessor. + * @category Types + */ export type ChatProcessOptions = { /** * Whether this is the initial turn in the conversation (prepends BOS token if @@ -76,6 +103,7 @@ export type ChatProcessOptions = { /** * Handles Jinja template formatting and media tensor preprocessing for chat turns. + * @category Typescript API * @param config Preprocessor configuration including template and preprocessor settings. * @returns Object with process and dispose methods. */ @@ -99,6 +127,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { }; const process = (message: ChatMessage, opts?: ChatProcessOptions): Prompt => { + 'worklet'; const isFirstTurn = opts?.isFirstTurn ?? false; const addGenerationPrompt = opts?.addGenerationPrompt ?? true; diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 1d0d584975..3834be3ba2 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -3,49 +3,82 @@ import { rnexecutorchJsi } from '../../native/bridge'; declare const llmRunnerBrand: unique symbol; -/** Configuration options for LLM text generation. */ +/** + * Configuration options for LLM text generation. + * @category Types + */ export type GenerationConfig = { + /** Whether to echo the prompt in the generated output. */ readonly echo?: boolean; + /** Whether to ignore EOS tokens during generation. */ readonly ignoreEos?: boolean; + /** Maximum number of new tokens to generate. */ readonly maxNewTokens?: number; + /** Sampling temperature for token selection. */ readonly temperature?: number; }; -/** Execution and performance statistics for a generation call. */ +/** + * Execution and performance statistics for a generation call. + * @category Types + */ export type GenerationStats = { + /** Number of tokens in the input prompt. */ readonly numPromptTokens: number; + /** Number of newly generated tokens. */ readonly numGeneratedTokens: number; + /** Time elapsed in milliseconds to generate the first token. */ readonly firstTokenMs: number; + /** Timestamp in milliseconds when inference started. */ readonly inferenceStartMs: number; + /** Timestamp in milliseconds when inference completed. */ readonly inferenceEndMs: number; + /** Timestamp in milliseconds when model loading started. */ readonly modelLoadStartMs: number; + /** Timestamp in milliseconds when model loading completed. */ readonly modelLoadEndMs: number; }; -/** Supported non-text input modality keys (e.g. `'image'`, `'audio'`). */ +/** + * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). + * @category Types + */ export type Modality = 'image' | 'audio'; -/** Low-level non-text media input tensor payloads. */ +/** + * Low-level non-text media input tensor payloads. + * @category Types + */ export type MediaInput = | { readonly kind: 'image'; readonly image: Tensor } | { readonly kind: 'audio'; readonly audio: Tensor }; -/** Text or interleaved multimodal prompt input for a low-level LLM runner. */ +/** + * Text or interleaved multimodal prompt input for a low-level LLM runner. + * @category Types + */ export type Prompt = string | readonly (string | MediaInput)[]; -/** Handle to a native ExecuTorch LLM runner. */ +/** + * Handle to a native ExecuTorch LLM runner. + * @category Types + */ export type LLMRunner = { /** Path to the local model file. */ readonly modelPath: string; /** Path to the local tokenizer configuration file. */ readonly tokenizerPath: string; - /** List of supported non-text input modalities for this runner (e.g. 'image', 'audio'). */ + /** List of supported non-text input modalities for this runner (e.g. `['image']`). */ readonly modalities: readonly Modality[]; - /** Disposes the native LLM runner and releases the loaded model memory. */ + /** + * Disposes the native LLM runner and releases the loaded model memory. + */ dispose(): void; - /** Interrupts and stops any active generation call on this runner. */ + /** + * Interrupts and stops any active generation call on this runner. + */ stop(): void; /** @@ -76,8 +109,9 @@ export type LLMRunner = { /** * Creates a native ExecuTorch LLM runner instance. - * @param modelPath Path to the local .pte model file. - * @param tokenizerPath Path to the local tokenizer configuration file (e.g. tokenizer.json). + * @category Typescript API + * @param modelPath Path to the local `.pte` model file. + * @param tokenizerPath Path to the local tokenizer configuration file (e.g. `tokenizer.json`). * @param modalities List of supported input non-text modalities (e.g. `['image']`). * Defaults to text-only. * @returns A native LLMRunner instance. diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index e5ba572305..54d257eeea 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -135,6 +135,7 @@ function generateChatTurnWorklet( /** * Instantiates an LLM chat session using background thread execution. + * @category Typescript API * @param config Model configuration containing model, tokenizer, and tokenizer config paths. * @param options Custom generation and state options. * @param runtime The worklet runtime thread to run native generation on. From 85798b7f1b7b86659945f669bdc3bbb1708ad37e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:38:16 +0200 Subject: [PATCH 32/43] revert(nlp): simplify ModelPicker component and remove NestedModelPicker --- apps/nlp/components/ModelPicker.tsx | 116 ---------------------------- 1 file changed, 116 deletions(-) diff --git a/apps/nlp/components/ModelPicker.tsx b/apps/nlp/components/ModelPicker.tsx index fa02662f12..ae3893eb2c 100644 --- a/apps/nlp/components/ModelPicker.tsx +++ b/apps/nlp/components/ModelPicker.tsx @@ -79,119 +79,3 @@ const styles = StyleSheet.create({ fontWeight: '600', }, }); - -function hasModelConfig(obj: any): boolean { - if (!obj || typeof obj !== 'object') return false; - if (typeof obj.modelPath === 'string') return true; - for (const key of Object.keys(obj)) { - if (hasModelConfig(obj[key])) return true; - } - return false; -} - -function getSubOptions(node: any): { key: string; value: any }[] { - if (!node || typeof node !== 'object') return []; - return Object.keys(node) - .filter((key) => hasModelConfig(node[key])) - .map((key) => ({ key, value: node[key] })); -} - -export function findPath(node: any, target: any, currentPath: string[] = []): string[] | null { - if (!node || typeof node !== 'object') return null; - - // Search child nodes first to match the deepest leaf node, preventing early returns on parent container objects - for (const key of Object.keys(node)) { - if (hasModelConfig(node[key])) { - const found = findPath(node[key], target, [...currentPath, key]); - if (found) return found; - } - } - - if (node === target) return currentPath; - if (target && typeof node.modelPath === 'string' && node.modelPath === target.modelPath) { - return currentPath; - } - - return null; -} - -function getDefaultPath(node: any, currentPath: string[] = []): string[] { - const subOptions = getSubOptions(node); - if (subOptions.length === 0) return currentPath; - const first = subOptions[0]!; - return getDefaultPath(first.value, [...currentPath, first.key]); -} - -function getValueAtPath(registry: any, path: string[]): any { - let current = registry; - for (const key of path) { - if (current && typeof current === 'object') { - current = current[key]; - } else { - return null; - } - } - return current; -} - -export interface NestedModelPickerProps { - labelPrefix?: string; - registry: any; - selectedValue: any; - onValueChange: (value: any) => void; -} - -export function NestedModelPicker({ - labelPrefix = '', - registry, - selectedValue, - onValueChange, -}: NestedModelPickerProps) { - const path = findPath(registry, selectedValue) || getDefaultPath(registry); - const pickers: React.ReactNode[] = []; - let currentNode = registry; - - for (let i = 0; i <= path.length; i++) { - const subOptions = getSubOptions(currentNode); - if (subOptions.length === 0) break; - - const selectedKey = path[i]; - const options = subOptions.map((opt) => ({ - label: opt.key, - value: opt.key, - })); - - const label = - i === 0 - ? `${labelPrefix ? labelPrefix + ' ' : ''}Family` - : i === 1 - ? `${labelPrefix ? labelPrefix + ' ' : ''}Variant` - : `${labelPrefix ? labelPrefix + ' ' : ''}Subvariant`; - - const levelIndex = i; - pickers.push( - { - const newPath = [...path.slice(0, levelIndex), newKey]; - const newNode = getValueAtPath(registry, newPath); - const leafPath = [...newPath, ...getDefaultPath(newNode)]; - const leafValue = getValueAtPath(registry, leafPath); - onValueChange(leafValue); - }} - /> - ); - - const nextKey = selectedKey || subOptions[0]?.key; - if (nextKey && currentNode[nextKey]) { - currentNode = currentNode[nextKey]; - } else { - break; - } - } - - return {pickers}; -} From d87e527f0e32c7e44d4255f19c5f18d422c190bf Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:42:45 +0200 Subject: [PATCH 33/43] docs(llm): add @category tags to tokenizerConfig --- .../src/extensions/llm/tokenizerConfig.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts index e06b3113db..ce07ca4101 100644 --- a/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts +++ b/packages/react-native-executorch/src/extensions/llm/tokenizerConfig.ts @@ -1,6 +1,9 @@ import { RnExecuTorchError } from '../../core/error'; -/** Model chat template configuration resolved from tokenizer config file. */ +/** + * Model chat template configuration resolved from tokenizer config file. + * @category Types + */ export type TokenizerChatConfig = { readonly chatTemplate: string; readonly bosToken?: string; @@ -17,6 +20,7 @@ function resolveToken(token: unknown): string | undefined { /** * Parses raw JSON configuration from `tokenizer_config.json` into a normalized format. + * @category Utils * @param config Raw JSON object from tokenizer_config.json. * @returns A parsed TokenizerChatConfig object. */ From ae5e20b3d42c4717ed41039989620e340d36380f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:48:54 +0200 Subject: [PATCH 34/43] fix --- apps/nlp/app/llm/index.tsx | 4 ++-- .../src/extensions/llm/chatPreprocessor.ts | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index dddb21af7a..688886115e 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -23,7 +23,7 @@ import { import ScreenWrapper from '../../components/ScreenWrapper'; import { getImage, skImageToBuffer } from '../../utils'; -const MODEL = models.llm.GEMMA4_E2B; +const MODEL = models.llm.LFM2_5_350M; const SYSTEM_PROMPT = 'You are a helpful multimodal assistant by Liquid AI.'; const INITIAL_MESSAGES: ChatMessage[] = [{ role: 'system', content: SYSTEM_PROMPT }]; const GENERATION_CONFIG = { temperature: 0.7, maxNewTokens: 512, echo: false }; @@ -103,7 +103,7 @@ function LLMContent() { if (currentImage) { payload = [ { kind: 'image' as const, image: currentImage.buffer }, - textMessage || 'What is in this image?', + textMessage || "What's in this image?", ]; } else { payload = textMessage; diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index cc6fb786a2..63c8e86537 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -150,17 +150,13 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { continue; } - if (modalities !== undefined && !modalities.includes(item.kind)) { + if (!modalities || !modalities.includes(item.kind)) { throw RnExecuTorchError( 'INVALID_ARGUMENT', `Modality '${item.kind}' is not supported by this model instance.` ); } - if (!preprocessorConfig || !(item.kind in preprocessorConfig)) { - throw RnExecuTorchError('INVALID_ARGUMENT', `Modality '${item.kind}' not supported`); - } - if (item.kind === 'image' && 'image' in item) { if (!preprocessorConfig?.image || !imgPreprocessor || !imgShape) { throw RnExecuTorchError( From fcbb0566bd09866e3cfb6840facfba191793a1bc Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:54:05 +0200 Subject: [PATCH 35/43] fix --- packages/react-native-executorch/src/models.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 05b8fe9d42..5a1866c50e 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -14,7 +14,7 @@ import { type WhisperSttModel, WHISPER_LANGUAGES, } from './extensions/speech/tasks/whisperSpeechToText'; -import type { LLMModel, LLMMediaPreprocessorConfig } from './extensions/llm/tasks/llmChatSession'; +import type { LLMModel } from './extensions/llm/tasks/llmChatSession'; import { IMAGENET_NORM, IMAGENET1K_LABELS, @@ -853,7 +853,7 @@ const LFM2_5_350M_MLX_INT4: LLMModel = { tokenizerConfigPath: `${LFM2_5_BASE_URL}/350m/tokenizer_config.json`, }; -const LFM2_5_VL_PREPROCESSOR_CONFIG: LLMMediaPreprocessorConfig = { +const LFM2_5_VL_PREPROCESSOR_CONFIG = { image: { token: { start: '<|image_start|>', end: '<|image_end|>' }, targetShape: [3, 512, 512] as const, From 39a8dae681bf605a643ddad9a069684124a3dcd2 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 00:58:19 +0200 Subject: [PATCH 36/43] refactor(llm): derive Modality type directly from MediaInput['kind'] --- .../src/extensions/llm/llmRunner.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index 3834be3ba2..dd5246e242 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -39,12 +39,6 @@ export type GenerationStats = { readonly modelLoadEndMs: number; }; -/** - * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). - * @category Types - */ -export type Modality = 'image' | 'audio'; - /** * Low-level non-text media input tensor payloads. * @category Types @@ -53,6 +47,12 @@ export type MediaInput = | { readonly kind: 'image'; readonly image: Tensor } | { readonly kind: 'audio'; readonly audio: Tensor }; +/** + * Supported non-text input modality keys (e.g. `'image'`, `'audio'`). + * @category Types + */ +export type Modality = MediaInput['kind']; + /** * Text or interleaved multimodal prompt input for a low-level LLM runner. * @category Types From 41e2fa69f7f7f214d60200bc37af15f16dedf1d8 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 01:04:13 +0200 Subject: [PATCH 37/43] fix --- .../react-native-executorch/cpp/extensions/llm/llm_runner.cpp | 2 +- .../src/extensions/llm/chatPreprocessor.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp index b564d652a3..ce32532593 100644 --- a/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp +++ b/packages/react-native-executorch/cpp/extensions/llm/llm_runner.cpp @@ -255,7 +255,7 @@ jsi::Value LLMRunnerHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &nam auto inputs = parsePrompt(rt, "LLMRunner.prefill", args[0], self->modalities_); auto result = self->runner_->prefill(inputs); - if (result.error() != executorch::runtime::Error::Ok) { + if (!result.ok()) { std::string errorMsg = executorch::runtime::to_string(result.error()); throw error::ExecutionFailed(std::format("LLMRunner.prefill: Failed: {}", errorMsg), result.error()); } diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index 63c8e86537..913d5e8db3 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -206,7 +206,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { const tail = renderedContent.slice(lastIndex); if (tail.length > 0) prompt.push(tail); - return prompt as unknown as Prompt; + return prompt; }; return { dispose, process }; From 1021096746dbb8eb73fdc6bf33de929895949cfb Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 01:04:56 +0200 Subject: [PATCH 38/43] update --- .../src/extensions/llm/chatPreprocessor.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index 913d5e8db3..ddb36f2f1c 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -157,7 +157,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { ); } - if (item.kind === 'image' && 'image' in item) { + if (item.kind === 'image') { if (!preprocessorConfig?.image || !imgPreprocessor || !imgShape) { throw RnExecuTorchError( 'INVALID_ARGUMENT', @@ -177,7 +177,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { mediaInputs.push({ kind: 'image', image: tImage }); } - if (item.kind === 'audio' && 'audio' in item) { + if (item.kind === 'audio') { throw RnExecuTorchError('INVALID_ARGUMENT', 'Audio input not yet supported'); } } From 3031d0646b85b5e8baf5859eab7812f87274bad5 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 01:08:19 +0200 Subject: [PATCH 39/43] small fix --- .../src/extensions/llm/chatPreprocessor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index ddb36f2f1c..f0c40c184a 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -114,7 +114,7 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { let imgPreprocessor: ReturnType | undefined; let imgShape: [number, number, number] | undefined; - if (preprocessorConfig !== undefined && preprocessorConfig.image !== undefined) { + if (preprocessorConfig?.image !== undefined) { imgShape = [...preprocessorConfig.image.targetShape]; imgPreprocessor = createImagePreprocessor(preprocessorConfig.image.preprocessorOpts, imgShape); } From a677df7b0ee52189c2974d3c38fa03a094465459 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 01:19:18 +0200 Subject: [PATCH 40/43] fix: mutable history --- .../src/extensions/llm/tasks/llmChatSession.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index 54d257eeea..fb325560f1 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -220,6 +220,6 @@ export async function createLLMChatSession( stop, dispose, sendMessage, - getHistory: () => history, + getHistory: () => [...history], }; } From 09d777d89a066c68da6ffae0a6c71d9b09690f41 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 01:57:47 +0200 Subject: [PATCH 41/43] update --- .../src/extensions/llm/tasks/llmChatSession.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index fb325560f1..1283cf8346 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -194,19 +194,19 @@ export async function createLLMChatSession( onToken?: (token: string) => void, genConfig?: GenerationConfig ): Promise => { - let msg: ChatMessage; + let input: ChatMessage; if (typeof message === 'object' && 'role' in message) { - msg = message; + input = message; } else { - msg = { role: 'user', content: message }; + input = { role: 'user', content: message }; } - const prompt = chatPreprocessor.process(msg, { + const prompt = chatPreprocessor.process(input, { isFirstTurn: history.length === 0, addGenerationPrompt: true, }); - history.push(msg); + history.push(input); const opts = { genConfig: { ...defaultGenerationConfig, ...genConfig }, stopTokens, onToken }; const { response, stats } = await generateChatTurn(runner, prompt, opts); From cb5530a7b866c20b23150e912f9e6344a1e2d8ac Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 02:27:51 +0200 Subject: [PATCH 42/43] fix: remove worklet --- .../src/extensions/llm/chatPreprocessor.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index f0c40c184a..4b422593e5 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -127,7 +127,6 @@ export function createChatPreprocessor(config: ChatPreprocessorConfig) { }; const process = (message: ChatMessage, opts?: ChatProcessOptions): Prompt => { - 'worklet'; const isFirstTurn = opts?.isFirstTurn ?? false; const addGenerationPrompt = opts?.addGenerationPrompt ?? true; From dde32f93a612f71874a1d547a103aa920308a822 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 14 Aug 2026 02:28:34 +0200 Subject: [PATCH 43/43] simplify --- .../src/extensions/llm/chatPreprocessor.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts index 4b422593e5..e6437cad32 100644 --- a/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts +++ b/packages/react-native-executorch/src/extensions/llm/chatPreprocessor.ts @@ -31,24 +31,13 @@ export type ChatMessage = { readonly content: ChatMessageContent; }; -/** - * Sentinel token delimiters framing media placeholders in chat templates. - * @category Types - */ -export type LLMMediaTokenConfig = { - /** Opening token delimiter (e.g. `<|image_start|>`). */ - readonly start: string; - /** Closing token delimiter (e.g. `<|image_end|>`). */ - readonly end: string; -}; - /** * Image preprocessing and sentinel token config for vision-language LLMs. * @category Types */ export type LLMImagePreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ - readonly token: LLMMediaTokenConfig; + readonly token: { readonly start: string; readonly end: string }; /** Image preprocessing options (normalization, resize mode, interpolation). */ readonly preprocessorOpts: ImagePreprocessorOptions; /** Fixed target shape expected by native LLM `[C, H, W]`. */ @@ -61,7 +50,7 @@ export type LLMImagePreprocessorConfig = { */ export type LLMAudioPreprocessorConfig = { /** Sentinel token delimiters inserted into Jinja prompts. */ - readonly token: LLMMediaTokenConfig; + readonly token: { readonly start: string; readonly end: string }; }; /**