diff --git a/docs/guide/troubleshooting.md b/docs/guide/troubleshooting.md index f60a7745..71865b1b 100644 --- a/docs/guide/troubleshooting.md +++ b/docs/guide/troubleshooting.md @@ -164,3 +164,50 @@ Ensure you're not using the `Administrator` user for `npm install` nor to run th To do that, go to `Settings > Update & Security > For developers` and enable `Developer mode`. After that, delete the `.cache` folder under your user directory and try building the app again. + +## Customizing `postinstall` Behavior {#postinstall-behavior} +When installing `node-llama-cpp`, its `postinstall` script checks whether the prebuilt binaries +are compatible with current machine (which they almost always are, at least the CPU-only ones which are the last resort fallback), +and when not, attempts [building the native bindings from source](./building-from-source.md). + +When attempting to [build from source](./building-from-source.md), if the machine lacks the required build tools, +the build will fail and indicative error messages will direct you to the specific commands you need to run +or packages you need to install in order for the build process to succeed. + +If you want to customize the `postinstall` behavior, you can do so using any of the following methods: +* Passing the `--node-llama-cpp-postinstall=` flag to the `npm install` command. +* Setting the `NODE_LLAMA_CPP_POSTINSTALL` environment variable to `` before running `npm install`. +* Configuring `config.nodeLlamaCppPostinstall` on your project's `package.json` to ``. +
+ This will only work when your module is installed globally using `npm install -g` or for a non-library project when you run `npm install` in the project root; it will not work when your module is installed as a dependency of another module. + +Where `` can be one of the following options: +* **`auto` (default)**: the default behavior explained above. +* **`ignoreFailedBuild`**: same as the default behavior, + but a failed build will not throw an error and will be ignored, which means the installation will succeed. + Using [`getLlama`](../api/functions/getLlama.md) for the first time will attempt building from source again by default. +* **`skip`**: skip the entire `postinstall` script. + If the prebuilt binaries are incompatible with the current machine, + using [`getLlama`](../api/functions/getLlama.md) for the first time will attempt building from source by default. + +::: code-group +```shell [npm install flag] +npm install --node-llama-cpp-postinstall=ignoreFailedBuild +``` + +```shell [env var (bash)] +NODE_LLAMA_CPP_POSTINSTALL=ignoreFailedBuild npm install +``` + +```shell [env var (using cross-env)] +npx --yes cross-env NODE_LLAMA_CPP_POSTINSTALL=ignoreFailedBuild npm install +``` + +```json [package.json] +{ + "config": { + "nodeLlamaCppPostinstall": "ignoreFailedBuild" + } +} +``` +::: diff --git a/src/cli/commands/OnPostInstallCommand.ts b/src/cli/commands/OnPostInstallCommand.ts index e81e9b0b..9731b71b 100644 --- a/src/cli/commands/OnPostInstallCommand.ts +++ b/src/cli/commands/OnPostInstallCommand.ts @@ -1,10 +1,16 @@ +import path from "path"; +import {fileURLToPath} from "url"; import {CommandModule} from "yargs"; import chalk from "chalk"; -import {defaultSkipDownload, documentationPageUrls} from "../../config.js"; +import {defaultSkipDownload, documentationPageUrls, defaultNodeLlamaCppPostinstall} from "../../config.js"; import {getLlamaForOptions} from "../../bindings/getLlama.js"; import {setForceShowConsoleLogPrefix} from "../../state.js"; import {isRunningUnderRosetta} from "../utils/isRunningUnderRosetta.js"; import {getConsoleLogPrefix} from "../../utils/getConsoleLogPrefix.js"; +import {parsePackageJsonConfig, resolvePackageJsonConfig} from "../utils/packageJsonConfig.js"; +import {detectCurrentPackageManager} from "../utils/packageManager.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); type OnPostInstallCommand = null; @@ -13,7 +19,22 @@ export const OnPostInstallCommand: CommandModule = describe: false, async handler() { if (defaultSkipDownload) - return; + return void process.exit(0); + + const nlcConfig = parsePackageJsonConfig(await resolvePackageJsonConfig(__dirname)); + const postinstallConfig = (defaultNodeLlamaCppPostinstall == null || defaultNodeLlamaCppPostinstall === "auto") + ? nlcConfig.nodeLlamaCppPostinstall ?? defaultNodeLlamaCppPostinstall + : defaultNodeLlamaCppPostinstall; + + // set via a `--node-llama-cpp-postinstall=skip` flag on an `npm install` command + // (prefer `--node-llama-cpp-postinstall=ignoreFailedBuild` if you really need it) + if (postinstallConfig === "skip") { + console.info( + getConsoleLogPrefix(false, false), + "Skipping node-llama-cpp postinstall due to a 'skip' configuration" + ); + return void process.exit(0); + } setForceShowConsoleLogPrefix(true); @@ -34,7 +55,10 @@ export const OnPostInstallCommand: CommandModule = "troubleshooting: " + documentationPageUrls.troubleshooting.RosettaIllegalHardwareInstruction ); - process.exit(1); + if (postinstallConfig === "ignoreFailedBuild") + process.exit(0); + else + process.exit(1); } try { @@ -47,7 +71,25 @@ export const OnPostInstallCommand: CommandModule = process.exit(0); } catch (err) { console.error(err); - process.exit(1); + + const packageManager = detectCurrentPackageManager(); + if (postinstallConfig === "auto" && packageManager === "npm") + console.info( + getConsoleLogPrefix(false, false), + "To disable node-llama-cpp's postinstall for this 'npm install', use the '--node-llama-cpp-postinstall=skip' flag when running 'npm install' command" + ); + + if (postinstallConfig === "auto") + console.info( + getConsoleLogPrefix(false, false), + "To customize node-llama-cpp's postinstall behavior, see the troubleshooting guide: " + + documentationPageUrls.troubleshooting.PostinstallBehavior + ); + + if (postinstallConfig === "ignoreFailedBuild") + process.exit(0); + else + process.exit(1); } } }; diff --git a/src/cli/utils/packageJsonConfig.ts b/src/cli/utils/packageJsonConfig.ts new file mode 100644 index 00000000..f9ed820d --- /dev/null +++ b/src/cli/utils/packageJsonConfig.ts @@ -0,0 +1,70 @@ +import path from "path"; +import fs from "fs-extra"; +import {NodeLlamaCppPostinstallBehavior} from "../../types.js"; + +export async function resolvePackageJsonConfig(startDir: string) { + const currentConfig: Record = {}; + + let currentDirPath = path.resolve(startDir); + while (true) { + applyConfig(currentConfig, await readPackageJsonConfig(path.join(currentDirPath, "package.json"))); + + const parentDirPath = path.dirname(currentDirPath); + if (parentDirPath === currentDirPath) + break; + + currentDirPath = parentDirPath; + } + + const npmPackageJsonPath = process.env["npm_package_json"] ?? ""; + if (npmPackageJsonPath !== "") + applyConfig(currentConfig, await readPackageJsonConfig(npmPackageJsonPath)); + + return currentConfig; +} + +export function parsePackageJsonConfig(config: Record) { + const res: NlcPackageJsonConfig = {}; + + const castedConfig = config as NlcPackageJsonConfig; + + if (castedConfig.nodeLlamaCppPostinstall === "auto" || + castedConfig.nodeLlamaCppPostinstall === "ignoreFailedBuild" || + castedConfig.nodeLlamaCppPostinstall === "skip" + ) + res.nodeLlamaCppPostinstall = castedConfig.nodeLlamaCppPostinstall; + else + void (castedConfig.nodeLlamaCppPostinstall satisfies undefined); + + return res; +} + +export type NlcPackageJsonConfig = { + nodeLlamaCppPostinstall?: NodeLlamaCppPostinstallBehavior +}; + +async function readPackageJsonConfig(packageJsonPath: string) { + try { + if (!(await fs.pathExists(packageJsonPath))) + return {}; + + const packageJsonContent = await fs.readFile(packageJsonPath, "utf8"); + const packageJson = JSON.parse(packageJsonContent); + const config = packageJson?.config; + if (typeof config === "object") + return config; + + return {}; + } catch (err) { + return {}; + } +} + +function applyConfig(baseConfig: Record, newConfig: Record) { + for (const key of Object.keys(newConfig)) { + if (Object.hasOwn(baseConfig, key)) + continue; + + baseConfig[key] = newConfig[key]; + } +} diff --git a/src/cli/utils/packageManager.ts b/src/cli/utils/packageManager.ts new file mode 100644 index 00000000..0b76eef1 --- /dev/null +++ b/src/cli/utils/packageManager.ts @@ -0,0 +1,16 @@ +export function detectCurrentPackageManager(): "npm" | "bun" | "pnpm" | "deno" | "yarn" | undefined { + const userAgent = (process.env["npm_config_user_agent"] ?? "").toLowerCase(); + + if (userAgent.startsWith("bun/")) + return "bun"; + else if (userAgent.startsWith("pnpm/")) + return "pnpm"; + else if (userAgent.startsWith("yarn/")) + return "yarn"; + else if (userAgent.startsWith("deno/")) + return "deno"; + else if (userAgent.startsWith("npm/")) + return "npm"; + + return undefined; +} diff --git a/src/config.ts b/src/config.ts index 631df45c..5337d012 100644 --- a/src/config.ts +++ b/src/config.ts @@ -8,6 +8,7 @@ import {getBinariesGithubRelease} from "./bindings/utils/binariesGithubRelease.j import { nodeLlamaCppGpuOptions, LlamaLogLevel, LlamaLogLevelValues, parseNodeLlamaCppGpuOption, nodeLlamaCppGpuOffStringOptions } from "./bindings/types.js"; +import type {NodeLlamaCppPostinstallBehavior} from "./types.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -75,6 +76,15 @@ export const defaultLlamaCppDebugMode = env.get("NODE_LLAMA_CPP_DEBUG") export const defaultSkipDownload = env.get("NODE_LLAMA_CPP_SKIP_DOWNLOAD") .default("false") .asBool(); + +// set via a `--node-llama-cpp-postinstall=ignoreFailedBuild` flag on an `npm install` command +export const defaultNodeLlamaCppPostinstall = env.get("NODE_LLAMA_CPP_POSTINSTALL") + .default( + env.get("npm_config_node_llama_cpp_postinstall") + .default("auto") + .asEnum(["auto", "ignoreFailedBuild", "skip"] as const satisfies NodeLlamaCppPostinstallBehavior[]) + ) + .asEnum(["auto", "ignoreFailedBuild", "skip"] as const satisfies NodeLlamaCppPostinstallBehavior[]); export const defaultBindingTestLogLevel = env.get("NODE_LLAMA_CPP_BINDING_TEST_LOG_LEVEL") .default(LlamaLogLevel.error) .asEnum(LlamaLogLevelValues); @@ -125,7 +135,8 @@ export const documentationPageUrls = { } }, troubleshooting: { - RosettaIllegalHardwareInstruction: documentationUrl + "/guide/troubleshooting#illegal-hardware-instruction" + RosettaIllegalHardwareInstruction: documentationUrl + "/guide/troubleshooting#illegal-hardware-instruction", + PostinstallBehavior: documentationUrl + "/guide/troubleshooting#postinstall-behavior" } } as const; export const newGithubIssueUrl = "https://github.com/withcatai/node-llama-cpp/issues"; diff --git a/src/gguf/types/GgufMetadataTypes.ts b/src/gguf/types/GgufMetadataTypes.ts index b4c4b458..a2cd41b9 100644 --- a/src/gguf/types/GgufMetadataTypes.ts +++ b/src/gguf/types/GgufMetadataTypes.ts @@ -193,7 +193,8 @@ export enum GgufFileType { MOSTLY_Q4_0_8_8 = 35, // deprecated MOSTLY_TQ1_0 = 36, MOSTLY_TQ2_0 = 37, - MOSTLY_MXFP4_MOE = 38 + MOSTLY_MXFP4_MOE = 38, + MOSTLY_NVFP4 = 39 } diff --git a/src/gguf/types/GgufTensorInfoTypes.ts b/src/gguf/types/GgufTensorInfoTypes.ts index ed750329..1ada8204 100644 --- a/src/gguf/types/GgufTensorInfoTypes.ts +++ b/src/gguf/types/GgufTensorInfoTypes.ts @@ -60,5 +60,6 @@ export const enum GgmlType { IQ4_NL_4_4 = 36, IQ4_NL_4_8 = 37, IQ4_NL_8_8 = 38, - MXFP4 = 39 // MXFP4 (1 block) + MXFP4 = 39, // MXFP4 (1 block) + NVFP4 = 40 // NVFP4 (4 blocks, E4M3 scale) } diff --git a/src/gguf/utils/ggufQuantNames.ts b/src/gguf/utils/ggufQuantNames.ts index abff8a8f..3e2c5c65 100644 --- a/src/gguf/utils/ggufQuantNames.ts +++ b/src/gguf/utils/ggufQuantNames.ts @@ -4,6 +4,7 @@ export const ggufQuantNames = new Map([ ["Q4_0", GgufFileType.MOSTLY_Q4_0], ["Q4_1", GgufFileType.MOSTLY_Q4_1], ["MXFP4", GgufFileType.MOSTLY_MXFP4_MOE], + ["NVFP4", GgufFileType.MOSTLY_MXFP4_MOE], ["Q5_0", GgufFileType.MOSTLY_Q5_0], ["Q5_1", GgufFileType.MOSTLY_Q5_1], ["IQ2_XXS", GgufFileType.MOSTLY_IQ2_XXS], diff --git a/src/types.ts b/src/types.ts index 4d24d155..630da6c1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -477,3 +477,5 @@ export type LLamaContextualDryRepeatPenalty = { */ sequenceBreakers?: string[] }; + +export type NodeLlamaCppPostinstallBehavior = "auto" | "ignoreFailedBuild" | "skip";