From f9cf8527e068151c3f8b1e2ce3354a21c3ac4504 Mon Sep 17 00:00:00 2001 From: Glucksberg Date: Sat, 22 Aug 2026 14:38:49 -0400 Subject: [PATCH 1/5] kernel: clamp zero-size host registrations; surface WDDM lock-pool guidance on pin failure cudaHostRegister rejects a 0-byte request with cudaErrorInvalidValue. The NVFP4/FP8 offload loaders register per-layer scale banks whose padded size can round to zero on dense layers, which aborts the whole pin pipeline with a cryptic 'cudaHostRegister failed for 0.0 GiB'. Clamp to 1 byte so the bank maps like every other one (parity with alloc_pinned_tensor, which already does 'nbytes == 0 ? 1 : nbytes'). Also re-raise HostBank.pin failures with the bank size and an actionable hint: on Windows/WDDM the pageable-locking pool is roughly half of system RAM, so large MoE models can exceed it even with free RAM available. Repro: RTX 3080 Ti 12 GB / 32 GB RAM / driver 591.86 (CUDA 13.1), serving ornith-ai/Ornith-1.5-35B-A3B-NVFP4 -- fails at PinPipeline.wait() with 'failed for 0.0 GiB' before this change. --- python/freetoken/kernel/csrc/pinned_tensor.cpp | 11 ++++++++--- python/freetoken/moe/host_banks.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/python/freetoken/kernel/csrc/pinned_tensor.cpp b/python/freetoken/kernel/csrc/pinned_tensor.cpp index c3947adf..7be0ca7a 100644 --- a/python/freetoken/kernel/csrc/pinned_tensor.cpp +++ b/python/freetoken/kernel/csrc/pinned_tensor.cpp @@ -95,9 +95,14 @@ int64_t host_device_ptr(int64_t host_ptr) { } void host_register(int64_t addr, int64_t nbytes) { - const cudaError_t err = - cudaHostRegister(reinterpret_cast(addr), static_cast(nbytes), - cudaHostRegisterPortable | cudaHostRegisterMapped); + // Zero-size registrations are rejected by the driver with cudaErrorInvalidValue. + // Callers pin-after-fill small scale banks whose padded size can round to zero + // on dense layers; clamp so they register as a 1-byte mapped region instead. + const size_t reg_nbytes = + static_cast(nbytes > 0 ? nbytes : 1); + const cudaError_t err = cudaHostRegister( + reinterpret_cast(addr), reg_nbytes, + cudaHostRegisterPortable | cudaHostRegisterMapped); TORCH_CHECK(err == cudaSuccess, "cudaHostRegister failed: ", cudaGetErrorString(err)); } diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py index 5b50b25e..47851238 100644 --- a/python/freetoken/moe/host_banks.py +++ b/python/freetoken/moe/host_banks.py @@ -88,8 +88,19 @@ def pin(self) -> None: try: host_register(self.addr, len(self._buf)) except RuntimeError as exc: + msg = str(exc) + if "out of memory" in msg: + hint = ( + "the driver refused to lock more host memory (on Windows/WDDM " + "the pageable-locking pool is roughly half of system RAM); " + "close other processes, add RAM, or serve some layers with " + "--moe-backend cpu" + ) + else: + hint = "see the driver error above" raise RuntimeError( - f"cudaHostRegister failed for {len(self._buf) / 2**30:.1f} GiB" + f"cudaHostRegister failed for a {len(self._buf) / 2**30:.2f} GiB bank " + f"({msg}): {hint}" ) from exc self._pinned = True From 12303b4c7fe5cb1c2c1c464e6e5cddd8eaba3a1e Mon Sep 17 00:00:00 2001 From: Glucksberg Date: Sat, 22 Aug 2026 14:42:12 -0400 Subject: [PATCH 2/5] docs: bodies da issue #55 e PR #56 --- issue_body.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ pr_body.md | 18 ++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 issue_body.md create mode 100644 pr_body.md diff --git a/issue_body.md b/issue_body.md new file mode 100644 index 00000000..42b62d07 --- /dev/null +++ b/issue_body.md @@ -0,0 +1,47 @@ +### Environment + +- FreeToken 0.1.1 (Windows wheel from `FreeToken-Web@beta`) and cross-checked against `main` (0.1.2) +- Windows 11 + WSL2-free native run, RTX 3080 Ti 12 GB, driver 591.86 (CUDA 13.1), 32 GB RAM +- Model: `ornith-ai/Ornith-1.5-35B-A3B-NVFP4` (36 B MoE, qwen3_5_moe arch, ~17 GB of host bank sources) + +### Summary + +On a 32 GB Windows machine, serving any MoE model whose total bank sources exceed ~14 GB is currently impossible through the offload backend, because: + +1. the NVFP4/FP8 source banks are allocated unpinned (`alloc_layer_banks` → lazy mmaps) and only mapped later via `HostBank.pin()` → `cudaHostRegister`; +2. WDDM enforces a pageable-locking pool of roughly half of system RAM (~13.8 GB measured here), so registering all banks raises `cudaHostRegister failed: out of memory` partway through the pipeline; +3. that failure aborts the whole serve with `RuntimeError: cudaHostRegister failed for X GiB` at `PinPipeline.wait()`, and `_build_copy_plan` then has no usable device pointers; +4. the designed escape hatch for exactly this situation (LOCKED/PAGEABLE residency served by the CPU executor) is scaffolded but not implemented: + - `HostResidency` docstring: *"their movement paths are not implemented here"* (`moe/host_banks.py`, `HostResidency`) + - `HostBank.lock()` raises `NotImplementedError` + - `OffloadMoeCache.set_bank_sources` raises `NotImplementedError` for any non-PINNED layer (`moe/offload_cache.py`) + +This matches the on-machine observation in the tweet thread where FreeToken was announced ("windows supports dma, we can pin half of the memory") — half of RAM is simply not enough for 35 B-class models on 32 GB boxes. + +### Repro + +``` +ft serve --model D:\models\Ornith-1.5-35B-A3B-NVFP4 --expert-load parallel +``` + +→ `RuntimeError: cudaHostRegister failed ... out of memory` once cumulative registrations reach ~13.8 GB. With an instrumented build that catches the per-bank failures, we measured across several runs: + +- device_ptr failures: 240 tensors / 16.93 GB of bank sources need mapped pointers +- successful in-place `cudaHostRegister`: ceiling consistently ~13.77–13.82 GB, then OOM +- every unmapped source left in the copy plan poisons execution later (`CUDA error: invalid argument` surfacing asynchronously at KV-pool init) + +### Additional smaller issues found along the way + +1. **Zero-size registrations**: `host_register` rejects a 0-byte request (`cudaErrorInvalidValue`); small scale banks whose padded size rounds to zero trip this. `alloc_pinned_tensor` already clamps (`nbytes == 0 ? 1 : nbytes`); `host_register` does not. +2. **KV pool sizing ignores `--max-seq-len-override`**: with a 16 K override the pool still requests 146 450 pages (~2.8 GB bf16). `--num-tokens` works as an escape hatch. +3. **`--moe-cpu-layers` does not reduce bank/copy-plan membership** (tested with explicit id lists and counts): all layers' sources still go through `set_bank_sources`/`device_ptr`, so the flag cannot be used to fit under the quota today. + +### Proposed direction + +Implement the missing LOCKED/PAGEABLE movement paths for Windows: + +- `HostBank.lock()` via `VirtualLock` (+ `SetProcessWorkingSetSize` to raise the working-set floor); Linux via `mlock` +- graceful per-bank fallback inside `PinPipeline._run`: on register failure mark the bank LOCKED/PAGEABLE instead of failing the drain +- plumb real `layer_residency` from the loaders into `ExpertBanks` → `set_bank_sources` (accepting LOCKED) and route non-PINNED layers to the CPU executor, reusing the existing `--moe-cpu-layers` routing + +We validated the mechanics end-to-end with a runtime harness on the affected machine: in-place registration up to quota succeeds, VirtualLock-based locking of the remaining banks keeps them resident, and the CPU-executor route initializes cleanly (`--moe-backend cpu` completes init with zero copy-plan entries). Happy to contribute the implementation if the approach sounds right. diff --git a/pr_body.md b/pr_body.md new file mode 100644 index 00000000..dded9978 --- /dev/null +++ b/pr_body.md @@ -0,0 +1,18 @@ +### What + +Two small, self-contained fixes found while debugging Windows/WDDM pin-quota failures with large MoE models (full repro and measurements in #55): + +1. **`host_register`: clamp zero-byte registrations to 1 byte.** `cudaHostRegister` rejects a 0-byte request with `cudaErrorInvalidValue`. The NVFP4/FP8 offload loaders register per-layer scale banks whose padded size can round to zero on dense layers, which aborts the entire pin pipeline with a cryptic `cudaHostRegister failed for 0.0 GiB`. `alloc_pinned_tensor` in the same file already clamps (`nbytes == 0 ? 1 : nbytes`) — this makes `host_register` consistent with it. + +2. **`HostBank.pin`: actionable error on lock-pool exhaustion.** On Windows/WDDM the pageable-locking pool is roughly half of system RAM, so registering ~17 GB of MoE banks on a 32 GB box fails with a bare driver OOM long before system RAM is exhausted. The re-raised error now carries the bank size plus a hint (free RAM / add RAM / `--moe-backend cpu`), instead of just `cudaHostRegister failed for 13.77 GiB`. + +### Validation + +- Repro machine: RTX 3080 Ti 12 GB / 32 GB RAM / driver 591.86 (CUDA 13.1), serving `ornith-ai/Ornith-1.5-35B-A3B-NVFP4`. +- Before: serve aborts at `PinPipeline.wait()` with `RuntimeError: cudaHostRegister failed for 0.0 GiB`. +- With an equivalent runtime patch (clamp + graceful handling), registration proceeds normally until the WDDM pool ceiling (~13.8 GB measured) — see #55 for the instrumented numbers. +- The clamp itself is exercised by the same path (`alloc_pinned_tensor` has used the identical guard since it shipped). + +### Notes + +These unblock diagnosing the bigger residency gap tracked in #55 (LOCKED/PAGEABLE movement paths are scaffolded but not implemented). Not included here to keep the diff minimal. From ebecb642c12fa9f197281cc45b859ec3fb4823b9 Mon Sep 17 00:00:00 2001 From: Glucksberg Date: Sat, 22 Aug 2026 14:42:30 -0400 Subject: [PATCH 3/5] chore: mover bodies de issue/PR para fora do repositorio --- issue_body.md | 47 ----------------------------------------------- pr_body.md | 18 ------------------ 2 files changed, 65 deletions(-) delete mode 100644 issue_body.md delete mode 100644 pr_body.md diff --git a/issue_body.md b/issue_body.md deleted file mode 100644 index 42b62d07..00000000 --- a/issue_body.md +++ /dev/null @@ -1,47 +0,0 @@ -### Environment - -- FreeToken 0.1.1 (Windows wheel from `FreeToken-Web@beta`) and cross-checked against `main` (0.1.2) -- Windows 11 + WSL2-free native run, RTX 3080 Ti 12 GB, driver 591.86 (CUDA 13.1), 32 GB RAM -- Model: `ornith-ai/Ornith-1.5-35B-A3B-NVFP4` (36 B MoE, qwen3_5_moe arch, ~17 GB of host bank sources) - -### Summary - -On a 32 GB Windows machine, serving any MoE model whose total bank sources exceed ~14 GB is currently impossible through the offload backend, because: - -1. the NVFP4/FP8 source banks are allocated unpinned (`alloc_layer_banks` → lazy mmaps) and only mapped later via `HostBank.pin()` → `cudaHostRegister`; -2. WDDM enforces a pageable-locking pool of roughly half of system RAM (~13.8 GB measured here), so registering all banks raises `cudaHostRegister failed: out of memory` partway through the pipeline; -3. that failure aborts the whole serve with `RuntimeError: cudaHostRegister failed for X GiB` at `PinPipeline.wait()`, and `_build_copy_plan` then has no usable device pointers; -4. the designed escape hatch for exactly this situation (LOCKED/PAGEABLE residency served by the CPU executor) is scaffolded but not implemented: - - `HostResidency` docstring: *"their movement paths are not implemented here"* (`moe/host_banks.py`, `HostResidency`) - - `HostBank.lock()` raises `NotImplementedError` - - `OffloadMoeCache.set_bank_sources` raises `NotImplementedError` for any non-PINNED layer (`moe/offload_cache.py`) - -This matches the on-machine observation in the tweet thread where FreeToken was announced ("windows supports dma, we can pin half of the memory") — half of RAM is simply not enough for 35 B-class models on 32 GB boxes. - -### Repro - -``` -ft serve --model D:\models\Ornith-1.5-35B-A3B-NVFP4 --expert-load parallel -``` - -→ `RuntimeError: cudaHostRegister failed ... out of memory` once cumulative registrations reach ~13.8 GB. With an instrumented build that catches the per-bank failures, we measured across several runs: - -- device_ptr failures: 240 tensors / 16.93 GB of bank sources need mapped pointers -- successful in-place `cudaHostRegister`: ceiling consistently ~13.77–13.82 GB, then OOM -- every unmapped source left in the copy plan poisons execution later (`CUDA error: invalid argument` surfacing asynchronously at KV-pool init) - -### Additional smaller issues found along the way - -1. **Zero-size registrations**: `host_register` rejects a 0-byte request (`cudaErrorInvalidValue`); small scale banks whose padded size rounds to zero trip this. `alloc_pinned_tensor` already clamps (`nbytes == 0 ? 1 : nbytes`); `host_register` does not. -2. **KV pool sizing ignores `--max-seq-len-override`**: with a 16 K override the pool still requests 146 450 pages (~2.8 GB bf16). `--num-tokens` works as an escape hatch. -3. **`--moe-cpu-layers` does not reduce bank/copy-plan membership** (tested with explicit id lists and counts): all layers' sources still go through `set_bank_sources`/`device_ptr`, so the flag cannot be used to fit under the quota today. - -### Proposed direction - -Implement the missing LOCKED/PAGEABLE movement paths for Windows: - -- `HostBank.lock()` via `VirtualLock` (+ `SetProcessWorkingSetSize` to raise the working-set floor); Linux via `mlock` -- graceful per-bank fallback inside `PinPipeline._run`: on register failure mark the bank LOCKED/PAGEABLE instead of failing the drain -- plumb real `layer_residency` from the loaders into `ExpertBanks` → `set_bank_sources` (accepting LOCKED) and route non-PINNED layers to the CPU executor, reusing the existing `--moe-cpu-layers` routing - -We validated the mechanics end-to-end with a runtime harness on the affected machine: in-place registration up to quota succeeds, VirtualLock-based locking of the remaining banks keeps them resident, and the CPU-executor route initializes cleanly (`--moe-backend cpu` completes init with zero copy-plan entries). Happy to contribute the implementation if the approach sounds right. diff --git a/pr_body.md b/pr_body.md deleted file mode 100644 index dded9978..00000000 --- a/pr_body.md +++ /dev/null @@ -1,18 +0,0 @@ -### What - -Two small, self-contained fixes found while debugging Windows/WDDM pin-quota failures with large MoE models (full repro and measurements in #55): - -1. **`host_register`: clamp zero-byte registrations to 1 byte.** `cudaHostRegister` rejects a 0-byte request with `cudaErrorInvalidValue`. The NVFP4/FP8 offload loaders register per-layer scale banks whose padded size can round to zero on dense layers, which aborts the entire pin pipeline with a cryptic `cudaHostRegister failed for 0.0 GiB`. `alloc_pinned_tensor` in the same file already clamps (`nbytes == 0 ? 1 : nbytes`) — this makes `host_register` consistent with it. - -2. **`HostBank.pin`: actionable error on lock-pool exhaustion.** On Windows/WDDM the pageable-locking pool is roughly half of system RAM, so registering ~17 GB of MoE banks on a 32 GB box fails with a bare driver OOM long before system RAM is exhausted. The re-raised error now carries the bank size plus a hint (free RAM / add RAM / `--moe-backend cpu`), instead of just `cudaHostRegister failed for 13.77 GiB`. - -### Validation - -- Repro machine: RTX 3080 Ti 12 GB / 32 GB RAM / driver 591.86 (CUDA 13.1), serving `ornith-ai/Ornith-1.5-35B-A3B-NVFP4`. -- Before: serve aborts at `PinPipeline.wait()` with `RuntimeError: cudaHostRegister failed for 0.0 GiB`. -- With an equivalent runtime patch (clamp + graceful handling), registration proceeds normally until the WDDM pool ceiling (~13.8 GB measured) — see #55 for the instrumented numbers. -- The clamp itself is exercised by the same path (`alloc_pinned_tensor` has used the identical guard since it shipped). - -### Notes - -These unblock diagnosing the bigger residency gap tracked in #55 (LOCKED/PAGEABLE movement paths are scaffolded but not implemented). Not included here to keep the diff minimal. From 815c673447c306fff3b28c80982c40c8eed4fbe3 Mon Sep 17 00:00:00 2001 From: Glucksberg Date: Sun, 23 Aug 2026 15:11:53 -0400 Subject: [PATCH 4/5] host_banks: platform-aware pin-failure hint (Linux RLIMIT_MEMLOCK branch) Per review on #56: on Linux the equivalent ceiling is RLIMIT_MEMLOCK (default RAM/8), and the failure surfaces minutes into a load with no hint about memlock. The re-raised error now reports the current soft limit and points at ulimit -l / /etc/security/limits.d/*.conf, plus the ControlMaster gotcha (limits apply at session start; ssh -O exit first). Windows branch unchanged. --- python/freetoken/moe/host_banks.py | 33 +++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py index 47851238..9d7a9617 100644 --- a/python/freetoken/moe/host_banks.py +++ b/python/freetoken/moe/host_banks.py @@ -22,6 +22,7 @@ import mmap import os import queue +import sys import threading from concurrent.futures import ThreadPoolExecutor from enum import Enum @@ -89,13 +90,31 @@ def pin(self) -> None: host_register(self.addr, len(self._buf)) except RuntimeError as exc: msg = str(exc) - if "out of memory" in msg: - hint = ( - "the driver refused to lock more host memory (on Windows/WDDM " - "the pageable-locking pool is roughly half of system RAM); " - "close other processes, add RAM, or serve some layers with " - "--moe-backend cpu" - ) + if "out of memory" in msg or "lock" in msg.lower(): + if sys.platform == "win32": + hint = ( + "the driver refused to lock more host memory (on Windows/WDDM " + "the pageable-locking pool is roughly half of system RAM); " + "close other processes, add RAM, or serve some layers with " + "--moe-backend cpu" + ) + else: + try: + import resource + + soft = resource.getrlimit(resource.RLIMIT_MEMLOCK)[0] + rlimit = ( + f"{soft // 1048576} MB" if soft != resource.RLIM_INFINITY else "unlimited" + ) + except Exception: + rlimit = "? MB" + hint = ( + "the kernel refused to lock more host memory: RLIMIT_MEMLOCK " + f"is {rlimit} (often RAM/8 by default). Check `ulimit -l`, " + "raise it via /etc/security/limits.d/*.conf and re-login -- " + "an existing SSH ControlMaster session keeps serving the old " + "limit until `ssh -O exit `" + ) else: hint = "see the driver error above" raise RuntimeError( From 6f0f6aa8f492da8214a9b93739cf95f59d59c31f Mon Sep 17 00:00:00 2001 From: Glucksberg Date: Sun, 23 Aug 2026 15:16:56 -0400 Subject: [PATCH 5/5] host_banks: tighten lock-word matching (word-boundary, avoid 'block' false positive); human-readable RLIMIT formatting --- python/freetoken/moe/host_banks.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py index 9d7a9617..6c00718f 100644 --- a/python/freetoken/moe/host_banks.py +++ b/python/freetoken/moe/host_banks.py @@ -89,8 +89,9 @@ def pin(self) -> None: try: host_register(self.addr, len(self._buf)) except RuntimeError as exc: - msg = str(exc) - if "out of memory" in msg or "lock" in msg.lower(): + msg = str(exc).lower() + lock_words = {"lock", "locked", "locking", "memlock"} + if "out of memory" in msg or bool(lock_words & set(msg.split())) or "memlock" in msg: if sys.platform == "win32": hint = ( "the driver refused to lock more host memory (on Windows/WDDM " @@ -103,11 +104,14 @@ def pin(self) -> None: import resource soft = resource.getrlimit(resource.RLIMIT_MEMLOCK)[0] - rlimit = ( - f"{soft // 1048576} MB" if soft != resource.RLIM_INFINITY else "unlimited" - ) + if soft == resource.RLIM_INFINITY: + rlimit = "unlimited" + elif soft >= 1073741824: + rlimit = f"{soft / 1073741824:.0f} GB" + else: + rlimit = f"{soft // 1048576} MB" except Exception: - rlimit = "? MB" + rlimit = "?" hint = ( "the kernel refused to lock more host memory: RLIMIT_MEMLOCK " f"is {rlimit} (often RAM/8 by default). Check `ulimit -l`, "