Apple Silicon Metal backend (ft serve on macOS arm64)

.gitignore

@@ -227,3 +227,4 @@ benchmarks/cross_framework
 
 # local e2e/bench artifacts (harnesses may run with repo cwd)
 /results/
+/.build

README.md

@@ -49,6 +49,22 @@ uv venv && source .venv/bin/activate
 uv pip install -e ".[accel]"
 ```
 
+### macOS (Apple Silicon, Metal)
+
+The native engine is CUDA-only; on a Mac `ft serve` (same as `ft serve-metal`)
+runs the OpenAI/Anthropic/Responses API backed by Apple's Metal runtimes
+(MLX or llama.cpp) instead of porting the CUDA scheduler:
+
+```bash
+uv venv && source .venv/bin/activate
+uv pip install -e .                      # core only; skips CUDA-only deps
+uv pip install mlx-lm                     # Metal engine (or: brew install llama.cpp)
+ft serve --model mlx-community/Qwen3-0.6B-4bit
+```
+
+See [Install on macOS](https://github.com/FlashML-org/FreeToken/blob/main/docs/install.md) and
+[serve-metal in the CLI reference](https://github.com/FlashML-org/FreeToken/blob/main/docs/cli.md).
+
 For More details:
 
 - [Install FreeToken](https://github.com/FlashML-org/FreeToken/blob/main/docs/install.md)

benchmarks/bench_decode_moe.py

@@ -77,7 +77,13 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     p.add_argument(
         "--backend",
         default="offload",
-        help="comma list of offload|cpu|hybrid; one server per backend",
+        help="comma list of offload|cpu|hybrid|metal; one server per backend (metal = Apple Silicon)",
+    )
+    p.add_argument(
+        "--metal-engine",
+        default="mlx",
+        choices=["mlx", "llama"],
+        help="Metal engine for --backend metal (mlx or llama.cpp; default mlx)",
     )
     p.add_argument(
         "--aime",
@@ -172,6 +178,16 @@ def free_port() -> int:
 
 
 def serve_cmd(args: argparse.Namespace, backend: str, port: int) -> list[str]:
+    if backend == "metal":
+        # Apple Silicon: the same serving path through the Metal backend (mlx /
+        # llama.cpp upstream). CUDA engine flags do not apply, and the Metal
+        # server reports readiness through /health like the CUDA one.
+        return [
+            sys.executable, "-m", "freetoken.cli", "serve-metal",
+            "--model", args.model,
+            "--backend", args.metal_engine,
+            "--host", "127.0.0.1", "--port", str(port),
+        ]
     cmd = [
         sys.executable, "-m", "freetoken.cli", "serve",
         "--model", args.model,
@@ -209,7 +225,10 @@ def wait_ready(origin: str, proc: subprocess.Popen, log_path: str, timeout: floa
             continue
         if health.get("status") == "error":
             die_with_log(f"server reported startup error: {health}", log_path)
-        if health.get("maintenance") == "serving":
+        # The CUDA server reports its lifecycle via `maintenance`; the Metal
+        # backend is "serving" as soon as /health is ok (its upstream engine
+        # finished loading before the API ever bound).
+        if health.get("maintenance") in ("serving", None) and health.get("status") == "ok":
             return
         time.sleep(1.0)
     die_with_log(f"server not ready after {timeout:.0f}s", log_path)
@@ -339,7 +358,15 @@ def run_one(args: argparse.Namespace, backend: str) -> dict:
         sys.exit(f"[bench] need >=2 token events to measure decode, got {len(stamps)}")
     completion = usage["completion_tokens"]
     if completion != args.decode:
-        print(f"[bench] WARNING: completion_tokens={completion} != --decode {args.decode}", flush=True)
+        why = (
+            " -- Metal upstream stops at EOS (ignore_eos is not honored)"
+            if backend == "metal"
+            else ""
+        )
+        print(
+            f"[bench] WARNING: completion_tokens={completion} != --decode {args.decode}{why}",
+            flush=True,
+        )
     steps = completion - 1
     decode_time = stamps[-1] - stamps[0]
     gaps = sorted((b - a) * 1e3 for a, b in zip(stamps, stamps[1:]))
@@ -369,6 +396,12 @@ def run_one(args: argparse.Namespace, backend: str) -> dict:
           f"(event p50 {row['event_ms_p50']:.3f} / p99 {row['event_ms_p99']:.3f} ms, "
           f"{len(stamps)} events)")
     print(f"  vram (server)     : {row['vram_gib']:8.2f} GiB")
+    if backend == "metal":
+        print(
+            "  note              : Metal/mlx coalesces tokens into fewer SSE events and\n"
+            "                      does not honor ignore_eos, so tok/s is an upper bound\n"
+            "                      over coalesced bursts, and steps may be < --decode."
+        )
     sha_note = "greedy" if args.greedy else "sampled, per-server deterministic"
     print(f"  output sha1       : {row['output_sha1']}  ({sha_note}; compare across backends)")
     print(f"  output sample     : {r['text'][:240]!r}")
@@ -378,7 +411,7 @@ def run_one(args: argparse.Namespace, backend: str) -> dict:
 def main(argv: list[str] | None = None) -> int:
     args = parse_args(argv)
     backends = [b.strip() for b in args.backend.split(",") if b.strip()]
-    unknown = [b for b in backends if b not in ("offload", "cpu", "hybrid")]
+    unknown = [b for b in backends if b not in ("offload", "cpu", "hybrid", "metal")]
     if unknown:
         sys.exit(f"unknown backend(s): {unknown}")
 

docs/cli.md

@@ -7,6 +7,7 @@ ft <command> [args]
 | Command | Purpose |
 |---|---|
 | `ft serve` | Start the API server (OpenAI `/v1/*`, Anthropic `/v1/messages`, Responses) |
+| `ft serve-metal` | Start the same API surface over an Apple Silicon Metal backend (mlx or llama.cpp) |
 | `ft shell` | Chat with a server in the terminal |
 | `ft ctl` | Query and manage a running server over HTTP |
 | `ft launch` | Configure and launch a coding agent against a server |
@@ -81,6 +82,42 @@ See [models.md](models.md#moe-backends) for what each backend does.
 | `--reasoning-parser` | auto | Splits chain-of-thought into `reasoning_content`; auto-inferred; `off` disables |
 | `--enable-cache-report` | off | Report prefix-cache hits in each response's usage block |
 
+## ft serve-metal
+
+`ft serve-metal` runs the same OpenAI `/v1/*`, Anthropic `/v1/messages`, and
+Responses API surface, but backs it with an **Apple Silicon Metal runtime**
+instead of FreeToken's CUDA scheduler. It does not import FreeToken's CUDA
+stack, so it works standalone on a Mac (no triton/flashinfer/sglang-kernel, no
+`flashlib`) by proxying to a Metal engine that already implements the protocol.
+
+```bash
+ft serve-metal --model mlx-community/Qwen3-0.6B-4bit --backend mlx --port 1919
+ft serve-metal --model ~/models/MyModel.Q4_K_M.gguf --backend llama --port 1919
+```
+
+The API surface (chat/completions/models/health, streaming included) is
+bit-for-bit the same FreeToken wire, so the desktop app, `ft shell`, `ft launch`
+agents, and OpenAI/Anthropic clients all work unchanged.
+
+| Flag | Default | Meaning |
+|---|---|---|
+| `--model` | required | Model path or HF id for the Metal engine |
+| `--backend` | `auto` | `mlx` (Apple's `mlx_lm.server`) or `llama` (llama.cpp `llama-server`); `auto` prefers mlx |
+| `--host` | 127.0.0.1 | FreeToken API bind address |
+| `--port` | 1919 | FreeToken API port (user-facing) |
+| `--metal-port` | 0 | Upstream Metal engine port; a free loopback port is chosen when `0` |
+
+Notes:
+
+- **mlx** requires the model loadable by `mlx-lm` (HF-converted or MLX weights,
+  e.g. `mlx-community/*`). `mlx_lm` uses the Apple GPU (Metal) directly.
+- **llama** requires the `llama-server` binary on PATH and a GGUF model (the
+  same format FreeToken already parses). It runs with the Metal backend
+  (`-ngl 999`).
+- `ft serve --backend mlx|llama` also switches between these backends on a CUDA
+  box, but still imports the CUDA config stack; `ft serve-metal` is the
+  standalone, CUDA-free form for Apple Silicon.
+
 ## ft shell
 
 ```bash

docs/install.md

@@ -2,11 +2,22 @@
 
 ## Requirements
 
+**Linux (CUDA — full engine)**
+
 - Linux x86_64, NVIDIA GPU, driver r580+ (CUDA 13)
 - Python >= 3.10, with [uv](https://docs.astral.sh/uv/) recommended (plain
   `pip` + `venv` works too)
 
-## Method 1: Install from PyPI
+**macOS (Apple Silicon — Metal backend)**
+
+- macOS on Apple Silicon (M1/M2/M3/M4)
+- Python >= 3.10
+- One of the Metal engines: `mlx-lm` (recommended) or `llama.cpp`'s
+  `llama-server` on PATH
+- No CUDA, triton, flashinfer, sglang-kernel, or `flashlib` needed — the
+  Metal path does not import FreeToken's CUDA stack at all
+
+## Method 1: Install from PyPI (Linux/CUDA)
 
 ```bash
 uv venv && source .venv/bin/activate
@@ -15,15 +26,75 @@ uv pip install "freetoken[accel]"
 
 CUDA kernels are JIT-compiled on first use, need a CUDA 13 toolkit with `nvcc` on PATH.
 
-## Method 2: Install from source
+## Method 2: Install from source (Linux/CUDA)
 
 ```bash
 git clone https://github.com/FlashML-org/FreeToken.git && cd FreeToken
 uv venv && source .venv/bin/activate
 uv pip install -e ".[accel]"
 ```
 
-## Verify
+## Method 3: macOS / Apple Silicon (Metal backend)
+
+FreeToken's native engine is CUDA-only. On a Mac, `ft serve-metal` runs the
+same API surface backed by Apple's own Metal runtimes instead, so nothing
+needs porting: it launches Apple's `mlx_lm.server` or llama.cpp's
+`llama-server` as the upstream engine and proxies the FreeToken wire surface.
+
+```bash
+git clone https://github.com/FlashML-org/FreeToken.git && cd FreeToken
+uv venv && source .venv/bin/activate
+
+# Core package only — skips flashlib, triton, sglang-kernel and the CUDA-linked
+# torch index pins, none of which have macOS wheels.
+uv pip install -e .
+
+# Then add one Metal engine:
+uv pip install mlx-lm            # option A: MLX (Apple's framework)
+brew install llama.cpp           # option B: llama.cpp's llama-server (Metal)
+```
+
+> **Why the CUDA deps are skipped:** `flashlib`, `triton==3.6.0`, and the cu130
+> torch pins are marked `platform_system == 'Linux'` in pyproject.toml — they
+> only serve the native scheduler, and the Metal path never imports them. A
+> plain `uv pip install -e .` resolves everything else from PyPI.
+
+```bash
+# Verify (no model needed):
+ft --version
+ft serve --help          # on Apple Silicon this is the Metal backend
+# or: ft serve-metal --help
+```
+
+Or one shot from this repo:
+
+```bash
+./install.sh             # Metal path on Darwin arm64; CUDA wheels on Linux
+ft serve --model mlx-community/Qwen3-0.6B-4bit
+```
+
+### Troubleshooting: "no wheels with a matching Python implementation tag"
+
+Your venv is an **x86_64 build running under Rosetta**, not native arm64 — no
+arm64-only wheel (mlx, macOS torch) can ever install into it. This happens when
+`uv venv` / `python3 -m venv` is run inside an x86_64 (Intel) terminal session.
+
+Fix — recreate the venv natively:
+
+```bash
+deactivate 2>/dev/null; rm -rf .venv
+# make sure the shell itself is native: arch -arm64 zsh   (or use an arm64 Terminal)
+uv venv && source .venv/bin/activate
+file .venv/bin/python   # must say: arm64   (not x86_64)
+uv pip install -e . && uv pip install mlx-lm
+```
+
+Also run installs from the repo root — `uv pip install -e .` needs the
+`pyproject.toml` in the current directory, which is why a stray
+`error: Requesting extras requires a pyproject.toml ...` means you are in the
+wrong directory (or asked for a nonexistent extra; the only extra is `dev`).
+
+## Verify (Linux/CUDA)
 
 ```bash
 source .venv/bin/activate

docs/quickstart.md

@@ -13,6 +13,31 @@ and MoE backends, cache sizes, tool-call and reasoning parsers — resolves from
 the checkpoint and the GPU; see [cli.md](cli.md) for the flags. The server is
 ready when the log reaches `API server is ready to serve on 127.0.0.1:1919`.
 
+### macOS (Apple Silicon)
+
+```bash
+scripts/start-metal.sh                             # serve + chat in one command
+scripts/start-metal.sh mlx-community/Llama-3.2-1B-Instruct-4bit
+scripts/start-metal.sh ~/models/MyModel.Q4_K_M.gguf --backend llama
+```
+
+`scripts/start-metal.sh` checks the environment, starts the Metal server,
+waits for readiness, and opens `ft shell` for interactive testing (the server
+stops when the chat exits; `NO_CHAT=1` runs the API alone). Or drive the
+pieces directly:
+
+```bash
+ft serve-metal --model mlx-community/Qwen3-0.6B-4bit --backend mlx
+```
+
+`ft serve-metal` serves the same API surface as `ft serve`, backed by MLX or
+llama.cpp on Metal (see [install.md](install.md#method-3-macos--apple-silicon-metal-backend)).
+`--model` takes an MLX/HF repo id (`mlx-community/*`) or, with `--backend llama`,
+a local GGUF file. The rest of this page (`/v1/models`, chat completions, `ft shell`,
+`ft launch`) works unchanged against it — including `ft shell --model <id>`, which
+starts the Metal engine and chats in one process on a Mac, and the `/model <id>`
+shell command to switch models without restarting.
+
 ## Send a request
 
 Check what is being served:
@@ -49,7 +74,9 @@ ft shell --model ~/models/Qwen3.6-35B-A3B   # start an engine and chat, one proc
 ```
 
 `/help` lists the in-shell commands. Attach mode needs no GPU, so it also drives
-a server on another machine (`--server URL`).
+a server on another machine (`--server URL`). On a Mac, `ft shell --model <mlx/hf id>`
+starts the Metal backend instead (see [quickstart.md](quickstart.md)); `/model <id>`
+switches the served model on servers that support it (Metal).
 
 ## Use a coding agent
 

install.sh

@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
 #
-# FreeToken engine installer (Linux, NVIDIA CUDA) — user-facing, wheel-based.
+# FreeToken engine installer — user-facing.
 #
-# Installs the `freetoken` runtime (the `ft` CLI) and its prebuilt kernel-cache
-# wheel into a managed venv, then wires it up so FreeToken Desktop can find it.
-# Dependencies come from PyPI via uv, except torch and sglang-kernel whose cu130
-# wheels live on dedicated indexes (see CU_INDEX_ARGS below).
+# Linux (NVIDIA CUDA): installs the `freetoken` runtime (the `ft` CLI) and its
+# prebuilt kernel-cache wheel into a managed venv, then wires it up so FreeToken
+# Desktop can find it. Dependencies come from PyPI via uv, except torch and
+# sglang-kernel whose cu130 wheels live on dedicated indexes (see CU_INDEX_ARGS).
+#
+# macOS (Apple Silicon): no CUDA. Installs the core package + mlx-lm and runs
+# the Metal backend (`ft serve` routes to serve-metal). No kernel-cache wheel.
 #
 # Typical use (once a release exists):
 #   curl -fsSL https://<host>/install.sh | bash
@@ -153,6 +156,70 @@ else
 fi
 say "uv $("$UV" --version | awk '{print $2}')"
 
+install_metal_macos() {
+  [ "$(uname -s)" = Darwin ] || return 1
+  [ "$(uname -m)" = arm64 ] || die "FreeToken on macOS requires Apple Silicon (arm64). This machine is $(uname -m)."
+
+  local script_dir src
+  script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd -P)"
+
+  say "macOS Apple Silicon — Metal backend (no CUDA, no kernel-cache wheel)"
+  mkdir -p "$FT_HOME"
+  "$UV" venv "$VENV" --python "$PY_VERSION" --clear
+
+  if [ -f "$script_dir/pyproject.toml" ]; then
+    src="$script_dir"
+    say "installing from source checkout: $src"
+    "$UV" pip install --python "$VENV" -e "$src"
+  else
+    # Pin the Metal branch until it lands on default. A piped `curl | bash` of
+    # this script has no adjacent checkout, so "git+...FreeToken.git" would
+    # otherwise install CUDA-only main and fail on macOS. Prefer FlashML-org;
+    # fall back to the working fork while the PR is open.
+    src="git+https://github.com/FlashML-org/FreeToken.git@feat/apple-metal-backend"
+    src_fork="git+https://github.com/jasonkneen/FreeToken.git@feat/apple-metal-backend"
+    say "installing from $src"
+    if ! "$UV" pip install --python "$VENV" "$src"; then
+      say "FlashML-org does not have feat/apple-metal-backend yet; using $src_fork"
+      "$UV" pip install --python "$VENV" "$src_fork"
+    fi
+  fi
+  say "installing mlx-lm (Apple Metal engine)"
+  "$UV" pip install --python "$VENV" mlx-lm
+
+  local ft_bin="$VENV/bin/ft"
+  [ -x "$ft_bin" ] || die "install finished but $ft_bin is missing."
+  mkdir -p "$BIN_DIR"
+  ln -sf "$ft_bin" "$BIN_DIR/ft"
+  say "symlinked $BIN_DIR/ft -> $ft_bin"
+  mkdir -p "$ENV_DIR"
+  printf 'FREETOKEN_FT_BIN=%s\n' "$ft_bin" > "$ENV_DIR/50-freetoken.conf"
+  say "wrote $ENV_DIR/50-freetoken.conf (FREETOKEN_FT_BIN)"
+
+  if "$ft_bin" --help >/dev/null 2>&1; then
+    say "self-check: \`ft --help\` OK"
+  else
+    warn "self-check: \`ft --help\` returned non-zero — inspect with: $ft_bin --help"
+  fi
+
+  cat <<EOF
+
+${C_GREEN}FreeToken (Metal) installed.${C_RESET}
+
+  ft binary        $ft_bin
+  on PATH as       $BIN_DIR/ft   (ensure $BIN_DIR is on PATH)
+
+Run:
+  ft serve --model mlx-community/Qwen3-0.6B-4bit --port 1919
+
+EOF
+}
+
+if [ "$(uname -s)" = Darwin ]; then
+  install_metal_macos
+  exit 0
+fi
+
 # Resolve the runtime wheel: explicit env → ./dist bundle → build from a source checkout.
 find_bundled_wheel
 build_from_repo_if_needed