sync: engine 0.2.0 (open-core source)

base-convert/Cargo.toml

@@ -12,7 +12,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.1.7"
+version = "0.2.0"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/basecompute/baseRT"

base-convert/crates/base-arch/src/lib.rs

@@ -81,15 +81,20 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe
         // (canonical_arch="llama" → llama model class). Validated end-to-end on
         // Mistral-7B-Instruct-v0.3 and Ministral-8B-Instruct-2410.
         //
-        // Phi-3 is NOT enabled. The converter side is ready (SplittingProvider
-        // splits its fused qkv_proj/gate_up_proj; weights convert; HD=96
-        // attention is fine — regression-tested) and the generation_config eos
-        // merge below makes it stop cleanly. BUT Phi-3 chat output degenerates
-        // (raw completions are coherent; chat floods/repeats and is incoherent at
-        // both Q4 and Q8, temp 0 and 0.7) — a chat-path issue (same class as
-        // SmolLM2) that's not yet root-caused. Re-enable "phi3" once that's
-        // fixed. (Phi-3.5 additionally needs LongRoPE — engine is linear-only.)
-        "llama" | "mistral" => Some(&llama::LlamaHfMapper),
+        // Phi-3-mini (4k, standard RoPE) is Llama-shaped and reuses the Llama
+        // mapper: the SplittingProvider (base-convert) slices its fused
+        // self_attn.qkv_proj / mlp.gate_up_proj into the canonical split names
+        // the mapper consumes, HD=96 is fine (regression-tested), and the
+        // generation_config eos merge makes it stop cleanly on <|end|> (32007).
+        // The chat-path flood that used to gate it (same class as SmolLM2) is
+        // resolved by two landed tokenizer fixes — the added-token lstrip/rstrip
+        // handling in split_special_tokens (Phi-3's `<|end|>` rstrip absorbs the
+        // trailing newline; tokenizer.cpp) and the GPT2* add_bos default
+        // (tokenizer_defaults.h). REVALIDATED on microsoft/Phi-3-mini-4k-instruct
+        // (converted Q8): chat output is coherent across probes, no flood.
+        // Phi-3.5 stays OUT — it needs LongRoPE, which the engine (linear scaling
+        // only) does not implement.
+        "llama" | "mistral" | "phi3" => Some(&llama::LlamaHfMapper),
         "qwen2" | "qwen3" => Some(&qwen::QwenHfMapper),
         "qwen2_moe" | "qwen3_moe" => Some(&qwen::QwenMoeHfMapper),
         // Qwen3.5 / 3.6: hybrid Gated-DeltaNet + full-attention decoder
@@ -107,7 +112,13 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe
         // actually google/gemma-3n-E2B-it. The canonical Gemma 4 lives
         // under google/gemma-4-{E2B,E4B}-it and uses model_type=gemma4
         // with text_config.model_type=gemma4_text.
-        "gemma4" | "gemma4_text" => Some(&gemma::Gemma4HfMapper),
+        // gemma4_unified (gemma-4-12B-it): the encoder-free multimodal
+        // variant — a standard gemma4 text stack under
+        // `model.language_model.*` plus ~10 small modality-projection
+        // tensors (embed_vision/embed_audio/vision_embedder) that the
+        // text conversion skips like any other non-text tower. Config is
+        // gemma4-shaped (uniform head_dim, rope_parameters, layer_types).
+        "gemma4" | "gemma4_text" | "gemma4_unified" => Some(&gemma::Gemma4HfMapper),
         _ => None,
     }
 }
@@ -118,6 +129,7 @@ pub fn hf_mapper_for_model_type(model_type: &str) -> Option<&'static dyn HfMappe
 pub const SUPPORTED_HF_MODEL_TYPES: &[&str] = &[
     "llama",
     "mistral",
+    "phi3",
     "qwen2",
     "qwen3",
     "qwen2_moe",
@@ -135,6 +147,7 @@ pub const SUPPORTED_HF_MODEL_TYPES: &[&str] = &[
     "gemma3_text",
     "gemma4",
     "gemma4_text",
+    "gemma4_unified",
 ];
 
 pub trait GgufMapper: Sync {

base-convert/crates/base-convert/src/hub.rs

@@ -175,12 +175,83 @@ fn installed_single_path(reg: &MergedRegistry, id: &str) -> Result<Option<PathBu
                 .with_context(|| format!("installed model `{id}` has no artifact path"))?,
         )),
         many => {
+            // Multiple variants installed — e.g. a universal `default-q4` cached
+            // before the backend-qualified catalog entries existed, plus a native
+            // `cuda-q4mix` pulled after. Rather than erroring, apply the same
+            // backend preference the catalog resolver uses (backend-native >
+            // universal) so bare `serve <id>` keeps Just Working. The installed
+            // variant dir name carries the backend slot (`cuda-*`/`rocm-*`/`cpu-*`
+            // vs the universal `default-*`); on a backend that can't load a
+            // foreign-native bundle it simply won't be present.
+            let backend = base_hub::registry::CatalogRegistry::client_backend();
+            let native_prefix = format!("{backend}-");
+            let native: Vec<&ModelEntry> =
+                many.iter().copied().filter(|r| r.variant.starts_with(&native_prefix)).collect();
+            let pick = if native.len() == 1 {
+                Some(native[0])
+            } else if native.is_empty() {
+                let uni: Vec<&ModelEntry> =
+                    many.iter().copied().filter(|r| r.variant.starts_with("default-")).collect();
+                (uni.len() == 1).then(|| uni[0])
+            } else {
+                None
+            };
+            if let Some(one) = pick {
+                return Ok(Some(
+                    one.path
+                        .clone()
+                        .with_context(|| format!("installed model `{id}` has no artifact path"))?,
+                ));
+            }
             let variants = many.iter().map(|r| r.variant.as_str()).collect::<Vec<_>>().join(", ");
             bail!("model `{id}` has multiple installed variants ({variants}) — specify one as `{id}:<variant>`")
         }
     }
 }
 
+/// Installed artifact for `id` matching the requested variant `want`. Tries an
+/// EXACT variant-dir match first (honors `:default-q4` / `:cuda-q4mix`); failing
+/// that, matches by BIT WIDTH so the documented short `:q4` finds an installed
+/// `cuda-q4mix` or `default-q4` (the exact dir is named for the quant slot, not
+/// the bare bits), preferring the backend-native slot over universal — the same
+/// rule the catalog resolver's quant_matches + backend preference apply. Returns
+/// `None` when nothing installed matches (caller then auto-pulls).
+fn installed_best_variant(reg: &MergedRegistry, id: &str, want: &str) -> Option<PathBuf> {
+    if let Some(p) = reg.local.installed_path(id, want) {
+        return Some(p); // exact variant name present
+    }
+    let want_bits = base_hub::registry::quant_bits(want).unwrap_or(want);
+    let backend = base_hub::registry::CatalogRegistry::client_backend();
+    let native_prefix = format!("{backend}-");
+    // A foreign-native slot (e.g. `cuda-q4mix` on macOS) must be EXCLUDED, not
+    // just deprioritized: on a copied/shared cache it can't be loaded, and a bit-
+    // width tie would otherwise pick the lexicographically-earlier CUDA artifact
+    // over the runnable universal one. Runnable = universal (`default-*`), this
+    // client's native slot (`<backend>-*`), or a bare bit-width dir (no slot).
+    const KNOWN_BACKENDS: [&str; 4] = ["cuda", "rocm", "cpu", "metal"];
+    let runnable = |v: &str| -> bool {
+        match v.split_once('-') {
+            Some((slot, _)) if KNOWN_BACKENDS.contains(&slot) => slot == backend, // native only
+            _ => true,                                                            // default-*, bare bits, etc.
+        }
+    };
+    let installed = reg.local.list().ok()?;
+    let mut matches: Vec<&ModelEntry> = installed
+        .iter()
+        .filter(|r| {
+            r.id == id
+                && runnable(&r.variant)
+                && base_hub::registry::quant_bits(&r.variant).unwrap_or(r.variant.as_str()) == want_bits
+        })
+        .collect();
+    if matches.is_empty() {
+        return None;
+    }
+    // Backend-native slot first (stable so a deterministic pick when both exist).
+    matches.sort_by_key(|r| u8::from(!r.variant.starts_with(&native_prefix)));
+    matches[0].path.clone()
+}
+
 /// Fetch a not-yet-installed model on demand, then return its artifact path.
 /// Prefers the pre-converted basecompute mirror; otherwise converts the source
 /// repo on pull. Progress (download + quantization) is shown by `cmd_pull`.
@@ -245,7 +316,7 @@ fn resolve_hub_model(token: &str, default_variant: Option<&str>) -> Result<PathB
     let reg = MergedRegistry::load()?;
 
     if let Some(v) = variant {
-        if let Some(p) = reg.local.installed_path(id, v) {
+        if let Some(p) = installed_best_variant(&reg, id, v) {
             return Ok(p);
         }
         return auto_pull_and_resolve(&reg, id, Some(v));

base-convert/crates/base-convert/src/main.rs

@@ -2015,6 +2015,10 @@ fn to_canonical_name(name: &str, arch: &str) -> Option<Canonical> {
         || mm_name.starts_with("visual.")
         || mm_name.starts_with("embed_audio")
         || mm_name.starts_with("embed_vision")
+        // gemma4_unified (gemma-4-12B-it) names its encoder-free patch
+        // embedder `vision_embedder.*` (pos_embedding, patch_dense,
+        // patch_ln1/2, pos_norm) rather than `vision_tower.*`.
+        || mm_name.starts_with("vision_embedder")
         || mm_name.starts_with("multi_modal_projector")
     {
         let canonical = base_arch::gemma::map_gemma4_mmproj_name(mm_name).unwrap_or_else(|| mm_name.to_string());

base-convert/crates/base-format/src/header.rs

@@ -183,6 +183,11 @@ pub enum TargetBackend {
     /// layouts.
     CudaSm89,
     CudaSm90,
+    /// NVIDIA GB10 (DGX Spark, consumer Blackwell). Serializes to
+    /// "cuda_sm121" — one of the two tags the C++ reader's CUDA build
+    /// accepts and Metal builds reject; used for bundles that carry
+    /// CUDA-only kernel requirements (base_q6 MoE experts today).
+    CudaSm121,
     /// AMD CDNA3 (MI300). MFMA tile layout.
     RocmCdna3,
     /// CPU AVX2 / NEON fallback paths. Row-major contiguous packing.

base-convert/crates/base-format/src/writer.rs

@@ -1,4 +1,4 @@
-use crate::header::{Header, MmprojBundle, TensorEntry};
+use crate::header::{Header, MmprojBundle, TensorDtype, TensorEntry};
 use crate::slots::{write_slots, Slot};
 use crate::{Error, Result, BLOB_ALIGNMENT, FORMAT_VERSION, MAGIC, PREFIX_LEN};
 use std::fs::File;
@@ -127,6 +127,23 @@ impl<W: Write + Seek> BaseWriter<W> {
         }
         self.header.tensors = entries;
 
+        // Stamp `target_backend` from CONTENT instead of trusting the
+        // caller's default (every construction site used to hardcode
+        // Metal, so CUDA-only bundles carried a `metal` tag and failed a
+        // kernel lookup only after a full download + load). The one
+        // CUDA-only content class today: base_q6 MoE expert slabs
+        // (`*_exps.*` tensors) — Metal ships no q6 MoE kernels. bf16-scale
+        // q8/q4 bundles stay `metal` (universal): both backends carry the
+        // `_sbf16` kernel families.
+        if self
+            .header
+            .tensors
+            .iter()
+            .any(|t| t.name.contains("_exps.") && t.dtype == TensorDtype::BaseQ6)
+        {
+            self.header.target_backend = crate::header::TargetBackend::CudaSm121;
+        }
+
         // Multimodal sub-bundle entries land in the same weights blob,
         // continuing past the LM tensors. Their entries go into
         // `header.mmproj.tensors` so the runtime can decide whether to

base-convert/crates/base-format/tests/roundtrip.rs

@@ -456,3 +456,36 @@ fn rejects_unknown_version() {
     let err = expect_err(BaseReader::open(tmp.path()));
     assert!(matches!(err, base_format::Error::UnsupportedVersion(999, _)));
 }
+
+#[test]
+fn target_backend_stamped_from_content() {
+    // q6 MoE expert slabs are CUDA-only (no Metal q6 MoE kernels): the
+    // writer must override the caller's Metal default so Metal builds
+    // reject the bundle at open, not at kernel lookup.
+    let tmp = tempfile::NamedTempFile::new().unwrap();
+    let header = make_header();
+    let mut writer = BaseWriter::create(tmp.path(), header).unwrap();
+    let mut e = entry("layers.0.ffn_gate_exps.weight", vec![4, 8]);
+    e.dtype = TensorDtype::BaseQ6;
+    writer.add_tensor(TensorPayload {
+        entry: e,
+        data: vec![0u8; 24],
+    });
+    writer.finish().unwrap();
+    let reader = BaseReader::open(tmp.path()).unwrap();
+    assert_eq!(reader.header().target_backend, TargetBackend::CudaSm121);
+
+    // ...but a DENSE q6 tensor (or q6 outside the expert slabs) stays
+    // universal — Metal has dense q6 kernels.
+    let tmp2 = tempfile::NamedTempFile::new().unwrap();
+    let mut writer = BaseWriter::create(tmp2.path(), make_header()).unwrap();
+    let mut e = entry("layers.0.self_attn.q_proj.weight", vec![4, 8]);
+    e.dtype = TensorDtype::BaseQ6;
+    writer.add_tensor(TensorPayload {
+        entry: e,
+        data: vec![0u8; 24],
+    });
+    writer.finish().unwrap();
+    let reader = BaseReader::open(tmp2.path()).unwrap();
+    assert_eq!(reader.header().target_backend, TargetBackend::Metal);
+}

base-convert/crates/base-hub/src/catalog.rs

@@ -68,6 +68,14 @@ pub struct CatalogEntry {
     /// Optional integrity check for the downloaded `.base`.
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub sha256: Option<String>,
+    /// Backend requirement. `None` = universal (runs on every backend —
+    /// f16/bf16 weights and bf16-scale q4/q8 bundles). `Some("cuda")` /
+    /// `Some("metal")` restricts resolution to clients of that backend
+    /// (e.g. base_q6-expert MoE bundles are CUDA-only). Checked at
+    /// resolve time so an incompatible pull refuses BEFORE the download,
+    /// not at load after 30 GB.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub backend: Option<String>,
 }
 
 fn default_file() -> String {
@@ -179,6 +187,15 @@ impl Catalog {
             .or_else(|| self.models.iter().find(|e| e.id.eq_ignore_ascii_case(id)))
     }
 
+    /// True if ANY row (any quant/backend) carries this id. Lets the resolver
+    /// tell "this is a catalog model we can't run on this backend" (refuse
+    /// pre-download) apart from "unknown id" (fall through to a raw HF repo).
+    pub fn has_id(&self, id: &str) -> bool {
+        self.models
+            .iter()
+            .any(|e| e.id == id || e.id.eq_ignore_ascii_case(id))
+    }
+
     /// Check the catalog is internally consistent: every entry well-formed, and
     /// no duplicate `(id, quant)` pair (which would shadow in `find`/listing).
     /// Run by tests/CI to catch a malformed catalog edit before it ships and
@@ -198,8 +215,17 @@ impl Catalog {
             if e.quant.is_empty() {
                 anyhow::bail!("{}: empty quant", e.id);
             }
-            if !seen.insert((e.id.as_str(), e.quant.as_str())) {
-                anyhow::bail!("duplicate catalog entry: {} [{}]", e.id, e.quant);
+            // A model may carry a universal row AND a per-backend variant of the
+            // same quant (the resolver prefers the backend-native one); those are
+            // distinguished by `backend`, so the uniqueness key includes it. Two
+            // rows identical in (id, quant, backend) are still a real duplicate.
+            if !seen.insert((e.id.as_str(), e.quant.as_str(), e.backend.as_deref())) {
+                anyhow::bail!(
+                    "duplicate catalog entry: {} [{}] backend={:?}",
+                    e.id,
+                    e.quant,
+                    e.backend
+                );
             }
         }
         Ok(())