Merge branch 'ServeurpersoCom:master' into master

Author: lmangani

src/backend.h

@@ -22,10 +22,22 @@ struct BackendPair {
     int gpu_cc; // CUDA compute capability (e.g. 720 for sm_72), 0 if not CUDA
 };
 
+// Cached backend state (shared across all modules in the same binary)
+static BackendPair g_backend_cache = {};
+static int         g_backend_refs  = 0;
+
 // Initialize backends: load all available (CUDA, Metal, Vulkan...),
 // pick the best one, keep CPU as fallback.
 // label: log prefix, e.g. "DiT", "VAE", "LM"
+// Subsequent calls reuse the same backend (single VMM pool).
 static BackendPair backend_init(const char * label) {
+    if (g_backend_refs > 0) {
+        g_backend_refs++;
+        fprintf(stderr, "[Load] %s backend: %s (shared)\n",
+                label, ggml_backend_name(g_backend_cache.backend));
+        return g_backend_cache;
+    }
+
     ggml_backend_load_all();
     BackendPair bp = {};
     bp.backend = ggml_backend_init_best();
@@ -54,9 +66,22 @@ static BackendPair backend_init(const char * label) {
     }
 #endif
 
+    g_backend_cache = bp;
+    g_backend_refs = 1;
     return bp;
 }
 
+// Release a backend reference. Frees GPU + CPU backends when refcount hits 0.
+static void backend_release(ggml_backend_t backend, ggml_backend_t cpu_backend) {
+    if (g_backend_refs <= 0) return;
+    g_backend_refs--;
+    if (g_backend_refs == 0) {
+        if (backend && backend != cpu_backend) ggml_backend_free(backend);
+        if (cpu_backend) ggml_backend_free(cpu_backend);
+        g_backend_cache = {};
+    }
+}
+
 // Create a scheduler from a backend pair.
 // max_nodes: graph size hint (4096 for small models, 8192 for large)
 static ggml_backend_sched_t backend_sched_new(BackendPair bp, int max_nodes) {

src/cond-enc.h

@@ -356,8 +356,7 @@ static void cond_ggml_forward(CondGGML * m,
 // Free
 static void cond_ggml_free(CondGGML * m) {
     if (m->sched) ggml_backend_sched_free(m->sched);
-    if (m->backend && m->backend != m->cpu_backend) ggml_backend_free(m->backend);
-    if (m->cpu_backend) ggml_backend_free(m->cpu_backend);
+    backend_release(m->backend, m->cpu_backend);
     wctx_free(&m->wctx);
     *m = {};
 }

src/dit.h

@@ -410,8 +410,7 @@ bool dit_ggml_load_lora(DiTGGML * m, const char * lora_path, float scale);
 
 static void dit_ggml_free(DiTGGML * m) {
     if (m->sched) ggml_backend_sched_free(m->sched);
-    if (m->backend && m->backend != m->cpu_backend) ggml_backend_free(m->backend);
-    if (m->cpu_backend) ggml_backend_free(m->cpu_backend);
+    backend_release(m->backend, m->cpu_backend);
     wctx_free(&m->wctx);
     if (m->lora_wctx.ctx) wctx_free(&m->lora_wctx);
     *m = {};

src/qwen3-enc.h

@@ -467,8 +467,7 @@ static void qwen3_embed_lookup(Qwen3GGML * m, const int * token_ids, int S, floa
 // Free
 static void qwen3_free(Qwen3GGML * m) {
     if (m->sched) ggml_backend_sched_free(m->sched);
-    if (m->backend && m->backend != m->cpu_backend) ggml_backend_free(m->backend);
-    if (m->cpu_backend) ggml_backend_free(m->cpu_backend);
+    backend_release(m->backend, m->cpu_backend);
     wctx_free(&m->wctx);
     *m = {};
 }

src/qwen3-lm.h

@@ -275,7 +275,8 @@ static struct ggml_tensor * qw3lm_build_attn(
         int kv_pos,
         int kv_len,
         int n_tokens,
-        bool use_flash_attn = true) {
+        bool use_flash_attn = true,
+        bool clamp_fp16 = false) {
 
     int D   = c.head_dim;
     int Nh  = c.n_heads;
@@ -328,6 +329,12 @@ static struct ggml_tensor * qw3lm_build_attn(
     k = ggml_cont(ctx, k);
     v = ggml_cont(ctx, v);
 
+    // Clamp V before F16 cast: sub-Ampere tensor cores accumulate in FP16,
+    // V projection can overflow to inf which corrupts all subsequent attention
+    if (clamp_fp16) {
+        v = ggml_clamp(ctx, v, -65504.0f, 65504.0f);
+    }
+
     // Write K,V to cache at kv_pos
     // Cache layout: [D, max_seq, Nkv] f16
     size_t nb1 = (size_t)D * ggml_type_size(GGML_TYPE_F16);
@@ -410,10 +417,13 @@ static void qw3lm_forward(Qwen3LM * m, const int * token_ids, int n_tokens,
         struct ggml_tensor * attn = qw3lm_build_attn(
             ctx, gf, c, ly, norm, positions, mask,
             m->kv_k[kv_set][l], m->kv_v[kv_set][l],
-            kv_pos, kv_len, n_tokens, m->use_flash_attn);
+            kv_pos, kv_len, n_tokens, m->use_flash_attn, m->clamp_fp16);
 
         // Residual
         hidden = ggml_add(ctx, hidden, attn);
+        if (m->clamp_fp16) {
+            hidden = ggml_clamp(ctx, hidden, -65504.0f, 65504.0f);
+        }
 
         // Post-attention norm + MLP
         norm = qwen3_rms_norm(ctx, hidden, ly->post_attn_layernorm, c.rms_norm_eps);
@@ -577,6 +587,11 @@ static void qw3lm_forward_batch(Qwen3LM * m, const int * token_ids,
         k = ggml_cont(ctx, k);
         v = ggml_cont(ctx, v);
 
+        // Clamp V before F16 cast (sub-Ampere FP16 accumulation overflow)
+        if (m->clamp_fp16) {
+            v = ggml_clamp(ctx, v, -65504.0f, 65504.0f);
+        }
+
         // Batched attention with 4D KV cache
         float scale = 1.0f / sqrtf((float)D);
 
@@ -633,6 +648,9 @@ static void qw3lm_forward_batch(Qwen3LM * m, const int * token_ids,
         // Batched O proj
         struct ggml_tensor * attn_out = qwen3_linear(ctx, ly->o_proj, attn_cat);
         hidden = ggml_add(ctx, hidden, attn_out);
+        if (m->clamp_fp16) {
+            hidden = ggml_clamp(ctx, hidden, -65504.0f, 65504.0f);
+        }
 
         // Batched FFN
         norm = qwen3_rms_norm(ctx, hidden, ly->post_attn_layernorm, c.rms_norm_eps);
@@ -706,8 +724,7 @@ static void qw3lm_free(Qwen3LM * m) {
     if (m->sched) ggml_backend_sched_free(m->sched);
     if (m->kv_buf) ggml_backend_buffer_free(m->kv_buf);
     if (m->kv_ctx) ggml_free(m->kv_ctx);
-    if (m->backend && m->backend != m->cpu_backend) ggml_backend_free(m->backend);
-    if (m->cpu_backend) ggml_backend_free(m->cpu_backend);
+    backend_release(m->backend, m->cpu_backend);
     wctx_free(&m->wctx);
     *m = {};
 }

src/vae.h

@@ -552,8 +552,7 @@ static void vae_ggml_free(VAEGGML * m) {
     if (m->sched) ggml_backend_sched_free(m->sched);
     if (m->buf) ggml_backend_buffer_free(m->buf);
     if (m->weight_ctx) ggml_free(m->weight_ctx);
-    if (m->backend && m->backend != m->cpu_backend) ggml_backend_free(m->backend);
-    if (m->cpu_backend) ggml_backend_free(m->cpu_backend);
+    backend_release(m->backend, m->cpu_backend);
     *m = {};
 }