build_corpus: lossless slot-mode identifier encoding

src/builtins.c

@@ -29,6 +29,33 @@
 #include "model_internal.h"
 #endif
 
+/* How many bindings the runtime itself installs.
+ *
+ * register_builtins fills the global env from slot 0 upward and nothing in the
+ * language ever unbinds a name, so slots [0, g_builtin_binding_count) are
+ * exactly the runtime's own registry and everything a user script defines lands
+ * after it. Set at the end of register_builtins; 0 until then.
+ *
+ * This exists because "is this name part of the language?" cannot be answered
+ * by looking up the LIVE global env: by the time a script calls a builtin, its
+ * own top-level functions are in that env too, and they are indistinguishable
+ * from stdlib entries by Value type alone. */
+static int g_builtin_binding_count = 0;
+
+/* True iff `name` is a name the runtime registered, as opposed to one the
+ * running script defined. Deliberately allocation-free — the suite gates on a
+ * zero leak tally, and a process-lifetime name snapshot would show up there. */
+int eigs_is_registered_builtin(const char *name) {
+    if (!name || !g_global_env) return 0;
+    int n = g_builtin_binding_count;
+    if (n > g_global_env->count) n = g_global_env->count;
+    for (int i = 0; i < n; i++) {
+        if (g_global_env->names[i] && strcmp(g_global_env->names[i], name) == 0)
+            return 1;
+    }
+    return 0;
+}
+
 /* Internal helpers defined in eigenscript.c. */
 Value* make_num_permanent(double n);
 const char* val_type_name(ValType t);
@@ -2276,6 +2303,37 @@ Value* builtin_build_corpus(Value *arg) {
 
     int top_n = (int)topn_val->data.num;
     int n_files = file_list->data.list.count;
+
+    /* ---- SLOT MODE (optional 6th arg: slot_count > 0) --------------------
+     * Measured on the 2026-07 ecosystem corpus: of 379,321 identifier
+     * occurrences, only 7.1% are builtin/stdlib names (219 distinct) and
+     * 92.9% are local names (25,355 distinct). A correctness requirement
+     * attaches only to the first group -- you cannot call `pront` -- while
+     * ANY consistent renaming of a local yields an equally correct program.
+     *
+     * Frequency mode spends the whole budget memorising the 25,355 arbitrary
+     * names and still drops 45.7% of occurrences into one fallback token, so
+     * distinct variables become indistinguishable and a correct program is
+     * not expressible: `v0 is v0 + v1` collapses to <ident> is <ident> +
+     * <ident>.
+     *
+     * Slot mode instead gives every builtin an exact token and encodes each
+     * local by WHICH local it is, via an LRU slot table. Distinct locals per
+     * real 32-token window: median 5, p99 10, max 17 -- so 20 slots cover
+     * 100% of windows, and no eviction can occur inside a window (all of a
+     * window's locals are more recently used than anything it would evict).
+     * Vocabulary lands comparable to frequency mode's -- measured 362 on the
+     * 2026-07 ecosystem corpus at slot_count=20, against 341 -- with ZERO
+     * identifier information lost. The win is losslessness, not vocab size.
+     * (That 362 was measured before the registry fix below, so it counts four
+     * driver-defined names that no longer qualify; the corrected figure has not
+     * been re-measured on the full corpus.) */
+    int slot_count = 0;
+    if (arg->data.list.count >= 6) {
+        Value *sv = arg->data.list.items[5];
+        if (sv && sv->type == VAL_NUM) slot_count = (int)sv->data.num;
+    }
+    if (slot_count < 0) slot_count = 0;
     /* Source of truth: the size of the TokType enum. Hardcoding 54 (which is
      * what this was before) silently aliases TOK_LBRACKET (=54) onto the most
      * common extended ident slot whenever the enum grows. */
@@ -2342,7 +2400,33 @@ Value* builtin_build_corpus(Value *arg) {
     fprintf(stderr, "\nFiles: %d/%d\n", files_found, n_files);
     fprintf(stderr, "Distinct identifiers: %d\n", n_idents);
 
-    /* ---- Pass 2: pick top-N identifiers ---- */
+    /* ---- Pass 2: pick the exact-token identifiers ----
+     * Slot mode asks the RUNTIME'S OWN registry which names are builtins,
+     * rather than carrying a hardcoded list that would silently drift as
+     * builtins are added. Everything else becomes a slot.
+     *
+     * The registry is the set captured at REGISTRATION time, not the live
+     * global env. Reading the live env instead promotes the driver script's own
+     * top-level functions to exact tokens — measured: `collect_eigs`,
+     * `_skip_dir`, `_skip_repo` and `_has_pathological_repetition` from
+     * iLambdaAi's build_corpus_v3.eigs landed in the vocabulary beside `print`
+     * and `len`. That is wrong twice over: those names are not part of the
+     * language, and it makes the vocabulary a function of whichever script
+     * built the corpus, so adding a helper to the driver silently renumbers
+     * tokens and quietly invalidates comparison against every model trained on
+     * an earlier build. */
+    if (slot_count > 0) {
+        int keep = 0;
+        for (int i = 0; i < n_idents; i++) {
+            if (eigs_is_registered_builtin(idents[i].name)) {
+                if (keep != i) { IdentEntry tmp = idents[keep]; idents[keep] = idents[i]; idents[i] = tmp; }
+                keep++;
+            }
+        }
+        top_n = keep;
+        fprintf(stderr, "Slot mode: %d builtin names kept exact, %d locals -> %d slots\n",
+                keep, n_idents - keep, slot_count);
+    }
     int actual_top = top_n < n_idents ? top_n : n_idents;
     if (actual_top <= 0) actual_top = 0;
     char **top_names = xcalloc(actual_top > 0 ? actual_top : 1, sizeof(char*));
@@ -2352,18 +2436,37 @@ Value* builtin_build_corpus(Value *arg) {
     int *work_counts = xmalloc_array(n_idents, sizeof(int));
     for (int i = 0; i < n_idents; i++) work_counts[i] = idents[i].count;
 
-    for (int t = 0; t < actual_top; t++) {
-        int best_idx = -1, best_val = -1;
-        for (int j = 0; j < n_idents; j++) {
-            if (work_counts[j] > best_val) { best_val = work_counts[j]; best_idx = j; }
+    if (slot_count > 0) {
+        /* Slot mode already partitioned the builtins to the front; take them
+         * verbatim. Re-selecting by frequency here would hand the exact tokens
+         * to the most COMMON names (which are locals like `total`/`items`) and
+         * push the builtins into slots -- exactly backwards, since a local's
+         * name is arbitrary and a builtin's is not. */
+        for (int t = 0; t < actual_top; t++) {
+            top_names[t] = idents[t].name;
+            top_ids[t] = FIRST_IDENT + t;
+        }
+    } else {
+        for (int t = 0; t < actual_top; t++) {
+            int best_idx = -1, best_val = -1;
+            for (int j = 0; j < n_idents; j++) {
+                if (work_counts[j] > best_val) { best_val = work_counts[j]; best_idx = j; }
+            }
+            if (best_idx < 0) break;
+            top_names[t] = idents[best_idx].name;
+            top_ids[t] = FIRST_IDENT + t;
+            work_counts[best_idx] = -1;
         }
-        if (best_idx < 0) break;
-        top_names[t] = idents[best_idx].name;
-        top_ids[t] = FIRST_IDENT + t;
-        work_counts[best_idx] = -1;
     }
     free(work_counts);
 
+    /* LRU slot table state (slot mode only). */
+    char **slot_names = NULL; long *slot_used = NULL; long slot_clock = 0;
+    if (slot_count > 0) {
+        slot_names = xcalloc(slot_count, sizeof(char*));
+        slot_used  = xcalloc(slot_count, sizeof(long));
+    }
+
     fprintf(stderr, "\nTop %d identifiers:\n", actual_top);
     for (int i = 0; i < 10 && i < actual_top; i++) {
         fprintf(stderr, "  %3d  %-20s  %d uses\n", top_ids[i], top_names[i],
@@ -2407,12 +2510,33 @@ Value* builtin_build_corpus(Value *arg) {
             int tid = tl.tokens[i].type;
             /* Replace known identifiers with extended IDs */
             if (tid == TOK_IDENT && tl.tokens[i].str_val && tl.tokens[i].str_val[0]) {
+                int matched = 0;
                 for (int j = 0; j < actual_top; j++) {
                     if (strcmp(top_names[j], tl.tokens[i].str_val) == 0) {
                         tid = top_ids[j];
+                        matched = 1;
                         break;
                     }
                 }
+                if (!matched && slot_count > 0) {
+                    /* LRU slot table: a name keeps its slot until evicted, so
+                     * every occurrence inside a window maps to the SAME token
+                     * -- which is precisely what lets the model express "the
+                     * variable I just read" and what the fallback destroyed. */
+                    int hit = -1;
+                    for (int j = 0; j < slot_count; j++)
+                        if (slot_names[j] && strcmp(slot_names[j], tl.tokens[i].str_val) == 0) { hit = j; break; }
+                    if (hit < 0) {
+                        int lru = 0;
+                        for (int j = 1; j < slot_count; j++)
+                            if (slot_used[j] < slot_used[lru]) lru = j;
+                        free(slot_names[lru]);
+                        slot_names[lru] = xstrdup(tl.tokens[i].str_val);
+                        hit = lru;
+                    }
+                    slot_used[hit] = ++slot_clock;
+                    tid = FIRST_IDENT + actual_top + hit;
+                }
             }
             double d = (double)tid;
             fwrite(&d, sizeof(double), 1, stream_file);
@@ -2424,6 +2548,10 @@ Value* builtin_build_corpus(Value *arg) {
     }
 
     fclose(stream_file);
+    if (slot_names) {
+        for (int j = 0; j < slot_count; j++) free(slot_names[j]);
+        free(slot_names); free(slot_used);
+    }
     fprintf(stderr, "\nWritten: %s (%d tokens)\n", stream_path_val->data.str, stream_pos);
 
     /* ---- Write identifier vocab JSON ----
@@ -2434,8 +2562,9 @@ Value* builtin_build_corpus(Value *arg) {
     FILE *vocab_file = xfopen_write(vocab_path_val->data.str, "w");
     if (vocab_file) {
         fprintf(vocab_file,
-                "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"top_n\": %d",
-                FIRST_IDENT, BASE_VOCAB + actual_top, BASE_VOCAB, actual_top);
+                "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"top_n\": %d",
+                FIRST_IDENT, BASE_VOCAB + actual_top + slot_count, BASE_VOCAB,
+                slot_count, FIRST_IDENT + actual_top, actual_top);
         fprintf(vocab_file, ", \"structural_ids\": {\"newline\": %d, \"indent\": %d, \"dedent\": %d, \"eof\": %d, \"ident_fallback\": %d}",
                 (int)TOK_NEWLINE, (int)TOK_INDENT, (int)TOK_DEDENT, (int)TOK_EOF, (int)TOK_IDENT);
         fprintf(vocab_file, ", \"base_names\": [");
@@ -6769,4 +6898,8 @@ void register_builtins(Env *env) {
     if (getenv("EIGS_BORROW_GUARD_SELFTEST"))
         env_set_local_owned(env, "__borrow_guard_selftest", make_builtin(builtin_borrow_guard_selftest));
 #endif
+
+    /* Everything bound above this line is the language; everything bound after
+     * it belongs to whatever program is running. See eigs_is_registered_builtin. */
+    g_builtin_binding_count = env->count;
 }

src/model_train.c

@@ -14,6 +14,31 @@
  * Existing matmul kernels work unchanged on the result.
  * ================================================================ */
 
+/* Ternary code-flip instrumentation (EIGS_TERNARY_FLIPS=1).
+ *
+ * QAT snaps every weight to {-alpha, 0, +alpha} after each step. A gradient
+ * update smaller than the threshold (alpha*0.5) leaves the code unchanged and
+ * is therefore ERASED as far as the forward pass is concerned -- the master
+ * weight moved, the model did not. If that is happening to most weights most of
+ * the time, the effective model is frozen no matter what the loss curve says.
+ *
+ * Counts how many of the {-1,0,+1} codes actually change per requantise, which
+ * is the only quantity that distinguishes "learning" from "accumulating
+ * sub-threshold noise in the master weights". */
+static int64_t g_tern_flips = 0, g_tern_total = 0;
+static int tern_count_enabled(void) {
+    static int v = -1;
+    if (v < 0) v = getenv("EIGS_TERNARY_FLIPS") ? 1 : 0;
+    return v;
+}
+void eigs_tern_report(int step) {
+    if (!tern_count_enabled() || g_tern_total == 0) return;
+    fprintf(stderr, "[tern] step=%d flips=%lld/%lld (%.4f%%)\n",
+            step, (long long)g_tern_flips, (long long)g_tern_total,
+            100.0 * (double)g_tern_flips / (double)g_tern_total);
+    g_tern_flips = 0; g_tern_total = 0;
+}
+
 void quantize_ternary(float *dst, float *src, int64_t n, float *out_alpha) {
     if (n <= 0) { if (out_alpha) *out_alpha = 0.0f; return; }
     float abs_sum = 0.0f;
@@ -23,16 +48,27 @@ void quantize_ternary(float *dst, float *src, int64_t n, float *out_alpha) {
     }
     float alpha = abs_sum / (float)n;
     float threshold = alpha * 0.5f;
+    int _tc = tern_count_enabled();
     for (int64_t i = 0; i < n; i++) {
         float w = src[i];
         float aw = (w < 0.0f) ? -w : w;
+        float nv;
         if (aw <= threshold) {
-            dst[i] = 0.0f;
+            nv = 0.0f;
         } else if (w > 0.0f) {
-            dst[i] = alpha;
+            nv = alpha;
         } else {
-            dst[i] = -alpha;
+            nv = -alpha;
+        }
+        if (_tc) {
+            /* Compare CODES, not values: alpha is recomputed every step, so the
+             * stored magnitude drifts even when the code is unchanged. */
+            int oldc = (dst[i] > 0.0f) ? 1 : ((dst[i] < 0.0f) ? -1 : 0);
+            int newc = (nv    > 0.0f) ? 1 : ((nv    < 0.0f) ? -1 : 0);
+            if (oldc != newc) g_tern_flips++;
+            g_tern_total++;
         }
+        dst[i] = nv;
     }
     if (out_alpha) *out_alpha = alpha;
 }
@@ -1501,6 +1537,7 @@ static int native_train_step_impl(int *input_ids, int input_len, int *output_ids
     /* Re-quantize master weights → ternary copies for next forward pass (if ternary format) */
     if (g_model.weight_format == WEIGHT_FORMAT_TERNARY) {
         requantize_all_layers(&g_model);
+        if ((g_training_samples % 100) == 0) eigs_tern_report(g_training_samples);
     }
 
     g_model_age += num_tokens;

tests/run_all_tests.sh

@@ -1710,6 +1710,20 @@ echo "[43a2] Builtin Argument Errors (26 checks)"
 check_eigs_suite "builtin argument errors" test_builtin_errors.eigs "All builtin_errors tests passed" 30
 check_eigs_suite "module-boundary write insulation (#373)" test_module_scope.eigs "All module-scope tests passed" 9
 check_eigs_suite "import top-level scope insulation vs load_file current-scope contract (#589)" test_import_toplevel_scope.eigs "All import top-level scope tests passed" 11
+
+echo "[43a2b] build_corpus slot-mode identifier encoding (6 checks)"
+CS_OUTPUT=$(bash "$TESTS_DIR/test_corpus_slots.sh" 2>&1)
+CS_PASS=$(echo "$CS_OUTPUT" | grep -c "PASS:" || true)
+CS_FAIL=$(echo "$CS_OUTPUT" | grep -c "FAIL:" || true)
+TOTAL=$((TOTAL + CS_PASS + CS_FAIL))
+PASS=$((PASS + CS_PASS))
+FAIL=$((FAIL + CS_FAIL))
+if [ "$CS_FAIL" -gt 0 ]; then
+    echo "  FAIL: $CS_FAIL slot-encoding check(s) failed"
+    echo "$CS_OUTPUT" | grep "FAIL:" | head -4
+else
+    echo "  PASS: slot mode is lossless and preserves identifier identity"
+fi
 echo ""
 
 # [43a3] EigenStore header-validation / corruption error paths (ext_store.c)