diff --git a/src/builtins.c b/src/builtins.c index 7a3963e..26e53a5 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -29,6 +29,33 @@ #include "model_internal.h" #endif +/* How many bindings the runtime itself installs. + * + * register_builtins fills the global env from slot 0 upward and nothing in the + * language ever unbinds a name, so slots [0, g_builtin_binding_count) are + * exactly the runtime's own registry and everything a user script defines lands + * after it. Set at the end of register_builtins; 0 until then. + * + * This exists because "is this name part of the language?" cannot be answered + * by looking up the LIVE global env: by the time a script calls a builtin, its + * own top-level functions are in that env too, and they are indistinguishable + * from stdlib entries by Value type alone. */ +static int g_builtin_binding_count = 0; + +/* True iff `name` is a name the runtime registered, as opposed to one the + * running script defined. Deliberately allocation-free — the suite gates on a + * zero leak tally, and a process-lifetime name snapshot would show up there. */ +int eigs_is_registered_builtin(const char *name) { + if (!name || !g_global_env) return 0; + int n = g_builtin_binding_count; + if (n > g_global_env->count) n = g_global_env->count; + for (int i = 0; i < n; i++) { + if (g_global_env->names[i] && strcmp(g_global_env->names[i], name) == 0) + return 1; + } + return 0; +} + /* Internal helpers defined in eigenscript.c. */ Value* make_num_permanent(double n); const char* val_type_name(ValType t); @@ -2276,6 +2303,37 @@ Value* builtin_build_corpus(Value *arg) { int top_n = (int)topn_val->data.num; int n_files = file_list->data.list.count; + + /* ---- SLOT MODE (optional 6th arg: slot_count > 0) -------------------- + * Measured on the 2026-07 ecosystem corpus: of 379,321 identifier + * occurrences, only 7.1% are builtin/stdlib names (219 distinct) and + * 92.9% are local names (25,355 distinct). A correctness requirement + * attaches only to the first group -- you cannot call `pront` -- while + * ANY consistent renaming of a local yields an equally correct program. + * + * Frequency mode spends the whole budget memorising the 25,355 arbitrary + * names and still drops 45.7% of occurrences into one fallback token, so + * distinct variables become indistinguishable and a correct program is + * not expressible: `v0 is v0 + v1` collapses to is + + * . + * + * Slot mode instead gives every builtin an exact token and encodes each + * local by WHICH local it is, via an LRU slot table. Distinct locals per + * real 32-token window: median 5, p99 10, max 17 -- so 20 slots cover + * 100% of windows, and no eviction can occur inside a window (all of a + * window's locals are more recently used than anything it would evict). + * Vocabulary lands comparable to frequency mode's -- measured 362 on the + * 2026-07 ecosystem corpus at slot_count=20, against 341 -- with ZERO + * identifier information lost. The win is losslessness, not vocab size. + * (That 362 was measured before the registry fix below, so it counts four + * driver-defined names that no longer qualify; the corrected figure has not + * been re-measured on the full corpus.) */ + int slot_count = 0; + if (arg->data.list.count >= 6) { + Value *sv = arg->data.list.items[5]; + if (sv && sv->type == VAL_NUM) slot_count = (int)sv->data.num; + } + if (slot_count < 0) slot_count = 0; /* Source of truth: the size of the TokType enum. Hardcoding 54 (which is * what this was before) silently aliases TOK_LBRACKET (=54) onto the most * common extended ident slot whenever the enum grows. */ @@ -2342,7 +2400,33 @@ Value* builtin_build_corpus(Value *arg) { fprintf(stderr, "\nFiles: %d/%d\n", files_found, n_files); fprintf(stderr, "Distinct identifiers: %d\n", n_idents); - /* ---- Pass 2: pick top-N identifiers ---- */ + /* ---- Pass 2: pick the exact-token identifiers ---- + * Slot mode asks the RUNTIME'S OWN registry which names are builtins, + * rather than carrying a hardcoded list that would silently drift as + * builtins are added. Everything else becomes a slot. + * + * The registry is the set captured at REGISTRATION time, not the live + * global env. Reading the live env instead promotes the driver script's own + * top-level functions to exact tokens — measured: `collect_eigs`, + * `_skip_dir`, `_skip_repo` and `_has_pathological_repetition` from + * iLambdaAi's build_corpus_v3.eigs landed in the vocabulary beside `print` + * and `len`. That is wrong twice over: those names are not part of the + * language, and it makes the vocabulary a function of whichever script + * built the corpus, so adding a helper to the driver silently renumbers + * tokens and quietly invalidates comparison against every model trained on + * an earlier build. */ + if (slot_count > 0) { + int keep = 0; + for (int i = 0; i < n_idents; i++) { + if (eigs_is_registered_builtin(idents[i].name)) { + if (keep != i) { IdentEntry tmp = idents[keep]; idents[keep] = idents[i]; idents[i] = tmp; } + keep++; + } + } + top_n = keep; + fprintf(stderr, "Slot mode: %d builtin names kept exact, %d locals -> %d slots\n", + keep, n_idents - keep, slot_count); + } int actual_top = top_n < n_idents ? top_n : n_idents; if (actual_top <= 0) actual_top = 0; char **top_names = xcalloc(actual_top > 0 ? actual_top : 1, sizeof(char*)); @@ -2352,18 +2436,37 @@ Value* builtin_build_corpus(Value *arg) { int *work_counts = xmalloc_array(n_idents, sizeof(int)); for (int i = 0; i < n_idents; i++) work_counts[i] = idents[i].count; - for (int t = 0; t < actual_top; t++) { - int best_idx = -1, best_val = -1; - for (int j = 0; j < n_idents; j++) { - if (work_counts[j] > best_val) { best_val = work_counts[j]; best_idx = j; } + if (slot_count > 0) { + /* Slot mode already partitioned the builtins to the front; take them + * verbatim. Re-selecting by frequency here would hand the exact tokens + * to the most COMMON names (which are locals like `total`/`items`) and + * push the builtins into slots -- exactly backwards, since a local's + * name is arbitrary and a builtin's is not. */ + for (int t = 0; t < actual_top; t++) { + top_names[t] = idents[t].name; + top_ids[t] = FIRST_IDENT + t; + } + } else { + for (int t = 0; t < actual_top; t++) { + int best_idx = -1, best_val = -1; + for (int j = 0; j < n_idents; j++) { + if (work_counts[j] > best_val) { best_val = work_counts[j]; best_idx = j; } + } + if (best_idx < 0) break; + top_names[t] = idents[best_idx].name; + top_ids[t] = FIRST_IDENT + t; + work_counts[best_idx] = -1; } - if (best_idx < 0) break; - top_names[t] = idents[best_idx].name; - top_ids[t] = FIRST_IDENT + t; - work_counts[best_idx] = -1; } free(work_counts); + /* LRU slot table state (slot mode only). */ + char **slot_names = NULL; long *slot_used = NULL; long slot_clock = 0; + if (slot_count > 0) { + slot_names = xcalloc(slot_count, sizeof(char*)); + slot_used = xcalloc(slot_count, sizeof(long)); + } + fprintf(stderr, "\nTop %d identifiers:\n", actual_top); for (int i = 0; i < 10 && i < actual_top; i++) { fprintf(stderr, " %3d %-20s %d uses\n", top_ids[i], top_names[i], @@ -2407,12 +2510,33 @@ Value* builtin_build_corpus(Value *arg) { int tid = tl.tokens[i].type; /* Replace known identifiers with extended IDs */ if (tid == TOK_IDENT && tl.tokens[i].str_val && tl.tokens[i].str_val[0]) { + int matched = 0; for (int j = 0; j < actual_top; j++) { if (strcmp(top_names[j], tl.tokens[i].str_val) == 0) { tid = top_ids[j]; + matched = 1; break; } } + if (!matched && slot_count > 0) { + /* LRU slot table: a name keeps its slot until evicted, so + * every occurrence inside a window maps to the SAME token + * -- which is precisely what lets the model express "the + * variable I just read" and what the fallback destroyed. */ + int hit = -1; + for (int j = 0; j < slot_count; j++) + if (slot_names[j] && strcmp(slot_names[j], tl.tokens[i].str_val) == 0) { hit = j; break; } + if (hit < 0) { + int lru = 0; + for (int j = 1; j < slot_count; j++) + if (slot_used[j] < slot_used[lru]) lru = j; + free(slot_names[lru]); + slot_names[lru] = xstrdup(tl.tokens[i].str_val); + hit = lru; + } + slot_used[hit] = ++slot_clock; + tid = FIRST_IDENT + actual_top + hit; + } } double d = (double)tid; fwrite(&d, sizeof(double), 1, stream_file); @@ -2424,6 +2548,10 @@ Value* builtin_build_corpus(Value *arg) { } fclose(stream_file); + if (slot_names) { + for (int j = 0; j < slot_count; j++) free(slot_names[j]); + free(slot_names); free(slot_used); + } fprintf(stderr, "\nWritten: %s (%d tokens)\n", stream_path_val->data.str, stream_pos); /* ---- Write identifier vocab JSON ---- @@ -2434,8 +2562,9 @@ Value* builtin_build_corpus(Value *arg) { FILE *vocab_file = xfopen_write(vocab_path_val->data.str, "w"); if (vocab_file) { fprintf(vocab_file, - "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"top_n\": %d", - FIRST_IDENT, BASE_VOCAB + actual_top, BASE_VOCAB, actual_top); + "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"top_n\": %d", + FIRST_IDENT, BASE_VOCAB + actual_top + slot_count, BASE_VOCAB, + slot_count, FIRST_IDENT + actual_top, actual_top); fprintf(vocab_file, ", \"structural_ids\": {\"newline\": %d, \"indent\": %d, \"dedent\": %d, \"eof\": %d, \"ident_fallback\": %d}", (int)TOK_NEWLINE, (int)TOK_INDENT, (int)TOK_DEDENT, (int)TOK_EOF, (int)TOK_IDENT); fprintf(vocab_file, ", \"base_names\": ["); @@ -6769,4 +6898,8 @@ void register_builtins(Env *env) { if (getenv("EIGS_BORROW_GUARD_SELFTEST")) env_set_local_owned(env, "__borrow_guard_selftest", make_builtin(builtin_borrow_guard_selftest)); #endif + + /* Everything bound above this line is the language; everything bound after + * it belongs to whatever program is running. See eigs_is_registered_builtin. */ + g_builtin_binding_count = env->count; } diff --git a/src/model_train.c b/src/model_train.c index bcab0e9..3ce5f24 100644 --- a/src/model_train.c +++ b/src/model_train.c @@ -14,6 +14,31 @@ * Existing matmul kernels work unchanged on the result. * ================================================================ */ +/* Ternary code-flip instrumentation (EIGS_TERNARY_FLIPS=1). + * + * QAT snaps every weight to {-alpha, 0, +alpha} after each step. A gradient + * update smaller than the threshold (alpha*0.5) leaves the code unchanged and + * is therefore ERASED as far as the forward pass is concerned -- the master + * weight moved, the model did not. If that is happening to most weights most of + * the time, the effective model is frozen no matter what the loss curve says. + * + * Counts how many of the {-1,0,+1} codes actually change per requantise, which + * is the only quantity that distinguishes "learning" from "accumulating + * sub-threshold noise in the master weights". */ +static int64_t g_tern_flips = 0, g_tern_total = 0; +static int tern_count_enabled(void) { + static int v = -1; + if (v < 0) v = getenv("EIGS_TERNARY_FLIPS") ? 1 : 0; + return v; +} +void eigs_tern_report(int step) { + if (!tern_count_enabled() || g_tern_total == 0) return; + fprintf(stderr, "[tern] step=%d flips=%lld/%lld (%.4f%%)\n", + step, (long long)g_tern_flips, (long long)g_tern_total, + 100.0 * (double)g_tern_flips / (double)g_tern_total); + g_tern_flips = 0; g_tern_total = 0; +} + void quantize_ternary(float *dst, float *src, int64_t n, float *out_alpha) { if (n <= 0) { if (out_alpha) *out_alpha = 0.0f; return; } float abs_sum = 0.0f; @@ -23,16 +48,27 @@ void quantize_ternary(float *dst, float *src, int64_t n, float *out_alpha) { } float alpha = abs_sum / (float)n; float threshold = alpha * 0.5f; + int _tc = tern_count_enabled(); for (int64_t i = 0; i < n; i++) { float w = src[i]; float aw = (w < 0.0f) ? -w : w; + float nv; if (aw <= threshold) { - dst[i] = 0.0f; + nv = 0.0f; } else if (w > 0.0f) { - dst[i] = alpha; + nv = alpha; } else { - dst[i] = -alpha; + nv = -alpha; + } + if (_tc) { + /* Compare CODES, not values: alpha is recomputed every step, so the + * stored magnitude drifts even when the code is unchanged. */ + int oldc = (dst[i] > 0.0f) ? 1 : ((dst[i] < 0.0f) ? -1 : 0); + int newc = (nv > 0.0f) ? 1 : ((nv < 0.0f) ? -1 : 0); + if (oldc != newc) g_tern_flips++; + g_tern_total++; } + dst[i] = nv; } if (out_alpha) *out_alpha = alpha; } @@ -1501,6 +1537,7 @@ static int native_train_step_impl(int *input_ids, int input_len, int *output_ids /* Re-quantize master weights → ternary copies for next forward pass (if ternary format) */ if (g_model.weight_format == WEIGHT_FORMAT_TERNARY) { requantize_all_layers(&g_model); + if ((g_training_samples % 100) == 0) eigs_tern_report(g_training_samples); } g_model_age += num_tokens; diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 2017d35..9d54df5 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1710,6 +1710,20 @@ echo "[43a2] Builtin Argument Errors (26 checks)" check_eigs_suite "builtin argument errors" test_builtin_errors.eigs "All builtin_errors tests passed" 30 check_eigs_suite "module-boundary write insulation (#373)" test_module_scope.eigs "All module-scope tests passed" 9 check_eigs_suite "import top-level scope insulation vs load_file current-scope contract (#589)" test_import_toplevel_scope.eigs "All import top-level scope tests passed" 11 + +echo "[43a2b] build_corpus slot-mode identifier encoding (6 checks)" +CS_OUTPUT=$(bash "$TESTS_DIR/test_corpus_slots.sh" 2>&1) +CS_PASS=$(echo "$CS_OUTPUT" | grep -c "PASS:" || true) +CS_FAIL=$(echo "$CS_OUTPUT" | grep -c "FAIL:" || true) +TOTAL=$((TOTAL + CS_PASS + CS_FAIL)) +PASS=$((PASS + CS_PASS)) +FAIL=$((FAIL + CS_FAIL)) +if [ "$CS_FAIL" -gt 0 ]; then + echo " FAIL: $CS_FAIL slot-encoding check(s) failed" + echo "$CS_OUTPUT" | grep "FAIL:" | head -4 +else + echo " PASS: slot mode is lossless and preserves identifier identity" +fi echo "" # [43a3] EigenStore header-validation / corruption error paths (ext_store.c) diff --git a/tests/test_corpus_slots.sh b/tests/test_corpus_slots.sh new file mode 100755 index 0000000..018ef4a --- /dev/null +++ b/tests/test_corpus_slots.sh @@ -0,0 +1,172 @@ +#!/bin/bash +# build_corpus slot mode: identifier encoding that can express a correct program. +# +# Frequency mode promotes the top-N identifiers by count and drops everything +# else onto ONE fallback token (bare TOK_IDENT). Measured on the 2026-07 +# ecosystem corpus at top_n=256 that is 45.7% of all identifier occurrences — +# so distinct variables become indistinguishable and `total is total + items[i]` +# encodes as ` is + []`. No model can express +# "assign the variable I just read" through that, which is the property that +# separates well-formed output from correct output. +# +# Slot mode (optional 6th arg) instead gives every BUILTIN an exact token — the +# runtime's own registry decides, so the list cannot drift — and encodes each +# local as one of N reusable LRU slots. Locals are arbitrary by definition: any +# consistent renaming is an equally correct program, so only identity matters. +# +# The assertions below pin the three properties that make it lossless. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +WORK=$(mktemp -d /tmp/eigs_slots_XXXXXX) +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT + +cat > "$WORK/sample.eigs" <<'SRC' +define accumulate(total, items) as: + for i in range of (len of items): + total is total + items[i] + return total +SRC + +# NOTE: the driver deliberately defines a top-level function whose name also +# occurs in the corpus. Reading the LIVE global env to decide "is this a +# builtin?" cannot tell `_driver_helper` from `print`, so it would promote the +# driver's own helper to an exact token — which is what SL05 pins against. +cat > "$WORK/build.eigs" <<'DRV' +define _driver_helper(x) as: + return x + +a is args of null +files is [a[0] + "/sample.eigs"] +r is build_corpus of [files, 4, a[0] + "/s.bin", a[0] + "/v.json", a[0] + "/i.json", 8] +print of ("TOKENS " + (str of r[0])) +DRV + +# `_driver_helper` must exist in the corpus too, or its absence from the vocab +# would prove nothing. +echo '_driver_helper is 1' >> "$WORK/sample.eigs" + +if ! "$EIGS" "$WORK/build.eigs" "$WORK" >"$WORK/out.log" 2>&1; then + fail "SL00 slot-mode build_corpus ran" "see $WORK/out.log" + echo "CORPUSSLOTS: 0 passed, 1 failed" + exit 1 +fi +ok "SL00 slot-mode build_corpus ran" + +# Decode the stream against the vocab and check the three properties. +cat > "$WORK/check.eigs" <<'CHK' +a is args of null +v is json_decode of (read_text of (a[0] + "/v.json")) +stream is tensor_load of (a[0] + "/s.bin") +base is v["base_vocab"] +fs is v["first_slot_id"] +nslot is v["slot_count"] +fb is v["structural_ids"]["ident_fallback"] + +# 1. no identifier may fall back +nfb is 0 +for i in range of (len of stream): + if stream[i] == fb: + nfb is nfb + 1 +print of ("FALLBACK " + (str of nfb)) + +# 2. slots must stay inside the declared range +bad is 0 +for i in range of (len of stream): + t is stream[i] + if t >= fs: + if t >= (fs + nslot): + bad is bad + 1 +print of ("OUTOFRANGE " + (str of bad)) + +# 3. identity: `total` occurs 4x in the sample and must be ONE repeated token. +# Count distinct ids that appear at least 3 times in the slot range. +counts is zeros of 64 +for i in range of (len of stream): + t is stream[i] + if t >= fs: + if t < (fs + nslot): + counts[t - fs] is counts[t - fs] + 1 +rep is 0 +for i in range of nslot: + if counts[i] >= 3: + rep is rep + 1 +print of ("REPEATED " + (str of rep)) +print of ("SLOTCOUNT " + (str of nslot)) +CHK + +OUT=$("$EIGS" "$WORK/check.eigs" "$WORK" 2>/dev/null) +g() { printf '%s\n' "$OUT" | sed -n "s/^$1 //p"; } + +if [ "$(g FALLBACK)" = "0" ]; then + ok "SL01 slot mode emits ZERO fallback tokens" +else + fail "SL01 slot mode still emits the fallback token" "count=$(g FALLBACK)" +fi + +if [ "$(g OUTOFRANGE)" = "0" ]; then + ok "SL02 every slot id lands inside [first_slot_id, +slot_count)" +else + fail "SL02 slot id out of declared range" "count=$(g OUTOFRANGE)" +fi + +# `total` appears 4x and `items` 3x — a repeated slot proves the LRU table +# returns the SAME token for the same name rather than a fresh one each time. +# Exactly two locals clear the 3-occurrence bar, so pin 2: `>= 1` would still +# pass with half the identity mapping broken, and identity is the one property +# this whole encoding exists to provide. +if [ "$(g REPEATED)" = "2" ]; then + ok "SL03 both repeated locals keep ONE slot each (identity preserved)" +else + fail "SL03 wrong number of reused slots — identity not preserved" "repeated=$(g REPEATED), want 2" +fi + +# Regression guard: frequency mode (no 6th arg) must be untouched. +cat > "$WORK/freq.eigs" <<'FRQ' +a is args of null +files is [a[0] + "/sample.eigs"] +r is build_corpus of [files, 2, a[0] + "/sf.bin", a[0] + "/vf.json", a[0] + "/if.json"] +v is json_decode of (read_text of (a[0] + "/vf.json")) +print of ("SLOTS " + (str of v["slot_count"])) +FRQ +FOUT=$("$EIGS" "$WORK/freq.eigs" "$WORK" 2>/dev/null | sed -n 's/^SLOTS //p') +if [ "$FOUT" = "0" ]; then + ok "SL04 frequency mode unchanged (slot_count 0 when no 6th arg)" +else + fail "SL04 frequency mode changed" "slot_count=$FOUT" +fi + +# The exact-token list must be the LANGUAGE's names, not the driver script's. +# Deciding this from the live global env cannot distinguish them — by the time +# build_corpus runs, the driver's own `define`s are bound in that same env — so +# `_driver_helper` would be handed an exact token beside `print` and `len`. +# Beyond being wrong, it makes the vocabulary depend on whichever script built +# the corpus: add a helper to the driver and every token id shifts, silently +# invalidating comparison against models trained on an earlier build. +cat > "$WORK/vocab.eigs" <<'VOC' +a is args of null +v is json_decode of (read_text of (a[0] + "/v.json")) +names is v["names"] +hit is 0 +for i in range of (len of names): + if names[i] == "_driver_helper": + hit is 1 +print of ("DRIVERLEAK " + (str of hit)) +VOC +VOUT=$("$EIGS" "$WORK/vocab.eigs" "$WORK" 2>/dev/null | sed -n 's/^DRIVERLEAK //p') +if [ "$VOUT" = "0" ]; then + ok "SL05 driver script's own functions stay OUT of the exact-token vocab" +else + fail "SL05 driver-defined function leaked into the builtin vocabulary" "_driver_helper promoted" +fi + +echo "CORPUSSLOTS: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]