build_corpus: exact tokens for small integer literals

src/builtins.c

@@ -2334,6 +2334,32 @@ Value* builtin_build_corpus(Value *arg) {
         if (sv && sv->type == VAL_NUM) slot_count = (int)sv->data.num;
     }
     if (slot_count < 0) slot_count = 0;
+
+    /* ---- INTEGER LITERALS (optional 7th arg: int_count > 0) -------------
+     * Every numeric literal tokenizes to a single TOK_NUM, so the value is
+     * ERASED from the stream: `[1,2,3]` and `[0,0,0]` become the identical
+     * `<num>,<num>,<num>`. Measured on the 2026-07 corpus this is doubly
+     * costly. (1) A spec written as assertions is uninformative -- `(f of 4)
+     * == 1` and `(f of 3) == 0` reach the model as one sequence, so it cannot
+     * learn what the test demands. (2) It MANUFACTURES the degenerate
+     * repetition the model collapses into: measured, essentially all of the
+     * corpus's `0,0,0,0` / `"s","s"` cycles (5.3% of tokens) route through the
+     * collapsed literal tokens, and integers 0..31 are 76% of all numeric
+     * literals. Giving small integers exact tokens both makes tests readable
+     * and breaks that false repetition -- `[1,2,3,4]` stops looking like a
+     * cycle -- without discarding any real code a repetition filter would cut.
+     *
+     * int_count == N gives exact tokens to the integers [0, N). N=32 covers
+     * 76% of numeric-literal occurrences for +32 tokens; larger N has a long
+     * tail (0..255 is 86%). Non-integer, negative, and >=N literals keep the
+     * lossy TOK_NUM fallback -- lossless is not the goal here, breaking the
+     * high-frequency collision is. Opt-in: 0 leaves the stream byte-identical. */
+    int int_count = 0;
+    if (arg->data.list.count >= 7) {
+        Value *iv = arg->data.list.items[6];
+        if (iv && iv->type == VAL_NUM) int_count = (int)iv->data.num;
+    }
+    if (int_count < 0) int_count = 0;
     /* Source of truth: the size of the TokType enum. Hardcoding 54 (which is
      * what this was before) silently aliases TOK_LBRACKET (=54) onto the most
      * common extended ident slot whenever the enum grows. */
@@ -2537,6 +2563,14 @@ Value* builtin_build_corpus(Value *arg) {
                     slot_used[hit] = ++slot_clock;
                     tid = FIRST_IDENT + actual_top + hit;
                 }
+            } else if (int_count > 0 && tid == TOK_NUM) {
+                /* Give small non-negative integer literals exact tokens; the
+                 * region sits directly after the slots. floor==value rejects
+                 * 3.5 etc.; the lexer emits '-' as its own token so num_val is
+                 * never negative for a literal, but guard anyway. */
+                double nv = tl.tokens[i].num_val;
+                if (nv >= 0.0 && nv < (double)int_count && nv == (double)(long)nv)
+                    tid = FIRST_IDENT + actual_top + slot_count + (int)nv;
             }
             double d = (double)tid;
             fwrite(&d, sizeof(double), 1, stream_file);
@@ -2562,9 +2596,10 @@ Value* builtin_build_corpus(Value *arg) {
     FILE *vocab_file = xfopen_write(vocab_path_val->data.str, "w");
     if (vocab_file) {
         fprintf(vocab_file,
-                "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"top_n\": %d",
-                FIRST_IDENT, BASE_VOCAB + actual_top + slot_count, BASE_VOCAB,
-                slot_count, FIRST_IDENT + actual_top, actual_top);
+                "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"int_count\": %d, \"first_int_id\": %d, \"top_n\": %d",
+                FIRST_IDENT, BASE_VOCAB + actual_top + slot_count + int_count, BASE_VOCAB,
+                slot_count, FIRST_IDENT + actual_top,
+                int_count, FIRST_IDENT + actual_top + slot_count, actual_top);
         fprintf(vocab_file, ", \"structural_ids\": {\"newline\": %d, \"indent\": %d, \"dedent\": %d, \"eof\": %d, \"ident_fallback\": %d}",
                 (int)TOK_NEWLINE, (int)TOK_INDENT, (int)TOK_DEDENT, (int)TOK_EOF, (int)TOK_IDENT);
         fprintf(vocab_file, ", \"base_names\": [");

tests/run_all_tests.sh

@@ -1726,6 +1726,21 @@ else
 fi
 echo ""
 
+echo "[43a2c] build_corpus integer-literal encoding (5 checks)"
+CI_OUTPUT=$(bash "$TESTS_DIR/test_corpus_ints.sh" 2>&1)
+CI_PASS=$(echo "$CI_OUTPUT" | grep -c "PASS:" || true)
+CI_FAIL=$(echo "$CI_OUTPUT" | grep -c "FAIL:" || true)
+TOTAL=$((TOTAL + CI_PASS + CI_FAIL))
+PASS=$((PASS + CI_PASS))
+FAIL=$((FAIL + CI_FAIL))
+if [ "$CI_FAIL" -gt 0 ]; then
+    echo "  FAIL: $CI_FAIL integer-encoding check(s) failed"
+    echo "$CI_OUTPUT" | grep "FAIL:" | head -4
+else
+    echo "  PASS: integer literals get exact tokens; genuine repetition preserved"
+fi
+echo ""
+
 # [43a3] EigenStore header-validation / corruption error paths (ext_store.c)
 echo "[43a3] Store Corruption Errors (12 checks)"
 check_eigs_suite "store corruption errors" test_store_corruption.eigs "All store_corruption tests passed" 12

tests/test_corpus_ints.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# build_corpus integer-literal encoding: give small integers exact tokens.
+#
+# Every numeric literal tokenizes to a single TOK_NUM, erasing its value: the
+# stream cannot tell `[1,2,3]` from `[0,0,0]`. Measured on the 2026-07 corpus
+# that erasure is doubly costly -- an assertion-as-spec carries no information
+# (`(f of 4)==1` and `(f of 3)==0` become one sequence), AND it manufactures the
+# degenerate `0,0,0` repetition the model collapses into. The optional 7th arg
+# int_count=N gives exact tokens to integers [0, N); everything else keeps the
+# lossy TOK_NUM fallback.
+#
+# The assertions pin the properties that make the fix do its job.
+
+set -u
+TESTS_DIR="$(cd "$(dirname "$0")" && pwd)"
+EIGS="$TESTS_DIR/../src/eigenscript"
+
+PASS=0
+FAIL=0
+ok()   { echo "  PASS: $1"; PASS=$((PASS+1)); }
+fail() { echo "  FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); }
+
+WORK=$(mktemp -d /tmp/eigs_ints_XXXXXX)
+cleanup() { rm -rf "$WORK"; }
+trap cleanup EXIT
+
+# Distinct ascending ints, a genuine zeros run, an out-of-range int, and a float.
+cat > "$WORK/sample.eigs" <<'SRC'
+a is [1, 2, 3, 4, 5]
+b is [0, 0, 0, 0, 0]
+c is 100
+d is 3.5
+SRC
+
+cat > "$WORK/build.eigs" <<'DRV'
+a is args of null
+files is [a[0] + "/sample.eigs"]
+r is build_corpus of [files, 4, a[0] + "/s.bin", a[0] + "/v.json", a[0] + "/i.json", 8, 32]
+print of ("TOKENS " + (str of r[0]))
+DRV
+
+if ! "$EIGS" "$WORK/build.eigs" "$WORK" >"$WORK/out.log" 2>&1; then
+    fail "IL00 int-mode build_corpus ran" "see $WORK/out.log"
+    echo "CORPUSINTS: 0 passed, 1 failed"
+    exit 1
+fi
+ok "IL00 int-mode build_corpus ran"
+
+# Decode against the vocab and check the four properties.
+cat > "$WORK/check.eigs" <<'CHK'
+a is args of null
+v is json_decode of (read_text of (a[0] + "/v.json"))
+stream is tensor_load of (a[0] + "/s.bin")
+fi is v["first_int_id"]
+nint is v["int_count"]
+toknum is v["structural_ids"]["eof"]   # placeholder; real TOK_NUM found below
+
+# 1. the five ascending literals 1..5 must be five DISTINCT tokens in the int
+#    range -- this is the collision break. Count distinct int-range tokens that
+#    appear exactly once and are NOT the zero token.
+counts is zeros of 64
+for i in range of (len of stream):
+    t is stream[i]
+    if t >= fi:
+        if t < (fi + nint):
+            counts[t - fi] is counts[t - fi] + 1
+# distinct nonzero int values present:
+distinct is 0
+for i in range of nint:
+    if i > 0:
+        if counts[i] >= 1:
+            distinct is distinct + 1
+print of ("DISTINCT_NONZERO " + (str of distinct))
+
+# 2. the zeros run: INT0 must appear 5x (genuine repetition preserved as ONE token)
+print of ("ZERO_COUNT " + (str of counts[0]))
+
+# 3. no int token may exceed the declared range
+bad is 0
+for i in range of (len of stream):
+    t is stream[i]
+    if t >= (fi + nint):
+        if t < fi:
+            bad is bad + 1
+print of ("MAXINT " + (str of (fi + nint - 1)))
+CHK
+
+OUT=$("$EIGS" "$WORK/check.eigs" "$WORK" 2>/dev/null)
+g() { printf '%s\n' "$OUT" | sed -n "s/^$1 //p"; }
+
+# 1..5 are five distinct nonzero int tokens: the collision `[1,2,3]`==`[0,0,0]` is broken.
+if [ "$(g DISTINCT_NONZERO)" = "5" ]; then
+    ok "IL01 ascending literals 1..5 encode as 5 distinct tokens (collision broken)"
+else
+    fail "IL01 literals 1..5 not distinct" "distinct_nonzero=$(g DISTINCT_NONZERO), want 5"
+fi
+
+# genuine zeros run stays ONE repeated token -- real repetition is not destroyed.
+if [ "$(g ZERO_COUNT)" = "5" ]; then
+    ok "IL02 a genuine zeros run stays one repeated token (real repetition kept)"
+else
+    fail "IL02 zeros run wrong" "zero_count=$(g ZERO_COUNT), want 5"
+fi
+
+# 100 (>= int_count) and 3.5 (non-integer) must NOT occupy int tokens: exactly
+# 6 distinct int tokens total (0..5), nothing more.
+cat > "$WORK/range.eigs" <<'RNG'
+a is args of null
+v is json_decode of (read_text of (a[0] + "/v.json"))
+stream is tensor_load of (a[0] + "/s.bin")
+fi is v["first_int_id"]
+nint is v["int_count"]
+seen is zeros of 64
+for i in range of (len of stream):
+    t is stream[i]
+    if t >= fi:
+        if t < (fi + nint):
+            seen[t - fi] is 1
+tot is 0
+for i in range of nint:
+    tot is tot + seen[i]
+print of ("DISTINCT_TOTAL " + (str of tot))
+RNG
+ROUT=$("$EIGS" "$WORK/range.eigs" "$WORK" 2>/dev/null | sed -n 's/^DISTINCT_TOTAL //p')
+if [ "$ROUT" = "6" ]; then
+    ok "IL03 out-of-range (100) and non-integer (3.5) literals stay lossy, not int tokens"
+else
+    fail "IL03 wrong distinct int total" "distinct=$ROUT, want 6 (values 0..5 only)"
+fi
+
+# Backward compat: no 7th arg -> no int region, stream byte-identical to slot-only.
+cat > "$WORK/noint.eigs" <<'NOI'
+a is args of null
+files is [a[0] + "/sample.eigs"]
+r is build_corpus of [files, 4, a[0] + "/sn.bin", a[0] + "/vn.json", a[0] + "/in.json", 8]
+v is json_decode of (read_text of (a[0] + "/vn.json"))
+print of ("INTCOUNT " + (str of v["int_count"]))
+NOI
+NOUT=$("$EIGS" "$WORK/noint.eigs" "$WORK" 2>/dev/null | sed -n 's/^INTCOUNT //p')
+if [ "$NOUT" = "0" ]; then
+    ok "IL04 no 7th arg -> int_count 0 (opt-in, stream unchanged)"
+else
+    fail "IL04 int mode leaked into the no-arg call" "int_count=$NOUT"
+fi
+
+echo "CORPUSINTS: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]