diff --git a/src/builtins.c b/src/builtins.c index 26e53a5..9082181 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -2334,6 +2334,32 @@ Value* builtin_build_corpus(Value *arg) { if (sv && sv->type == VAL_NUM) slot_count = (int)sv->data.num; } if (slot_count < 0) slot_count = 0; + + /* ---- INTEGER LITERALS (optional 7th arg: int_count > 0) ------------- + * Every numeric literal tokenizes to a single TOK_NUM, so the value is + * ERASED from the stream: `[1,2,3]` and `[0,0,0]` become the identical + * `,,`. Measured on the 2026-07 corpus this is doubly + * costly. (1) A spec written as assertions is uninformative -- `(f of 4) + * == 1` and `(f of 3) == 0` reach the model as one sequence, so it cannot + * learn what the test demands. (2) It MANUFACTURES the degenerate + * repetition the model collapses into: measured, essentially all of the + * corpus's `0,0,0,0` / `"s","s"` cycles (5.3% of tokens) route through the + * collapsed literal tokens, and integers 0..31 are 76% of all numeric + * literals. Giving small integers exact tokens both makes tests readable + * and breaks that false repetition -- `[1,2,3,4]` stops looking like a + * cycle -- without discarding any real code a repetition filter would cut. + * + * int_count == N gives exact tokens to the integers [0, N). N=32 covers + * 76% of numeric-literal occurrences for +32 tokens; larger N has a long + * tail (0..255 is 86%). Non-integer, negative, and >=N literals keep the + * lossy TOK_NUM fallback -- lossless is not the goal here, breaking the + * high-frequency collision is. Opt-in: 0 leaves the stream byte-identical. */ + int int_count = 0; + if (arg->data.list.count >= 7) { + Value *iv = arg->data.list.items[6]; + if (iv && iv->type == VAL_NUM) int_count = (int)iv->data.num; + } + if (int_count < 0) int_count = 0; /* Source of truth: the size of the TokType enum. Hardcoding 54 (which is * what this was before) silently aliases TOK_LBRACKET (=54) onto the most * common extended ident slot whenever the enum grows. */ @@ -2537,6 +2563,14 @@ Value* builtin_build_corpus(Value *arg) { slot_used[hit] = ++slot_clock; tid = FIRST_IDENT + actual_top + hit; } + } else if (int_count > 0 && tid == TOK_NUM) { + /* Give small non-negative integer literals exact tokens; the + * region sits directly after the slots. floor==value rejects + * 3.5 etc.; the lexer emits '-' as its own token so num_val is + * never negative for a literal, but guard anyway. */ + double nv = tl.tokens[i].num_val; + if (nv >= 0.0 && nv < (double)int_count && nv == (double)(long)nv) + tid = FIRST_IDENT + actual_top + slot_count + (int)nv; } double d = (double)tid; fwrite(&d, sizeof(double), 1, stream_file); @@ -2562,9 +2596,10 @@ Value* builtin_build_corpus(Value *arg) { FILE *vocab_file = xfopen_write(vocab_path_val->data.str, "w"); if (vocab_file) { fprintf(vocab_file, - "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"top_n\": %d", - FIRST_IDENT, BASE_VOCAB + actual_top + slot_count, BASE_VOCAB, - slot_count, FIRST_IDENT + actual_top, actual_top); + "{\"first_ident_id\": %d, \"ext_vocab_size\": %d, \"base_vocab\": %d, \"slot_count\": %d, \"first_slot_id\": %d, \"int_count\": %d, \"first_int_id\": %d, \"top_n\": %d", + FIRST_IDENT, BASE_VOCAB + actual_top + slot_count + int_count, BASE_VOCAB, + slot_count, FIRST_IDENT + actual_top, + int_count, FIRST_IDENT + actual_top + slot_count, actual_top); fprintf(vocab_file, ", \"structural_ids\": {\"newline\": %d, \"indent\": %d, \"dedent\": %d, \"eof\": %d, \"ident_fallback\": %d}", (int)TOK_NEWLINE, (int)TOK_INDENT, (int)TOK_DEDENT, (int)TOK_EOF, (int)TOK_IDENT); fprintf(vocab_file, ", \"base_names\": ["); diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index 9d54df5..10e80d8 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -1726,6 +1726,21 @@ else fi echo "" +echo "[43a2c] build_corpus integer-literal encoding (5 checks)" +CI_OUTPUT=$(bash "$TESTS_DIR/test_corpus_ints.sh" 2>&1) +CI_PASS=$(echo "$CI_OUTPUT" | grep -c "PASS:" || true) +CI_FAIL=$(echo "$CI_OUTPUT" | grep -c "FAIL:" || true) +TOTAL=$((TOTAL + CI_PASS + CI_FAIL)) +PASS=$((PASS + CI_PASS)) +FAIL=$((FAIL + CI_FAIL)) +if [ "$CI_FAIL" -gt 0 ]; then + echo " FAIL: $CI_FAIL integer-encoding check(s) failed" + echo "$CI_OUTPUT" | grep "FAIL:" | head -4 +else + echo " PASS: integer literals get exact tokens; genuine repetition preserved" +fi +echo "" + # [43a3] EigenStore header-validation / corruption error paths (ext_store.c) echo "[43a3] Store Corruption Errors (12 checks)" check_eigs_suite "store corruption errors" test_store_corruption.eigs "All store_corruption tests passed" 12 diff --git a/tests/test_corpus_ints.sh b/tests/test_corpus_ints.sh new file mode 100755 index 0000000..9fff887 --- /dev/null +++ b/tests/test_corpus_ints.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# build_corpus integer-literal encoding: give small integers exact tokens. +# +# Every numeric literal tokenizes to a single TOK_NUM, erasing its value: the +# stream cannot tell `[1,2,3]` from `[0,0,0]`. Measured on the 2026-07 corpus +# that erasure is doubly costly -- an assertion-as-spec carries no information +# (`(f of 4)==1` and `(f of 3)==0` become one sequence), AND it manufactures the +# degenerate `0,0,0` repetition the model collapses into. The optional 7th arg +# int_count=N gives exact tokens to integers [0, N); everything else keeps the +# lossy TOK_NUM fallback. +# +# The assertions pin the properties that make the fix do its job. + +set -u +TESTS_DIR="$(cd "$(dirname "$0")" && pwd)" +EIGS="$TESTS_DIR/../src/eigenscript" + +PASS=0 +FAIL=0 +ok() { echo " PASS: $1"; PASS=$((PASS+1)); } +fail() { echo " FAIL: $1${2:+ ($2)}"; FAIL=$((FAIL+1)); } + +WORK=$(mktemp -d /tmp/eigs_ints_XXXXXX) +cleanup() { rm -rf "$WORK"; } +trap cleanup EXIT + +# Distinct ascending ints, a genuine zeros run, an out-of-range int, and a float. +cat > "$WORK/sample.eigs" <<'SRC' +a is [1, 2, 3, 4, 5] +b is [0, 0, 0, 0, 0] +c is 100 +d is 3.5 +SRC + +cat > "$WORK/build.eigs" <<'DRV' +a is args of null +files is [a[0] + "/sample.eigs"] +r is build_corpus of [files, 4, a[0] + "/s.bin", a[0] + "/v.json", a[0] + "/i.json", 8, 32] +print of ("TOKENS " + (str of r[0])) +DRV + +if ! "$EIGS" "$WORK/build.eigs" "$WORK" >"$WORK/out.log" 2>&1; then + fail "IL00 int-mode build_corpus ran" "see $WORK/out.log" + echo "CORPUSINTS: 0 passed, 1 failed" + exit 1 +fi +ok "IL00 int-mode build_corpus ran" + +# Decode against the vocab and check the four properties. +cat > "$WORK/check.eigs" <<'CHK' +a is args of null +v is json_decode of (read_text of (a[0] + "/v.json")) +stream is tensor_load of (a[0] + "/s.bin") +fi is v["first_int_id"] +nint is v["int_count"] +toknum is v["structural_ids"]["eof"] # placeholder; real TOK_NUM found below + +# 1. the five ascending literals 1..5 must be five DISTINCT tokens in the int +# range -- this is the collision break. Count distinct int-range tokens that +# appear exactly once and are NOT the zero token. +counts is zeros of 64 +for i in range of (len of stream): + t is stream[i] + if t >= fi: + if t < (fi + nint): + counts[t - fi] is counts[t - fi] + 1 +# distinct nonzero int values present: +distinct is 0 +for i in range of nint: + if i > 0: + if counts[i] >= 1: + distinct is distinct + 1 +print of ("DISTINCT_NONZERO " + (str of distinct)) + +# 2. the zeros run: INT0 must appear 5x (genuine repetition preserved as ONE token) +print of ("ZERO_COUNT " + (str of counts[0])) + +# 3. no int token may exceed the declared range +bad is 0 +for i in range of (len of stream): + t is stream[i] + if t >= (fi + nint): + if t < fi: + bad is bad + 1 +print of ("MAXINT " + (str of (fi + nint - 1))) +CHK + +OUT=$("$EIGS" "$WORK/check.eigs" "$WORK" 2>/dev/null) +g() { printf '%s\n' "$OUT" | sed -n "s/^$1 //p"; } + +# 1..5 are five distinct nonzero int tokens: the collision `[1,2,3]`==`[0,0,0]` is broken. +if [ "$(g DISTINCT_NONZERO)" = "5" ]; then + ok "IL01 ascending literals 1..5 encode as 5 distinct tokens (collision broken)" +else + fail "IL01 literals 1..5 not distinct" "distinct_nonzero=$(g DISTINCT_NONZERO), want 5" +fi + +# genuine zeros run stays ONE repeated token -- real repetition is not destroyed. +if [ "$(g ZERO_COUNT)" = "5" ]; then + ok "IL02 a genuine zeros run stays one repeated token (real repetition kept)" +else + fail "IL02 zeros run wrong" "zero_count=$(g ZERO_COUNT), want 5" +fi + +# 100 (>= int_count) and 3.5 (non-integer) must NOT occupy int tokens: exactly +# 6 distinct int tokens total (0..5), nothing more. +cat > "$WORK/range.eigs" <<'RNG' +a is args of null +v is json_decode of (read_text of (a[0] + "/v.json")) +stream is tensor_load of (a[0] + "/s.bin") +fi is v["first_int_id"] +nint is v["int_count"] +seen is zeros of 64 +for i in range of (len of stream): + t is stream[i] + if t >= fi: + if t < (fi + nint): + seen[t - fi] is 1 +tot is 0 +for i in range of nint: + tot is tot + seen[i] +print of ("DISTINCT_TOTAL " + (str of tot)) +RNG +ROUT=$("$EIGS" "$WORK/range.eigs" "$WORK" 2>/dev/null | sed -n 's/^DISTINCT_TOTAL //p') +if [ "$ROUT" = "6" ]; then + ok "IL03 out-of-range (100) and non-integer (3.5) literals stay lossy, not int tokens" +else + fail "IL03 wrong distinct int total" "distinct=$ROUT, want 6 (values 0..5 only)" +fi + +# Backward compat: no 7th arg -> no int region, stream byte-identical to slot-only. +cat > "$WORK/noint.eigs" <<'NOI' +a is args of null +files is [a[0] + "/sample.eigs"] +r is build_corpus of [files, 4, a[0] + "/sn.bin", a[0] + "/vn.json", a[0] + "/in.json", 8] +v is json_decode of (read_text of (a[0] + "/vn.json")) +print of ("INTCOUNT " + (str of v["int_count"])) +NOI +NOUT=$("$EIGS" "$WORK/noint.eigs" "$WORK" 2>/dev/null | sed -n 's/^INTCOUNT //p') +if [ "$NOUT" = "0" ]; then + ok "IL04 no 7th arg -> int_count 0 (opt-in, stream unchanged)" +else + fail "IL04 int mode leaked into the no-arg call" "int_count=$NOUT" +fi + +echo "CORPUSINTS: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ]