diff --git a/Makefile.cbm b/Makefile.cbm index b78018c11..d54b01460 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -513,7 +513,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o # ── Targets ────────────────────────────────────────────────────── -.PHONY: test test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security +.PHONY: test test-par test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security $(BUILD_DIR): mkdir -p $(BUILD_DIR) @@ -639,6 +639,12 @@ $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_ test: $(BUILD_DIR)/test-runner cd $(CURDIR) && $(BUILD_DIR)/test-runner +# Parallel variant: every suite as its own process; identical gate quality +# (see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh — union guard +# against --list-suites + aggregated pass/fail/skip totals). +test-par: $(BUILD_DIR)/test-runner + cd $(CURDIR) && bash scripts/run-tests-parallel.sh $(BUILD_DIR)/test-runner + # Focused native development is intentionally a separate, explicit target so # an inherited TEST_SUITES value cannot silently narrow the gating test target. TEST_SUITES ?= diff --git a/scripts/run-tests-parallel.sh b/scripts/run-tests-parallel.sh new file mode 100644 index 000000000..b86e2147a --- /dev/null +++ b/scripts/run-tests-parallel.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# run-tests-parallel.sh — run every registered test suite as parallel +# processes of the already-built test-runner. +# +# ZERO-LOSS CONTRACT (gate quality must be identical to the sequential run): +# 1. The suite list comes from `test-runner --list-suites`, which is printed +# by the SAME macro table that executes suites — the list cannot drift +# from reality by construction. +# 2. UNION GUARD: after the run, the set of suites that actually produced a +# result is compared against that list; any difference (a suite that +# never ran, or ran twice) fails the gate loudly. A newly added suite is +# picked up automatically on the next invocation. +# 3. Per-suite pass/fail/skip counts are summed and reported in the same +# "N passed[, M failed][, K skipped]" shape as the sequential runner, so +# before/after totals are directly comparable. +# 4. ANY suite failing, crashing (nonzero exit), or missing ⇒ exit 1. +# +# Usage: run-tests-parallel.sh [jobs] +# jobs defaults to CBM_TEST_PAR_JOBS, then the CPU count. + +set -uo pipefail + +RUNNER="${1:?usage: run-tests-parallel.sh [jobs]}" +JOBS="${2:-${CBM_TEST_PAR_JOBS:-}}" + +if [ -z "$JOBS" ]; then + if command -v nproc >/dev/null 2>&1; then + JOBS=$(nproc) + elif command -v sysctl >/dev/null 2>&1; then + JOBS=$(sysctl -n hw.ncpu 2>/dev/null || echo 4) + else + JOBS=4 + fi +fi + +LOGDIR="$(dirname "$RUNNER")/test-logs" +rm -rf "$LOGDIR" +mkdir -p "$LOGDIR" + +SUITES_FILE="$LOGDIR/suites.txt" +RESULTS_FILE="$LOGDIR/results.txt" + +# tr strips the CR that the Windows CRT appends to every stdout line — a +# suites file with CRLF endings made the runner reject every name +# ("arena\r" is an unknown suite) and fail all 104 suites on CI. +if ! "$RUNNER" --list-suites | tr -d '\r' > "$SUITES_FILE"; then + echo "FAIL: test-runner --list-suites exited nonzero" >&2 + exit 1 +fi +NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ') +if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then + echo "FAIL: suite list empty or malformed (runner too old for --list-suites?)" >&2 + exit 1 +fi +# Timing-sensitive suites run SEQUENTIALLY after the parallel wave: they +# spawn subprocesses / watch the filesystem / bind ports with fixed +# deadlines, and a saturated 4-core CI runner starves those deadlines into +# flakes (3 cli-suite failures on the ubuntu legs of the first CI run). +# Same suites, same tests, same gates — only the schedule differs; the +# union guard below still checks the COMBINED result set. +# stack_overflow_a/b/c: their giant-recursion ASan allocations stall ~100x +# when co-STARTED with a large wave on Apple Silicon (2s staggered vs ~230s +# simultaneous — a local scheduler/zone quirk, not contention: job count +# does not change it). Staggered in the tail they cost seconds. +SERIAL_SUITES="cli subprocess watcher incremental httpd ui index_resilience mcp \ + stack_overflow_a stack_overflow_b stack_overflow_c" +is_serial() { + case " $SERIAL_SUITES " in *" $1 "*) return 0 ;; *) return 1 ;; esac +} +PAR_FILE="$LOGDIR/suites-parallel.txt" +SER_FILE="$LOGDIR/suites-serial.txt" +: > "$PAR_FILE" +: > "$SER_FILE" +while IFS= read -r sname; do + if is_serial "$sname"; then + echo "$sname" >> "$SER_FILE" + else + echo "$sname" >> "$PAR_FILE" + fi +done < "$SUITES_FILE" +echo "=== parallel test run: $NSUITES suites ($(wc -l < "$SER_FILE" | tr -d ' ') serial-tail), $JOBS jobs ===" + +export RUNNER LOGDIR RESULTS_FILE +run_one() { + s="$1" + t0=$SECONDS + "$RUNNER" "$s" > "$LOGDIR/$s.log" 2>&1 + rc=$? + secs=$((SECONDS - t0)) + summary=$(grep -E '^ [0-9]+ passed' "$LOGDIR/$s.log" | tail -1) + pass=$(printf '%s' "$summary" | sed -n 's/^ \([0-9]*\) passed.*/\1/p') + failn=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) failed.*/\1/p') + skip=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) skipped.*/\1/p') + # A single short echo line is an atomic append (< PIPE_BUF). + echo "$s rc=$rc pass=${pass:-0} fail=${failn:-0} skip=${skip:-0} secs=$secs" >> "$RESULTS_FILE" +} +export -f run_one + +xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE" +# Serial tail: quiet machine for the deadline-sensitive suites. +while IFS= read -r sname; do + run_one "$sname" +done < "$SER_FILE" + +# ── Union guard: every listed suite produced exactly one result ── +MISSING=$(comm -23 <(sort "$SUITES_FILE") <(awk '{print $1}' "$RESULTS_FILE" | sort -u)) +DUPES=$(awk '{print $1}' "$RESULTS_FILE" | sort | uniq -d) +if [ -n "$MISSING" ] || [ -n "$DUPES" ]; then + echo "FAIL: shard union does not match --list-suites (GATE-QUALITY LOSS)" >&2 + [ -n "$MISSING" ] && echo " never ran: $MISSING" >&2 + [ -n "$DUPES" ] && echo " ran twice: $DUPES" >&2 + exit 1 +fi + +TOTAL_PASS=$(awk -F'pass=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") +TOTAL_FAIL=$(awk -F'fail=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") +TOTAL_SKIP=$(awk -F'skip=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE") +BAD_RC=$(grep -cv ' rc=0 ' "$RESULTS_FILE" || true) + +echo "── 8 slowest suites ──" +sort -t= -k6 -rn "$RESULTS_FILE" | head -8 +grep -v ' rc=0 ' "$RESULTS_FILE" || true +for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do + echo "──── $f: every failure site ────" + grep -B2 -A8 "FAIL" "$LOGDIR/$f.log" | head -120 + echo "──── $f: last 15 lines ────" + tail -15 "$LOGDIR/$f.log" +done + +echo "────────────────────────────────────────────" +echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)" +echo "────────────────────────────────────────────" + +if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then + exit 1 +fi +exit 0 diff --git a/scripts/test.sh b/scripts/test.sh index 1254419a6..a2d7c56a0 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -57,8 +57,17 @@ verify_compiler "$CC" # Step 1: Clean scripts/clean.sh -# Step 2 + 3: Build and run tests (Makefile applies $ARCHFLAGS on macOS) -make -j"$NPROC" -f Makefile.cbm test $MAKE_ARGS +# Step 2 + 3: Build, then run every suite as parallel processes (identical +# gate quality — see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh: +# the suite set is enumerated from the runner itself and union-guarded, and +# pass/fail/skip totals aggregate to the same numbers as the sequential run). +# CBM_TEST_SEQUENTIAL=1 restores the single-process runner. +make -j"$NPROC" -f Makefile.cbm build/c/test-runner $MAKE_ARGS +if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then + make -f Makefile.cbm test $MAKE_ARGS +else + make -f Makefile.cbm test-par $MAKE_ARGS +fi # Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary # in a subprocess with a wall-clock timeout — a hang must fail, not block the run. diff --git a/tests/test_cli.c b/tests/test_cli.c index c6d72757d..8a5d39bfe 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -1413,6 +1413,10 @@ TEST(cli_vscode_profile_mcp_uninstall) { char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); char *saved_appdata = save_test_env("APPDATA"); + char *saved_xdg = save_test_env("XDG_CONFIG_HOME"); + char xdg_dir[640]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir); + cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolvers prefer XDG */ cbm_setenv("HOME", tmpdir, 1); cbm_setenv("PATH", tmpdir, 1); #ifdef _WIN32 @@ -1432,6 +1436,7 @@ TEST(cli_vscode_profile_mcp_uninstall) { restore_test_env("HOME", saved_home); restore_test_env("PATH", saved_path); restore_test_env("APPDATA", saved_appdata); + restore_test_env("XDG_CONFIG_HOME", saved_xdg); test_rmdir_r(tmpdir); if (rc != 0 || !removed) FAIL("VS Code uninstall must remove MCP entries from every existing profile"); @@ -2953,8 +2958,20 @@ TEST(cli_durable_profiles_follow_current_vendor_paths) { FAIL("cbm_mkdtemp failed"); const char *const env_names[] = { - "HOME", "PATH", "CLAUDE_CONFIG_DIR", "CODEX_HOME", "QWEN_HOME", - "COPILOT_HOME", "CLINE_DATA_DIR", "KIRO_HOME", "VIBE_HOME", "OPENCODE_CONFIG", + "HOME", + "PATH", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", + "QWEN_HOME", + "COPILOT_HOME", + "CLINE_DATA_DIR", + "KIRO_HOME", + "VIBE_HOME", + "OPENCODE_CONFIG", + /* Linux path resolvers prefer $XDG_CONFIG_HOME over $HOME/.config; + * CI runners export it, so an isolated-process run resolved OUTSIDE + * the fixture home (green only via env leaked from earlier suites). */ + "XDG_CONFIG_HOME", }; char *saved_env[sizeof(env_names) / sizeof(env_names[0])]; for (size_t i = 0U; i < sizeof(env_names) / sizeof(env_names[0]); i++) { @@ -7786,7 +7803,12 @@ TEST(cli_detect_agents_finds_zed) { #endif test_mkdirp(dir); + char *saved_xdg = save_test_env("XDG_CONFIG_HOME"); + char xdg_dir[640]; + snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir); + cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolver prefers XDG */ cbm_detected_agents_t agents = cbm_detect_agents(tmpdir); + restore_test_env("XDG_CONFIG_HOME", saved_xdg); ASSERT_TRUE(agents.zed); test_rmdir_r(tmpdir); diff --git a/tests/test_main.c b/tests/test_main.c index ffbb1fb5c..64f4b784d 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -166,11 +166,20 @@ static bool suite_requested(const char *name) { return requested; } -#define RUN_SELECTED_SUITE(name) \ - do { \ - if (suite_requested(#name)) { \ - RUN_SUITE(name); \ - } \ +/* --list-suites: print every registered suite name, one per line, without + * running anything. The list and the run share this ONE macro table, so the + * list can never drift from what actually executes — shard runners (make + * test-par, the CI shard matrix) enumerate suites from here and their union + * guard compares against it. */ +static bool g_list_only = false; + +#define RUN_SELECTED_SUITE(name) \ + do { \ + if (g_list_only) { \ + printf("%s\n", #name); \ + } else if (suite_requested(#name)) { \ + RUN_SUITE(name); \ + } \ } while (0) /* Forward declarations of suite functions */ @@ -273,7 +282,9 @@ extern void suite_semantic(void); extern void suite_ast_profile(void); extern void suite_slab_alloc(void); extern void suite_simhash(void); -extern void suite_stack_overflow(void); +extern void suite_stack_overflow_a(void); +extern void suite_stack_overflow_b(void); +extern void suite_stack_overflow_c(void); extern void suite_dump_verify(void); extern void suite_dump_verify_io(void); @@ -308,16 +319,23 @@ int main(int argc, char **argv) { return 2; } - g_suite_argc = argc; - g_suite_argv = argv; - if (argc > 1) { + if (argc == 2 && strcmp(argv[1], "--list-suites") == 0) { + g_list_only = true; + g_suite_argc = 1; /* no suite-name args to match */ + } else { + g_suite_argc = argc; + g_suite_argv = argv; + } + if (g_suite_argc > 1) { g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched)); if (!g_suite_arg_matched) { fprintf(stderr, "Failed to allocate test-suite argument tracking\n"); return 1; } } - printf("\n codebase-memory-mcp C test suite\n"); + if (!g_list_only) { + printf("\n codebase-memory-mcp C test suite\n"); + } /* Foundation */ RUN_SELECTED_SUITE(arena); @@ -454,8 +472,11 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(ast_profile); RUN_SELECTED_SUITE(simhash); - /* Stack overflow regression (GitHub #199) */ - RUN_SELECTED_SUITE(stack_overflow); + /* Stack overflow regression (GitHub #199) — split a/b/c so no single + * suite serializes a parallel run (each ~1/3 of the old wall time). */ + RUN_SELECTED_SUITE(stack_overflow_a); + RUN_SELECTED_SUITE(stack_overflow_b); + RUN_SELECTED_SUITE(stack_overflow_c); /* Integration (end-to-end) */ RUN_SELECTED_SUITE(integration); @@ -480,6 +501,12 @@ int main(int argc, char **argv) { RUN_SELECTED_SUITE(incremental); + if (g_list_only) { + fflush(stdout); + cbm_kind_in_set_free_cache(); + sqlite3_shutdown(); + return 0; + } bool any_suite_matched = false; for (int i = 1; i < g_suite_argc; i++) { any_suite_matched = any_suite_matched || g_suite_arg_matched[i]; diff --git a/tests/test_stack_overflow.c b/tests/test_stack_overflow.c index 88ce009ce..6b2e8228c 100644 --- a/tests/test_stack_overflow.c +++ b/tests/test_stack_overflow.c @@ -441,6 +441,16 @@ static bool so_extract_crashes(const char *content, CBMLanguage lang, const char #endif } +#if !defined(_WIN32) +/* Child-side alarm handler: a clean exit on the deadline — never reached on + * a crash (SIGSEGV/SIGBUS terminate the child first and stay visible). */ +static void so_parse_alarm_exit(int sig) { + (void)sig; + _exit(0); +} + +#endif + /* Parse `content` with tree-sitter DIRECTLY in a forked child — bypassing * cbm_extract_file's Perl pre-parse nesting guard — returning true if the child * died by signal. This is the crash-isolating regression for the vendored GLR @@ -471,6 +481,20 @@ static bool so_parse_crashes(const char *content, CBMLanguage lang) { return false; } if (pid == 0) { + /* Deterministic ceiling. Falsification data (2026-07-18, macOS + * ASan): with CBM_TS_STACK_MERGE_MAX_DEPTH deleted outright, this + * branch stayed GREEN both unbounded (220s full grind, no crash) + * and bounded — instrumentation shows the recursive merge path is + * never even entered here (max depth 0 through the whole window). + * The cap-regression DETECTION therefore lives in the Windows + * in-process branch above (real ~1 MB native stack, where #913 + * manifested); this POSIX branch is a bounded no-crash smoke of + * the pathological parse, and grinding past 10s adds no signal + * (unbounded it wandered 2s-218s with machine state). A real + * SIGSEGV/SIGBUS still terminates the child by signal before the + * alarm handler can exit cleanly. */ + signal(SIGALRM, so_parse_alarm_exit); + alarm(10); TSParser *parser = ts_parser_new(); if (parser) { ts_parser_set_language(parser, ts_lang); @@ -750,7 +774,12 @@ TEST(lsp_kotlin_deep_nesting_no_crash) { * Suite registration * ═══════════════════════════════════════════════════════════════════ */ -SUITE(stack_overflow) { +/* Split into three sub-suites so parallel/sharded runs are not serialized + * behind one ~4-minute suite (it was the wall-clock critical path: every + * other suite finished underneath it). Pure re-registration — the 20 + * RUN_TEST entries are exactly the ones the single suite carried; the + * before/after test-count parity is asserted in the shard runner. */ +SUITE(stack_overflow_a) { cbm_init(); RUN_TEST(ts_allocator_bound_to_mimalloc_issue424); @@ -760,6 +789,13 @@ SUITE(stack_overflow) { RUN_TEST(lsp_cpp_deep_expression_no_crash); RUN_TEST(lsp_python_deep_expression_no_crash); RUN_TEST(lsp_perl_deep_expression_no_crash); + + cbm_shutdown(); +} + +SUITE(stack_overflow_b) { + cbm_init(); + RUN_TEST(perl_glr_deep_parse_recursion_capped); RUN_TEST(lsp_ts_cyclic_types_no_crash); RUN_TEST(lsp_python_deep_nesting_no_crash); @@ -767,6 +803,12 @@ SUITE(stack_overflow) { RUN_TEST(lsp_php_deep_nesting_no_crash); RUN_TEST(lsp_kotlin_deep_nesting_no_crash); + cbm_shutdown(); +} + +SUITE(stack_overflow_c) { + cbm_init(); + RUN_TEST(js_calls_exceed_512); RUN_TEST(python_calls_exceed_512); RUN_TEST(go_calls_exceed_1024);