Skip to content
8 changes: 7 additions & 1 deletion Makefile.cbm
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@ PP_OBJ_TSAN = $(BUILD_DIR)/tsan_preprocessor.o

# ── Targets ──────────────────────────────────────────────────────

.PHONY: test test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security
.PHONY: test test-par test-repro test-foundation test-tsan cbm cbm-with-ui frontend embed clean-c lint lint-tidy lint-cppcheck lint-format security

$(BUILD_DIR):
mkdir -p $(BUILD_DIR)
Expand Down Expand Up @@ -639,6 +639,12 @@ $(BUILD_DIR)/test-runner: $(ALL_TEST_SRCS) $(PROD_SRCS) $(EXTRACTION_SRCS) $(AC_
test: $(BUILD_DIR)/test-runner
cd $(CURDIR) && $(BUILD_DIR)/test-runner

# Parallel variant: every suite as its own process; identical gate quality
# (see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh — union guard
# against --list-suites + aggregated pass/fail/skip totals).
test-par: $(BUILD_DIR)/test-runner
cd $(CURDIR) && bash scripts/run-tests-parallel.sh $(BUILD_DIR)/test-runner

# Focused native development is intentionally a separate, explicit target so
# an inherited TEST_SUITES value cannot silently narrow the gating test target.
TEST_SUITES ?=
Expand Down
137 changes: 137 additions & 0 deletions scripts/run-tests-parallel.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
# run-tests-parallel.sh — run every registered test suite as parallel
# processes of the already-built test-runner.
#
# ZERO-LOSS CONTRACT (gate quality must be identical to the sequential run):
# 1. The suite list comes from `test-runner --list-suites`, which is printed
# by the SAME macro table that executes suites — the list cannot drift
# from reality by construction.
# 2. UNION GUARD: after the run, the set of suites that actually produced a
# result is compared against that list; any difference (a suite that
# never ran, or ran twice) fails the gate loudly. A newly added suite is
# picked up automatically on the next invocation.
# 3. Per-suite pass/fail/skip counts are summed and reported in the same
# "N passed[, M failed][, K skipped]" shape as the sequential runner, so
# before/after totals are directly comparable.
# 4. ANY suite failing, crashing (nonzero exit), or missing ⇒ exit 1.
#
# Usage: run-tests-parallel.sh <path-to-test-runner> [jobs]
# jobs defaults to CBM_TEST_PAR_JOBS, then the CPU count.

set -uo pipefail

RUNNER="${1:?usage: run-tests-parallel.sh <path-to-test-runner> [jobs]}"
JOBS="${2:-${CBM_TEST_PAR_JOBS:-}}"

if [ -z "$JOBS" ]; then
if command -v nproc >/dev/null 2>&1; then
JOBS=$(nproc)
elif command -v sysctl >/dev/null 2>&1; then
JOBS=$(sysctl -n hw.ncpu 2>/dev/null || echo 4)
else
JOBS=4
fi
fi

LOGDIR="$(dirname "$RUNNER")/test-logs"
rm -rf "$LOGDIR"
mkdir -p "$LOGDIR"

SUITES_FILE="$LOGDIR/suites.txt"
RESULTS_FILE="$LOGDIR/results.txt"

# tr strips the CR that the Windows CRT appends to every stdout line — a
# suites file with CRLF endings made the runner reject every name
# ("arena\r" is an unknown suite) and fail all 104 suites on CI.
if ! "$RUNNER" --list-suites | tr -d '\r' > "$SUITES_FILE"; then
echo "FAIL: test-runner --list-suites exited nonzero" >&2
exit 1
fi
NSUITES=$(wc -l < "$SUITES_FILE" | tr -d ' ')
if [ "$NSUITES" -lt 1 ] || grep -qvE '^[a-z0-9_]+$' "$SUITES_FILE"; then
echo "FAIL: suite list empty or malformed (runner too old for --list-suites?)" >&2
exit 1
fi
# Timing-sensitive suites run SEQUENTIALLY after the parallel wave: they
# spawn subprocesses / watch the filesystem / bind ports with fixed
# deadlines, and a saturated 4-core CI runner starves those deadlines into
# flakes (3 cli-suite failures on the ubuntu legs of the first CI run).
# Same suites, same tests, same gates — only the schedule differs; the
# union guard below still checks the COMBINED result set.
# stack_overflow_a/b/c: their giant-recursion ASan allocations stall ~100x
# when co-STARTED with a large wave on Apple Silicon (2s staggered vs ~230s
# simultaneous — a local scheduler/zone quirk, not contention: job count
# does not change it). Staggered in the tail they cost seconds.
SERIAL_SUITES="cli subprocess watcher incremental httpd ui index_resilience mcp \
stack_overflow_a stack_overflow_b stack_overflow_c"
is_serial() {
case " $SERIAL_SUITES " in *" $1 "*) return 0 ;; *) return 1 ;; esac
}
PAR_FILE="$LOGDIR/suites-parallel.txt"
SER_FILE="$LOGDIR/suites-serial.txt"
: > "$PAR_FILE"
: > "$SER_FILE"
while IFS= read -r sname; do
if is_serial "$sname"; then
echo "$sname" >> "$SER_FILE"
else
echo "$sname" >> "$PAR_FILE"
fi
done < "$SUITES_FILE"
echo "=== parallel test run: $NSUITES suites ($(wc -l < "$SER_FILE" | tr -d ' ') serial-tail), $JOBS jobs ==="

export RUNNER LOGDIR RESULTS_FILE
run_one() {
s="$1"
t0=$SECONDS
"$RUNNER" "$s" > "$LOGDIR/$s.log" 2>&1
rc=$?
secs=$((SECONDS - t0))
summary=$(grep -E '^ [0-9]+ passed' "$LOGDIR/$s.log" | tail -1)
pass=$(printf '%s' "$summary" | sed -n 's/^ \([0-9]*\) passed.*/\1/p')
failn=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) failed.*/\1/p')
skip=$(printf '%s' "$summary" | sed -n 's/.* \([0-9]*\) skipped.*/\1/p')
# A single short echo line is an atomic append (< PIPE_BUF).
echo "$s rc=$rc pass=${pass:-0} fail=${failn:-0} skip=${skip:-0} secs=$secs" >> "$RESULTS_FILE"
}
export -f run_one

xargs -P "$JOBS" -I{} bash -c 'run_one "$@"' _ {} < "$PAR_FILE"
# Serial tail: quiet machine for the deadline-sensitive suites.
while IFS= read -r sname; do
run_one "$sname"
done < "$SER_FILE"

# ── Union guard: every listed suite produced exactly one result ──
MISSING=$(comm -23 <(sort "$SUITES_FILE") <(awk '{print $1}' "$RESULTS_FILE" | sort -u))
DUPES=$(awk '{print $1}' "$RESULTS_FILE" | sort | uniq -d)
if [ -n "$MISSING" ] || [ -n "$DUPES" ]; then
echo "FAIL: shard union does not match --list-suites (GATE-QUALITY LOSS)" >&2
[ -n "$MISSING" ] && echo " never ran: $MISSING" >&2
[ -n "$DUPES" ] && echo " ran twice: $DUPES" >&2
exit 1
fi

TOTAL_PASS=$(awk -F'pass=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
TOTAL_FAIL=$(awk -F'fail=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
TOTAL_SKIP=$(awk -F'skip=' '{split($2,a," "); s+=a[1]} END{print s+0}' "$RESULTS_FILE")
BAD_RC=$(grep -cv ' rc=0 ' "$RESULTS_FILE" || true)

echo "── 8 slowest suites ──"
sort -t= -k6 -rn "$RESULTS_FILE" | head -8
grep -v ' rc=0 ' "$RESULTS_FILE" || true
for f in $(grep -v ' rc=0 ' "$RESULTS_FILE" | awk '{print $1}'); do
echo "──── $f: every failure site ────"
grep -B2 -A8 "FAIL" "$LOGDIR/$f.log" | head -120
echo "──── $f: last 15 lines ────"
tail -15 "$LOGDIR/$f.log"
done

echo "────────────────────────────────────────────"
echo " $TOTAL_PASS passed, $TOTAL_FAIL failed, $TOTAL_SKIP skipped ($NSUITES suites, $JOBS jobs)"
echo "────────────────────────────────────────────"

if [ "$TOTAL_FAIL" -gt 0 ] || [ "$BAD_RC" -gt 0 ]; then
exit 1
fi
exit 0
13 changes: 11 additions & 2 deletions scripts/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,17 @@ verify_compiler "$CC"
# Step 1: Clean
scripts/clean.sh

# Step 2 + 3: Build and run tests (Makefile applies $ARCHFLAGS on macOS)
make -j"$NPROC" -f Makefile.cbm test $MAKE_ARGS
# Step 2 + 3: Build, then run every suite as parallel processes (identical
# gate quality — see the ZERO-LOSS CONTRACT in scripts/run-tests-parallel.sh:
# the suite set is enumerated from the runner itself and union-guarded, and
# pass/fail/skip totals aggregate to the same numbers as the sequential run).
# CBM_TEST_SEQUENTIAL=1 restores the single-process runner.
make -j"$NPROC" -f Makefile.cbm build/c/test-runner $MAKE_ARGS
if [ "${CBM_TEST_SEQUENTIAL:-0}" = "1" ]; then
make -f Makefile.cbm test $MAKE_ARGS
else
make -f Makefile.cbm test-par $MAKE_ARGS
fi

# Step 4: C++ large-TU index-hang regression guard (#410). Runs the PROD binary
# in a subprocess with a wall-clock timeout — a hang must fail, not block the run.
Expand Down
26 changes: 24 additions & 2 deletions tests/test_cli.c
Original file line number Diff line number Diff line change
Expand Up @@ -1413,6 +1413,10 @@ TEST(cli_vscode_profile_mcp_uninstall) {
char *saved_home = save_test_env("HOME");
char *saved_path = save_test_env("PATH");
char *saved_appdata = save_test_env("APPDATA");
char *saved_xdg = save_test_env("XDG_CONFIG_HOME");
char xdg_dir[640];
snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir);
cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolvers prefer XDG */
cbm_setenv("HOME", tmpdir, 1);
cbm_setenv("PATH", tmpdir, 1);
#ifdef _WIN32
Expand All @@ -1432,6 +1436,7 @@ TEST(cli_vscode_profile_mcp_uninstall) {
restore_test_env("HOME", saved_home);
restore_test_env("PATH", saved_path);
restore_test_env("APPDATA", saved_appdata);
restore_test_env("XDG_CONFIG_HOME", saved_xdg);
test_rmdir_r(tmpdir);
if (rc != 0 || !removed)
FAIL("VS Code uninstall must remove MCP entries from every existing profile");
Expand Down Expand Up @@ -2953,8 +2958,20 @@ TEST(cli_durable_profiles_follow_current_vendor_paths) {
FAIL("cbm_mkdtemp failed");

const char *const env_names[] = {
"HOME", "PATH", "CLAUDE_CONFIG_DIR", "CODEX_HOME", "QWEN_HOME",
"COPILOT_HOME", "CLINE_DATA_DIR", "KIRO_HOME", "VIBE_HOME", "OPENCODE_CONFIG",
"HOME",
"PATH",
"CLAUDE_CONFIG_DIR",
"CODEX_HOME",
"QWEN_HOME",
"COPILOT_HOME",
"CLINE_DATA_DIR",
"KIRO_HOME",
"VIBE_HOME",
"OPENCODE_CONFIG",
/* Linux path resolvers prefer $XDG_CONFIG_HOME over $HOME/.config;
* CI runners export it, so an isolated-process run resolved OUTSIDE
* the fixture home (green only via env leaked from earlier suites). */
"XDG_CONFIG_HOME",
};
char *saved_env[sizeof(env_names) / sizeof(env_names[0])];
for (size_t i = 0U; i < sizeof(env_names) / sizeof(env_names[0]); i++) {
Expand Down Expand Up @@ -7786,7 +7803,12 @@ TEST(cli_detect_agents_finds_zed) {
#endif
test_mkdirp(dir);

char *saved_xdg = save_test_env("XDG_CONFIG_HOME");
char xdg_dir[640];
snprintf(xdg_dir, sizeof(xdg_dir), "%s/.config", tmpdir);
cbm_setenv("XDG_CONFIG_HOME", xdg_dir, 1); /* Linux resolver prefers XDG */
cbm_detected_agents_t agents = cbm_detect_agents(tmpdir);
restore_test_env("XDG_CONFIG_HOME", saved_xdg);
ASSERT_TRUE(agents.zed);

test_rmdir_r(tmpdir);
Expand Down
51 changes: 39 additions & 12 deletions tests/test_main.c
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,20 @@ static bool suite_requested(const char *name) {
return requested;
}

#define RUN_SELECTED_SUITE(name) \
do { \
if (suite_requested(#name)) { \
RUN_SUITE(name); \
} \
/* --list-suites: print every registered suite name, one per line, without
* running anything. The list and the run share this ONE macro table, so the
* list can never drift from what actually executes — shard runners (make
* test-par, the CI shard matrix) enumerate suites from here and their union
* guard compares against it. */
static bool g_list_only = false;

#define RUN_SELECTED_SUITE(name) \
do { \
if (g_list_only) { \
printf("%s\n", #name); \
} else if (suite_requested(#name)) { \
RUN_SUITE(name); \
} \
} while (0)

/* Forward declarations of suite functions */
Expand Down Expand Up @@ -273,7 +282,9 @@ extern void suite_semantic(void);
extern void suite_ast_profile(void);
extern void suite_slab_alloc(void);
extern void suite_simhash(void);
extern void suite_stack_overflow(void);
extern void suite_stack_overflow_a(void);
extern void suite_stack_overflow_b(void);
extern void suite_stack_overflow_c(void);
extern void suite_dump_verify(void);
extern void suite_dump_verify_io(void);

Expand Down Expand Up @@ -308,16 +319,23 @@ int main(int argc, char **argv) {
return 2;
}

g_suite_argc = argc;
g_suite_argv = argv;
if (argc > 1) {
if (argc == 2 && strcmp(argv[1], "--list-suites") == 0) {
g_list_only = true;
g_suite_argc = 1; /* no suite-name args to match */
} else {
g_suite_argc = argc;
g_suite_argv = argv;
}
if (g_suite_argc > 1) {
g_suite_arg_matched = calloc((size_t)argc, sizeof(*g_suite_arg_matched));
if (!g_suite_arg_matched) {
fprintf(stderr, "Failed to allocate test-suite argument tracking\n");
return 1;
}
}
printf("\n codebase-memory-mcp C test suite\n");
if (!g_list_only) {
printf("\n codebase-memory-mcp C test suite\n");
}

/* Foundation */
RUN_SELECTED_SUITE(arena);
Expand Down Expand Up @@ -454,8 +472,11 @@ int main(int argc, char **argv) {
RUN_SELECTED_SUITE(ast_profile);
RUN_SELECTED_SUITE(simhash);

/* Stack overflow regression (GitHub #199) */
RUN_SELECTED_SUITE(stack_overflow);
/* Stack overflow regression (GitHub #199) — split a/b/c so no single
* suite serializes a parallel run (each ~1/3 of the old wall time). */
RUN_SELECTED_SUITE(stack_overflow_a);
RUN_SELECTED_SUITE(stack_overflow_b);
RUN_SELECTED_SUITE(stack_overflow_c);

/* Integration (end-to-end) */
RUN_SELECTED_SUITE(integration);
Expand All @@ -480,6 +501,12 @@ int main(int argc, char **argv) {

RUN_SELECTED_SUITE(incremental);

if (g_list_only) {
fflush(stdout);
cbm_kind_in_set_free_cache();
sqlite3_shutdown();
return 0;
}
bool any_suite_matched = false;
for (int i = 1; i < g_suite_argc; i++) {
any_suite_matched = any_suite_matched || g_suite_arg_matched[i];
Expand Down
Loading
Loading