From 32c8f08a6f7ff5a433c13e8cbe34d904944bf30d Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 11 Aug 2026 11:28:39 -0300 Subject: [PATCH 1/3] ci(gpu): cap each GPU prover test target with a wall clock A panic on a device-only cliff assert can leave the prover hung instead of aborting: the panicking thread unwinds while its siblings stay parked in CUDA driver waits, so the process never exits. Observed repeatedly on rented 5090s under VRAM pressure. The merge-queue GPU job runs the Makefile targets through scripts/gpu_test.sh with no per-target limit, so one hang holds the box until the workflow timeout kills the whole job with no indication of which group stalled. Wrap the four cuda targets that run the prover in `timeout -k 30 2700`. 45 minutes is well above their normal runtime and well below the job timeout, and timeout's 124 exit fails the target, so gpu_test.sh names the stalled group and the merge is blocked. test-math-cuda is left alone: kernel parity never enters the prover. --- Makefile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a4b05b507..aeb67114b 100644 --- a/Makefile +++ b/Makefile @@ -561,6 +561,14 @@ test-disk-spill: cargo test --release -p stark --features disk-spill disk_spill FORCE_DISK_SPILL=1 cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths +# Per-target wall clock for the GPU prover targets below. A panic on a device-only +# cliff assert can leave the prover hung rather than aborting — the panicking thread +# unwinds while its siblings stay parked in CUDA driver waits, and the process never +# exits — which would hold the rented merge-queue box until the workflow timeout. +# 45 min is generous against their normal runtime; the SIGKILL follows 30s later, and +# timeout's 124 exit fails the target so gpu_test.sh reports the group as failed. +GPU_TEST_TIMEOUT := timeout -k 30 2700 + # math-cuda parity tests (requires NVIDIA GPU + nvcc) test-math-cuda: cargo test -p math-cuda --release @@ -570,13 +578,13 @@ test-math-cuda: # --test-threads=1: these tests reset and assert on process-global GPU call # counters, so they must run serially or one test's reset races another's read. test-cuda-integration: - cargo test -p lambda-vm-prover --release --features cuda \ + $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features cuda \ --test cuda_path_integration -- --ignored --nocapture --test-threads=1 # GPU error-path coverage (requires NVIDIA GPU + nvcc). # Forces cuda dispatch errors and asserts the CPU fallback still produces a verifying proof. test-cuda-fallback: - cargo test -p lambda-vm-prover --release --features test-cuda-faults \ + $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \ --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1 # The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA @@ -586,14 +594,14 @@ test-cuda-fallback: # compile-recursion-elfs: this unfiltered run executes the non-ignored recursion # smoke tests, which read prebuilt guest ELFs; scripts/gpu_test.sh otherwise never builds them. test-prover-cuda: compile-recursion-elfs - cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \ + $(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \ --features lambda-vm-prover/cuda -- --test-threads=1 # The comprehensive all-instructions prove (ignored by default) on the GPU path (requires # NVIDIA GPU + nvcc). GPU counterpart of the all-instructions half of CPU CI's merge-queue-only # comprehensive job (the CPU job also runs test_recursion_execute; recursion has no GPU leg yet). test-prover-comprehensive-cuda: - cargo test --release -p lambda-vm-prover --features cuda \ + $(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover --features cuda \ test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture # math-cuda quick microbench (median of 10 runs) From c0d61effc91905fb6da4aaac2c156a260a6d0bfa Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 11 Aug 2026 11:29:55 -0300 Subject: [PATCH 2/3] feat(gpu): count the resident-aux drain-and-retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The R1 resident-aux path retries the device LDE after a full device drain when the first attempt declines, and that retry usually succeeds — which is the problem: a successful retry left no trace anywhere except an eprintln, so how often the device actually declines under VRAM pressure was unmeasurable in production, where nobody is reading stderr. GPU_RESIDENT_AUX_RETRIES makes the decline rate observable and separates it from its consequence: retries with no downgrades means the drain absorbed the pressure, while the two rising together means the drain is no longer enough. --- crypto/stark/src/gpu_lde.rs | 12 ++++++++++++ crypto/stark/src/prover.rs | 2 ++ 2 files changed, 14 insertions(+) diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index ced0bc530..d9832a1b8 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -117,6 +117,7 @@ pub fn reset_all_gpu_call_counters() { GPU_OPENING_GATHER_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_DOWNGRADES.store(0, Ordering::Relaxed); + GPU_RESIDENT_AUX_RETRIES.store(0, Ordering::Relaxed); } pub(crate) static GPU_EXTEND_HALVES_CALLS: AtomicU64 = AtomicU64::new(0); @@ -1477,6 +1478,17 @@ pub fn gpu_device_only_downgrades() -> u64 { GPU_DEVICE_ONLY_DOWNGRADES.load(Ordering::Relaxed) } +/// Times the R1 resident-aux LDE declined and the prover drained the device to +/// retry it (prover.rs). Nonzero means the device hit transient VRAM pressure — +/// the retry is what keeps a decline from becoming the host downgrade counted +/// by [`GPU_DEVICE_ONLY_DOWNGRADES`], so a run with retries but no downgrades +/// paid nothing but the drain. Counts declines, not outcomes: it is bumped +/// before the retry, whether or not the retry then succeeds. +pub(crate) static GPU_RESIDENT_AUX_RETRIES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_resident_aux_retries() -> u64 { + GPU_RESIDENT_AUX_RETRIES.load(Ordering::Relaxed) +} + /// Recover a device-only table for the host path: download the resident main /// and aux LDEs from their device handles into the host buffers and clear the /// device-only flag. A side whose host buffer is already populated (a mixed diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index 8659cf730..232e1faaf 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -3469,6 +3469,8 @@ pub trait IsStarkProver< // drain releases those peaks, so one retry // tends to keep the table fully resident // instead of paying the host downgrade. + crate::gpu_lde::GPU_RESIDENT_AUX_RETRIES + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); eprintln!( "[gpu] resident aux LDE declined: table={} \ (retrying after device drain)", From 715bc39c28b9da3644b3b5b2ba0770c16c5a7415 Mon Sep 17 00:00:00 2001 From: MauroFab Date: Tue, 11 Aug 2026 11:33:49 -0300 Subject: [PATCH 3/3] fix(gpu): split the downgrade counter by site GPU_DEVICE_ONLY_DOWNGRADES counted two unrelated events: the R2 device-only downgrade in materialize_lde_trace_host, which is always a device-only gate miss, and the R1 resident-aux downgrade in materialize_aux_trace_host, which is entered whenever aux_resident() is set and so fires on tables the gate never marked device-only. A GPU run made that concrete: a preprocessed BITWISE table took the R1 downgrade despite never being device-only, and the combined counter reported it as a gate miss with nothing to distinguish it from one. Keep GPU_DEVICE_ONLY_DOWNGRADES on the R2 site alone and add GPU_RESIDENT_AUX_DOWNGRADES for the R1 site, so a nonzero value names its own fix: mirror the missing condition into the gate for the former, relieve VRAM pressure for the latter. The integration test now asserts both are zero with per-site messages, and the gate docs say which counter each round bumps. --- crypto/stark/src/gpu_lde.rs | 59 +++++++++++++++++---------- prover/tests/cuda_path_integration.rs | 18 +++++--- 2 files changed, 51 insertions(+), 26 deletions(-) diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index d9832a1b8..b510b33d1 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -118,6 +118,7 @@ pub fn reset_all_gpu_call_counters() { GPU_DEVICE_ONLY_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_DOWNGRADES.store(0, Ordering::Relaxed); GPU_RESIDENT_AUX_RETRIES.store(0, Ordering::Relaxed); + GPU_RESIDENT_AUX_DOWNGRADES.store(0, Ordering::Relaxed); } pub(crate) static GPU_EXTEND_HALVES_CALLS: AtomicU64 = AtomicU64::new(0); @@ -219,11 +220,12 @@ pub(crate) fn device_only_disabled() -> bool { /// transient GPU error), what happens depends on the round. R2 and the R1 /// resident-aux commit recover: they download what the host arms need (the /// resident LDEs at R2, the resident aux trace plus the main LDE at R1), bump -/// [`GPU_DEVICE_ONLY_DOWNGRADES`] and continue host-backed — slower, never -/// wrong — aborting only when the resident handles cannot serve the data. R3 -/// and R4 have no such recovery: the R3 barycentric arms assert on the buffer -/// they are about to read and the R4 guards on `host_trace_empty`, both -/// failing loudly rather than reading an empty host trace. +/// their site's counter ([`GPU_DEVICE_ONLY_DOWNGRADES`] at R2, +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] at R1) and continue host-backed — slower, +/// never wrong — aborting only when the resident handles cannot serve the +/// data. R3 and R4 have no such recovery: the R3 barycentric arms assert on +/// the buffer they are about to read and the R4 guards on `host_trace_empty`, +/// both failing loudly rather than reading an empty host trace. /// /// `zerofier_uniform` must be the R1-derived conservative form (all constraints /// share `end_exemptions == 0`), which implies `ZerofierEvaluations::is_uniform` @@ -235,7 +237,8 @@ pub(crate) fn device_only_disabled() -> bool { /// costs every gate-true table either a hard-abort at R3/R4 — loud, but an /// avoidable crash — or, at R2 and the R1 resident-aux commit, a silent /// downgrade to the host path, which is what [`GPU_DEVICE_ONLY_DOWNGRADES`] -/// exists to surface. +/// exists to surface (an R1 decline lands in [`GPU_RESIDENT_AUX_DOWNGRADES`], +/// which the gate does not govern). pub(crate) fn device_only_gate( lde_size: usize, n: usize, @@ -1463,27 +1466,41 @@ pub fn gpu_fri_calls() -> u64 { /// are counted here, so a single failed dispatch does not necessarily lower /// the total; R3's fallbacks are CPU-only, so a failure there does. pub(crate) static GPU_BATCH_INVERT_CALLS: AtomicU64 = AtomicU64::new(0); -/// Times a table had to fall back to a host trace whose data first had to be -/// downloaded off the device, because a device path declined at runtime (see -/// [`materialize_lde_trace_host`] and [`materialize_aux_trace_host`]). -/// Nonzero means a device dispatch declined and the table continued -/// host-backed — correct but slower. Not every one is a gate miss: the R1 -/// resident-aux site is entered whenever `aux_resident()` is set, whatever -/// the device-only gate said, so it also counts declines on tables that were -/// never device-only. Mirroring the missing condition into the gate is the -/// fix for the device-only case; a resident-aux decline is usually transient -/// VRAM pressure instead. +/// R2 downgrades, and only those: times a device-only table fell back to the +/// host evaluator and had its resident LDEs downloaded into the host buffers +/// first ([`materialize_lde_trace_host`], the sole site that bumps this). +/// Nonzero means the device-only gate cleared a table whose R2 dispatch then +/// declined at runtime — the table continued host-backed, correct but slower — +/// so every count is a gate miss, and the fix is to mirror the missing +/// condition into the gate. The R1 resident-aux downgrade is counted by +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] instead: it fires on tables the gate never +/// marked device-only, so summing the two would blame the gate for declines it +/// never made. pub(crate) static GPU_DEVICE_ONLY_DOWNGRADES: AtomicU64 = AtomicU64::new(0); pub fn gpu_device_only_downgrades() -> u64 { GPU_DEVICE_ONLY_DOWNGRADES.load(Ordering::Relaxed) } +/// R1 downgrades, and only those: times the resident aux trace was downloaded +/// so the aux commit could continue on the host arms, after the device aux LDE +/// declined and the drain-and-retry either did not run or declined again +/// ([`materialize_aux_trace_host`], the sole site that bumps this). Independent +/// of the device-only gate — the site is entered whenever `aux_resident()` is +/// set, whatever the gate said — so a table that was never device-only can land +/// here, and a nonzero value points at sustained VRAM pressure rather than a +/// gate miss. Read it against [`GPU_RESIDENT_AUX_RETRIES`]: retries alone mean +/// the drain absorbed the pressure, retries plus downgrades mean it did not. +pub(crate) static GPU_RESIDENT_AUX_DOWNGRADES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_resident_aux_downgrades() -> u64 { + GPU_RESIDENT_AUX_DOWNGRADES.load(Ordering::Relaxed) +} + /// Times the R1 resident-aux LDE declined and the prover drained the device to /// retry it (prover.rs). Nonzero means the device hit transient VRAM pressure — -/// the retry is what keeps a decline from becoming the host downgrade counted -/// by [`GPU_DEVICE_ONLY_DOWNGRADES`], so a run with retries but no downgrades -/// paid nothing but the drain. Counts declines, not outcomes: it is bumped -/// before the retry, whether or not the retry then succeeds. +/// the retry is what keeps a decline from becoming a +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] host downgrade, so a run with retries but no +/// downgrades paid nothing but the drain. Counts declines, not outcomes: it is +/// bumped before the retry, whether or not the retry then succeeds. pub(crate) static GPU_RESIDENT_AUX_RETRIES: AtomicU64 = AtomicU64::new(0); pub fn gpu_resident_aux_retries() -> u64 { GPU_RESIDENT_AUX_RETRIES.load(Ordering::Relaxed) @@ -1706,7 +1723,7 @@ where return false; } trace.aux_resident = None; - GPU_DEVICE_ONLY_DOWNGRADES.fetch_add(1, Ordering::Relaxed); + GPU_RESIDENT_AUX_DOWNGRADES.fetch_add(1, Ordering::Relaxed); true } diff --git a/prover/tests/cuda_path_integration.rs b/prover/tests/cuda_path_integration.rs index 5b5e52fbc..7ae50afad 100644 --- a/prover/tests/cuda_path_integration.rs +++ b/prover/tests/cuda_path_integration.rs @@ -188,7 +188,8 @@ fn gpu_opening_gather_fires_and_verifies() { /// verify). A mis-gate that forces a host fallback shows up one of two ways: /// at R3/R4 it panics one of the guards, while at R2 and the R1 resident-aux /// commit it recovers silently and is caught by the downgrade-counter -/// assertion below. +/// assertions below — one per site, since the R1 counter also covers tables the +/// device-only gate never cleared. #[test] #[ignore = "requires GPU; run with --ignored --nocapture"] fn gpu_device_only_residency_fires_and_verifies() { @@ -202,10 +203,17 @@ fn gpu_device_only_residency_fires_and_verifies() { assert_eq!( stark::gpu_lde::gpu_device_only_downgrades(), 0, - "a table was downgraded back to a host trace on the happy path \ - (a device dispatch declined at runtime: on a device-only table the \ - gate should mirror the missing condition; a resident-aux decline is \ - usually VRAM pressure)" + "a device-only table was downgraded back to a host trace on the happy \ + path (its R2 dispatch declined at runtime: the gate should mirror the \ + missing condition)" + ); + assert_eq!( + stark::gpu_lde::gpu_resident_aux_downgrades(), + 0, + "a table's resident aux trace was downloaded back to the host on the \ + happy path (the device aux LDE declined and the drain-and-retry did \ + not recover it — usually VRAM pressure, and not gated on device-only, \ + so this can fire for a table that was never device-only)" ); assert!( verify(&proof, &elf).expect("verify"),