fix(gpu): cap the GPU test targets, count the aux retries, split the downgrade counter

Makefile

@@ -561,6 +561,14 @@ test-disk-spill:
 	cargo test --release -p stark --features disk-spill disk_spill
 	FORCE_DISK_SPILL=1 cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths
 
+# Per-target wall clock for the GPU prover targets below. A panic on a device-only
+# cliff assert can leave the prover hung rather than aborting — the panicking thread
+# unwinds while its siblings stay parked in CUDA driver waits, and the process never
+# exits — which would hold the rented merge-queue box until the workflow timeout.
+# 45 min is generous against their normal runtime; the SIGKILL follows 30s later, and
+# timeout's 124 exit fails the target so gpu_test.sh reports the group as failed.
+GPU_TEST_TIMEOUT := timeout -k 30 2700
+
 # math-cuda parity tests (requires NVIDIA GPU + nvcc)
 test-math-cuda:
 	cargo test -p math-cuda --release
@@ -570,13 +578,13 @@ test-math-cuda:
 # --test-threads=1: these tests reset and assert on process-global GPU call
 # counters, so they must run serially or one test's reset races another's read.
 test-cuda-integration:
-	cargo test -p lambda-vm-prover --release --features cuda \
+	$(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features cuda \
 	    --test cuda_path_integration -- --ignored --nocapture --test-threads=1
 
 # GPU error-path coverage (requires NVIDIA GPU + nvcc).
 # Forces cuda dispatch errors and asserts the CPU fallback still produces a verifying proof.
 test-cuda-fallback:
-	cargo test -p lambda-vm-prover --release --features test-cuda-faults \
+	$(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \
 	    --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1
 
 # The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA
@@ -586,14 +594,14 @@ test-cuda-fallback:
 # compile-recursion-elfs: this unfiltered run executes the non-ignored recursion
 # smoke tests, which read prebuilt guest ELFs; scripts/gpu_test.sh otherwise never builds them.
 test-prover-cuda: compile-recursion-elfs
-	cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \
+	$(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \
 	    --features lambda-vm-prover/cuda -- --test-threads=1
 
 # The comprehensive all-instructions prove (ignored by default) on the GPU path (requires
 # NVIDIA GPU + nvcc). GPU counterpart of the all-instructions half of CPU CI's merge-queue-only
 # comprehensive job (the CPU job also runs test_recursion_execute; recursion has no GPU leg yet).
 test-prover-comprehensive-cuda:
-	cargo test --release -p lambda-vm-prover --features cuda \
+	$(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover --features cuda \
 	    test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture
 
 # math-cuda quick microbench (median of 10 runs)

crypto/stark/src/gpu_lde.rs

@@ -117,6 +117,8 @@ pub fn reset_all_gpu_call_counters() {
     GPU_OPENING_GATHER_CALLS.store(0, Ordering::Relaxed);
     GPU_DEVICE_ONLY_CALLS.store(0, Ordering::Relaxed);
     GPU_DEVICE_ONLY_DOWNGRADES.store(0, Ordering::Relaxed);
+    GPU_RESIDENT_AUX_RETRIES.store(0, Ordering::Relaxed);
+    GPU_RESIDENT_AUX_DOWNGRADES.store(0, Ordering::Relaxed);
 }
 
 pub(crate) static GPU_EXTEND_HALVES_CALLS: AtomicU64 = AtomicU64::new(0);
@@ -218,11 +220,12 @@ pub(crate) fn device_only_disabled() -> bool {
 /// transient GPU error), what happens depends on the round. R2 and the R1
 /// resident-aux commit recover: they download what the host arms need (the
 /// resident LDEs at R2, the resident aux trace plus the main LDE at R1), bump
-/// [`GPU_DEVICE_ONLY_DOWNGRADES`] and continue host-backed — slower, never
-/// wrong — aborting only when the resident handles cannot serve the data. R3
-/// and R4 have no such recovery: the R3 barycentric arms assert on the buffer
-/// they are about to read and the R4 guards on `host_trace_empty`, both
-/// failing loudly rather than reading an empty host trace.
+/// their site's counter ([`GPU_DEVICE_ONLY_DOWNGRADES`] at R2,
+/// [`GPU_RESIDENT_AUX_DOWNGRADES`] at R1) and continue host-backed — slower,
+/// never wrong — aborting only when the resident handles cannot serve the
+/// data. R3 and R4 have no such recovery: the R3 barycentric arms assert on
+/// the buffer they are about to read and the R4 guards on `host_trace_empty`,
+/// both failing loudly rather than reading an empty host trace.
 ///
 /// `zerofier_uniform` must be the R1-derived conservative form (all constraints
 /// share `end_exemptions == 0`), which implies `ZerofierEvaluations::is_uniform`
@@ -234,7 +237,8 @@ pub(crate) fn device_only_disabled() -> bool {
 /// costs every gate-true table either a hard-abort at R3/R4 — loud, but an
 /// avoidable crash — or, at R2 and the R1 resident-aux commit, a silent
 /// downgrade to the host path, which is what [`GPU_DEVICE_ONLY_DOWNGRADES`]
-/// exists to surface.
+/// exists to surface (an R1 decline lands in [`GPU_RESIDENT_AUX_DOWNGRADES`],
+/// which the gate does not govern).
 pub(crate) fn device_only_gate<F, E>(
     lde_size: usize,
     n: usize,
@@ -1462,21 +1466,46 @@ pub fn gpu_fri_calls() -> u64 {
 /// are counted here, so a single failed dispatch does not necessarily lower
 /// the total; R3's fallbacks are CPU-only, so a failure there does.
 pub(crate) static GPU_BATCH_INVERT_CALLS: AtomicU64 = AtomicU64::new(0);
-/// Times a table had to fall back to a host trace whose data first had to be
-/// downloaded off the device, because a device path declined at runtime (see
-/// [`materialize_lde_trace_host`] and [`materialize_aux_trace_host`]).
-/// Nonzero means a device dispatch declined and the table continued
-/// host-backed — correct but slower. Not every one is a gate miss: the R1
-/// resident-aux site is entered whenever `aux_resident()` is set, whatever
-/// the device-only gate said, so it also counts declines on tables that were
-/// never device-only. Mirroring the missing condition into the gate is the
-/// fix for the device-only case; a resident-aux decline is usually transient
-/// VRAM pressure instead.
+/// R2 downgrades, and only those: times a device-only table fell back to the
+/// host evaluator and had its resident LDEs downloaded into the host buffers
+/// first ([`materialize_lde_trace_host`], the sole site that bumps this).
+/// Nonzero means the device-only gate cleared a table whose R2 dispatch then
+/// declined at runtime — the table continued host-backed, correct but slower —
+/// so every count is a gate miss, and the fix is to mirror the missing
+/// condition into the gate. The R1 resident-aux downgrade is counted by
+/// [`GPU_RESIDENT_AUX_DOWNGRADES`] instead: it fires on tables the gate never
+/// marked device-only, so summing the two would blame the gate for declines it
+/// never made.
 pub(crate) static GPU_DEVICE_ONLY_DOWNGRADES: AtomicU64 = AtomicU64::new(0);
 pub fn gpu_device_only_downgrades() -> u64 {
     GPU_DEVICE_ONLY_DOWNGRADES.load(Ordering::Relaxed)
 }
 
+/// R1 downgrades, and only those: times the resident aux trace was downloaded
+/// so the aux commit could continue on the host arms, after the device aux LDE
+/// declined and the drain-and-retry either did not run or declined again
+/// ([`materialize_aux_trace_host`], the sole site that bumps this). Independent
+/// of the device-only gate — the site is entered whenever `aux_resident()` is
+/// set, whatever the gate said — so a table that was never device-only can land
+/// here, and a nonzero value points at sustained VRAM pressure rather than a
+/// gate miss. Read it against [`GPU_RESIDENT_AUX_RETRIES`]: retries alone mean
+/// the drain absorbed the pressure, retries plus downgrades mean it did not.
+pub(crate) static GPU_RESIDENT_AUX_DOWNGRADES: AtomicU64 = AtomicU64::new(0);
+pub fn gpu_resident_aux_downgrades() -> u64 {
+    GPU_RESIDENT_AUX_DOWNGRADES.load(Ordering::Relaxed)
+}
+
+/// Times the R1 resident-aux LDE declined and the prover drained the device to
+/// retry it (prover.rs). Nonzero means the device hit transient VRAM pressure —
+/// the retry is what keeps a decline from becoming a
+/// [`GPU_RESIDENT_AUX_DOWNGRADES`] host downgrade, so a run with retries but no
+/// downgrades paid nothing but the drain. Counts declines, not outcomes: it is
+/// bumped before the retry, whether or not the retry then succeeds.
+pub(crate) static GPU_RESIDENT_AUX_RETRIES: AtomicU64 = AtomicU64::new(0);
+pub fn gpu_resident_aux_retries() -> u64 {
+    GPU_RESIDENT_AUX_RETRIES.load(Ordering::Relaxed)
+}
+
 /// Recover a device-only table for the host path: download the resident main
 /// and aux LDEs from their device handles into the host buffers and clear the
 /// device-only flag. A side whose host buffer is already populated (a mixed
@@ -1694,7 +1723,7 @@ where
         return false;
     }
     trace.aux_resident = None;
-    GPU_DEVICE_ONLY_DOWNGRADES.fetch_add(1, Ordering::Relaxed);
+    GPU_RESIDENT_AUX_DOWNGRADES.fetch_add(1, Ordering::Relaxed);
     true
 }
 

crypto/stark/src/prover.rs

@@ -3469,6 +3469,8 @@ pub trait IsStarkProver<
                                 // drain releases those peaks, so one retry
                                 // tends to keep the table fully resident
                                 // instead of paying the host downgrade.
+                                crate::gpu_lde::GPU_RESIDENT_AUX_RETRIES
+                                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
                                 eprintln!(
                                     "[gpu] resident aux LDE declined: table={} \
                                      (retrying after device drain)",

prover/tests/cuda_path_integration.rs

@@ -188,7 +188,8 @@ fn gpu_opening_gather_fires_and_verifies() {
 /// verify). A mis-gate that forces a host fallback shows up one of two ways:
 /// at R3/R4 it panics one of the guards, while at R2 and the R1 resident-aux
 /// commit it recovers silently and is caught by the downgrade-counter
-/// assertion below.
+/// assertions below — one per site, since the R1 counter also covers tables the
+/// device-only gate never cleared.
 #[test]
 #[ignore = "requires GPU; run with --ignored --nocapture"]
 fn gpu_device_only_residency_fires_and_verifies() {
@@ -202,10 +203,17 @@ fn gpu_device_only_residency_fires_and_verifies() {
     assert_eq!(
         stark::gpu_lde::gpu_device_only_downgrades(),
         0,
-        "a table was downgraded back to a host trace on the happy path \
-         (a device dispatch declined at runtime: on a device-only table the \
-          gate should mirror the missing condition; a resident-aux decline is \
-          usually VRAM pressure)"
+        "a device-only table was downgraded back to a host trace on the happy \
+         path (its R2 dispatch declined at runtime: the gate should mirror the \
+          missing condition)"
+    );
+    assert_eq!(
+        stark::gpu_lde::gpu_resident_aux_downgrades(),
+        0,
+        "a table's resident aux trace was downloaded back to the host on the \
+         happy path (the device aux LDE declined and the drain-and-retry did \
+          not recover it — usually VRAM pressure, and not gated on device-only, \
+          so this can fire for a table that was never device-only)"
     );
     assert!(
         verify(&proof, &elf).expect("verify"),