diff --git a/Makefile b/Makefile index a4b05b507..aeb67114b 100644 --- a/Makefile +++ b/Makefile @@ -561,6 +561,14 @@ test-disk-spill: cargo test --release -p stark --features disk-spill disk_spill FORCE_DISK_SPILL=1 cargo test --release -p lambda-vm-prover --features disk-spill -- disk_spill count_table_lengths +# Per-target wall clock for the GPU prover targets below. A panic on a device-only +# cliff assert can leave the prover hung rather than aborting — the panicking thread +# unwinds while its siblings stay parked in CUDA driver waits, and the process never +# exits — which would hold the rented merge-queue box until the workflow timeout. +# 45 min is generous against their normal runtime; the SIGKILL follows 30s later, and +# timeout's 124 exit fails the target so gpu_test.sh reports the group as failed. +GPU_TEST_TIMEOUT := timeout -k 30 2700 + # math-cuda parity tests (requires NVIDIA GPU + nvcc) test-math-cuda: cargo test -p math-cuda --release @@ -570,13 +578,13 @@ test-math-cuda: # --test-threads=1: these tests reset and assert on process-global GPU call # counters, so they must run serially or one test's reset races another's read. test-cuda-integration: - cargo test -p lambda-vm-prover --release --features cuda \ + $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features cuda \ --test cuda_path_integration -- --ignored --nocapture --test-threads=1 # GPU error-path coverage (requires NVIDIA GPU + nvcc). # Forces cuda dispatch errors and asserts the CPU fallback still produces a verifying proof. test-cuda-fallback: - cargo test -p lambda-vm-prover --release --features test-cuda-faults \ + $(GPU_TEST_TIMEOUT) cargo test -p lambda-vm-prover --release --features test-cuda-faults \ --test cuda_fallback_tests -- --ignored --nocapture --test-threads=1 # The prover/stark/crypto/ecsm test suite with the GPU (cuda) path enabled (requires NVIDIA @@ -586,14 +594,14 @@ test-cuda-fallback: # compile-recursion-elfs: this unfiltered run executes the non-ignored recursion # smoke tests, which read prebuilt guest ELFs; scripts/gpu_test.sh otherwise never builds them. test-prover-cuda: compile-recursion-elfs - cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \ + $(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover -p stark -p crypto -p ecsm \ --features lambda-vm-prover/cuda -- --test-threads=1 # The comprehensive all-instructions prove (ignored by default) on the GPU path (requires # NVIDIA GPU + nvcc). GPU counterpart of the all-instructions half of CPU CI's merge-queue-only # comprehensive job (the CPU job also runs test_recursion_execute; recursion has no GPU leg yet). test-prover-comprehensive-cuda: - cargo test --release -p lambda-vm-prover --features cuda \ + $(GPU_TEST_TIMEOUT) cargo test --release -p lambda-vm-prover --features cuda \ test_prove_elfs_all_instructions_64_full -- --ignored --test-threads=1 --nocapture # math-cuda quick microbench (median of 10 runs) diff --git a/crypto/stark/src/gpu_lde.rs b/crypto/stark/src/gpu_lde.rs index ced0bc530..b510b33d1 100644 --- a/crypto/stark/src/gpu_lde.rs +++ b/crypto/stark/src/gpu_lde.rs @@ -117,6 +117,8 @@ pub fn reset_all_gpu_call_counters() { GPU_OPENING_GATHER_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_CALLS.store(0, Ordering::Relaxed); GPU_DEVICE_ONLY_DOWNGRADES.store(0, Ordering::Relaxed); + GPU_RESIDENT_AUX_RETRIES.store(0, Ordering::Relaxed); + GPU_RESIDENT_AUX_DOWNGRADES.store(0, Ordering::Relaxed); } pub(crate) static GPU_EXTEND_HALVES_CALLS: AtomicU64 = AtomicU64::new(0); @@ -218,11 +220,12 @@ pub(crate) fn device_only_disabled() -> bool { /// transient GPU error), what happens depends on the round. R2 and the R1 /// resident-aux commit recover: they download what the host arms need (the /// resident LDEs at R2, the resident aux trace plus the main LDE at R1), bump -/// [`GPU_DEVICE_ONLY_DOWNGRADES`] and continue host-backed — slower, never -/// wrong — aborting only when the resident handles cannot serve the data. R3 -/// and R4 have no such recovery: the R3 barycentric arms assert on the buffer -/// they are about to read and the R4 guards on `host_trace_empty`, both -/// failing loudly rather than reading an empty host trace. +/// their site's counter ([`GPU_DEVICE_ONLY_DOWNGRADES`] at R2, +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] at R1) and continue host-backed — slower, +/// never wrong — aborting only when the resident handles cannot serve the +/// data. R3 and R4 have no such recovery: the R3 barycentric arms assert on +/// the buffer they are about to read and the R4 guards on `host_trace_empty`, +/// both failing loudly rather than reading an empty host trace. /// /// `zerofier_uniform` must be the R1-derived conservative form (all constraints /// share `end_exemptions == 0`), which implies `ZerofierEvaluations::is_uniform` @@ -234,7 +237,8 @@ pub(crate) fn device_only_disabled() -> bool { /// costs every gate-true table either a hard-abort at R3/R4 — loud, but an /// avoidable crash — or, at R2 and the R1 resident-aux commit, a silent /// downgrade to the host path, which is what [`GPU_DEVICE_ONLY_DOWNGRADES`] -/// exists to surface. +/// exists to surface (an R1 decline lands in [`GPU_RESIDENT_AUX_DOWNGRADES`], +/// which the gate does not govern). pub(crate) fn device_only_gate( lde_size: usize, n: usize, @@ -1462,21 +1466,46 @@ pub fn gpu_fri_calls() -> u64 { /// are counted here, so a single failed dispatch does not necessarily lower /// the total; R3's fallbacks are CPU-only, so a failure there does. pub(crate) static GPU_BATCH_INVERT_CALLS: AtomicU64 = AtomicU64::new(0); -/// Times a table had to fall back to a host trace whose data first had to be -/// downloaded off the device, because a device path declined at runtime (see -/// [`materialize_lde_trace_host`] and [`materialize_aux_trace_host`]). -/// Nonzero means a device dispatch declined and the table continued -/// host-backed — correct but slower. Not every one is a gate miss: the R1 -/// resident-aux site is entered whenever `aux_resident()` is set, whatever -/// the device-only gate said, so it also counts declines on tables that were -/// never device-only. Mirroring the missing condition into the gate is the -/// fix for the device-only case; a resident-aux decline is usually transient -/// VRAM pressure instead. +/// R2 downgrades, and only those: times a device-only table fell back to the +/// host evaluator and had its resident LDEs downloaded into the host buffers +/// first ([`materialize_lde_trace_host`], the sole site that bumps this). +/// Nonzero means the device-only gate cleared a table whose R2 dispatch then +/// declined at runtime — the table continued host-backed, correct but slower — +/// so every count is a gate miss, and the fix is to mirror the missing +/// condition into the gate. The R1 resident-aux downgrade is counted by +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] instead: it fires on tables the gate never +/// marked device-only, so summing the two would blame the gate for declines it +/// never made. pub(crate) static GPU_DEVICE_ONLY_DOWNGRADES: AtomicU64 = AtomicU64::new(0); pub fn gpu_device_only_downgrades() -> u64 { GPU_DEVICE_ONLY_DOWNGRADES.load(Ordering::Relaxed) } +/// R1 downgrades, and only those: times the resident aux trace was downloaded +/// so the aux commit could continue on the host arms, after the device aux LDE +/// declined and the drain-and-retry either did not run or declined again +/// ([`materialize_aux_trace_host`], the sole site that bumps this). Independent +/// of the device-only gate — the site is entered whenever `aux_resident()` is +/// set, whatever the gate said — so a table that was never device-only can land +/// here, and a nonzero value points at sustained VRAM pressure rather than a +/// gate miss. Read it against [`GPU_RESIDENT_AUX_RETRIES`]: retries alone mean +/// the drain absorbed the pressure, retries plus downgrades mean it did not. +pub(crate) static GPU_RESIDENT_AUX_DOWNGRADES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_resident_aux_downgrades() -> u64 { + GPU_RESIDENT_AUX_DOWNGRADES.load(Ordering::Relaxed) +} + +/// Times the R1 resident-aux LDE declined and the prover drained the device to +/// retry it (prover.rs). Nonzero means the device hit transient VRAM pressure — +/// the retry is what keeps a decline from becoming a +/// [`GPU_RESIDENT_AUX_DOWNGRADES`] host downgrade, so a run with retries but no +/// downgrades paid nothing but the drain. Counts declines, not outcomes: it is +/// bumped before the retry, whether or not the retry then succeeds. +pub(crate) static GPU_RESIDENT_AUX_RETRIES: AtomicU64 = AtomicU64::new(0); +pub fn gpu_resident_aux_retries() -> u64 { + GPU_RESIDENT_AUX_RETRIES.load(Ordering::Relaxed) +} + /// Recover a device-only table for the host path: download the resident main /// and aux LDEs from their device handles into the host buffers and clear the /// device-only flag. A side whose host buffer is already populated (a mixed @@ -1694,7 +1723,7 @@ where return false; } trace.aux_resident = None; - GPU_DEVICE_ONLY_DOWNGRADES.fetch_add(1, Ordering::Relaxed); + GPU_RESIDENT_AUX_DOWNGRADES.fetch_add(1, Ordering::Relaxed); true } diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index 8659cf730..232e1faaf 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -3469,6 +3469,8 @@ pub trait IsStarkProver< // drain releases those peaks, so one retry // tends to keep the table fully resident // instead of paying the host downgrade. + crate::gpu_lde::GPU_RESIDENT_AUX_RETRIES + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); eprintln!( "[gpu] resident aux LDE declined: table={} \ (retrying after device drain)", diff --git a/prover/tests/cuda_path_integration.rs b/prover/tests/cuda_path_integration.rs index 5b5e52fbc..7ae50afad 100644 --- a/prover/tests/cuda_path_integration.rs +++ b/prover/tests/cuda_path_integration.rs @@ -188,7 +188,8 @@ fn gpu_opening_gather_fires_and_verifies() { /// verify). A mis-gate that forces a host fallback shows up one of two ways: /// at R3/R4 it panics one of the guards, while at R2 and the R1 resident-aux /// commit it recovers silently and is caught by the downgrade-counter -/// assertion below. +/// assertions below — one per site, since the R1 counter also covers tables the +/// device-only gate never cleared. #[test] #[ignore = "requires GPU; run with --ignored --nocapture"] fn gpu_device_only_residency_fires_and_verifies() { @@ -202,10 +203,17 @@ fn gpu_device_only_residency_fires_and_verifies() { assert_eq!( stark::gpu_lde::gpu_device_only_downgrades(), 0, - "a table was downgraded back to a host trace on the happy path \ - (a device dispatch declined at runtime: on a device-only table the \ - gate should mirror the missing condition; a resident-aux decline is \ - usually VRAM pressure)" + "a device-only table was downgraded back to a host trace on the happy \ + path (its R2 dispatch declined at runtime: the gate should mirror the \ + missing condition)" + ); + assert_eq!( + stark::gpu_lde::gpu_resident_aux_downgrades(), + 0, + "a table's resident aux trace was downloaded back to the host on the \ + happy path (the device aux LDE declined and the drain-and-retry did \ + not recover it — usually VRAM pressure, and not gated on device-only, \ + so this can fire for a table that was never device-only)" ); assert!( verify(&proof, &elf).expect("verify"),