benchmark-gpu.yml

name: Benchmark GPU (PR)

# Rent an RTX 5090 on Vast.ai (hourly) and run the drift-free A/B/B/A (ABBA) paired
# prover benchmark — the same method as the CPU `/bench-abba` (scripts/bench_abba.sh) —
# but with the CUDA prover path enabled (BENCH_FEATURES=jemalloc-stats,prover/cuda).
# It builds the cli at the PR head and at main, runs N interleaved pairs on the GPU,
# posts the paired-t + Wilcoxon verdict back to the PR, then ALWAYS destroys the box.
#
# Triggered by a "/bench-gpu [N]" comment on a PR (N = pair count, default 14) or via
# workflow_dispatch.
#
# Workload: the real block (see tooling/ethrex-block-converter/README.md), proven with
# --continuations at the calibrated epoch size below.
# Orchestration runs on a GitHub-hosted runner; all GPU work happens on the rented
# Vast box (provisioned by the template onstart).
#
# Requires repo secrets:
#   VAST_API_KEY        — https://cloud.vast.ai/manage-keys/
#   VAST_TEMPLATE_HASH  — hash of the "NVIDIA CUDA Lambda VM 64GB" template

on:
  workflow_dispatch:
    inputs:
      pairs:
        description: "Number of A/B/B/A pairs"
        default: "14"
  issue_comment:
    types: [created]

permissions:
  contents: read
  pull-requests: write
  issues: write

concurrency:
  # See bench-verify.yml: this workflow fires on EVERY issue_comment, and GitHub
  # claims the concurrency group when the run is CREATED — before the job-level
  # `if` skips it. With the old plain per-issue group, any comment on the PR
  # evicted a running GPU ABBA mid-rental (2026-08-03: a `/bench 5` comment
  # killed the run started 20 minutes earlier, ~40 min of paid box). Real
  # /bench-gpu comments share the per-issue group, so a deliberate re-fire still
  # replaces a stale run (cancel-in-progress stays true for that case — one
  # rental per PR, newest request wins); every other comment and
  # workflow_dispatch falls to a throwaway group and cannot evict anything.
  group: ${{ startsWith(github.event.comment.body, '/bench-gpu') && format('benchmark-gpu-{0}', github.event.issue.number) || format('benchmark-gpu-ignore-{0}', github.run_id) }}
  cancel-in-progress: true

env:
  # Vast offer search: RTX 5090, >=16 cores, >=48GB RAM, >=64GB disk, verified +
  # rentable, Blackwell-capable driver, <= cap. gpu_frac=1 (whole-machine, dedicated
  # host) — see the query step for why.
  GPU_NAME: RTX_5090
  PRICE_CAP: "1"
  VAST_IMAGE_DISK: "64"
  # cli features for the ABBA build — the GPU (cuda) prover path plus jemalloc heap stats.
  BENCH_FEATURES: "jemalloc-stats,prover/cuda"
  # Continuation epoch for the REAL-BLOCK path, from the RTX 5090 calibration on
  # 2026-07-31 against main @9ccdaf2 (raw traces:
  # ~/workspace/lambda_vm_bench_cache/gpu_epoch_calib_2026-07-31/, PROVENANCE.txt).
  # Measured on the 32,607 MiB card, same fixture and CLI, one prove per setting:
  #
  #   2^21   70.52 s wall   19,193 MiB VRAM (58.9%)   25 epochs   1.65 GB proof
  #   2^22   59.87 s wall   23,193 MiB VRAM (71.1%)   13 epochs   1.12 GB proof
  #   2^23   OOM at 32,079 MiB (98.4%) after 9.7 s — needs ~44 GiB
  #
  # So 2^22 is the largest setting that fits a 32 GiB card, and it is ~15% faster than
  # 2^21 (equivalently, 2^21 is ~18% slower) with 28.9% VRAM headroom left. 2^23 is out
  # of reach for every card below 48 GiB, not just this one.
  #
  # GPU PATH ONLY, and deliberately so. It is NOT pushed into bench_abba.sh's default
  # (2^20, which the CPU /bench-abba uses) nor into the CLI's
  # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~32 GiB of HOST memory on
  # a CPU build (measured peak RSS on the 124 GiB calibration box; the CUDA path's host
  # peak is a different number, ~36 GB — see the cpu_ram floor below), which would break
  # laptops. VRAM is the binding constraint here and host RAM is the binding constraint
  # there, so the two defaults are not the same question.
  GPU_REAL_EPOCH_LOG2: "22"
  # Unique per-run label set on the instance, for easy identification in the Vast console.
  RUN_LABEL: "gpu-bench-${{ github.run_id }}-${{ github.run_attempt }}"
  # Pin the Vast CLI to an immutable commit (a PyPI version can be re-published; a commit
  # hash can't) — avoids pulling untrusted code at run time.
  VAST_CLI_COMMIT: "28494d92c6c03d887f8375085243c22eb68c5874"

jobs:
  benchmark-gpu:
    runs-on: ubuntu-latest
    # Skip unless: workflow_dispatch, or a "/bench-gpu" comment from a privileged author.
    if: >-
      github.event_name == 'workflow_dispatch' ||
      (github.event_name == 'issue_comment' &&
       github.event.issue.pull_request &&
       startsWith(github.event.comment.body, '/bench-gpu') &&
       contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association))
    # Provisioning + dual cuda build + 2*pairs real-block proves (~2 min each,
    # host-CPU-dependent). Sized for the 32-pair worst case with generous headroom;
    # teardown still always destroys the box.
    timeout-minutes: 330
    steps:
      - name: Resolve PR ref + bench config
        id: config
        env:
          GH_TOKEN: ${{ github.token }}
          EVENT_NAME: ${{ github.event_name }}
          COMMENT_BODY: ${{ github.event.comment.body }}
          PR_NUM: ${{ github.event.issue.number }}
          DISPATCH_PAIRS: ${{ github.event.inputs.pairs }}
          DISPATCH_REF: ${{ github.ref_name }}
        run: |
          if [ "$EVENT_NAME" = "issue_comment" ]; then
            # Pin the head SHA (works for fork PRs; avoids a force-push race mid-run).
            HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
            OUT_PR_NUM="$PR_NUM"; OUT_HEAD_SHA="$HEAD_SHA"; OUT_BRANCH=""
            # Everything after "/bench-gpu" on its line: the only token is an
            # optional pair count.
            ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-gpu||p' | head -n1)
            PAIRS=14
          else
            # workflow_dispatch: compare this branch vs main.
            OUT_PR_NUM=""; OUT_HEAD_SHA=""; OUT_BRANCH="$DISPATCH_REF"
            ARGS=""
            PAIRS=${DISPATCH_PAIRS:-14}
          fi
          set -f  # tokens must not glob-expand against the runner's CWD
          for tok in $ARGS; do
            case "$tok" in
              [0-9]*)     PAIRS="$tok" ;;
              *)          echo "::warning::ignoring unrecognized token '$tok'" ;;
            esac
          done
          # PAIRS is interpolated into the remote bash -lc below: enforce digits-only.
          case "$PAIRS" in
            ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 14"; PAIRS=14 ;;
          esac
          # Clamp to [2,32]; out-of-range -> default. 14 ~ resolves a 2% delta. The ceiling
          # keeps the worst-case run (64 proves + provisioning + dual build) under the job
          # timeout above.
          if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 32 ]; then
            echo "::warning::pair count out of range [2,32], defaulting to 14"
            PAIRS=14
          fi
          # Even is ideal so the AB/BA orders balance; round an odd request up by one.
          if [ "$((PAIRS % 2))" -ne 0 ]; then
            PAIRS=$((PAIRS + 1))
            echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance"
          fi
          WORKLOAD="ethrex real block, continuations"
          {
            echo "pr_num=$OUT_PR_NUM"
            echo "head_sha=$OUT_HEAD_SHA"
            echo "branch=$OUT_BRANCH"
            echo "pairs=$PAIRS"
            echo "workload=$WORKLOAD"
          } >> "$GITHUB_OUTPUT"
          echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD"

      - name: Acknowledge (react + occupancy notice)
        if: github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        env:
          PAIRS: ${{ steps.config.outputs.pairs }}
          WORKLOAD: ${{ steps.config.outputs.workload }}
        with:
          script: |
            await github.rest.reactions.createForIssueComment({
              owner: context.repo.owner, repo: context.repo.repo,
              comment_id: context.payload.comment.id, content: 'eyes'
            });
            // Post the "started" notice under the SAME marker the result step uses, so the
            // result updates this comment in place (and re-runs reuse it rather than stacking).
            const marker = 'GPU Benchmark (ABBA)';
            // Reference: 4 pairs measured 20 min 11 s end-to-end — 3 min 56 s of rental,
            // checkout and dual cuda build, then 4.06 min per pair, since a pair is TWO
            // proves at ~2 min each. Per-prove wall varies with the rented host's CPU
            // (the prover is partly host-CPU-bound), so the slope is the measured one
            // and the intercept carries slack for a colder box.
            const mins = 12 + Number(process.env.PAIRS) * 4;
            const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. Rough ETA ~${mins} min. The result will replace this comment.`;
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner, repo: context.repo.repo,
              issue_number: context.issue.number, per_page: 100,
            });
            const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner, repo: context.repo.repo,
                comment_id: existing.id, body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: context.issue.number, body,
              });
            }

      - name: Install Vast CLI
        # No secrets in this step's env: install-time code can't read the API key during pip
        # install. Pinned to an immutable commit (see VAST_CLI_COMMIT) for the same reason.
        # --break-system-packages: the ephemeral runner's Python may be PEP-668 "externally
        # managed"; safe to override on a disposable runner.
        run: pip install --quiet --break-system-packages "git+https://github.com/vast-ai/vast-cli.git@${VAST_CLI_COMMIT}"

      - name: Authenticate Vast CLI
        env:
          VAST_API_KEY: ${{ secrets.VAST_API_KEY }}
        run: vastai set api-key "$VAST_API_KEY"

      - name: Generate ephemeral SSH key
        id: sshkey
        run: |
          mkdir -p "$HOME/.ssh"
          KEY="$HOME/.ssh/vast_bench"
          ssh-keygen -t ed25519 -N "" -f "$KEY" -C "gh-actions-bench-${GITHUB_RUN_ID}" >/dev/null
          echo "key_path=$KEY" >> "$GITHUB_OUTPUT"

      - name: Pick a Vast offer
        id: offer
        env:
          # Retry the same query to ride out transient scarcity. Requiring gpu_frac=1
          # (dedicated host) shrinks the rentable pool (~7 vs ~28 fractional), so give it
          # more attempts to find a free whole-machine box. Total wait ~= ATTEMPTS * INTERVAL.
          OFFER_ATTEMPTS: "20"
          OFFER_INTERVAL: "30"
          # Require driver >= this major so cudarc (default cuda-version-from-build-system)
          # matches the runtime driver. Older drivers (e.g. 575) lack newer symbols like
          # cuCtxGetDevice_v2 and the GPU path falls back to CPU. Filtered client-side in jq
          # because vast can't numerically compare the driver_version string server-side.
          MIN_DRIVER: "580"
        run: |
          # cpu_ram filter is in GB. Floor 48 GB: the real block at epoch 2^22 peaks at
          # ~36 GB host RSS on the CUDA path (measured, main vintage) — ~25% headroom.
          # Continuation peak is set by the epoch size, not the block, so bigger blocks
          # don't move it; raising the epoch would (see the calibration tables in
          # tooling/ethrex-block-converter/README.md).
          # gpu_frac=1 requires a WHOLE-MACHINE offer (you rent every GPU on the host), so
          # Vast places no other tenant on the box: CPU cores, RAM/memory bandwidth, PCIe,
          # and NVMe are fully dedicated. Without it the "most expensive" sort below lands on
          # 1-of-8 slices (gpu_frac=0.125) on big multi-GPU servers — the GPU die is still
          # whole, but up to 7 noisy neighbors share the host CPU/PCIe and add per-pair
          # variance that ABBA pairing can't cancel (it's not static drift). Dedicated boxes
          # exist in the same pool, just priced lower per slot.
          QUERY="gpu_name=${GPU_NAME} num_gpus=1 gpu_frac=1 cpu_cores_effective>=16 cpu_cores_effective<=32 cpu_ram>=48 disk_space>=64 verified=true rentable=true cuda_max_good>=12.8 dph_total<=${PRICE_CAP}"
          echo "Query: $QUERY (+ client-side driver_version major >= $MIN_DRIVER)"
          # Keep only offers whose driver major >= MIN_DRIVER, then most expensive first
          # (within the price cap). Within the now whole-machine pool, price just tracks
          # core/RAM size; the priciest box gives the most headroom. The cheapest boxes were
          # flaky (slow image pulls, OOM), so bias high.
          # `try ... catch 0` so a malformed/null driver_version on one offer is treated as 0
          # (filtered out) rather than erroring the whole jq and wasting the attempt.
          SELECT="map(select((try (.driver_version|split(\".\")[0]|tonumber) catch 0) >= ${MIN_DRIVER})) | sort_by(.dph_total) | reverse"
          OFFER_ID=""
          for attempt in $(seq 1 "$OFFER_ATTEMPTS"); do
            vastai search offers "$QUERY" --raw -o dph_total > offers.json || true
            OFFER_ID=$(jq -r "$SELECT | .[0].id // empty" offers.json)
            OFFER_PRICE=$(jq -r "$SELECT | .[0].dph_total // empty" offers.json)
            if [ -n "$OFFER_ID" ]; then
              echo "Selected offer $OFFER_ID at \$${OFFER_PRICE}/hr (attempt $attempt)"
              break
            fi
            echo "No matching offer (attempt $attempt/$OFFER_ATTEMPTS); retrying in ${OFFER_INTERVAL}s..."
            sleep "$OFFER_INTERVAL"
          done
          if [ -z "$OFFER_ID" ]; then
            echo "::error::No RTX 5090 offer matched after $OFFER_ATTEMPTS attempts (>=16 cores, >=48GB RAM, >=64GB disk, driver>=${MIN_DRIVER}, <= \$${PRICE_CAP}/hr)"
            exit 1
          fi
          echo "id=$OFFER_ID" >> "$GITHUB_OUTPUT"
          echo "price=$OFFER_PRICE" >> "$GITHUB_OUTPUT"

      - name: Create instance
        id: instance
        env:
          VAST_TEMPLATE_HASH: ${{ secrets.VAST_TEMPLATE_HASH }}
          OFFER_ID: ${{ steps.offer.outputs.id }}
        run: |
          vastai create instance "$OFFER_ID" \
            --template_hash "$VAST_TEMPLATE_HASH" \
            --disk "$VAST_IMAGE_DISK" \
            --label "$RUN_LABEL" \
            --ssh --direct --raw > create.json
          # Log only the fields we need rather than the full --raw response, which could carry
          # an unexpected sensitive field into the (collaborator-/world-readable) run log.
          jq '{success, new_contract: (.new_contract // .instances.new_contract)}' create.json
          IID=$(jq -r '.new_contract // .instances.new_contract // empty' create.json)
          if [ -z "$IID" ]; then
            echo "::error::Failed to create Vast instance"
            exit 1
          fi
          # Persist immediately so teardown runs even if later steps fail.
          echo "$IID" > "$RUNNER_TEMP/vast_instance_id"
          echo "id=$IID" >> "$GITHUB_OUTPUT"
          echo "Created instance $IID (label $RUN_LABEL)"

      - name: Attach SSH key to instance
        env:
          IID: ${{ steps.instance.outputs.id }}
          KEY: ${{ steps.sshkey.outputs.key_path }}
        run: |
          # Attach the ephemeral pubkey to THIS instance only (added to its authorized_keys).
          # It's removed when the instance is destroyed, so no account-level key to clean up.
          # Retry: the instance may not accept the attach immediately after create.
          PUB="$(cat "$KEY.pub")"
          for attempt in $(seq 1 12); do
            if vastai attach ssh "$IID" "$PUB"; then
              echo "Attached ssh key (attempt $attempt)"; exit 0
            fi
            echo "attach failed (attempt $attempt/12); retrying in 10s..."
            sleep 10
          done
          echo "::error::Failed to attach ssh key to instance $IID"
          exit 1

      - name: Wait for SSH
        id: ssh
        env:
          IID: ${{ steps.instance.outputs.id }}
        run: |
          echo "Waiting for instance $IID to reach 'running' with SSH endpoint..."
          HOST=""; PORT=""
          # The base CUDA image is large; some hosts sit in 'loading' (image pull) a while.
          for _ in $(seq 1 180); do   # ~30 min
            vastai show instance "$IID" --raw > inst.json || true
            STATUS=$(jq -r '.actual_status // empty' inst.json)
            # We create with --direct, so SSH straight to the public IP + the host port
            # mapped to container port 22. The .ssh_host/.ssh_port proxy fields are
            # unreliable (observed off-by-one vs the real proxy port), so use the direct
            # mapping — same endpoint `vastai ssh-url` reports.
            HOST=$(jq -r '.public_ipaddr // empty' inst.json)
            PORT=$(jq -r '.ports["22/tcp"][0].HostPort // empty' inst.json)
            echo "  status=$STATUS ssh=$HOST:$PORT"
            if [ "$STATUS" = "running" ] && [ -n "$HOST" ] && [ -n "$PORT" ]; then
              break
            fi
            sleep 10
          done
          if [ "$STATUS" != "running" ] || [ -z "$HOST" ] || [ -z "$PORT" ]; then
            echo "::error::Instance never became reachable (status=$STATUS host=$HOST port=$PORT)"
            exit 1
          fi
          echo "host=$HOST" >> "$GITHUB_OUTPUT"
          echo "port=$PORT" >> "$GITHUB_OUTPUT"

          # Wait for sshd to accept our key.
          for _ in $(seq 1 30); do
            if ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes \
                 -i "${{ steps.sshkey.outputs.key_path }}" -p "$PORT" "root@$HOST" true 2>/dev/null; then
              echo "sshd reachable"; exit 0
            fi
            sleep 10
          done
          echo "::error::sshd did not accept connections in time"
          exit 1

      - name: Wait for onstart provisioning
        env:
          HOST: ${{ steps.ssh.outputs.host }}
          PORT: ${{ steps.ssh.outputs.port }}
          KEY: ${{ steps.sshkey.outputs.key_path }}
        run: |
          SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST"

          echo "Waiting for the template onstart script to finish (Rust + LLVM + sysroot + clone)..."
          # The bootstrap's final stdout line is "=== done ===". Vast captures onstart
          # output to /var/log/onstart.log; fall back to checking the artifacts it leaves.
          for _ in $(seq 1 120); do   # ~20 min
            if $SSH 'grep -q "=== done ===" /var/log/onstart.log 2>/dev/null'; then
              echo "onstart reported done"; exit 0
            fi
            # Fallback if the log marker isn't found: the late-stage artifacts (cargo + the
            # sysroot + the cloned repo) imply the earlier Rust/LLVM/toolchain install finished.
            # Deliberately no toolchain-date check — it would go stale when the repo bumps nightly.
            # shellcheck disable=SC2016  # $HOME must expand on the remote box, not the runner
            if $SSH 'test -x "$HOME/.cargo/bin/cargo" \
                  && test -f /opt/lambda-vm-sysroot/include/stdlib.h \
                  && test -d /workspace/lambda_vm/.git'; then
              echo "provisioning artifacts present"; exit 0
            fi
            sleep 10
          done
          echo "::error::onstart provisioning did not complete in time"
          exit 1

      - name: Run GPU ABBA benchmark
        id: bench
        env:
          HOST: ${{ steps.ssh.outputs.host }}
          PORT: ${{ steps.ssh.outputs.port }}
          KEY: ${{ steps.sshkey.outputs.key_path }}
          PR_NUM: ${{ steps.config.outputs.pr_num }}
          HEAD_SHA: ${{ steps.config.outputs.head_sha }}
          BRANCH: ${{ steps.config.outputs.branch }}
          PAIRS: ${{ steps.config.outputs.pairs }}
        run: |
          SSH="ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=10 -o BatchMode=yes -i $KEY -p $PORT root@$HOST"

          # Resolve the PR side (REF_A) and the fetch needed to make it resolvable on the box.
          if [ -n "$PR_NUM" ]; then
            FETCH="git fetch --force origin refs/pull/$PR_NUM/head"
            REF_A="$HEAD_SHA"
          else
            # Reject anything outside the git-ref-safe charset before it reaches the remote
            # `bash -lc` (defense-in-depth; workflow_dispatch is write-access only, but never
            # interpolate an unvalidated ref into a remote shell command).
            case "$BRANCH" in
              ''|*[!A-Za-z0-9._/-]*) echo "::error::invalid branch name: '$BRANCH'"; exit 1 ;;
            esac
            FETCH="git fetch --force origin $BRANCH"
            REF_A="origin/$BRANCH"
          fi

          # Run main's bench_abba.sh — the harness is the pinned measurement methodology, so a
          # PR can't alter how its own benchmark is computed. (The template clones the default
          # branch, so checking out origin/main is also what's already there; this makes it
          # explicit and robust to the template default changing.) The harness still builds the
          # cli at REF_A (the PR) and origin/main in isolated worktrees, runs PAIRS interleaved
          # A/B/B/A proves, and prints the paired-t CI + Wilcoxon verdict. BENCH_FEATURES routes
          # the build through the CUDA prover path. bench_abba.sh is shared with the CPU ABBA
          # flow and has its own defaults, so its knobs are pinned explicitly here: the real
          # block, continuations, the calibrated epoch.
          # The harness runs from main, so workflow/script changes take effect post-merge.
          # REBUILD=1: each Vast box is fresh, GPU-specific hardware — always rebuild both
          # binaries (cubin is compiled for the detected arch); never trust a cached binary.
          # CUDARC_PIN: compat shim for pre-pin baseline shas. cudarc's CUDA version is now pinned
          # permanently in crypto/math-cuda/Cargo.toml (cuda-12080), so this no-ops on shas that
          # carry the pin and only rewrites older baselines (where fallback-latest could request a
          # symbol the box's driver doesn't export, e.g. cuDevSmResourceSplit -> runtime panic).
          # MIN_DRIVER>=580 still guards the too-old end (older drivers lack cuCtxGetDevice_v2 and
          # the GPU path falls back to CPU). nvidia-smi is logged for diagnosing driver issues.
          REMOTE="set -e; cd /workspace/lambda_vm; \
            command -v python3 >/dev/null || { apt-get update -qq && apt-get install -y -qq python3; }; \
            nvidia-smi || true; \
            git fetch --force origin main; $FETCH; \
            git checkout -f origin/main; \
            REBUILD=1 CUDARC_PIN=cuda-12080 SYSROOT_DIR=/opt/lambda-vm-sysroot BENCH_FEATURES='$BENCH_FEATURES' \
            WORKLOAD=real CONTINUATIONS=1 EPOCH_SIZE_LOG2=$GPU_REAL_EPOCH_LOG2 \
            scripts/bench_abba.sh $REF_A origin/main $PAIRS"

          # Absolute seconds don't transfer between hosts (same-price 5090 rentals
          # span ~2x per prove with the host CPU; the paired Δ% cancels it), so every
          # number this run prints arrives with its host attached. Captured before
          # the bench so a failed run still records the host it failed on.
          $SSH "lscpu" || true
          CPU_MODEL=$($SSH "lscpu 2>/dev/null | sed -n 's/^Model name:[[:space:]]*//p' | head -1" 2>/dev/null || true)
          [ -n "$CPU_MODEL" ] || CPU_MODEL=$($SSH "sed -n 's/^model name[[:space:]]*: //p' /proc/cpuinfo | head -1" 2>/dev/null || true)
          CPU_THREADS=$($SSH "nproc" 2>/dev/null || true)
          echo "Host CPU: ${CPU_MODEL:-unknown} (${CPU_THREADS:-?} threads)"
          {
            echo "cpu_model=${CPU_MODEL}"
            echo "cpu_threads=${CPU_THREADS}"
          } >> "$GITHUB_OUTPUT"

          # pipefail so a failed remote bench (e.g. a prove that dies) propagates through the
          # tee pipe and fails this step, instead of being masked by tee's exit 0.
          set -o pipefail
          $SSH "bash -lc \"$REMOTE\"" | tee "$RUNNER_TEMP/abba_out.txt"
          # Extract the result section for the PR comment (same marker bench-abba.yml uses).
          sed -n '/=== ABBA paired result/,$p' "$RUNNER_TEMP/abba_out.txt" > "$RUNNER_TEMP/abba_result.txt"

      - name: Write run summary
        # Always run so a failure (incl. workflow_dispatch, which has no PR comment step) is
        # visible in the Actions run summary instead of only the raw step log.
        if: always() && (steps.bench.outcome == 'success' || steps.bench.outcome == 'failure')
        env:
          OUTCOME: ${{ steps.bench.outcome }}
          WORKLOAD: ${{ steps.config.outputs.workload }}
          CPU_MODEL: ${{ steps.bench.outputs.cpu_model }}
          CPU_THREADS: ${{ steps.bench.outputs.cpu_threads }}
        run: |
          {
            echo "## GPU ABBA — ${WORKLOAD:-ethrex} (vs main)"
            [ -n "$CPU_MODEL" ] && echo "Host: $CPU_MODEL (${CPU_THREADS:-?} threads)"
            if [ "$OUTCOME" = "success" ] && [ -s "$RUNNER_TEMP/abba_result.txt" ]; then
              echo '```'
              cat "$RUNNER_TEMP/abba_result.txt"
              echo '```'
            else
              echo "❌ Run outcome: ${OUTCOME:-unknown}. Last log lines:"
              echo '```'
              tail -n 30 "$RUNNER_TEMP/abba_out.txt" 2>/dev/null || echo "(no output captured)"
              echo '```'
            fi
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Comment ABBA result on PR
        if: always() && github.event_name == 'issue_comment'
        uses: actions/github-script@v7
        env:
          HEAD_SHA: ${{ steps.config.outputs.head_sha }}
          PAIRS: ${{ steps.config.outputs.pairs }}
          OUTCOME: ${{ steps.bench.outcome }}
          GPU_NAME: ${{ env.GPU_NAME }}
          OFFER_PRICE: ${{ steps.offer.outputs.price }}
          WORKLOAD: ${{ steps.config.outputs.workload }}
          CPU_MODEL: ${{ steps.bench.outputs.cpu_model }}
          CPU_THREADS: ${{ steps.bench.outputs.cpu_threads }}
        with:
          script: |
            const fs = require('fs');
            const tmp = process.env.RUNNER_TEMP;
            const read = (p) => { try { return fs.readFileSync(p, 'utf8').trim(); } catch { return ''; } };
            const head = (process.env.HEAD_SHA || '').slice(0, 10);
            const pairs = process.env.PAIRS;
            const gpu = (process.env.GPU_NAME || '').replace('_', ' ');
            const price = process.env.OFFER_PRICE;
            const workload = process.env.WORKLOAD || 'ethrex';
            // Absolute seconds vary ~2x with the rented host's CPU — name the host
            // they belong to (on failure too, so failures correlate with hosts).
            const cpuModel = process.env.CPU_MODEL;
            const cpuThreads = process.env.CPU_THREADS;
            const host = cpuModel ? ` · ${cpuModel}${cpuThreads ? ` (${cpuThreads} threads)` : ''}` : '';

            let body = `## GPU Benchmark (ABBA) — \`${head}\` vs \`main\` (${pairs} pairs)\n\n`;
            body += `<sub>${gpu}${host} · Vast.ai datacenter${price ? ` @ \$${price}/hr` : ''} · \`prover/cuda\` · ${workload} · drift-free A/B/B/A</sub>\n\n`;
            if (process.env.OUTCOME === 'success') {
              const res = read(`${tmp}/abba_result.txt`) || read(`${tmp}/abba_out.txt`);
              body += '```\n' + res + '\n```\n';
              body += '\n<sub>- = PR faster. Trust the verdict when paired-t and Wilcoxon agree.</sub>\n';
            } else {
              const tail = read(`${tmp}/abba_out.txt`).split('\n').slice(-30).join('\n');
              body += `❌ Run failed. Last log lines:\n\n` + '```\n' + tail + '\n```\n';
            }

            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner, repo: context.repo.repo,
              issue_number: context.issue.number, per_page: 100,
            });
            const marker = 'GPU Benchmark (ABBA)';
            const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(marker));
            if (existing) {
              await github.rest.issues.updateComment({
                owner: context.repo.owner, repo: context.repo.repo,
                comment_id: existing.id, body,
              });
            } else {
              await github.rest.issues.createComment({
                owner: context.repo.owner, repo: context.repo.repo,
                issue_number: context.issue.number, body,
              });
            }

      # --- Teardown: ALWAYS destroy the instance (cost guardrail) ---
      - name: Destroy instance
        if: always()
        run: |
          # Retry transient failures (network/auth) so a paid box isn't stranded.
          # --yes: skip the interactive [y/N] confirm (CI has no tty).
          destroy() {
            iid="$1"; destroyed=""
            for attempt in 1 2 3; do
              if vastai destroy instance "$iid" --yes; then destroyed=1; break; fi
              echo "destroy attempt $attempt failed; retrying in 10s..."
              sleep 10
            done
            [ -n "$destroyed" ] || echo "::warning::Failed to destroy instance $iid after 3 attempts — check the Vast console (label $RUN_LABEL)"
          }
          if [ -f "$RUNNER_TEMP/vast_instance_id" ]; then
            IID=$(cat "$RUNNER_TEMP/vast_instance_id")
            echo "Destroying instance $IID"
            destroy "$IID"
          else
            # The id file is written only AFTER create succeeds AND its JSON parses, so a box can
            # exist unrecorded if the run was cancelled in that window (concurrency cancel) or the
            # parse failed. Fall back to destroying by our unique RUN_LABEL so the box can't leak
            # (bill indefinitely). RUN_LABEL is unique per run, so this never touches another run's box.
            echo "No instance id recorded; searching Vast for any box labelled $RUN_LABEL..."
            vastai show instances --raw > all_inst.json 2>/dev/null || echo '[]' > all_inst.json
            # Tolerate either a bare array or {instances:[...]}; match our exact label.
            LEAKED=$(jq -r --arg L "$RUN_LABEL" \
              '(if type=="array" then . else (.instances // []) end) | .[] | select(.label == $L) | .id' \
              all_inst.json 2>/dev/null || true)
            if [ -z "$LEAKED" ]; then
              echo "No instance labelled $RUN_LABEL found; nothing to destroy."
            else
              for IID in $LEAKED; do
                echo "Destroying leaked instance $IID (label $RUN_LABEL)"
                destroy "$IID"
              done
            fi
          fi