From e5cbd4dda68d267ba14237e5e7b61c84293e3ed7 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 13:31:43 -0300 Subject: [PATCH 01/16] Add hyperfine benchmarks for CLI prove command --- .github/workflows/hyperfine_prove.yaml | 206 +++++++++++++++++++++++++ Makefile | 6 +- scripts/bench_prove.sh | 119 ++++++++++++++ 3 files changed, 330 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/hyperfine_prove.yaml create mode 100755 scripts/bench_prove.sh diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml new file mode 100644 index 000000000..04d1a6c37 --- /dev/null +++ b/.github/workflows/hyperfine_prove.yaml @@ -0,0 +1,206 @@ +name: Hyperfine Prove Benchmark + +on: + workflow_dispatch: + inputs: + program: + description: 'Bench program to prove (e.g. keccak)' + required: false + default: 'keccak' + runs: + description: 'Number of hyperfine runs' + required: false + default: '3' + pull_request: + types: [labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + BENCH_PROGRAM: ${{ github.event.inputs.program || 'keccak' }} + BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} + +jobs: + should-run: + name: Check trigger + runs-on: ubuntu-24.04 + outputs: + run: ${{ steps.check.outputs.run }} + steps: + - id: check + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + elif [ "${{ github.event.label.name }}" = "bench-prove" ]; then + echo "run=true" >> "$GITHUB_OUTPUT" + else + echo "run=false" >> "$GITHUB_OUTPUT" + fi + + build-program: + name: Build bench program + needs: should-run + if: needs.should-run.outputs.run == 'true' + runs-on: ubuntu-24.04 + outputs: + bench-hash: ${{ steps.export-hash.outputs.bench-hash }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Fetch from cache + uses: actions/cache@v4 + id: cache + with: + path: bench_programs/*.elf + key: bench-prove-${{ hashFiles('executor/programs/bench/*') }} + + - name: Setup Rust Environment + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-rust + + - name: Build programs + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bench_programs + make -j compile-bench + cp executor/program_artifacts/bench/*.elf bench_programs/ + + - name: Export bench hash + id: export-hash + run: echo "bench-hash=${{ hashFiles('executor/programs/bench/*') }}" >> "$GITHUB_OUTPUT" + + build-binaries: + strategy: + matrix: + branch: [base, head] + name: Build CLI for ${{ matrix.branch }} + needs: should-run + if: needs.should-run.outputs.run == 'true' + runs-on: ubuntu-24.04 + steps: + - name: Fetch from cache + uses: actions/cache@v4 + id: cache + with: + path: bench-bin/cli-${{ matrix.branch }} + key: binary-prove-${{ github.event.pull_request[matrix.branch].sha || github.sha }} + + - name: Checkout + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request[matrix.branch].sha || github.sha }} + + - name: Setup Rust Environment + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-rust + + - name: Fetch Rust cache + if: steps.cache.outputs.cache-hit != 'true' + uses: Swatinem/rust-cache@v2 + with: + shared-key: "lambda-vm" + + - name: Build binary + if: steps.cache.outputs.cache-hit != 'true' + run: | + cargo build --release --bin cli + mkdir -p bench-bin + cp target/release/cli bench-bin/cli-${{ matrix.branch }} + + run-prove-benchmark: + name: Run prove benchmark + needs: [build-program, build-binaries] + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - name: Install Hyperfine + uses: taiki-e/install-action@v2 + with: + tool: hyperfine@1.19 + + - name: Fetch base binary + uses: actions/cache/restore@v4 + with: + path: bench-bin/cli-base + key: binary-prove-${{ github.event.pull_request.base.sha || github.sha }} + + - name: Fetch head binary + uses: actions/cache/restore@v4 + with: + path: bench-bin/cli-head + key: binary-prove-${{ github.event.pull_request.head.sha || github.sha }} + + - name: Fetch bench programs + uses: actions/cache/restore@v4 + with: + path: bench_programs/*.elf + key: bench-prove-${{ needs.build-program.outputs.bench-hash }} + + - name: Benchmark prove + id: run-benchmark + run: | + sudo swapoff -a + chmod +x bench-bin/cli-base bench-bin/cli-head + + PROGRAM="${{ env.BENCH_PROGRAM }}" + ELF="bench_programs/${PROGRAM}.elf" + + if [ ! -f "$ELF" ]; then + echo "::error::Program $PROGRAM not found" + exit 1 + fi + + PROOF_BASE="/tmp/proof_base.cbor" + PROOF_HEAD="/tmp/proof_head.cbor" + + hyperfine \ + --warmup 0 \ + --runs "${{ env.BENCH_RUNS }}" \ + --prepare "rm -f $PROOF_BASE $PROOF_HEAD" \ + -n "base prove $PROGRAM" \ + "./bench-bin/cli-base prove $ELF --output $PROOF_BASE --security fast" \ + -n "head prove $PROGRAM" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ + --export-markdown results.md + + echo "has_results=true" >> "$GITHUB_OUTPUT" + + - name: Build comment body + if: steps.run-benchmark.outputs.has_results == 'true' + run: | + { + echo "Prove Benchmark Results :lock:" + echo "" + echo "Program: \`${{ env.BENCH_PROGRAM }}\` | Security: \`fast\` | Runs: \`${{ env.BENCH_RUNS }}\`" + echo "" + cat results.md + } > comment_body.md + + - name: Find comment + if: steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Prove Benchmark Results + + - name: Create comment + if: steps.fc.outputs.comment-id == '' && steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body-path: comment_body.md + + - name: Update comment + if: steps.fc.outputs.comment-id != '' && steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + body-path: comment_body.md + edit-mode: replace diff --git a/Makefile b/Makefile index 8ac4d5694..b0745dfd0 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: deps deps-linux deps-macos prepare-test-data compile-programs-asm compile-programs-rust compile-bench \ compile-programs clean-asm clean-rust clean-bench clean-shared clean test test-asm test-no-compile \ test-asm-no-compile test-rust test-rust-no-compile test-executor flamegraph-prover \ -test-fast test-prover test-prover-all build check +test-fast test-prover test-prover-all build check bench-prove UNAME := $(shell uname) @@ -170,5 +170,9 @@ build: check: cargo check --workspace +bench-prove: compile-bench + cargo build --release -p cli + ./scripts/bench_prove.sh $(BENCH_PROVE_PROGRAM) + flamegraph-prover: cd crypto/stark && samply record cargo bench --bench profile_prover --features parallel diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh new file mode 100755 index 000000000..894219923 --- /dev/null +++ b/scripts/bench_prove.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# +# Prove Benchmark Script +# Benchmarks the CLI prove command on the current branch using hyperfine. +# +# Usage: +# ./bench_prove.sh # Benchmark all bench programs +# ./bench_prove.sh # Benchmark a specific program (e.g. keccak) +# +# Environment variables: +# BENCH_PROVE_RUNS Number of hyperfine runs (default: 3) +# BENCH_PROVE_WARMUP Number of warmup runs (default: 0) +# BENCH_PROVE_SECURITY Security preset: fast|standard|maximum (default: fast) +# +# Requires: hyperfine +# + +set -euo pipefail + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TMP_DIR="/tmp/bench_prove" +BENCH_ARTIFACTS_DIR="$ROOT_DIR/executor/program_artifacts/bench" + +RUNS="${BENCH_PROVE_RUNS:-3}" +WARMUP="${BENCH_PROVE_WARMUP:-0}" +SECURITY="${BENCH_PROVE_SECURITY:-fast}" + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + echo "Prove Benchmark Script" + echo "Benchmarks the CLI prove command using hyperfine." + echo "" + echo "Usage:" + echo " ./bench_prove.sh # Benchmark all bench programs" + echo " ./bench_prove.sh # Benchmark a specific program (e.g. keccak)" + echo "" + echo "Environment variables:" + echo " BENCH_PROVE_RUNS Number of runs (default: 3)" + echo " BENCH_PROVE_WARMUP Number of warmup runs (default: 0)" + echo " BENCH_PROVE_SECURITY Security preset (default: fast)" + exit 0 +fi + +PROGRAM="${1:-}" + +# Validate security preset +case "$SECURITY" in + fast|standard|maximum) ;; + *) + echo -e "${RED}Error: Invalid security preset '$SECURITY'. Use fast, standard, or maximum.${NC}" + exit 1 + ;; +esac + +echo -e "${GREEN}=== Prove Benchmark ===${NC}" +echo -e "${YELLOW}Runs: $RUNS | Warmup: $WARMUP | Security: $SECURITY${NC}" + +# Find CLI binary +CLI="$ROOT_DIR/target/release/cli" +if [ ! -f "$CLI" ]; then + echo -e "${RED}Error: CLI binary not found at $CLI. Build with: cargo build --release -p cli${NC}" + exit 1 +fi + +# Collect ELFs to benchmark +if [ -n "$PROGRAM" ]; then + ELF="$BENCH_ARTIFACTS_DIR/$PROGRAM.elf" + if [ ! -f "$ELF" ]; then + echo -e "${RED}Error: Program '$PROGRAM' not found at $ELF${NC}" + exit 1 + fi + ELFS=("$ELF") +else + ELFS=("$BENCH_ARTIFACTS_DIR"/*.elf) + if [ ${#ELFS[@]} -eq 0 ]; then + echo -e "${RED}Error: No ELF files found in $BENCH_ARTIFACTS_DIR. Run: make compile-bench${NC}" + exit 1 + fi +fi + +# Setup output directory +mkdir -p "$TMP_DIR" + +# Run benchmarks +for elf in "${ELFS[@]}"; do + name=$(basename "$elf" .elf) + proof_file="$TMP_DIR/${name}_proof.cbor" + echo -e "${YELLOW}--- $name ---${NC}" + hyperfine \ + --warmup "$WARMUP" \ + --runs "$RUNS" \ + --prepare "rm -f $proof_file" \ + -n "prove $name ($SECURITY)" \ + "$CLI prove $elf --output $proof_file --security $SECURITY" \ + --export-markdown "$TMP_DIR/$name.md" \ + --export-json "$TMP_DIR/$name.json" +done + +# Summary +echo "" +echo -e "${GREEN}=== Results ===${NC}" +for elf in "${ELFS[@]}"; do + name=$(basename "$elf" .elf) + if [ -f "$TMP_DIR/$name.json" ]; then + mean=$(jq -r '.results[0].mean' "$TMP_DIR/$name.json") + stddev=$(jq -r '.results[0].stddev' "$TMP_DIR/$name.json") + printf "%-20s mean: %8.2fs stddev: %6.2fs\n" "$name" "$mean" "$stddev" + fi +done + +echo "" +echo -e "${GREEN}Markdown reports: $TMP_DIR/*.md${NC}" +echo -e "${GREEN}JSON results: $TMP_DIR/*.json${NC}" From a1de212a5a250d08691a0bf472d111d37af44dc3 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 15:01:08 -0300 Subject: [PATCH 02/16] Fix review issues: workflow_dispatch, nullglob, jq dep, binary verification --- .github/workflows/hyperfine_prove.yaml | 48 +++++++++++++++++++------- scripts/bench_prove.sh | 11 +++++- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 04d1a6c37..0e821d5e8 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -79,7 +79,7 @@ jobs: branch: [base, head] name: Build CLI for ${{ matrix.branch }} needs: should-run - if: needs.should-run.outputs.run == 'true' + if: needs.should-run.outputs.run == 'true' && (matrix.branch == 'head' || github.event_name != 'workflow_dispatch') runs-on: ubuntu-24.04 steps: - name: Fetch from cache @@ -124,10 +124,11 @@ jobs: tool: hyperfine@1.19 - name: Fetch base binary + if: github.event_name != 'workflow_dispatch' uses: actions/cache/restore@v4 with: path: bench-bin/cli-base - key: binary-prove-${{ github.event.pull_request.base.sha || github.sha }} + key: binary-prove-${{ github.event.pull_request.base.sha }} - name: Fetch head binary uses: actions/cache/restore@v4 @@ -141,11 +142,22 @@ jobs: path: bench_programs/*.elf key: bench-prove-${{ needs.build-program.outputs.bench-hash }} + - name: Verify binaries + run: | + if [ ! -f bench-bin/cli-head ]; then + echo "::error::Head binary not found - cache restore failed" + exit 1 + fi + if [ "${{ github.event_name }}" != "workflow_dispatch" ] && [ ! -f bench-bin/cli-base ]; then + echo "::error::Base binary not found - cache restore failed" + exit 1 + fi + chmod +x bench-bin/cli-* + - name: Benchmark prove id: run-benchmark run: | sudo swapoff -a - chmod +x bench-bin/cli-base bench-bin/cli-head PROGRAM="${{ env.BENCH_PROGRAM }}" ELF="bench_programs/${PROGRAM}.elf" @@ -155,18 +167,28 @@ jobs: exit 1 fi - PROOF_BASE="/tmp/proof_base.cbor" PROOF_HEAD="/tmp/proof_head.cbor" - hyperfine \ - --warmup 0 \ - --runs "${{ env.BENCH_RUNS }}" \ - --prepare "rm -f $PROOF_BASE $PROOF_HEAD" \ - -n "base prove $PROGRAM" \ - "./bench-bin/cli-base prove $ELF --output $PROOF_BASE --security fast" \ - -n "head prove $PROGRAM" \ - "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ - --export-markdown results.md + if [ -f bench-bin/cli-base ]; then + PROOF_BASE="/tmp/proof_base.cbor" + hyperfine \ + --warmup 0 \ + --runs "${{ env.BENCH_RUNS }}" \ + --prepare "rm -f $PROOF_BASE $PROOF_HEAD" \ + -n "base prove $PROGRAM" \ + "./bench-bin/cli-base prove $ELF --output $PROOF_BASE --security fast" \ + -n "head prove $PROGRAM" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ + --export-markdown results.md + else + hyperfine \ + --warmup 0 \ + --runs "${{ env.BENCH_RUNS }}" \ + --prepare "rm -f $PROOF_HEAD" \ + -n "prove $PROGRAM" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ + --export-markdown results.md + fi echo "has_results=true" >> "$GITHUB_OUTPUT" diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh index 894219923..c9f5336ee 100755 --- a/scripts/bench_prove.sh +++ b/scripts/bench_prove.sh @@ -12,11 +12,18 @@ # BENCH_PROVE_WARMUP Number of warmup runs (default: 0) # BENCH_PROVE_SECURITY Security preset: fast|standard|maximum (default: fast) # -# Requires: hyperfine +# Requires: hyperfine, jq # set -euo pipefail +for cmd in hyperfine jq; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: $cmd is required but not installed." + exit 1 + fi +done + # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -77,7 +84,9 @@ if [ -n "$PROGRAM" ]; then fi ELFS=("$ELF") else + shopt -s nullglob ELFS=("$BENCH_ARTIFACTS_DIR"/*.elf) + shopt -u nullglob if [ ${#ELFS[@]} -eq 0 ]; then echo -e "${RED}Error: No ELF files found in $BENCH_ARTIFACTS_DIR. Run: make compile-bench${NC}" exit 1 From a1444bd1d24c812f0ad32a91c71d3469009cd7f5 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 16:28:59 -0300 Subject: [PATCH 03/16] Split build-binaries into separate jobs to fix workflow_dispatch --- .github/workflows/hyperfine_prove.yaml | 64 ++++++++++++++++++++------ 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 0e821d5e8..b0462f393 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -29,15 +29,19 @@ jobs: runs-on: ubuntu-24.04 outputs: run: ${{ steps.check.outputs.run }} + is-comparison: ${{ steps.check.outputs.is-comparison }} steps: - id: check run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "run=true" >> "$GITHUB_OUTPUT" + echo "is-comparison=false" >> "$GITHUB_OUTPUT" elif [ "${{ github.event.label.name }}" = "bench-prove" ]; then echo "run=true" >> "$GITHUB_OUTPUT" + echo "is-comparison=true" >> "$GITHUB_OUTPUT" else echo "run=false" >> "$GITHUB_OUTPUT" + echo "is-comparison=false" >> "$GITHUB_OUTPUT" fi build-program: @@ -73,27 +77,60 @@ jobs: id: export-hash run: echo "bench-hash=${{ hashFiles('executor/programs/bench/*') }}" >> "$GITHUB_OUTPUT" - build-binaries: - strategy: - matrix: - branch: [base, head] - name: Build CLI for ${{ matrix.branch }} + build-base-binary: + name: Build CLI for base needs: should-run - if: needs.should-run.outputs.run == 'true' && (matrix.branch == 'head' || github.event_name != 'workflow_dispatch') + if: needs.should-run.outputs.run == 'true' && needs.should-run.outputs.is-comparison == 'true' runs-on: ubuntu-24.04 steps: - name: Fetch from cache uses: actions/cache@v4 id: cache with: - path: bench-bin/cli-${{ matrix.branch }} - key: binary-prove-${{ github.event.pull_request[matrix.branch].sha || github.sha }} + path: bench-bin/cli-base + key: binary-prove-${{ github.event.pull_request.base.sha }} + + - name: Checkout + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + + - name: Setup Rust Environment + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-rust + + - name: Fetch Rust cache + if: steps.cache.outputs.cache-hit != 'true' + uses: Swatinem/rust-cache@v2 + with: + shared-key: "lambda-vm" + + - name: Build binary + if: steps.cache.outputs.cache-hit != 'true' + run: | + cargo build --release --bin cli + mkdir -p bench-bin + cp target/release/cli bench-bin/cli-base + + build-head-binary: + name: Build CLI for head + needs: should-run + if: needs.should-run.outputs.run == 'true' + runs-on: ubuntu-24.04 + steps: + - name: Fetch from cache + uses: actions/cache@v4 + id: cache + with: + path: bench-bin/cli-head + key: binary-prove-${{ github.event.pull_request.head.sha || github.sha }} - name: Checkout if: steps.cache.outputs.cache-hit != 'true' uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request[matrix.branch].sha || github.sha }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Setup Rust Environment if: steps.cache.outputs.cache-hit != 'true' @@ -110,11 +147,12 @@ jobs: run: | cargo build --release --bin cli mkdir -p bench-bin - cp target/release/cli bench-bin/cli-${{ matrix.branch }} + cp target/release/cli bench-bin/cli-head run-prove-benchmark: name: Run prove benchmark - needs: [build-program, build-binaries] + needs: [should-run, build-program, build-base-binary, build-head-binary] + if: "!failure() && !cancelled()" runs-on: ubuntu-24.04 timeout-minutes: 120 steps: @@ -124,7 +162,7 @@ jobs: tool: hyperfine@1.19 - name: Fetch base binary - if: github.event_name != 'workflow_dispatch' + if: needs.should-run.outputs.is-comparison == 'true' uses: actions/cache/restore@v4 with: path: bench-bin/cli-base @@ -148,7 +186,7 @@ jobs: echo "::error::Head binary not found - cache restore failed" exit 1 fi - if [ "${{ github.event_name }}" != "workflow_dispatch" ] && [ ! -f bench-bin/cli-base ]; then + if [ "${{ needs.should-run.outputs.is-comparison }}" = "true" ] && [ ! -f bench-bin/cli-base ]; then echo "::error::Base binary not found - cache restore failed" exit 1 fi From fcd8bb233178e0995c58cb32cfab952ffc956322 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 16:41:25 -0300 Subject: [PATCH 04/16] Remove swapoff to avoid OOM kills during proving --- .github/workflows/hyperfine_prove.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index b0462f393..5e3ba0896 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -195,8 +195,6 @@ jobs: - name: Benchmark prove id: run-benchmark run: | - sudo swapoff -a - PROGRAM="${{ env.BENCH_PROGRAM }}" ELF="bench_programs/${PROGRAM}.elf" From e45433f6c3bf6f75dd5b03b34e51b3f555c41db4 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 16:57:08 -0300 Subject: [PATCH 05/16] Add dry run to debug prove failure in CI --- .github/workflows/hyperfine_prove.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 5e3ba0896..764b17a7a 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -205,6 +205,10 @@ jobs: PROOF_HEAD="/tmp/proof_head.cbor" + # Dry run to catch errors before hyperfine + ./bench-bin/cli-head prove "$ELF" --output /tmp/proof_dryrun.cbor --security fast + rm -f /tmp/proof_dryrun.cbor + if [ -f bench-bin/cli-base ]; then PROOF_BASE="/tmp/proof_base.cbor" hyperfine \ From 764299879b33e2da3d1d9469ed1e9b20b8435188 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 17:03:36 -0300 Subject: [PATCH 06/16] Remove dry run debug step --- .github/workflows/hyperfine_prove.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 764b17a7a..5e3ba0896 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -205,10 +205,6 @@ jobs: PROOF_HEAD="/tmp/proof_head.cbor" - # Dry run to catch errors before hyperfine - ./bench-bin/cli-head prove "$ELF" --output /tmp/proof_dryrun.cbor --security fast - rm -f /tmp/proof_dryrun.cbor - if [ -f bench-bin/cli-base ]; then PROOF_BASE="/tmp/proof_base.cbor" hyperfine \ From 5ef4d79971122ec4bcae9e3cc04892f6bc2ab963 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 18:29:18 -0300 Subject: [PATCH 07/16] Use vector as default prove benchmark program --- .github/workflows/hyperfine_prove.yaml | 13 +++++++------ scripts/bench_prove.sh | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 5e3ba0896..a346171dd 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -6,7 +6,7 @@ on: program: description: 'Bench program to prove (e.g. keccak)' required: false - default: 'keccak' + default: 'vector' runs: description: 'Number of hyperfine runs' required: false @@ -20,7 +20,7 @@ concurrency: env: CARGO_TERM_COLOR: always - BENCH_PROGRAM: ${{ github.event.inputs.program || 'keccak' }} + BENCH_PROGRAM: ${{ github.event.inputs.program || 'vector' }} BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} jobs: @@ -60,7 +60,7 @@ jobs: id: cache with: path: bench_programs/*.elf - key: bench-prove-${{ hashFiles('executor/programs/bench/*') }} + key: prove-programs-${{ hashFiles('executor/programs/rust/*') }} - name: Setup Rust Environment if: steps.cache.outputs.cache-hit != 'true' @@ -70,12 +70,13 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | mkdir -p bench_programs - make -j compile-bench - cp executor/program_artifacts/bench/*.elf bench_programs/ + make prepare-sysroot + make -j compile-programs-rust + cp executor/program_artifacts/rust/*.elf bench_programs/ - name: Export bench hash id: export-hash - run: echo "bench-hash=${{ hashFiles('executor/programs/bench/*') }}" >> "$GITHUB_OUTPUT" + run: echo "bench-hash=${{ hashFiles('executor/programs/rust/*') }}" >> "$GITHUB_OUTPUT" build-base-binary: name: Build CLI for base diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh index c9f5336ee..80f4d6c2f 100755 --- a/scripts/bench_prove.sh +++ b/scripts/bench_prove.sh @@ -5,7 +5,7 @@ # # Usage: # ./bench_prove.sh # Benchmark all bench programs -# ./bench_prove.sh # Benchmark a specific program (e.g. keccak) +# ./bench_prove.sh # Benchmark a specific program (e.g. vector) # # Environment variables: # BENCH_PROVE_RUNS Number of hyperfine runs (default: 3) From 12c47472632c8af6d70dd153787aea5e39edcdc6 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 18:47:13 -0300 Subject: [PATCH 08/16] Use ASM programs with bench_32k as default prove benchmark --- .github/workflows/hyperfine_prove.yaml | 17 ++++++++--------- Makefile | 2 +- scripts/bench_prove.sh | 8 ++++---- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index a346171dd..662d10095 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: program: - description: 'Bench program to prove (e.g. keccak)' + description: 'Bench program to prove (e.g. bench_32k)' required: false - default: 'vector' + default: 'bench_32k' runs: description: 'Number of hyperfine runs' required: false @@ -20,7 +20,7 @@ concurrency: env: CARGO_TERM_COLOR: always - BENCH_PROGRAM: ${{ github.event.inputs.program || 'vector' }} + BENCH_PROGRAM: ${{ github.event.inputs.program || 'bench_32k' }} BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} jobs: @@ -60,7 +60,7 @@ jobs: id: cache with: path: bench_programs/*.elf - key: prove-programs-${{ hashFiles('executor/programs/rust/*') }} + key: prove-programs-${{ hashFiles('executor/programs/asm/*') }} - name: Setup Rust Environment if: steps.cache.outputs.cache-hit != 'true' @@ -70,13 +70,12 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | mkdir -p bench_programs - make prepare-sysroot - make -j compile-programs-rust - cp executor/program_artifacts/rust/*.elf bench_programs/ + make -j compile-programs-asm + cp executor/program_artifacts/asm/*.elf bench_programs/ - name: Export bench hash id: export-hash - run: echo "bench-hash=${{ hashFiles('executor/programs/rust/*') }}" >> "$GITHUB_OUTPUT" + run: echo "bench-hash=${{ hashFiles('executor/programs/asm/*') }}" >> "$GITHUB_OUTPUT" build-base-binary: name: Build CLI for base @@ -179,7 +178,7 @@ jobs: uses: actions/cache/restore@v4 with: path: bench_programs/*.elf - key: bench-prove-${{ needs.build-program.outputs.bench-hash }} + key: prove-programs-${{ needs.build-program.outputs.bench-hash }} - name: Verify binaries run: | diff --git a/Makefile b/Makefile index b0745dfd0..0d12110a0 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ build: check: cargo check --workspace -bench-prove: compile-bench +bench-prove: compile-programs-asm cargo build --release -p cli ./scripts/bench_prove.sh $(BENCH_PROVE_PROGRAM) diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh index 80f4d6c2f..4ef590ecc 100755 --- a/scripts/bench_prove.sh +++ b/scripts/bench_prove.sh @@ -5,7 +5,7 @@ # # Usage: # ./bench_prove.sh # Benchmark all bench programs -# ./bench_prove.sh # Benchmark a specific program (e.g. vector) +# ./bench_prove.sh # Benchmark a specific program (e.g. bench_32k) # # Environment variables: # BENCH_PROVE_RUNS Number of hyperfine runs (default: 3) @@ -33,7 +33,7 @@ NC='\033[0m' SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" TMP_DIR="/tmp/bench_prove" -BENCH_ARTIFACTS_DIR="$ROOT_DIR/executor/program_artifacts/bench" +BENCH_ARTIFACTS_DIR="$ROOT_DIR/executor/program_artifacts/asm" RUNS="${BENCH_PROVE_RUNS:-3}" WARMUP="${BENCH_PROVE_WARMUP:-0}" @@ -45,7 +45,7 @@ if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then echo "" echo "Usage:" echo " ./bench_prove.sh # Benchmark all bench programs" - echo " ./bench_prove.sh # Benchmark a specific program (e.g. keccak)" + echo " ./bench_prove.sh # Benchmark a specific program (e.g. bench_32k)" echo "" echo "Environment variables:" echo " BENCH_PROVE_RUNS Number of runs (default: 3)" @@ -88,7 +88,7 @@ else ELFS=("$BENCH_ARTIFACTS_DIR"/*.elf) shopt -u nullglob if [ ${#ELFS[@]} -eq 0 ]; then - echo -e "${RED}Error: No ELF files found in $BENCH_ARTIFACTS_DIR. Run: make compile-bench${NC}" + echo -e "${RED}Error: No ELF files found in $BENCH_ARTIFACTS_DIR. Run: make compile-programs-asm${NC}" exit 1 fi fi From 86d3e19ffee6550b7068031df25067c7be88b548 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 18:57:26 -0300 Subject: [PATCH 09/16] Quote paths in hyperfine command to prevent shell injection --- scripts/bench_prove.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh index 4ef590ecc..c7cd7062f 100755 --- a/scripts/bench_prove.sh +++ b/scripts/bench_prove.sh @@ -104,9 +104,9 @@ for elf in "${ELFS[@]}"; do hyperfine \ --warmup "$WARMUP" \ --runs "$RUNS" \ - --prepare "rm -f $proof_file" \ + --prepare "rm -f '$proof_file'" \ -n "prove $name ($SECURITY)" \ - "$CLI prove $elf --output $proof_file --security $SECURITY" \ + "'$CLI' prove '$elf' --output '$proof_file' --security $SECURITY" \ --export-markdown "$TMP_DIR/$name.md" \ --export-json "$TMP_DIR/$name.json" done From 5a34396795c1cc99c1ae772c65e584b76fbe1004 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 19:41:20 -0300 Subject: [PATCH 10/16] Use all_instructions_64 as default prove benchmark --- .github/workflows/hyperfine_prove.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 662d10095..fc60eb81e 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: program: - description: 'Bench program to prove (e.g. bench_32k)' + description: 'Bench program to prove (e.g. all_instructions_64, bench_32k)' required: false - default: 'bench_32k' + default: 'all_instructions_64' runs: description: 'Number of hyperfine runs' required: false @@ -20,7 +20,7 @@ concurrency: env: CARGO_TERM_COLOR: always - BENCH_PROGRAM: ${{ github.event.inputs.program || 'bench_32k' }} + BENCH_PROGRAM: ${{ github.event.inputs.program || 'all_instructions_64' }} BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} jobs: From 817e71a1e9a453a8efcb999a3f496a79c47dd57d Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 20:13:17 -0300 Subject: [PATCH 11/16] Simplify workflow: matrix builds, artifacts, remove should-run job --- .github/workflows/hyperfine_prove.yaml | 140 +++++++------------------ 1 file changed, 36 insertions(+), 104 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index fc60eb81e..25c7dbc2d 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -24,33 +24,10 @@ env: BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} jobs: - should-run: - name: Check trigger - runs-on: ubuntu-24.04 - outputs: - run: ${{ steps.check.outputs.run }} - is-comparison: ${{ steps.check.outputs.is-comparison }} - steps: - - id: check - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - echo "run=true" >> "$GITHUB_OUTPUT" - echo "is-comparison=false" >> "$GITHUB_OUTPUT" - elif [ "${{ github.event.label.name }}" = "bench-prove" ]; then - echo "run=true" >> "$GITHUB_OUTPUT" - echo "is-comparison=true" >> "$GITHUB_OUTPUT" - else - echo "run=false" >> "$GITHUB_OUTPUT" - echo "is-comparison=false" >> "$GITHUB_OUTPUT" - fi - build-program: name: Build bench program - needs: should-run - if: needs.should-run.outputs.run == 'true' + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove' runs-on: ubuntu-24.04 - outputs: - bench-hash: ${{ steps.export-hash.outputs.bench-hash }} steps: - name: Checkout uses: actions/checkout@v4 @@ -73,85 +50,50 @@ jobs: make -j compile-programs-asm cp executor/program_artifacts/asm/*.elf bench_programs/ - - name: Export bench hash - id: export-hash - run: echo "bench-hash=${{ hashFiles('executor/programs/asm/*') }}" >> "$GITHUB_OUTPUT" + - name: Upload programs + uses: actions/upload-artifact@v4 + with: + name: bench-programs + path: bench_programs/*.elf - build-base-binary: - name: Build CLI for base - needs: should-run - if: needs.should-run.outputs.run == 'true' && needs.should-run.outputs.is-comparison == 'true' + build-binary: + strategy: + matrix: + branch: [base, head] + name: Build CLI for ${{ matrix.branch }} + if: >- + (github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove') && + (matrix.branch == 'head' || github.event.label.name == 'bench-prove') runs-on: ubuntu-24.04 steps: - - name: Fetch from cache - uses: actions/cache@v4 - id: cache - with: - path: bench-bin/cli-base - key: binary-prove-${{ github.event.pull_request.base.sha }} - - name: Checkout - if: steps.cache.outputs.cache-hit != 'true' uses: actions/checkout@v4 with: - ref: ${{ github.event.pull_request.base.sha }} + ref: ${{ matrix.branch == 'base' && github.event.pull_request.base.sha || github.event.pull_request.head.sha || github.sha }} - name: Setup Rust Environment - if: steps.cache.outputs.cache-hit != 'true' uses: ./.github/actions/setup-rust - name: Fetch Rust cache - if: steps.cache.outputs.cache-hit != 'true' uses: Swatinem/rust-cache@v2 with: shared-key: "lambda-vm" - name: Build binary - if: steps.cache.outputs.cache-hit != 'true' run: | cargo build --release --bin cli mkdir -p bench-bin - cp target/release/cli bench-bin/cli-base + cp target/release/cli bench-bin/cli-${{ matrix.branch }} - build-head-binary: - name: Build CLI for head - needs: should-run - if: needs.should-run.outputs.run == 'true' - runs-on: ubuntu-24.04 - steps: - - name: Fetch from cache - uses: actions/cache@v4 - id: cache - with: - path: bench-bin/cli-head - key: binary-prove-${{ github.event.pull_request.head.sha || github.sha }} - - - name: Checkout - if: steps.cache.outputs.cache-hit != 'true' - uses: actions/checkout@v4 + - name: Upload binary + uses: actions/upload-artifact@v4 with: - ref: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Setup Rust Environment - if: steps.cache.outputs.cache-hit != 'true' - uses: ./.github/actions/setup-rust - - - name: Fetch Rust cache - if: steps.cache.outputs.cache-hit != 'true' - uses: Swatinem/rust-cache@v2 - with: - shared-key: "lambda-vm" - - - name: Build binary - if: steps.cache.outputs.cache-hit != 'true' - run: | - cargo build --release --bin cli - mkdir -p bench-bin - cp target/release/cli bench-bin/cli-head + name: cli-${{ matrix.branch }} + path: bench-bin/cli-${{ matrix.branch }} run-prove-benchmark: name: Run prove benchmark - needs: [should-run, build-program, build-base-binary, build-head-binary] + needs: [build-program, build-binary] if: "!failure() && !cancelled()" runs-on: ubuntu-24.04 timeout-minutes: 120 @@ -161,40 +103,30 @@ jobs: with: tool: hyperfine@1.19 - - name: Fetch base binary - if: needs.should-run.outputs.is-comparison == 'true' - uses: actions/cache/restore@v4 + - name: Download head binary + uses: actions/download-artifact@v4 with: - path: bench-bin/cli-base - key: binary-prove-${{ github.event.pull_request.base.sha }} + name: cli-head + path: bench-bin - - name: Fetch head binary - uses: actions/cache/restore@v4 + - name: Download base binary + if: github.event.label.name == 'bench-prove' + uses: actions/download-artifact@v4 with: - path: bench-bin/cli-head - key: binary-prove-${{ github.event.pull_request.head.sha || github.sha }} + name: cli-base + path: bench-bin - - name: Fetch bench programs - uses: actions/cache/restore@v4 + - name: Download bench programs + uses: actions/download-artifact@v4 with: - path: bench_programs/*.elf - key: prove-programs-${{ needs.build-program.outputs.bench-hash }} - - - name: Verify binaries - run: | - if [ ! -f bench-bin/cli-head ]; then - echo "::error::Head binary not found - cache restore failed" - exit 1 - fi - if [ "${{ needs.should-run.outputs.is-comparison }}" = "true" ] && [ ! -f bench-bin/cli-base ]; then - echo "::error::Base binary not found - cache restore failed" - exit 1 - fi - chmod +x bench-bin/cli-* + name: bench-programs + path: bench_programs - name: Benchmark prove id: run-benchmark run: | + chmod +x bench-bin/cli-* + PROGRAM="${{ env.BENCH_PROGRAM }}" ELF="bench_programs/${PROGRAM}.elf" From b46c642527a6d2eb86575e71c53899deea2a4456 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 6 Feb 2026 20:16:50 -0300 Subject: [PATCH 12/16] Split matrix into separate jobs (matrix context not allowed in job-level if) --- .github/workflows/hyperfine_prove.yaml | 51 +++++++++++++++++++------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index 25c7dbc2d..a7312b230 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -56,20 +56,15 @@ jobs: name: bench-programs path: bench_programs/*.elf - build-binary: - strategy: - matrix: - branch: [base, head] - name: Build CLI for ${{ matrix.branch }} - if: >- - (github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove') && - (matrix.branch == 'head' || github.event.label.name == 'bench-prove') + build-base-binary: + name: Build CLI for base + if: github.event.label.name == 'bench-prove' runs-on: ubuntu-24.04 steps: - name: Checkout uses: actions/checkout@v4 with: - ref: ${{ matrix.branch == 'base' && github.event.pull_request.base.sha || github.event.pull_request.head.sha || github.sha }} + ref: ${{ github.event.pull_request.base.sha }} - name: Setup Rust Environment uses: ./.github/actions/setup-rust @@ -83,17 +78,47 @@ jobs: run: | cargo build --release --bin cli mkdir -p bench-bin - cp target/release/cli bench-bin/cli-${{ matrix.branch }} + cp target/release/cli bench-bin/cli-base - name: Upload binary uses: actions/upload-artifact@v4 with: - name: cli-${{ matrix.branch }} - path: bench-bin/cli-${{ matrix.branch }} + name: cli-base + path: bench-bin/cli-base + + build-head-binary: + name: Build CLI for head + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Setup Rust Environment + uses: ./.github/actions/setup-rust + + - name: Fetch Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "lambda-vm" + + - name: Build binary + run: | + cargo build --release --bin cli + mkdir -p bench-bin + cp target/release/cli bench-bin/cli-head + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: cli-head + path: bench-bin/cli-head run-prove-benchmark: name: Run prove benchmark - needs: [build-program, build-binary] + needs: [build-program, build-base-binary, build-head-binary] if: "!failure() && !cancelled()" runs-on: ubuntu-24.04 timeout-minutes: 120 From fe6e0fbd4b156a6213c8924f09dae065d75d190a Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Mon, 9 Feb 2026 11:23:46 -0300 Subject: [PATCH 13/16] Only comment on PR when prove benchmark regresses >5% --- .github/workflows/hyperfine_prove.yaml | 32 ++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index a7312b230..ebe3d9d8f 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -172,7 +172,8 @@ jobs: "./bench-bin/cli-base prove $ELF --output $PROOF_BASE --security fast" \ -n "head prove $PROGRAM" \ "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ - --export-markdown results.md + --export-markdown results.md \ + --export-json results.json else hyperfine \ --warmup 0 \ @@ -185,8 +186,29 @@ jobs: echo "has_results=true" >> "$GITHUB_OUTPUT" - - name: Build comment body + - name: Detect regression + id: detect-regression if: steps.run-benchmark.outputs.has_results == 'true' + run: | + if [ ! -f results.json ]; then + echo "No base vs head comparison (dispatch-only run), skipping regression check" + exit 0 + fi + + BASE_MEAN=$(jq '.results[0].mean' results.json) + HEAD_MEAN=$(jq '.results[1].mean' results.json) + + # Flag regression if head is >5% slower than base + REGRESSED=$(awk -v bm="$BASE_MEAN" -v hm="$HEAD_MEAN" 'BEGIN { + pct = ((hm - bm) / bm) * 100 + printf "base=%.2fs head=%.2fs change=%+.1f%%\n", bm, hm, pct > "/dev/stderr" + print (pct > 5 ? "true" : "false") + }') + + echo "regressed=$REGRESSED" >> "$GITHUB_OUTPUT" + + - name: Build comment body + if: steps.detect-regression.outputs.regressed == 'true' run: | { echo "Prove Benchmark Results :lock:" @@ -197,7 +219,7 @@ jobs: } > comment_body.md - name: Find comment - if: steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + if: steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number uses: peter-evans/find-comment@v3 id: fc with: @@ -206,14 +228,14 @@ jobs: body-includes: Prove Benchmark Results - name: Create comment - if: steps.fc.outputs.comment-id == '' && steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + if: steps.fc.outputs.comment-id == '' && steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number uses: peter-evans/create-or-update-comment@v4 with: issue-number: ${{ github.event.pull_request.number }} body-path: comment_body.md - name: Update comment - if: steps.fc.outputs.comment-id != '' && steps.run-benchmark.outputs.has_results == 'true' && github.event.pull_request.number + if: steps.fc.outputs.comment-id != '' && steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number uses: peter-evans/create-or-update-comment@v4 with: comment-id: ${{ steps.fc.outputs.comment-id }} From 8297738168d362b5498b54d1b0e69e3e7e9bc5aa Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Mon, 9 Feb 2026 11:45:59 -0300 Subject: [PATCH 14/16] Fix workflow_dispatch path and division by zero in regression check --- .github/workflows/hyperfine_prove.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index ebe3d9d8f..eacc204d8 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -119,7 +119,7 @@ jobs: run-prove-benchmark: name: Run prove benchmark needs: [build-program, build-base-binary, build-head-binary] - if: "!failure() && !cancelled()" + if: always() && needs.build-program.result == 'success' && needs.build-head-binary.result == 'success' && needs.build-base-binary.result != 'failure' runs-on: ubuntu-24.04 timeout-minutes: 120 steps: @@ -200,6 +200,7 @@ jobs: # Flag regression if head is >5% slower than base REGRESSED=$(awk -v bm="$BASE_MEAN" -v hm="$HEAD_MEAN" 'BEGIN { + if (bm <= 0) { print "false"; exit } pct = ((hm - bm) / bm) * 100 printf "base=%.2fs head=%.2fs change=%+.1f%%\n", bm, hm, pct > "/dev/stderr" print (pct > 5 ? "true" : "false") From 594ba6f91e9d76278332b8b00d0a560abe5bdef7 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Mon, 9 Feb 2026 12:38:01 -0300 Subject: [PATCH 15/16] Set 1-day retention on ephemeral benchmark artifacts --- .github/workflows/hyperfine_prove.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index eacc204d8..e206ffd08 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -55,6 +55,7 @@ jobs: with: name: bench-programs path: bench_programs/*.elf + retention-days: 1 build-base-binary: name: Build CLI for base @@ -85,6 +86,7 @@ jobs: with: name: cli-base path: bench-bin/cli-base + retention-days: 1 build-head-binary: name: Build CLI for head @@ -115,6 +117,7 @@ jobs: with: name: cli-head path: bench-bin/cli-head + retention-days: 1 run-prove-benchmark: name: Run prove benchmark From 3116ec090c9c03138dedd7da8b09327c207782d1 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Mon, 9 Feb 2026 15:44:03 -0300 Subject: [PATCH 16/16] Remove --security flag from bench prove scripts --- .github/workflows/hyperfine_prove.yaml | 8 ++++---- scripts/bench_prove.sh | 18 +++--------------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml index e206ffd08..e2fd3e8e2 100644 --- a/.github/workflows/hyperfine_prove.yaml +++ b/.github/workflows/hyperfine_prove.yaml @@ -172,9 +172,9 @@ jobs: --runs "${{ env.BENCH_RUNS }}" \ --prepare "rm -f $PROOF_BASE $PROOF_HEAD" \ -n "base prove $PROGRAM" \ - "./bench-bin/cli-base prove $ELF --output $PROOF_BASE --security fast" \ + "./bench-bin/cli-base prove $ELF --output $PROOF_BASE" \ -n "head prove $PROGRAM" \ - "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD" \ --export-markdown results.md \ --export-json results.json else @@ -183,7 +183,7 @@ jobs: --runs "${{ env.BENCH_RUNS }}" \ --prepare "rm -f $PROOF_HEAD" \ -n "prove $PROGRAM" \ - "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD --security fast" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD" \ --export-markdown results.md fi @@ -217,7 +217,7 @@ jobs: { echo "Prove Benchmark Results :lock:" echo "" - echo "Program: \`${{ env.BENCH_PROGRAM }}\` | Security: \`fast\` | Runs: \`${{ env.BENCH_RUNS }}\`" + echo "Program: \`${{ env.BENCH_PROGRAM }}\` | Runs: \`${{ env.BENCH_RUNS }}\`" echo "" cat results.md } > comment_body.md diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh index c7cd7062f..37749c3cb 100755 --- a/scripts/bench_prove.sh +++ b/scripts/bench_prove.sh @@ -10,7 +10,6 @@ # Environment variables: # BENCH_PROVE_RUNS Number of hyperfine runs (default: 3) # BENCH_PROVE_WARMUP Number of warmup runs (default: 0) -# BENCH_PROVE_SECURITY Security preset: fast|standard|maximum (default: fast) # # Requires: hyperfine, jq # @@ -37,7 +36,6 @@ BENCH_ARTIFACTS_DIR="$ROOT_DIR/executor/program_artifacts/asm" RUNS="${BENCH_PROVE_RUNS:-3}" WARMUP="${BENCH_PROVE_WARMUP:-0}" -SECURITY="${BENCH_PROVE_SECURITY:-fast}" if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then echo "Prove Benchmark Script" @@ -50,23 +48,13 @@ if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then echo "Environment variables:" echo " BENCH_PROVE_RUNS Number of runs (default: 3)" echo " BENCH_PROVE_WARMUP Number of warmup runs (default: 0)" - echo " BENCH_PROVE_SECURITY Security preset (default: fast)" exit 0 fi PROGRAM="${1:-}" -# Validate security preset -case "$SECURITY" in - fast|standard|maximum) ;; - *) - echo -e "${RED}Error: Invalid security preset '$SECURITY'. Use fast, standard, or maximum.${NC}" - exit 1 - ;; -esac - echo -e "${GREEN}=== Prove Benchmark ===${NC}" -echo -e "${YELLOW}Runs: $RUNS | Warmup: $WARMUP | Security: $SECURITY${NC}" +echo -e "${YELLOW}Runs: $RUNS | Warmup: $WARMUP${NC}" # Find CLI binary CLI="$ROOT_DIR/target/release/cli" @@ -105,8 +93,8 @@ for elf in "${ELFS[@]}"; do --warmup "$WARMUP" \ --runs "$RUNS" \ --prepare "rm -f '$proof_file'" \ - -n "prove $name ($SECURITY)" \ - "'$CLI' prove '$elf' --output '$proof_file' --security $SECURITY" \ + -n "prove $name" \ + "'$CLI' prove '$elf' --output '$proof_file'" \ --export-markdown "$TMP_DIR/$name.md" \ --export-json "$TMP_DIR/$name.json" done