diff --git a/.github/workflows/hyperfine_prove.yaml b/.github/workflows/hyperfine_prove.yaml new file mode 100644 index 000000000..e2fd3e8e2 --- /dev/null +++ b/.github/workflows/hyperfine_prove.yaml @@ -0,0 +1,247 @@ +name: Hyperfine Prove Benchmark + +on: + workflow_dispatch: + inputs: + program: + description: 'Bench program to prove (e.g. all_instructions_64, bench_32k)' + required: false + default: 'all_instructions_64' + runs: + description: 'Number of hyperfine runs' + required: false + default: '3' + pull_request: + types: [labeled] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + BENCH_PROGRAM: ${{ github.event.inputs.program || 'all_instructions_64' }} + BENCH_RUNS: ${{ github.event.inputs.runs || '3' }} + +jobs: + build-program: + name: Build bench program + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Fetch from cache + uses: actions/cache@v4 + id: cache + with: + path: bench_programs/*.elf + key: prove-programs-${{ hashFiles('executor/programs/asm/*') }} + + - name: Setup Rust Environment + if: steps.cache.outputs.cache-hit != 'true' + uses: ./.github/actions/setup-rust + + - name: Build programs + if: steps.cache.outputs.cache-hit != 'true' + run: | + mkdir -p bench_programs + make -j compile-programs-asm + cp executor/program_artifacts/asm/*.elf bench_programs/ + + - name: Upload programs + uses: actions/upload-artifact@v4 + with: + name: bench-programs + path: bench_programs/*.elf + retention-days: 1 + + build-base-binary: + name: Build CLI for base + if: github.event.label.name == 'bench-prove' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + + - name: Setup Rust Environment + uses: ./.github/actions/setup-rust + + - name: Fetch Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "lambda-vm" + + - name: Build binary + run: | + cargo build --release --bin cli + mkdir -p bench-bin + cp target/release/cli bench-bin/cli-base + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: cli-base + path: bench-bin/cli-base + retention-days: 1 + + build-head-binary: + name: Build CLI for head + if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'bench-prove' + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Setup Rust Environment + uses: ./.github/actions/setup-rust + + - name: Fetch Rust cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "lambda-vm" + + - name: Build binary + run: | + cargo build --release --bin cli + mkdir -p bench-bin + cp target/release/cli bench-bin/cli-head + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: cli-head + path: bench-bin/cli-head + retention-days: 1 + + run-prove-benchmark: + name: Run prove benchmark + needs: [build-program, build-base-binary, build-head-binary] + if: always() && needs.build-program.result == 'success' && needs.build-head-binary.result == 'success' && needs.build-base-binary.result != 'failure' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + steps: + - name: Install Hyperfine + uses: taiki-e/install-action@v2 + with: + tool: hyperfine@1.19 + + - name: Download head binary + uses: actions/download-artifact@v4 + with: + name: cli-head + path: bench-bin + + - name: Download base binary + if: github.event.label.name == 'bench-prove' + uses: actions/download-artifact@v4 + with: + name: cli-base + path: bench-bin + + - name: Download bench programs + uses: actions/download-artifact@v4 + with: + name: bench-programs + path: bench_programs + + - name: Benchmark prove + id: run-benchmark + run: | + chmod +x bench-bin/cli-* + + PROGRAM="${{ env.BENCH_PROGRAM }}" + ELF="bench_programs/${PROGRAM}.elf" + + if [ ! -f "$ELF" ]; then + echo "::error::Program $PROGRAM not found" + exit 1 + fi + + PROOF_HEAD="/tmp/proof_head.cbor" + + if [ -f bench-bin/cli-base ]; then + PROOF_BASE="/tmp/proof_base.cbor" + hyperfine \ + --warmup 0 \ + --runs "${{ env.BENCH_RUNS }}" \ + --prepare "rm -f $PROOF_BASE $PROOF_HEAD" \ + -n "base prove $PROGRAM" \ + "./bench-bin/cli-base prove $ELF --output $PROOF_BASE" \ + -n "head prove $PROGRAM" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD" \ + --export-markdown results.md \ + --export-json results.json + else + hyperfine \ + --warmup 0 \ + --runs "${{ env.BENCH_RUNS }}" \ + --prepare "rm -f $PROOF_HEAD" \ + -n "prove $PROGRAM" \ + "./bench-bin/cli-head prove $ELF --output $PROOF_HEAD" \ + --export-markdown results.md + fi + + echo "has_results=true" >> "$GITHUB_OUTPUT" + + - name: Detect regression + id: detect-regression + if: steps.run-benchmark.outputs.has_results == 'true' + run: | + if [ ! -f results.json ]; then + echo "No base vs head comparison (dispatch-only run), skipping regression check" + exit 0 + fi + + BASE_MEAN=$(jq '.results[0].mean' results.json) + HEAD_MEAN=$(jq '.results[1].mean' results.json) + + # Flag regression if head is >5% slower than base + REGRESSED=$(awk -v bm="$BASE_MEAN" -v hm="$HEAD_MEAN" 'BEGIN { + if (bm <= 0) { print "false"; exit } + pct = ((hm - bm) / bm) * 100 + printf "base=%.2fs head=%.2fs change=%+.1f%%\n", bm, hm, pct > "/dev/stderr" + print (pct > 5 ? "true" : "false") + }') + + echo "regressed=$REGRESSED" >> "$GITHUB_OUTPUT" + + - name: Build comment body + if: steps.detect-regression.outputs.regressed == 'true' + run: | + { + echo "Prove Benchmark Results :lock:" + echo "" + echo "Program: \`${{ env.BENCH_PROGRAM }}\` | Runs: \`${{ env.BENCH_RUNS }}\`" + echo "" + cat results.md + } > comment_body.md + + - name: Find comment + if: steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number + uses: peter-evans/find-comment@v3 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: "github-actions[bot]" + body-includes: Prove Benchmark Results + + - name: Create comment + if: steps.fc.outputs.comment-id == '' && steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number + uses: peter-evans/create-or-update-comment@v4 + with: + issue-number: ${{ github.event.pull_request.number }} + body-path: comment_body.md + + - name: Update comment + if: steps.fc.outputs.comment-id != '' && steps.detect-regression.outputs.regressed == 'true' && github.event.pull_request.number + uses: peter-evans/create-or-update-comment@v4 + with: + comment-id: ${{ steps.fc.outputs.comment-id }} + body-path: comment_body.md + edit-mode: replace diff --git a/Makefile b/Makefile index c02bffc49..7627a06e1 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: deps deps-linux deps-macos prepare-test-data compile-programs-asm compile-programs-rust compile-bench \ compile-programs clean-asm clean-rust clean-bench clean-shared clean test test-asm test-no-compile \ test-asm-no-compile test-rust test-rust-no-compile test-executor flamegraph-prover \ -test-fast test-prover test-prover-all build check clippy fmt lint +test-fast test-prover test-prover-all build check bench-prove clippy fmt lint UNAME := $(shell uname) @@ -174,6 +174,10 @@ build: check: cargo check --workspace +bench-prove: compile-programs-asm + cargo build --release -p cli + ./scripts/bench_prove.sh $(BENCH_PROVE_PROGRAM) + # === Linting === # op_ref: We pass big integers (U256/U384) and field elements by reference since operator # impls delegate to &self internally, avoiding unnecessary 32-48 byte copies. diff --git a/scripts/bench_prove.sh b/scripts/bench_prove.sh new file mode 100755 index 000000000..37749c3cb --- /dev/null +++ b/scripts/bench_prove.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Prove Benchmark Script +# Benchmarks the CLI prove command on the current branch using hyperfine. +# +# Usage: +# ./bench_prove.sh # Benchmark all bench programs +# ./bench_prove.sh # Benchmark a specific program (e.g. bench_32k) +# +# Environment variables: +# BENCH_PROVE_RUNS Number of hyperfine runs (default: 3) +# BENCH_PROVE_WARMUP Number of warmup runs (default: 0) +# +# Requires: hyperfine, jq +# + +set -euo pipefail + +for cmd in hyperfine jq; do + if ! command -v "$cmd" &>/dev/null; then + echo "Error: $cmd is required but not installed." + exit 1 + fi +done + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +TMP_DIR="/tmp/bench_prove" +BENCH_ARTIFACTS_DIR="$ROOT_DIR/executor/program_artifacts/asm" + +RUNS="${BENCH_PROVE_RUNS:-3}" +WARMUP="${BENCH_PROVE_WARMUP:-0}" + +if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then + echo "Prove Benchmark Script" + echo "Benchmarks the CLI prove command using hyperfine." + echo "" + echo "Usage:" + echo " ./bench_prove.sh # Benchmark all bench programs" + echo " ./bench_prove.sh # Benchmark a specific program (e.g. bench_32k)" + echo "" + echo "Environment variables:" + echo " BENCH_PROVE_RUNS Number of runs (default: 3)" + echo " BENCH_PROVE_WARMUP Number of warmup runs (default: 0)" + exit 0 +fi + +PROGRAM="${1:-}" + +echo -e "${GREEN}=== Prove Benchmark ===${NC}" +echo -e "${YELLOW}Runs: $RUNS | Warmup: $WARMUP${NC}" + +# Find CLI binary +CLI="$ROOT_DIR/target/release/cli" +if [ ! -f "$CLI" ]; then + echo -e "${RED}Error: CLI binary not found at $CLI. Build with: cargo build --release -p cli${NC}" + exit 1 +fi + +# Collect ELFs to benchmark +if [ -n "$PROGRAM" ]; then + ELF="$BENCH_ARTIFACTS_DIR/$PROGRAM.elf" + if [ ! -f "$ELF" ]; then + echo -e "${RED}Error: Program '$PROGRAM' not found at $ELF${NC}" + exit 1 + fi + ELFS=("$ELF") +else + shopt -s nullglob + ELFS=("$BENCH_ARTIFACTS_DIR"/*.elf) + shopt -u nullglob + if [ ${#ELFS[@]} -eq 0 ]; then + echo -e "${RED}Error: No ELF files found in $BENCH_ARTIFACTS_DIR. Run: make compile-programs-asm${NC}" + exit 1 + fi +fi + +# Setup output directory +mkdir -p "$TMP_DIR" + +# Run benchmarks +for elf in "${ELFS[@]}"; do + name=$(basename "$elf" .elf) + proof_file="$TMP_DIR/${name}_proof.cbor" + echo -e "${YELLOW}--- $name ---${NC}" + hyperfine \ + --warmup "$WARMUP" \ + --runs "$RUNS" \ + --prepare "rm -f '$proof_file'" \ + -n "prove $name" \ + "'$CLI' prove '$elf' --output '$proof_file'" \ + --export-markdown "$TMP_DIR/$name.md" \ + --export-json "$TMP_DIR/$name.json" +done + +# Summary +echo "" +echo -e "${GREEN}=== Results ===${NC}" +for elf in "${ELFS[@]}"; do + name=$(basename "$elf" .elf) + if [ -f "$TMP_DIR/$name.json" ]; then + mean=$(jq -r '.results[0].mean' "$TMP_DIR/$name.json") + stddev=$(jq -r '.results[0].stddev' "$TMP_DIR/$name.json") + printf "%-20s mean: %8.2fs stddev: %6.2fs\n" "$name" "$mean" "$stddev" + fi +done + +echo "" +echo -e "${GREEN}Markdown reports: $TMP_DIR/*.md${NC}" +echo -e "${GREEN}JSON results: $TMP_DIR/*.json${NC}"