From 0d0604ad06b0908ba400b069c00184ac2d14d4ec Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Tue, 25 Aug 2026 17:03:31 +0100 Subject: [PATCH 1/9] Run benchmarks in CI --- .github/workflows/cargo-test.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 0099fe02d..613cc5b1a 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -40,3 +40,29 @@ jobs: uses: ./.github/actions/codecov with: token: ${{ secrets.CODECOV_TOKEN }} + + benchmark: + name: Run benchmarks + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + - uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Run cargo bench + shell: bash + run: cargo bench --no-fail-fast | tee output.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: "cargo" + output-file-path: output.txt + external-data-json-path: ./cache/benchmark-data.json + fail-on-alert: true + # GitHub API token to make a commit comment + github-token: ${{ secrets.GITHUB_TOKEN }} + # Enable alert commit comment + comment-on-alert: true + # Enable Job Summary for PRs + summary-always: true From bc62017b22e6891b6caa5aba3cea66005823f85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Diego=20Alonso=20=C3=81lvarez?= <6095790+dalonsoa@users.noreply.github.com> Date: Tue, 25 Aug 2026 18:24:28 +0100 Subject: [PATCH 2/9] Update cargo-test.yml --- .github/workflows/cargo-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 613cc5b1a..b3a5db447 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -44,14 +44,14 @@ jobs: benchmark: name: Run benchmarks runs-on: ubuntu-latest - + timeout-minutes: 15 steps: - uses: actions/checkout@v7 - uses: actions-rust-lang/setup-rust-toolchain@v1 - name: Run cargo bench shell: bash - run: cargo bench --no-fail-fast | tee output.txt + run: cargo bench --features bench --no-fail-fast | tee output.txt - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 From c83686793ea34d3f36ac1e32368841bff81758e6 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 26 Aug 2026 09:05:23 +0100 Subject: [PATCH 3/9] Adapt Citerion output to what action needs --- .github/workflows/cargo-test.yml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index b3a5db447..f2973bed3 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -53,11 +53,38 @@ jobs: shell: bash run: cargo bench --features bench --no-fail-fast | tee output.txt + - name: Convert Criterion results to JSON + shell: python3 {0} + run: | + import json + from pathlib import Path + + criterion_dir = Path("target/criterion") + results = [] + + # Walk every new/estimates.json produced by Criterion. + # The path structure is: + # target/criterion///new/estimates.json + # where Criterion sanitises any '/' in a group name to '_'. + # Stripping the leading criterion_dir and trailing new/estimates.json + # gives a stable, human-readable name that is consistent across runs. + for estimates_file in sorted(criterion_dir.rglob("new/estimates.json")): + parts = estimates_file.relative_to(criterion_dir).parts[:-2] + name = "/".join(parts) + data = json.loads(estimates_file.read_text()) + value = data["median"]["point_estimate"] + results.append({"name": name, "value": value, "unit": "ns"}) + + Path("benchmark-results.json").write_text(json.dumps(results, indent=2)) + print(f"Converted {len(results)} Criterion benchmark results to benchmark-results.json") + for r in results: + print(f" {r['name']}: {r['value']:.0f} ns") + - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - tool: "cargo" - output-file-path: output.txt + tool: "customSmallerIsBetter" + output-file-path: benchmark-results.json external-data-json-path: ./cache/benchmark-data.json fail-on-alert: true # GitHub API token to make a commit comment From fbfdaf60449b4dd24bc4bdca22d5c2d3424cc816 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 26 Aug 2026 09:56:11 +0100 Subject: [PATCH 4/9] Cache the results --- .github/workflows/cargo-test.yml | 15 +++++++++++++++ .gitignore | 3 +++ 2 files changed, 18 insertions(+) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index f2973bed3..13c30be88 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -49,6 +49,18 @@ jobs: - uses: actions/checkout@v7 - uses: actions-rust-lang/setup-rust-toolchain@v1 + - name: Restore benchmark data + uses: actions/cache@v4 + with: + path: cache/benchmark-data.json + # Unique key per run so each run saves a new entry; restore-keys finds + # the most recent previous entry when the exact key is not yet cached. + key: benchmark-data-${{ github.run_id }} + restore-keys: benchmark-data- + + - name: Ensure benchmark cache directory exists + run: mkdir -p cache + - name: Run cargo bench shell: bash run: cargo bench --features bench --no-fail-fast | tee output.txt @@ -93,3 +105,6 @@ jobs: comment-on-alert: true # Enable Job Summary for PRs summary-always: true + + # actions/cache automatically saves cache/benchmark-data.json at the end + # of the job via its post step, keyed with the run-specific key above. diff --git a/.gitignore b/.gitignore index 1679c5113..fd9a73fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ settings.toml debug/ target/ +# GitHub Actions benchmark cache (managed by actions/cache, not git) +cache/ + # These are backup files generated by rustfmt **/*.rs.bk From 58b1a172321614811a304afdd3d78ebda6272347 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 26 Aug 2026 10:20:06 +0100 Subject: [PATCH 5/9] Update alert conditions and add comments --- .github/workflows/cargo-test.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 13c30be88..a706c643f 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -98,12 +98,18 @@ jobs: tool: "customSmallerIsBetter" output-file-path: benchmark-results.json external-data-json-path: ./cache/benchmark-data.json + # Fail the workflow if any benchmark regresses by more than 30% relative + # to the stored baseline. CI runners have variable load, so a threshold + # that is too tight will cause spurious failures; 130% is a reasonable + # starting point and can be tightened once baseline stability is known. + alert-threshold: "130%" fail-on-alert: true # GitHub API token to make a commit comment github-token: ${{ secrets.GITHUB_TOKEN }} - # Enable alert commit comment + # Post a comment on the commit when an alert is triggered, listing which + # benchmarks regressed and by how much. comment-on-alert: true - # Enable Job Summary for PRs + # Always write a job summary table, not only when a regression is found. summary-always: true # actions/cache automatically saves cache/benchmark-data.json at the end From 8166e6847d4bc073f6285a04fa2210eca162e17a Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Wed, 26 Aug 2026 10:41:18 +0100 Subject: [PATCH 6/9] Remove unnecessary pipe --- .github/workflows/cargo-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index a706c643f..00bbb762a 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -63,7 +63,7 @@ jobs: - name: Run cargo bench shell: bash - run: cargo bench --features bench --no-fail-fast | tee output.txt + run: cargo bench --features bench --no-fail-fast - name: Convert Criterion results to JSON shell: python3 {0} From c853c75c1fa813ce6686f911af124f881b75ffd2 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 27 Aug 2026 07:04:50 +0100 Subject: [PATCH 7/9] Use baseline benchmark results --- .github/workflows/cargo-test.yml | 56 ++++++++------------------- .gitignore | 3 -- benchmark/convert_criterion_output.py | 39 +++++++++++++++++++ 3 files changed, 55 insertions(+), 43 deletions(-) create mode 100644 benchmark/convert_criterion_output.py diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 00bbb762a..6ff9f4932 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -45,59 +45,26 @@ jobs: name: Run benchmarks runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: write # needed to push the updated baseline back to the repo + pull-requests: write # needed to post a comment when an alert is triggered steps: - uses: actions/checkout@v7 - uses: actions-rust-lang/setup-rust-toolchain@v1 - - name: Restore benchmark data - uses: actions/cache@v4 - with: - path: cache/benchmark-data.json - # Unique key per run so each run saves a new entry; restore-keys finds - # the most recent previous entry when the exact key is not yet cached. - key: benchmark-data-${{ github.run_id }} - restore-keys: benchmark-data- - - - name: Ensure benchmark cache directory exists - run: mkdir -p cache - - name: Run cargo bench shell: bash run: cargo bench --features bench --no-fail-fast - name: Convert Criterion results to JSON - shell: python3 {0} - run: | - import json - from pathlib import Path - - criterion_dir = Path("target/criterion") - results = [] - - # Walk every new/estimates.json produced by Criterion. - # The path structure is: - # target/criterion///new/estimates.json - # where Criterion sanitises any '/' in a group name to '_'. - # Stripping the leading criterion_dir and trailing new/estimates.json - # gives a stable, human-readable name that is consistent across runs. - for estimates_file in sorted(criterion_dir.rglob("new/estimates.json")): - parts = estimates_file.relative_to(criterion_dir).parts[:-2] - name = "/".join(parts) - data = json.loads(estimates_file.read_text()) - value = data["median"]["point_estimate"] - results.append({"name": name, "value": value, "unit": "ns"}) - - Path("benchmark-results.json").write_text(json.dumps(results, indent=2)) - print(f"Converted {len(results)} Criterion benchmark results to benchmark-results.json") - for r in results: - print(f" {r['name']}: {r['value']:.0f} ns") + run: python3 benchmark/convert_criterion_output.py - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: tool: "customSmallerIsBetter" output-file-path: benchmark-results.json - external-data-json-path: ./cache/benchmark-data.json + external-data-json-path: ./benchmark/benchmark-data.json # Fail the workflow if any benchmark regresses by more than 30% relative # to the stored baseline. CI runners have variable load, so a threshold # that is too tight will cause spurious failures; 130% is a reasonable @@ -112,5 +79,14 @@ jobs: # Always write a job summary table, not only when a regression is found. summary-always: true - # actions/cache automatically saves cache/benchmark-data.json at the end - # of the job via its post step, keyed with the run-specific key above. + - name: Commit updated baseline + # Only update the stored baseline when code is merged to main. On pull + # requests the job reads the baseline for comparison but must not overwrite + # it, since the PR branch has not yet been reviewed and merged. + if: github.ref == 'refs/heads/main' && github.event_name == 'push' + run: | + git config user.name "GitHub Actions" + git config user.email "github-actions@github.com" + git add benchmark/benchmark-data.json + git diff --staged --quiet || git commit -m "chore: update benchmark baseline [skip ci]" + git push diff --git a/.gitignore b/.gitignore index fd9a73fc6..1679c5113 100644 --- a/.gitignore +++ b/.gitignore @@ -13,9 +13,6 @@ settings.toml debug/ target/ -# GitHub Actions benchmark cache (managed by actions/cache, not git) -cache/ - # These are backup files generated by rustfmt **/*.rs.bk diff --git a/benchmark/convert_criterion_output.py b/benchmark/convert_criterion_output.py new file mode 100644 index 000000000..88f5968b7 --- /dev/null +++ b/benchmark/convert_criterion_output.py @@ -0,0 +1,39 @@ +"""Convert Criterion benchmark results to the JSON format expected by +github-action-benchmark (customSmallerIsBetter). + +Criterion writes one estimates.json file per benchmark under: + target/criterion///new/estimates.json + +where any '/' in a group name is sanitised to '_' by Criterion when creating +directory names. The benchmark name used for tracking is derived by stripping +the leading target/criterion/ prefix and the trailing /new/estimates.json +suffix, then joining the remaining path components with '/'. + +The output is written to benchmark-results.json in the repository root, in the +format: + [{"name": "", "value": , "unit": "ns"}, ...] + +All paths are resolved relative to the repository root (the parent directory of +the directory containing this script), so the script can be invoked from any +working directory. +""" + +import json +from pathlib import Path + +repo_root = Path(__file__).parent.parent +criterion_dir = repo_root / "target" / "criterion" +output_file = repo_root / "benchmark-results.json" + +results = [] +for estimates_file in sorted(criterion_dir.rglob("new/estimates.json")): + parts = estimates_file.relative_to(criterion_dir).parts[:-2] + name = "/".join(parts) + data = json.loads(estimates_file.read_text()) + value = data["median"]["point_estimate"] + results.append({"name": name, "value": value, "unit": "ns"}) + +output_file.write_text(json.dumps(results, indent=2)) +print(f"Converted {len(results)} Criterion benchmark results to {output_file}") +for r in results: + print(f" {r['name']}: {r['value']:.0f} ns") From 47c4385ea6cabd0157c92665861c90376dd3baaa Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 27 Aug 2026 07:09:43 +0100 Subject: [PATCH 8/9] Update paths --- .github/workflows/cargo-test.yml | 6 +++--- {benchmark => benches}/convert_criterion_output.py | 0 2 files changed, 3 insertions(+), 3 deletions(-) rename {benchmark => benches}/convert_criterion_output.py (100%) diff --git a/.github/workflows/cargo-test.yml b/.github/workflows/cargo-test.yml index 6ff9f4932..f28420db3 100644 --- a/.github/workflows/cargo-test.yml +++ b/.github/workflows/cargo-test.yml @@ -57,14 +57,14 @@ jobs: run: cargo bench --features bench --no-fail-fast - name: Convert Criterion results to JSON - run: python3 benchmark/convert_criterion_output.py + run: python3 benches/convert_criterion_output.py - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: tool: "customSmallerIsBetter" output-file-path: benchmark-results.json - external-data-json-path: ./benchmark/benchmark-data.json + external-data-json-path: ./benches/benchmark-data.json # Fail the workflow if any benchmark regresses by more than 30% relative # to the stored baseline. CI runners have variable load, so a threshold # that is too tight will cause spurious failures; 130% is a reasonable @@ -87,6 +87,6 @@ jobs: run: | git config user.name "GitHub Actions" git config user.email "github-actions@github.com" - git add benchmark/benchmark-data.json + git add benches/benchmark-data.json git diff --staged --quiet || git commit -m "chore: update benchmark baseline [skip ci]" git push diff --git a/benchmark/convert_criterion_output.py b/benches/convert_criterion_output.py similarity index 100% rename from benchmark/convert_criterion_output.py rename to benches/convert_criterion_output.py From 22c38a3a4844a945de4aa145a100f33b2b6eb8c5 Mon Sep 17 00:00:00 2001 From: Diego Alonso Alvarez Date: Thu, 27 Aug 2026 07:39:21 +0100 Subject: [PATCH 9/9] Simplify benchmarks --- benches/assets.rs | 82 +++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/benches/assets.rs b/benches/assets.rs index 431e64054..bc7bb2f92 100644 --- a/benches/assets.rs +++ b/benches/assets.rs @@ -32,9 +32,11 @@ const YEAR: u32 = 2030; /// competing candidate technologies (petrol, diesel, electric and hybrid cars), making it a /// representative, non-trivial case for asset selection. const COMMODITY_ID: &str = "TPASKM"; -/// The range of numbers of competing candidate technologies to benchmark, to see how -/// [`select_best_assets`] scales with the size of the search space. -const N_TECHNOLOGIES_RANGE: std::ops::RangeInclusive = 1..=20; +/// The numbers of competing candidate technologies to benchmark, stepped in fives, to see how +/// [`select_best_assets`] scales with the size of the search space. Values are zero-padded to +/// two digits when used as benchmark IDs so that names sort correctly on the filesystem +/// (e.g. `05` before `10` rather than `10` before `5`). +const N_TECHNOLOGIES: &[usize] = &[1, 5, 10, 15, 20]; /// Extract the `two_outputs` example model to a temporary directory, load it, and create a /// (non-debug) [`DataWriter`] for it. @@ -163,7 +165,7 @@ fn criterion_benchmark(c: &mut Criterion) { ); // Real candidate technologies for this market, used as templates to build up to - // `N_TECHNOLOGIES_RANGE.end()` synthetic competing technologies + // `N_TECHNOLOGIES` synthetic competing technologies let templates: Vec> = agent .iter_search_space(region_id, &commodity.id, YEAR) .cloned() @@ -184,7 +186,7 @@ fn criterion_benchmark(c: &mut Criterion) { .sample_size(20) .measurement_time(Duration::from_secs(3)); - for n in N_TECHNOLOGIES_RANGE { + for &n in N_TECHNOLOGIES { // Give the agent a synthetic search space of `n` competing technologies let mut agent = agent.clone(); agent.search_space.insert( @@ -210,40 +212,44 @@ fn criterion_benchmark(c: &mut Criterion) { commodity_portion, ); - group.bench_with_input(BenchmarkId::from_parameter(n), &n, |b, _| { - b.iter_batched( - || { - ( - opt_assets.clone(), - agent_addition_limits.clone(), - demand.clone(), - ) - }, - |(opt_assets, agent_addition_limits, demand)| { - let run = || { - select_best_assets( - black_box(&model), - opt_assets, - agent_addition_limits, - black_box(commodity), - black_box(&agent), - black_box(region_id), - black_box(&prices), - demand, - black_box(YEAR), - &mut writer, + group.bench_with_input( + BenchmarkId::from_parameter(format!("{n:02}")), + &n, + |b, _| { + b.iter_batched( + || { + ( + opt_assets.clone(), + agent_addition_limits.clone(), + demand.clone(), ) - .expect("select_best_assets failed") - }; - if *use_parallel { - run() - } else { - sequential_pool.install(run) - } - }, - BatchSize::SmallInput, - ); - }); + }, + |(opt_assets, agent_addition_limits, demand)| { + let run = || { + select_best_assets( + black_box(&model), + opt_assets, + agent_addition_limits, + black_box(commodity), + black_box(&agent), + black_box(region_id), + black_box(&prices), + demand, + black_box(YEAR), + &mut writer, + ) + .expect("select_best_assets failed") + }; + if *use_parallel { + run() + } else { + sequential_pool.install(run) + } + }, + BatchSize::SmallInput, + ); + }, + ); } group.finish(); }