From 600445c325ade517631d50a5f625923f56387f1c Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Tue, 4 Aug 2026 13:01:11 -0700 Subject: [PATCH 1/2] feat(perf): persist release report artifacts for rerendering - Store deterministic comparison CSV with an adjacent schema-versioned provenance sidecar before temporary worktrees are removed. - Render and promote reports only from validated artifact reloads, with fail-closed path checks and transactional rollback. - Add performance-rerender and document artifact retention, GitHub assets, and the release workflow. - Refresh Rust, Python, contributor-tool, and GitHub Action pins, and separate Dependabot security update groups. Closes #205 --- .github/actions/setup-just/action.yml | 2 +- .github/dependabot.yml | 17 +- .github/workflows/audit.yml | 4 +- .github/workflows/benchmarks.yml | 2 +- .github/workflows/ci.yml | 14 +- .github/workflows/codecov.yml | 4 +- .github/workflows/codeql.yml | 6 +- .github/workflows/release-benchmarks.yml | 2 +- .github/workflows/rust-clippy.yml | 6 +- .github/workflows/semgrep-sarif.yml | 2 +- CONTRIBUTING.md | 2 +- Cargo.lock | 18 +- Cargo.toml | 2 +- README.md | 2 +- docs/BENCHMARKING.md | 45 +- docs/RELEASING.md | 24 +- justfile | 15 +- pyproject.toml | 5 +- scripts/README.md | 15 +- scripts/archive_performance.py | 551 ++++++++++--- scripts/bench_compare.py | 428 +++++++++- scripts/performance_artifacts.py | 841 ++++++++++++++++++++ scripts/tests/test_archive_performance.py | 382 ++++++++- scripts/tests/test_bench_compare.py | 80 +- scripts/tests/test_performance_artifacts.py | 365 +++++++++ uv.lock | 152 ++-- 26 files changed, 2687 insertions(+), 299 deletions(-) create mode 100644 scripts/performance_artifacts.py create mode 100644 scripts/tests/test_performance_artifacts.py diff --git a/.github/actions/setup-just/action.yml b/.github/actions/setup-just/action.yml index c2152a8..ac62725 100644 --- a/.github/actions/setup-just/action.yml +++ b/.github/actions/setup-just/action.yml @@ -46,6 +46,6 @@ runs: echo "version=$version" >> "$GITHUB_OUTPUT" - name: Install just - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: just@${{ steps.resolve.outputs.version }} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 54d9ab4..354b81b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,11 @@ updates: - "github-actions" groups: github-actions: + applies-to: version-updates + patterns: + - "*" + github-actions-security: + applies-to: security-updates patterns: - "*" @@ -33,9 +38,14 @@ updates: labels: - "dependencies" - "rust" - # Group all updates together + # Group version and security updates separately groups: dependencies: + applies-to: version-updates + patterns: + - "*" + rust-security: + applies-to: security-updates patterns: - "*" @@ -55,5 +65,10 @@ updates: - "python" groups: dependencies: + applies-to: version-updates + patterns: + - "*" + python-security: + applies-to: security-updates patterns: - "*" diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index c7befe9..29d94c6 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -56,7 +56,7 @@ jobs: restore-keys: advisory-db- - name: Install cargo-audit - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-audit@${{ env.CARGO_AUDIT_VERSION }} @@ -87,7 +87,7 @@ jobs: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) - uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: sarif_file: audit-results.sarif category: cargo-audit diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e386cc8..6142327 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -89,7 +89,7 @@ jobs: echo "version=$version" >> "$GITHUB_OUTPUT" - name: Install cargo-nextest - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-nextest@${{ steps.cargo_nextest_version.outputs.version }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67a0455..652a9c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,19 +123,19 @@ jobs: run: uv sync --locked --group dev - name: Install dprint - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: dprint@${{ steps.tool_versions.outputs.DPRINT_VERSION }} - name: Install rumdl - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: rumdl@${{ steps.tool_versions.outputs.RUMDL_VERSION }} - name: Install taplo id: install-taplo continue-on-error: ${{ matrix.os == 'windows-latest' }} - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: taplo-cli@${{ steps.tool_versions.outputs.TAPLO_VERSION }} @@ -147,22 +147,22 @@ jobs: TAPLO_VERSION: ${{ steps.tool_versions.outputs.TAPLO_VERSION }} - name: Install typos - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: typos-cli@${{ steps.tool_versions.outputs.TYPOS_VERSION }} - name: Install zizmor - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: zizmor@${{ steps.tool_versions.outputs.ZIZMOR_VERSION }} - name: Install cargo-machete - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-machete@${{ steps.tool_versions.outputs.CARGO_MACHETE_VERSION }} - name: Install cargo-nextest - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-nextest@${{ steps.tool_versions.outputs.CARGO_NEXTEST_VERSION }} diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index ac9d111..3ecb049 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -68,12 +68,12 @@ jobs: run: rustup component add llvm-tools-preview - name: Install cargo-llvm-cov - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-llvm-cov@${{ steps.tool_versions.outputs.CARGO_LLVM_COV_VERSION }} - name: Install cargo-nextest - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-nextest@${{ steps.tool_versions.outputs.CARGO_NEXTEST_VERSION }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index eb7d33e..6c73ffc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,20 +35,20 @@ jobs: - name: Initialize CodeQL if: matrix.language != 'rust' - uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} queries: security-extended - name: Initialize CodeQL (Rust) if: matrix.language == 'rust' - uses: github/codeql-action/init@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} build-mode: none queries: security-extended - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/release-benchmarks.yml b/.github/workflows/release-benchmarks.yml index bff247f..8f436ec 100644 --- a/.github/workflows/release-benchmarks.yml +++ b/.github/workflows/release-benchmarks.yml @@ -56,7 +56,7 @@ jobs: echo "version=$version" >> "$GITHUB_OUTPUT" - name: Install cargo-nextest - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: cargo-nextest@${{ steps.cargo_nextest_version.outputs.version }} diff --git a/.github/workflows/rust-clippy.yml b/.github/workflows/rust-clippy.yml index 1a31da7..4f82dfd 100644 --- a/.github/workflows/rust-clippy.yml +++ b/.github/workflows/rust-clippy.yml @@ -74,12 +74,12 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Install clippy-sarif - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: clippy-sarif@${{ steps.tool_versions.outputs.CLIPPY_SARIF_VERSION }} - name: Install sarif-fmt - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 + uses: taiki-e/cache-cargo-install-action@9ee83daaa7b96a6fab930949ecf1122bba04a389 # v3.0.8 with: tool: sarif-fmt@${{ steps.tool_versions.outputs.SARIF_FMT_VERSION }} @@ -104,7 +104,7 @@ jobs: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) - uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: sarif_file: rust-clippy-results.sarif category: "clippy" diff --git a/.github/workflows/semgrep-sarif.yml b/.github/workflows/semgrep-sarif.yml index 7e606df..01731e7 100644 --- a/.github/workflows/semgrep-sarif.yml +++ b/.github/workflows/semgrep-sarif.yml @@ -81,7 +81,7 @@ jobs: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository ) - uses: github/codeql-action/upload-sarif@f205ea1c3313d32999d8d6a48b4f6530d4437b38 # v4.37.4 + uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: sarif_file: semgrep-results.sarif category: semgrep-repository-rules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ec2c5ad..0e52f24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,7 @@ Install Rust 1.97.1 through [rustup](https://rustup.rs/), Git, Python 3.14, pinned `just` version from its locked dependency graph: ```bash -cargo install --locked just --version 1.57.0 +cargo install --locked just --version 1.58.0 ``` Set up the remaining development tools and validate the checkout: diff --git a/Cargo.lock b/Cargo.lock index 5c56e06..8538035 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] @@ -571,9 +571,9 @@ checksum = "f70749695b063ecbf6b62949ccccde2e733ec3ecbbd71d467dca4e5c6c97cca0" [[package]] name = "glam" -version = "0.33.2" +version = "0.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f22fb22f065b308be0d8724e3706c7fa3fc2a6c7d6899df4cad7860e7a75436" +checksum = "7360bd2cd76e0cd9032d42cf2922155cecea2685b0cfa4630c3246df030bcfd6" [[package]] name = "half" @@ -696,7 +696,7 @@ dependencies = [ "glam 0.30.10", "glam 0.31.1", "glam 0.32.1", - "glam 0.33.2", + "glam 0.33.3", "matrixmultiply", "num-complex", "num-rational", @@ -1107,9 +1107,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -1228,9 +1228,9 @@ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "simba" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f45c644a9f3a386f9288625d9f0c1e999e1acf07a37df35d0516c7f199d9cb2" +checksum = "834a9952211d4a60ff10ac1d20cc01640a75bdc0a0c688d62f359ea7d17459f3" dependencies = [ "approx", "num-complex", diff --git a/Cargo.toml b/Cargo.toml index 332769a..bcc6830 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ include = [ [dependencies] # All runtime deps are optional; see [features] below. # Must stay in sync with num-rational -num-bigint = { version = "0.4.6", optional = true } +num-bigint = { version = "0.4.8", optional = true } num-rational = { version = "0.4.2", features = [ "num-bigint-std" ], optional = true } num-traits = { version = "0.2.19", optional = true } diff --git a/README.md b/README.md index 894297a..785ec5e 100644 --- a/README.md +++ b/README.md @@ -575,7 +575,7 @@ Install Rust 1.97.1 through [rustup](https://rustup.rs/), Git, Python 3.14, `just` release from its locked dependency graph: ```bash -cargo install --locked just --version 1.57.0 +cargo install --locked just --version 1.58.0 just setup # install/verify dev tools + sync Python deps just check # lint/validate (non-mutating) just fix # apply auto-fixes (mutating) diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index db15e5a..77e9ef1 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -30,6 +30,7 @@ the commands measure and where their outputs go. | Full crate comparison | `just bench-vs-linalg` | | README table and plot | `just plot-vs-linalg-readme` | | Release report | `just performance-release v0.4.4 v0.4.3` | +| Re-render retained release inputs | `just performance-rerender` | | Published-asset comparison | `just performance-github-assets v0.4.4 v0.4.3` | Rule of thumb: @@ -39,6 +40,8 @@ Rule of thumb: - Use `bench-vs-linalg` plus plotting when updating README crate-to-crate comparisons. - Use `performance-release` only when preparing committed release artifacts. +- Use `performance-rerender` for report-format changes after a valid release + dataset has already been retained. ## Benchmark Suites @@ -200,11 +203,33 @@ With no arguments, `just performance-release` infers the current release tag from `Cargo.toml` and discovers the previous stable published release. During release preparation, passing both tags explicitly removes ambiguity. -This command creates temporary worktrees, generates the comparison, writes -`docs/PERFORMANCE.md`, and archives the previous committed report under +This command creates temporary worktrees, validates the complete comparison, +and writes the exact report inputs to +`target/bench-reports/performance.csv` with adjacent +`performance.provenance.json`. The CSV records deterministic benchmark keys, +coverage status and notes, and baseline/current median estimates with complete +confidence intervals in nanoseconds. The JSON sidecar binds the CSV digest and +row count to the release pair, source states, commands, toolchain, Criterion +version, harness/configuration digests, host, and schema version. + +The pair is validated and published before the temporary worktree is removed. +`docs/PERFORMANCE.md` is then rendered from a validated reload of that retained +pair, and the previous committed report is archived under `docs/archive/performance/`. Archive filenames are release-pair names such as -`v0.4.2-vs-v0.4.1.md`. Publication fails rather than emitting a partial report -when a selected suite or required dimension is absent. +`v0.4.2-vs-v0.4.1.md`. Serialization, validation, rendering, coverage, or +promotion failures preserve the previous valid report and artifact pair. + +To reproduce and promote the report without running Cargo or creating Git +worktrees, use: + +```bash +just performance-rerender +``` + +This command fails closed on a missing, partial, malformed, mismatched, or +unsupported artifact pair. Use it for presentation-only report corrections; +changes to benchmark inputs, code, toolchains, or measurement configuration +require a fresh `performance-release` run. ### Compare Published Release Artifacts @@ -235,7 +260,11 @@ shared-harness workflow before attributing a difference solely to library code. |------|------------|----------|---------| | `target/criterion/` | No | `cargo bench`, `bench-save-*` | Local Criterion measurements and named baselines. | | `target/bench-reports/performance.md` | No | `bench-compare`, `performance-local*` | Local comparison report. | +| `target/bench-reports/performance.csv` | No | `performance-local*`, `performance-release` | Validated tabular inputs for the release comparison. | +| `target/bench-reports/performance.provenance.json` | No | `performance-local*`, `performance-release` | Schema, release, source, command, toolchain, host, digest, and harness provenance. | | `target/bench-reports/github-assets-performance.md` | No | `performance-github-assets` | Local report from published release artifacts. | +| `target/bench-reports/github-assets-performance.csv` | No | `performance-github-assets` | Tabular inputs derived from published native archives. | +| `target/bench-reports/github-assets-performance.provenance.json` | No | `performance-github-assets` | Provenance for the published-asset report inputs. | | `docs/PERFORMANCE.md` | Yes | `performance-release` | Latest curated release-to-release comparison. | | `docs/archive/performance/` | Yes | `performance-release` | Older curated release-to-release comparisons. | | `docs/assets/bench/` | Yes | `plot-vs-linalg-readme` | README benchmark CSV/SVG assets and JSON provenance. | @@ -244,6 +273,14 @@ shared-harness workflow before attributing a difference solely to library code. Published baseline assets use the filename `la-stack-$TAG-criterion-baseline.tar.gz`. +Everything under `target/bench-reports/` is reproducible local scratch owned by +the performance-report workflows. It survives temporary-worktree cleanup but +may be removed by `just clean` or `cargo clean`; retain or copy the CSV/JSON pair +while reviewing or rerendering a release PR. The compact CSV is the analysis +and Markdown-reproduction layer. It does not replace the full native Criterion +`.tar.gz` archive attached to each GitHub Release, which remains the durable raw +baseline for post-release comparisons. + ## `vs_linalg` Methodology `vs_linalg` is a per-kernel comparison, not a single aggregate score. Each row diff --git a/docs/RELEASING.md b/docs/RELEASING.md index ec78875..deaf26b 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -140,9 +140,21 @@ just performance-release Review `docs/PERFORMANCE.md` for the latest release-to-release comparison. Older committed comparisons are archived under `docs/archive/performance/` with lexicographically sorted filenames such as `v0.4.2-vs-v0.4.1.md`. Iterative -local reports still live under `target/bench-reports/`. For an explicit release -repair, run `just performance-release `. To compare -the stored GitHub Actions release assets instead of running cargo locally, use +local reports still live under `target/bench-reports/`. + +The release command retains `target/bench-reports/performance.csv` and +`performance.provenance.json` before its temporary worktrees are removed, then +renders and promotes Markdown from a validated reload of that pair. Review the +CSV coverage/timing rows and JSON release, revision, command, toolchain, host, +harness, and digest metadata alongside the Markdown. For a presentation-only +correction, run `just performance-rerender`; it reproduces and promotes the +report from those files without invoking Cargo or creating worktrees. The pair +is local scratch and may be removed by `just clean` or `cargo clean`, so do not +clean `target/` until the release report review is complete. + +For an explicit measurement repair, run +`just performance-release `. To compare the stored +GitHub Actions release assets instead of running Cargo locally, use `just performance-github-assets`. The local release workflow validates and then compiles both library revisions with the current checkout's hashed benchmark harness, recording source-state, environment, toolchain, dependency, Criterion, @@ -154,8 +166,10 @@ After the GitHub Release is published, the `Release Benchmarks` workflow checks out the release tag, runs the independent benchmark-input tests, saves a full Criterion baseline, and attaches `la-stack-$TAG-criterion-baseline.tar.gz` to the release. That release asset is -the durable archive for historical baseline comparisons; the workflow also -uploads a short-lived Actions artifact for debugging the run. +the durable native Criterion archive for historical baseline comparisons. The +compact release-report CSV is an analysis and report-reproduction layer, not a +replacement for that raw archive. The workflow also uploads a short-lived +Actions artifact for debugging the run. See `docs/BENCHMARKING.md` for local saved-baseline workflows and the full comparison command reference. diff --git a/justfile b/justfile index f502113..636d49b 100644 --- a/justfile +++ b/justfile @@ -23,11 +23,11 @@ cargo_nextest_version := "0.9.140" clippy_sarif_version := "0.8.0" dprint_version := "0.55.2" git_cliff_version := "2.13.1" -just_version := "1.57.0" -rumdl_version := "0.2.48" +just_version := "1.58.0" +rumdl_version := "0.2.50" sarif_fmt_version := "0.8.0" taplo_version := "0.10.0" -typos_version := "1.48.0" +typos_version := "1.49.0" uv_version := "0.12.1" zizmor_version := "1.29.0" @@ -459,6 +459,7 @@ help-workflows: @echo " just performance-local # Compare current tree against latest release locally" @echo " just performance-local-vs-linalg # Compare current non-exact kernels locally" @echo " just performance-release # Promote local release performance docs" + @echo " just performance-rerender # Re-render release docs from retained CSV/JSON" @echo " just bench-save-last # Save full baseline as 'last'" @echo " just bench-vs-linalg # Run vs_linalg bench (optional filter)" @echo " just bench-vs-linalg-la-stack # Run la-stack rows from vs_linalg" @@ -586,9 +587,9 @@ performance-github-assets current_tag="" baseline_tag="": python-sync echo "current_tag and baseline_tag must be provided together" >&2 exit 2 fi - uv run --locked archive-performance "$current_tag" "$baseline_tag" --github-assets --generate-in-temp-worktree --worktree-ref "$current_tag" --output-only --output target/bench-reports/github-assets-performance.md + uv run --locked archive-performance "$current_tag" "$baseline_tag" --github-assets --generate-in-temp-worktree --worktree-ref "$current_tag" --output-only --output target/bench-reports/github-assets-performance.md --artifact-csv target/bench-reports/github-assets-performance.csv --artifact-provenance target/bench-reports/github-assets-performance.provenance.json else - uv run --locked archive-performance --published-latest --github-assets --generate-in-temp-worktree --output-only --output target/bench-reports/github-assets-performance.md + uv run --locked archive-performance --published-latest --github-assets --generate-in-temp-worktree --output-only --output target/bench-reports/github-assets-performance.md --artifact-csv target/bench-reports/github-assets-performance.csv --artifact-provenance target/bench-reports/github-assets-performance.provenance.json fi # Compare the current tree against the latest published release locally. @@ -627,6 +628,10 @@ performance-release current_tag="" baseline_tag="": python-sync uv run --locked archive-performance --infer-release --generate-in-temp-worktree --worktree-ref HEAD fi +# Re-render and promote release performance docs from retained report inputs. +performance-rerender: python-sync + uv run --locked archive-performance --rerender + # Plot: generate a single time-vs-dimension SVG from Criterion results. plot-vs-linalg metric="lu_solve" stat="median" sample="new" log_y="false" allow_partial="false": python-sync #!/usr/bin/env bash diff --git a/pyproject.toml b/pyproject.toml index 5b97db7..58b56b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ check-docs-version-sync = "check_docs_version_sync:main" # Configure setuptools to find modules in scripts/ directory. [tool.setuptools] package-dir = { "" = "scripts" } -py-modules = [ "archive_changelog", "archive_performance", "bench_compare", "check_docs_version_sync", "check_semgrep_fixtures", "criterion_dim_plot", "postprocess_changelog", "subprocess_utils", "tag_release" ] +py-modules = [ "archive_changelog", "archive_performance", "bench_compare", "check_docs_version_sync", "check_semgrep_fixtures", "criterion_dim_plot", "performance_artifacts", "postprocess_changelog", "subprocess_utils", "tag_release" ] [tool.ruff] line-length = 160 @@ -92,6 +92,7 @@ known-first-party = [ "bench_compare", "check_semgrep_fixtures", "criterion_dim_plot", + "performance_artifacts", "postprocess_changelog", "subprocess_utils", "tag_release", @@ -149,6 +150,6 @@ dev = [ "semgrep==1.172.0", "shellcheck-py==0.11.0.1", "shfmt-py==4.0.0", - "ty==0.0.65", + "ty==0.0.66", "yamllint==1.38.0", ] diff --git a/scripts/README.md b/scripts/README.md index f10efbe..d7591d6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,6 +53,9 @@ just performance-local # Release PR: update docs/PERFORMANCE.md and archive the previous report just performance-release +# Re-render the release report from retained CSV/JSON inputs +just performance-rerender + # GitHub Actions release assets, without local cargo benchmark runs just performance-github-assets ``` @@ -61,9 +64,14 @@ The local release workflows run the independent benchmark-input correctness gate and then measure both library revisions with one hashed current benchmark harness. Reports record source-state, environment, toolchain, dependency, Criterion, harness, and validation provenance and fail on incomplete selected -coverage. Direct comparisons of separately published artifacts retain their -original per-release harnesses and label unavailable historical measurement -metadata explicitly. +coverage. `performance-release` retains schema-versioned `performance.csv` and +`performance.provenance.json` inputs under `target/bench-reports/`, then renders +the promoted Markdown from their validated reload. `performance-rerender` uses +that pair without Cargo or temporary worktrees. These files are reproducible +scratch and may be removed with `target/`; native Criterion release archives +remain the durable raw baselines. Direct comparisons of separately published +artifacts retain their original per-release harnesses and label unavailable +historical measurement metadata explicitly. See `docs/BENCHMARKING.md` for the current command matrix, local saved-baseline workflow, explicit tag arguments, output locations, and release-artifact @@ -219,6 +227,7 @@ tag-annotation size limit. |---|---| | `archive_changelog.py` | Split completed changelog minor series into archives | | `archive_performance.py` | Promote release performance docs and archive older comparisons | +| `performance_artifacts.py` | Validate and publish schema-versioned release-report CSV/JSON inputs | | `bench_compare.py` | Compare Criterion benchmark baselines and render Markdown reports | | `check_docs_version_sync.py` | Verify versioned documentation links and snippets stay synchronized | | `criterion_dim_plot.py` | Plot Criterion benchmark results (CSV + SVG + README table) | diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index 48fa948..e331281 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -5,16 +5,16 @@ - ``target/bench-reports/performance.md`` is local scratch output for the current machine and branch. + - ``target/bench-reports/performance.csv`` and the adjacent provenance JSON + are validated, rerenderable release-report inputs. - ``docs/PERFORMANCE.md`` is the latest curated release-to-release comparison. - ``docs/archive/performance/*.md`` stores older curated comparisons. -This script copies a freshly generated local report into ``docs/PERFORMANCE.md`` -and archives the previous committed report under a filename derived from the -report metadata, such as ``v0.4.2-vs-v0.4.1.md``. +This script renders from a validated artifact reload, copies the result into +``docs/PERFORMANCE.md``, and archives the previous committed report under a +filename derived from the report metadata, such as ``v0.4.2-vs-v0.4.1.md``. """ -from __future__ import annotations - import argparse import hashlib import json @@ -34,6 +34,8 @@ from pathlib import Path from typing import Any, Literal, cast +from bench_compare import render_release_artifacts +from performance_artifacts import ArtifactPaths, PerformanceBundle, ensure_distinct_paths, load_bundle, publish_bundle from subprocess_utils import ExecutableNotFoundError, run_git_command, run_git_command_with_input, run_safe_command _VERSION_RE = re.compile(r"^\*\*la-stack\*\* v(?P[^\s`]+)", re.MULTILINE) @@ -47,6 +49,8 @@ _SEMVER_PARTS_RE = re.compile(r"^v?(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)\.(?P0|[1-9][0-9]*)$") _DEFAULT_SOURCE = "target/bench-reports/performance.md" +_DEFAULT_ARTIFACT_CSV = "target/bench-reports/performance.csv" +_DEFAULT_ARTIFACT_PROVENANCE = "target/bench-reports/performance.provenance.json" _DEFAULT_CURRENT = "docs/PERFORMANCE.md" _DEFAULT_ARCHIVE_DIR = "docs/archive/performance" _DEFAULT_SUITE = "all" @@ -89,6 +93,18 @@ def archive_name(self) -> str: return f"{self.current_tag}-vs-{self.baseline_tag}.md" +@dataclass(frozen=True) +class PromotionRequest: + """Destinations and expected identity for one report promotion.""" + + current: Path + archive_dir: Path + expected: ReportId + source_path: Path | None = None + output: Path | None = None + reserved_paths: Mapping[str, Path] | None = None + + @dataclass(frozen=True) class GenerationConfig: """Configuration for benchmark report generation in a temp worktree.""" @@ -143,6 +159,15 @@ class ArchivePaths: current: Path output: Path archive_dir: Path + artifacts: ArtifactPaths + + +@dataclass(frozen=True) +class GeneratedReport: + """A report rendered from a validated, durable artifact bundle.""" + + text: str + bundle: PerformanceBundle @dataclass(frozen=True) @@ -150,7 +175,7 @@ class ArchiveResult: """Result and destination metadata for a completed archive operation.""" report_id: ReportId - action: Literal["output", "promote-generated", "promote-source"] + action: Literal["output", "promote-generated", "promote-source", "rerender"] @dataclass(frozen=True) @@ -362,6 +387,9 @@ def _how_to_update_section() -> str: "# Release PR: update docs/PERFORMANCE.md and archive the previous report", "just performance-release", "", + "# Re-render and promote from retained CSV/JSON inputs (no benchmarks)", + "just performance-rerender", + "", "# GitHub Actions release assets", "just performance-github-assets", "", @@ -371,6 +399,7 @@ def _how_to_update_section() -> str: "", "`just performance-local` writes `target/bench-reports/performance.md`.", "`just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`.", + "`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report.", "", "Older curated release-to-release reports are archived in `docs/archive/performance/`.", "", @@ -413,6 +442,53 @@ def _write_text(path: Path, text: str) -> None: tmp_path.unlink() +def _restore_file(path: Path, payload: bytes | None) -> None: + """Restore one file snapshot without relying on the publication writer.""" + if payload is None: + path.unlink(missing_ok=True) + return + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile( + "wb", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".restore", + delete=False, + ) as tmp: + restore_path = Path(tmp.name) + tmp.write(payload) + tmp.flush() + os.fsync(tmp.fileno()) + try: + restore_path.replace(path) + finally: + restore_path.unlink(missing_ok=True) + + +def _snapshot_regular_file(path: Path, *, label: str) -> bytes | None: + """Snapshot a mutation target while rejecting symlinks and non-files.""" + if path.is_symlink(): + msg = f"{label} must not be a symlink: {path}" + raise ValueError(msg) + if not path.exists(): + return None + if not path.is_file(): + msg = f"{label} must be a regular file: {path}" + raise ValueError(msg) + return path.read_bytes() + + +def _restore_snapshots(snapshots: tuple[tuple[Path, bytes | None], ...]) -> tuple[BaseException, ...]: + """Attempt every file restoration and return all rollback failures.""" + errors: list[BaseException] = [] + for path, payload in reversed(snapshots): + try: + _restore_file(path, payload) + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + return tuple(errors) + + def _archive_readme(archive_dir: Path) -> str: reports = sorted(path.name for path in archive_dir.glob("*.md") if path.name != "README.md") lines = [ @@ -1152,45 +1228,48 @@ def _has_current_release_signal_tooling(worktree: Path) -> bool: return re.search(r"(?m)^bench-latest(?:[ :]|$)", justfile_text) is not None and '"--suite"' in bench_compare_text and '"--scope"' in bench_compare_text -def _render_report(*, worktree: Path, report: Path, config: GenerationConfig) -> None: - if _has_current_release_signal_tooling(worktree): - _run_tool( - "uv", - [ - "run", - "--locked", - "bench-compare", - config.baseline_tag, - "--suite", - config.suite, - "--scope", - config.scope, - "--output", - str(report), - ], - cwd=worktree, - timeout=_COMMAND_TIMEOUT_SECONDS, - ) - else: - _run_tool( - "uv", - [ - "run", - "--locked", - "bench-compare", - config.baseline_tag, - "--output", - str(report), - ], - cwd=worktree, - timeout=_COMMAND_TIMEOUT_SECONDS, - ) +def _render_report( + *, + worktree: Path, + report: Path, + artifacts: ArtifactPaths, + config: GenerationConfig, +) -> None: + """Export report inputs and render Markdown from their validated reload.""" + _run_tool( + "uv", + [ + "run", + "--locked", + "--project", + str(config.repo_root), + "bench-compare", + config.baseline_tag, + "--repo-root", + str(worktree), + "--criterion-dir", + str(worktree / "target" / "criterion"), + "--suite", + config.suite, + "--scope", + config.scope, + "--csv-output", + str(artifacts.csv), + "--provenance-output", + str(artifacts.provenance), + "--output", + str(report), + ], + cwd=config.repo_root, + timeout=_COMMAND_TIMEOUT_SECONDS, + ) def _run_benchmarks_and_render_report( *, worktree: Path, report: Path, + artifacts: ArtifactPaths, config: GenerationConfig, baseline_run: BaselineRun, ) -> None: @@ -1217,17 +1296,32 @@ def _run_benchmarks_and_render_report( baseline_run=baseline_run, current_command=current_command, ) - _render_report(worktree=worktree, report=report, config=config) + _render_report(worktree=worktree, report=report, artifacts=artifacts, config=config) + + +def _default_artifact_paths(repo_root: Path) -> ArtifactPaths: + """Return the canonical retained release-report artifact pair.""" + return ArtifactPaths( + csv=repo_root / _DEFAULT_ARTIFACT_CSV, + provenance=repo_root / _DEFAULT_ARTIFACT_PROVENANCE, + ) -def _generate_report_in_temp_worktree( +@contextmanager +def _generated_report_in_temp_worktree( *, config: GenerationConfig, -) -> str: + published_artifacts: ArtifactPaths, +) -> Iterator[GeneratedReport]: + """Generate, publish, and expose a report before its worktree is removed.""" with tempfile.TemporaryDirectory(prefix="la-stack-performance-") as tmp: tmp_dir = Path(tmp) worktree = tmp_dir / "worktree" report = tmp_dir / f"{config.current_tag}-vs-{config.baseline_tag}.md" + temporary_artifacts = ArtifactPaths( + csv=tmp_dir / "performance.csv", + provenance=tmp_dir / "performance.provenance.json", + ) with _temporary_detached_worktree( repo_root=config.repo_root, @@ -1260,7 +1354,12 @@ def _generate_report_in_temp_worktree( config=config, baseline_run=baseline_run, ) - _render_report(worktree=worktree, report=report, config=config) + _render_report( + worktree=worktree, + report=report, + artifacts=temporary_artifacts, + config=config, + ) else: baseline_run = _prepare_local_release_baseline( baseline_tag=config.baseline_tag, @@ -1272,53 +1371,178 @@ def _generate_report_in_temp_worktree( _run_benchmarks_and_render_report( worktree=worktree, report=report, + artifacts=temporary_artifacts, config=config, baseline_run=baseline_run, ) - return _read_text(report) + bundle = load_bundle(temporary_artifacts) + report_text = _read_text(report) + with publish_bundle(published_artifacts, bundle): + durable_text = render_release_artifacts(published_artifacts) + if durable_text != report_text: + msg = "durable release-performance artifacts did not reproduce the generated Markdown" + raise ValueError(msg) + yield GeneratedReport(text=durable_text, bundle=bundle) -def promote_report( +def _current_archive_state( *, - source: Path, current: Path, + source_id: ReportId, archive_dir: Path, - expected_current_tag: str, - expected_baseline_tag: str, -) -> ReportId: - """Archive the old committed report and promote *source* as the current one.""" - source_text = _normalize_how_to_update(_read_text(source)) - source_id = parse_report_id(source_text) - expected_source_id = ReportId( - current_tag=normalize_tag(expected_current_tag), - baseline_tag=normalize_tag(expected_baseline_tag), +) -> tuple[str | None, ReportId | None, Path | None]: + """Return normalized current-report state and its prospective archive path.""" + if not current.exists(): + return None, None, None + current_text = _normalize_how_to_update(_read_text(current)) + current_id = parse_report_id(current_text) + archive_path = archive_dir / current_id.archive_name if current_id != source_id else None + return current_text, current_id, archive_path + + +def _validate_promotion_paths(request: PromotionRequest, *, index_path: Path, archive_path: Path | None) -> None: + """Reject aliases and archive-contained scratch output before mutation.""" + named_paths: dict[str, Path] = { + "current report": request.current, + "archive index": index_path, + } + if request.source_path is not None: + named_paths["source report"] = request.source_path + if request.output is not None: + named_paths["rendered output"] = request.output + if request.output.resolve(strict=False).is_relative_to(request.archive_dir.resolve(strict=False)): + msg = f"rendered output must not be written inside the archive directory: {request.output}" + raise ValueError(msg) + if archive_path is not None: + named_paths["archived report"] = archive_path + if request.reserved_paths is not None: + named_paths.update(request.reserved_paths) + ensure_distinct_paths(named_paths) + + +def _existing_archive_matches(*, archive_path: Path | None, current_id: ReportId | None, current_text: str | None) -> bool: + """Validate a pre-existing canonical archive before trusting it.""" + if archive_path is None or (not archive_path.exists() and not archive_path.is_symlink()): + return False + if current_id is None or current_text is None: + msg = "existing archive validation invariant violated" + raise AssertionError(msg) + archived_payload = _snapshot_regular_file(archive_path, label="existing archived report") + if archived_payload is None: + msg = "existing archive snapshot invariant violated" + raise AssertionError(msg) + archived_text = _normalize_how_to_update(archived_payload.decode("utf-8")) + try: + archived_id = parse_report_id(archived_text) + except (TypeError, ValueError) as exc: + msg = f"existing archived report does not match the current report being replaced: {archive_path}" + raise ValueError(msg) from exc + if archived_id != current_id or archived_text != current_text: + msg = f"existing archived report does not match the current report being replaced: {archive_path}" + raise ValueError(msg) + return True + + +def _promotion_snapshots( + request: PromotionRequest, + *, + index_path: Path, + archive_path: Path | None, + archive_exists: bool, +) -> tuple[tuple[Path, bytes | None], ...]: + """Snapshot every path mutated by a report promotion.""" + snapshots: list[tuple[Path, bytes | None]] = [] + if archive_path is not None and not archive_exists: + snapshots.append((archive_path, None)) + snapshots.extend( + ( + (request.current, _snapshot_regular_file(request.current, label="current report")), + (index_path, _snapshot_regular_file(index_path, label="archive index")), + ) ) - if source_id != expected_source_id: + if request.output is not None: + snapshots.append((request.output, _snapshot_regular_file(request.output, label="rendered output"))) + return tuple(snapshots) + + +def _promote_report_text(*, source_text: str, request: PromotionRequest) -> ReportId: + """Archive the old report and atomically promote validated Markdown text.""" + source_text = _normalize_how_to_update(source_text) + source_id = parse_report_id(source_text) + if source_id != request.expected: msg = ( "benchmark report does not match requested release pair: " f"found {source_id.current_tag} vs {source_id.baseline_tag}, " - f"expected {expected_source_id.current_tag} vs {expected_source_id.baseline_tag}" + f"expected {request.expected.current_tag} vs {request.expected.baseline_tag}" ) raise ValueError(msg) - if current.exists(): - current_text = _normalize_how_to_update(_read_text(current)) - current_id = parse_report_id(current_text) - if current_id != source_id: - archive_path = archive_dir / current_id.archive_name - if not archive_path.exists(): - _write_text(archive_path, current_text) - - _write_text(current, source_text) - update_archive_index(archive_dir) + current_text, current_id, archive_path = _current_archive_state( + current=request.current, + source_id=source_id, + archive_dir=request.archive_dir, + ) + index_path = request.archive_dir / "README.md" + _validate_promotion_paths(request, index_path=index_path, archive_path=archive_path) + archive_exists = _existing_archive_matches( + archive_path=archive_path, + current_id=current_id, + current_text=current_text, + ) + snapshots = _promotion_snapshots( + request, + index_path=index_path, + archive_path=archive_path, + archive_exists=archive_exists, + ) + try: + if archive_path is not None and not archive_exists: + if current_text is None: + msg = "archive promotion invariant violated" + raise AssertionError(msg) + _write_text(archive_path, current_text) + _write_text(request.current, source_text) + update_archive_index(request.archive_dir) + if request.output is not None: + _write_text(request.output, source_text) + except BaseException as promotion_error: + rollback_errors = _restore_snapshots(snapshots) + if rollback_errors: + group_message = "performance report promotion and rollback failed" + raise BaseExceptionGroup(group_message, [promotion_error, *rollback_errors]) from None + raise return source_id +def promote_report( + *, + source: Path, + current: Path, + archive_dir: Path, + expected_current_tag: str, + expected_baseline_tag: str, +) -> ReportId: + """Archive the old committed report and promote *source* as the current one.""" + return _promote_report_text( + source_text=_read_text(source), + request=PromotionRequest( + current=current, + archive_dir=archive_dir, + expected=ReportId( + current_tag=normalize_tag(expected_current_tag), + baseline_tag=normalize_tag(expected_baseline_tag), + ), + source_path=source, + ), + ) + + def generate_and_promote_worktree_report( *, current: Path, archive_dir: Path, config: GenerationConfig, + artifacts: ArtifactPaths | None = None, ) -> ReportId: """Generate a comparison in a temp worktree, then promote it.""" current_tag = normalize_tag(config.current_tag) @@ -1333,29 +1557,43 @@ def generate_and_promote_worktree_report( apply_current_diff=config.apply_current_diff, baseline_source=config.baseline_source, ) - report_text = _generate_report_in_temp_worktree( - config=config, + published_artifacts = artifacts or _default_artifact_paths(config.repo_root) + expected = ReportId(current_tag=current_tag, baseline_tag=baseline_tag) + reserved_paths = { + "artifact CSV": published_artifacts.csv, + "artifact provenance": published_artifacts.provenance, + } + request = PromotionRequest( + current=current, + archive_dir=archive_dir, + expected=expected, + reserved_paths=reserved_paths, ) - with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".md", delete=False) as tmp: - source = Path(tmp.name) - tmp.write(report_text) - try: - return promote_report( - source=source, - current=current, - archive_dir=archive_dir, - expected_current_tag=current_tag, - expected_baseline_tag=baseline_tag, + _current_text, _current_id, archive_path = _current_archive_state( + current=current, + source_id=expected, + archive_dir=archive_dir, + ) + _validate_promotion_paths( + request, + index_path=archive_dir / "README.md", + archive_path=archive_path, + ) + with _generated_report_in_temp_worktree( + config=config, + published_artifacts=published_artifacts, + ) as generated: + return _promote_report_text( + source_text=generated.text, + request=request, ) - finally: - if source.exists(): - source.unlink() def generate_worktree_report( *, output: Path, config: GenerationConfig, + artifacts: ArtifactPaths | None = None, ) -> ReportId: """Generate a comparison in a temp worktree and write it to *output*.""" current_tag = normalize_tag(config.current_tag) @@ -1370,18 +1608,70 @@ def generate_worktree_report( apply_current_diff=config.apply_current_diff, baseline_source=config.baseline_source, ) - report_text = _normalize_how_to_update(_generate_report_in_temp_worktree(config=config)) - report_id = parse_report_id(report_text) - expected = ReportId(current_tag=current_tag, baseline_tag=baseline_tag) - if report_id != expected: + published_artifacts = artifacts or ArtifactPaths( + csv=output.with_suffix(".csv"), + provenance=output.with_suffix(".provenance.json"), + ) + ensure_distinct_paths( + { + "rendered output": output, + "artifact CSV": published_artifacts.csv, + "artifact provenance": published_artifacts.provenance, + } + ) + with _generated_report_in_temp_worktree( + config=config, + published_artifacts=published_artifacts, + ) as generated: + report_text = _normalize_how_to_update(generated.text) + report_id = parse_report_id(report_text) + expected = ReportId(current_tag=current_tag, baseline_tag=baseline_tag) + if report_id != expected: + msg = ( + "benchmark report does not match requested release pair: " + f"found {report_id.current_tag} vs {report_id.baseline_tag}, " + f"expected {expected.current_tag} vs {expected.baseline_tag}" + ) + raise ValueError(msg) + _write_text(output, report_text) + return report_id + + +def rerender_and_promote_artifacts( + *, + artifacts: ArtifactPaths, + output: Path, + current: Path, + archive_dir: Path, +) -> ReportId: + """Reproduce and promote Markdown using only retained CSV/JSON inputs.""" + bundle = load_bundle(artifacts) + report_text = _normalize_how_to_update(render_release_artifacts(artifacts)) + expected = ReportId( + current_tag=normalize_tag(bundle.context.release.current), + baseline_tag=normalize_tag(bundle.context.release.baseline), + ) + observed = parse_report_id(report_text) + if observed != expected: msg = ( - "benchmark report does not match requested release pair: " - f"found {report_id.current_tag} vs {report_id.baseline_tag}, " + "rerendered benchmark report does not match retained release pair: " + f"found {observed.current_tag} vs {observed.baseline_tag}, " f"expected {expected.current_tag} vs {expected.baseline_tag}" ) raise ValueError(msg) - _write_text(output, report_text) - return report_id + return _promote_report_text( + source_text=report_text, + request=PromotionRequest( + current=current, + archive_dir=archive_dir, + expected=expected, + output=output, + reserved_paths={ + "artifact CSV": artifacts.csv, + "artifact provenance": artifacts.provenance, + }, + ), + ) def resolve_archive_request(options: ArchiveRequestOptions) -> ResolvedArchiveRequest: @@ -1472,6 +1762,16 @@ def build_parser() -> argparse.ArgumentParser: default=_DEFAULT_SOURCE, help=f"Generated report path for --output-only (default: {_DEFAULT_SOURCE})", ) + parser.add_argument( + "--artifact-csv", + default=_DEFAULT_ARTIFACT_CSV, + help=f"Retained release-report CSV path (default: {_DEFAULT_ARTIFACT_CSV})", + ) + parser.add_argument( + "--artifact-provenance", + default=_DEFAULT_ARTIFACT_PROVENANCE, + help=f"Retained release-report JSON provenance path (default: {_DEFAULT_ARTIFACT_PROVENANCE})", + ) parser.add_argument( "--archive-dir", default=_DEFAULT_ARCHIVE_DIR, @@ -1507,6 +1807,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Write the generated report to --output without promoting docs/PERFORMANCE.md.", ) + parser.add_argument( + "--rerender", + action="store_true", + help="Render and promote from retained CSV/JSON artifacts without benchmarks or worktrees.", + ) parser.add_argument( "--worktree-ref", default="HEAD", @@ -1537,6 +1842,8 @@ def _resolve_cli_paths(root: Path, args: argparse.Namespace) -> ArchivePaths: current = Path(args.current) output = Path(args.output) archive_dir = Path(args.archive_dir) + artifact_csv = Path(args.artifact_csv) + artifact_provenance = Path(args.artifact_provenance) if not source.is_absolute(): source = root / source if not current.is_absolute(): @@ -1545,7 +1852,17 @@ def _resolve_cli_paths(root: Path, args: argparse.Namespace) -> ArchivePaths: output = root / output if not archive_dir.is_absolute(): archive_dir = root / archive_dir - return ArchivePaths(source=source, current=current, output=output, archive_dir=archive_dir) + if not artifact_csv.is_absolute(): + artifact_csv = root / artifact_csv + if not artifact_provenance.is_absolute(): + artifact_provenance = root / artifact_provenance + return ArchivePaths( + source=source, + current=current, + output=output, + archive_dir=archive_dir, + artifacts=ArtifactPaths(csv=artifact_csv, provenance=artifact_provenance), + ) def _fetch_required_tags(*, request: ResolvedArchiveRequest, repo_root: Path, include_current: bool) -> None: @@ -1578,6 +1895,7 @@ def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, reque report_id=generate_worktree_report( output=paths.output, config=config, + artifacts=paths.artifacts, ), action="output", ) @@ -1586,6 +1904,7 @@ def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, reque current=paths.current, archive_dir=paths.archive_dir, config=config, + artifacts=paths.artifacts, ), action="promote-generated", ) @@ -1616,19 +1935,45 @@ def main(argv: list[str] | None = None) -> int: paths = _resolve_cli_paths(root, args) try: - request = resolve_archive_request( - ArchiveRequestOptions( - current_tag=args.current_tag, - baseline_tag=args.baseline_tag, - published_latest=args.published_latest, - infer_release=args.infer_release, - current_vs_latest=args.current_vs_latest, - worktree_ref=args.worktree_ref, - repo_root=root, + if args.rerender: + if any( + ( + args.current_tag, + args.baseline_tag, + args.published_latest, + args.infer_release, + args.current_vs_latest, + args.github_assets, + args.generate_in_temp_worktree, + args.output_only, + ) + ): + msg = "--rerender cannot be combined with release selection, generation, GitHub-asset, or output-only options" + raise ValueError(msg) + result = ArchiveResult( + report_id=rerender_and_promote_artifacts( + artifacts=paths.artifacts, + output=paths.output, + current=paths.current, + archive_dir=paths.archive_dir, + ), + action="rerender", ) - ) - result = _run_archive_request(args=args, paths=paths, request=request, repo_root=root) + else: + request = resolve_archive_request( + ArchiveRequestOptions( + current_tag=args.current_tag, + baseline_tag=args.baseline_tag, + published_latest=args.published_latest, + infer_release=args.infer_release, + current_vs_latest=args.current_vs_latest, + worktree_ref=args.worktree_ref, + repo_root=root, + ) + ) + result = _run_archive_request(args=args, paths=paths, request=request, repo_root=root) except ( + ExceptionGroup, ExecutableNotFoundError, OSError, TypeError, @@ -1644,6 +1989,8 @@ def main(argv: list[str] | None = None) -> int: print(f"Generated benchmark report in a temporary worktree and wrote it to {paths.output}") elif result.action == "promote-generated": print(f"Generated benchmark report in a temporary worktree and promoted it to {paths.current}") + elif result.action == "rerender": + print(f"Re-rendered {paths.output} from {paths.artifacts.csv} and promoted it to {paths.current}") else: print(f"Promoted {paths.source} to {paths.current}") print(f"Current performance report: {result.report_id.current_tag} vs {result.report_id.baseline_tag}") diff --git a/scripts/bench_compare.py b/scripts/bench_compare.py index d7f9c99..d958748 100644 --- a/scripts/bench_compare.py +++ b/scripts/bench_compare.py @@ -4,7 +4,8 @@ Reads Criterion output under: target/criterion/{group}/{bench}/{sample}/estimates.json -And writes a local markdown performance report. +Writes a local Markdown performance report and, for release workflows, a +schema-versioned CSV plus JSON provenance pair that can reproduce it. Typical workflow (see docs/RELEASING.md): @@ -20,8 +21,6 @@ just bench-compare """ -from __future__ import annotations - import argparse import json import math @@ -35,6 +34,18 @@ from typing import Literal, Protocol, cast from criterion_dim_plot import METRICS +from performance_artifacts import ( + ArtifactContext, + ArtifactPaths, + PerformanceBundle, + PerformanceRow, + ReleasePair, + ReportSource, + TimingEstimate, + ensure_distinct_paths, + load_bundle, + write_bundle, +) from subprocess_utils import ExecutableNotFoundError, run_git_command # --------------------------------------------------------------------------- @@ -457,44 +468,65 @@ def _read_harness_provenance( msg = f"expected JSON object in {provenance_path}" raise TypeError(msg) + return _parse_harness_provenance( + data, + path=provenance_path, + expected_baseline=expected_baseline, + expected=expected, + ) + + +def _parse_harness_provenance( + data: dict[str, object], + *, + path: Path, + expected_baseline: str, + expected: CriterionSelection, +) -> HarnessProvenance: + """Parse already-decoded harness provenance at a trusted report boundary.""" + schema = data.get("schema") - mode = data.get("mode") - baseline = _required_metadata_string(data, "baseline", provenance_path) + mode_value = data.get("mode") + if not isinstance(mode_value, str): + msg = f"unsupported or missing mode in {path}: {mode_value!r}" + raise TypeError(msg) + mode = mode_value + baseline = _required_metadata_string(data, "baseline", path) if baseline != expected_baseline: - msg = f"benchmark harness provenance baseline {baseline!r} does not match requested Criterion baseline {expected_baseline!r} in {provenance_path}" + msg = f"benchmark harness provenance baseline {baseline!r} does not match requested Criterion baseline {expected_baseline!r} in {path}" raise ValueError(msg) if not isinstance(schema, bool) and schema == 1: if mode != "shared-current-harness": - msg = f"unsupported or missing mode in {provenance_path}: {mode!r}" + msg = f"unsupported or missing mode in {path}: {mode!r}" raise ValueError(msg) - sha256 = _required_sha256(data, "sha256", provenance_path) + sha256 = _required_sha256(data, "sha256", path) return HarnessProvenance(schema=1, mode=mode, sha256=sha256, baseline=baseline) if isinstance(schema, bool) or schema != 2: - msg = f"unsupported or missing schema in {provenance_path}: expected 1 or 2, got {schema!r}" + msg = f"unsupported or missing schema in {path}: expected 1 or 2, got {schema!r}" raise ValueError(msg) if mode not in {"shared-current-harness", "historical-assets"}: - msg = f"unsupported or missing mode in {provenance_path}: {mode!r}" + msg = f"unsupported or missing mode in {path}: {mode!r}" raise ValueError(msg) - measurement = _required_metadata_object(data, "measurement", provenance_path) - publication = _required_metadata_object(data, "publication", provenance_path) - criterion_data = _required_metadata_object(data, "criterion", provenance_path) - validation = _required_metadata_object(data, "validation", provenance_path) - _validate_measurement_metadata(measurement, mode=mode, path=provenance_path) - _validate_environment_metadata(publication, path=provenance_path, context="publication") + measurement = _required_metadata_object(data, "measurement", path) + publication = _required_metadata_object(data, "publication", path) + criterion_data = _required_metadata_object(data, "criterion", path) + validation = _required_metadata_object(data, "validation", path) + _validate_measurement_metadata(measurement, mode=mode, path=path) + _validate_environment_metadata(publication, path=path, context="publication") criterion = _parse_criterion_metadata( criterion_data, - path=provenance_path, + path=path, expected=expected, ) - _validate_validation_metadata(validation, path=provenance_path) - _validate_baseline_api_compatibility(validation, baseline=baseline, path=provenance_path) + _validate_validation_metadata(validation, path=path) + _validate_baseline_api_compatibility(validation, baseline=baseline, path=path) sha256: str | None = None if measurement.get("status") == "recorded": - sha256 = _required_sha256(measurement, "harness_sha256", provenance_path) + sha256 = _required_sha256(measurement, "harness_sha256", path) return HarnessProvenance( schema=2, mode=mode, @@ -1287,6 +1319,160 @@ def _coverage_table(gaps: list[CoverageGap], baseline_name: str) -> str: return "\n".join(lines) +def _artifact_timing(estimate: CriterionEstimate) -> TimingEstimate: + """Convert a Criterion estimate into the stricter persisted timing model.""" + if estimate.ci_lo_ns is None or estimate.ci_hi_ns is None: + msg = "release-performance artifacts require complete Criterion confidence intervals" + raise ValueError(msg) + return TimingEstimate( + median_ns=estimate.point_ns, + ci_lower_ns=estimate.ci_lo_ns, + ci_upper_ns=estimate.ci_hi_ns, + ) + + +def _criterion_timing(estimate: TimingEstimate) -> CriterionEstimate: + """Convert a validated persisted timing back into the report model.""" + return CriterionEstimate( + point_ns=estimate.median_ns, + ci_lo_ns=estimate.ci_lower_ns, + ci_hi_ns=estimate.ci_upper_ns, + ) + + +def _comparison_artifact_rows( + comparisons: list[Comparison], + *, + scope: str, +) -> list[PerformanceRow]: + """Convert complete comparisons into deterministic schema rows.""" + return [ + PerformanceRow( + suite=comparison.suite, + scope=scope, + benchmark_id=f"{comparison.group}/{comparison.bench}", + group=comparison.group, + benchmark=comparison.bench, + baseline_benchmark=comparison.baseline_bench or comparison.bench, + coverage_status="comparable", + coverage_note="", + baseline=_artifact_timing(comparison.baseline), + current=_artifact_timing(comparison.current), + baseline_nalgebra=(None if comparison.baseline_nalgebra is None else _artifact_timing(comparison.baseline_nalgebra)), + baseline_faer=None if comparison.baseline_faer is None else _artifact_timing(comparison.baseline_faer), + ) + for comparison in comparisons + ] + + +def _unavailable_artifact_rows( # noqa: PLR0913 + criterion_dir: Path, + *, + baseline_name: str, + stat: str, + suite: str, + scope: str, + policy: ComparisonPolicy, +) -> list[PerformanceRow]: + """Retain correctness-excluded baseline rows as explicit current-only data.""" + compatibility = policy.baseline_api_compatibility + unavailable = _UNAVAILABLE_BASELINE_ROWS_BY_COMPATIBILITY.get(compatibility or "", frozenset()) + rows: list[PerformanceRow] = [] + for group, bench in sorted(unavailable): + row_suite = "exact" if group.startswith("exact_") else "vs_linalg" + if suite not in ("all", row_suite): + continue + if scope == "release-signal" and row_suite == "exact" and group not in EXACT_RELEASE_SIGNAL_GROUPS: + continue + if scope == "release-signal" and row_suite == "vs_linalg": + dim = _dim_from_vs_linalg_group(group) + if dim is None or bench not in VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dim, []): + continue + current_path = criterion_dir / group / bench / "new" / "estimates.json" + if not current_path.is_file(): + continue + current = _read_estimate(current_path, stat) + rows.append( + PerformanceRow( + suite=row_suite, + scope=scope, + benchmark_id=f"{group}/{bench}", + group=group, + benchmark=bench, + baseline_benchmark=bench, + coverage_status="current-only", + coverage_note=( + f"Baseline {baseline_name} has no correctness-compatible benchmark row under {compatibility or 'the selected compatibility policy'}." + ), + baseline=None, + current=_artifact_timing(current), + ) + ) + return rows + + +def _artifact_comparisons(rows: tuple[PerformanceRow, ...]) -> list[Comparison]: + """Reconstruct comparison rows solely from the retained CSV model.""" + comparisons: list[Comparison] = [] + for row in rows: + if row.coverage_status != "comparable": + continue + if row.baseline is None or row.current is None: + msg = "comparable artifact row lost a required timing" + raise AssertionError(msg) + baseline = _criterion_timing(row.baseline) + current = _criterion_timing(row.current) + comparisons.append( + Comparison( + suite=row.suite, + group=row.group, + bench=row.benchmark, + baseline=baseline, + current=current, + assessment=_assess_change(baseline, current), + baseline_bench=(row.baseline_benchmark if row.baseline_benchmark != row.benchmark else None), + baseline_nalgebra=(None if row.baseline_nalgebra is None else _criterion_timing(row.baseline_nalgebra)), + baseline_faer=None if row.baseline_faer is None else _criterion_timing(row.baseline_faer), + ) + ) + return comparisons + + +def _artifact_coverage_table(rows: tuple[PerformanceRow, ...], baseline_name: str) -> str: + """Render every one-sided measurement and note from retained CSV fields.""" + one_sided = [row for row in rows if row.coverage_status != "comparable"] + if not one_sided: + return "" + lines = [ + "## Coverage Notes", + "", + "One-sided rows retain the available measurement but are excluded from point-estimate change and ratio calculations.", + "", + f"| Benchmark | Coverage | {baseline_name} (point + CI) | Latest (point + CI) | Note |", + "|:----------|:---------|-----------------------------:|--------------------:|:-----|", + ] + for row in one_sided: + baseline = "—" if row.baseline is None else _format_estimate(_criterion_timing(row.baseline)) + current = "—" if row.current is None else _format_estimate(_criterion_timing(row.current)) + lines.append(f"| {row.benchmark_id} | {row.coverage_status} | {baseline} | {current} | {row.coverage_note} |") + return "\n".join(lines) + + +def _artifact_tables(bundle: PerformanceBundle) -> str: + """Render comparison and coverage tables from one validated artifact bundle.""" + comparisons = _artifact_comparisons(bundle.sorted_rows) + sections: list[str] = [] + if comparisons: + sections.append(_comparison_tables(comparisons, bundle.context.release.baseline)) + coverage = _artifact_coverage_table(bundle.sorted_rows, bundle.context.release.baseline) + if coverage: + sections.append(coverage) + if not sections: + msg = "release-performance artifact contains no renderable rows" + raise ValueError(msg) + return "\n\n".join(sections) + + def _read_cargo_version(root: Path) -> str: cargo_toml = root / "Cargo.toml" if not cargo_toml.exists(): @@ -1350,20 +1536,40 @@ def _get_git_source_date(root: Path) -> str: return parsed.astimezone(UTC).strftime("%Y-%m-%d %H:%M:%S UTC") +def _report_source(root: Path) -> ReportSource: + """Capture stable source metadata before report artifacts leave a worktree.""" + short_hash, branch = _get_git_info(root) + return ReportSource( + version=_read_cargo_version(root), + commit=short_hash, + ref=branch, + revision_timestamp=_get_git_source_date(root), + ) + + def _generate_markdown( root: Path, table: str, settings: ReportSettings, + *, + source: ReportSource | None = None, + coverage_from_artifacts: bool = False, ) -> str: """Generate the complete benchmark report content.""" - version = _read_cargo_version(root) - short_hash, branch = _get_git_info(root) - source_date = _get_git_source_date(root) + if source is None: + version = _read_cargo_version(root) + short_hash, branch = _get_git_info(root) + source_date = _get_git_source_date(root) + else: + version = source.version + short_hash = source.commit + branch = source.ref + source_date = source.revision_timestamp lines = [ "# Benchmark Performance", "", - f"**la-stack** v{version} · `{short_hash}` ({branch})", + f"**la-stack** v{version.removeprefix('v')} · `{short_hash}` ({branch})", f"**Source revision timestamp**: {source_date} (deterministic report metadata; not the benchmark measurement time)", "**Benchmark measurement timestamp**: not recorded by Criterion; use the provenance below to identify the measured revisions and environment.", f"**Statistic**: {settings.stat}", @@ -1395,7 +1601,12 @@ def _generate_markdown( "fixture validation are unknown." ) else: - lines.extend(_provenance_markdown(settings.harness_provenance)) + lines.extend( + _provenance_markdown( + settings.harness_provenance, + include_compatibility_rows=not coverage_from_artifacts, + ) + ) else: lines.append("Current performance snapshot (no baseline comparison).") @@ -1414,6 +1625,9 @@ def _generate_markdown( "# Release PR: update docs/PERFORMANCE.md and archive the previous report", "just performance-release", "", + "# Re-render and promote from retained CSV/JSON inputs (no benchmarks)", + "just performance-rerender", + "", "# GitHub Actions release assets", "just performance-github-assets", "", @@ -1423,6 +1637,7 @@ def _generate_markdown( "", "`just performance-local` writes `target/bench-reports/performance.md`.", "`just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`.", + "`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report.", "", "Older curated release-to-release reports are archived in `docs/archive/performance/`.", "", @@ -1433,7 +1648,11 @@ def _generate_markdown( return "\n".join(lines) + "\n" -def _provenance_markdown(provenance: HarnessProvenance) -> list[str]: +def _provenance_markdown( + provenance: HarnessProvenance, + *, + include_compatibility_rows: bool = True, +) -> list[str]: """Render validated provenance without implying facts absent from metadata.""" if provenance.schema == 1: return [ @@ -1519,10 +1738,10 @@ def _provenance_markdown(provenance: HarnessProvenance) -> list[str]: lines.extend( [ f"- Baseline API compatibility: `{compatibility}` selects only source-compatible benchmark calls;", - " rows outside the baseline's correctness domain remain explicitly unavailable.", + " one-sided rows outside the baseline's correctness domain are identified by the retained CSV coverage status and note.", ] ) - if compatibility == _V0_4_3_API_COMPATIBILITY and criterion.suite in {"all", "vs_linalg"}: + if include_compatibility_rows and compatibility == _V0_4_3_API_COMPATIBILITY and criterion.suite in {"all", "vs_linalg"}: lines.extend( [ "- Baseline-unavailable rows: `d8/la_stack_det_from_lu_balanced_range` and", @@ -1530,7 +1749,7 @@ def _provenance_markdown(provenance: HarnessProvenance) -> list[str]: " fixture whose exact determinant is one; current samples remain required, but no speedup is claimed.", ] ) - if compatibility == _V0_4_3_API_COMPATIBILITY and criterion.suite in {"all", "exact"}: + if include_compatibility_rows and compatibility == _V0_4_3_API_COMPATIBILITY and criterion.suite in {"all", "exact"}: lines.extend( [ "- Baseline-unavailable rows: `exact_d2/det_direct_with_errbound`,", @@ -1541,6 +1760,94 @@ def _provenance_markdown(provenance: HarnessProvenance) -> list[str]: return lines +def _read_raw_harness_provenance(criterion_dir: Path) -> dict[str, object]: + """Read the already-validated harness provenance for durable embedding.""" + path = criterion_dir / ".la-stack-benchmark-harness.json" + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + msg = f"malformed benchmark harness provenance JSON in {path}: {exc}" + raise ValueError(msg) from exc + if not isinstance(raw, dict) or not all(isinstance(key, str) for key in raw): + msg = f"expected JSON object in {path}" + raise TypeError(msg) + return cast("dict[str, object]", raw) + + +def _release_artifact_bundle( + *, + root: Path, + criterion_dir: Path, + baseline_name: str, + settings: ReportSettings, + collection: ComparisonCollection, +) -> PerformanceBundle: + """Build the trusted report dataset after Criterion and coverage validation.""" + provenance = settings.harness_provenance + if provenance is None or provenance.schema != 2: + msg = "release-performance artifacts require complete schema-2 benchmark provenance" + raise ValueError(msg) + policy = _comparison_policy(settings.scope, provenance) + rows = _comparison_artifact_rows(collection.comparisons, scope=settings.scope) + rows.extend( + _unavailable_artifact_rows( + criterion_dir, + baseline_name=baseline_name, + stat=settings.stat, + suite=settings.suite, + scope=settings.scope, + policy=policy, + ) + ) + source = _report_source(root) + return PerformanceBundle( + context=ArtifactContext( + release=ReleasePair( + current=f"v{source.version.removeprefix('v')}", + baseline=baseline_name, + ), + statistic="median", + suite=settings.suite, + scope=settings.scope, + source=source, + benchmark_provenance=_read_raw_harness_provenance(criterion_dir), + ), + rows=tuple(rows), + ) + + +def render_release_artifacts(paths: ArtifactPaths) -> str: + """Reload, validate, and render a report without Criterion, Cargo, or Git.""" + bundle = load_bundle(paths) + context = bundle.context + selection = CriterionSelection( + suite=cast("BenchmarkSuite", context.suite), + scope=cast("ComparisonScope", context.scope), + statistic="median", + sample="new", + ) + harness_provenance = _parse_harness_provenance( + context.benchmark_provenance, + path=paths.provenance, + expected_baseline=context.release.baseline, + expected=selection, + ) + settings = ReportSettings( + baseline_name=context.release.baseline, + stat="median", + suite=cast("BenchmarkSuite", context.suite), + scope=cast("ComparisonScope", context.scope), + harness_provenance=harness_provenance, + ) + return _generate_markdown( + paths.csv.parent, + _artifact_tables(bundle), + settings, + source=context.source, + coverage_from_artifacts=True, + ) + + # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- @@ -1584,11 +1891,23 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: default="target/criterion", help="Criterion output directory (default: target/criterion).", ) + parser.add_argument( + "--repo-root", + help="Source checkout used for report metadata (default: the checkout containing this script).", + ) parser.add_argument( "--output", default="target/bench-reports/performance.md", help="Output markdown file (default: target/bench-reports/performance.md).", ) + parser.add_argument( + "--csv-output", + help="Write schema-versioned release-report CSV input; requires an adjacent, distinct --provenance-output.", + ) + parser.add_argument( + "--provenance-output", + help="Write an adjacent release-report JSON provenance sidecar; requires a distinct --csv-output.", + ) return parser.parse_args(argv) @@ -1619,7 +1938,7 @@ def _save_baseline_hint(suite: str, baseline: str) -> str: return f"just bench-save-baseline {baseline}" -def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911 +def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, PLR0915 """Generate a benchmark snapshot or comparison report from CLI arguments.""" args = _parse_args(sys.argv[1:] if argv is None else argv) stat = cast("Statistic", args.stat) @@ -1627,10 +1946,35 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911 scope = cast("ComparisonScope", args.scope) selection = CriterionSelection(suite=suite, scope=scope, statistic=stat, sample="new") - root = _repo_root() + root = Path(args.repo_root).resolve() if args.repo_root is not None else _repo_root() criterion_dir = root / args.criterion_dir output_path = Path(args.output) if Path(args.output).is_absolute() else root / args.output + if (args.csv_output is None) != (args.provenance_output is None): + print("--csv-output and --provenance-output must be provided together", file=sys.stderr) + return 2 + if args.csv_output is not None and (args.snapshot or stat != "median"): + print("release-performance artifacts require a median baseline comparison", file=sys.stderr) + return 2 + + artifact_paths: ArtifactPaths | None = None + if args.csv_output is not None and args.provenance_output is not None: + try: + artifact_paths = ArtifactPaths( + csv=Path(args.csv_output) if Path(args.csv_output).is_absolute() else root / args.csv_output, + provenance=(Path(args.provenance_output) if Path(args.provenance_output).is_absolute() else root / args.provenance_output), + ) + ensure_distinct_paths( + { + "Markdown output": output_path, + "artifact CSV": artifact_paths.csv, + "artifact provenance": artifact_paths.provenance, + } + ) + except (OSError, ValueError) as err: + print(f"Invalid release-performance artifact paths: {err}", file=sys.stderr) + return 2 + if not criterion_dir.is_dir(): print( f"No Criterion results found at {criterion_dir}.\nRun benchmarks first:\n {_run_bench_hint(args.suite)}\n", @@ -1640,6 +1984,7 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911 baseline_name = None if args.snapshot else args.baseline harness_provenance: HarnessProvenance | None = None + collection: ComparisonCollection | None = None if baseline_name: try: @@ -1718,7 +2063,26 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911 scope=scope, harness_provenance=harness_provenance, ) - md = _generate_markdown(root, table, settings) + if args.csv_output is not None and args.provenance_output is not None: + if baseline_name is None or collection is None or artifact_paths is None: + msg = "release-performance artifact invariant violated" + raise AssertionError(msg) + try: + bundle = _release_artifact_bundle( + root=root, + criterion_dir=criterion_dir, + baseline_name=baseline_name, + settings=settings, + collection=collection, + ) + write_bundle(artifact_paths, bundle) + md = render_release_artifacts(artifact_paths) + except (ExceptionGroup, OSError, KeyError, TypeError, ValueError) as err: + print(f"Invalid release-performance artifact data: {err}", file=sys.stderr) + return 2 + print(f"📊 Wrote {artifact_paths.csv} and {artifact_paths.provenance}") + else: + md = _generate_markdown(root, table, settings) output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(md, encoding="utf-8") diff --git a/scripts/performance_artifacts.py b/scripts/performance_artifacts.py new file mode 100644 index 0000000..cd7b040 --- /dev/null +++ b/scripts/performance_artifacts.py @@ -0,0 +1,841 @@ +"""Schema-versioned release-performance CSV and provenance artifacts.""" + +import csv +import hashlib +import io +import json +import math +import os +import tempfile +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Literal, cast + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + +SCHEMA_VERSION = 1 +SUITES = ("all", "exact", "vs_linalg") +SCOPES = ("release-signal", "all-benches") +COVERAGE_STATES = ("comparable", "current-only", "baseline-only") + +type CoverageState = Literal["comparable", "current-only", "baseline-only"] + +CSV_COLUMNS = ( + "schema_version", + "suite", + "scope", + "benchmark_id", + "group", + "benchmark", + "baseline_benchmark", + "coverage_status", + "coverage_note", + "baseline_median_ns", + "baseline_ci_lower_ns", + "baseline_ci_upper_ns", + "current_median_ns", + "current_ci_lower_ns", + "current_ci_upper_ns", + "baseline_nalgebra_median_ns", + "baseline_nalgebra_ci_lower_ns", + "baseline_nalgebra_ci_upper_ns", + "baseline_faer_median_ns", + "baseline_faer_ci_lower_ns", + "baseline_faer_ci_upper_ns", +) + + +@dataclass(frozen=True, slots=True) +class TimingEstimate: + """A finite positive median and its complete confidence interval.""" + + median_ns: float + ci_lower_ns: float + ci_upper_ns: float + + def __post_init__(self) -> None: + """Reject non-finite, non-positive, or inconsistent timing intervals.""" + for field, value in ( + ("median_ns", self.median_ns), + ("ci_lower_ns", self.ci_lower_ns), + ("ci_upper_ns", self.ci_upper_ns), + ): + if not math.isfinite(value) or value <= 0: + msg = f"{field} must be finite and positive: {value!r}" + raise ValueError(msg) + if not self.ci_lower_ns <= self.median_ns <= self.ci_upper_ns: + msg = f"confidence interval must contain the median: {self.ci_lower_ns} <= {self.median_ns} <= {self.ci_upper_ns}" + raise ValueError(msg) + + +@dataclass(frozen=True, slots=True) +class PerformanceRow: + """One validated row in the release-performance reporting dataset.""" + + suite: str + scope: str + benchmark_id: str + group: str + benchmark: str + baseline_benchmark: str + coverage_status: CoverageState + coverage_note: str + baseline: TimingEstimate | None + current: TimingEstimate | None + baseline_nalgebra: TimingEstimate | None = None + baseline_faer: TimingEstimate | None = None + + def __post_init__(self) -> None: # noqa: C901 + """Preserve identity, coverage, and optional-timing invariants.""" + if self.suite not in SUITES: + msg = f"unsupported suite: {self.suite!r}" + raise ValueError(msg) + if self.scope not in SCOPES: + msg = f"unsupported scope: {self.scope!r}" + raise ValueError(msg) + for field, value in ( + ("benchmark_id", self.benchmark_id), + ("group", self.group), + ("benchmark", self.benchmark), + ("baseline_benchmark", self.baseline_benchmark), + ): + if not value.strip(): + msg = f"{field} must not be empty" + raise ValueError(msg) + expected_id = f"{self.group}/{self.benchmark}" + if self.benchmark_id != expected_id: + msg = f"benchmark_id must be {expected_id!r}, got {self.benchmark_id!r}" + raise ValueError(msg) + if self.coverage_status not in COVERAGE_STATES: + msg = f"unsupported coverage status: {self.coverage_status!r}" + raise ValueError(msg) + + expected_presence = { + "comparable": (True, True), + "current-only": (False, True), + "baseline-only": (True, False), + }[self.coverage_status] + observed_presence = (self.baseline is not None, self.current is not None) + if observed_presence != expected_presence: + msg = f"coverage status {self.coverage_status!r} requires baseline/current presence {expected_presence}, got {observed_presence}" + raise ValueError(msg) + if self.coverage_status == "comparable" and self.coverage_note: + msg = "comparable rows must not contain a coverage note" + raise ValueError(msg) + if self.coverage_status != "comparable" and not self.coverage_note.strip(): + msg = f"{self.coverage_status} rows require a coverage note" + raise ValueError(msg) + if (self.baseline_nalgebra is not None or self.baseline_faer is not None) and self.baseline is None: + msg = "baseline peer timings require a baseline la-stack timing" + raise ValueError(msg) + + +@dataclass(frozen=True, slots=True) +class ReleasePair: + """The current and baseline release identifiers represented by a report.""" + + current: str + baseline: str + + def __post_init__(self) -> None: + """Require two distinct, non-empty release identifiers.""" + if not self.current.strip() or not self.baseline.strip(): + msg = "current and baseline releases must not be empty" + raise ValueError(msg) + if self.current == self.baseline: + msg = "current and baseline releases must differ" + raise ValueError(msg) + + +@dataclass(frozen=True, slots=True) +class ReportSource: + """Stable source metadata needed to reproduce the report header.""" + + version: str + commit: str + ref: str + revision_timestamp: str + + def __post_init__(self) -> None: + """Reject incomplete source metadata before artifact publication.""" + for field, value in ( + ("version", self.version), + ("commit", self.commit), + ("ref", self.ref), + ("revision_timestamp", self.revision_timestamp), + ): + if not value.strip() or value == "unknown": + msg = f"report source {field} is incomplete: {value!r}" + raise ValueError(msg) + + +@dataclass(frozen=True, slots=True) +class ArtifactContext: + """Validated non-tabular metadata for a release-performance dataset.""" + + release: ReleasePair + statistic: Literal["median"] + suite: str + scope: str + source: ReportSource + benchmark_provenance: dict[str, object] + + def __post_init__(self) -> None: + """Bind report settings and benchmark provenance to the release pair.""" + if self.statistic != "median": + msg = f"unsupported release-report statistic: {self.statistic!r}" + raise ValueError(msg) + if self.suite not in SUITES: + msg = f"unsupported suite: {self.suite!r}" + raise ValueError(msg) + if self.scope not in SCOPES: + msg = f"unsupported scope: {self.scope!r}" + raise ValueError(msg) + _validate_benchmark_provenance( + self.benchmark_provenance, + context=self, + ) + + +@dataclass(frozen=True, slots=True) +class PerformanceBundle: + """A complete validated release-performance dataset and its provenance.""" + + context: ArtifactContext + rows: tuple[PerformanceRow, ...] + + def __post_init__(self) -> None: + """Require non-empty, unique rows bound to one report selection.""" + if not self.rows: + msg = "release-performance dataset must contain at least one row" + raise ValueError(msg) + keys: set[tuple[str, str, str]] = set() + for row in self.rows: + if row.suite not in _selected_suites(self.context.suite): + msg = f"row suite {row.suite!r} is outside selected suite {self.context.suite!r}" + raise ValueError(msg) + if row.scope != self.context.scope: + msg = f"row scope {row.scope!r} does not match report scope {self.context.scope!r}" + raise ValueError(msg) + key = (row.suite, row.scope, row.benchmark_id) + if key in keys: + msg = f"duplicate benchmark key: {key!r}" + raise ValueError(msg) + keys.add(key) + + @property + def sorted_rows(self) -> tuple[PerformanceRow, ...]: + """Return rows in deterministic suite/group/benchmark order.""" + return tuple(sorted(self.rows, key=lambda row: (row.suite, row.group, row.benchmark, row.baseline_benchmark))) + + +@dataclass(frozen=True, slots=True) +class ArtifactPaths: + """Adjacent CSV and JSON provenance destinations.""" + + csv: Path + provenance: Path + + def __post_init__(self) -> None: + """Require distinct adjacent artifact files.""" + ensure_distinct_paths({"CSV": self.csv, "provenance": self.provenance}) + if self.csv.resolve(strict=False).parent != self.provenance.resolve(strict=False).parent: + msg = "CSV and provenance sidecar must be adjacent" + raise ValueError(msg) + + +def _selected_suites(suite: str) -> frozenset[str]: + return frozenset({"exact", "vs_linalg"} if suite == "all" else {suite}) + + +def _paths_alias(first: Path, second: Path) -> bool: + """Return whether two paths resolve to the same filesystem target.""" + if first.exists() and second.exists() and first.samefile(second): + return True + return first.resolve(strict=False) == second.resolve(strict=False) + + +def ensure_distinct_paths(paths: Mapping[str, Path]) -> None: + """Reject lexical, symlink, or existing-file aliases among named paths.""" + items = tuple(paths.items()) + for index, (first_name, first_path) in enumerate(items): + for second_name, second_path in items[index + 1 :]: + if _paths_alias(first_path, second_path): + msg = f"{first_name} and {second_name} must use distinct paths: {first_path}" + raise ValueError(msg) + + +def _required_provenance_object(data: Mapping[str, object], field: str, *, context: str) -> dict[str, object]: + value = data.get(field) + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + msg = f"benchmark provenance requires a {field} object in {context}" + raise TypeError(msg) + return cast("dict[str, object]", value) + + +def _required_provenance_string(data: Mapping[str, object], field: str, *, context: str) -> str: + value = data.get(field) + if not isinstance(value, str) or not value.strip(): + msg = f"benchmark provenance requires a non-empty {context}.{field} string" + raise ValueError(msg) + return value + + +def _required_provenance_sha256(data: Mapping[str, object], field: str, *, context: str) -> str: + value = _required_provenance_string(data, field, context=context) + if len(value) != 64 or any(char not in "0123456789abcdef" for char in value): + msg = f"benchmark provenance {context}.{field} must be a lowercase SHA-256 digest" + raise ValueError(msg) + return value + + +def _required_provenance_bool(data: Mapping[str, object], field: str, *, context: str) -> bool: + value = data.get(field) + if not isinstance(value, bool): + msg = f"benchmark provenance requires a boolean {context}.{field}" + raise TypeError(msg) + return value + + +def _required_provenance_command(data: Mapping[str, object], field: str, *, context: str) -> tuple[str, ...]: + value = data.get(field) + if not isinstance(value, list) or not value or not all(isinstance(part, str) and part for part in value): + msg = f"benchmark provenance requires a non-empty {context}.{field} command" + raise ValueError(msg) + return tuple(cast("list[str]", value)) + + +def _validate_environment_provenance(data: Mapping[str, object], *, context: str) -> None: + for field in ("cpu", "os", "rustc", "commit"): + _required_provenance_string(data, field, context=context) + for field in ("cargo_lock_sha256", "harness_sha256", "source_state_sha256"): + _required_provenance_sha256(data, field, context=context) + _required_provenance_bool(data, "git_clean", context=context) + gate = _required_provenance_string(data, "correctness_gate", context=context) + if gate != "passed": + msg = f"benchmark provenance {context}.correctness_gate must be 'passed', got {gate!r}" + raise ValueError(msg) + + +def _require_matching_provenance( + first: Mapping[str, object], + second: Mapping[str, object], + field: str, + *, + first_context: str, + second_context: str, +) -> None: + if first.get(field) != second.get(field): + msg = f"benchmark provenance {first_context}.{field} {first.get(field)!r} does not match {second_context}.{field} {second.get(field)!r}" + raise ValueError(msg) + + +def _validate_criterion_provenance(criterion: Mapping[str, object], *, context: ArtifactContext) -> None: + """Validate complete Criterion settings and commands.""" + expected = { + "statistic": context.statistic, + "suite": context.suite, + "scope": context.scope, + "sample": "new", + } + for field, expected_value in expected.items(): + if criterion.get(field) != expected_value: + msg = f"benchmark provenance criterion.{field} {criterion.get(field)!r} does not match report value {expected_value!r}" + raise ValueError(msg) + _required_provenance_string(criterion, "criterion_version", context="criterion") + _required_provenance_command(criterion, "baseline_command", context="criterion") + _required_provenance_command(criterion, "current_command", context="criterion") + + +def _validate_measurement_provenance(measurement: Mapping[str, object], *, mode: object) -> str: + """Validate recorded or explicitly unavailable measurement metadata.""" + measurement_status = _required_provenance_string(measurement, "status", context="measurement") + if measurement_status == "recorded": + if mode != "shared-current-harness": + msg = "recorded benchmark measurement provenance requires shared-current-harness mode" + raise ValueError(msg) + for field in ("cpu", "os", "rustc", "current_commit", "baseline_commit"): + _required_provenance_string(measurement, field, context="measurement") + for field in ( + "cargo_lock_sha256", + "harness_sha256", + "current_source_state_sha256", + "baseline_source_state_sha256", + ): + _required_provenance_sha256(measurement, field, context="measurement") + for field in ("current_git_clean", "baseline_git_clean"): + _required_provenance_bool(measurement, field, context="measurement") + elif measurement_status == "unavailable": + _required_provenance_string(measurement, "reason", context="measurement") + else: + msg = f"unsupported benchmark provenance measurement.status: {measurement_status!r}" + raise ValueError(msg) + return measurement_status + + +def _validate_validation_provenance(validation: Mapping[str, object]) -> str: + """Validate fixture-gate evidence for both compared revisions.""" + if validation.get("command") != ["just", "test-bench-inputs"]: + msg = "benchmark provenance validation.command must be ['just', 'test-bench-inputs']" + raise ValueError(msg) + for field in ("current_revision", "baseline_revision"): + result = _required_provenance_string(validation, field, context="validation") + if result != "passed": + msg = f"benchmark provenance validation.{field} must be 'passed', got {result!r}" + raise ValueError(msg) + for field in ("current_commit", "baseline_commit"): + _required_provenance_string(validation, field, context="validation") + for field in ("current_source_state_sha256", "baseline_source_state_sha256"): + _required_provenance_sha256(validation, field, context="validation") + for field in ("current_git_clean", "baseline_git_clean"): + _required_provenance_bool(validation, field, context="validation") + compatibility = _required_provenance_string(validation, "baseline_api_compatibility", context="validation") + harness = _required_provenance_string(validation, "harness", context="validation") + if harness != "shared-current": + msg = f"benchmark provenance validation.harness must be 'shared-current', got {harness!r}" + raise ValueError(msg) + return compatibility + + +def _validate_current_revision_consistency( + publication: Mapping[str, object], + validation: Mapping[str, object], +) -> None: + """Bind validation of the current revision to publication metadata.""" + for field in ("commit", "git_clean", "source_state_sha256"): + validation_field = f"current_{field}" if field != "commit" else "current_commit" + publication_field = field + if validation.get(validation_field) != publication.get(publication_field): + msg = ( + f"benchmark provenance validation.{validation_field} {validation.get(validation_field)!r} does not match " + f"publication.{publication_field} {publication.get(publication_field)!r}" + ) + raise ValueError(msg) + + +def _validate_recorded_measurement_consistency( + measurement: Mapping[str, object], + publication: Mapping[str, object], + validation: Mapping[str, object], + *, + compatibility: str, +) -> None: + """Bind recorded timing metadata to publication and validation evidence.""" + for field in ("cpu", "os", "rustc", "cargo_lock_sha256", "harness_sha256"): + _require_matching_provenance( + measurement, + publication, + field, + first_context="measurement", + second_context="publication", + ) + for measurement_field, publication_field in ( + ("current_commit", "commit"), + ("current_git_clean", "git_clean"), + ("current_source_state_sha256", "source_state_sha256"), + ): + if measurement.get(measurement_field) != publication.get(publication_field): + msg = ( + f"benchmark provenance measurement.{measurement_field} {measurement.get(measurement_field)!r} does not match " + f"publication.{publication_field} {publication.get(publication_field)!r}" + ) + raise ValueError(msg) + for field in ("baseline_commit", "baseline_git_clean", "baseline_source_state_sha256"): + _require_matching_provenance( + measurement, + validation, + field, + first_context="measurement", + second_context="validation", + ) + measurement_compatibility = measurement.get("baseline_api_compatibility") + if measurement_compatibility is not None and measurement_compatibility != compatibility: + msg = "benchmark provenance measurement.baseline_api_compatibility does not match validation.baseline_api_compatibility" + raise ValueError(msg) + + +def _validate_benchmark_provenance(data: Mapping[str, object], *, context: ArtifactContext) -> None: + """Reject incomplete or mismatched embedded benchmark provenance.""" + schema = data.get("schema") + if type(schema) is not int or schema != 2: + msg = f"benchmark provenance schema must be 2, got {data.get('schema')!r}" + raise ValueError(msg) + mode = data.get("mode") + if mode not in {"shared-current-harness", "historical-assets"}: + msg = f"unsupported benchmark provenance mode: {mode!r}" + raise ValueError(msg) + if data.get("baseline") != context.release.baseline: + msg = f"benchmark provenance baseline {data.get('baseline')!r} does not match release baseline {context.release.baseline!r}" + raise ValueError(msg) + + criterion = _required_provenance_object(data, "criterion", context="root") + measurement = _required_provenance_object(data, "measurement", context="root") + publication = _required_provenance_object(data, "publication", context="root") + validation = _required_provenance_object(data, "validation", context="root") + _validate_criterion_provenance(criterion, context=context) + _validate_environment_provenance(publication, context="publication") + source_commit = _required_provenance_string(publication, "commit", context="publication") + if source_commit != "unavailable" and not source_commit.startswith(context.source.commit): + msg = f"benchmark provenance publication.commit {source_commit!r} does not match report source commit {context.source.commit!r}" + raise ValueError(msg) + measurement_status = _validate_measurement_provenance(measurement, mode=mode) + compatibility = _validate_validation_provenance(validation) + _validate_current_revision_consistency(publication, validation) + if measurement_status == "recorded": + _validate_recorded_measurement_consistency( + measurement, + publication, + validation, + compatibility=compatibility, + ) + + +def _timing_fields(prefix: str, estimate: TimingEstimate | None) -> dict[str, str]: + if estimate is None: + return { + f"{prefix}_median_ns": "", + f"{prefix}_ci_lower_ns": "", + f"{prefix}_ci_upper_ns": "", + } + return { + f"{prefix}_median_ns": format(estimate.median_ns, ".17g"), + f"{prefix}_ci_lower_ns": format(estimate.ci_lower_ns, ".17g"), + f"{prefix}_ci_upper_ns": format(estimate.ci_upper_ns, ".17g"), + } + + +def _row_to_csv(row: PerformanceRow) -> dict[str, str]: + values = { + "schema_version": str(SCHEMA_VERSION), + "suite": row.suite, + "scope": row.scope, + "benchmark_id": row.benchmark_id, + "group": row.group, + "benchmark": row.benchmark, + "baseline_benchmark": row.baseline_benchmark, + "coverage_status": row.coverage_status, + "coverage_note": row.coverage_note, + } + values.update(_timing_fields("baseline", row.baseline)) + values.update(_timing_fields("current", row.current)) + values.update(_timing_fields("baseline_nalgebra", row.baseline_nalgebra)) + values.update(_timing_fields("baseline_faer", row.baseline_faer)) + return values + + +def _serialize_csv(bundle: PerformanceBundle) -> bytes: + output = io.StringIO(newline="") + writer = csv.DictWriter(output, fieldnames=CSV_COLUMNS, lineterminator="\n") + writer.writeheader() + for row in bundle.sorted_rows: + writer.writerow(_row_to_csv(row)) + return output.getvalue().encode("utf-8") + + +def _serialize_provenance(bundle: PerformanceBundle, csv_payload: bytes) -> bytes: + context = bundle.context + payload = { + "benchmark_provenance": context.benchmark_provenance, + "csv": { + "columns": list(CSV_COLUMNS), + "row_count": len(bundle.rows), + "sha256": hashlib.sha256(csv_payload).hexdigest(), + }, + "release": { + "baseline": context.release.baseline, + "current": context.release.current, + }, + "report": { + "scope": context.scope, + "source": { + "commit": context.source.commit, + "ref": context.source.ref, + "revision_timestamp": context.source.revision_timestamp, + "version": context.source.version, + }, + "statistic": context.statistic, + "suite": context.suite, + }, + "schema_version": SCHEMA_VERSION, + } + return (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode() + + +def serialize_bundle(bundle: PerformanceBundle) -> tuple[bytes, bytes]: + """Serialize and self-validate one deterministic artifact pair.""" + csv_payload = _serialize_csv(bundle) + provenance_payload = _serialize_provenance(bundle, csv_payload) + parsed = load_bundle_bytes(csv_payload, provenance_payload, source="serialized artifact pair") + if parsed != PerformanceBundle(context=bundle.context, rows=bundle.sorted_rows): + msg = "release-performance artifact round trip changed the validated dataset" + raise ValueError(msg) + return csv_payload, provenance_payload + + +def _parse_required_string(data: Mapping[str, object], field: str, *, source: str) -> str: + value = data.get(field) + if not isinstance(value, str) or not value.strip(): + msg = f"invalid or missing {field} in {source}" + raise ValueError(msg) + return value + + +def _parse_required_object(data: Mapping[str, object], field: str, *, source: str) -> dict[str, object]: + value = data.get(field) + if not isinstance(value, dict) or not all(isinstance(key, str) for key in value): + msg = f"invalid or missing {field} object in {source}" + raise ValueError(msg) + return cast("dict[str, object]", value) + + +def _parse_float(value: str, field: str, *, row_number: int, source: str) -> float: + try: + parsed = float(value) + except ValueError as exc: + msg = f"{field} in CSV row {row_number} of {source} is not numeric: {value!r}" + raise ValueError(msg) from exc + if not math.isfinite(parsed) or parsed <= 0: + msg = f"{field} in CSV row {row_number} of {source} must be finite and positive: {value!r}" + raise ValueError(msg) + return parsed + + +def _parse_timing(row: Mapping[str, str], prefix: str, *, row_number: int, source: str) -> TimingEstimate | None: + field_names = ( + f"{prefix}_median_ns", + f"{prefix}_ci_lower_ns", + f"{prefix}_ci_upper_ns", + ) + values = tuple(row[field] for field in field_names) + if not any(values): + return None + if not all(values): + msg = f"incomplete {prefix} confidence interval in CSV row {row_number} of {source}" + raise ValueError(msg) + return TimingEstimate( + median_ns=_parse_float(values[0], field_names[0], row_number=row_number, source=source), + ci_lower_ns=_parse_float(values[1], field_names[1], row_number=row_number, source=source), + ci_upper_ns=_parse_float(values[2], field_names[2], row_number=row_number, source=source), + ) + + +def _parse_rows(csv_payload: bytes, *, source: str) -> tuple[PerformanceRow, ...]: + try: + text = csv_payload.decode("utf-8") + except UnicodeDecodeError as exc: + msg = f"release-performance CSV is not valid UTF-8 in {source}: {exc}" + raise ValueError(msg) from exc + reader = csv.DictReader(io.StringIO(text, newline="")) + if reader.fieldnames != list(CSV_COLUMNS): + msg = f"unsupported release-performance CSV columns in {source}: {reader.fieldnames!r}" + raise ValueError(msg) + rows: list[PerformanceRow] = [] + for row_number, raw in enumerate(reader, start=2): + if None in raw or any(value is None for value in raw.values()): + msg = f"malformed release-performance CSV row {row_number} in {source}" + raise ValueError(msg) + row = cast("dict[str, str]", raw) + if row["schema_version"] != str(SCHEMA_VERSION): + msg = f"unsupported CSV schema version in row {row_number} of {source}: {row['schema_version']!r}" + raise ValueError(msg) + coverage = row["coverage_status"] + if coverage not in COVERAGE_STATES: + msg = f"unsupported coverage status in row {row_number} of {source}: {coverage!r}" + raise ValueError(msg) + rows.append( + PerformanceRow( + suite=row["suite"], + scope=row["scope"], + benchmark_id=row["benchmark_id"], + group=row["group"], + benchmark=row["benchmark"], + baseline_benchmark=row["baseline_benchmark"], + coverage_status=coverage, + coverage_note=row["coverage_note"], + baseline=_parse_timing(row, "baseline", row_number=row_number, source=source), + current=_parse_timing(row, "current", row_number=row_number, source=source), + baseline_nalgebra=_parse_timing(row, "baseline_nalgebra", row_number=row_number, source=source), + baseline_faer=_parse_timing(row, "baseline_faer", row_number=row_number, source=source), + ) + ) + return tuple(rows) + + +def _parse_provenance(payload: bytes, *, source: str) -> tuple[ArtifactContext, str, int, tuple[str, ...]]: + try: + raw = json.loads(payload) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + msg = f"malformed release-performance provenance JSON in {source}: {exc}" + raise ValueError(msg) from exc + if not isinstance(raw, dict): + msg = f"release-performance provenance must be a JSON object in {source}" + raise TypeError(msg) + data = cast("dict[str, object]", raw) + schema_version = data.get("schema_version") + if type(schema_version) is not int or schema_version != SCHEMA_VERSION: + msg = f"unsupported release-performance provenance schema in {source}: {data.get('schema_version')!r}" + raise ValueError(msg) + + release_data = _parse_required_object(data, "release", source=source) + report_data = _parse_required_object(data, "report", source=source) + source_data = _parse_required_object(report_data, "source", source=source) + csv_data = _parse_required_object(data, "csv", source=source) + benchmark_provenance = _parse_required_object(data, "benchmark_provenance", source=source) + + release = ReleasePair( + current=_parse_required_string(release_data, "current", source=source), + baseline=_parse_required_string(release_data, "baseline", source=source), + ) + statistic = _parse_required_string(report_data, "statistic", source=source) + if statistic != "median": + msg = f"unsupported report statistic in {source}: {statistic!r}" + raise ValueError(msg) + context = ArtifactContext( + release=release, + statistic="median", + suite=_parse_required_string(report_data, "suite", source=source), + scope=_parse_required_string(report_data, "scope", source=source), + source=ReportSource( + version=_parse_required_string(source_data, "version", source=source), + commit=_parse_required_string(source_data, "commit", source=source), + ref=_parse_required_string(source_data, "ref", source=source), + revision_timestamp=_parse_required_string(source_data, "revision_timestamp", source=source), + ), + benchmark_provenance=benchmark_provenance, + ) + expected_current = f"v{context.source.version.removeprefix('v')}" + if context.release.current != expected_current: + msg = f"release current {context.release.current!r} does not match report source version {context.source.version!r} in {source}" + raise ValueError(msg) + + digest = _parse_required_string(csv_data, "sha256", source=source) + if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest): + msg = f"invalid csv.sha256 in {source}" + raise ValueError(msg) + row_count = csv_data.get("row_count") + if isinstance(row_count, bool) or not isinstance(row_count, int) or row_count <= 0: + msg = f"invalid csv.row_count in {source}: {row_count!r}" + raise ValueError(msg) + columns = csv_data.get("columns") + if columns != list(CSV_COLUMNS): + msg = f"provenance CSV columns do not match schema {SCHEMA_VERSION} in {source}" + raise ValueError(msg) + return context, digest, row_count, tuple(cast("list[str]", columns)) + + +def load_bundle_bytes(csv_payload: bytes, provenance_payload: bytes, *, source: str) -> PerformanceBundle: + """Parse and validate an artifact pair before it reaches report rendering.""" + context, expected_digest, expected_count, _columns = _parse_provenance(provenance_payload, source=source) + observed_digest = hashlib.sha256(csv_payload).hexdigest() + if observed_digest != expected_digest: + msg = f"CSV digest mismatch in {source}: expected {expected_digest}, got {observed_digest}" + raise ValueError(msg) + rows = _parse_rows(csv_payload, source=source) + if len(rows) != expected_count: + msg = f"CSV row count mismatch in {source}: expected {expected_count}, got {len(rows)}" + raise ValueError(msg) + return PerformanceBundle(context=context, rows=rows) + + +def load_bundle(paths: ArtifactPaths) -> PerformanceBundle: + """Load an adjacent CSV/provenance pair and fail closed on partial publication.""" + missing = [str(path) for path in (paths.csv, paths.provenance) if not path.is_file()] + if missing: + msg = f"release-performance artifact pair is incomplete; missing: {', '.join(missing)}" + raise FileNotFoundError(msg) + return load_bundle_bytes( + paths.csv.read_bytes(), + paths.provenance.read_bytes(), + source=f"{paths.csv} and {paths.provenance}", + ) + + +def _stage_payload(path: Path, payload: bytes) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("wb", dir=path.parent, prefix=f".{path.name}.", suffix=".tmp", delete=False) as tmp: + tmp.write(payload) + tmp.flush() + os.fsync(tmp.fileno()) + return Path(tmp.name) + + +def _atomic_restore(path: Path, payload: bytes | None) -> None: + if payload is None: + path.unlink(missing_ok=True) + return + staged = _stage_payload(path, payload) + try: + _replace_path(staged, path) + finally: + staged.unlink(missing_ok=True) + + +def _publish_payloads(paths: ArtifactPaths, csv_payload: bytes, provenance_payload: bytes) -> None: + staged_csv = _stage_payload(paths.csv, csv_payload) + staged_provenance = _stage_payload(paths.provenance, provenance_payload) + previous_csv = paths.csv.read_bytes() if paths.csv.is_file() else None + previous_provenance = paths.provenance.read_bytes() if paths.provenance.is_file() else None + try: + _replace_path(staged_csv, paths.csv) + _replace_path(staged_provenance, paths.provenance) + load_bundle(paths) + except BaseException as publication_error: + rollback_errors = _restore_artifact_pair(paths, previous_csv, previous_provenance) + if rollback_errors: + group_message = "release-performance artifact publication and rollback failed" + raise BaseExceptionGroup( + group_message, + [publication_error, *rollback_errors], + ) from None + raise + finally: + staged_csv.unlink(missing_ok=True) + staged_provenance.unlink(missing_ok=True) + + +def _replace_path(source: Path, destination: Path) -> None: + """Atomically replace *destination* with a staged file.""" + source.replace(destination) + + +def _restore_artifact_pair( + paths: ArtifactPaths, + previous_csv: bytes | None, + previous_provenance: bytes | None, +) -> tuple[BaseException, ...]: + """Attempt both artifact restorations and return every rollback failure.""" + errors: list[BaseException] = [] + for path, payload in ((paths.csv, previous_csv), (paths.provenance, previous_provenance)): + try: + _atomic_restore(path, payload) + except BaseException as exc: # noqa: BLE001 + errors.append(exc) + return tuple(errors) + + +@contextmanager +def publish_bundle(paths: ArtifactPaths, bundle: PerformanceBundle) -> Iterator[None]: + """Publish a validated pair and roll it back if a downstream promotion fails.""" + csv_payload, provenance_payload = serialize_bundle(bundle) + previous_csv = paths.csv.read_bytes() if paths.csv.is_file() else None + previous_provenance = paths.provenance.read_bytes() if paths.provenance.is_file() else None + _publish_payloads(paths, csv_payload, provenance_payload) + try: + yield + except BaseException as downstream_error: + rollback_errors = _restore_artifact_pair(paths, previous_csv, previous_provenance) + if rollback_errors: + group_message = "release-performance downstream publication and rollback failed" + raise BaseExceptionGroup( + group_message, + [downstream_error, *rollback_errors], + ) from None + raise + + +def write_bundle(paths: ArtifactPaths, bundle: PerformanceBundle) -> None: + """Atomically publish a complete validated artifact pair.""" + csv_payload, provenance_payload = serialize_bundle(bundle) + _publish_payloads(paths, csv_payload, provenance_payload) diff --git a/scripts/tests/test_archive_performance.py b/scripts/tests/test_archive_performance.py index 2e56c4d..9e210eb 100644 --- a/scripts/tests/test_archive_performance.py +++ b/scripts/tests/test_archive_performance.py @@ -1,7 +1,5 @@ """Tests for archive_performance.py.""" -from __future__ import annotations - import io import json import subprocess @@ -22,6 +20,16 @@ parse_report_id, promote_report, ) +from performance_artifacts import ( + ArtifactContext, + ArtifactPaths, + PerformanceBundle, + PerformanceRow, + ReleasePair, + ReportSource, + TimingEstimate, + write_bundle, +) if TYPE_CHECKING: from collections.abc import Sequence @@ -58,6 +66,62 @@ def _normalized_report(version: str, baseline: str) -> str: return archive_performance._normalize_how_to_update(_report(version, baseline)) +def _write_fake_rendered_artifacts(args: Sequence[str], *, version: str, baseline: str) -> None: + criterion_dir = Path(args[args.index("--criterion-dir") + 1]) + suite = args[args.index("--suite") + 1] + scope = args[args.index("--scope") + 1] + row_suite = "vs_linalg" if suite == "vs_linalg" else "exact" + group = "d2" if row_suite == "vs_linalg" else "exact_d2" + benchmark = "la_stack_lu" if row_suite == "vs_linalg" else "det_exact" + timing = TimingEstimate(median_ns=10.0, ci_lower_ns=9.0, ci_upper_ns=11.0) + paths = ArtifactPaths( + csv=Path(args[args.index("--csv-output") + 1]), + provenance=Path(args[args.index("--provenance-output") + 1]), + ) + benchmark_provenance = json.loads((criterion_dir / archive_performance._BENCHMARK_HARNESS_METADATA).read_text(encoding="utf-8")) + assert isinstance(benchmark_provenance, dict) + bundle = PerformanceBundle( + context=ArtifactContext( + release=ReleasePair(current=f"v{version}", baseline=baseline), + statistic="median", + suite=suite, + scope=scope, + source=ReportSource( + version=version, + commit="abc1234", + ref="release/test", + revision_timestamp="2026-06-08 12:00:00 UTC", + ), + benchmark_provenance=benchmark_provenance, + ), + rows=( + PerformanceRow( + suite=row_suite, + scope=scope, + benchmark_id=f"{group}/{benchmark}", + group=group, + benchmark=benchmark, + baseline_benchmark=benchmark, + coverage_status="comparable", + coverage_note="", + baseline=timing, + current=TimingEstimate(median_ns=9.0, ci_lower_ns=8.0, ci_upper_ns=10.0), + ), + ), + ) + write_bundle(paths, bundle) + output = Path(args[args.index("--output") + 1]) + output.write_text(archive_performance.render_release_artifacts(paths), encoding="utf-8") + + +def _retained_report(root: Path, *, stem: str = "performance") -> str: + paths = ArtifactPaths( + csv=root / "target" / "bench-reports" / f"{stem}.csv", + provenance=root / "target" / "bench-reports" / f"{stem}.provenance.json", + ) + return archive_performance._normalize_how_to_update(archive_performance.render_release_artifacts(paths)) + + def test_normalized_report_links_archived_performance_reports() -> None: text = _normalized_report("0.4.3", "v0.4.2") assert "Older curated release-to-release reports are archived in `docs/archive/performance/`." in text @@ -771,7 +835,7 @@ def test_promote_report_does_not_overwrite_existing_archive(tmp_path: Path) -> N current.parent.mkdir(parents=True) current.write_text(_report("0.4.1", "v0.4.0"), encoding="utf-8") archive_dir.mkdir(parents=True) - archived.write_text("already archived\n", encoding="utf-8") + archived.write_text(_normalized_report("0.4.1", "v0.4.0"), encoding="utf-8") promote_report( source=source, @@ -781,7 +845,68 @@ def test_promote_report_does_not_overwrite_existing_archive(tmp_path: Path) -> N expected_baseline_tag="v0.4.1", ) - assert archived.read_text(encoding="utf-8") == "already archived\n" + assert archived.read_text(encoding="utf-8") == _normalized_report("0.4.1", "v0.4.0") + + +def test_promote_report_rejects_mismatched_existing_archive_without_mutation(tmp_path: Path) -> None: + source = tmp_path / "performance-new.md" + current = tmp_path / "docs" / "PERFORMANCE.md" + archive_dir = tmp_path / "docs" / "archive" / "performance" + archived = archive_dir / "v0.4.1-vs-v0.4.0.md" + index = archive_dir / "README.md" + original = _report("0.4.1", "v0.4.0") + + source.write_text(_report("0.4.2", "v0.4.1"), encoding="utf-8") + current.parent.mkdir(parents=True) + current.write_text(original, encoding="utf-8") + archive_dir.mkdir(parents=True) + archived.write_text("unrelated archive contents\n", encoding="utf-8") + index.write_text("original index\n", encoding="utf-8") + + with pytest.raises(ValueError, match="does not match the current report"): + promote_report( + source=source, + current=current, + archive_dir=archive_dir, + expected_current_tag="v0.4.2", + expected_baseline_tag="v0.4.1", + ) + + assert current.read_text(encoding="utf-8") == original + assert archived.read_text(encoding="utf-8") == "unrelated archive contents\n" + assert index.read_text(encoding="utf-8") == "original index\n" + + +@pytest.mark.parametrize("collision", ["directory", "symlink"]) +def test_promote_report_rejects_non_file_archive_collision_without_mutation(tmp_path: Path, collision: str) -> None: + source = tmp_path / "performance-new.md" + current = tmp_path / "docs" / "PERFORMANCE.md" + archive_dir = tmp_path / "docs" / "archive" / "performance" + archived = archive_dir / "v0.4.1-vs-v0.4.0.md" + original = _report("0.4.1", "v0.4.0") + + source.write_text(_report("0.4.2", "v0.4.1"), encoding="utf-8") + current.parent.mkdir(parents=True) + current.write_text(original, encoding="utf-8") + archive_dir.mkdir(parents=True) + if collision == "directory": + archived.mkdir() + else: + target = tmp_path / "unrelated.md" + target.write_text("unrelated target\n", encoding="utf-8") + archived.symlink_to(target) + + with pytest.raises(ValueError, match=r"must (?:not be a symlink|be a regular file)"): + promote_report( + source=source, + current=current, + archive_dir=archive_dir, + expected_current_tag="v0.4.2", + expected_baseline_tag="v0.4.1", + ) + + assert current.read_text(encoding="utf-8") == original + assert not (archive_dir / "README.md").exists() def test_promote_report_rejects_unexpected_release_pair(tmp_path: Path) -> None: @@ -901,6 +1026,63 @@ def test_main_reports_release_pair_mismatch_to_stderr( assert not current.exists() +@pytest.mark.parametrize("alias", ["artifact-csv", "current-report"]) +def test_main_rerender_rejects_output_alias_without_mutation( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + alias: str, +) -> None: + artifacts = ArtifactPaths( + csv=tmp_path / "target" / "bench-reports" / "performance.csv", + provenance=tmp_path / "target" / "bench-reports" / "performance.provenance.json", + ) + artifacts.csv.parent.mkdir(parents=True) + artifacts.csv.write_bytes(b"original csv\n") + artifacts.provenance.write_bytes(b"original provenance\n") + current = tmp_path / "docs" / "PERFORMANCE.md" + current.parent.mkdir(parents=True) + current.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + archive_dir = tmp_path / "docs" / "archive" / "performance" + output = artifacts.csv if alias == "artifact-csv" else current + bundle = SimpleNamespace( + context=SimpleNamespace( + release=SimpleNamespace(current="v0.4.4", baseline="v0.4.3"), + ) + ) + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(archive_performance, "load_bundle", lambda _paths: bundle) + monkeypatch.setattr( + archive_performance, + "render_release_artifacts", + lambda _paths: _report("0.4.4", "v0.4.3"), + ) + + rc = main( + [ + "--rerender", + "--artifact-csv", + str(artifacts.csv), + "--artifact-provenance", + str(artifacts.provenance), + "--output", + str(output), + "--current", + str(current), + "--archive-dir", + str(archive_dir), + ] + ) + + captured = capsys.readouterr() + assert rc == 1 + assert "must use distinct paths" in captured.err + assert artifacts.csv.read_bytes() == b"original csv\n" + assert artifacts.provenance.read_bytes() == b"original provenance\n" + assert current.read_text(encoding="utf-8") == _report("0.4.3", "v0.4.2") + assert not archive_dir.exists() + + def test_main_reraises_unexpected_errors(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: def fail_unexpected(*, args: object, paths: object, request: object, repo_root: Path) -> archive_performance.ArchiveResult: msg = "unexpected test failure" @@ -938,8 +1120,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** criterion_dir.mkdir(parents=True) (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") if command == "uv": - output = Path(args[args.index("--output") + 1]) - output.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.chdir(tmp_path) @@ -968,7 +1149,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** captured = capsys.readouterr() assert rc == 0 - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) assert "Generated benchmark report in a temporary worktree" in captured.out assert "target/bench-reports/performance.md" not in captured.out assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.3" for kind, args, _ in calls) @@ -978,6 +1159,22 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert not any(kind == "git" and args[:1] == ("read-tree",) for kind, args, _ in calls) assert not any(kind == "git-stdin" for kind, _, _ in calls) + external_call_count = len(calls) + rerender_rc = main( + [ + "--rerender", + "--current", + str(current), + "--archive-dir", + str(archive_dir), + ] + ) + rerendered = capsys.readouterr() + assert rerender_rc == 0 + assert "Re-rendered" in rerendered.out + assert len(calls) == external_call_count + assert (tmp_path / "target" / "bench-reports" / "performance.md").read_text(encoding="utf-8") == current.read_text(encoding="utf-8") + def test_temp_worktree_is_removed_when_benchmark_command_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: current = tmp_path / "docs" / "PERFORMANCE.md" @@ -1141,18 +1338,18 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert not (criterion_dir / "new").exists() assert (criterion_dir / "v0.4.2" / "estimates.json").is_file() if command == "uv": - assert cwd is not None - metadata = json.loads((cwd / "target" / "criterion" / archive_performance._BENCHMARK_HARNESS_METADATA).read_text(encoding="utf-8")) + report_root = Path(args[args.index("--repo-root") + 1]) + criterion_root = Path(args[args.index("--criterion-dir") + 1]) + metadata = json.loads((criterion_root / archive_performance._BENCHMARK_HARNESS_METADATA).read_text(encoding="utf-8")) assert metadata["baseline"] == "v0.4.2" assert metadata["mode"] == "shared-current-harness" assert metadata["schema"] == 2 - assert metadata["measurement"]["harness_sha256"] == archive_performance._benchmark_harness_digest(cwd) - assert metadata["measurement"]["current_source_state_sha256"] == archive_performance._source_state_digest(cwd) + assert metadata["measurement"]["harness_sha256"] == archive_performance._benchmark_harness_digest(report_root) + assert metadata["measurement"]["current_source_state_sha256"] == archive_performance._source_state_digest(report_root) assert metadata["criterion"]["criterion_version"] == "manifest requirement 0.7.0" assert metadata["validation"]["baseline_revision"] == "passed" assert metadata["validation"]["current_revision"] == "passed" - output = Path(args[args.index("--output") + 1]) - output.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.chdir(tmp_path) @@ -1178,7 +1375,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** captured = capsys.readouterr() assert rc == 0 assert captured.err == "" - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) assert not any(kind == "gh" for kind, _, _ in calls) assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.2") for kind, args, _ in calls) assert any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) @@ -1233,8 +1430,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** if command == "just" and args == ["bench-vs-linalg-la-stack"]: assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" if command == "uv": - report = Path(args[args.index("--output") + 1]) - report.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) @@ -1254,7 +1450,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** ) assert report_id.archive_name == "v0.4.3-vs-v0.4.2.md" - assert output.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert output.read_text(encoding="utf-8") == _retained_report(tmp_path) assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.2", "vs_linalg") for kind, args, _ in calls) assert any(kind == "just" and args == ("bench-vs-linalg-la-stack",) for kind, args, _ in calls) assert not any(kind == "cargo" for kind, _, _ in calls) @@ -1286,8 +1482,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** criterion_dir.mkdir(parents=True) (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") if command == "uv": - report = Path(args[args.index("--output") + 1]) - report.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) @@ -1344,8 +1539,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** tag = args[2] _write_baseline_archive(download_dir / f"la-stack-{tag}-criterion-baseline.tar.gz") if command == "uv": - output = Path(args[args.index("--output") + 1]) - output.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.chdir(tmp_path) @@ -1367,7 +1561,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** ) assert rc == 0 - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) assert any( kind == "git" and args @@ -1410,8 +1604,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** tag = args[2] _write_baseline_archive(download_dir / f"la-stack-{tag}-criterion-baseline.tar.gz") if command == "uv": - report = Path(args[args.index("--output") + 1]) - report.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.chdir(tmp_path) @@ -1438,7 +1631,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** ) assert rc == 0 - assert output.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert output.read_text(encoding="utf-8") == _retained_report(tmp_path) assert not current.exists() assert any( kind == "git" @@ -1545,6 +1738,131 @@ def fail_replace(src: Path, dst: Path) -> None: assert not list(current.parent.glob(".PERFORMANCE.md.*.tmp")) +def test_failed_archive_index_update_rolls_back_report_and_new_archive( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + source = tmp_path / "performance-new.md" + current = tmp_path / "docs" / "PERFORMANCE.md" + archive_dir = tmp_path / "docs" / "archive" / "performance" + original = _normalized_report("0.4.2", "v0.4.1") + source.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + current.parent.mkdir(parents=True) + current.write_text(original, encoding="utf-8") + + def fail_index(_archive_dir: Path) -> None: + msg = "simulated archive index failure" + raise OSError(msg) + + monkeypatch.setattr(archive_performance, "update_archive_index", fail_index) + + with pytest.raises(OSError, match="simulated archive index failure"): + promote_report( + source=source, + current=current, + archive_dir=archive_dir, + expected_current_tag="v0.4.3", + expected_baseline_tag="v0.4.2", + ) + + assert current.read_text(encoding="utf-8") == original + assert not (archive_dir / "v0.4.2-vs-v0.4.1.md").exists() + assert not (archive_dir / "README.md").exists() + + +def test_failed_rerender_output_write_rolls_back_report_archive_and_index( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + current = tmp_path / "docs" / "PERFORMANCE.md" + archive_dir = tmp_path / "docs" / "archive" / "performance" + index = archive_dir / "README.md" + output = tmp_path / "target" / "bench-reports" / "performance.md" + original_current = _report("0.4.2", "v0.4.1") + original_index = "original index\n" + original_output = "original output\n" + current.parent.mkdir(parents=True) + current.write_text(original_current, encoding="utf-8") + archive_dir.mkdir(parents=True) + index.write_text(original_index, encoding="utf-8") + output.parent.mkdir(parents=True) + output.write_text(original_output, encoding="utf-8") + real_write = archive_performance._write_text + + def fail_output_write(path: Path, text: str) -> None: + if path == output: + msg = "simulated rerender output failure" + raise OSError(msg) + real_write(path, text) + + monkeypatch.setattr(archive_performance, "_write_text", fail_output_write) + + with pytest.raises(OSError, match="simulated rerender output failure"): + archive_performance._promote_report_text( + source_text=_report("0.4.3", "v0.4.2"), + request=archive_performance.PromotionRequest( + current=current, + archive_dir=archive_dir, + expected=archive_performance.ReportId(current_tag="v0.4.3", baseline_tag="v0.4.2"), + output=output, + ), + ) + + assert current.read_text(encoding="utf-8") == original_current + assert index.read_text(encoding="utf-8") == original_index + assert output.read_text(encoding="utf-8") == original_output + assert not (archive_dir / "v0.4.2-vs-v0.4.1.md").exists() + + +def test_generate_and_promote_rejects_artifact_alias_before_worktree(tmp_path: Path) -> None: + current = tmp_path / "docs" / "PERFORMANCE.md" + archive_dir = tmp_path / "docs" / "archive" / "performance" + current.parent.mkdir(parents=True) + current.write_text(_report("0.4.2", "v0.4.1"), encoding="utf-8") + artifacts = ArtifactPaths( + csv=current, + provenance=current.with_suffix(".provenance.json"), + ) + + with pytest.raises(ValueError, match="must use distinct paths"): + generate_and_promote_worktree_report( + current=current, + archive_dir=archive_dir, + config=GenerationConfig( + repo_root=tmp_path, + current_tag="v0.4.3", + baseline_tag="v0.4.2", + worktree_ref="HEAD", + ), + artifacts=artifacts, + ) + + assert current.read_text(encoding="utf-8") == _report("0.4.2", "v0.4.1") + assert not archive_dir.exists() + + +def test_generate_output_only_rejects_artifact_alias_before_worktree(tmp_path: Path) -> None: + output = tmp_path / "target" / "bench-reports" / "performance.md" + artifacts = ArtifactPaths( + csv=output, + provenance=output.with_suffix(".provenance.json"), + ) + + with pytest.raises(ValueError, match="must use distinct paths"): + generate_worktree_report( + output=output, + config=GenerationConfig( + repo_root=tmp_path, + current_tag="v0.4.3", + baseline_tag="v0.4.2", + worktree_ref="HEAD", + ), + artifacts=artifacts, + ) + + assert not output.exists() + + def test_generate_and_promote_uses_temp_worktree_and_current_diff(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") @@ -1581,8 +1899,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** if command == "just" and args == ["bench-latest"]: assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" if command == "uv": - output = Path(args[args.index("--output") + 1]) - output.write_text(_report("0.4.3", "v0.4.2"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") return _result() monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) @@ -1602,7 +1919,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** ) assert report_id.archive_name == "v0.4.3-vs-v0.4.2.md" - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.3", "v0.4.2") + assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) assert (archive_dir / "v0.4.2-vs-v0.4.1.md").exists() assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "HEAD" for kind, args, _ in calls) assert any(kind == "git-stdin" and args == ("apply", "--binary") for kind, args, _ in calls) @@ -1635,8 +1952,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** criterion_dir.mkdir(parents=True) (criterion_dir / "baseline.txt").write_text("baseline\n", encoding="utf-8") if command == "uv": - output = Path(args[args.index("--output") + 1]) - output.write_text(_report("0.4.2", "v0.4.1"), encoding="utf-8") + _write_fake_rendered_artifacts(args, version="0.4.2", baseline="v0.4.1") return _result() monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) @@ -1656,12 +1972,12 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** ) assert report_id.archive_name == "v0.4.2-vs-v0.4.1.md" - assert current.read_text(encoding="utf-8") == _normalized_report("0.4.2", "v0.4.1") + assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.2" for kind, args, _ in calls) assert any(kind == "cargo" and args == ("bench", "--locked", "--features", "bench,exact") for kind, args, _ in calls) assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) assert not any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) - assert not any(kind == "uv" and "--suite" in args for kind, args, _ in calls) - assert not any(kind == "uv" and "--scope" in args for kind, args, _ in calls) + assert any(kind == "uv" and "--suite" in args for kind, args, _ in calls) + assert any(kind == "uv" and "--scope" in args for kind, args, _ in calls) assert not any(kind == "git" and args[:1] == ("read-tree",) for kind, args, _ in calls) assert not any(kind == "git-stdin" for kind, _, _ in calls) diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 0c5cba3..387b99a 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -1035,7 +1035,51 @@ def test_main_rejects_overflowing_timing_without_writing_or_traceback( assert not output.exists() -def test_main_v043_comparison_allows_only_unavailable_balanced_baselines(tmp_path: Path) -> None: +@pytest.mark.parametrize("invalid_paths", ["same-pair", "non-adjacent", "markdown-alias"]) +def test_main_rejects_invalid_artifact_paths_without_writing( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + invalid_paths: str, +) -> None: + output = tmp_path / "report.md" + csv_output = tmp_path / "artifacts" / "performance.csv" + provenance_output = tmp_path / "artifacts" / "performance.provenance.json" + if invalid_paths == "same-pair": + provenance_output = csv_output + elif invalid_paths == "non-adjacent": + provenance_output = tmp_path / "other" / "performance.provenance.json" + else: + output = csv_output + + paths = {output, csv_output, provenance_output} + for path in paths: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(f"original {path.name}\n".encode()) + before = {path: path.read_bytes() for path in paths} + + rc = bench_compare.main( + [ + "v0.4.3", + "--repo-root", + str(tmp_path), + "--output", + str(output), + "--csv-output", + str(csv_output), + "--provenance-output", + str(provenance_output), + ] + ) + + assert rc == 2 + assert "Invalid release-performance artifact paths" in capsys.readouterr().err + assert {path: path.read_bytes() for path in paths} == before + + +def test_main_v043_comparison_allows_only_unavailable_balanced_baselines( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: criterion_dir = tmp_path / "criterion" unavailable = bench_compare._V0_4_3_UNAVAILABLE_BASELINE_ROWS for dimension in bench_compare.VS_LINALG_CANONICAL_DIMS: @@ -1055,14 +1099,44 @@ def test_main_v043_comparison_allows_only_unavailable_balanced_baselines(tmp_pat cast("dict[str, object]", criterion)["suite"] = "vs_linalg" (criterion_dir / ".la-stack-benchmark-harness.json").write_text(json.dumps(provenance), encoding="utf-8") output = tmp_path / "report.md" + artifact_paths = bench_compare.ArtifactPaths( + csv=tmp_path / "performance.csv", + provenance=tmp_path / "performance.provenance.json", + ) + monkeypatch.setattr( + bench_compare, + "_report_source", + lambda _root: bench_compare.ReportSource( + version="0.4.4", + commit="current-commit", + ref="test", + revision_timestamp="2026-08-04 12:00:00 UTC", + ), + ) - rc = bench_compare.main(["v0.4.3", "--suite", "vs_linalg", "--criterion-dir", str(criterion_dir), "--output", str(output)]) + rc = bench_compare.main( + [ + "v0.4.3", + "--suite", + "vs_linalg", + "--criterion-dir", + str(criterion_dir), + "--csv-output", + str(artifact_paths.csv), + "--provenance-output", + str(artifact_paths.provenance), + "--output", + str(output), + ] + ) assert rc == 0 rendered = output.read_text(encoding="utf-8") + assert rendered == bench_compare.render_release_artifacts(artifact_paths) assert "d8/la_stack_det_from_lu_balanced_range" in rendered assert "d8/la_stack_det_from_ldlt_balanced_range" in rendered - assert "no speedup is claimed" in rendered + assert "current-only" in rendered + assert "no correctness-compatible benchmark row" in rendered def test_generate_markdown_labels_absent_provenance_unavailable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/scripts/tests/test_performance_artifacts.py b/scripts/tests/test_performance_artifacts.py new file mode 100644 index 0000000..f0ff6a1 --- /dev/null +++ b/scripts/tests/test_performance_artifacts.py @@ -0,0 +1,365 @@ +"""Tests for durable release-performance report artifacts.""" + +import csv +import hashlib +import io +import json +from pathlib import Path + +import pytest + +import performance_artifacts +from performance_artifacts import ( + ArtifactContext, + ArtifactPaths, + PerformanceBundle, + PerformanceRow, + ReleasePair, + ReportSource, + TimingEstimate, + load_bundle, + load_bundle_bytes, + publish_bundle, + serialize_bundle, + write_bundle, +) + + +def _timing(value: float) -> TimingEstimate: + return TimingEstimate( + median_ns=value, + ci_lower_ns=value * 0.9, + ci_upper_ns=value * 1.1, + ) + + +def _context(*, current: str = "v0.4.4", baseline: str = "v0.4.3") -> ArtifactContext: + return ArtifactContext( + release=ReleasePair(current=current, baseline=baseline), + statistic="median", + suite="exact", + scope="release-signal", + source=ReportSource( + version=current.removeprefix("v"), + commit="abc1234", + ref="HEAD", + revision_timestamp="2026-08-04 12:00:00 UTC", + ), + benchmark_provenance={ + "baseline": baseline, + "criterion": { + "baseline_command": ["just", "bench-save-baseline", baseline], + "criterion_version": "0.7.0", + "current_command": ["just", "bench-latest"], + "sample": "new", + "scope": "release-signal", + "statistic": "median", + "suite": "exact", + }, + "measurement": { + "baseline_api_compatibility": "none", + "baseline_commit": "def5678", + "baseline_git_clean": True, + "baseline_source_state_sha256": "d" * 64, + "cargo_lock_sha256": "b" * 64, + "cpu": "test-cpu", + "current_commit": "abc1234", + "current_git_clean": True, + "current_source_state_sha256": "c" * 64, + "harness_sha256": "a" * 64, + "os": "TestOS 1 x86_64", + "rustc": "rustc 1.97.1", + "status": "recorded", + }, + "mode": "shared-current-harness", + "publication": { + "cargo_lock_sha256": "b" * 64, + "commit": "abc1234", + "correctness_gate": "passed", + "cpu": "test-cpu", + "git_clean": True, + "harness_sha256": "a" * 64, + "os": "TestOS 1 x86_64", + "rustc": "rustc 1.97.1", + "source_state_sha256": "c" * 64, + }, + "schema": 2, + "validation": { + "baseline_api_compatibility": "none", + "baseline_commit": "def5678", + "baseline_git_clean": True, + "baseline_revision": "passed", + "baseline_source_state_sha256": "d" * 64, + "command": ["just", "test-bench-inputs"], + "current_commit": "abc1234", + "current_git_clean": True, + "current_revision": "passed", + "current_source_state_sha256": "c" * 64, + "harness": "shared-current", + }, + }, + ) + + +def _bundle(*, current_value: float = 8.0) -> PerformanceBundle: + rows = ( + PerformanceRow( + suite="exact", + scope="release-signal", + benchmark_id="exact_d2/det_exact", + group="exact_d2", + benchmark="det_exact", + baseline_benchmark="det_exact", + coverage_status="comparable", + coverage_note="", + baseline=_timing(10.0), + current=_timing(current_value), + ), + PerformanceRow( + suite="exact", + scope="release-signal", + benchmark_id="exact_d3/current_only", + group="exact_d3", + benchmark="current_only", + baseline_benchmark="current_only", + coverage_status="current-only", + coverage_note="The baseline does not expose this correctness-compatible row.", + baseline=None, + current=_timing(12.0), + ), + PerformanceRow( + suite="exact", + scope="release-signal", + benchmark_id="exact_d4/baseline_only", + group="exact_d4", + benchmark="baseline_only", + baseline_benchmark="baseline_only", + coverage_status="baseline-only", + coverage_note="The current run does not expose this row.", + baseline=_timing(15.0), + current=None, + ), + ) + return PerformanceBundle(context=_context(), rows=rows) + + +def _replace_csv_and_digest(csv_payload: bytes, provenance_payload: bytes) -> tuple[bytes, bytes]: + provenance = json.loads(provenance_payload) + provenance["csv"]["sha256"] = hashlib.sha256(csv_payload).hexdigest() + return csv_payload, (json.dumps(provenance, indent=2, sort_keys=True) + "\n").encode() + + +def _simulate_promotion_failure(paths: ArtifactPaths) -> None: + with publish_bundle(paths, _bundle(current_value=7.0)): + msg = "simulated promotion failure" + raise RuntimeError(msg) + + +def test_artifact_round_trip_preserves_comparable_and_one_sided_rows() -> None: + bundle = _bundle() + + csv_payload, provenance_payload = serialize_bundle(bundle) + parsed = load_bundle_bytes(csv_payload, provenance_payload, source="round-trip fixture") + + assert parsed == PerformanceBundle(context=bundle.context, rows=bundle.sorted_rows) + assert [row.coverage_status for row in parsed.rows] == ["comparable", "current-only", "baseline-only"] + assert csv_payload.endswith(b"\n") + assert provenance_payload.endswith(b"\n") + + +def test_artifact_serialization_is_independent_of_input_row_order() -> None: + bundle = _bundle() + reordered = PerformanceBundle(context=bundle.context, rows=tuple(reversed(bundle.rows))) + + assert serialize_bundle(reordered) == serialize_bundle(bundle) + + +def test_artifact_loader_rejects_incomplete_confidence_interval() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + reader = csv.DictReader(io.StringIO(csv_payload.decode(), newline="")) + rows = list(reader) + rows[0]["current_ci_upper_ns"] = "" + output = io.StringIO(newline="") + fieldnames = reader.fieldnames + assert fieldnames is not None + writer = csv.DictWriter(output, fieldnames=fieldnames, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + malformed = output.getvalue().encode() + malformed, provenance_payload = _replace_csv_and_digest(malformed, provenance_payload) + + with pytest.raises(ValueError, match="incomplete current confidence interval"): + load_bundle_bytes(malformed, provenance_payload, source="incomplete interval fixture") + + +def test_artifact_loader_rejects_mismatched_release_pair() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + provenance["release"]["baseline"] = "v0.4.2" + malformed = (json.dumps(provenance, indent=2, sort_keys=True) + "\n").encode() + + with pytest.raises(ValueError, match="benchmark provenance baseline"): + load_bundle_bytes(csv_payload, malformed, source="mismatched release fixture") + + +def test_artifact_loader_rejects_malformed_csv_schema() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + malformed = csv_payload.replace(b"benchmark_id", b"benchmark_key", 1) + malformed, provenance_payload = _replace_csv_and_digest(malformed, provenance_payload) + + with pytest.raises(ValueError, match="unsupported release-performance CSV columns"): + load_bundle_bytes(malformed, provenance_payload, source="malformed CSV fixture") + + +def test_artifact_loader_rejects_malformed_provenance_json() -> None: + csv_payload, _provenance_payload = serialize_bundle(_bundle()) + + with pytest.raises(ValueError, match="malformed release-performance provenance JSON"): + load_bundle_bytes(csv_payload, b"{not-json}\n", source="malformed provenance fixture") + + +@pytest.mark.parametrize("invalid_schema", [True, 1.0]) +def test_artifact_loader_rejects_non_integer_outer_schema(invalid_schema: object) -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + provenance["schema_version"] = invalid_schema + + with pytest.raises(ValueError, match="unsupported release-performance provenance schema"): + load_bundle_bytes( + csv_payload, + (json.dumps(provenance) + "\n").encode(), + source="invalid outer schema fixture", + ) + + +@pytest.mark.parametrize("invalid_schema", [True, 2.0]) +def test_artifact_loader_rejects_non_integer_benchmark_schema(invalid_schema: object) -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + provenance["benchmark_provenance"]["schema"] = invalid_schema + + with pytest.raises(ValueError, match="benchmark provenance schema must be 2"): + load_bundle_bytes( + csv_payload, + (json.dumps(provenance) + "\n").encode(), + source="invalid benchmark schema fixture", + ) + + +def test_artifact_loader_rejects_incomplete_nested_provenance() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + del provenance["benchmark_provenance"]["criterion"]["criterion_version"] + + with pytest.raises(ValueError, match=r"criterion\.criterion_version"): + load_bundle_bytes( + csv_payload, + (json.dumps(provenance) + "\n").encode(), + source="incomplete nested provenance fixture", + ) + + +def test_artifact_loader_rejects_contradictory_current_revision() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + provenance["benchmark_provenance"]["validation"]["current_commit"] = "different-commit" + + with pytest.raises(ValueError, match=r"validation\.current_commit.*publication\.commit"): + load_bundle_bytes( + csv_payload, + (json.dumps(provenance) + "\n").encode(), + source="contradictory current revision fixture", + ) + + +def test_bundle_rejects_duplicate_benchmark_keys() -> None: + bundle = _bundle() + + with pytest.raises(ValueError, match="duplicate benchmark key"): + PerformanceBundle(context=bundle.context, rows=(bundle.rows[0], bundle.rows[0])) + + +@pytest.mark.parametrize("value", [0.0, -1.0, float("inf"), float("nan")]) +def test_timing_rejects_non_positive_or_non_finite_values(value: float) -> None: + with pytest.raises(ValueError, match="must be finite and positive"): + TimingEstimate(median_ns=value, ci_lower_ns=1.0, ci_upper_ns=2.0) + + +def test_artifact_loader_fails_closed_on_partial_pair(tmp_path: Path) -> None: + paths = ArtifactPaths( + csv=tmp_path / "performance.csv", + provenance=tmp_path / "performance.provenance.json", + ) + paths.csv.write_text("partial\n", encoding="utf-8") + + with pytest.raises(FileNotFoundError, match="artifact pair is incomplete"): + load_bundle(paths) + + +def test_failed_second_replace_restores_prior_valid_pair(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + paths = ArtifactPaths( + csv=tmp_path / "performance.csv", + provenance=tmp_path / "performance.provenance.json", + ) + original = _bundle(current_value=8.0) + write_bundle(paths, original) + real_replace = performance_artifacts._replace_path + failed = False + + def fail_provenance_once(source: Path, destination: Path) -> None: + nonlocal failed + if Path(destination) == paths.provenance and not failed: + failed = True + msg = "simulated provenance replace failure" + raise OSError(msg) + real_replace(source, destination) + + monkeypatch.setattr(performance_artifacts, "_replace_path", fail_provenance_once) + + with pytest.raises(OSError, match="simulated provenance replace failure"): + write_bundle(paths, _bundle(current_value=7.0)) + + assert load_bundle(paths) == PerformanceBundle(context=original.context, rows=original.sorted_rows) + + +def test_downstream_promotion_failure_rolls_back_artifact_pair(tmp_path: Path) -> None: + paths = ArtifactPaths( + csv=tmp_path / "performance.csv", + provenance=tmp_path / "performance.provenance.json", + ) + original = _bundle(current_value=8.0) + write_bundle(paths, original) + + with pytest.raises(RuntimeError, match="simulated promotion failure"): + _simulate_promotion_failure(paths) + + assert load_bundle(paths) == PerformanceBundle(context=original.context, rows=original.sorted_rows) + + +def test_rollback_attempts_both_artifacts_when_first_restoration_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + paths = ArtifactPaths( + csv=tmp_path / "performance.csv", + provenance=tmp_path / "performance.provenance.json", + ) + write_bundle(paths, _bundle(current_value=8.0)) + real_restore = performance_artifacts._atomic_restore + restored: list[Path] = [] + + def fail_csv_restore(path: Path, payload: bytes | None) -> None: + restored.append(path) + if path == paths.csv: + msg = "simulated CSV restoration failure" + raise OSError(msg) + real_restore(path, payload) + + monkeypatch.setattr(performance_artifacts, "_atomic_restore", fail_csv_restore) + + with pytest.raises(BaseExceptionGroup, match="downstream publication and rollback failed") as raised: + _simulate_promotion_failure(paths) + + assert restored == [paths.csv, paths.provenance] + assert any(isinstance(error, RuntimeError) for error in raised.value.exceptions) + assert any(isinstance(error, OSError) for error in raised.value.exceptions) diff --git a/uv.lock b/uv.lock index c001d6d..107ddc9 100644 --- a/uv.lock +++ b/uv.lock @@ -67,61 +67,61 @@ wheels = [ [[package]] name = "cffi" -version = "2.1.0" +version = "2.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pycparser", marker = "implementation_name != 'PyPy'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/57/5f/ff100cae70ebe9d8df1c01a00e510e45d9adb5c1fdda84791b199141de97/cffi-2.1.0.tar.gz", hash = "sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9", size = 531036, upload-time = "2026-07-06T21:34:30.382Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/f0/81478e482afa03f6d18dc8f2afb5edc45b3080853b634b5ed91961be0998/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a", size = 194142, upload-time = "2026-07-06T21:33:23.657Z" }, - { url = "https://files.pythonhosted.org/packages/7d/95/8de304305cd9204974b0ca051b86d307cafca13aa575a0ef1b44d92c0d8c/cffi-2.1.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3", size = 196819, upload-time = "2026-07-06T21:33:25.007Z" }, - { url = "https://files.pythonhosted.org/packages/20/71/7c8372d30e42415602ed9f268f7cfd66f1b855fed881ecd168bcb45dbc0b/cffi-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d", size = 184965, upload-time = "2026-07-06T21:33:26.605Z" }, - { url = "https://files.pythonhosted.org/packages/d6/5c/584e626835f0375c928176c04137c96927165cb8733cdb3150ec04e5ee5e/cffi-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac", size = 184952, upload-time = "2026-07-06T21:33:27.823Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d2/065fcae1c73979fac8e054462478d0ff8a29c40cdc2ed7ea5676a061df53/cffi-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6", size = 222353, upload-time = "2026-07-06T21:33:29.178Z" }, - { url = "https://files.pythonhosted.org/packages/ed/a5/e8bbb1ce5b3ac2f53ad6a10bde44318a5a8d99d4f4a000d44a6e39aeb3e4/cffi-2.1.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913", size = 210051, upload-time = "2026-07-06T21:33:30.534Z" }, - { url = "https://files.pythonhosted.org/packages/28/ed/c127d3ac36e899c965e3361357c3befacd6578c03f40125183e41c3b219e/cffi-2.1.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d", size = 208630, upload-time = "2026-07-06T21:33:31.753Z" }, - { url = "https://files.pythonhosted.org/packages/cc/d7/97d3136f81db489ec8d1d67748c110d6c994268fd7528014aa9f2b085e4e/cffi-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5", size = 221593, upload-time = "2026-07-06T21:33:33.044Z" }, - { url = "https://files.pythonhosted.org/packages/d3/27/93195977168ee63aed233a1a0993a2178798654d1f4bddcdd321d6fd3b21/cffi-2.1.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce", size = 225146, upload-time = "2026-07-06T21:33:34.224Z" }, - { url = "https://files.pythonhosted.org/packages/b3/c1/6dbd291ee2ae5a50a034aa057207081f545923bbf15dad4511e985aafff5/cffi-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326", size = 223240, upload-time = "2026-07-06T21:33:35.57Z" }, - { url = "https://files.pythonhosted.org/packages/0f/6f/ade5ce9863a57992a6ea3d0d10d7e29b8749fc127204b3d493d667b2815f/cffi-2.1.0-cp314-cp314-win32.whl", hash = "sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd", size = 177723, upload-time = "2026-07-06T21:33:51.626Z" }, - { url = "https://files.pythonhosted.org/packages/41/de/92b9eeed4ae4a21d6fd9b2a2c8505cbed573299902ea73981cc13f7ff62c/cffi-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb", size = 187937, upload-time = "2026-07-06T21:33:53.403Z" }, - { url = "https://files.pythonhosted.org/packages/2e/1a/cc6ae6c2913a03aab8898eee57963cf1035b8df5872ed8b9115fcc7e2be8/cffi-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804", size = 183001, upload-time = "2026-07-06T21:33:54.74Z" }, - { url = "https://files.pythonhosted.org/packages/14/f0/134c00ce0779ec86dea2aa1aac69339c2741a8045072676763512363a2ea/cffi-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714", size = 188538, upload-time = "2026-07-06T21:33:36.792Z" }, - { url = "https://files.pythonhosted.org/packages/50/d8/3b86aba791cb610d24e8a3e1b2cd529e71fa15096b04e4d4e360049d4a4c/cffi-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376", size = 188230, upload-time = "2026-07-06T21:33:38.011Z" }, - { url = "https://files.pythonhosted.org/packages/14/d0/117dcd9209255ad8571fbc8c92ef32593a1d294dcec91ddc4e4db50606f2/cffi-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98", size = 223899, upload-time = "2026-07-06T21:33:39.514Z" }, - { url = "https://files.pythonhosted.org/packages/b6/3d/f20f8b886b254e3ad10e15cd4186d3aed49f3e6a35ab37aab9f8f25f7c03/cffi-2.1.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13", size = 211652, upload-time = "2026-07-06T21:33:40.851Z" }, - { url = "https://files.pythonhosted.org/packages/28/3b/fad54de07260b93ddeef4b96d0131d57ea900675df1d410ae1deee52d7a6/cffi-2.1.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d", size = 210755, upload-time = "2026-07-06T21:33:42.183Z" }, - { url = "https://files.pythonhosted.org/packages/cc/82/3d5c705acb7abbba9bbd7d79b8e62e0f25b6120eb7ae6ac49f1b721722fe/cffi-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056", size = 223933, upload-time = "2026-07-06T21:33:43.603Z" }, - { url = "https://files.pythonhosted.org/packages/6c/d0/47e338384ab6b1004241002fa616301020cea4fc95f283506565d252f276/cffi-2.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4", size = 226749, upload-time = "2026-07-06T21:33:45.046Z" }, - { url = "https://files.pythonhosted.org/packages/70/25/65bd5b58ea4bfdfc15cde02cb5365f89ef8ab8b2adfb8fe5c4bd4233382f/cffi-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94", size = 225703, upload-time = "2026-07-06T21:33:46.374Z" }, - { url = "https://files.pythonhosted.org/packages/dc/78/aa01ac599a8a4322533d45a1f9bc93b338276d2d59dabbe7c6d92a775c81/cffi-2.1.0-cp314-cp314t-win32.whl", hash = "sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76", size = 182857, upload-time = "2026-07-06T21:33:47.696Z" }, - { url = "https://files.pythonhosted.org/packages/b9/26/d00496b22de4d4228f32dde94ad996f350c8aad676d63bcca0743c8dea4d/cffi-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5", size = 194065, upload-time = "2026-07-06T21:33:48.953Z" }, - { url = "https://files.pythonhosted.org/packages/d5/dd/0c7dbf815a579ff005008a2d815a55d6bb047c349eef536d9dc53d3f0a8d/cffi-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8", size = 186404, upload-time = "2026-07-06T21:33:50.309Z" }, - { url = "https://files.pythonhosted.org/packages/55/c7/8c8c50cb11c6750051daf12164098a9a6f027ac4356967fd4d800a07f242/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c", size = 194121, upload-time = "2026-07-06T21:33:56.109Z" }, - { url = "https://files.pythonhosted.org/packages/99/e2/67680bf19a6b60d2bb7ff83baefa2a4c3d2d7dc0f3277034b802e1fc504c/cffi-2.1.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001", size = 196820, upload-time = "2026-07-06T21:33:57.288Z" }, - { url = "https://files.pythonhosted.org/packages/ed/da/4bbe583a3b3a5c8c60892124fe17f3fa3656523faf0d3484eae90f091853/cffi-2.1.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3", size = 184936, upload-time = "2026-07-06T21:33:58.765Z" }, - { url = "https://files.pythonhosted.org/packages/e5/4b/1f4c36ab273980d7aa75bb126ea4f8971f24a96108acad3a0a084028c57b/cffi-2.1.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc", size = 185045, upload-time = "2026-07-06T21:34:00.085Z" }, - { url = "https://files.pythonhosted.org/packages/ef/c3/ad299dc38f3583f8d916b299f028af418a9ec98bc695fcbebeae7420691c/cffi-2.1.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699", size = 222342, upload-time = "2026-07-06T21:34:01.814Z" }, - { url = "https://files.pythonhosted.org/packages/eb/d8/df4543cc087245044ed02ef3ad8e0a26619d0075ac7a77a12dc81177851b/cffi-2.1.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022", size = 210073, upload-time = "2026-07-06T21:34:03.255Z" }, - { url = "https://files.pythonhosted.org/packages/2c/0e/fac738d73728c6cea2a88a2883dca54892496cbba88a1dc1f2909cb8a6f5/cffi-2.1.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0", size = 208551, upload-time = "2026-07-06T21:34:04.433Z" }, - { url = "https://files.pythonhosted.org/packages/e6/3f/0b04a700dd64f465c93020253a793a82c9b4dff9961f48facd0df945d9b8/cffi-2.1.0-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1", size = 221649, upload-time = "2026-07-06T21:34:06.157Z" }, - { url = "https://files.pythonhosted.org/packages/5d/7c/b7379a5704c79eda57ce075869ba70a0368d1c850f803b3c0d078d39dcaf/cffi-2.1.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28", size = 225203, upload-time = "2026-07-06T21:34:07.489Z" }, - { url = "https://files.pythonhosted.org/packages/5a/02/d5e6c43ea85c41bda2a184a3418f195fe7cf602967a8d2b94e085b83deef/cffi-2.1.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629", size = 223263, upload-time = "2026-07-06T21:34:08.712Z" }, - { url = "https://files.pythonhosted.org/packages/2c/d8/772b8259bf75749adffb1c546828978381fb516f60cf701f6c83daf60c85/cffi-2.1.0-cp315-cp315-win32.whl", hash = "sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6", size = 177696, upload-time = "2026-07-06T21:34:26.355Z" }, - { url = "https://files.pythonhosted.org/packages/2f/dd/afa2191fc6d57fedd26e5844a2fe2fcc0bbfa00961bbaa5a41e4921e7cca/cffi-2.1.0-cp315-cp315-win_amd64.whl", hash = "sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853", size = 187914, upload-time = "2026-07-06T21:34:27.58Z" }, - { url = "https://files.pythonhosted.org/packages/05/ef/6cd4f8c671517162379dc79cfae5aea9106bc38abb89628d5c16adf6a838/cffi-2.1.0-cp315-cp315-win_arm64.whl", hash = "sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda", size = 183004, upload-time = "2026-07-06T21:34:28.905Z" }, - { url = "https://files.pythonhosted.org/packages/11/b6/12fc55092817a5faa26fb8c40c7f9d662e11a46ee248c137aafc42517d92/cffi-2.1.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc", size = 188378, upload-time = "2026-07-06T21:34:09.926Z" }, - { url = "https://files.pythonhosted.org/packages/8d/2e/cdac88979f295fde5daa69622c7d2111e56e7ceb94f211357fbe452339e4/cffi-2.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca", size = 188319, upload-time = "2026-07-06T21:34:11.101Z" }, - { url = "https://files.pythonhosted.org/packages/e0/27/1d0b408497e41a74795af122d7b603c418c5fed0171450f899afd04e594f/cffi-2.1.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d", size = 223904, upload-time = "2026-07-06T21:34:12.606Z" }, - { url = "https://files.pythonhosted.org/packages/8b/31/e115c985105dd7ffb32444505f18ceb874bb42d992af05d5dced7ecf1980/cffi-2.1.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8", size = 211554, upload-time = "2026-07-06T21:34:13.987Z" }, - { url = "https://files.pythonhosted.org/packages/5a/67/9e6e09409336d9e515c58367e7cfcf4f89df06ad25252675595a58eb59d5/cffi-2.1.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd", size = 210795, upload-time = "2026-07-06T21:34:15.972Z" }, - { url = "https://files.pythonhosted.org/packages/19/e5/d3cc82a4a0be7902af279c04181ad038449c096734464a5ae1de3e1401bd/cffi-2.1.0-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f", size = 223843, upload-time = "2026-07-06T21:34:17.509Z" }, - { url = "https://files.pythonhosted.org/packages/b9/65/b434abc97ce7cecc2c640fde160507c0ecc7e21544b483ba3325d2e2ea17/cffi-2.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc", size = 226773, upload-time = "2026-07-06T21:34:19.05Z" }, - { url = "https://files.pythonhosted.org/packages/b5/9f/d4dc66ca651eb1145a133314cda721abf13cfac3d28c4a0402263ae6ad75/cffi-2.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9", size = 225719, upload-time = "2026-07-06T21:34:20.576Z" }, - { url = "https://files.pythonhosted.org/packages/68/5a/e536c528bc8057496c360c0978559a2dc45653f89dd6151078aa7d8fca1a/cffi-2.1.0-cp315-cp315t-win32.whl", hash = "sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b", size = 182760, upload-time = "2026-07-06T21:34:22.059Z" }, - { url = "https://files.pythonhosted.org/packages/d3/0b/0ffe8b82d3875bced5fa1e7986a7a46b748262a40ab7f60b475eb9fb1bb3/cffi-2.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5", size = 193769, upload-time = "2026-07-06T21:34:23.589Z" }, - { url = "https://files.pythonhosted.org/packages/a0/17/1073b53b68c9b5ca6914adf5f8bf55aacc2d3be102418c90700160ea8605/cffi-2.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210", size = 186405, upload-time = "2026-07-06T21:34:24.857Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] [[package]] @@ -419,7 +419,7 @@ dev = [ { name = "semgrep", specifier = "==1.172.0" }, { name = "shellcheck-py", specifier = "==0.11.0.1" }, { name = "shfmt-py", specifier = "==4.0.0" }, - { name = "ty", specifier = "==0.0.65" }, + { name = "ty", specifier = "==0.0.66" }, { name = "yamllint", specifier = "==1.38.0" }, ] @@ -606,11 +606,11 @@ wheels = [ [[package]] name = "packaging" -version = "26.2" +version = "26.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, ] [[package]] @@ -1118,27 +1118,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.65" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cf561927e8e9ab5c1892a833b664aa9cd6f051a75f6280c66d8047246bda/ty-0.0.65.tar.gz", hash = "sha256:b7134bffcc00b715fa8291e84d845782ced810a998dc1f7f11d71c85c4046325", size = 6460098, upload-time = "2026-07-29T18:31:03.27Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/4e/71e2d325d2b53a1afad81624ad076b2ede413213fc4a18cb05b78c568571/ty-0.0.65-py3-none-linux_armv6l.whl", hash = "sha256:dc556c9f05408bef4c4ef02b2cc382e4e5f797b4b20d64410289848f0d76705f", size = 12298466, upload-time = "2026-07-29T18:30:12.744Z" }, - { url = "https://files.pythonhosted.org/packages/57/77/fec8f29647c55794efa430a7f365e44f5ce7ffb6459d9445a87fac569bec/ty-0.0.65-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:29d2e0d34cc0a28a17ef0cf81135c5ebabc3562131f9079138ba5e7bae0f56bd", size = 11942421, upload-time = "2026-07-29T18:30:16.076Z" }, - { url = "https://files.pythonhosted.org/packages/13/09/7f3766aef9dc627e2698cf4e3e59cf53389dcae3812040d33c1aa931230f/ty-0.0.65-py3-none-macosx_11_0_arm64.whl", hash = "sha256:685f49a9312bbf69d5b65bbb66384fed1f927403ea030c217b9289092d7e46c4", size = 11451922, upload-time = "2026-07-29T18:30:19.155Z" }, - { url = "https://files.pythonhosted.org/packages/cb/7b/1a77cd50e0befb50f55b8bf9bd3ed3eddf184bf28c61b56727039e0774fc/ty-0.0.65-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f564b5ebe78e2f3a8e7b8eacb1292eb88b7c0f3c8630671cfca31abc0709cd9", size = 11994999, upload-time = "2026-07-29T18:30:22.315Z" }, - { url = "https://files.pythonhosted.org/packages/63/7b/feda16f3a4a0a99be27431e0c9598eeeec0db1eb2fec9a15976698209418/ty-0.0.65-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c983e156fe9e113fb56389e13d327b6b8549fe866de9b269684723a88e9b732d", size = 12090662, upload-time = "2026-07-29T18:30:24.93Z" }, - { url = "https://files.pythonhosted.org/packages/ed/3e/3f69bf9c9307dbdc0771719f65ce5b556e7bdeeaccbdd599d4f57866d801/ty-0.0.65-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3663b7396e8b1a9954e20e732de7ccb0192bf4118473069b4945920d6923921", size = 12822094, upload-time = "2026-07-29T18:30:28.012Z" }, - { url = "https://files.pythonhosted.org/packages/90/38/8fa791b3bb503ee2b46ad81690cd1bdd54519582df6d805cee57fe143e85/ty-0.0.65-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:306ed01f29d6e108e98feb233dbbf5878a027603b71bd3743b343977933a9f16", size = 13357833, upload-time = "2026-07-29T18:30:31.122Z" }, - { url = "https://files.pythonhosted.org/packages/c1/73/4dda396a201e1dd0ed3594a9b48e559cb41c4bc048c6cd4c4d1b39eb4313/ty-0.0.65-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28bcfc8898c94f079a9100e684bcf312b6a64ad3a7d4ebb35a4591546030a2cd", size = 12977303, upload-time = "2026-07-29T18:30:33.944Z" }, - { url = "https://files.pythonhosted.org/packages/a5/26/c250c2c569adc53a8591716641388397bcb2a442e4a30b952ae81b50c0e0/ty-0.0.65-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a75bd0c245c38802a8f488378e74f92feb7dd33db7d63fbdd6fdf82791ba730", size = 12579338, upload-time = "2026-07-29T18:30:37.199Z" }, - { url = "https://files.pythonhosted.org/packages/d3/94/4a5647d44753ca218fc930d7e4d9bf468d0ed4a0ad4b3d57588bc1bbacf7/ty-0.0.65-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:9e5e1bdea9662d2b5312b4e99f319f4e6e2ea427511b5fbc546141b79ec53f76", size = 12957731, upload-time = "2026-07-29T18:30:39.937Z" }, - { url = "https://files.pythonhosted.org/packages/36/b6/1e22fa11a1e0dfb20b1c7f3cbfd8170273aada2a82f9ecd3055275370c44/ty-0.0.65-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:03a88493d4842889f65280ae241e06b399d57eb3c63571054cad21a4c33b3b69", size = 11938625, upload-time = "2026-07-29T18:30:42.603Z" }, - { url = "https://files.pythonhosted.org/packages/5c/0a/fe5f22ef62b193201bc5566762e22049762cd485bfafb5095a7050760054/ty-0.0.65-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:600b8bf6f4940cf7ffb2f43d3716faaf38dcb97cd8617c55771451bc0276408f", size = 12105592, upload-time = "2026-07-29T18:30:45.419Z" }, - { url = "https://files.pythonhosted.org/packages/76/fd/922b3a6e9d697452cdbb4b7e3f636868add5ec652154518a736e4364f3b7/ty-0.0.65-py3-none-musllinux_1_2_i686.whl", hash = "sha256:0c28007bc79d648c1ddaf1e65885d07baec48eb87240da442f608e4107c1b7d8", size = 12387335, upload-time = "2026-07-29T18:30:48.405Z" }, - { url = "https://files.pythonhosted.org/packages/77/22/a1a08ebc84c083db2fb55e3b5cd186db0c067692f4921146f601360231e2/ty-0.0.65-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:c852da96091ad22361e6586b7c7ba98e1334dcd4d8ffb67e47f4fb673de33f77", size = 12682710, upload-time = "2026-07-29T18:30:51.364Z" }, - { url = "https://files.pythonhosted.org/packages/81/14/eaaa410a25bbdea19722109b5422380a0e211b3afcf3071d15953ddbd5db/ty-0.0.65-py3-none-win32.whl", hash = "sha256:cf529d538f1403b14b0511e6ec3cdb95d3d974adabf24cc76cedc533368c3edc", size = 11692341, upload-time = "2026-07-29T18:30:54.35Z" }, - { url = "https://files.pythonhosted.org/packages/bc/0f/6d48f206dce9d7e53fe3b5ea0f0ab5800dd9d2365b2b48f736783436c43f/ty-0.0.65-py3-none-win_amd64.whl", hash = "sha256:234a321e33c7cbbfbd67bfa0b01b685dd9c21f1841781a21e5ca1fa0b25f1d5d", size = 12729355, upload-time = "2026-07-29T18:30:57.275Z" }, - { url = "https://files.pythonhosted.org/packages/96/aa/7446f7725e303cf78e058c893af1f0552b9451895454908706f4c6c3494b/ty-0.0.65-py3-none-win_arm64.whl", hash = "sha256:b9424be1ec56d93ff18609fb1c0a0a2283fe1282cd6d1c7604f97d73b94d61f2", size = 12051375, upload-time = "2026-07-29T18:31:00.579Z" }, +version = "0.0.66" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/58/4f6ab2a86589e422a3cf840bcf6114c565e4c39ddf4d0b7cd328af5b52b4/ty-0.0.66.tar.gz", hash = "sha256:24bddd4479ce445b51ac015410dd2d34af1cadd62a77f5b3cb269149ed83f9b5", size = 6520402, upload-time = "2026-08-04T01:09:47.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/4d/bbc28310d6d887ef73e5800f062c4bf54caa35a1b47d70c7b03d0515ecf1/ty-0.0.66-py3-none-linux_armv6l.whl", hash = "sha256:8b46450438b54b732338e4d7a78a7d2f5e1a012a13d77d121aacaca20fb814e2", size = 12409743, upload-time = "2026-08-04T01:09:01.229Z" }, + { url = "https://files.pythonhosted.org/packages/73/4e/c3d2eb2242fa0a2ef445ca2c7009dc9118e5b3dcb8b8a8bec70d58c8e4bc/ty-0.0.66-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:8e4adbe662bc3c62b52b83d46b07f703fdc3c123bb72601606c58be5ea017ed3", size = 12078362, upload-time = "2026-08-04T01:09:04.233Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/7a4b5e45d701a8de6b714dacf6ffca91b0411baaceaa781d494037623a18/ty-0.0.66-py3-none-macosx_11_0_arm64.whl", hash = "sha256:776814351735847eb934f9a3cbea21d2278ba14aa0fc099f683da11bc2d5c90a", size = 11583289, upload-time = "2026-08-04T01:09:06.973Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4a/fbb1f71ee2999f981f8c4b3b139231e4bcff4ede0c3b37e858fd1334ed26/ty-0.0.66-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cca1da877f613965b954bfd22d495386e260ec767c5536c8022cca98a260ea5d", size = 12137274, upload-time = "2026-08-04T01:09:09.592Z" }, + { url = "https://files.pythonhosted.org/packages/d5/07/9e0662f6603a5ac171ae6b314396e677ead4b45695fc948efb0a4837051f/ty-0.0.66-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aa523e777bc36c0fbf8ded5844096f46cbfc3712fe5a003351a94259b7e86cf4", size = 12207047, upload-time = "2026-08-04T01:09:12.533Z" }, + { url = "https://files.pythonhosted.org/packages/bb/cc/12e01bc2ea47fa7bf20fc6cdd3249d6a340be9be4edc52b1d77256245dd0/ty-0.0.66-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f7993c1f95f80a4e44056e6aaf8e46e608d778db131ae5ce59262ab59358f9a", size = 12919304, upload-time = "2026-08-04T01:09:15.175Z" }, + { url = "https://files.pythonhosted.org/packages/75/88/c6c0d3a8e71c9cdb560cc5c627a4bba6c47b5eec460b2f8c449b57c6d03a/ty-0.0.66-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f9b29354bcbac9f53b6952d8f46789bd81eeb9bdd7a68df7d26e654ca7498c3c", size = 13470963, upload-time = "2026-08-04T01:09:17.849Z" }, + { url = "https://files.pythonhosted.org/packages/27/c2/2f8c18063412ad80e1b3f87afce7bd50982b4a048166607b67f80660a9fb/ty-0.0.66-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fbf4e5325f7f584d9c346946e7c415b2e3cd3b8f1119d468082a90a6afd020ce", size = 13244773, upload-time = "2026-08-04T01:09:20.59Z" }, + { url = "https://files.pythonhosted.org/packages/ed/f6/fdae2b95831116dffc055ad53a99a8f21437263651047b3ad46def4950a3/ty-0.0.66-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bd304363764bd723c22fb20b17035c345420fdf66fee856934b158ebde08a91", size = 12751343, upload-time = "2026-08-04T01:09:23.225Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/7e75f0371d11463a256dc2bee98b0df401e0fa06021e200b8b601d21949d/ty-0.0.66-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:205d8589bd957ea9718d488b731b2fbdd0d1b1cefd37c79f52c0deb7cafddfef", size = 13068057, upload-time = "2026-08-04T01:09:26.052Z" }, + { url = "https://files.pythonhosted.org/packages/b4/84/f20f24518f6f0bea936e2e668ede250b9ce0774624559931599bd1f42772/ty-0.0.66-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:7304a1df54741a2343801354f41d7ad87974acb541ee3e93c26cb6a1a0b863af", size = 12082318, upload-time = "2026-08-04T01:09:28.877Z" }, + { url = "https://files.pythonhosted.org/packages/08/20/70ca0eac2427d4a58a81a3a9426b20e46fb4a5a13aefc9edc2e5172e1243/ty-0.0.66-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:cf2c062863e5da0f588b0b120fec0da2e81c0913ee4cd07b50d77aa60ffd8deb", size = 12228978, upload-time = "2026-08-04T01:09:31.905Z" }, + { url = "https://files.pythonhosted.org/packages/7c/01/5a461ab34456d788248780830aed673c9e0e796f6e9dd92d3dbf02e04d7f/ty-0.0.66-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a58d32c879d86428978332adf21e85009ec269314a23d330c3483c65b57aedc7", size = 12471917, upload-time = "2026-08-04T01:09:34.418Z" }, + { url = "https://files.pythonhosted.org/packages/46/ed/f8eb5eff7c9ee644490c6d2626425935c097acc54871adf659ea70624a2c/ty-0.0.66-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b2f810fa3516c977630d78dbe9161592b3c27029fa9bb81074366624d9b5e4b6", size = 12858086, upload-time = "2026-08-04T01:09:37.602Z" }, + { url = "https://files.pythonhosted.org/packages/02/7f/8a78361f752274a550a7f6f07f127b246ece7d7fdde31121d22074e46eab/ty-0.0.66-py3-none-win32.whl", hash = "sha256:d28c3df565a387c1c5ea359aa452a46d19163616ef71be819058a424a62f1ae1", size = 11782494, upload-time = "2026-08-04T01:09:40.136Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e2/deed22b823ce309b8410d50414fd15afe9e90aee8b8ca04789ba55c21231/ty-0.0.66-py3-none-win_amd64.whl", hash = "sha256:e3a457f3312c078f24c47d0da6e4f73de34d0a77ed2de22571c066c80b2fd5e7", size = 12893338, upload-time = "2026-08-04T01:09:42.825Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ce/6c828f42ef1ed39f53f57f9c0cdcdf03e23666fa7a29d799042a2c34bcb4/ty-0.0.66-py3-none-win_arm64.whl", hash = "sha256:2f62ae247b9c75674fcc060635f9f00210357da7681de59234d97b50fb9e9e94", size = 12227381, upload-time = "2026-08-04T01:09:45.365Z" }, ] [[package]] From 8b8537b86226285a343976ea987cf2759bf3dd7d Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Tue, 4 Aug 2026 20:29:37 -0700 Subject: [PATCH 2/2] fix(perf): harden release artifact promotion - Treat CRLF and LF archive reports as equivalent while preserving stored bytes. - Share comparison selection and report-update guidance across generation paths. - Isolate artifact path resolution and retained-artifact rendering. - Clarify that durable benchmark downloads are GitHub Release assets. --- docs/RELEASING.md | 2 +- scripts/archive_performance.py | 49 +--- scripts/bench_compare.py | 236 +++++++++++++------- scripts/tests/test_archive_performance.py | 10 +- scripts/tests/test_bench_compare.py | 36 +++ scripts/tests/test_performance_artifacts.py | 18 ++ 6 files changed, 225 insertions(+), 126 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index deaf26b..5d4d670 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -154,7 +154,7 @@ clean `target/` until the release report review is complete. For an explicit measurement repair, run `just performance-release `. To compare the stored -GitHub Actions release assets instead of running Cargo locally, use +GitHub Release assets instead of running Cargo locally, use `just performance-github-assets`. The local release workflow validates and then compiles both library revisions with the current checkout's hashed benchmark harness, recording source-state, environment, toolchain, dependency, Criterion, diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index e331281..f5f51d2 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -34,7 +34,7 @@ from pathlib import Path from typing import Any, Literal, cast -from bench_compare import render_release_artifacts +from bench_compare import HOW_TO_UPDATE_SECTION, render_release_artifacts from performance_artifacts import ArtifactPaths, PerformanceBundle, ensure_distinct_paths, load_bundle, publish_bundle from subprocess_utils import ExecutableNotFoundError, run_git_command, run_git_command_with_input, run_safe_command @@ -374,46 +374,10 @@ def _read_text(path: Path) -> str: return path.read_text(encoding="utf-8") -def _how_to_update_section() -> str: - lines = [ - "## How to Update", - "", - "Local performance reports are generated in isolated temporary worktrees:", - "", - "```bash", - "# Local development: compare the current tree with the latest release", - "just performance-local", - "", - "# Release PR: update docs/PERFORMANCE.md and archive the previous report", - "just performance-release", - "", - "# Re-render and promote from retained CSV/JSON inputs (no benchmarks)", - "just performance-rerender", - "", - "# GitHub Actions release assets", - "just performance-github-assets", - "", - "# Explicit repair", - "just performance-release ", - "```", - "", - "`just performance-local` writes `target/bench-reports/performance.md`.", - "`just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`.", - "`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report.", - "", - "Older curated release-to-release reports are archived in `docs/archive/performance/`.", - "", - "See `docs/BENCHMARKING.md` for the full comparison workflow.", - "", - ] - return "\n".join(lines) - - def _normalize_how_to_update(text: str) -> str: - section = _how_to_update_section() if _HOW_TO_UPDATE_RE.search(text): - return _HOW_TO_UPDATE_RE.sub(section, text) - return f"{text.rstrip()}\n\n{section}" + return _HOW_TO_UPDATE_RE.sub(HOW_TO_UPDATE_SECTION, text) + return f"{text.rstrip()}\n\n{HOW_TO_UPDATE_SECTION}" def _replace_file(src: Path, dst: Path) -> None: @@ -478,6 +442,11 @@ def _snapshot_regular_file(path: Path, *, label: str) -> bytes | None: return path.read_bytes() +def _decode_text_snapshot(payload: bytes) -> str: + """Decode UTF-8 bytes with the universal-newline behavior of ``Path.read_text``.""" + return payload.decode("utf-8").replace("\r\n", "\n").replace("\r", "\n") + + def _restore_snapshots(snapshots: tuple[tuple[Path, bytes | None], ...]) -> tuple[BaseException, ...]: """Attempt every file restoration and return all rollback failures.""" errors: list[BaseException] = [] @@ -1431,7 +1400,7 @@ def _existing_archive_matches(*, archive_path: Path | None, current_id: ReportId if archived_payload is None: msg = "existing archive snapshot invariant violated" raise AssertionError(msg) - archived_text = _normalize_how_to_update(archived_payload.decode("utf-8")) + archived_text = _normalize_how_to_update(_decode_text_snapshot(archived_payload)) try: archived_id = parse_report_id(archived_text) except (TypeError, ValueError) as exc: diff --git a/scripts/bench_compare.py b/scripts/bench_compare.py index d958748..7066643 100644 --- a/scripts/bench_compare.py +++ b/scripts/bench_compare.py @@ -185,6 +185,36 @@ SUITE_CHOICES: tuple[str, ...] = ("all", "exact", "vs_linalg") SCOPE_CHOICES: tuple[str, ...] = ("release-signal", "all-benches") +HOW_TO_UPDATE_SECTION = """## How to Update + +Local performance reports are generated in isolated temporary worktrees: + +```bash +# Local development: compare the current tree with the latest release +just performance-local + +# Release PR: update docs/PERFORMANCE.md and archive the previous report +just performance-release + +# Re-render and promote from retained CSV/JSON inputs (no benchmarks) +just performance-rerender + +# GitHub Actions release assets +just performance-github-assets + +# Explicit repair +just performance-release +``` + +`just performance-local` writes `target/bench-reports/performance.md`. +`just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`. +`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report. + +Older curated release-to-release reports are archived in `docs/archive/performance/`. + +See `docs/BENCHMARKING.md` for the full comparison workflow. +""" + type ChangeAssessment = Literal["improvement", "regression", "inconclusive", "unknown"] type BenchmarkSuite = Literal["all", "exact", "vs_linalg"] type ComparisonScope = Literal["release-signal", "all-benches"] @@ -390,6 +420,36 @@ def _dim_from_vs_linalg_group(name: str) -> int | None: return int(suffix) +def _comparison_row_suite(group: str) -> Literal["exact", "vs_linalg"]: + """Return the benchmark suite that owns one comparison row.""" + if group in EXACT_GROUPS: + return "exact" + if _dim_from_vs_linalg_group(group) is not None: + return "vs_linalg" + msg = f"unsupported comparison group: {group!r}" + raise ValueError(msg) + + +def _is_selected_comparison_row( + group: str, + bench: str, + *, + suite: str, + scope: str, +) -> bool: + """Return whether a comparison row belongs to the requested suite and scope.""" + row_suite = _comparison_row_suite(group) + if suite not in ("all", row_suite): + return False + if scope != "release-signal": + return True + if row_suite == "exact": + return group in EXACT_RELEASE_SIGNAL_GROUPS + + dim = _dim_from_vs_linalg_group(group) + return dim is not None and (bench in VS_LINALG_LA_STACK_BENCHES or bench in VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dim, [])) + + def _read_estimate(estimates_json: Path, stat: str = "median") -> CriterionEstimate: """Read and validate a Criterion point estimate and confidence interval.""" try: @@ -809,6 +869,7 @@ def _collect_exact_comparisons( criterion_dir: Path, baseline_name: str, stat: str, + suite: str, policy: ComparisonPolicy, ) -> ComparisonCollection: """Compare exact results while retaining every missing expected row.""" @@ -819,12 +880,15 @@ def _collect_exact_comparisons( frozenset(), ) + selected_groups: list[str] = [] for group, benches in EXACT_GROUPS.items(): - if policy.scope == "release-signal" and group not in EXACT_RELEASE_SIGNAL_GROUPS: + selected_benches = [bench for bench in benches if _is_selected_comparison_row(group, bench, suite=suite, scope=policy.scope)] + if not selected_benches: continue + selected_groups.append(group) group_dir = criterion_dir / group - for bench in benches: + for bench in selected_benches: new_path = group_dir / bench / "new" / "estimates.json" baseline_bench, base_path = _exact_baseline_path(group_dir, bench, baseline_name) missing_current = not new_path.exists() @@ -875,14 +939,19 @@ def _collect_exact_comparisons( ) ) - expected_groups = [group for group in EXACT_GROUPS if policy.scope != "release-signal" or group in EXACT_RELEASE_SIGNAL_GROUPS] - if not any((criterion_dir / group).is_dir() for group in expected_groups): + if selected_groups and not any((criterion_dir / group).is_dir() for group in selected_groups): gaps.append(_entire_suite_gap("exact")) return ComparisonCollection(comparisons=comparisons, gaps=gaps) -def _ordered_vs_linalg_comparison_benches(group_dir: Path, baseline_name: str, scope: str) -> list[str]: +def _ordered_vs_linalg_comparison_benches( + group_dir: Path, + baseline_name: str, + *, + suite: str, + scope: str, +) -> list[str]: """Return expected or discovered comparison rows in stable order.""" dim = _dim_from_vs_linalg_group(group_dir.name) if scope == "release-signal": @@ -896,8 +965,9 @@ def _ordered_vs_linalg_comparison_benches(group_dir: Path, baseline_name: str, s if child.is_dir() and ((child / "new" / "estimates.json").exists() or (child / baseline_name / "estimates.json").exists()) } - ordered = [bench for bench in VS_LINALG_BENCH_ORDER if bench in present] - extras = sorted(present.difference(VS_LINALG_BENCH_ORDER)) + selected = {bench for bench in present if _is_selected_comparison_row(group_dir.name, bench, suite=suite, scope=scope)} + ordered = [bench for bench in VS_LINALG_BENCH_ORDER if bench in selected] + extras = sorted(selected.difference(VS_LINALG_BENCH_ORDER)) return [*ordered, *extras] @@ -951,6 +1021,7 @@ def _collect_vs_linalg_comparisons( criterion_dir: Path, baseline_name: str, stat: str, + suite: str, policy: ComparisonPolicy, ) -> ComparisonCollection: """Compare vs_linalg results while retaining one-sided rows.""" @@ -963,7 +1034,12 @@ def _collect_vs_linalg_comparisons( dim_groups = _vs_linalg_dimension_groups(criterion_dir, policy.scope) for _dim, group_dir in sorted(dim_groups, key=lambda item: item[0]): - expected_benches = _ordered_vs_linalg_comparison_benches(group_dir, baseline_name, policy.scope) + expected_benches = _ordered_vs_linalg_comparison_benches( + group_dir, + baseline_name, + suite=suite, + scope=policy.scope, + ) for bench in expected_benches: new_path = group_dir / bench / "new" / "estimates.json" base_path = group_dir / bench / baseline_name / "estimates.json" @@ -1105,11 +1181,11 @@ def _collect_comparisons( comparisons: list[Comparison] = [] gaps: list[CoverageGap] = [] if suite in ("all", "exact"): - exact = _collect_exact_comparisons(criterion_dir, baseline_name, stat, policy) + exact = _collect_exact_comparisons(criterion_dir, baseline_name, stat, suite, policy) comparisons.extend(exact.comparisons) gaps.extend(exact.gaps) if suite in ("all", "vs_linalg"): - vs_linalg = _collect_vs_linalg_comparisons(criterion_dir, baseline_name, stat, policy) + vs_linalg = _collect_vs_linalg_comparisons(criterion_dir, baseline_name, stat, suite, policy) comparisons.extend(vs_linalg.comparisons) gaps.extend(vs_linalg.gaps) return ComparisonCollection(comparisons=comparisons, gaps=gaps) @@ -1379,15 +1455,9 @@ def _unavailable_artifact_rows( # noqa: PLR0913 unavailable = _UNAVAILABLE_BASELINE_ROWS_BY_COMPATIBILITY.get(compatibility or "", frozenset()) rows: list[PerformanceRow] = [] for group, bench in sorted(unavailable): - row_suite = "exact" if group.startswith("exact_") else "vs_linalg" - if suite not in ("all", row_suite): - continue - if scope == "release-signal" and row_suite == "exact" and group not in EXACT_RELEASE_SIGNAL_GROUPS: + if not _is_selected_comparison_row(group, bench, suite=suite, scope=scope): continue - if scope == "release-signal" and row_suite == "vs_linalg": - dim = _dim_from_vs_linalg_group(group) - if dim is None or bench not in VS_LINALG_RELEASE_SIGNAL_BENCHES_BY_DIM.get(dim, []): - continue + row_suite = _comparison_row_suite(group) current_path = criterion_dir / group / bench / "new" / "estimates.json" if not current_path.is_file(): continue @@ -1612,38 +1682,7 @@ def _generate_markdown( lines.extend(["", table, ""]) - lines.extend( - [ - "## How to Update", - "", - "Local performance reports are generated in isolated temporary worktrees:", - "", - "```bash", - "# Local development: compare the current tree with the latest release", - "just performance-local", - "", - "# Release PR: update docs/PERFORMANCE.md and archive the previous report", - "just performance-release", - "", - "# Re-render and promote from retained CSV/JSON inputs (no benchmarks)", - "just performance-rerender", - "", - "# GitHub Actions release assets", - "just performance-github-assets", - "", - "# Explicit repair", - "just performance-release ", - "```", - "", - "`just performance-local` writes `target/bench-reports/performance.md`.", - "`just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`.", - "`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report.", - "", - "Older curated release-to-release reports are archived in `docs/archive/performance/`.", - "", - "See `docs/BENCHMARKING.md` for the full comparison workflow.", - ] - ) + lines.append(HOW_TO_UPDATE_SECTION.rstrip("\n")) return "\n".join(lines) + "\n" @@ -1938,6 +1977,65 @@ def _save_baseline_hint(suite: str, baseline: str) -> str: return f"just bench-save-baseline {baseline}" +def _resolve_artifact_paths( + args: argparse.Namespace, + *, + root: Path, + output_path: Path, + stat: Statistic, +) -> ArtifactPaths | None: + """Resolve and validate the optional release-performance artifact pair.""" + if (args.csv_output is None) != (args.provenance_output is None): + msg = "--csv-output and --provenance-output must be provided together" + raise ValueError(msg) + if args.csv_output is None: + return None + if args.snapshot or stat != "median": + msg = "release-performance artifacts require a median baseline comparison" + raise ValueError(msg) + + try: + paths = ArtifactPaths( + csv=Path(args.csv_output) if Path(args.csv_output).is_absolute() else root / args.csv_output, + provenance=(Path(args.provenance_output) if Path(args.provenance_output).is_absolute() else root / args.provenance_output), + ) + ensure_distinct_paths( + { + "Markdown output": output_path, + "artifact CSV": paths.csv, + "artifact provenance": paths.provenance, + } + ) + except (OSError, ValueError) as err: + msg = f"Invalid release-performance artifact paths: {err}" + raise ValueError(msg) from err + return paths + + +def _write_and_render_artifacts( + paths: ArtifactPaths, + *, + root: Path, + criterion_dir: Path, + settings: ReportSettings, + collection: ComparisonCollection | None, +) -> str: + """Write a validated artifact pair and render its reloaded report.""" + baseline_name = settings.baseline_name + if baseline_name is None or collection is None: + msg = "release-performance artifacts require a completed baseline comparison" + raise ValueError(msg) + bundle = _release_artifact_bundle( + root=root, + criterion_dir=criterion_dir, + baseline_name=baseline_name, + settings=settings, + collection=collection, + ) + write_bundle(paths, bundle) + return render_release_artifacts(paths) + + def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, PLR0915 """Generate a benchmark snapshot or comparison report from CLI arguments.""" args = _parse_args(sys.argv[1:] if argv is None else argv) @@ -1950,31 +2048,12 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, criterion_dir = root / args.criterion_dir output_path = Path(args.output) if Path(args.output).is_absolute() else root / args.output - if (args.csv_output is None) != (args.provenance_output is None): - print("--csv-output and --provenance-output must be provided together", file=sys.stderr) - return 2 - if args.csv_output is not None and (args.snapshot or stat != "median"): - print("release-performance artifacts require a median baseline comparison", file=sys.stderr) + try: + artifact_paths = _resolve_artifact_paths(args, root=root, output_path=output_path, stat=stat) + except ValueError as err: + print(err, file=sys.stderr) return 2 - artifact_paths: ArtifactPaths | None = None - if args.csv_output is not None and args.provenance_output is not None: - try: - artifact_paths = ArtifactPaths( - csv=Path(args.csv_output) if Path(args.csv_output).is_absolute() else root / args.csv_output, - provenance=(Path(args.provenance_output) if Path(args.provenance_output).is_absolute() else root / args.provenance_output), - ) - ensure_distinct_paths( - { - "Markdown output": output_path, - "artifact CSV": artifact_paths.csv, - "artifact provenance": artifact_paths.provenance, - } - ) - except (OSError, ValueError) as err: - print(f"Invalid release-performance artifact paths: {err}", file=sys.stderr) - return 2 - if not criterion_dir.is_dir(): print( f"No Criterion results found at {criterion_dir}.\nRun benchmarks first:\n {_run_bench_hint(args.suite)}\n", @@ -2063,20 +2142,15 @@ def main(argv: list[str] | None = None) -> int: # noqa: C901, PLR0911, PLR0912, scope=scope, harness_provenance=harness_provenance, ) - if args.csv_output is not None and args.provenance_output is not None: - if baseline_name is None or collection is None or artifact_paths is None: - msg = "release-performance artifact invariant violated" - raise AssertionError(msg) + if artifact_paths is not None: try: - bundle = _release_artifact_bundle( + md = _write_and_render_artifacts( + artifact_paths, root=root, criterion_dir=criterion_dir, - baseline_name=baseline_name, settings=settings, collection=collection, ) - write_bundle(artifact_paths, bundle) - md = render_release_artifacts(artifact_paths) except (ExceptionGroup, OSError, KeyError, TypeError, ValueError) as err: print(f"Invalid release-performance artifact data: {err}", file=sys.stderr) return 2 diff --git a/scripts/tests/test_archive_performance.py b/scripts/tests/test_archive_performance.py index 9e210eb..96226ab 100644 --- a/scripts/tests/test_archive_performance.py +++ b/scripts/tests/test_archive_performance.py @@ -122,9 +122,10 @@ def _retained_report(root: Path, *, stem: str = "performance") -> str: return archive_performance._normalize_how_to_update(archive_performance.render_release_artifacts(paths)) -def test_normalized_report_links_archived_performance_reports() -> None: +def test_normalized_report_uses_shared_how_to_update_section() -> None: text = _normalized_report("0.4.3", "v0.4.2") - assert "Older curated release-to-release reports are archived in `docs/archive/performance/`." in text + + assert text.endswith(archive_performance.HOW_TO_UPDATE_SECTION) def _legacy_report(version: str, baseline: str) -> str: @@ -835,7 +836,8 @@ def test_promote_report_does_not_overwrite_existing_archive(tmp_path: Path) -> N current.parent.mkdir(parents=True) current.write_text(_report("0.4.1", "v0.4.0"), encoding="utf-8") archive_dir.mkdir(parents=True) - archived.write_text(_normalized_report("0.4.1", "v0.4.0"), encoding="utf-8") + archived_payload = _normalized_report("0.4.1", "v0.4.0").replace("\n", "\r\n").encode() + archived.write_bytes(archived_payload) promote_report( source=source, @@ -845,7 +847,7 @@ def test_promote_report_does_not_overwrite_existing_archive(tmp_path: Path) -> N expected_baseline_tag="v0.4.1", ) - assert archived.read_text(encoding="utf-8") == _normalized_report("0.4.1", "v0.4.0") + assert archived.read_bytes() == archived_payload def test_promote_report_rejects_mismatched_existing_archive_without_mutation(tmp_path: Path) -> None: diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 387b99a..8929096 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -966,6 +966,7 @@ def test_main_snapshot_writes_output(tmp_path: Path) -> None: assert "just performance-release " in text assert "Older curated release-to-release reports are archived in `docs/archive/performance/`." in text assert "git checkout" not in text + assert text.endswith(bench_compare.HOW_TO_UPDATE_SECTION) def test_main_no_criterion_dir(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -1076,6 +1077,41 @@ def test_main_rejects_invalid_artifact_paths_without_writing( assert {path: path.read_bytes() for path in paths} == before +@pytest.mark.parametrize( + ("invalid_options", "expected_error"), + [ + ("csv-only", "--csv-output and --provenance-output must be provided together"), + ("provenance-only", "--csv-output and --provenance-output must be provided together"), + ("snapshot", "release-performance artifacts require a median baseline comparison"), + ("mean", "release-performance artifacts require a median baseline comparison"), + ], +) +def test_main_rejects_invalid_artifact_option_combinations( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + invalid_options: str, + expected_error: str, +) -> None: + csv_output = tmp_path / "performance.csv" + provenance_output = tmp_path / "performance.provenance.json" + artifact_args = ["--csv-output", str(csv_output), "--provenance-output", str(provenance_output)] + if invalid_options == "csv-only": + artifact_args = artifact_args[:2] + elif invalid_options == "provenance-only": + artifact_args = artifact_args[2:] + elif invalid_options == "snapshot": + artifact_args.append("--snapshot") + else: + artifact_args.extend(["--stat", "mean"]) + + rc = bench_compare.main(["--repo-root", str(tmp_path), *artifact_args]) + + assert rc == 2 + assert expected_error in capsys.readouterr().err + assert not csv_output.exists() + assert not provenance_output.exists() + + def test_main_v043_comparison_allows_only_unavailable_balanced_baselines( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, diff --git a/scripts/tests/test_performance_artifacts.py b/scripts/tests/test_performance_artifacts.py index f0ff6a1..9113ca6 100644 --- a/scripts/tests/test_performance_artifacts.py +++ b/scripts/tests/test_performance_artifacts.py @@ -174,6 +174,24 @@ def test_artifact_serialization_is_independent_of_input_row_order() -> None: assert serialize_bundle(reordered) == serialize_bundle(bundle) +def test_artifact_loader_rejects_csv_digest_mismatch() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + malformed = csv_payload.replace(b"exact_d2/det_exact", b"exact_d2/det_changed", 1) + + with pytest.raises(ValueError, match="CSV digest mismatch"): + load_bundle_bytes(malformed, provenance_payload, source="digest mismatch fixture") + + +def test_artifact_loader_rejects_csv_row_count_mismatch() -> None: + csv_payload, provenance_payload = serialize_bundle(_bundle()) + provenance = json.loads(provenance_payload) + provenance["csv"]["row_count"] += 1 + malformed = (json.dumps(provenance, indent=2, sort_keys=True) + "\n").encode() + + with pytest.raises(ValueError, match="CSV row count mismatch"): + load_bundle_bytes(csv_payload, malformed, source="row count mismatch fixture") + + def test_artifact_loader_rejects_incomplete_confidence_interval() -> None: csv_payload, provenance_payload = serialize_bundle(_bundle()) reader = csv.DictReader(io.StringIO(csv_payload.decode(), newline=""))