From c601c4146b01868a29288b7221e413706e3e01c7 Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Wed, 12 Aug 2026 13:42:57 -0700 Subject: [PATCH] fix(perf)!: unify local and release performance workflows - Retain local Markdown, CSV, and provenance artifacts while allowing same-version comparisons of tracked changes. - Exclude untracked files explicitly and isolate the narrowed non-exact comparison bundle. - Promote distinct-release artifacts through performance-doc and keep performance-release atomic and rollback-capable. - Render one consolidated table per benchmark suite and reject invalid release publication states. BREAKING CHANGE: rename performance-rerender to performance-doc and performance-local-vs-linalg to performance-local-non-exact, replace --rerender with --promote-artifacts, and remove performance-archive-published. --- CONTRIBUTING.md | 8 +- docs/BENCHMARKING.md | 103 +++++-- docs/PERFORMANCE.md | 10 +- docs/RELEASING.md | 34 ++- justfile | 26 +- scripts/README.md | 41 ++- scripts/archive_performance.py | 203 ++++++++----- scripts/bench_compare.py | 75 +++-- scripts/check_docs_version_sync.py | 2 +- scripts/performance_artifacts.py | 11 +- scripts/tests/test_archive_performance.py | 269 +++++++++++++++--- scripts/tests/test_bench_compare.py | 39 +-- scripts/tests/test_check_docs_version_sync.py | 4 +- scripts/tests/test_performance_artifacts.py | 14 + 14 files changed, 601 insertions(+), 238 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca4f5f9..5c65679 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -183,9 +183,11 @@ just performance-local ``` It writes `target/bench-reports/performance.md` without changing committed -release documentation. Treat regressions as design feedback. If a slowdown is -intentional, explain the correctness, API clarity, or composability benefit -that justifies it. +release documentation or requiring a version bump. Staged and unstaged changes +to tracked files participate; untracked files are excluded, so stage any new +benchmark-relevant file first. Treat regressions as design feedback. If a +slowdown is intentional, explain the correctness, API clarity, or composability +benefit that justifies it. Core Rust, Cargo, or public-behavior changes must pass `just ci` before a pull request is ready. Documentation, configuration, Python, test-only, diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index d306ece..77bd4db 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -25,12 +25,12 @@ the commands measure and where their outputs go. | Goal | Recipe | |------|--------| | Latest-release local audit | `just performance-local` | -| Release-signal check against tags | `just performance-local-vs-linalg v0.4.4 v0.4.3` | +| Non-exact release-signal check against tags | `just performance-local-non-exact v0.4.4 v0.4.3` | | Fast saved-baseline loop | `just bench-save-baseline ` then `just bench-compare all-benches` | | Full crate comparison | `just bench-vs-linalg` | | README table and plot | `just plot-vs-linalg-readme` | | Release report | `just performance-release v0.4.4 v0.4.3` | -| Re-render retained release inputs | `just performance-rerender` | +| Build docs from retained release inputs | `just performance-doc` | | Published-asset comparison | `just performance-github-assets v0.4.4 v0.4.3` | Rule of thumb: @@ -40,13 +40,34 @@ Rule of thumb: - Use `bench-vs-linalg` plus plotting when updating README crate-to-crate comparisons. - Use `performance-release` only when preparing committed release artifacts. -- Use `performance-rerender` for report-format changes after a valid release - dataset has already been retained. +- Use `performance-doc` for report-format changes after a valid, promotable + comparison dataset has already been retained. + +The three canonical workflows compose around one artifact schema, metric set, +and renderer: + +| Recipe | Measure | Retain CSV/JSON | Promote release docs | +|--------|---------|-----------------|----------------------| +| `performance-local` | Yes | Yes | No | +| `performance-doc` | No | Consumes retained inputs | Yes | +| `performance-release` | Yes | Yes | Yes | + +For a distinct release pair with no intervening source or configuration changes, +running `performance-local` followed by `performance-doc` produces the same +report and committed documentation as `performance-release`. +`performance-release` exists as the safer one-step release operation: it keeps +fresh measurement, validated artifact publication, and rollback-capable document +promotion in one command. ## Benchmark Suites `la-stack` has two Criterion benchmark suites. +Newly rendered reports use one table per selected suite. Dimension and +adversarial-input group appear in a `Case` column instead of creating a separate +table for every group. The `vs_linalg` table is the one wider variant because it +adds nalgebra and faer context columns where matching peer measurements exist. + **`vs_linalg`** (`benches/vs_linalg.rs`) compares `la-stack` against `nalgebra` and `faer` across D=2-64 for LU, solve, determinant, dot, norm, and SPD factorization operations. Use this suite to answer "why choose la-stack over @@ -76,16 +97,20 @@ just performance-local This creates isolated temporary worktrees and runs both library revisions on the same machine with the current checkout's benchmark sources, manifests, lockfile, -benchmark-input tests, recipes, and Rust toolchain. Only the baseline library -implementation comes from the release tag. Before either timing run, the command -runs `just test-bench-inputs` against that revision under the shared current -fixture harness. This is a prerequisite correctness gate over the deterministic -fixtures and operations, not validation of each timed Criterion sample. It -writes `target/bench-reports/performance.md` and records both -commits, CPU, operating system, Rust toolchain, lockfile and harness digests, -Criterion selection/commands, and both correctness-gate results. The report -reader rejects malformed or mismatched provenance and incomplete selected-suite -coverage. +benchmark-input tests, recipes, and Rust toolchain. Staged and unstaged changes +to tracked files are applied to the current worktree. Untracked files are +excluded; stage a new file before running the command if it must participate in +the comparison. Only the baseline library implementation comes from the release +tag. Before either timing run, the command runs `just test-bench-inputs` against +that revision under the shared current fixture harness. This is a prerequisite +correctness gate over the deterministic fixtures and operations, not validation +of each timed Criterion sample. It writes +`target/bench-reports/performance.md` plus retained `performance.csv` and +`performance.provenance.json` comparison inputs. The report and sidecar embed +both commits, CPU, operating system, Rust toolchain, lockfile and harness +digests, Criterion selection/commands, and both correctness-gate results. The +report reader rejects malformed or mismatched provenance and incomplete +selected-suite coverage. The shared harness carries an explicit v0.4.3-only API adapter for renamed or ownership-adjusted calls (`det_sign_exact`, `Tolerance`, and vector dot @@ -116,17 +141,18 @@ timing, current validation, and current timing, so a long comparison exposes completed samples and its active phase instead of remaining silent until the final report is rendered. -If the checkout's package version is identical to the latest published release, -the command now stops before creating worktrees or running benchmarks because a -release report requires two distinct identifiers. For repeated optimization -within one package version, use the named-baseline loop below instead. +The local report may compare a checkout whose package version is identical to +the latest published release. Commit/ref and source-state provenance distinguish +the modified checkout from the tagged baseline even though both display the same +package version. Release artifact publication remains stricter and requires two +distinct release identifiers. ### Compare Current Code With A Specific Release For a narrower non-exact check against a known release pair, run: ```bash -just performance-local-vs-linalg v0.4.4 v0.4.3 +just performance-local-non-exact v0.4.4 v0.4.3 ``` This generates a local `v0.4.3` `vs_linalg` baseline, measures the current @@ -134,6 +160,14 @@ la-stack `vs_linalg` rows, and renders a `vs_linalg` report. The report includes saved baseline nalgebra/faer timings as context where matching peer rows exist, without rerunning current peer crates. +This narrowed peer-context view uses the same metrics and renderer but writes a +separate `performance-non-exact.*` scratch bundle so it cannot replace the +canonical full comparison inputs accidentally. + +When tags are provided explicitly, the current tag must match the package +version in the `HEAD` checkout. A mismatch is rejected before tags are fetched, +worktrees are created, or benchmarks run. + ### Iterate Against A Local Saved Baseline Use local saved baselines when tuning one kernel and comparing several edits @@ -240,13 +274,18 @@ To reproduce and promote the report without running Cargo or creating Git worktrees, use: ```bash -just performance-rerender +just performance-doc ``` This command fails closed on a missing, partial, malformed, mismatched, or -unsupported artifact pair. Use it for presentation-only report corrections; -changes to benchmark inputs, code, toolchains, or measurement configuration -require a fresh `performance-release` run. +unsupported artifact pair. It consumes the default CSV/JSON pair retained by a +successful `performance-local` or `performance-release` run, rewrites the +scratch Markdown, promotes it to `docs/PERFORMANCE.md`, and archives the previous +committed report when the release pair changes. Promotion requires distinct +current and baseline package versions, so a same-version local comparison is +retained and reproducible but cannot become release documentation. Use promotion +for presentation-only report corrections; changes to benchmark inputs, code, +toolchains, or measurement configuration require a fresh local or release run. ### Compare Published Release Artifacts @@ -276,14 +315,15 @@ shared-harness workflow before attributing a difference solely to library code. | Path | Committed? | Producer | Purpose | |------|------------|----------|---------| | `target/criterion/` | No | `cargo bench`, `bench-save-*` | Local Criterion measurements and named baselines. | -| `target/bench-reports/performance.md` | No | `bench-compare`, `performance-local*` | Local comparison report. | -| `target/bench-reports/performance.csv` | No | `performance-local*`, `performance-release` | Validated tabular inputs for the release comparison. | -| `target/bench-reports/performance.provenance.json` | No | `performance-local*`, `performance-release` | Schema, release, source, command, toolchain, host, digest, and harness provenance. | +| `target/bench-reports/performance.md` | No | `bench-compare`, `performance-local`, `performance-release`, `performance-doc` | Canonical local comparison report. | +| `target/bench-reports/performance.csv` | No | `performance-local`, `performance-release` | Validated tabular inputs for the canonical comparison. | +| `target/bench-reports/performance.provenance.json` | No | `performance-local`, `performance-release` | Schema, package identifiers, source, command, toolchain, host, digest, and harness provenance. | +| `target/bench-reports/performance-non-exact.*` | No | `performance-local-non-exact` | Narrowed non-exact report and retained peer-context comparison inputs. | | `target/bench-reports/github-assets-performance.md` | No | `performance-github-assets` | Local report from published release artifacts. | | `target/bench-reports/github-assets-performance.csv` | No | `performance-github-assets` | Tabular inputs derived from published native archives. | | `target/bench-reports/github-assets-performance.provenance.json` | No | `performance-github-assets` | Provenance for the published-asset report inputs. | -| `docs/PERFORMANCE.md` | Yes | `performance-release` | Latest curated release-to-release comparison. | -| `docs/archive/performance/` | Yes | `performance-release` | Older curated release-to-release comparisons. | +| `docs/PERFORMANCE.md` | Yes | `performance-release`, `performance-doc` | Latest curated release-to-release comparison. | +| `docs/archive/performance/` | Yes | `performance-release`, `performance-doc` | Older curated release-to-release comparisons. | | `docs/assets/bench/` | Yes | `plot-vs-linalg-readme` | README benchmark CSV/SVG assets and JSON provenance. | | GitHub Release | Remote | `.github/workflows/release-benchmarks.yml` | Criterion baseline archive. | @@ -293,9 +333,10 @@ Published baseline assets use the filename Everything under `target/bench-reports/` is reproducible local scratch owned by the performance-report workflows. It survives temporary-worktree cleanup but may be removed by `just clean` or `cargo clean`; retain or copy the CSV/JSON pair -while reviewing or rerendering a release PR. The compact CSV is the analysis -and Markdown-reproduction layer. It does not replace the full native Criterion -`.tar.gz` archive attached to each GitHub Release, which remains the durable raw +while reviewing or re-rendering a release PR. The compact CSV is the analysis and +Markdown-reproduction layer for both local and release comparisons. It does not +replace the full native Criterion `.tar.gz` archive attached to each GitHub +Release, which remains the durable raw baseline for post-release comparisons. ## `vs_linalg` Methodology diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 23a5c33..9009a30 100644 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -360,6 +360,9 @@ just performance-local # Release PR: update docs/PERFORMANCE.md and archive the previous report just performance-release +# Build release docs from retained CSV/JSON inputs (no benchmarks) +just performance-doc + # GitHub Actions release assets just performance-github-assets @@ -367,8 +370,13 @@ just performance-github-assets just performance-release ``` -`just performance-local` writes `target/bench-reports/performance.md`. +`just performance-local` writes `performance.md` plus retained `performance.csv` and +`performance.provenance.json` comparison inputs under `target/bench-reports/` without promoting documentation. +It applies staged and unstaged tracked changes; untracked files are excluded. `just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`. +`just performance-release` performs the same measurement and retention work, then promotes distinct-release documentation. +`just performance-doc` consumes the retained pair from either workflow without benchmarking and promotes it when the package versions differ. +For a distinct pair, `performance-local` followed by `performance-doc` is equivalent to the atomic `performance-release` workflow. Older curated release-to-release reports are archived in `docs/archive/performance/`. diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 0568fda..8f27353 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -140,20 +140,36 @@ incomplete. just performance-release ``` +Run this only after the package version has been bumped. The no-argument form +uses the `Cargo.toml` version as the current release and discovers the previous +stable published release. For an explicit repair, the supplied current tag must +match the package version in the `HEAD` checkout; mismatches are rejected before +tag fetching, worktree creation, or benchmarking. Release report publication +also requires distinct current and baseline release identifiers. + Review `docs/PERFORMANCE.md` for the latest release-to-release comparison. Older committed comparisons are archived under `docs/archive/performance/` with lexicographically sorted filenames such as `v0.4.2-vs-v0.4.1.md`. Iterative local reports still live under `target/bench-reports/`. +The temporary current worktree includes staged and unstaged changes to tracked +files. Untracked files are excluded, so stage every new benchmark-relevant file +before running the release comparison. + The release command retains `target/bench-reports/performance.csv` and `performance.provenance.json` before its temporary worktrees are removed, then renders and promotes Markdown from a validated reload of that pair. Review the CSV coverage/timing rows and JSON release, revision, command, toolchain, host, harness, and digest metadata alongside the Markdown. For a presentation-only -correction, run `just performance-rerender`; it reproduces and promotes the -report from those files without invoking Cargo or creating worktrees. The pair -is local scratch and may be removed by `just clean` or `cargo clean`, so do not -clean `target/` until the release report review is complete. +correction, run `just performance-doc`; it reproduces and promotes the +report from those files without invoking Cargo or creating worktrees. The same +command can consume a pair retained by `performance-local` after the version +bump. It rejects same-version local artifacts because committed performance docs +must compare distinct releases. The command updates `docs/PERFORMANCE.md` and the +performance archive, so review and stage those changes as release artifacts. +The retained pair is local scratch and may be removed by `just clean` or +`cargo clean`, so do not clean `target/` until the release report review is +complete. For an explicit measurement repair, run `just performance-release `. To compare the stored @@ -165,6 +181,16 @@ and validation provenance. Stored release assets retain their original per-release harnesses; unavailable historical measurement metadata is labelled explicitly rather than treated as an isolated library-code comparison. +`just performance-local` is the non-promoting half of the release workflow: it +measures and writes `target/bench-reports/performance.md` plus the adjacent +retained CSV/JSON pair. The local report may compare revisions with the same +package version because commit/ref and source-state provenance distinguish them. +For a distinct pair, following it with `performance-doc` is equivalent to +`performance-release`; the latter performs fresh measurement, retention, and +rollback-capable promotion atomically. `performance-local-non-exact` uses the +same metric and report model for the narrowed peer-context view but writes a +separate `performance-non-exact.*` scratch bundle. + After the GitHub Release is published, the `Release Benchmarks` workflow checks out the release tag, runs the independent benchmark-input tests, saves a full Criterion baseline, and attaches diff --git a/justfile b/justfile index 30c63ae..ddbdeb1 100644 --- a/justfile +++ b/justfile @@ -457,9 +457,9 @@ help-workflows: @echo " just bench-vs-linalg-latest-vs # Run non-exact latest and compare against last" @echo " just performance-github-assets # Compare stored GitHub Actions release assets" @echo " just performance-local # Compare current tree against latest release locally" - @echo " just performance-local-vs-linalg # Compare current non-exact kernels locally" - @echo " just performance-release # Promote local release performance docs" - @echo " just performance-rerender # Re-render release docs from retained CSV/JSON" + @echo " just performance-local-non-exact # Compare current non-exact kernels locally" + @echo " just performance-release # Measure, retain, and publish release docs" + @echo " just performance-doc # Build release docs from retained CSV/JSON" @echo " just bench-save-last # Save full baseline as 'last'" @echo " just bench-vs-linalg # Run vs_linalg bench (optional filter)" @echo " just bench-vs-linalg-la-stack # Run la-stack rows from vs_linalg" @@ -572,10 +572,6 @@ markdown-fix: _ensure-rumdl markdown-lint: markdown-check -# Backward-compatible alias for the GitHub Actions release-asset comparison. -performance-archive-published current_tag="" baseline_tag="": - just performance-github-assets {{ quote(current_tag) }} {{ quote(baseline_tag) }} - # Compare stored GitHub Actions release benchmark assets without local cargo runs. performance-github-assets current_tag="" baseline_tag="": python-sync #!/usr/bin/env bash @@ -592,12 +588,12 @@ performance-github-assets current_tag="" baseline_tag="": python-sync uv run --locked archive-performance --published-latest --github-assets --generate-in-temp-worktree --output-only --output target/bench-reports/github-assets-performance.md --artifact-csv target/bench-reports/github-assets-performance.csv --artifact-provenance target/bench-reports/github-assets-performance.provenance.json fi -# Compare the current tree against the latest published release locally. +# Compare the current tree against the latest release; untracked files are excluded. performance-local: python-sync - uv run --locked archive-performance --current-vs-latest --generate-in-temp-worktree --output-only --output target/bench-reports/performance.md + uv run --locked archive-performance --current-vs-latest --generate-in-temp-worktree --output-only --local-report --output target/bench-reports/performance.md # Compare current non-exact kernels locally without rerunning current peer crates. -performance-local-vs-linalg current_tag="" baseline_tag="": python-sync +performance-local-non-exact current_tag="" baseline_tag="": python-sync #!/usr/bin/env bash set -euo pipefail current_tag={{ quote(current_tag) }} @@ -607,9 +603,9 @@ performance-local-vs-linalg current_tag="" baseline_tag="": python-sync echo "current_tag and baseline_tag must be provided together" >&2 exit 2 fi - uv run --locked archive-performance "$current_tag" "$baseline_tag" --suite vs_linalg --generate-in-temp-worktree --worktree-ref HEAD --output-only --output target/bench-reports/performance.md + uv run --locked archive-performance "$current_tag" "$baseline_tag" --suite vs_linalg --generate-in-temp-worktree --worktree-ref HEAD --output-only --local-report --output target/bench-reports/performance-non-exact.md --artifact-csv target/bench-reports/performance-non-exact.csv --artifact-provenance target/bench-reports/performance-non-exact.provenance.json else - uv run --locked archive-performance --current-vs-latest --suite vs_linalg --generate-in-temp-worktree --output-only --output target/bench-reports/performance.md + uv run --locked archive-performance --current-vs-latest --suite vs_linalg --generate-in-temp-worktree --output-only --local-report --output target/bench-reports/performance-non-exact.md --artifact-csv target/bench-reports/performance-non-exact.csv --artifact-provenance target/bench-reports/performance-non-exact.provenance.json fi # Generate local release-signal measurements in a temp worktree, then promote/archive docs. @@ -628,9 +624,9 @@ performance-release current_tag="" baseline_tag="": python-sync uv run --locked archive-performance --infer-release --generate-in-temp-worktree --worktree-ref HEAD fi -# Re-render and promote release performance docs from retained report inputs. -performance-rerender: python-sync - uv run --locked archive-performance --rerender +# Build and promote release performance docs from retained report inputs. +performance-doc: python-sync + uv run --locked archive-performance --promote-artifacts # Plot: generate a single time-vs-dimension SVG from Criterion results. plot-vs-linalg metric="lu_solve" stat="median" sample="new" log_y="false" allow_partial="false": python-sync diff --git a/scripts/README.md b/scripts/README.md index 652673d..fd1f4af 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -53,8 +53,8 @@ just performance-local # Release PR: update docs/PERFORMANCE.md and archive the previous report just performance-release -# Re-render the release report from retained CSV/JSON inputs -just performance-rerender +# Build release docs from retained CSV/JSON inputs +just performance-doc # GitHub Actions release assets, without local cargo benchmark runs just performance-github-assets @@ -62,22 +62,35 @@ just performance-github-assets Local benchmark generation streams Cargo and Criterion progress while retaining the existing fail-closed report and provenance checks. Lines prefixed with -`[performance]` identify the active validation or timing phase. A -current-vs-latest request whose package and release identifiers match is -rejected before benchmark work starts. +`[performance]` identify the active validation or timing phase. Staged and +unstaged changes to tracked files participate. Untracked files are excluded; +stage a new file before running the command if it must participate. A local +current-vs-latest report may use the same package and release identifier because +its commit/ref and source-state provenance still distinguish the revisions. The local release workflows run the independent benchmark-input correctness gate and then measure both library revisions with one hashed current benchmark harness. Reports record source-state, environment, toolchain, dependency, Criterion, harness, and validation provenance and fail on incomplete selected -coverage. `performance-release` retains schema-versioned `performance.csv` and -`performance.provenance.json` inputs under `target/bench-reports/`, then renders -the promoted Markdown from their validated reload. `performance-rerender` uses -that pair without Cargo or temporary worktrees. These files are reproducible -scratch and may be removed with `target/`; native Criterion release archives -remain the durable raw baselines. Direct comparisons of separately published -artifacts retain their original per-release harnesses and label unavailable -historical measurement metadata explicitly. +coverage. `performance-local` writes Markdown plus schema-versioned +`performance.csv` and `performance.provenance.json` inputs under +`target/bench-reports/` without promoting documentation. `performance-release` +does the same measurement and retention work, requires distinct releases, and +promotes the validated result. `performance-doc` consumes the retained pair +from either workflow without Cargo or temporary worktrees, then promotes the +result into `docs/PERFORMANCE.md` and the archive. Same-version local artifacts +remain valid comparison evidence but cannot be promoted as a release report. +These files are reproducible scratch and may be removed with `target/`; native +Criterion release archives remain the durable raw baselines. Direct comparisons +of separately published artifacts retain their original per-release harnesses +and label unavailable historical measurement metadata explicitly. + +Operationally, `performance-release` is the atomic composition of +`performance-local` and `performance-doc`: measure, retain the common +comparison inputs, render, and promote. The narrowed +`performance-local-non-exact` view uses the same metrics and renderer with +nalgebra/faer context, but writes `performance-non-exact.*` so it does not +overwrite the canonical full comparison bundle. See `docs/BENCHMARKING.md` for the current command matrix, local saved-baseline workflow, explicit tag arguments, output locations, and release-artifact @@ -233,7 +246,7 @@ validates SemVer, and handles GitHub's 125KB tag-annotation size limit. |---|---| | `archive_changelog.py` | Split completed changelog minor series into archives | | `archive_performance.py` | Promote release performance docs and archive older comparisons | -| `performance_artifacts.py` | Validate and publish schema-versioned release-report CSV/JSON inputs | +| `performance_artifacts.py` | Validate and publish schema-versioned performance-comparison CSV/JSON inputs | | `bench_compare.py` | Compare Criterion benchmark baselines and render Markdown reports | | `check_docs_version_sync.py` | Verify versioned documentation links and snippets stay synchronized | | `criterion_dim_plot.py` | Plot Criterion benchmark results (CSV + SVG + README table) | diff --git a/scripts/archive_performance.py b/scripts/archive_performance.py index 269283f..5af3f22 100644 --- a/scripts/archive_performance.py +++ b/scripts/archive_performance.py @@ -1,12 +1,12 @@ #!/usr/bin/env -S uv run --locked -"""Promote a benchmark report into docs/PERFORMANCE.md and archive the old one. +"""Generate, retain, render, and promote benchmark comparison reports. Release performance docs have two different lifetimes: - ``target/bench-reports/performance.md`` is local scratch output for the current machine and branch. - ``target/bench-reports/performance.csv`` and the adjacent provenance JSON - are validated, rerenderable release-report inputs. + are validated, reproducible performance-comparison inputs. - ``docs/PERFORMANCE.md`` is the latest curated release-to-release comparison. - ``docs/archive/performance/*.md`` stores older curated comparisons. @@ -35,7 +35,7 @@ from typing import Any, Literal, cast from bench_compare import HOW_TO_UPDATE_SECTION, render_release_artifacts -from performance_artifacts import ArtifactPaths, PerformanceBundle, ensure_distinct_paths, load_bundle, publish_bundle +from performance_artifacts import ArtifactPaths, ensure_distinct_paths, load_bundle, publish_bundle from subprocess_utils import ExecutableNotFoundError, cpu_description, run_git_command, run_git_command_with_input, run_safe_command _VERSION_RE = re.compile(r"^\*\*la-stack\*\* v(?P[^\s`]+)", re.MULTILINE) @@ -173,10 +173,9 @@ class ArchivePaths: @dataclass(frozen=True) class GeneratedReport: - """A report rendered from a validated, durable artifact bundle.""" + """A generated benchmark report that remains valid after worktree cleanup.""" text: str - bundle: PerformanceBundle @dataclass(frozen=True) @@ -184,7 +183,7 @@ class ArchiveResult: """Result and destination metadata for a completed archive operation.""" report_id: ReportId - action: Literal["output", "promote-generated", "promote-source", "rerender"] + action: Literal["output", "promote-generated", "promote-source", "promote-artifacts"] @dataclass(frozen=True) @@ -341,10 +340,6 @@ def _previous_release_from_list(stable_releases: list[PublishedRelease], current return previous_releases[-1] -def _previous_published_release(repo_root: Path, current_tag: str) -> PublishedRelease: - return _previous_release_from_list(_published_stable_releases(repo_root), current_tag) - - def _normalize_worktree_ref_for_tag(worktree_ref: str, current_tag: str) -> str: try: normalized_ref = normalize_tag(worktree_ref) @@ -378,6 +373,25 @@ def _published_release_pair(repo_root: Path) -> ReportId: return ReportId(current_tag=current.tag, baseline_tag=previous.tag) +def _unpublished_package_release_pair(repo_root: Path) -> ReportId: + """Pair an unpublished package version with its newest prior release.""" + current = _current_package_tag(repo_root) + stable_releases = _published_stable_releases(repo_root) + if not stable_releases: + msg = "expected at least one published stable semver release" + raise RuntimeError(msg) + latest_version = max(stable_releases, key=lambda release: _semver_sort_key(release.tag)) + if _semver_sort_key(current) <= _semver_sort_key(latest_version.tag): + msg = ( + f"current package tag {current} must be newer than latest published stable release " + f"{latest_version.tag}; update the package version before running --infer-release, " + "or pass explicit tags for a repair" + ) + raise ValueError(msg) + previous = _previous_release_from_list(stable_releases, current) + return ReportId(current_tag=current, baseline_tag=previous.tag) + + def _read_text(path: Path) -> str: return path.read_text(encoding="utf-8") @@ -1244,31 +1258,35 @@ def _render_report( artifacts: ArtifactPaths, config: GenerationConfig, ) -> None: - """Export report inputs and render Markdown from their validated reload.""" - _run_tool( - "uv", + """Render Markdown and export validated comparison artifacts.""" + command = [ + "run", + "--locked", + "--project", + str(config.repo_root), + "bench-compare", + config.baseline_tag, + "--repo-root", + str(worktree), + "--criterion-dir", + str(worktree / "target" / "criterion"), + "--suite", + config.suite, + "--scope", + config.scope, + ] + command.extend( [ - "run", - "--locked", - "--project", - str(config.repo_root), - "bench-compare", - config.baseline_tag, - "--repo-root", - str(worktree), - "--criterion-dir", - str(worktree / "target" / "criterion"), - "--suite", - config.suite, - "--scope", - config.scope, "--csv-output", str(artifacts.csv), "--provenance-output", str(artifacts.provenance), - "--output", - str(report), - ], + ] + ) + command.extend(["--output", str(report)]) + _run_tool( + "uv", + command, cwd=config.repo_root, ) @@ -1313,7 +1331,7 @@ def _run_benchmarks_and_render_report( def _default_artifact_paths(repo_root: Path) -> ArtifactPaths: - """Return the canonical retained release-report artifact pair.""" + """Return the canonical retained comparison-artifact pair.""" return ArtifactPaths( csv=repo_root / _DEFAULT_ARTIFACT_CSV, provenance=repo_root / _DEFAULT_ARTIFACT_PROVENANCE, @@ -1326,7 +1344,7 @@ def _generated_report_in_temp_worktree( config: GenerationConfig, published_artifacts: ArtifactPaths, ) -> Iterator[GeneratedReport]: - """Generate, publish, and expose a report before its worktree is removed.""" + """Generate and expose a report before its temporary worktree is removed.""" if config.baseline_source == "local": _require_recorded_measurement_cpu() with tempfile.TemporaryDirectory(prefix="la-stack-performance-") as tmp: @@ -1345,6 +1363,7 @@ def _generated_report_in_temp_worktree( label="temporary worktree", ): if config.apply_current_diff: + _progress("applying staged and unstaged tracked changes; untracked files are excluded") _apply_current_diff_to_worktree(repo_root=config.repo_root, worktree=worktree) if config.baseline_source == "github-assets": _prepare_github_release_assets( @@ -1390,14 +1409,14 @@ def _generated_report_in_temp_worktree( config=config, baseline_run=baseline_run, ) - bundle = load_bundle(temporary_artifacts) report_text = _read_text(report) + bundle = load_bundle(temporary_artifacts) with publish_bundle(published_artifacts, bundle): durable_text = render_release_artifacts(published_artifacts) if durable_text != report_text: - msg = "durable release-performance artifacts did not reproduce the generated Markdown" + msg = "durable performance-comparison artifacts did not reproduce the generated Markdown" raise ValueError(msg) - yield GeneratedReport(text=durable_text, bundle=bundle) + yield GeneratedReport(text=durable_text) def _current_archive_state( @@ -1482,6 +1501,9 @@ def _promotion_snapshots( def _promote_report_text(*, source_text: str, request: PromotionRequest) -> ReportId: """Archive the old report and atomically promote validated Markdown text.""" + if request.expected.current_tag == request.expected.baseline_tag: + msg = "cannot promote a same-version local performance comparison as release documentation" + raise ValueError(msg) source_text = _normalize_how_to_update(source_text) source_id = parse_report_id(source_text) if source_id != request.expected: @@ -1554,6 +1576,7 @@ def promote_report( def generate_and_promote_worktree_report( *, + output: Path, current: Path, archive_dir: Path, config: GenerationConfig, @@ -1562,6 +1585,9 @@ def generate_and_promote_worktree_report( """Generate a comparison in a temp worktree, then promote it.""" current_tag = normalize_tag(config.current_tag) baseline_tag = normalize_tag(config.baseline_tag) + if current_tag == baseline_tag: + msg = "release performance promotion requires distinct current and baseline releases" + raise ValueError(msg) config = GenerationConfig( repo_root=config.repo_root, current_tag=current_tag, @@ -1582,6 +1608,7 @@ def generate_and_promote_worktree_report( current=current, archive_dir=archive_dir, expected=expected, + output=output, reserved_paths=reserved_paths, ) _current_text, _current_id, archive_path = _current_archive_state( @@ -1652,7 +1679,7 @@ def generate_worktree_report( return report_id -def rerender_and_promote_artifacts( +def render_and_promote_artifacts( *, artifacts: ArtifactPaths, output: Path, @@ -1661,15 +1688,18 @@ def rerender_and_promote_artifacts( ) -> ReportId: """Reproduce and promote Markdown using only retained CSV/JSON inputs.""" bundle = load_bundle(artifacts) - report_text = _normalize_how_to_update(render_release_artifacts(artifacts)) expected = ReportId( current_tag=normalize_tag(bundle.context.release.current), baseline_tag=normalize_tag(bundle.context.release.baseline), ) + if expected.current_tag == expected.baseline_tag: + msg = "cannot promote a same-version local performance comparison as release documentation" + raise ValueError(msg) + report_text = _normalize_how_to_update(render_release_artifacts(artifacts)) observed = parse_report_id(report_text) if observed != expected: msg = ( - "rerendered benchmark report does not match retained release pair: " + "rendered benchmark report does not match retained release pair: " f"found {observed.current_tag} vs {observed.baseline_tag}, " f"expected {expected.current_tag} vs {expected.baseline_tag}" ) @@ -1720,13 +1750,12 @@ def resolve_archive_request(options: ArchiveRequestOptions) -> ResolvedArchiveRe if current_tag is not None or baseline_tag is not None: msg = "do not pass current_tag or baseline_tag with --infer-release" raise ValueError(msg) - inferred_current = _current_package_tag(repo_root) - inferred_baseline = _previous_published_release(repo_root, inferred_current).tag + inferred_pair = _unpublished_package_release_pair(repo_root) return ResolvedArchiveRequest( - current_tag=inferred_current, - baseline_tag=inferred_baseline, + current_tag=inferred_pair.current_tag, + baseline_tag=inferred_pair.baseline_tag, worktree_ref=worktree_ref, - tags_to_fetch=(inferred_baseline,), + tags_to_fetch=(inferred_pair.baseline_tag,), ) if current_vs_latest: @@ -1735,14 +1764,6 @@ def resolve_archive_request(options: ArchiveRequestOptions) -> ResolvedArchiveRe raise ValueError(msg) inferred_current = _current_package_tag(repo_root) latest = _latest_published_release(repo_root).tag - if inferred_current == latest: - msg = ( - f"current package tag and latest published release are both {latest}; " - "a release-performance report requires distinct identifiers. " - "Use a named local Criterion baseline for same-version worktree comparisons, " - "or rerun after the maintainer updates the package version." - ) - raise ValueError(msg) return ResolvedArchiveRequest( current_tag=inferred_current, baseline_tag=latest, @@ -1766,9 +1787,9 @@ def resolve_archive_request(options: ArchiveRequestOptions) -> ResolvedArchiveRe def build_parser() -> argparse.ArgumentParser: """Build the CLI argument parser.""" parser = argparse.ArgumentParser( - description="Promote a benchmark comparison into docs/PERFORMANCE.md and archive the previous report.", + description="Generate local benchmark comparisons or promote validated comparison artifacts into release documentation.", ) - parser.add_argument("current_tag", nargs="?", help="Release tag for the new report, e.g. v0.4.3") + parser.add_argument("current_tag", nargs="?", help="Current package-version label, e.g. v0.4.3") parser.add_argument("baseline_tag", nargs="?", help="Previous release tag used as the comparison baseline, e.g. v0.4.2") parser.add_argument( "--source", @@ -1783,17 +1804,17 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--output", default=_DEFAULT_SOURCE, - help=f"Generated report path for --output-only (default: {_DEFAULT_SOURCE})", + help=f"Rendered local comparison report path (default: {_DEFAULT_SOURCE})", ) parser.add_argument( "--artifact-csv", default=_DEFAULT_ARTIFACT_CSV, - help=f"Retained release-report CSV path (default: {_DEFAULT_ARTIFACT_CSV})", + help=f"Retained comparison CSV path (default: {_DEFAULT_ARTIFACT_CSV})", ) parser.add_argument( "--artifact-provenance", default=_DEFAULT_ARTIFACT_PROVENANCE, - help=f"Retained release-report JSON provenance path (default: {_DEFAULT_ARTIFACT_PROVENANCE})", + help=f"Retained comparison JSON provenance path (default: {_DEFAULT_ARTIFACT_PROVENANCE})", ) parser.add_argument( "--archive-dir", @@ -1803,7 +1824,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--generate-in-temp-worktree", action="store_true", - help="Generate the comparison in a temporary detached worktree before promoting it.", + help="Generate the comparison in a temporary detached worktree.", ) parser.add_argument( "--published-latest", @@ -1813,7 +1834,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument( "--infer-release", action="store_true", - help="Infer current_tag from Cargo.toml and baseline_tag from the previous stable published release.", + help="Require an unpublished Cargo.toml version and compare it with the previous stable published release.", ) parser.add_argument( "--current-vs-latest", @@ -1831,7 +1852,15 @@ def build_parser() -> argparse.ArgumentParser: help="Write the generated report to --output without promoting docs/PERFORMANCE.md.", ) parser.add_argument( - "--rerender", + "--local-report", + action="store_true", + help=( + "Generate a local report and retained comparison artifacts without promoting release documentation; " + "the current package version may match the baseline release." + ), + ) + parser.add_argument( + "--promote-artifacts", action="store_true", help="Render and promote from retained CSV/JSON artifacts without benchmarks or worktrees.", ) @@ -1847,13 +1876,13 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--suite", - default=_DEFAULT_SUITE, + default=None, choices=_SUPPORTED_SUITES, help=f"Benchmark suite for --generate-in-temp-worktree (default: {_DEFAULT_SUITE})", ) parser.add_argument( "--scope", - default=_DEFAULT_SCOPE, + default=None, choices=_SUPPORTED_SCOPES, help=f"Comparison scope for --generate-in-temp-worktree (default: {_DEFAULT_SCOPE})", ) @@ -1902,14 +1931,36 @@ def _generation_config(*, args: argparse.Namespace, request: ResolvedArchiveRequ current_tag=request.current_tag, baseline_tag=request.baseline_tag, worktree_ref=request.worktree_ref, - suite=cast("BenchmarkSuite", args.suite), - scope=cast("ComparisonScope", args.scope), + suite=cast("BenchmarkSuite", args.suite or _DEFAULT_SUITE), + scope=cast("ComparisonScope", args.scope or _DEFAULT_SCOPE), apply_current_diff=not args.no_apply_current_diff and not args.github_assets, baseline_source="github-assets" if args.github_assets else "local", ) +def _validate_generation_request( + *, + args: argparse.Namespace, + request: ResolvedArchiveRequest, + repo_root: Path, +) -> None: + """Reject invalid release identities before fetching tags or benchmarking.""" + if not args.local_report and request.current_tag == request.baseline_tag: + msg = "release performance artifacts require distinct current and baseline releases" + raise ValueError(msg) + + if args.generate_in_temp_worktree and not args.github_assets and args.current_tag is not None and request.worktree_ref == "HEAD": + package_tag = _current_package_tag(repo_root) + if request.current_tag != package_tag: + msg = ( + f"explicit current tag {request.current_tag} does not match the current Cargo package tag " + f"{package_tag}; update the package version or pass the matching current tag" + ) + raise ValueError(msg) + + def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, request: ResolvedArchiveRequest, repo_root: Path) -> ArchiveResult: + _validate_generation_request(args=args, request=request, repo_root=repo_root) if args.generate_in_temp_worktree: _fetch_required_tags(request=request, repo_root=repo_root, include_current=args.github_assets) config = _generation_config(args=args, request=request, repo_root=repo_root) @@ -1924,6 +1975,7 @@ def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, reque ) return ArchiveResult( report_id=generate_and_promote_worktree_report( + output=paths.output, current=paths.current, archive_dir=paths.archive_dir, config=config, @@ -1953,7 +2005,7 @@ def _run_archive_request(*, args: argparse.Namespace, paths: ArchivePaths, reque def _validate_cli_preflight(args: argparse.Namespace) -> None: """Reject locally invalid options before release discovery or tag fetching.""" - if args.rerender and any( + if args.promote_artifacts and any( ( args.current_tag, args.baseline_tag, @@ -1963,13 +2015,22 @@ def _validate_cli_preflight(args: argparse.Namespace) -> None: args.github_assets, args.generate_in_temp_worktree, args.output_only, + args.local_report, + args.suite, + args.scope, ) ): - msg = "--rerender cannot be combined with release selection, generation, GitHub-asset, or output-only options" + msg = "--promote-artifacts cannot be combined with release selection, generation, GitHub-asset, output-only, suite, or scope options" raise ValueError(msg) if args.output_only and not args.generate_in_temp_worktree: msg = "--output-only requires --generate-in-temp-worktree" raise ValueError(msg) + if args.local_report and not args.output_only: + msg = "--local-report requires --output-only" + raise ValueError(msg) + if args.local_report and args.github_assets: + msg = "--local-report cannot be combined with --github-assets" + raise ValueError(msg) if args.github_assets and not args.generate_in_temp_worktree: msg = "--github-assets requires --generate-in-temp-worktree" raise ValueError(msg) @@ -1983,15 +2044,15 @@ def main(argv: list[str] | None = None) -> int: try: _validate_cli_preflight(args) paths = _resolve_cli_paths(root, args) - if args.rerender: + if args.promote_artifacts: result = ArchiveResult( - report_id=rerender_and_promote_artifacts( + report_id=render_and_promote_artifacts( artifacts=paths.artifacts, output=paths.output, current=paths.current, archive_dir=paths.archive_dir, ), - action="rerender", + action="promote-artifacts", ) else: request = resolve_archive_request( @@ -2022,9 +2083,9 @@ def main(argv: list[str] | None = None) -> int: if result.action == "output": print(f"Generated benchmark report in a temporary worktree and wrote it to {paths.output}") elif result.action == "promote-generated": - print(f"Generated benchmark report in a temporary worktree and promoted it to {paths.current}") - elif result.action == "rerender": - print(f"Re-rendered {paths.output} from {paths.artifacts.csv} and promoted it to {paths.current}") + print(f"Generated benchmark report in a temporary worktree, wrote it to {paths.output}, and promoted it to {paths.current}") + elif result.action == "promote-artifacts": + print(f"Rendered {paths.output} from {paths.artifacts.csv} and promoted it to {paths.current}") else: print(f"Promoted {paths.source} to {paths.current}") print(f"Current performance report: {result.report_id.current_tag} vs {result.report_id.baseline_tag}") diff --git a/scripts/bench_compare.py b/scripts/bench_compare.py index b5329d7..b8a88b2 100644 --- a/scripts/bench_compare.py +++ b/scripts/bench_compare.py @@ -200,8 +200,8 @@ # Release PR: update docs/PERFORMANCE.md and archive the previous report just performance-release -# Re-render and promote from retained CSV/JSON inputs (no benchmarks) -just performance-rerender +# Build release docs from retained CSV/JSON inputs (no benchmarks) +just performance-doc # GitHub Actions release assets just performance-github-assets @@ -210,9 +210,13 @@ just performance-release ``` -`just performance-local` writes `target/bench-reports/performance.md`. +`just performance-local` writes `performance.md` plus retained `performance.csv` and +`performance.provenance.json` comparison inputs under `target/bench-reports/` without promoting documentation. +It applies staged and unstaged tracked changes; untracked files are excluded. `just performance-github-assets` writes `target/bench-reports/github-assets-performance.md`. -`just performance-release` also retains `performance.csv` and `performance.provenance.json` beside the local report. +`just performance-release` performs the same measurement and retention work, then promotes distinct-release documentation. +`just performance-doc` consumes the retained pair from either workflow without benchmarking and promotes it when the package versions differ. +For a distinct pair, `performance-local` followed by `performance-doc` is equivalent to the atomic `performance-release` workflow. Older curated release-to-release reports are archived in `docs/archive/performance/`. @@ -1377,59 +1381,52 @@ def _group_heading_for_suite(suite: str, group: str) -> str: def _snapshot_tables(results: list[BenchResult], stat: str) -> str: - """Generate per-dimension markdown tables for a single set of results.""" + """Generate one Markdown table per suite for a single set of results.""" stat_label = stat.capitalize() sections: list[str] = [] for suite, suite_items in _group_by_suite(results).items(): sections.append(f"## {_suite_heading(suite)}") + lines = [ + f"| Case | Benchmark | {stat_label} | Criterion CI |", + "|:-----|:----------|-------:|-------:|", + ] for group, items in _group_by_group(suite_items).items(): - lines = [ - f"### {_group_heading_for_suite(suite, group)}", - "", - f"| Benchmark | {stat_label} | Criterion CI |", - "|-----------|-------:|-------:|", - ] + case = _group_heading_for_suite(suite, group) for r in items: ci_range = _format_confidence_interval(r.estimate) - lines.append(f"| {r.bench} | {_format_time(r.point_ns)} | {ci_range} |") - sections.append("\n".join(lines)) + lines.append(f"| {case} | {r.bench} | {_format_time(r.point_ns)} | {ci_range} |") + sections.append("\n".join(lines)) return "\n\n".join(sections) def _comparison_tables(comparisons: list[Comparison], baseline_name: str) -> str: - """Generate per-dimension markdown tables comparing baseline vs current.""" + """Generate one Markdown comparison table per suite.""" sections: list[str] = [] for suite, suite_items in _group_by_suite(comparisons).items(): sections.append(f"## {_suite_heading(suite)}") - for group, items in _group_by_group(suite_items).items(): - has_peer_context = any(item.baseline_nalgebra_ns is not None or item.baseline_faer_ns is not None for item in items) + has_peer_context = any(item.baseline_nalgebra_ns is not None or item.baseline_faer_ns is not None for item in suite_items) + if has_peer_context: + header = ( + f"| Case | Benchmark | {baseline_name} (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | " + f"Point-estimate ratio | {baseline_name} nalgebra | {baseline_name} faer |" + ) lines = [ - f"### {_group_heading_for_suite(suite, group)}", - "", + header, + "|:-----|:----------|-------:|-------:|-------:|:-----------|--------:|-------:|-------:|", ] - if has_peer_context: - header = ( - f"| Benchmark | {baseline_name} (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | Point-estimate ratio | " - f"{baseline_name} nalgebra | {baseline_name} faer |" - ) - lines.extend( - [ - header, - "|-----------|-------:|-------:|-------:|:-----------|--------:|-------:|-------:|", - ] - ) - else: - lines.extend( - [ - f"| Benchmark | {baseline_name} (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | Point-estimate ratio |", - "|-----------|-------:|-------:|-------:|:-----------|--------:|", - ] - ) + else: + lines = [ + f"| Case | Benchmark | {baseline_name} (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | Point-estimate ratio |", + "|:-----|:----------|-------:|-------:|-------:|:-----------|--------:|", + ] + for group, items in _group_by_group(suite_items).items(): + case = _group_heading_for_suite(suite, group) for c in items: cells = [ + case, _comparison_bench_label(c), _format_estimate(c.baseline), _format_estimate(c.current), @@ -1445,7 +1442,7 @@ def _comparison_tables(comparisons: list[Comparison], baseline_name: str) -> str ] ) lines.append(f"| {' | '.join(cells)} |") - sections.append("\n".join(lines)) + sections.append("\n".join(lines)) return "\n\n".join(sections) @@ -2023,11 +2020,11 @@ def _parse_args(argv: list[str]) -> argparse.Namespace: ) parser.add_argument( "--csv-output", - help="Write schema-versioned release-report CSV input; requires an adjacent, distinct --provenance-output.", + help="Write schema-versioned comparison CSV input; requires an adjacent, distinct --provenance-output.", ) parser.add_argument( "--provenance-output", - help="Write an adjacent release-report JSON provenance sidecar; requires a distinct --csv-output.", + help="Write an adjacent comparison JSON provenance sidecar; requires a distinct --csv-output.", ) return parser.parse_args(argv) diff --git a/scripts/check_docs_version_sync.py b/scripts/check_docs_version_sync.py index a292378..9a59b0c 100644 --- a/scripts/check_docs_version_sync.py +++ b/scripts/check_docs_version_sync.py @@ -332,7 +332,7 @@ def _readme_tag_references(path: Path) -> list[VersionReference]: _BENCHMARK_CURRENT_TAG_RE = re.compile( - r"just performance-(?:github-assets|local-vs-linalg|release)\s+v" + r"just performance-(?:github-assets|local-non-exact|release)\s+v" r"(?P[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)(?=\s|`)" ) diff --git a/scripts/performance_artifacts.py b/scripts/performance_artifacts.py index dd1887e..6772a3b 100644 --- a/scripts/performance_artifacts.py +++ b/scripts/performance_artifacts.py @@ -1,4 +1,4 @@ -"""Schema-versioned release-performance CSV and provenance artifacts.""" +"""Schema-versioned performance-comparison CSV and provenance artifacts.""" import csv import hashlib @@ -136,19 +136,16 @@ def __post_init__(self) -> None: # noqa: C901 @dataclass(frozen=True, slots=True) class ReleasePair: - """The current and baseline release identifiers represented by a report.""" + """The current and baseline package identifiers represented by a report.""" current: str baseline: str def __post_init__(self) -> None: - """Require two distinct, non-empty release identifiers.""" + """Require two non-empty identifiers; local comparisons may match.""" if not self.current.strip() or not self.baseline.strip(): msg = "current and baseline releases must not be empty" raise ValueError(msg) - if self.current == self.baseline: - msg = "current and baseline releases must differ" - raise ValueError(msg) @dataclass(frozen=True, slots=True) @@ -187,7 +184,7 @@ class ArtifactContext: def __post_init__(self) -> None: """Bind report settings and benchmark provenance to the release pair.""" if self.statistic != "median": - msg = f"unsupported release-report statistic: {self.statistic!r}" + msg = f"unsupported performance-comparison statistic: {self.statistic!r}" raise ValueError(msg) if self.suite not in SUITES: msg = f"unsupported suite: {self.suite!r}" diff --git a/scripts/tests/test_archive_performance.py b/scripts/tests/test_archive_performance.py index 6416f8a..7dc9551 100644 --- a/scripts/tests/test_archive_performance.py +++ b/scripts/tests/test_archive_performance.py @@ -73,6 +73,11 @@ def _normalized_report(version: str, baseline: str) -> str: def _write_fake_rendered_artifacts(args: Sequence[str], *, version: str, baseline: str) -> None: + output = Path(args[args.index("--output") + 1]) + if "--csv-output" not in args: + output.write_text(_report(version, baseline), encoding="utf-8") + return + criterion_dir = Path(args[args.index("--criterion-dir") + 1]) suite = args[args.index("--suite") + 1] scope = args[args.index("--scope") + 1] @@ -116,7 +121,6 @@ def _write_fake_rendered_artifacts(args: Sequence[str], *, version: str, baselin ), ) write_bundle(paths, bundle) - output = Path(args[args.index("--output") + 1]) output.write_text(archive_performance.render_release_artifacts(paths), encoding="utf-8") @@ -670,6 +674,41 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert request.tags_to_fetch == ("v0.4.2",) +@pytest.mark.parametrize("package_version", ["0.4.4", "0.4.3"]) +def test_resolve_archive_request_infer_release_requires_unpublished_package_version( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + package_version: str, +) -> None: + (tmp_path / "Cargo.toml").write_text(f'[package]\nversion = "{package_version}"\n', encoding="utf-8") + + def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: + assert command == "gh" + assert args[:2] == ["release", "list"] + assert cwd == tmp_path + return _result( + "[" + '{"tagName":"v0.4.3","isDraft":false,"isPrerelease":false,"publishedAt":"2026-02-01T00:00:00Z"},' + '{"tagName":"v0.4.4","isDraft":false,"isPrerelease":false,"publishedAt":"2026-03-01T00:00:00Z"}' + "]" + ) + + monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) + + with pytest.raises(ValueError, match=r"must be newer than latest published stable release v0\.4\.4"): + archive_performance.resolve_archive_request( + archive_performance.ArchiveRequestOptions( + current_tag=None, + baseline_tag=None, + published_latest=False, + infer_release=True, + current_vs_latest=False, + worktree_ref="HEAD", + repo_root=tmp_path, + ) + ) + + def test_resolve_archive_request_current_vs_latest_uses_package_version_and_latest_release(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.3"\n', encoding="utf-8") @@ -704,7 +743,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert request.tags_to_fetch == ("v0.4.2",) -def test_resolve_archive_request_current_vs_latest_rejects_equal_release_tags_before_benchmarking( +def test_resolve_archive_request_current_vs_latest_allows_equal_release_tags_for_local_reports( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -718,18 +757,22 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) - with pytest.raises(ValueError, match=r"both v0\.4\.3"): - archive_performance.resolve_archive_request( - archive_performance.ArchiveRequestOptions( - current_tag=None, - baseline_tag=None, - published_latest=False, - infer_release=False, - current_vs_latest=True, - worktree_ref="HEAD", - repo_root=tmp_path, - ) + request = archive_performance.resolve_archive_request( + archive_performance.ArchiveRequestOptions( + current_tag=None, + baseline_tag=None, + published_latest=False, + infer_release=False, + current_vs_latest=True, + worktree_ref="HEAD", + repo_root=tmp_path, ) + ) + + assert request.current_tag == "v0.4.3" + assert request.baseline_tag == "v0.4.3" + assert request.worktree_ref == "HEAD" + assert request.tags_to_fetch == ("v0.4.3",) def test_benchmark_env_uses_current_repo_toolchain(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: @@ -1126,6 +1169,118 @@ def fail_discovery(_options: object) -> None: assert "--output-only requires --generate-in-temp-worktree" in capsys.readouterr().err +@pytest.mark.parametrize( + ("argv", "message"), + [ + (["--local-report"], "--local-report requires --output-only"), + ( + ["--local-report", "--output-only", "--generate-in-temp-worktree", "--github-assets"], + "--local-report cannot be combined with --github-assets", + ), + ], +) +def test_main_rejects_invalid_local_report_options_before_release_discovery( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + argv: list[str], + message: str, +) -> None: + monkeypatch.chdir(tmp_path) + + def fail_discovery(_options: object) -> None: + msg = "release discovery must not run" + raise AssertionError(msg) + + monkeypatch.setattr(archive_performance, "resolve_archive_request", fail_discovery) + + assert main(argv) == 1 + assert message in capsys.readouterr().err + + +@pytest.mark.parametrize("selection", [["--suite", "exact"], ["--scope", "all-benches"]]) +def test_main_rejects_artifact_promotion_benchmark_selectors_before_loading_artifacts( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, + selection: list[str], +) -> None: + monkeypatch.chdir(tmp_path) + + def fail_load(_paths: object) -> None: + msg = "artifact loading must not start" + raise AssertionError(msg) + + monkeypatch.setattr(archive_performance, "load_bundle", fail_load) + + assert main(["--promote-artifacts", *selection]) == 1 + assert "--promote-artifacts cannot be combined" in capsys.readouterr().err + + +def test_archive_performance_help_describes_local_and_release_workflows() -> None: + help_text = archive_performance.build_parser().format_help() + + assert "Generate local benchmark comparisons or promote validated comparison artifacts" in help_text + assert "Rendered local comparison report path" in help_text + assert "Retained comparison CSV path" in help_text + assert "before promoting it" not in help_text + + +def test_main_rejects_explicit_current_tag_mismatch_before_external_work( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.4"\n', encoding="utf-8") + monkeypatch.chdir(tmp_path) + + def fail_external_work(*_args: object, **_kwargs: object) -> None: + msg = "external work must not start" + raise AssertionError(msg) + + monkeypatch.setattr(archive_performance, "_fetch_required_tags", fail_external_work) + + rc = main( + [ + "v0.4.5", + "v0.4.4", + "--generate-in-temp-worktree", + "--output-only", + "--local-report", + ] + ) + + assert rc == 1 + assert "explicit current tag v0.4.5 does not match the current Cargo package tag v0.4.4" in capsys.readouterr().err + + +def test_main_rejects_equal_release_artifact_tags_before_external_work( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.4"\n', encoding="utf-8") + monkeypatch.chdir(tmp_path) + + def fail_external_work(*_args: object, **_kwargs: object) -> None: + msg = "external work must not start" + raise AssertionError(msg) + + monkeypatch.setattr(archive_performance, "_fetch_required_tags", fail_external_work) + + rc = main( + [ + "v0.4.4", + "v0.4.4", + "--generate-in-temp-worktree", + "--output-only", + ] + ) + + assert rc == 1 + assert "release performance artifacts require distinct current and baseline releases" in capsys.readouterr().err + + def test_local_release_generation_rejects_unavailable_cpu_before_external_work( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -1155,7 +1310,7 @@ def fail_external_work(*_args: object, **_kwargs: object) -> None: @pytest.mark.parametrize("alias", ["artifact-csv", "current-report"]) -def test_main_rerender_rejects_output_alias_without_mutation( +def test_main_promote_artifacts_rejects_output_alias_without_mutation( tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, @@ -1188,7 +1343,7 @@ def test_main_rerender_rejects_output_alias_without_mutation( rc = main( [ - "--rerender", + "--promote-artifacts", "--artifact-csv", str(artifacts.csv), "--artifact-provenance", @@ -1278,8 +1433,10 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** captured = capsys.readouterr() assert rc == 0 assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) + output = tmp_path / "target" / "bench-reports" / "performance.md" + assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") assert "Generated benchmark report in a temporary worktree" in captured.out - assert "target/bench-reports/performance.md" not in captured.out + assert str(output) in captured.out assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.3" for kind, args, _ in calls) assert any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) assert any(kind == "uv" and "--suite" in args and args[args.index("--suite") + 1] == "exact" for kind, args, _ in calls) @@ -1288,23 +1445,24 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert not any(kind == "git-stdin" for kind, _, _ in calls) external_call_count = len(calls) - rerender_rc = main( + promote_rc = main( [ - "--rerender", + "--promote-artifacts", "--current", str(current), "--archive-dir", str(archive_dir), ] ) - rerendered = capsys.readouterr() - assert rerender_rc == 0 - assert "Re-rendered" in rerendered.out + promoted = capsys.readouterr() + assert promote_rc == 0 + assert "Rendered" in promoted.out assert len(calls) == external_call_count - assert (tmp_path / "target" / "bench-reports" / "performance.md").read_text(encoding="utf-8") == current.read_text(encoding="utf-8") + assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") def test_temp_worktree_is_removed_when_benchmark_command_fails(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: + (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.3"\n', encoding="utf-8") current = tmp_path / "docs" / "PERFORMANCE.md" archive_dir = tmp_path / "docs" / "archive" / "performance" calls: list[RunnerCall] = [] @@ -1426,6 +1584,7 @@ def test_generate_report_generates_release_baseline_locally( # noqa: PLR0915 capsys: pytest.CaptureFixture[str], ) -> None: monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) + (tmp_path / "Cargo.toml").write_text('[package]\nversion = "0.4.3"\n', encoding="utf-8") (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") current = tmp_path / "docs" / "PERFORMANCE.md" archive_dir = tmp_path / "docs" / "archive" / "performance" @@ -1532,7 +1691,10 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert sum(1 for kind, args, _ in calls if kind == "git" and args[:3] == ("worktree", "remove", "--force")) == 2 -def test_generate_report_vs_linalg_suite_uses_copied_current_baseline_recipe(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_generate_local_non_exact_report_retains_same_version_comparison_artifacts( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") output = tmp_path / "target" / "bench-reports" / "performance.md" @@ -1555,7 +1717,7 @@ def fake_run_git_with_input(args: Sequence[str], input_data: str, cwd: Path | No def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, **kwargs: Any) -> SimpleNamespace: calls.append((command, tuple(args), cwd)) - if command == "just" and args == ["bench-save-baseline", "v0.4.2", "vs_linalg"]: + if command == "just" and args == ["bench-save-baseline", "v0.4.3", "vs_linalg"]: assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" assert cwd is not None assert "bench-latest" in (cwd / "justfile").read_text(encoding="utf-8") @@ -1565,7 +1727,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** if command == "just" and args == ["bench-vs-linalg-la-stack"]: assert kwargs["env"]["RUSTUP_TOOLCHAIN"] == "1.97.0" if command == "uv": - _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.2") + _write_fake_rendered_artifacts(args, version="0.4.3", baseline="v0.4.3") return _result() monkeypatch.setattr(archive_performance, "run_git_command", fake_run_git) @@ -1577,23 +1739,39 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** config=GenerationConfig( repo_root=tmp_path, current_tag="v0.4.3", - baseline_tag="v0.4.2", + baseline_tag="v0.4.3", worktree_ref="HEAD", suite="vs_linalg", apply_current_diff=False, ), ) - assert report_id.archive_name == "v0.4.3-vs-v0.4.2.md" + assert report_id.archive_name == "v0.4.3-vs-v0.4.3.md" assert output.read_text(encoding="utf-8") == _retained_report(tmp_path) - assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.2", "vs_linalg") for kind, args, _ in calls) + assert output.with_suffix(".csv").is_file() + assert output.with_suffix(".provenance.json").is_file() + assert any(kind == "just" and args == ("bench-save-baseline", "v0.4.3", "vs_linalg") for kind, args, _ in calls) assert any(kind == "just" and args == ("bench-vs-linalg-la-stack",) for kind, args, _ in calls) + assert any(kind == "uv" and "--csv-output" in args and "--provenance-output" in args for kind, args, _ in calls) assert not any(kind == "cargo" for kind, _, _ in calls) assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) assert not any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) + current = tmp_path / "docs" / "PERFORMANCE.md" + with pytest.raises(ValueError, match="cannot promote a same-version local performance comparison"): + archive_performance.render_and_promote_artifacts( + artifacts=ArtifactPaths( + csv=output.with_suffix(".csv"), + provenance=output.with_suffix(".provenance.json"), + ), + output=output, + current=current, + archive_dir=tmp_path / "docs" / "archive" / "performance", + ) + assert not current.exists() + -def test_generate_report_vs_linalg_suite_uses_suite_aware_baseline_recipe(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_generate_local_report_artifacts_can_be_rendered_and_promoted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: output = tmp_path / "target" / "bench-reports" / "performance.md" calls: list[RunnerCall] = [] @@ -1642,6 +1820,19 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert not any(kind == "cargo" for kind, _, _ in calls) assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) + current = tmp_path / "docs" / "PERFORMANCE.md" + promoted = archive_performance.render_and_promote_artifacts( + artifacts=ArtifactPaths( + csv=output.with_suffix(".csv"), + provenance=output.with_suffix(".provenance.json"), + ), + output=output, + current=current, + archive_dir=tmp_path / "docs" / "archive" / "performance", + ) + assert promoted == report_id + assert current.read_text(encoding="utf-8") == output.read_text(encoding="utf-8") + def test_main_generates_latest_published_report_from_github_releases(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: current = tmp_path / "docs" / "PERFORMANCE.md" @@ -1905,7 +2096,7 @@ def fail_index(_archive_dir: Path) -> None: assert not (archive_dir / "README.md").exists() -def test_failed_rerender_output_write_rolls_back_report_archive_and_index( +def test_failed_artifact_promotion_output_write_rolls_back_report_archive_and_index( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -1926,13 +2117,13 @@ def test_failed_rerender_output_write_rolls_back_report_archive_and_index( def fail_output_write(path: Path, text: str) -> None: if path == output: - msg = "simulated rerender output failure" + msg = "simulated artifact promotion output failure" raise OSError(msg) real_write(path, text) monkeypatch.setattr(archive_performance, "_write_text", fail_output_write) - with pytest.raises(OSError, match="simulated rerender output failure"): + with pytest.raises(OSError, match="simulated artifact promotion output failure"): archive_performance._promote_report_text( source_text=_report("0.4.3", "v0.4.2"), request=archive_performance.PromotionRequest( @@ -1961,6 +2152,7 @@ def test_generate_and_promote_rejects_artifact_alias_before_worktree(tmp_path: P with pytest.raises(ValueError, match="must use distinct paths"): generate_and_promote_worktree_report( + output=tmp_path / "target" / "bench-reports" / "performance.md", current=current, archive_dir=archive_dir, config=GenerationConfig( @@ -1998,10 +2190,15 @@ def test_generate_output_only_rejects_artifact_alias_before_worktree(tmp_path: P assert not output.exists() -def test_generate_and_promote_uses_temp_worktree_and_current_diff(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_generate_and_promote_uses_temp_worktree_and_current_diff( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: monkeypatch.delenv("RUSTUP_TOOLCHAIN", raising=False) (tmp_path / "rust-toolchain.toml").write_text('[toolchain]\nchannel = "1.97.0"\n', encoding="utf-8") current = tmp_path / "docs" / "PERFORMANCE.md" + output = tmp_path / "target" / "bench-reports" / "performance.md" archive_dir = tmp_path / "docs" / "archive" / "performance" current.parent.mkdir(parents=True) current.write_text(_report("0.4.2", "v0.4.1"), encoding="utf-8") @@ -2042,6 +2239,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) report_id = generate_and_promote_worktree_report( + output=output, current=current, archive_dir=archive_dir, config=GenerationConfig( @@ -2055,15 +2253,18 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert report_id.archive_name == "v0.4.3-vs-v0.4.2.md" assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) + assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") assert (archive_dir / "v0.4.2-vs-v0.4.1.md").exists() assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "HEAD" for kind, args, _ in calls) assert any(kind == "git-stdin" and args == ("apply", "--binary") for kind, args, _ in calls) assert any(kind == "just" and args == ("bench-latest",) for kind, args, _ in calls) assert any(kind == "git" and args[:3] == ("worktree", "remove", "--force") for kind, args, _ in calls) + assert "applying staged and unstaged tracked changes; untracked files are excluded" in capsys.readouterr().err def test_generate_and_promote_legacy_published_tag_uses_legacy_commands(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: current = tmp_path / "docs" / "PERFORMANCE.md" + output = tmp_path / "target" / "bench-reports" / "performance.md" archive_dir = tmp_path / "docs" / "archive" / "performance" calls: list[RunnerCall] = [] @@ -2095,6 +2296,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** monkeypatch.setattr(archive_performance, "run_safe_command", fake_run_safe) report_id = generate_and_promote_worktree_report( + output=output, current=current, archive_dir=archive_dir, config=GenerationConfig( @@ -2108,6 +2310,7 @@ def fake_run_safe(command: str, args: Sequence[str], cwd: Path | None = None, ** assert report_id.archive_name == "v0.4.2-vs-v0.4.1.md" assert current.read_text(encoding="utf-8") == _retained_report(tmp_path) + assert output.read_text(encoding="utf-8") == current.read_text(encoding="utf-8") assert any(kind == "git" and args[:3] == ("worktree", "add", "--detach") and args[4] == "v0.4.2" for kind, args, _ in calls) assert any(kind == "cargo" and args == ("bench", "--locked", "--features", "bench,exact") for kind, args, _ in calls) assert not any(kind == "just" and args == ("bench-exact",) for kind, args, _ in calls) diff --git a/scripts/tests/test_bench_compare.py b/scripts/tests/test_bench_compare.py index 7bf84be..761a44e 100644 --- a/scripts/tests/test_bench_compare.py +++ b/scripts/tests/test_bench_compare.py @@ -702,26 +702,26 @@ def test_collect_vs_linalg_all_benches_includes_latest_peer_rows(tmp_path: Path) # --------------------------------------------------------------------------- -def test_snapshot_tables_per_dimension(tmp_path: Path) -> None: +def test_snapshot_uses_one_table_per_suite_with_case_column(tmp_path: Path) -> None: _build_criterion_tree(tmp_path) results = bench_compare._collect_results(tmp_path, "new", "median") tables = bench_compare._snapshot_tables(results, "median") - assert "### D=2" in tables - assert "### D=3" in tables - assert "### Random corpus D=3" in tables - assert "### Near-singular 3x3" in tables - assert "| Benchmark | Median | Criterion CI |" in tables + assert tables.count("| Case | Benchmark | Median | Criterion CI |") == 1 + assert "| D=2 |" in tables + assert "| D=3 |" in tables + assert "| Random corpus D=3 |" in tables + assert "| Near-singular 3x3 |" in tables def test_snapshot_tables_uses_stat_label(tmp_path: Path) -> None: _build_criterion_tree(tmp_path, stat="mean") results = bench_compare._collect_results(tmp_path, "new", "mean") tables = bench_compare._snapshot_tables(results, "mean") - assert "| Benchmark | Mean | Criterion CI |" in tables + assert "| Case | Benchmark | Mean | Criterion CI |" in tables assert "Median" not in tables -def test_comparison_tables_per_dimension(tmp_path: Path) -> None: +def test_comparison_uses_one_table_per_suite_with_case_column(tmp_path: Path) -> None: _build_criterion_tree(tmp_path) for d, det, det_exact in [(2, 2.0, 8000.0), (3, 10.0, 42000.0)]: group = tmp_path / f"exact_d{d}" @@ -732,9 +732,9 @@ def test_comparison_tables_per_dimension(tmp_path: Path) -> None: comparisons = bench_compare._collect_comparisons(tmp_path, "v0.3.0", "median").comparisons tables = bench_compare._comparison_tables(comparisons, "v0.3.0") - assert "### D=2" in tables - assert "### D=3" in tables - assert "| Benchmark | v0.3.0 (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | Point-estimate ratio |" in tables + assert tables.count("| Case | Benchmark | v0.3.0 (point + CI)") == 1 + assert "| D=2 |" in tables + assert "| D=3 |" in tables assert "det_exact_rounded_f64 (vs det_exact_f64)" in tables assert "solve_exact_f64_result (vs solve_exact_f64)" in tables @@ -745,16 +745,16 @@ def test_comparison_tables_include_vs_linalg_peer_context(tmp_path: Path) -> Non tables = bench_compare._comparison_tables(comparisons, "last") assert ( - "| Benchmark | last (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | " + "| Case | Benchmark | last (point + CI) | Latest (point + CI) | Point-estimate change | CI relation | " "Point-estimate ratio | last nalgebra | last faer |" in tables ) assert ( - "| la_stack_lu_solve | 20.0 ns [18.0 ns, 22.0 ns] | 10.0 ns [9.0 ns, 11.0 ns] | -50.0% | " + "| D=2 | la_stack_lu_solve | 20.0 ns [18.0 ns, 22.0 ns] | 10.0 ns [9.0 ns, 11.0 ns] | -50.0% | " "faster point estimate; marginal CIs separated | 2.00x | " "30.0 ns [27.0 ns, 33.0 ns] | 40.0 ns [36.0 ns, 44.0 ns] |" in tables ) assert ( - "| la_stack_ldlt_solve | 24.0 ns [21.6 ns, 26.4 ns] | 12.0 ns [10.8 ns, 13.2 ns] | -50.0% | " + "| D=2 | la_stack_ldlt_solve | 24.0 ns [21.6 ns, 26.4 ns] | 12.0 ns [10.8 ns, 13.2 ns] | -50.0% | " "faster point estimate; marginal CIs separated | 2.00x | " "36.0 ns [32.4 ns, 39.6 ns] | 48.0 ns [43.2 ns, 52.8 ns] |" in tables ) @@ -993,13 +993,18 @@ def test_main_snapshot_writes_output(tmp_path: Path) -> None: assert output.exists() text = output.read_text(encoding="utf-8") - assert "### D=2" in text - assert "### Random corpus D=3" in text - assert "### Near-singular 3x3" in text + assert "| D=2 |" in text + assert "| Random corpus D=3 |" in text + assert "| Near-singular 3x3 |" in text assert "just performance-local" in text assert "just performance-release" in text + assert "just performance-doc" in text assert "just performance-github-assets" in text assert "just performance-release " in text + assert "untracked files are excluded" in text + assert "performance-local` writes `performance.md` plus retained `performance.csv`" in text + assert "performance-doc` consumes the retained pair from either workflow" in text + assert "performance-local` followed by `performance-doc` is equivalent" in text assert "Older curated release-to-release reports are archived in `docs/archive/performance/`." in text assert "git checkout" not in text assert text.endswith(bench_compare.HOW_TO_UPDATE_SECTION) diff --git a/scripts/tests/test_check_docs_version_sync.py b/scripts/tests/test_check_docs_version_sync.py index efb9137..3629ef9 100644 --- a/scripts/tests/test_check_docs_version_sync.py +++ b/scripts/tests/test_check_docs_version_sync.py @@ -130,7 +130,7 @@ def test_readme_tag_references_reject_longer_non_semver_tags(tmp_path: Path, tag assert check_docs_version_sync._readme_tag_references(readme) == [] -@pytest.mark.parametrize("recipe", ["performance-github-assets", "performance-local-vs-linalg", "performance-release"]) +@pytest.mark.parametrize("recipe", ["performance-github-assets", "performance-local-non-exact", "performance-release"]) def test_find_version_mismatches_reports_stale_benchmark_current_tags(tmp_path: Path, recipe: str) -> None: _write_project(tmp_path) docs = tmp_path / "docs" @@ -138,7 +138,7 @@ def test_find_version_mismatches_reports_stale_benchmark_current_tags(tmp_path: workflows = docs / "workflows.md" workflows.write_text( f"| Release workflow | `just {recipe} v1.2.2 v1.2.1` |\n" - "```bash\njust performance-local-vs-linalg v1.2.3 v1.2.2\n```\n" + "```bash\njust performance-local-non-exact v1.2.3 v1.2.2\n```\n" "Historical v1.2.1 behavior remains documented.\n", encoding="utf-8", ) diff --git a/scripts/tests/test_performance_artifacts.py b/scripts/tests/test_performance_artifacts.py index 158c108..3821130 100644 --- a/scripts/tests/test_performance_artifacts.py +++ b/scripts/tests/test_performance_artifacts.py @@ -170,6 +170,20 @@ def test_artifact_round_trip_preserves_comparable_and_one_sided_rows() -> None: assert provenance_payload.endswith(b"\n") +def test_artifact_round_trip_allows_same_version_local_comparison() -> None: + original = _bundle() + bundle = PerformanceBundle( + context=_context(current="v0.4.4", baseline="v0.4.4"), + rows=original.rows, + ) + + csv_payload, provenance_payload = serialize_bundle(bundle) + parsed = load_bundle_bytes(csv_payload, provenance_payload, source="same-version local fixture") + + assert parsed.context.release == ReleasePair(current="v0.4.4", baseline="v0.4.4") + assert parsed.rows == bundle.sorted_rows + + def test_artifact_serialization_is_independent_of_input_row_order() -> None: bundle = _bundle() reordered = PerformanceBundle(context=bundle.context, rows=tuple(reversed(bundle.rows)))