diff --git a/cortex/tests/reference_images/README.md b/cortex/tests/reference_images/README.md new file mode 100644 index 00000000..075b308f --- /dev/null +++ b/cortex/tests/reference_images/README.md @@ -0,0 +1,73 @@ +# Reference images + +Stored renders that `cortex/tests/test_webgl_headless.py` asserts against, so a +change in rendering output fails a test instead of needing to be spotted by eye. + +## `alpha_dataviews/` + +Twelve images: each of the six public dataview classes (`Volume`, `Vertex`, +`Volume2D`, `Vertex2D`, `VolumeRGB`, `VertexRGB`) rendered through both paths -- +`qs_*` via `cortex.quickshow` (matplotlib) and `wg_*` via +`cortex.export.plot_panels` (headless WebGL). + +Between them they exercise every way pycortex encodes alpha: `Volume`/`Vertex` are +a no-alpha baseline, `Volume2D`/`Vertex2D` use the 2D alpha colormap +`RdBu_r_alpha`, and `VolumeRGB`/`VertexRGB` use the native `alpha=` keyword. All +six also composite the curvature underlay. + +## Provenance + +Generated on the `main` branch at commit `5af26a86`, deliberately: they are a +pre-change baseline, so a restructure of `cortex.dataset` has to reproduce main's +output pixel for pixel. + +| | | +| --- | --- | +| chromium | 151.0.7922.34 (headless shell, SwiftShader software rendering) | +| matplotlib | 3.11.1 | +| pillow | 12.3.0 | + +## Format + +Lossless WebP (`method=6`, `quality=100`, `exact=True`): bit-exact after decode, +and 59% the size of optimized PNG (669 KiB versus 1129 KiB for the set). + +Two alternatives were measured and rejected: + +- **Higher PNG compression** does nothing. With `optimize=True`, `compress_level=6` + and `compress_level=9` produce byte-identical output. +- **AVIF** is smaller again, but Pillow cannot write it losslessly. Even at + `qmin=0, qmax=0` it decoded with max\|difference\| 27-29 -- larger than the + test's own 16-unit per-pixel threshold, so it would corrupt the comparison it + exists to feed. + +Bit-exactness matters because the test compares all four channels; a format that +perturbs values would eat into the tolerance budget meant for genuine +environmental differences. + +## Distribution + +These are test fixtures with no runtime use, so they are **kept out of the wheel** +(`exclude_package_data` in `setup.py`) and **kept in the source tarball** +(`MANIFEST.in`'s `recursive-include cortex *`). A build from source can therefore +run the test; a `pip install` does not carry ~700 KiB of PNGs into site-packages +for data no user will read. + +The test skips, rather than fails, when the images are absent, so a test run +against an installed wheel degrades gracefully. + +## Regenerating + +The renders are deterministic: repeated runs on one machine produce bit-identical +output, including the WebGL ones under software rendering. They are, however, +coupled to the Chromium and matplotlib builds above, so a browser or matplotlib +upgrade can shift anti-aliasing and rasterization slightly. The test's tolerances +absorb that; if an upgrade moves output beyond them, inspect the `diff_*.png` +files the failure writes, confirm the change is cosmetic, then: + +``` +REGENERATE_REFERENCE_IMAGES=1 pytest cortex/tests/test_webgl_headless.py -k visual_comparison +``` + +Review the resulting diff before committing -- regenerating is how a real +regression gets silently blessed. diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_Vertex.webp b/cortex/tests/reference_images/alpha_dataviews/qs_Vertex.webp new file mode 100644 index 00000000..47e021e8 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_Vertex.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_Vertex2D.webp b/cortex/tests/reference_images/alpha_dataviews/qs_Vertex2D.webp new file mode 100644 index 00000000..2c03bae9 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_Vertex2D.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_VertexRGB.webp b/cortex/tests/reference_images/alpha_dataviews/qs_VertexRGB.webp new file mode 100644 index 00000000..ae476266 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_VertexRGB.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_Volume.webp b/cortex/tests/reference_images/alpha_dataviews/qs_Volume.webp new file mode 100644 index 00000000..85e4a2dc Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_Volume.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_Volume2D.webp b/cortex/tests/reference_images/alpha_dataviews/qs_Volume2D.webp new file mode 100644 index 00000000..eed1c57c Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_Volume2D.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/qs_VolumeRGB.webp b/cortex/tests/reference_images/alpha_dataviews/qs_VolumeRGB.webp new file mode 100644 index 00000000..d1140090 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/qs_VolumeRGB.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_Vertex.webp b/cortex/tests/reference_images/alpha_dataviews/wg_Vertex.webp new file mode 100644 index 00000000..b76d4c45 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_Vertex.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_Vertex2D.webp b/cortex/tests/reference_images/alpha_dataviews/wg_Vertex2D.webp new file mode 100644 index 00000000..f13f0276 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_Vertex2D.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_VertexRGB.webp b/cortex/tests/reference_images/alpha_dataviews/wg_VertexRGB.webp new file mode 100644 index 00000000..42d4fc66 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_VertexRGB.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_Volume.webp b/cortex/tests/reference_images/alpha_dataviews/wg_Volume.webp new file mode 100644 index 00000000..8d75bd2f Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_Volume.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_Volume2D.webp b/cortex/tests/reference_images/alpha_dataviews/wg_Volume2D.webp new file mode 100644 index 00000000..a7f42f36 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_Volume2D.webp differ diff --git a/cortex/tests/reference_images/alpha_dataviews/wg_VolumeRGB.webp b/cortex/tests/reference_images/alpha_dataviews/wg_VolumeRGB.webp new file mode 100644 index 00000000..7cb62047 Binary files /dev/null and b/cortex/tests/reference_images/alpha_dataviews/wg_VolumeRGB.webp differ diff --git a/cortex/tests/test_webgl_headless.py b/cortex/tests/test_webgl_headless.py index 90808412..8c2818e1 100644 --- a/cortex/tests/test_webgl_headless.py +++ b/cortex/tests/test_webgl_headless.py @@ -8,6 +8,8 @@ """ import os +import pathlib +import shutil import time import numpy as np @@ -617,25 +619,91 @@ def test_addData_no_crash(): # --------------------------------------------------------------------------- -# Group 10: Manual visual A/B comparison across all alpha-bearing dataviews +# Group 10: Visual regression across all alpha-bearing dataviews # --------------------------------------------------------------------------- +#: Stored renders this test asserts against. See that directory's README for how +#: they were produced and how to regenerate them. +REFERENCE_DIR = pathlib.Path(__file__).parent / "reference_images" / "alpha_dataviews" + +#: Lossless WebP: bit-exact after decode and 59% the size of optimized PNG. AVIF +#: is smaller still but Pillow cannot write it losslessly -- it was measured at +#: max|difference| 27-29, larger than DIFF_THRESHOLD below, so it would corrupt +#: the comparison it is meant to feed. Higher PNG compression levels are pointless +#: here: level 6 and level 9 produce byte-identical output. +REFERENCE_SUFFIX = ".webp" + +#: Rewrite the references from this run instead of comparing against them. +REGENERATE_REFERENCES = bool(os.environ.get("REGENERATE_REFERENCE_IMAGES")) + +# Tolerances. The renders are deterministic -- repeated runs on one machine are +# bit-identical -- so these are not absorbing noise. They exist because the +# references are coupled to the Chromium and matplotlib builds that produced them, +# and an upgrade can shift anti-aliasing and rasterization slightly. They are far +# tighter than any real regression: a wrong colormap, a dropped alpha channel or +# swapped colour channels all move large areas of the image by much more. +MAX_MEAN_ABS_DIFF = 2.0 # mean |difference| over all pixels/channels, of 255 +DIFF_THRESHOLD = 16 # a pixel "differs" if any channel moves by more +MAX_FRACTION_DIFFERING = 0.02 # at most this fraction of pixels may differ + + +def _check_against_reference(name, actual_path, debug_dir): + """Compare one render to its reference. + + Returns a description of the mismatch, or None if it matches. On mismatch, + writes the actual render and an amplified difference image into ``debug_dir`` + so the change can be inspected rather than guessed at. + """ + from PIL import Image + + ref_path = REFERENCE_DIR / f"{name}{REFERENCE_SUFFIX}" + + if REGENERATE_REFERENCES: + REFERENCE_DIR.mkdir(parents=True, exist_ok=True) + Image.open(actual_path).convert("RGBA").save( + ref_path, format="WEBP", lossless=True, method=6, quality=100, exact=True + ) + return None + + actual = np.asarray(Image.open(actual_path).convert("RGBA")).astype(np.int16) + ref = np.asarray(Image.open(ref_path).convert("RGBA")).astype(np.int16) + if actual.shape != ref.shape: + return f"{name}: render is {actual.shape}, reference is {ref.shape}" + + diff = np.abs(actual - ref) + mean_abs = float(diff.mean()) + fraction = float((diff.max(axis=-1) > DIFF_THRESHOLD).mean()) + if mean_abs <= MAX_MEAN_ABS_DIFF and fraction <= MAX_FRACTION_DIFFERING: + return None + + shutil.copyfile(actual_path, debug_dir / f"actual_{name}.png") + amplified = np.clip(diff[..., :3] * 8, 0, 255).astype("uint8") + Image.fromarray(amplified).save(debug_dir / f"diff_{name}.png") + return ( + f"{name}: mean|diff|={mean_abs:.3f} (limit {MAX_MEAN_ABS_DIFF}), " + f"{fraction:.2%} of pixels differ by more than {DIFF_THRESHOLD} " + f"(limit {MAX_FRACTION_DIFFERING:.0%})" + ) + -@pytest.mark.skipif( - not os.environ.get("RUN_VISUAL_COMPARISON"), - reason="Manual visual comparison; set RUN_VISUAL_COMPARISON=1 to run.", -) def test_visual_comparison_alpha_dataviews(tmp_path): - """Render all 6 dataview types via quickshow + webgl, side-by-side. + """Render all 6 dataview types via quickshow + webgl, and assert they match. + + Each of the six public dataview classes is rendered twice -- through + matplotlib (``cortex.quickshow``) and through the headless WebGL viewer + (``cortex.export.plot_panels``) -- and both renders are compared against a + stored reference produced on ``main``. Together they cover every way pycortex + encodes alpha, so this is the test that notices if a change to + ``cortex.dataset`` alters what actually gets drawn. - Skipped by default — set ``RUN_VISUAL_COMPARISON=1`` to run. Builds a - grid where each row is one dataview type (Volume, Vertex, Volume2D, - Vertex2D, VolumeRGB, VertexRGB) and the two columns are the matplotlib - (``cortex.quickshow``) reference vs the headless WebGL flatmap render. - Used as a manual smoke check that the alpha-blend fix - (``Package``-side premultiply for VertexRGB + cmap-LUT - ``premultiplyAlpha=true`` for the 2D-cmap path) keeps both viewers in - visual agreement across every alpha-encoding pattern. + It also still writes the side-by-side composite, which is the useful artifact + when something *has* changed: the printed path opens a grid of all twelve + renders. Mismatches additionally dump ``actual_*.png`` and an amplified + ``diff_*.png`` next to it. + + Previously this was gated behind ``RUN_VISUAL_COMPARISON`` and asserted only + that a file had been written, which meant a rendering regression could only be + caught by a human looking at the picture. Plain Volume / Vertex have no native per-element alpha (pycortex's bundled ``*_alpha`` colormaps are all 2D and only apply to the 2D @@ -770,6 +838,26 @@ def _bump(surf, seed, sigma): } ] + # Absent references are a skip, not a failure: the wheel ships this test but + # deliberately not the images (they are test fixtures, of no use at runtime), + # so an installed-package test run must degrade gracefully rather than error. + # Checked before rendering, which would otherwise waste ~30s to reach it. + if not REGENERATE_REFERENCES: + missing = [ + f"{prefix}_{name}{REFERENCE_SUFFIX}" + for name, _ in dataviews + for prefix in ("qs", "wg") + if not (REFERENCE_DIR / f"{prefix}_{name}{REFERENCE_SUFFIX}").exists() + ] + if missing: + pytest.skip( + "No reference images in %s (missing %d, e.g. %s). They ship in the " + "source tarball but not the wheel; see that directory's README." + % (REFERENCE_DIR, len(missing), missing[0]) + ) + + rendered: list[tuple[str, pathlib.Path]] = [] + for row, (name, view) in enumerate(dataviews): # quickshow → low-res PNG qs_path = tmp_path / f"qs_{name}.png" @@ -785,6 +873,7 @@ def _bump(surf, seed, sigma): ) qs_fig.savefig(qs_path, bbox_inches="tight", pad_inches=0, dpi=80) plt.close(qs_fig) + rendered.append((f"qs_{name}", qs_path)) # webgl → trimmed flatmap PNG via plot_panels (single flatmap panel) wg_path = str(tmp_path / f"wg_{name}.png") @@ -799,6 +888,7 @@ def _bump(surf, seed, sigma): headless=True, ) plt.close(wg_fig) + rendered.append((f"wg_{name}", pathlib.Path(wg_path))) ax_qs, ax_wg = axes[row] ax_qs.imshow(plt.imread(qs_path)) @@ -819,3 +909,32 @@ def _bump(surf, seed, sigma): print(f"\nVisual comparison saved to:\n {out_path}\n") assert out_path.exists() assert out_path.stat().st_size > 0 + + # Every render is compared, so a failure reports all of them at once rather + # than stopping at the first -- knowing whether one view or all six moved is + # most of the diagnosis. + failures = [ + msg + for msg in ( + _check_against_reference(rendered_name, rendered_path, tmp_path) + for rendered_name, rendered_path in rendered + ) + if msg is not None + ] + if REGENERATE_REFERENCES: + pytest.skip( + f"Regenerated {len(rendered)} reference images in {REFERENCE_DIR}. " + "Review the diff before committing." + ) + assert not failures, ( + "%d of %d renders differ from their reference:\n %s\n\n" + "Renders, and amplified diffs, are in %s\nIf the change is intended, see " + "%s/README.md for how to regenerate." + % ( + len(failures), + len(rendered), + "\n ".join(failures), + tmp_path, + REFERENCE_DIR.parent, + ) + ) diff --git a/setup.py b/setup.py index 86823c57..cfdde3f9 100644 --- a/setup.py +++ b/setup.py @@ -139,6 +139,13 @@ def run(self): # Don't use `extras_require` here. Put them in pyproject.toml . cmdclass=dict(install=my_install), include_package_data=True, + # Reference renders for the visual-regression test are fixtures, of no use + # at runtime. MANIFEST.in's recursive-include keeps them in the source + # tarball, so a build from source can still run the test; this keeps them + # out of the wheel, where they would just sit in every user's + # site-packages. The test skips when they are absent. + exclude_package_data={'cortex.tests': ['reference_images/*', + 'reference_images/*/*']}, classifiers=[ 'Development Status :: 6 - Mature', 'Intended Audience :: Science/Research',