From e016f863e7909c74249e4c7112611ec93cf54400 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 5 Mar 2026 21:37:34 +0000 Subject: [PATCH 1/4] Add skip_hash=True/False option to allow hash comparison to be skipped for cases where hybrid mode is used and a test is not giving deterministic hashes --- docs/configuration.rst | 24 ++++++++++++++++++ docs/hybrid_mode.rst | 18 +++++++++++++ pytest_mpl/plugin.py | 12 ++++++--- tests/test_hash_library.py | 52 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 4 deletions(-) diff --git a/docs/configuration.rst b/docs/configuration.rst index 1afce3af..b115f3f2 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -207,6 +207,30 @@ If its directory does not exist, it will be created along with any missing paren Configuring this option disables baseline image comparison. If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" `, you must explicitly set the :ref:`baseline directory configuration option `. +.. _skip-hash: + +Skip hash comparison for specific tests +--------------------------------------- +| **kwarg**: ``skip_hash=`` +| **CLI**: --- +| **INI**: --- +| Default: ``False`` + +When a global hash library is configured (via CLI or INI), you can disable hash comparison for specific tests by setting ``skip_hash=True``. +This is useful for tests that have non-deterministic output where exact hash matching is not possible, but baseline image comparison with a tolerance is acceptable. + +When ``skip_hash=True`` is set, the test will use baseline image comparison instead of hash comparison, even if a hash library is configured globally. + +.. code:: python + + @pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10) + def test_plot_with_tolerance(): + # This test will use baseline image comparison with tolerance, + # skipping hash comparison even if --mpl-hash-library is set + ... + +This option is particularly useful in :doc:`"hybrid mode" ` when most tests should use hash comparison for speed and reliability, but a few tests need tolerance-based image comparison due to platform-specific rendering differences. + .. _controlling-sensitivity: Controlling the sensitivity of the comparison diff --git a/docs/hybrid_mode.rst b/docs/hybrid_mode.rst index f5b1ebfb..5d7268d1 100644 --- a/docs/hybrid_mode.rst +++ b/docs/hybrid_mode.rst @@ -82,6 +82,24 @@ This is what the basic HTML summary looks like for the same test as above: :summary:`test_basic_html` +Skipping hash comparison for specific tests +============================================ + +In some cases, certain tests may produce images that are not deterministic across platforms or environments. +For these tests, you can use ``skip_hash=True`` to disable hash comparison and fall back to baseline image comparison with a tolerance: + +.. code-block:: python + + @pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10) + def test_plot_with_platform_differences(): + # This test will use baseline image comparison instead of hash comparison + fig, ax = plt.subplots() + ax.plot([1, 2, 3]) + return fig + +This allows you to use hash comparison for most tests (for speed and reliability), while allowing a few tests to use tolerance-based image comparison. +See the :ref:`skip_hash configuration option ` for more details. + Continue reading ================ diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 4367dd89..8b4b19b3 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -694,9 +694,11 @@ def save_figure(self, item, fig, filename): if deterministic is None: # The deterministic option should only matter for hash-based tests, - # so we first check if a hash library is being used + # so we first check if a hash library is being used (and not skipped) + skip_hash = compare.kwargs.get('skip_hash', False) + using_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash - if self.hash_library or compare.kwargs.get('hash_library', None): + if using_hash_library: if ext == 'png': if 'metadata' not in savefig_kwargs or 'Software' not in savefig_kwargs['metadata']: @@ -917,8 +919,10 @@ def pytest_runtest_call(self, item): # noqa # Only test figures if not generating images if self.generate_dir is None: - # Compare to hash library - if self.hash_library or compare.kwargs.get('hash_library', None): + # Compare to hash library (skip_hash=True disables for this test) + skip_hash = compare.kwargs.get('skip_hash', False) + use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash + if use_hash_library: msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary) # Compare against a baseline if specified diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py index 0afa0544..b00957c8 100644 --- a/tests/test_hash_library.py +++ b/tests/test_hash_library.py @@ -4,6 +4,56 @@ from helpers import pytester_path +def test_skip_hash(pytester): + """Test that skip_hash=True skips hash comparison and uses baseline instead.""" + path = pytester_path(pytester) + hash_library = path / "hash_library.json" + baseline_dir = path / "baseline" + + # Generate baseline image (no hash library needed for generation) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare() + def test_mpl(): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([1, 3, 2]) + return fig + """ + ) + pytester.runpytest(f"--mpl-generate-path={baseline_dir}") + + # Create hash library with bad hash + with open(hash_library, "w") as fp: + json.dump({"test_skip_hash.test_mpl": "bad-hash-value"}, fp) + + # Without skip_hash: should fail (hash mismatch) + result = pytester.runpytest("--mpl", + f"--mpl-hash-library={hash_library}", + f"--mpl-baseline-path={baseline_dir}") + result.assert_outcomes(failed=1) + + # With skip_hash=True: should pass (uses baseline comparison, skips hash) + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + @pytest.mark.mpl_image_compare(skip_hash=True) + def test_mpl(): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([1, 3, 2]) + return fig + """ + ) + result = pytester.runpytest("--mpl", + f"--mpl-hash-library={hash_library}", + f"--mpl-baseline-path={baseline_dir}") + result.assert_outcomes(passed=1) + + @pytest.mark.parametrize( "ini, cli, kwarg, success_expected", [ @@ -49,3 +99,5 @@ def test_mpl(): result.assert_outcomes(passed=1) else: result.assert_outcomes(failed=1) + + From 65181c02fa23ab6bf7412d08334b66074dafd872 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 5 Mar 2026 21:45:21 +0000 Subject: [PATCH 2/4] Don't output tests with skip_hash=True to hash library --- pytest_mpl/plugin.py | 12 ++++++++---- tests/test_hash_library.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 8b4b19b3..dd43c0db 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -912,10 +912,14 @@ def pytest_runtest_call(self, item): # noqa result_image.relative_to(self.results_dir).as_posix() if self.generate_hash_library is not None: - summary['hash_status'] = 'generated' - image_hash = self.generate_image_hash(item, fig) - self._generated_hash_library[test_name] = image_hash - summary['baseline_hash'] = image_hash + skip_hash = compare.kwargs.get('skip_hash', False) + if skip_hash: + summary['hash_status'] = 'skipped' + else: + summary['hash_status'] = 'generated' + image_hash = self.generate_image_hash(item, fig) + self._generated_hash_library[test_name] = image_hash + summary['baseline_hash'] = image_hash # Only test figures if not generating images if self.generate_dir is None: diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py index b00957c8..bf9dfaa6 100644 --- a/tests/test_hash_library.py +++ b/tests/test_hash_library.py @@ -54,6 +54,43 @@ def test_mpl(): result.assert_outcomes(passed=1) +def test_skip_hash_not_generated(pytester): + """Test that skip_hash=True tests are not included in generated hash library.""" + path = pytester_path(pytester) + hash_library = path / "hash_library.json" + + pytester.makepyfile( + """ + import matplotlib.pyplot as plt + import pytest + + @pytest.mark.mpl_image_compare() + def test_normal(): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([1, 2, 3]) + return fig + + @pytest.mark.mpl_image_compare(skip_hash=True) + def test_skip(): + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.plot([3, 2, 1]) + return fig + """ + ) + pytester.runpytest(f"--mpl-generate-hash-library={hash_library}") + + # Check generated hash library + with open(hash_library) as fp: + hashes = json.load(fp) + + # test_normal should be in the hash library + assert "test_skip_hash_not_generated.test_normal" in hashes + # test_skip should NOT be in the hash library + assert "test_skip_hash_not_generated.test_skip" not in hashes + + @pytest.mark.parametrize( "ini, cli, kwarg, success_expected", [ From 10c03aa3fafd71828d3bc3296f182499fed47839 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 5 Mar 2026 21:47:51 +0000 Subject: [PATCH 3/4] Change variable name --- pytest_mpl/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index dd43c0db..dde08770 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -696,9 +696,9 @@ def save_figure(self, item, fig, filename): # The deterministic option should only matter for hash-based tests, # so we first check if a hash library is being used (and not skipped) skip_hash = compare.kwargs.get('skip_hash', False) - using_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash + use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash - if using_hash_library: + if use_hash_library: if ext == 'png': if 'metadata' not in savefig_kwargs or 'Software' not in savefig_kwargs['metadata']: From f3db2fe00af14cc49e94452999c53f3b857a3ca7 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Thu, 5 Mar 2026 21:51:01 +0000 Subject: [PATCH 4/4] Fix codestyle --- tests/test_hash_library.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_hash_library.py b/tests/test_hash_library.py index bf9dfaa6..e0638812 100644 --- a/tests/test_hash_library.py +++ b/tests/test_hash_library.py @@ -136,5 +136,3 @@ def test_mpl(): result.assert_outcomes(passed=1) else: result.assert_outcomes(failed=1) - -