Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions docs/configuration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,30 @@ If its directory does not exist, it will be created along with any missing paren
Configuring this option disables baseline image comparison.
If you want to enable both hash and baseline image comparison, which we call :doc:`"hybrid mode" <hybrid_mode>`, you must explicitly set the :ref:`baseline directory configuration option <baseline-dir>`.

.. _skip-hash:

Skip hash comparison for specific tests
---------------------------------------
| **kwarg**: ``skip_hash=<bool>``
| **CLI**: ---
| **INI**: ---
| Default: ``False``

When a global hash library is configured (via CLI or INI), you can disable hash comparison for specific tests by setting ``skip_hash=True``.
This is useful for tests that have non-deterministic output where exact hash matching is not possible, but baseline image comparison with a tolerance is acceptable.

When ``skip_hash=True`` is set, the test will use baseline image comparison instead of hash comparison, even if a hash library is configured globally.

.. code:: python

@pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10)
def test_plot_with_tolerance():
# This test will use baseline image comparison with tolerance,
# skipping hash comparison even if --mpl-hash-library is set
...

This option is particularly useful in :doc:`"hybrid mode" <hybrid_mode>` when most tests should use hash comparison for speed and reliability, but a few tests need tolerance-based image comparison due to platform-specific rendering differences.

.. _controlling-sensitivity:

Controlling the sensitivity of the comparison
Expand Down
18 changes: 18 additions & 0 deletions docs/hybrid_mode.rst
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,24 @@ This is what the basic HTML summary looks like for the same test as above:

:summary:`test_basic_html`

Skipping hash comparison for specific tests
============================================

In some cases, certain tests may produce images that are not deterministic across platforms or environments.
For these tests, you can use ``skip_hash=True`` to disable hash comparison and fall back to baseline image comparison with a tolerance:

.. code-block:: python

@pytest.mark.mpl_image_compare(skip_hash=True, tolerance=10)
def test_plot_with_platform_differences():
# This test will use baseline image comparison instead of hash comparison
fig, ax = plt.subplots()
ax.plot([1, 2, 3])
return fig

This allows you to use hash comparison for most tests (for speed and reliability), while allowing a few tests to use tolerance-based image comparison.
See the :ref:`skip_hash configuration option <skip-hash>` for more details.

Continue reading
================

Expand Down
24 changes: 16 additions & 8 deletions pytest_mpl/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -694,9 +694,11 @@ def save_figure(self, item, fig, filename):
if deterministic is None:

# The deterministic option should only matter for hash-based tests,
# so we first check if a hash library is being used
# so we first check if a hash library is being used (and not skipped)
skip_hash = compare.kwargs.get('skip_hash', False)
use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash

if self.hash_library or compare.kwargs.get('hash_library', None):
if use_hash_library:

if ext == 'png':
if 'metadata' not in savefig_kwargs or 'Software' not in savefig_kwargs['metadata']:
Expand Down Expand Up @@ -910,15 +912,21 @@ def pytest_runtest_call(self, item): # noqa
result_image.relative_to(self.results_dir).as_posix()

if self.generate_hash_library is not None:
summary['hash_status'] = 'generated'
image_hash = self.generate_image_hash(item, fig)
self._generated_hash_library[test_name] = image_hash
summary['baseline_hash'] = image_hash
skip_hash = compare.kwargs.get('skip_hash', False)
if skip_hash:
summary['hash_status'] = 'skipped'
else:
summary['hash_status'] = 'generated'
image_hash = self.generate_image_hash(item, fig)
self._generated_hash_library[test_name] = image_hash
summary['baseline_hash'] = image_hash

# Only test figures if not generating images
if self.generate_dir is None:
# Compare to hash library
if self.hash_library or compare.kwargs.get('hash_library', None):
# Compare to hash library (skip_hash=True disables for this test)
skip_hash = compare.kwargs.get('skip_hash', False)
use_hash_library = (self.hash_library or compare.kwargs.get('hash_library', None)) and not skip_hash
if use_hash_library:
msg = self.compare_image_to_hash_library(item, fig, result_dir, summary=summary)

# Compare against a baseline if specified
Expand Down
87 changes: 87 additions & 0 deletions tests/test_hash_library.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,93 @@
from helpers import pytester_path


def test_skip_hash(pytester):
"""Test that skip_hash=True skips hash comparison and uses baseline instead."""
path = pytester_path(pytester)
hash_library = path / "hash_library.json"
baseline_dir = path / "baseline"

# Generate baseline image (no hash library needed for generation)
pytester.makepyfile(
"""
import matplotlib.pyplot as plt
import pytest
@pytest.mark.mpl_image_compare()
def test_mpl():
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot([1, 3, 2])
return fig
"""
)
pytester.runpytest(f"--mpl-generate-path={baseline_dir}")

# Create hash library with bad hash
with open(hash_library, "w") as fp:
json.dump({"test_skip_hash.test_mpl": "bad-hash-value"}, fp)

# Without skip_hash: should fail (hash mismatch)
result = pytester.runpytest("--mpl",
f"--mpl-hash-library={hash_library}",
f"--mpl-baseline-path={baseline_dir}")
result.assert_outcomes(failed=1)

# With skip_hash=True: should pass (uses baseline comparison, skips hash)
pytester.makepyfile(
"""
import matplotlib.pyplot as plt
import pytest
@pytest.mark.mpl_image_compare(skip_hash=True)
def test_mpl():
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot([1, 3, 2])
return fig
"""
)
result = pytester.runpytest("--mpl",
f"--mpl-hash-library={hash_library}",
f"--mpl-baseline-path={baseline_dir}")
result.assert_outcomes(passed=1)


def test_skip_hash_not_generated(pytester):
"""Test that skip_hash=True tests are not included in generated hash library."""
path = pytester_path(pytester)
hash_library = path / "hash_library.json"

pytester.makepyfile(
"""
import matplotlib.pyplot as plt
import pytest

@pytest.mark.mpl_image_compare()
def test_normal():
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot([1, 2, 3])
return fig

@pytest.mark.mpl_image_compare(skip_hash=True)
def test_skip():
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.plot([3, 2, 1])
return fig
"""
)
pytester.runpytest(f"--mpl-generate-hash-library={hash_library}")

# Check generated hash library
with open(hash_library) as fp:
hashes = json.load(fp)

# test_normal should be in the hash library
assert "test_skip_hash_not_generated.test_normal" in hashes
# test_skip should NOT be in the hash library
assert "test_skip_hash_not_generated.test_skip" not in hashes


@pytest.mark.parametrize(
"ini, cli, kwarg, success_expected",
[
Expand Down