Use streaming in all assertion comparisons consumers

changelog/14523.improvement.rst

@@ -0,0 +1,7 @@
+The assertion comparison helpers and the truncator now hand the
+explanation between them as an iterator, so the dispatcher only has to
+materialise lines the truncator actually keeps. For the typical small
+diff this is a no-op; for the rare case where two large collections
+fail to compare, peak memory drops by a few percent because the
+millions of lines that would have been built and immediately discarded
+are no longer built.

src/_pytest/assertion/__init__.py

@@ -181,13 +181,19 @@ def callbinrepr(op, left: object, right: object) -> str | None:
             config=item.config, op=op, left=left, right=right
         )
         for new_expl in hook_result:
-            if new_expl:
-                new_expl = truncate.truncate_if_required(new_expl, item)
-                new_expl = [line.replace("\n", "\\n") for line in new_expl]
-                res = "\n~".join(new_expl)
-                if item.config.getvalue("assertmode") == "rewrite":
-                    res = res.replace("%", "%%")
-                return res
+            if not new_expl:
+                continue
+            # Plugin-supplied lists are truncated here; the built-in
+            # impl truncates as it streams, so this is a no-op on its
+            # already-short output.
+            new_expl = truncate.materialize_with_truncation(new_expl, item.config)
+            if not new_expl:
+                continue
+            new_expl = [line.replace("\n", "\\n") for line in new_expl]
+            res = "\n~".join(new_expl)
+            if item.config.getvalue("assertmode") == "rewrite":
+                res = res.replace("%", "%%")
+            return res
         return None
 
     saved_assert_hooks = util._reprcompare, util._assertion_pass
@@ -218,16 +224,25 @@ def pytest_sessionfinish(session: Session) -> None:
 def pytest_assertrepr_compare(
     config: Config, op: str, left: Any, right: Any
 ) -> list[str] | None:
+    """Return an explanation for ``left op right``.
+
+    Internally ``util.assertrepr_compare`` is a generator; we feed it
+    through ``materialize_with_truncation`` so a huge comparison
+    short-circuits at the truncation threshold without building the
+    full diff, while still returning the ``list[str] | None`` shape
+    the hook spec advertises.
+    """
     if config.pluginmanager.has_plugin("terminalreporter"):
         highlighter = config.get_terminal_writer()._highlight
     else:
         # Keep it plaintext when not using terminalrepoterer (#14377).
         highlighter = util.dummy_highlighter
-    return util.assertrepr_compare(
+    lines = util.assertrepr_compare(
         op=op,
         left=left,
         right=right,
         verbose=config.get_verbosity(Config.VERBOSITY_ASSERTIONS),
         highlighter=highlighter,
         assertion_text_diff_style=util.get_assertion_text_diff_style(config),
     )
+    return truncate.materialize_with_truncation(lines, config) or None

src/_pytest/assertion/_compare_any.py

@@ -28,54 +28,52 @@ def _compare_eq_any(
     highlighter: _HighlightFunc,
     verbose: int,
     assertion_text_diff_style: _AssertionTextDiffStyle,
-) -> list[str]:
-    explanation = []
+) -> Iterator[str]:
+    """Yield the per-line explanation for ``left == right`` (without summary).
+
+    Yields nothing when no specialised explanation applies, so consumers
+    can stream the output and bail out early (e.g. for truncation) without
+    materialising the entire diff first.
+    """
     if istext(left) and istext(right):
-        explanation = list(
-            _compare_eq_text(
-                left,
-                right,
-                highlighter,
-                verbose,
-                assertion_text_diff_style,
-            )
+        yield from _compare_eq_text(
+            left,
+            right,
+            highlighter,
+            verbose,
+            assertion_text_diff_style,
         )
     else:
         from _pytest.python_api import ApproxBase
 
         # Although the common order should be obtained == approx(...), allow both ways.
         if isinstance(right, ApproxBase):
-            explanation = right._repr_compare(left)
+            yield from right._repr_compare(left)
         elif isinstance(left, ApproxBase):
-            explanation = left._repr_compare(right)
+            yield from left._repr_compare(right)
         elif type(left) is type(right) and (
             isdatacls(left) or isattrs(left) or isnamedtuple(left)
         ):
             # Note: unlike dataclasses/attrs, namedtuples compare only the
             # field values, not the type or field names. But this branch
             # intentionally only handles the same-type case, which was often
             # used in older code bases before dataclasses/attrs were available.
-            explanation = list(
-                _compare_eq_cls(
-                    left,
-                    right,
-                    highlighter,
-                    verbose,
-                    assertion_text_diff_style,
-                )
+            yield from _compare_eq_cls(
+                left,
+                right,
+                highlighter,
+                verbose,
+                assertion_text_diff_style,
             )
         elif issequence(left) and issequence(right):
-            explanation = list(_compare_eq_sequence(left, right, highlighter, verbose))
+            yield from _compare_eq_sequence(left, right, highlighter, verbose)
         elif isset(left) and isset(right):
-            explanation = _compare_eq_set(left, right, highlighter, verbose)
+            yield from _compare_eq_set(left, right, highlighter, verbose)
         elif ismapping(left) and ismapping(right):
-            explanation = list(_compare_eq_mapping(left, right, highlighter, verbose))
+            yield from _compare_eq_mapping(left, right, highlighter, verbose)
 
         if isiterable(left) and isiterable(right):
-            expl = _compare_eq_iterable(left, right, highlighter, verbose)
-            explanation.extend(expl)
-
-    return explanation
+            yield from _compare_eq_iterable(left, right, highlighter, verbose)
 
 
 def _compare_eq_cls(

src/_pytest/assertion/_compare_set.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from collections.abc import Iterator
 from collections.abc import Set as AbstractSet
 from typing import TypeAlias
 
@@ -13,80 +14,86 @@ def _set_one_sided_diff(
     set1: AbstractSet[object],
     set2: AbstractSet[object],
     highlighter: _HighlightFunc,
-) -> list[str]:
-    explanation = []
+) -> Iterator[str]:
     diff = set1 - set2
     if diff:
-        explanation.append(f"Extra items in the {posn} set:")
+        yield f"Extra items in the {posn} set:"
         for item in diff:
-            explanation.append(highlighter(saferepr(item)))
-    return explanation
+            yield highlighter(saferepr(item))
 
 
 def _compare_eq_set(
     left: AbstractSet[object],
     right: AbstractSet[object],
     highlighter: _HighlightFunc,
     verbose: int = 0,
-) -> list[str]:
-    explanation = []
-    explanation.extend(_set_one_sided_diff("left", left, right, highlighter))
-    explanation.extend(_set_one_sided_diff("right", right, left, highlighter))
-    return explanation
+) -> Iterator[str]:
+    yield from _set_one_sided_diff("left", left, right, highlighter)
+    yield from _set_one_sided_diff("right", right, left, highlighter)
 
 
-def _compare_gt_set(
+def _compare_gte_set(
     left: AbstractSet[object],
     right: AbstractSet[object],
     highlighter: _HighlightFunc,
     verbose: int = 0,
-) -> list[str]:
-    explanation = _compare_gte_set(left, right, highlighter)
-    if not explanation:
-        return ["Both sets are equal"]
-    return explanation
+) -> Iterator[str]:
+    yield from _set_one_sided_diff("right", right, left, highlighter)
 
 
-def _compare_lt_set(
+def _compare_lte_set(
     left: AbstractSet[object],
     right: AbstractSet[object],
     highlighter: _HighlightFunc,
     verbose: int = 0,
-) -> list[str]:
-    explanation = _compare_lte_set(left, right, highlighter)
-    if not explanation:
-        return ["Both sets are equal"]
-    return explanation
+) -> Iterator[str]:
+    yield from _set_one_sided_diff("left", left, right, highlighter)
 
 
-def _compare_gte_set(
+def _compare_gt_set(
     left: AbstractSet[object],
     right: AbstractSet[object],
     highlighter: _HighlightFunc,
     verbose: int = 0,
-) -> list[str]:
-    return _set_one_sided_diff("right", right, left, highlighter)
+) -> Iterator[str]:
+    if left == right:
+        yield "Both sets are equal"
+    else:
+        yield from _set_one_sided_diff("right", right, left, highlighter)
 
 
-def _compare_lte_set(
+def _compare_lt_set(
     left: AbstractSet[object],
     right: AbstractSet[object],
     highlighter: _HighlightFunc,
     verbose: int = 0,
-) -> list[str]:
-    return _set_one_sided_diff("left", left, right, highlighter)
+) -> Iterator[str]:
+    if left == right:
+        yield "Both sets are equal"
+    else:
+        yield from _set_one_sided_diff("left", left, right, highlighter)
 
 
 SetComparisonFunction: TypeAlias = Callable[
     [AbstractSet[object], AbstractSet[object], _HighlightFunc, int],
-    list[str],
+    Iterator[str],
 ]
 
+
+def _both_sets_are_equal(
+    left: AbstractSet[object],
+    right: AbstractSet[object],
+    highlighter: _HighlightFunc,
+    verbose: int = 0,
+) -> Iterator[str]:
+    yield "Both sets are equal"
+
+
 SET_COMPARISON_FUNCTIONS: dict[str, SetComparisonFunction] = {
     # == can't be done here without a prior refactor because there's an additional
     # explanation for iterable in _compare_eq_any
     # "==": _compare_eq_set,
-    "!=": lambda *a, **kw: ["Both sets are equal"],
+    "!=": _both_sets_are_equal,
     ">=": _compare_gte_set,
     "<=": _compare_lte_set,
     ">": _compare_gt_set,