fix: preserve region stability across theme drift

openadapt_flow/runtime/replayer.py

@@ -97,6 +97,12 @@
 # and the minimum template-match score to accept it.
 PC_TEMPLATE_SEARCH_PAD = 80
 PC_TEMPLATE_THRESHOLD = 0.9
+# REGION_STABLE asserts recorded structure, not palette.  The ordinary
+# grayscale template matcher remains the first (stricter) check; this edge-map
+# threshold is a color-invariant fallback.  Exact v1.16.1 MockMed evidence
+# measured correct theme+parameter substitution at 0.86 and a true blocking
+# modal at 0.24, leaving a deliberate margin on both sides.
+PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8
 
 # Typed-input verification: size of the "field region" diffed/OCRed around
 # the focusing click point after a TYPE action. Generous on purpose — the
@@ -3666,6 +3672,24 @@ def _postcondition_passes(
                 )
                 if match is not None:
                     return True
+                # A region-stability check is about the recorded structure,
+                # not its light/dark palette.  Use the same sealed template
+                # and localized search on edge maps before falling back to the
+                # exact-position structural pHash.  This preserves the
+                # postcondition: a blocking modal or changed layout still
+                # fails; only the representation becomes palette-invariant.
+                structural_matcher = getattr(
+                    self.vision, "find_structural_template", None
+                )
+                if callable(structural_matcher):
+                    match = structural_matcher(
+                        frame_png,
+                        template_png,
+                        search_region=search,
+                        threshold=PC_STRUCTURAL_TEMPLATE_THRESHOLD,
+                    )
+                    if match is not None:
+                        return True
             live = self.vision.phash_png(frame_png, region=region)
             distance = self.vision.phash_distance(live, pc.phash)
             return distance <= pc.phash_tolerance

openadapt_flow/vision/__init__.py

@@ -2,7 +2,7 @@
 
 Public API (see DESIGN.md "Vision API"):
 
-- :class:`Match`, :func:`find_template`
+- :class:`Match`, :func:`find_template`, :func:`find_structural_template`
 - :class:`OcrLine`, :func:`ocr`, :func:`find_text`, :func:`text_present`,
   :func:`upscale_png`
 - :func:`phash_png`, :func:`phash_distance`
@@ -11,7 +11,12 @@
 """
 
 from openadapt_flow.vision.hashing import phash_distance, phash_png
-from openadapt_flow.vision.match import Match, find_template, pixels_changed
+from openadapt_flow.vision.match import (
+    Match,
+    find_structural_template,
+    find_template,
+    pixels_changed,
+)
 from openadapt_flow.vision.ocr import (
     OcrLine,
     find_text,
@@ -24,6 +29,7 @@
 __all__ = [
     "Match",
     "OcrLine",
+    "find_structural_template",
     "find_template",
     "find_text",
     "ocr",

openadapt_flow/vision/match.py

@@ -1,7 +1,9 @@
-"""Multi-scale grayscale template matching.
+"""Multi-scale grayscale and structural-edge template matching.
 
 Implements the `find_template` rung of the resolution ladder using
 ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` over a small scale ladder.
+``find_structural_template`` applies the same matcher to Canny edge maps for
+postconditions whose recorded invariant is layout/content rather than palette.
 All returned coordinates are in *screen* (full-frame) pixel space, even when
 a ``search_region`` restricts the search.
 """
@@ -66,44 +68,22 @@ def _clamp_region(region: Region, width: int, height: int) -> Optional[Region]:
 # (repeated UI structure such as identical empty form fields produces exact
 # score ties at several positions).
 TIE_BREAK_EPS = 1e-3
+# A flat/near-flat edge crop has no discriminative structure and makes
+# normalized correlation degenerate.  Refuse that evidence and let the
+# caller's independent hash/semantic checks decide.
+STRUCTURAL_MIN_EDGE_PIXELS = 16
 
 
-def find_template(
-    screen_png: bytes,
-    template_png: bytes,
+def _find_template_arrays(
+    screen: np.ndarray,
+    template: np.ndarray,
     *,
     search_region: Region | None = None,
     scales: tuple[float, ...] = (0.85, 1.0, 1.18),
     threshold: float = 0.82,
     prefer_near: Point | None = None,
 ) -> Match | None:
-    """Locate a template crop on screen via multi-scale template matching.
-
-    Grayscale ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` is run at each
-    scale (the *template* is resized); the best-scoring location across all
-    scales wins if it clears ``threshold``. Scales at which the resized
-    template would not fit inside the search image are skipped.
-
-    Args:
-        screen_png: Full-frame screenshot as PNG bytes.
-        template_png: Template crop as PNG bytes.
-        search_region: Optional ``(x, y, w, h)`` sub-region of the screen to
-            search within (clamped to screen bounds). Returned coordinates
-            are still full-screen coordinates.
-        scales: Multiplicative scale factors applied to the template.
-        threshold: Minimum ``TM_CCOEFF_NORMED`` score to accept a match.
-        prefer_near: Optional expected match origin ``(x, y)`` in screen
-            coordinates. When several locations score within
-            ``TIE_BREAK_EPS`` of the best (repeated UI structure — e.g. two
-            identical empty inputs), the tie is broken in favor of the
-            location nearest this point instead of raster-scan order.
-
-    Returns:
-        The best :class:`Match` in screen coordinates, or ``None`` if no
-        scale produced a score at or above ``threshold``.
-    """
-    screen = _decode_gray(screen_png)
-    template = _decode_gray(template_png)
+    """Run the shared multi-scale matcher over two single-channel arrays."""
     sh, sw = screen.shape[:2]
 
     off_x, off_y = 0, 0
@@ -153,6 +133,82 @@ def find_template(
     return Match(point=point, region=region, confidence=score)
 
 
+def find_template(
+    screen_png: bytes,
+    template_png: bytes,
+    *,
+    search_region: Region | None = None,
+    scales: tuple[float, ...] = (0.85, 1.0, 1.18),
+    threshold: float = 0.82,
+    prefer_near: Point | None = None,
+) -> Match | None:
+    """Locate a grayscale template crop via multi-scale matching.
+
+    ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` is run at each scale (the
+    *template* is resized); the best-scoring location wins if it clears
+    ``threshold``. Scales at which the resized template would not fit inside
+    the search image are skipped.
+
+    Args:
+        screen_png: Full-frame screenshot as PNG bytes.
+        template_png: Template crop as PNG bytes.
+        search_region: Optional ``(x, y, w, h)`` sub-region of the screen to
+            search within (clamped to screen bounds). Returned coordinates
+            are still full-screen coordinates.
+        scales: Multiplicative scale factors applied to the template.
+        threshold: Minimum ``TM_CCOEFF_NORMED`` score to accept a match.
+        prefer_near: Optional expected match origin ``(x, y)`` in screen
+            coordinates. When several locations score within
+            ``TIE_BREAK_EPS`` of the best (repeated UI structure — e.g. two
+            identical empty inputs), the tie is broken in favor of the
+            location nearest this point instead of raster-scan order.
+
+    Returns:
+        The best :class:`Match` in screen coordinates, or ``None`` if no
+        scale produced a score at or above ``threshold``.
+    """
+    return _find_template_arrays(
+        _decode_gray(screen_png),
+        _decode_gray(template_png),
+        search_region=search_region,
+        scales=scales,
+        threshold=threshold,
+        prefer_near=prefer_near,
+    )
+
+
+def find_structural_template(
+    screen_png: bytes,
+    template_png: bytes,
+    *,
+    search_region: Region | None = None,
+    scales: tuple[float, ...] = (0.85, 1.0, 1.18),
+    threshold: float = 0.8,
+    prefer_near: Point | None = None,
+) -> Match | None:
+    """Locate recorded structure while ignoring foreground/background palette.
+
+    Both images are reduced to Canny edge maps before the same localized,
+    multi-scale normalized-correlation matcher used by :func:`find_template`.
+    A light/dark theme inversion therefore keeps borders and glyph geometry,
+    while a modal, missing panel, or changed layout alters the edge map and
+    remains a failed match. This is intended for ``REGION_STABLE`` outcome
+    checks; target resolution continues to use the stricter grayscale matcher.
+    """
+    screen = cv2.Canny(_decode_gray(screen_png), 50, 150)
+    template = cv2.Canny(_decode_gray(template_png), 50, 150)
+    if int(np.count_nonzero(template)) < STRUCTURAL_MIN_EDGE_PIXELS:
+        return None
+    return _find_template_arrays(
+        screen,
+        template,
+        search_region=search_region,
+        scales=scales,
+        threshold=threshold,
+        prefer_near=prefer_near,
+    )
+
+
 # pixels_changed: threshold below which a per-pixel grayscale difference is
 # attributed to encoder noise, and the minimum count of above-threshold
 # pixels for the region to count as visibly changed. Headless screenshots of

tests/e2e/test_region_stable_theme_regression.py

@@ -0,0 +1,112 @@
+"""Regression for the exact v1.16.1 theme + parameter false halt.
+
+The committed fixture is metadata-only.  This test regenerates the bundled
+synthetic MockMed evidence locally, so no screenshots or third-party
+application source enter the repository or package artifact.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from openadapt_flow.backends.playwright_backend import PlaywrightBackend
+from openadapt_flow.benchmark.dom_arm import verify_final_state
+from openadapt_flow.compiler import compile_recording
+from openadapt_flow.demo_driver import record_triage_demo
+from openadapt_flow.ir import Workflow
+from openadapt_flow.runtime import Replayer
+
+FIXTURE = (
+    Path(__file__).resolve().parents[1]
+    / "fixtures"
+    / "theme-region-stable-overhalt-v1.16.1.json"
+)
+
+
+def _load_regression() -> dict:
+    fixture = json.loads(FIXTURE.read_text())
+    assert fixture["artifact_policy"].startswith("metadata only")
+    assert fixture["observed_count"] == 3
+    assert fixture["outcomes"]["over_halt"] == 3
+    assert len(fixture["replay_notes"]) == fixture["observed_count"]
+    return fixture
+
+
+def _run(bundle: Path, url: str, note: str, run_dir: Path):
+    backend, close = PlaywrightBackend.launch(url)
+    try:
+        report = Replayer(backend).run(
+            Workflow.load(bundle),
+            params={"note": note},
+            bundle_dir=bundle,
+            run_dir=run_dir,
+        )
+        verdict = verify_final_state(backend.screenshot(), note)
+        return report, verdict
+    finally:
+        close()
+
+
+@pytest.fixture(scope="module")
+def regression_bundle(
+    mockmed_url: str, tmp_path_factory: pytest.TempPathFactory
+) -> tuple[dict, Path]:
+    regression = _load_regression()
+    root = tmp_path_factory.mktemp("theme-parameter-regression")
+    recording = record_triage_demo(
+        mockmed_url,
+        root / "recording",
+        note_text=regression["recording_note"],
+    )
+    bundle = root / "bundle"
+    compile_recording(recording, bundle, name="theme-parameter-regression")
+    return regression, bundle
+
+
+@pytest.mark.parametrize("note_index", range(3))
+def test_theme_parameter_substitution_keeps_structural_outcome_evidence(
+    mockmed_url: str,
+    tmp_path: Path,
+    regression_bundle: tuple[dict, Path],
+    note_index: int,
+) -> None:
+    regression, bundle = regression_bundle
+    note = regression["replay_notes"][note_index]
+
+    report, verdict = _run(
+        bundle,
+        f"{mockmed_url}{regression['condition']}",
+        note,
+        tmp_path / "theme-run",
+    )
+
+    assert verdict.success is True
+    assert verdict.wrong_action is False
+    assert report.success is True
+    assert sum(result.drift_oracle_calls for result in report.results) == 0
+    save = next(result for result in report.results if result.step_id == "step_010")
+    assert save.postconditions_ok is True
+
+
+def test_true_region_change_still_refuses_after_structural_theme_match(
+    mockmed_url: str, tmp_path: Path, regression_bundle: tuple[dict, Path]
+) -> None:
+    regression, bundle = regression_bundle
+
+    report, verdict = _run(
+        bundle,
+        f"{mockmed_url}?drift=theme,modal",
+        regression["replay_notes"][0],
+        tmp_path / "theme-modal-run",
+    )
+
+    assert verdict.success is False
+    assert report.success is False
+    assert sum(result.drift_oracle_calls for result in report.results) == 0
+    failure = next(result for result in report.results if not result.ok)
+    assert failure.step_id == regression["failed_step"]
+    assert "region_stable" in (failure.error or "")
+    assert "text_present" in (failure.error or "")

tests/fixtures/theme-region-stable-overhalt-v1.16.1.json

@@ -0,0 +1,35 @@
+{
+  "schema_version": 1,
+  "title": "theme plus parameter substitution caused a region_stable over-halt",
+  "source_eval": {
+    "repository": "OpenAdaptAI/openadapt-evals",
+    "pull_request": 270,
+    "runner_commit": "574bcdf",
+    "runner_sha256": "ac58c0b9a02cfc991144a1f5c7a2814c6b6b748bcba27b87f8e1027ee575970f"
+  },
+  "flow": {
+    "version": "1.16.1",
+    "commit": "113ce992b491576d77236f495b983165ce7a63bd",
+    "wheel_sha256": "c7073283475e7ae722db2478b499d962364851104e09edb71f254ba21c1310cd"
+  },
+  "task": "MockMed triage save",
+  "condition": "?drift=theme",
+  "recording_note": "Exact-current setup 3",
+  "replay_notes": [
+    "Referral letter faxed to cardiology. [A03]",
+    "Home BP log reviewed; stable readings. [A04]",
+    "Flu shot offered; patient declined. [A05]"
+  ],
+  "failed_step": "step_010",
+  "failed_postcondition": "region_stable",
+  "region": [0, 43, 968, 169],
+  "observed_count": 3,
+  "outcomes": {
+    "independent_effect_success": 3,
+    "runtime_reported_success": 0,
+    "over_halt": 3,
+    "silent_incorrect_success": 0,
+    "wrong_action": 0
+  },
+  "artifact_policy": "metadata only; no screenshots or third-party application source"
+}