diff --git a/openadapt_flow/runtime/replayer.py b/openadapt_flow/runtime/replayer.py index 8008fd0b..2ff36197 100644 --- a/openadapt_flow/runtime/replayer.py +++ b/openadapt_flow/runtime/replayer.py @@ -97,6 +97,12 @@ # and the minimum template-match score to accept it. PC_TEMPLATE_SEARCH_PAD = 80 PC_TEMPLATE_THRESHOLD = 0.9 +# REGION_STABLE asserts recorded structure, not palette. The ordinary +# grayscale template matcher remains the first (stricter) check; this edge-map +# threshold is a color-invariant fallback. Exact v1.16.1 MockMed evidence +# measured correct theme+parameter substitution at 0.86 and a true blocking +# modal at 0.24, leaving a deliberate margin on both sides. +PC_STRUCTURAL_TEMPLATE_THRESHOLD = 0.8 # Typed-input verification: size of the "field region" diffed/OCRed around # the focusing click point after a TYPE action. Generous on purpose — the @@ -3666,6 +3672,24 @@ def _postcondition_passes( ) if match is not None: return True + # A region-stability check is about the recorded structure, + # not its light/dark palette. Use the same sealed template + # and localized search on edge maps before falling back to the + # exact-position structural pHash. This preserves the + # postcondition: a blocking modal or changed layout still + # fails; only the representation becomes palette-invariant. + structural_matcher = getattr( + self.vision, "find_structural_template", None + ) + if callable(structural_matcher): + match = structural_matcher( + frame_png, + template_png, + search_region=search, + threshold=PC_STRUCTURAL_TEMPLATE_THRESHOLD, + ) + if match is not None: + return True live = self.vision.phash_png(frame_png, region=region) distance = self.vision.phash_distance(live, pc.phash) return distance <= pc.phash_tolerance diff --git a/openadapt_flow/vision/__init__.py b/openadapt_flow/vision/__init__.py index b25681a1..3fa73474 100644 --- a/openadapt_flow/vision/__init__.py +++ b/openadapt_flow/vision/__init__.py @@ -2,7 +2,7 @@ Public API (see DESIGN.md "Vision API"): -- :class:`Match`, :func:`find_template` +- :class:`Match`, :func:`find_template`, :func:`find_structural_template` - :class:`OcrLine`, :func:`ocr`, :func:`find_text`, :func:`text_present`, :func:`upscale_png` - :func:`phash_png`, :func:`phash_distance` @@ -11,7 +11,12 @@ """ from openadapt_flow.vision.hashing import phash_distance, phash_png -from openadapt_flow.vision.match import Match, find_template, pixels_changed +from openadapt_flow.vision.match import ( + Match, + find_structural_template, + find_template, + pixels_changed, +) from openadapt_flow.vision.ocr import ( OcrLine, find_text, @@ -24,6 +29,7 @@ __all__ = [ "Match", "OcrLine", + "find_structural_template", "find_template", "find_text", "ocr", diff --git a/openadapt_flow/vision/match.py b/openadapt_flow/vision/match.py index 9889105e..7b4c22c0 100644 --- a/openadapt_flow/vision/match.py +++ b/openadapt_flow/vision/match.py @@ -1,7 +1,9 @@ -"""Multi-scale grayscale template matching. +"""Multi-scale grayscale and structural-edge template matching. Implements the `find_template` rung of the resolution ladder using ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` over a small scale ladder. +``find_structural_template`` applies the same matcher to Canny edge maps for +postconditions whose recorded invariant is layout/content rather than palette. All returned coordinates are in *screen* (full-frame) pixel space, even when a ``search_region`` restricts the search. """ @@ -66,44 +68,22 @@ def _clamp_region(region: Region, width: int, height: int) -> Optional[Region]: # (repeated UI structure such as identical empty form fields produces exact # score ties at several positions). TIE_BREAK_EPS = 1e-3 +# A flat/near-flat edge crop has no discriminative structure and makes +# normalized correlation degenerate. Refuse that evidence and let the +# caller's independent hash/semantic checks decide. +STRUCTURAL_MIN_EDGE_PIXELS = 16 -def find_template( - screen_png: bytes, - template_png: bytes, +def _find_template_arrays( + screen: np.ndarray, + template: np.ndarray, *, search_region: Region | None = None, scales: tuple[float, ...] = (0.85, 1.0, 1.18), threshold: float = 0.82, prefer_near: Point | None = None, ) -> Match | None: - """Locate a template crop on screen via multi-scale template matching. - - Grayscale ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` is run at each - scale (the *template* is resized); the best-scoring location across all - scales wins if it clears ``threshold``. Scales at which the resized - template would not fit inside the search image are skipped. - - Args: - screen_png: Full-frame screenshot as PNG bytes. - template_png: Template crop as PNG bytes. - search_region: Optional ``(x, y, w, h)`` sub-region of the screen to - search within (clamped to screen bounds). Returned coordinates - are still full-screen coordinates. - scales: Multiplicative scale factors applied to the template. - threshold: Minimum ``TM_CCOEFF_NORMED`` score to accept a match. - prefer_near: Optional expected match origin ``(x, y)`` in screen - coordinates. When several locations score within - ``TIE_BREAK_EPS`` of the best (repeated UI structure — e.g. two - identical empty inputs), the tie is broken in favor of the - location nearest this point instead of raster-scan order. - - Returns: - The best :class:`Match` in screen coordinates, or ``None`` if no - scale produced a score at or above ``threshold``. - """ - screen = _decode_gray(screen_png) - template = _decode_gray(template_png) + """Run the shared multi-scale matcher over two single-channel arrays.""" sh, sw = screen.shape[:2] off_x, off_y = 0, 0 @@ -153,6 +133,82 @@ def find_template( return Match(point=point, region=region, confidence=score) +def find_template( + screen_png: bytes, + template_png: bytes, + *, + search_region: Region | None = None, + scales: tuple[float, ...] = (0.85, 1.0, 1.18), + threshold: float = 0.82, + prefer_near: Point | None = None, +) -> Match | None: + """Locate a grayscale template crop via multi-scale matching. + + ``cv2.matchTemplate`` with ``TM_CCOEFF_NORMED`` is run at each scale (the + *template* is resized); the best-scoring location wins if it clears + ``threshold``. Scales at which the resized template would not fit inside + the search image are skipped. + + Args: + screen_png: Full-frame screenshot as PNG bytes. + template_png: Template crop as PNG bytes. + search_region: Optional ``(x, y, w, h)`` sub-region of the screen to + search within (clamped to screen bounds). Returned coordinates + are still full-screen coordinates. + scales: Multiplicative scale factors applied to the template. + threshold: Minimum ``TM_CCOEFF_NORMED`` score to accept a match. + prefer_near: Optional expected match origin ``(x, y)`` in screen + coordinates. When several locations score within + ``TIE_BREAK_EPS`` of the best (repeated UI structure — e.g. two + identical empty inputs), the tie is broken in favor of the + location nearest this point instead of raster-scan order. + + Returns: + The best :class:`Match` in screen coordinates, or ``None`` if no + scale produced a score at or above ``threshold``. + """ + return _find_template_arrays( + _decode_gray(screen_png), + _decode_gray(template_png), + search_region=search_region, + scales=scales, + threshold=threshold, + prefer_near=prefer_near, + ) + + +def find_structural_template( + screen_png: bytes, + template_png: bytes, + *, + search_region: Region | None = None, + scales: tuple[float, ...] = (0.85, 1.0, 1.18), + threshold: float = 0.8, + prefer_near: Point | None = None, +) -> Match | None: + """Locate recorded structure while ignoring foreground/background palette. + + Both images are reduced to Canny edge maps before the same localized, + multi-scale normalized-correlation matcher used by :func:`find_template`. + A light/dark theme inversion therefore keeps borders and glyph geometry, + while a modal, missing panel, or changed layout alters the edge map and + remains a failed match. This is intended for ``REGION_STABLE`` outcome + checks; target resolution continues to use the stricter grayscale matcher. + """ + screen = cv2.Canny(_decode_gray(screen_png), 50, 150) + template = cv2.Canny(_decode_gray(template_png), 50, 150) + if int(np.count_nonzero(template)) < STRUCTURAL_MIN_EDGE_PIXELS: + return None + return _find_template_arrays( + screen, + template, + search_region=search_region, + scales=scales, + threshold=threshold, + prefer_near=prefer_near, + ) + + # pixels_changed: threshold below which a per-pixel grayscale difference is # attributed to encoder noise, and the minimum count of above-threshold # pixels for the region to count as visibly changed. Headless screenshots of diff --git a/tests/e2e/test_region_stable_theme_regression.py b/tests/e2e/test_region_stable_theme_regression.py new file mode 100644 index 00000000..c9d2bda8 --- /dev/null +++ b/tests/e2e/test_region_stable_theme_regression.py @@ -0,0 +1,112 @@ +"""Regression for the exact v1.16.1 theme + parameter false halt. + +The committed fixture is metadata-only. This test regenerates the bundled +synthetic MockMed evidence locally, so no screenshots or third-party +application source enter the repository or package artifact. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from openadapt_flow.backends.playwright_backend import PlaywrightBackend +from openadapt_flow.benchmark.dom_arm import verify_final_state +from openadapt_flow.compiler import compile_recording +from openadapt_flow.demo_driver import record_triage_demo +from openadapt_flow.ir import Workflow +from openadapt_flow.runtime import Replayer + +FIXTURE = ( + Path(__file__).resolve().parents[1] + / "fixtures" + / "theme-region-stable-overhalt-v1.16.1.json" +) + + +def _load_regression() -> dict: + fixture = json.loads(FIXTURE.read_text()) + assert fixture["artifact_policy"].startswith("metadata only") + assert fixture["observed_count"] == 3 + assert fixture["outcomes"]["over_halt"] == 3 + assert len(fixture["replay_notes"]) == fixture["observed_count"] + return fixture + + +def _run(bundle: Path, url: str, note: str, run_dir: Path): + backend, close = PlaywrightBackend.launch(url) + try: + report = Replayer(backend).run( + Workflow.load(bundle), + params={"note": note}, + bundle_dir=bundle, + run_dir=run_dir, + ) + verdict = verify_final_state(backend.screenshot(), note) + return report, verdict + finally: + close() + + +@pytest.fixture(scope="module") +def regression_bundle( + mockmed_url: str, tmp_path_factory: pytest.TempPathFactory +) -> tuple[dict, Path]: + regression = _load_regression() + root = tmp_path_factory.mktemp("theme-parameter-regression") + recording = record_triage_demo( + mockmed_url, + root / "recording", + note_text=regression["recording_note"], + ) + bundle = root / "bundle" + compile_recording(recording, bundle, name="theme-parameter-regression") + return regression, bundle + + +@pytest.mark.parametrize("note_index", range(3)) +def test_theme_parameter_substitution_keeps_structural_outcome_evidence( + mockmed_url: str, + tmp_path: Path, + regression_bundle: tuple[dict, Path], + note_index: int, +) -> None: + regression, bundle = regression_bundle + note = regression["replay_notes"][note_index] + + report, verdict = _run( + bundle, + f"{mockmed_url}{regression['condition']}", + note, + tmp_path / "theme-run", + ) + + assert verdict.success is True + assert verdict.wrong_action is False + assert report.success is True + assert sum(result.drift_oracle_calls for result in report.results) == 0 + save = next(result for result in report.results if result.step_id == "step_010") + assert save.postconditions_ok is True + + +def test_true_region_change_still_refuses_after_structural_theme_match( + mockmed_url: str, tmp_path: Path, regression_bundle: tuple[dict, Path] +) -> None: + regression, bundle = regression_bundle + + report, verdict = _run( + bundle, + f"{mockmed_url}?drift=theme,modal", + regression["replay_notes"][0], + tmp_path / "theme-modal-run", + ) + + assert verdict.success is False + assert report.success is False + assert sum(result.drift_oracle_calls for result in report.results) == 0 + failure = next(result for result in report.results if not result.ok) + assert failure.step_id == regression["failed_step"] + assert "region_stable" in (failure.error or "") + assert "text_present" in (failure.error or "") diff --git a/tests/fixtures/theme-region-stable-overhalt-v1.16.1.json b/tests/fixtures/theme-region-stable-overhalt-v1.16.1.json new file mode 100644 index 00000000..41bc4a1d --- /dev/null +++ b/tests/fixtures/theme-region-stable-overhalt-v1.16.1.json @@ -0,0 +1,35 @@ +{ + "schema_version": 1, + "title": "theme plus parameter substitution caused a region_stable over-halt", + "source_eval": { + "repository": "OpenAdaptAI/openadapt-evals", + "pull_request": 270, + "runner_commit": "574bcdf", + "runner_sha256": "ac58c0b9a02cfc991144a1f5c7a2814c6b6b748bcba27b87f8e1027ee575970f" + }, + "flow": { + "version": "1.16.1", + "commit": "113ce992b491576d77236f495b983165ce7a63bd", + "wheel_sha256": "c7073283475e7ae722db2478b499d962364851104e09edb71f254ba21c1310cd" + }, + "task": "MockMed triage save", + "condition": "?drift=theme", + "recording_note": "Exact-current setup 3", + "replay_notes": [ + "Referral letter faxed to cardiology. [A03]", + "Home BP log reviewed; stable readings. [A04]", + "Flu shot offered; patient declined. [A05]" + ], + "failed_step": "step_010", + "failed_postcondition": "region_stable", + "region": [0, 43, 968, 169], + "observed_count": 3, + "outcomes": { + "independent_effect_success": 3, + "runtime_reported_success": 0, + "over_halt": 3, + "silent_incorrect_success": 0, + "wrong_action": 0 + }, + "artifact_policy": "metadata only; no screenshots or third-party application source" +} diff --git a/tests/test_replayer.py b/tests/test_replayer.py index a5829383..17f3a247 100644 --- a/tests/test_replayer.py +++ b/tests/test_replayer.py @@ -51,6 +51,8 @@ class FakeVision: def __init__(self): self.template_results: list = [] self.template_calls: list = [] + self.structural_template_results: list = [] + self.structural_template_calls: list = [] # Template-crop bytes the resolver handed each find_template call # (the decrypted in-memory crop for an encrypted bundle). self.template_png_calls: list = [] @@ -84,6 +86,21 @@ def find_template( return self.template_results.pop(0) return None + def find_structural_template( + self, + screen_png, + template_png, + *, + search_region=None, + prefer_near=None, + scales=(0.85, 1.0, 1.18), + threshold=0.8, + ): + self.structural_template_calls.append(search_region) + if self.structural_template_results: + return self.structural_template_results.pop(0) + return None + def find_text(self, screen_png, text, *, region=None, min_ratio=0.8): self.text_calls.append(text) result = self.text_results.get(text) @@ -575,6 +592,44 @@ def test_region_stable_template_tolerates_layout_shift(bundle, run_dir): assert vision.template_calls[0] is not None +def test_region_stable_structural_template_tolerates_theme(bundle, run_dir): + """A strict edge-map match rescues palette drift when grayscale and the + exact-position pHash miss; the REGION_STABLE evidence remains armed.""" + vision = FakeVision() + vision.phash_dist = 99 + vision.structural_template_results = [ + Match(point=(120, 60), region=(80, 48, 100, 40), confidence=0.86) + ] + (bundle / "templates" / "pc.png").write_bytes(make_png((100, 40))) + pc = Postcondition( + kind=PostconditionKind.REGION_STABLE, + region=(80, 40, 100, 40), + phash="aa", + template="templates/pc.png", + timeout_s=0.2, + ) + workflow = Workflow( + name="wf", + steps=[ + Step( + id="k1", + intent="press enter", + action=ActionKind.KEY, + key="Enter", + expect=[pc], + ) + ], + ) + + report = Replayer(FakeBackend(), vision=vision, poll_interval_s=0.01).run( + workflow, bundle_dir=bundle, run_dir=run_dir + ) + + assert report.success is True + assert vision.template_calls + assert vision.structural_template_calls + + def test_region_stable_fails_when_template_and_phash_miss(bundle, run_dir): vision = FakeVision() vision.phash_dist = 99 diff --git a/tests/test_vision.py b/tests/test_vision.py index b82fbab6..0bd9637c 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -15,6 +15,7 @@ from openadapt_flow.vision import ( Match, + find_structural_template, find_template, find_text, ocr, @@ -139,6 +140,66 @@ def test_threshold_is_honored(self) -> None: assert m is not None and m.confidence < 0.82 +class TestFindStructuralTemplate: + def test_palette_inversion_preserves_structure(self) -> None: + recorded = blank(420, 220) + draw_button(recorded, 120, 80, 160, 48, "Save Encounter") + x, y, w, h = (105, 65, 190, 78) + template = to_png(recorded[y : y + h, x : x + w]) + themed = 255 - recorded + + # The ordinary grayscale matcher correctly rejects the palette flip; + # REGION_STABLE's edge representation recognizes the same structure. + assert ( + find_template( + to_png(themed), + template, + search_region=(80, 40, 260, 140), + threshold=0.9, + ) + is None + ) + match = find_structural_template( + to_png(themed), + template, + search_region=(80, 40, 260, 140), + threshold=0.8, + ) + assert match is not None + assert match.confidence > 0.95 + assert abs(match.region[0] - x) <= 1 + assert abs(match.region[1] - y) <= 1 + + def test_true_region_change_is_not_rescued(self) -> None: + recorded = blank(420, 220) + draw_button(recorded, 120, 80, 160, 48, "Save Encounter") + x, y, w, h = (105, 65, 190, 78) + template = to_png(recorded[y : y + h, x : x + w]) + changed = blank(420, 220, gray=30) + cv2.rectangle(changed, (100, 60), (300, 150), (220, 220, 220), -1) + + assert ( + find_structural_template( + to_png(changed), + template, + search_region=(80, 40, 260, 140), + threshold=0.8, + ) + is None + ) + + def test_flat_template_is_unverifiable_not_a_match(self) -> None: + assert ( + find_structural_template( + to_png(blank(420, 220, gray=30)), + to_png(blank(190, 78, gray=220)), + search_region=(80, 40, 260, 140), + threshold=0.8, + ) + is None + ) + + # -- ocr / find_text ----------------------------------------------------------