diff --git a/README.md b/README.md index bd05684..0758f4c 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,7 @@ from openadapt_capture import CaptureSession with CaptureSession.load("./my-capture") as capture: for action in capture.actions(): print(action.timestamp, action.type, action.x, action.y) + print(action.structural_observation) frame = action.screenshot ``` @@ -136,6 +137,27 @@ through a pipe, the selected video encoder, MP4 demuxing/muxing, PNG decoding/encoding, the `image2pipe` muxer, and the `select` video filter; Desktop provisions and probes that exact closure. +## Native structural observations + +On Windows, the recorder can retain a versioned UI Automation observation +beside the native action that produced it. When UIA exposes the information, +the observation includes the target's AutomationId, role/control type, name, +bounds, supported patterns, process/window identity, ancestry, and exact +candidate cardinality within the top-level window. Unavailable values remain +absent; capture never infers structural fields from coordinates or pixels. + +This evidence is stored in `recording.db`, exposed on raw events and processed +`Action` objects, and remains optional so existing recordings and non-Windows +hosts continue to load unchanged. It is enabled by default on Windows and can +be disabled with +`Recorder(..., capture_structural_observations=False)`. Applications can inject +another read-only observer through `Recorder(..., structural_observer=...)` +using the public `StructuralObserver` protocol. + +UIA describes the local Windows accessibility tree. It does not cross an +RDP/Citrix pixel boundary into the remote application; those demonstrations +retain window-scoped pixels and coordinates for Flow's remote visual compiler. + ## Window-scoped recording **Status: implemented and unit-proven on all CI platforms; live-validated @@ -206,11 +228,11 @@ Capture does not upload a session by default. The sharing command, remote transcription, and profiling transfer are explicit opt-in operations. Installing the `privacy` extra alone does not automatically scrub a recording. -The current desktop-to-Flow conversion has no field geometry for reliable -secret redaction and no live UIA locator. `openadapt-flow` therefore refuses its -desktop `--secret` option, and converted desktop workflows rely on retained -visual evidence unless a separate structural recording path arms them. Review -the desktop guide before recording sensitive workflows. +Structural observations can also contain sensitive control names, window +titles, and accessibility ancestry. They stay in the same local raw-capture +boundary. `openadapt-flow` still refuses desktop `--secret` authoring until its +source-time field-redaction contract can prove that sensitive values were not +retained. Review the desktop guide before recording sensitive workflows. The experimental Chrome extension can observe pages across its configured host permissions and can emit DOM text and keyboard events to a local WebSocket. @@ -220,8 +242,8 @@ Treat it as development code; do not deploy it in a sensitive browser profile. - Native recording requires a visible user session plus the operating system's screen-recording and input-monitoring permissions. -- Desktop capture records pixels and coordinates, not a structural accessibility - locator for each demonstrated target. +- Native Windows capture retains UIA evidence when the application exposes it; + opaque remote applications still require Flow's visual and OCR bindings. - The Flow adapter rejects unsupported input such as drag, non-left-click, and modifier-chord actions instead of silently compiling an incomplete workflow. - Browser-extension installation, security hardening, and compiler integration diff --git a/openadapt_capture/__init__.py b/openadapt_capture/__init__.py index 3fca6ff..79f28fd 100644 --- a/openadapt_capture/__init__.py +++ b/openadapt_capture/__init__.py @@ -83,6 +83,20 @@ PerfStat, plot_capture_performance, ) +from openadapt_capture.structural import ( + STRUCTURAL_OBSERVATION_SCHEMA_VERSION, + StructuralAncestor, + StructuralBounds, + StructuralCandidateContext, + StructuralElement, + StructuralObservation, + StructuralObservationRequest, + StructuralObserver, + StructuralProcessIdentity, + StructuralWindowIdentity, + create_structural_observer, + observe_structural_action, +) # Visualization from openadapt_capture.visualize import create_demo, create_html @@ -133,6 +147,19 @@ "Capture", "CaptureSession", "Action", + # Native structural observation + "STRUCTURAL_OBSERVATION_SCHEMA_VERSION", + "StructuralAncestor", + "StructuralBounds", + "StructuralCandidateContext", + "StructuralElement", + "StructuralObservation", + "StructuralObservationRequest", + "StructuralObserver", + "StructuralProcessIdentity", + "StructuralWindowIdentity", + "create_structural_observer", + "observe_structural_action", # Window-scoped capture "WindowTarget", "TargetWindow", diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 0df400e..b4a229b 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -36,6 +36,7 @@ MouseUpEvent, ) from openadapt_capture.processing import process_events +from openadapt_capture.structural import StructuralObservation if TYPE_CHECKING: from PIL import Image @@ -43,6 +44,17 @@ from openadapt_capture.browser_events import BrowserEvent +def _parse_structural_observation(raw: object) -> StructuralObservation | None: + """Load optional structural evidence without breaking legacy recordings.""" + + if raw is None: + return None + try: + return StructuralObservation.model_validate(raw) + except Exception: + return None + + def _convert_action_event(db_event) -> PydanticActionEvent | None: """Convert a SQLAlchemy ActionEvent to a Pydantic event. @@ -52,11 +64,16 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: Returns: Pydantic event or None if unrecognized. """ - ts = db_event.timestamp + common = { + "timestamp": db_event.timestamp, + "structural_observation": _parse_structural_observation( + getattr(db_event, "structural_observation", None) + ), + } if db_event.name == "move": return MouseMoveEvent( - timestamp=ts, + **common, x=db_event.mouse_x or 0, y=db_event.mouse_y or 0, ) @@ -65,14 +82,14 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: if db_event.mouse_pressed is True: return MouseDownEvent( - timestamp=ts, + **common, x=db_event.mouse_x or 0, y=db_event.mouse_y or 0, button=button, ) elif db_event.mouse_pressed is False: return MouseUpEvent( - timestamp=ts, + **common, x=db_event.mouse_x or 0, y=db_event.mouse_y or 0, button=button, @@ -81,7 +98,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: return None elif db_event.name == "scroll": return MouseScrollEvent( - timestamp=ts, + **common, x=db_event.mouse_x or 0, y=db_event.mouse_y or 0, dx=db_event.mouse_dx or 0, @@ -89,7 +106,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: ) elif db_event.name == "press": return KeyDownEvent( - timestamp=ts, + **common, key_name=db_event.key_name, key_char=db_event.key_char, key_vk=db_event.key_vk, @@ -99,7 +116,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: ) elif db_event.name == "release": return KeyUpEvent( - timestamp=ts, + **common, key_name=db_event.key_name, key_char=db_event.key_char, key_vk=db_event.key_vk, @@ -369,6 +386,11 @@ def button(self) -> str | None: return btn.value if hasattr(btn, "value") else str(btn) return None + @property + def structural_observation(self) -> StructuralObservation | None: + """Accessibility evidence captured at this action, when available.""" + return self.event.structural_observation + @property def screenshot(self) -> "Image" | None: """Get the screenshot at the time of this action. diff --git a/openadapt_capture/config.py b/openadapt_capture/config.py index ba0e7c8..f5fd54d 100644 --- a/openadapt_capture/config.py +++ b/openadapt_capture/config.py @@ -34,6 +34,9 @@ class Settings(BaseSettings): # Record and replay (from legacy OpenAdapt config.defaults.json) RECORD_WINDOW_DATA: bool = False RECORD_READ_ACTIVE_ELEMENT_STATE: bool = False + # Retain live native accessibility evidence beside actionable input. + # The platform factory is a no-op outside Windows. + RECORD_STRUCTURAL_OBSERVATIONS: bool = True RECORD_VIDEO: bool = True RECORD_AUDIO: bool = False RECORD_BROWSER_EVENTS: bool = False @@ -99,6 +102,7 @@ class Settings(BaseSettings): "capture_audio": "RECORD_AUDIO", "capture_images": "RECORD_IMAGES", "capture_window_data": "RECORD_WINDOW_DATA", + "capture_structural_observations": "RECORD_STRUCTURAL_OBSERVATIONS", "capture_browser_events": "RECORD_BROWSER_EVENTS", "window_owner": "RECORD_WINDOW_OWNER", "window_title": "RECORD_WINDOW_TITLE", @@ -124,6 +128,7 @@ class RecordingConfig: capture_audio: bool | None = None capture_images: bool | None = None capture_window_data: bool | None = None + capture_structural_observations: bool | None = None capture_browser_events: bool | None = None # Window-scoped capture selectors (see window_capture.WindowTarget). window_owner: str | None = None diff --git a/openadapt_capture/db/models.py b/openadapt_capture/db/models.py index 60c0768..59a9dc1 100644 --- a/openadapt_capture/db/models.py +++ b/openadapt_capture/db/models.py @@ -124,6 +124,9 @@ class ActionEvent(Base): canonical_key_vk = sa.Column(sa.String) parent_id = sa.Column(sa.Integer, sa.ForeignKey("action_event.id")) element_state = sa.Column(sa.JSON) + # Versioned optional accessibility evidence captured at action time. + # Nullable + additive migration keep older recording.db files readable. + structural_observation = sa.Column(sa.JSON) disabled = sa.Column(sa.Boolean, default=False) children = sa.orm.relationship("ActionEvent") diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index ac5ae5b..e22986f 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -12,6 +12,8 @@ from pydantic import BaseModel, Field +from openadapt_capture.structural import StructuralObservation + class EventType(str, Enum): """Event type identifiers.""" @@ -62,12 +64,21 @@ class BaseEvent(BaseModel): model_config = {"use_enum_values": True} +class ActionBaseEvent(BaseEvent): + """Base event for native actions with optional structural evidence.""" + + structural_observation: StructuralObservation | None = Field( + default=None, + description="Versioned accessibility evidence observed at action time", + ) + + # ============================================================================= # Mouse Events # ============================================================================= -class MouseMoveEvent(BaseEvent): +class MouseMoveEvent(ActionBaseEvent): """Mouse cursor movement event. Corresponds to OpenAdapt's ActionEvent with name="move". @@ -78,7 +89,7 @@ class MouseMoveEvent(BaseEvent): y: float = Field(description="Mouse Y position in pixels") -class MouseDownEvent(BaseEvent): +class MouseDownEvent(ActionBaseEvent): """Mouse button press event. Corresponds to OpenAdapt's ActionEvent with name="click" and mouse_pressed=True. @@ -90,7 +101,7 @@ class MouseDownEvent(BaseEvent): button: str = Field(description="Native mouse button name") -class MouseUpEvent(BaseEvent): +class MouseUpEvent(ActionBaseEvent): """Mouse button release event. Corresponds to OpenAdapt's ActionEvent with name="click" and mouse_pressed=False. @@ -102,7 +113,7 @@ class MouseUpEvent(BaseEvent): button: str = Field(description="Native mouse button name") -class MouseScrollEvent(BaseEvent): +class MouseScrollEvent(ActionBaseEvent): """Mouse scroll wheel event. Corresponds to OpenAdapt's ActionEvent with name="scroll". @@ -120,7 +131,7 @@ class MouseScrollEvent(BaseEvent): # ============================================================================= -class KeyDownEvent(BaseEvent): +class KeyDownEvent(ActionBaseEvent): """Keyboard key press event. Corresponds to OpenAdapt's ActionEvent with name="press". @@ -135,7 +146,7 @@ class KeyDownEvent(BaseEvent): canonical_key_vk: str | None = Field(default=None, description="Canonical virtual key code") -class KeyUpEvent(BaseEvent): +class KeyUpEvent(ActionBaseEvent): """Keyboard key release event. Corresponds to OpenAdapt's ActionEvent with name="release". @@ -192,7 +203,7 @@ class AudioChunkEvent(BaseEvent): # ============================================================================= -class MouseClickEvent(BaseEvent): +class MouseClickEvent(ActionBaseEvent): """Combined mouse click event (down + up). Corresponds to OpenAdapt's ActionEvent with name="singleclick". @@ -208,7 +219,7 @@ class MouseClickEvent(BaseEvent): ) -class MouseDoubleClickEvent(BaseEvent): +class MouseDoubleClickEvent(ActionBaseEvent): """Double click event. Corresponds to OpenAdapt's ActionEvent with name="doubleclick". @@ -224,7 +235,7 @@ class MouseDoubleClickEvent(BaseEvent): ) -class MouseDragEvent(BaseEvent): +class MouseDragEvent(ActionBaseEvent): """Mouse drag event (down + moves + up). Uses x/y for start position and dx/dy for displacement (like MouseScrollEvent). @@ -242,7 +253,7 @@ class MouseDragEvent(BaseEvent): ) -class KeyTypeEvent(BaseEvent): +class KeyTypeEvent(ActionBaseEvent): """Sequence of typed characters. Corresponds to OpenAdapt's ActionEvent with name="type". diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index 88a819d..8c31289 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -220,32 +220,64 @@ def _delivery_main(self) -> None: ) ) return - while True: - if ( - self._delivery_stop_requested.is_set() - and self._delivery_queue.empty() - ): - return - try: - item = self._delivery_queue.get(timeout=0.1) - except queue.Empty: - continue - try: - if item is self._delivery_sentinel: + start_hook = getattr( + self.callback, + "_openadapt_delivery_thread_start", + None, + ) + stop_hook = getattr( + self.callback, + "_openadapt_delivery_thread_stop", + None, + ) + try: + if callable(start_hook): + start_hook() + while True: + if ( + self._delivery_stop_requested.is_set() + and self._delivery_queue.empty() + ): return - self.callback(item) # type: ignore[arg-type] - except BaseException as exc: - failure = ( - exc - if isinstance(exc, InputObserverError) - else InputObserverError( - f"{type(self).__name__} input consumer failed: {exc}" + try: + item = self._delivery_queue.get(timeout=0.1) + except queue.Empty: + continue + try: + if item is self._delivery_sentinel: + return + self.callback(item) # type: ignore[arg-type] + except BaseException as exc: + failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError( + f"{type(self).__name__} input consumer failed: {exc}" + ) ) + self._fail(failure) + return + finally: + self._delivery_queue.task_done() + except BaseException as exc: + failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError( + f"{type(self).__name__} input delivery setup failed: {exc}" ) - self._fail(failure) - return - finally: - self._delivery_queue.task_done() + ) + self._fail(failure) + finally: + if callable(stop_hook): + try: + stop_hook() + except BaseException as exc: + self._fail( + InputObserverError( + f"{type(self).__name__} input delivery cleanup failed: {exc}" + ) + ) def _discard_delivery_queue(self) -> None: """Discard every event buffered by a startup that did not commit.""" diff --git a/openadapt_capture/processing.py b/openadapt_capture/processing.py index 2dd5d3f..f30feb6 100644 --- a/openadapt_capture/processing.py +++ b/openadapt_capture/processing.py @@ -22,6 +22,7 @@ MouseScrollEvent, MouseUpEvent, ) +from openadapt_capture.structural import StructuralObservation # Type variable for event types E = TypeVar("E", bound=Event) @@ -38,6 +39,17 @@ KEY_TYPE_MERGE_INTERVAL_SECONDS = 0.5 # merge adjacent KeyTypeEvents within this interval +def _first_structural_observation( + events: list[ActionEvent], +) -> StructuralObservation | None: + """Retain the first action-time observation when events are merged.""" + + for event in events: + if event.structural_observation is not None: + return event.structural_observation + return None + + # ============================================================================= # Event Processing Functions # ============================================================================= @@ -162,6 +174,9 @@ def flush_buffer() -> None: timestamp=first_event.timestamp, text=text, children=list(keyboard_buffer), + structural_observation=_first_structural_observation( + list(keyboard_buffer) + ), ) result.append(type_event) @@ -221,6 +236,9 @@ def flush_buffer() -> None: timestamp=first.timestamp, x=last.x, y=last.y, + structural_observation=_first_structural_observation( + list(move_buffer) + ), ) result.append(merged) @@ -270,6 +288,9 @@ def flush_buffer() -> None: y=first.y, dx=total_dx, dy=total_dy, + structural_observation=_first_structural_observation( + list(scroll_buffer) + ), ) result.append(merged) @@ -374,6 +395,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: y=down.y, button=down.button, children=[down, up, next_down, next_up], + structural_observation=_first_structural_observation( + [down, up, next_down, next_up] + ), ) result.append(double_click) skip_timestamps.add(up.timestamp) @@ -388,6 +412,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: y=down.y, button=down.button, children=[down, up], + structural_observation=_first_structural_observation( + [down, up] + ), ) result.append(single_click) skip_timestamps.add(up.timestamp) @@ -452,6 +479,9 @@ def calculate_distance(x1: float, y1: float, x2: float, y2: float) -> float: dy=event.y - down_event.y, button=down_event.button, children=[down_event] + moves + [event], + structural_observation=_first_structural_observation( + [down_event] + moves + [event] + ), ) result.append(drag) else: diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 8843529..9bb150a 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -54,6 +54,12 @@ add_exception_note, create_input_observer, ) +from openadapt_capture.structural import ( + StructuralObservationRequest, + StructuralObserver, + create_structural_observer, + observe_structural_action, +) from openadapt_capture.window_capture import ( WindowCaptureError, WindowCaptureScope, @@ -837,6 +843,7 @@ def trigger_action_event( action_event_args: dict[str, Any], window_scope: WindowCaptureScope | None = None, timestamp: float | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: """Triggers an action event and adds it to the event queue. @@ -849,12 +856,29 @@ def trigger_action_event( frames directly. timestamp: Native event-receipt time. Defaults to the current recording clock only for legacy/direct callers. + structural_observer: Optional accessibility observer. Evidence is + captured against global coordinates before any window translation. Returns: None """ + event_timestamp = utils.get_timestamp() if timestamp is None else timestamp x = action_event_args.get("mouse_x") y = action_event_args.get("mouse_y") + observation = observe_structural_action( + structural_observer, + StructuralObservationRequest( + event_timestamp=event_timestamp, + action_name=str(action_event_args.get("name") or "unknown"), + x=x, + y=y, + ), + ) + if observation is not None: + action_event_args["structural_observation"] = observation.model_dump( + mode="json", + exclude_none=True, + ) if x is not None and y is not None: if config.RECORD_READ_ACTIVE_ELEMENT_STATE: # element lookup needs GLOBAL coordinates: translate afterwards. @@ -874,7 +898,7 @@ def trigger_action_event( action_event_args["mouse_y"] = wy event_q.put( Event( - utils.get_timestamp() if timestamp is None else timestamp, + event_timestamp, "action", action_event_args, ) @@ -920,6 +944,7 @@ def on_click( pressed: bool, injected: bool = False, timestamp: float | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: """Handles the 'click' event. @@ -948,6 +973,7 @@ def on_click( }, window_scope, timestamp, + structural_observer if pressed else None, ) @@ -960,6 +986,7 @@ def on_scroll( dy: float, injected: bool = False, timestamp: float | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: """Handles the 'scroll' event. @@ -988,12 +1015,14 @@ def on_scroll( }, window_scope, timestamp, + structural_observer, ) def handle_key( event_q: queue.Queue, key: ObservedKey, + structural_observer: StructuralObserver | None = None, ) -> None: """Persist a normalized native key transition. @@ -1016,6 +1045,7 @@ def handle_key( "canonical_key_vk": key.canonical_key_vk, }, timestamp=key.timestamp, + structural_observer=structural_observer if key.pressed else None, ) @@ -1309,6 +1339,7 @@ def read_input_events( recording: Recording, started_event: threading.Event, window_scope: WindowCaptureScope | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: """Read globally ordered keyboard and mouse events from one native observer.""" stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] @@ -1335,6 +1366,7 @@ def on_observed(event: ObservedInput) -> None: event.pressed, event.injected, timestamp=event.timestamp, + structural_observer=structural_observer, ) return if isinstance(event, ObservedMouseScroll): @@ -1347,13 +1379,14 @@ def on_observed(event: ObservedInput) -> None: event.dy, event.injected, timestamp=event.timestamp, + structural_observer=structural_observer, ) return if event.injected: return logger.debug(f"{event=}") - handle_key(event_q, event) + handle_key(event_q, event, structural_observer) if not event.pressed: return @@ -1377,6 +1410,14 @@ def on_observed(event: ObservedInput) -> None: logger.info("Stop sequence entered! Stopping recording now.") stop_sequence_detected = True + if structural_observer is not None: + start_hook = getattr(structural_observer, "open_current_thread", None) + stop_hook = getattr(structural_observer, "close_current_thread", None) + if callable(start_hook): + setattr(on_observed, "_openadapt_delivery_thread_start", start_hook) + if callable(stop_hook): + setattr(on_observed, "_openadapt_delivery_thread_stop", stop_hook) + utils.set_start_time(recording.timestamp) observer = create_input_observer( on_observed, @@ -1625,6 +1666,7 @@ def record( send_profile: bool = False, window_owner: str | None = None, window_title: str | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: """Record Screenshots/ActionEvents/WindowEvents/BrowserEvents. @@ -1640,6 +1682,9 @@ def record( ``config.RECORD_WINDOW_OWNER``. window_title: Title substring for window-scoped capture. Falls back to ``config.RECORD_WINDOW_TITLE``. + structural_observer: Optional injected accessibility observer. When + omitted, the platform factory follows + ``RECORD_STRUCTURAL_OBSERVATIONS``. """ assert config.RECORD_VIDEO or config.RECORD_IMAGES, ( config.RECORD_VIDEO, @@ -1687,6 +1732,11 @@ def record( f"initial frame {initial_window_frame.size}" ) + if structural_observer is None: + structural_observer = create_structural_observer( + enabled=config.RECORD_STRUCTURAL_OBSERVATIONS, + ) + if capture_dir is None: capture_dir = os.path.join(os.getcwd(), "capture") recording, db_path = create_recording( @@ -1769,6 +1819,7 @@ def record( recording, task_started_events.setdefault("input_event_reader", threading.Event()), window_scope, + structural_observer, ) input_event_reader = threading.Thread( target=_run_task_fail_loud, @@ -2176,6 +2227,7 @@ def __init__( capture_audio: bool | None = None, capture_images: bool | None = None, capture_window_data: bool | None = None, + capture_structural_observations: bool | None = None, capture_browser_events: bool | None = None, capture_full_video: bool | None = None, video_encoding: str | None = None, @@ -2190,6 +2242,7 @@ def __init__( screen_capture_fps: float | None = None, send_profile: bool = False, window: dict | None = None, + structural_observer: StructuralObserver | None = None, ) -> None: from pathlib import Path @@ -2209,6 +2262,7 @@ def __init__( capture_audio=capture_audio, capture_images=capture_images, capture_window_data=capture_window_data, + capture_structural_observations=capture_structural_observations, capture_browser_events=capture_browser_events, capture_full_video=capture_full_video, video_encoding=video_encoding, @@ -2246,6 +2300,7 @@ def __init__( self._capture = None # lazy CaptureSession self._worker_error: BaseException | None = None self._worker_error_lock = threading.Lock() + self._structural_observer = structural_observer def _drain_status_pipe(self) -> None: """Background thread that reads status messages from record().""" @@ -2281,6 +2336,7 @@ def _run_record(self) -> None: num_browser_events=self._num_browser_events, num_video_events=self._num_video_events, send_profile=self._send_profile, + structural_observer=self._structural_observer, ) except BaseException as exc: # A setup exception must wake wait_for_ready() and let context-manager diff --git a/openadapt_capture/structural.py b/openadapt_capture/structural.py new file mode 100644 index 0000000..e0a053f --- /dev/null +++ b/openadapt_capture/structural.py @@ -0,0 +1,198 @@ +"""Versioned structural observations captured beside native input events. + +Structural observations are optional evidence. They describe what the native +accessibility API exposed at action time; absent values stay absent rather than +being guessed from pixels, coordinates, or neighboring controls. +""" + +from __future__ import annotations + +import logging +import sys +from dataclasses import dataclass +from typing import Literal, Protocol, runtime_checkable + +from pydantic import BaseModel, ConfigDict, Field + +STRUCTURAL_OBSERVATION_SCHEMA_VERSION = ( + "openadapt.capture.structural-observation/v1" +) + +_logger = logging.getLogger(__name__) + + +class StructuralBounds(BaseModel): + """Screen-space bounds reported by the accessibility provider.""" + + model_config = ConfigDict(extra="forbid") + + left: float + top: float + right: float + bottom: float + + +class StructuralElement(BaseModel): + """Stable and semantic fields exposed for one accessibility element.""" + + model_config = ConfigDict(extra="forbid") + + automation_id: str | None = None + role: str | None = None + role_source: Literal["pywinauto_friendly_class"] | None = None + control_type: str | None = None + name: str | None = None + class_name: str | None = None + framework_id: str | None = None + native_window_handle: int | None = None + bounds: StructuralBounds | None = None + supported_patterns: list[str] | None = None + + +class StructuralAncestor(BaseModel): + """One parent in the target's accessibility ancestry.""" + + model_config = ConfigDict(extra="forbid") + + automation_id: str | None = None + role: str | None = None + role_source: Literal["pywinauto_friendly_class"] | None = None + control_type: str | None = None + name: str | None = None + class_name: str | None = None + bounds: StructuralBounds | None = None + + +class StructuralProcessIdentity(BaseModel): + """Identity of the process that owned the observed element.""" + + model_config = ConfigDict(extra="forbid") + + process_id: int | None = Field(default=None, ge=0) + process_name: str | None = None + + +class StructuralWindowIdentity(BaseModel): + """Identity of the target's top-level window.""" + + model_config = ConfigDict(extra="forbid") + + title: str | None = None + automation_id: str | None = None + class_name: str | None = None + native_window_handle: int | None = None + bounds: StructuralBounds | None = None + + +class StructuralCandidateContext(BaseModel): + """How candidate cardinality was measured.""" + + model_config = ConfigDict(extra="forbid") + + scope: Literal["top_level_window"] + matched_fields: list[ + Literal["automation_id", "control_type", "name"] + ] = Field(min_length=1) + + +class StructuralObservation(BaseModel): + """UI structure retained beside one native action event.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: Literal[ + "openadapt.capture.structural-observation/v1" + ] = STRUCTURAL_OBSERVATION_SCHEMA_VERSION + provider: Literal["windows_uia"] + event_timestamp: float + observed_at: float + query_kind: Literal["point", "focused"] + element: StructuralElement + process: StructuralProcessIdentity | None = None + window: StructuralWindowIdentity | None = None + ancestry: list[StructuralAncestor] | None = None + candidate_count: int | None = Field(default=None, ge=0) + candidate_context: StructuralCandidateContext | None = None + + +@dataclass(frozen=True) +class StructuralObservationRequest: + """An action-time request passed to a platform structural observer.""" + + event_timestamp: float + action_name: str + x: float | None = None + y: float | None = None + + +@runtime_checkable +class StructuralObserver(Protocol): + """Injectable interface for platform accessibility observation.""" + + def observe( + self, + request: StructuralObservationRequest, + ) -> StructuralObservation | None: + """Return provider evidence for the action, or ``None`` if unavailable.""" + ... + + +def create_structural_observer( + *, + enabled: bool = True, + platform_name: str | None = None, +) -> StructuralObserver | None: + """Create the native observer without importing UIA on other platforms.""" + + if not enabled: + return None + resolved_platform = platform_name or sys.platform + if resolved_platform != "win32": + return None + + try: + from openadapt_capture.structural_observer.windows import ( + WindowsUIAStructuralObserver, + ) + + return WindowsUIAStructuralObserver() + except Exception as exc: + _logger.warning("Windows UIA structural observation is unavailable: %s", exc) + return None + + +def observe_structural_action( + observer: StructuralObserver | None, + request: StructuralObservationRequest, +) -> StructuralObservation | None: + """Call an observer conservatively and validate its public contract.""" + + if observer is None: + return None + try: + observation = observer.observe(request) + if observation is None: + return None + return StructuralObservation.model_validate(observation) + except Exception as exc: + # Native accessibility trees can disappear between input receipt and + # observation. Missing optional authoring evidence must not corrupt the + # otherwise valid screen/input recording. + _logger.warning("Structural observation omitted after provider failure: %s", exc) + return None + + +__all__ = [ + "STRUCTURAL_OBSERVATION_SCHEMA_VERSION", + "StructuralAncestor", + "StructuralBounds", + "StructuralCandidateContext", + "StructuralElement", + "StructuralObservation", + "StructuralObservationRequest", + "StructuralObserver", + "StructuralProcessIdentity", + "StructuralWindowIdentity", + "create_structural_observer", + "observe_structural_action", +] diff --git a/openadapt_capture/structural_observer/__init__.py b/openadapt_capture/structural_observer/__init__.py new file mode 100644 index 0000000..dec5f88 --- /dev/null +++ b/openadapt_capture/structural_observer/__init__.py @@ -0,0 +1 @@ +"""Platform structural-observer implementations.""" diff --git a/openadapt_capture/structural_observer/windows.py b/openadapt_capture/structural_observer/windows.py new file mode 100644 index 0000000..663b60e --- /dev/null +++ b/openadapt_capture/structural_observer/windows.py @@ -0,0 +1,503 @@ +"""Windows UI Automation observation without input injection. + +``pywinauto`` is imported only when the Windows observer is instantiated. +Importing this module on macOS, Linux, or a headless host is side-effect free. +""" + +from __future__ import annotations + +import ctypes +import gc +import logging +import math +import sys +import threading +import time +from collections.abc import Callable +from typing import Any + +from openadapt_capture.structural import ( + StructuralAncestor, + StructuralBounds, + StructuralCandidateContext, + StructuralElement, + StructuralObservation, + StructuralObservationRequest, + StructuralProcessIdentity, + StructuralWindowIdentity, +) + +_logger = logging.getLogger(__name__) + +_SUPPORTED_PATTERN_ATTRIBUTES = ( + ("ExpandCollapse", "iface_expand_collapse"), + ("Selection", "iface_selection"), + ("SelectionItem", "iface_selection_item"), + ("Invoke", "iface_invoke"), + ("Toggle", "iface_toggle"), + ("Text", "iface_text"), + ("Value", "iface_value"), + ("RangeValue", "iface_range_value"), + ("Grid", "iface_grid"), + ("GridItem", "iface_grid_item"), + ("Table", "iface_table"), + ("TableItem", "iface_table_item"), + ("ScrollItem", "iface_scroll_item"), + ("Scroll", "iface_scroll"), + ("Transform", "iface_transform"), + ("Window", "iface_window"), +) + +_COINIT_APARTMENTTHREADED = 0x2 +def _uia_point_coordinate(value: float) -> int: + """Quantize a native pixel coordinate for UIA's integer ``POINT`` ABI. + + Native Windows hook coordinates arrive as integer pixels but the public + cross-platform event contract represents them as floats. Half values are + rounded away from zero so negative-monitor coordinates are deterministic. + """ + + coordinate = float(value) + if not math.isfinite(coordinate): + raise ValueError("UIA point coordinates must be finite") + if coordinate >= 0: + return int(math.floor(coordinate + 0.5)) + return int(math.ceil(coordinate - 0.5)) + + +class _PywinautoRuntime: + """Exact pywinauto UIA adapter, created and used on one COM thread.""" + + def __init__(self) -> None: + from pywinauto import Desktop + from pywinauto.uia_defines import IUIA + from pywinauto.uia_element_info import UIAElementInfo + + self._desktop = Desktop(backend="uia", allow_magic_lookup=False) + self._iui = IUIA() + self._element_info_type = UIAElementInfo + + def from_point(self, x: float, y: float) -> Any: + """Return the element under the integer Windows screen point.""" + + return self._desktop.from_point( + _uia_point_coordinate(x), + _uia_point_coordinate(y), + ) + + def focused_element(self) -> Any: + """Return the exact focused UIA element, not merely the active window.""" + + element_info = self._element_info_type(self._iui.get_focused_element()) + return self._desktop.backend.generic_wrapper_class(element_info) + + def close(self) -> None: + """Release the service-owned pywinauto singleton on its COM thread.""" + + from pywinauto.uia_defines import IUIA + + singleton_instances = getattr(type(IUIA), "_instances", {}) + if singleton_instances.get(IUIA) is self._iui: + singleton_instances.pop(IUIA, None) + self._desktop = None + self._iui = None + self._element_info_type = None + + +class _WindowsCOMApartment: + """Balanced COM apartment owned by the UIA service thread.""" + + def __init__(self) -> None: + self._ole32: Any | None = None + self._comtypes_import_initialized = False + + def __enter__(self) -> "_WindowsCOMApartment": + # Importing comtypes initializes COM on the importing thread. Initialize + # explicitly first so this adapter owns a balanced lifecycle even when + # comtypes was already imported by the host process. + comtypes_was_loaded = "comtypes" in sys.modules + existing_uia_module = sys.modules.get("pywinauto.uia_defines") + if existing_uia_module is not None: + iui_type = getattr(existing_uia_module, "IUIA", None) + singleton_instances = getattr(type(iui_type), "_instances", {}) + if iui_type is not None and singleton_instances.get(iui_type) is not None: + raise RuntimeError( + "pywinauto UIA was initialized before the Capture UIA service; " + "refusing to reuse an apartment-bound singleton" + ) + ole32 = ctypes.OleDLL("ole32") + co_initialize_ex = ole32.CoInitializeEx + co_initialize_ex.argtypes = [ctypes.c_void_p, ctypes.c_uint32] + co_initialize_ex.restype = ctypes.c_long + result = int(co_initialize_ex(None, _COINIT_APARTMENTTHREADED)) + if result < 0: + raise OSError(f"CoInitializeEx failed with HRESULT 0x{result & 0xFFFFFFFF:08X}") + self._ole32 = ole32 + + self._comtypes_import_initialized = not comtypes_was_loaded + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + ole32 = self._ole32 + self._ole32 = None + if ole32 is None: + return + uia_module = sys.modules.get("pywinauto.uia_defines") + if uia_module is not None: + iui_type = getattr(uia_module, "IUIA", None) + singleton_instances = getattr(type(iui_type), "_instances", {}) + if iui_type is not None: + singleton_instances.pop(iui_type, None) + gc.collect() + # If this service thread performed comtypes' first import, balance the + # initialization that comtypes performs at module import as well as our + # explicit initialization. + if self._comtypes_import_initialized: + import atexit as _atexit + + from comtypes import CoUninitialize + from comtypes._post_coinit import _shutdown + + # comtypes assumes its first import happens on the process main + # thread and registers a process-exit uninitializer. This adapter + # imports it on its owned UIA thread, balances it here, and removes + # that mismatched process-exit callback. + _atexit.unregister(_shutdown) + + CoUninitialize() + co_uninitialize = ole32.CoUninitialize + co_uninitialize.argtypes = [] + co_uninitialize.restype = None + co_uninitialize() + + +def _present_string(value: Any) -> str | None: + if not isinstance(value, str): + return None + stripped = value.strip() + return stripped or None + + +def _present_int(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + parsed = int(value) + except (TypeError, ValueError): + return None + return parsed if parsed >= 0 else None + + +def _safe_value(obj: Any, name: str) -> Any: + try: + return getattr(obj, name) + except Exception: + return None + + +def _safe_call(obj: Any, name: str) -> Any: + value = _safe_value(obj, name) + if not callable(value): + return None + try: + return value() + except Exception: + return None + + +def _bounds(value: Any) -> StructuralBounds | None: + if value is None: + return None + + def read(name: str) -> Any: + if isinstance(value, dict): + return value.get(name) + return _safe_value(value, name) + + raw = {name: read(name) for name in ("left", "top", "right", "bottom")} + if any(item is None for item in raw.values()): + return None + try: + return StructuralBounds(**raw) + except (TypeError, ValueError): + return None + + +def _element_info(wrapper: Any) -> Any: + return _safe_value(wrapper, "element_info") + + +def _element_fields(wrapper: Any) -> dict[str, Any]: + info = _element_info(wrapper) + if info is None: + return {} + role = _present_string(_safe_call(wrapper, "friendly_class_name")) + return { + "automation_id": _present_string( + _safe_value(info, "automation_id") + or _safe_value(info, "auto_id") + ), + "role": role, + "role_source": "pywinauto_friendly_class" if role else None, + "control_type": _present_string(_safe_value(info, "control_type")), + "name": _present_string(_safe_value(info, "name")), + "class_name": _present_string(_safe_value(info, "class_name")), + "framework_id": _present_string(_safe_value(info, "framework_id")), + "native_window_handle": _present_int( + _safe_value(info, "handle") or _safe_value(wrapper, "handle") + ), + "bounds": _bounds(_safe_value(info, "rectangle")), + } + + +def _supported_patterns(wrapper: Any) -> list[str]: + supported: list[str] = [] + for name, attribute in _SUPPORTED_PATTERN_ATTRIBUTES: + try: + if getattr(wrapper, attribute) is not None: + supported.append(name) + except Exception: + continue + return supported + + +def _as_element(wrapper: Any) -> StructuralElement: + return StructuralElement( + **_element_fields(wrapper), + supported_patterns=_supported_patterns(wrapper), + ) + + +def _as_ancestor(wrapper: Any) -> StructuralAncestor | None: + fields = _element_fields(wrapper) + fields.pop("framework_id", None) + fields.pop("native_window_handle", None) + ancestor = StructuralAncestor(**fields) + if not ancestor.model_dump(exclude_none=True): + return None + return ancestor + + +def _ancestry(wrapper: Any, *, maximum_depth: int) -> list[StructuralAncestor] | None: + ancestors: list[StructuralAncestor] = [] + current = wrapper + for _ in range(maximum_depth): + parent = _safe_call(current, "parent") + if parent is None: + break + ancestor = _as_ancestor(parent) + if ancestor is not None: + ancestors.append(ancestor) + current = parent + return ancestors or None + + +def _window_identity(wrapper: Any) -> StructuralWindowIdentity | None: + top = _safe_call(wrapper, "top_level_parent") + if top is None: + return None + fields = _element_fields(top) + title = _present_string(_safe_call(top, "window_text")) or fields.get("name") + identity = StructuralWindowIdentity( + title=title, + automation_id=fields.get("automation_id"), + class_name=fields.get("class_name"), + native_window_handle=fields.get("native_window_handle"), + bounds=fields.get("bounds"), + ) + if not identity.model_dump(exclude_none=True): + return None + return identity + + +def _candidate_cardinality( + target: Any, +) -> tuple[int | None, StructuralCandidateContext | None]: + target_fields = _element_fields(target) + automation_id = target_fields.get("automation_id") + control_type = target_fields.get("control_type") + name = target_fields.get("name") + + search_kwargs: dict[str, str] = {} + matched_fields: list[str] = [] + if automation_id: + # pywinauto 0.6.9's UIA descendants() cannot filter by auto_id. + # Narrow with its supported control-type condition, then compare the + # stable AutomationId exactly below. + matched_fields.append("automation_id") + if control_type: + search_kwargs["control_type"] = control_type + matched_fields.append("control_type") + elif control_type and name: + search_kwargs["control_type"] = control_type + search_kwargs["title"] = name + matched_fields.extend(("control_type", "name")) + else: + return None, None + + top = _safe_call(target, "top_level_parent") + if top is None: + return None, None + try: + candidates = list(top.descendants(**search_kwargs)) + except Exception: + return None, None + + def exact_match(candidate: Any) -> bool: + fields = _element_fields(candidate) + return all( + { + "automation_id": fields.get("automation_id") == automation_id, + "control_type": fields.get("control_type") == control_type, + "name": fields.get("name") == name, + }[field] + for field in matched_fields + ) + + count = sum(1 for candidate in candidates if exact_match(candidate)) + if exact_match(top): + count += 1 + return count, StructuralCandidateContext( + scope="top_level_window", + matched_fields=matched_fields, + ) + + +def _resolve_process_name(process_id: int) -> str | None: + try: + import psutil + + return _present_string(psutil.Process(process_id).name()) + except Exception: + return None + + +def _observe_with_runtime( + runtime: Any, + request: StructuralObservationRequest, + *, + process_name_resolver: Callable[[int], str | None], + clock: Callable[[], float], + maximum_ancestry_depth: int, +) -> StructuralObservation | None: + """Build one observation while every UIA wrapper stays on its owner thread.""" + + if request.x is not None and request.y is not None: + target = runtime.from_point(request.x, request.y) + query_kind = "point" + else: + target = runtime.focused_element() + query_kind = "focused" + if target is None: + return None + + element = _as_element(target) + info = _element_info(target) + process_id = _present_int(_safe_value(info, "process_id")) + process = None + if process_id is not None: + process = StructuralProcessIdentity( + process_id=process_id, + process_name=process_name_resolver(process_id), + ) + candidate_count, candidate_context = _candidate_cardinality(target) + return StructuralObservation( + provider="windows_uia", + event_timestamp=request.event_timestamp, + observed_at=clock(), + query_kind=query_kind, + element=element, + process=process, + window=_window_identity(target), + ancestry=_ancestry( + target, + maximum_depth=maximum_ancestry_depth, + ), + candidate_count=candidate_count, + candidate_context=candidate_context, + ) + + +class WindowsUIAStructuralObserver: + """Read Windows UIA evidence at a pointer or focused-element action.""" + + def __init__( + self, + *, + runtime: Any | None = None, + runtime_factory: Callable[[], Any] = _PywinautoRuntime, + apartment_factory: Callable[[], Any] = _WindowsCOMApartment, + process_name_resolver: Callable[[int], str | None] | None = None, + clock: Callable[[], float] = time.time, + maximum_ancestry_depth: int = 12, + ) -> None: + self._runtime = runtime + self._runtime_factory = runtime_factory + self._apartment_factory = apartment_factory + self._thread_state = threading.local() + self._process_name_resolver = process_name_resolver or _resolve_process_name + self._clock = clock + self._maximum_ancestry_depth = maximum_ancestry_depth + + def open_current_thread(self) -> None: + """Initialize the observing thread's COM apartment and UIA runtime.""" + + if self._runtime is not None: + return + if getattr(self._thread_state, "unavailable", False): + return + if getattr(self._thread_state, "runtime", None) is not None: + return + apartment = self._apartment_factory() + opened = False + try: + apartment.__enter__() + opened = True + runtime = self._runtime_factory() + except Exception as exc: + if opened: + apartment.__exit__(*sys.exc_info()) + self._thread_state.unavailable = True + _logger.warning("Windows UIA observation is unavailable: %s", exc) + return + self._thread_state.apartment = apartment + self._thread_state.runtime = runtime + + def close_current_thread(self) -> None: + """Release UIA and COM on the same thread that initialized them.""" + + if self._runtime is not None: + return + runtime = getattr(self._thread_state, "runtime", None) + apartment = getattr(self._thread_state, "apartment", None) + if runtime is None or apartment is None: + return + try: + close = getattr(runtime, "close", None) + if callable(close): + close() + finally: + self._thread_state.runtime = None + self._thread_state.apartment = None + apartment.__exit__(None, None, None) + + def observe( + self, + request: StructuralObservationRequest, + ) -> StructuralObservation | None: + runtime = self._runtime + if runtime is None: + self.open_current_thread() + runtime = getattr(self._thread_state, "runtime", None) + if runtime is None: + return None + return _observe_with_runtime( + runtime, + request, + process_name_resolver=self._process_name_resolver, + clock=self._clock, + maximum_ancestry_depth=self._maximum_ancestry_depth, + ) + + +__all__ = ["WindowsUIAStructuralObserver"] diff --git a/pyproject.toml b/pyproject.toml index 5bb24ba..de6cecc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,9 @@ dependencies = [ "matplotlib>=3.10.8", # Native macOS window/accessibility observation via permissive PyObjC. "pyobjc-framework-ApplicationServices>=12.2.1; sys_platform == 'darwin'", + # Native Windows structural observation. Platform-gated so non-Windows + # installs and imports do not load the Windows UIA stack. + "pywinauto>=0.6.9; sys_platform == 'win32'", ] [project.optional-dependencies] diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 069e3ab..55fb93c 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -147,6 +147,31 @@ def test_setup_events_are_delivered_in_order_only_after_start_commits() -> None: assert delivered == observer.events +def test_delivery_thread_balances_callback_lifecycle_hooks() -> None: + lifecycle: list[tuple[str, int]] = [] + delivered = threading.Event() + + class Callback: + def _openadapt_delivery_thread_start(self) -> None: + lifecycle.append(("start", threading.get_ident())) + + def __call__(self, _event) -> None: + lifecycle.append(("event", threading.get_ident())) + delivered.set() + + def _openadapt_delivery_thread_stop(self) -> None: + lifecycle.append(("stop", threading.get_ident())) + + observer = _ReadyObserver(Callback()) + observer.start() + observer._emit(ObservedKey(pressed=True, key_char="a", timestamp=1.0)) + assert delivered.wait(timeout=1) + observer.stop() + + assert [name for name, _thread_id in lifecycle] == ["start", "event", "stop"] + assert len({thread_id for _name, thread_id in lifecycle}) == 1 + + def test_setup_events_are_discarded_when_start_fails() -> None: delivered = [] observer = _SetupEmittingObserver(delivered.append, fail_setup=True) @@ -434,6 +459,19 @@ def test_recorder_uses_one_observer_and_preserves_cross_device_receipt_order( ), ] factory_calls = [] + structural_lifecycle = [] + + class FakeStructuralObserver: + def open_current_thread(self) -> None: + structural_lifecycle.append("start") + + def close_current_thread(self) -> None: + structural_lifecycle.append("stop") + + def observe(self, _request): + return None + + structural_observer = FakeStructuralObserver() class FakeObserver: def __init__(self, callback) -> None: @@ -452,6 +490,8 @@ def stop(self) -> None: def create(callback, **kwargs): factory_calls.append(kwargs) + callback._openadapt_delivery_thread_start() + callback._openadapt_delivery_thread_stop() return FakeObserver(callback) monkeypatch.setattr(recorder_module, "create_input_observer", create) @@ -460,6 +500,7 @@ def create(callback, **kwargs): terminate, SimpleNamespace(timestamp=100.0), started, + structural_observer=structural_observer, ) persisted = [event_q.get_nowait() for _ in range(event_q.qsize())] @@ -471,6 +512,7 @@ def create(callback, **kwargs): } ] assert started.is_set() + assert structural_lifecycle == ["start", "stop"] assert [event.timestamp for event in persisted] == [100.1, 100.2, 100.3] assert [event.data["name"] for event in persisted] == [ "click", diff --git a/tests/test_structural_observation.py b/tests/test_structural_observation.py new file mode 100644 index 0000000..fb23822 --- /dev/null +++ b/tests/test_structural_observation.py @@ -0,0 +1,444 @@ +"""Contract tests for optional native structural observations.""" + +from __future__ import annotations + +import queue +import sqlite3 +import threading +from pathlib import Path +from types import SimpleNamespace + +from openadapt_capture.capture import CaptureSession +from openadapt_capture.db import create_db, crud +from openadapt_capture.recorder import on_click, write_action_event +from openadapt_capture.structural import ( + StructuralElement, + StructuralObservation, + StructuralObservationRequest, + create_structural_observer, +) +from openadapt_capture.structural_observer.windows import ( + WindowsUIAStructuralObserver, + _PywinautoRuntime, +) + + +class _Wrapper: + def __init__( + self, + *, + automation_id: str | None = None, + control_type: str | None = None, + name: str | None = None, + role: str | None = None, + process_id: int | None = None, + parent: "_Wrapper | None" = None, + top: "_Wrapper | None" = None, + descendants: list["_Wrapper"] | None = None, + title: str | None = None, + patterns: tuple[str, ...] = (), + ) -> None: + self.element_info = SimpleNamespace( + automation_id=automation_id, + control_type=control_type, + name=name, + class_name=None, + framework_id=None, + handle=None, + process_id=process_id, + rectangle=SimpleNamespace(left=10, top=20, right=110, bottom=60), + ) + self._role = role + self._parent = parent + self._top = top + self._descendants = descendants or [] + self._title = title + self.descendant_queries: list[dict[str, object]] = [] + for pattern in patterns: + setattr(self, f"iface_{pattern}", object()) + + def friendly_class_name(self) -> str | None: + return self._role + + def parent(self) -> "_Wrapper | None": + return self._parent + + def top_level_parent(self) -> "_Wrapper": + return self._top or self + + def descendants(self, **kwargs) -> list["_Wrapper"]: + self.descendant_queries.append(kwargs) + return self._descendants + + def window_text(self) -> str | None: + return self._title + + +def _observation(event_timestamp: float) -> StructuralObservation: + return StructuralObservation( + provider="windows_uia", + event_timestamp=event_timestamp, + observed_at=event_timestamp + 0.001, + query_kind="point", + element=StructuralElement( + automation_id="submit-order", + role="Button", + control_type="Button", + name="Submit", + ), + ) + + +class _Observer: + def observe( + self, + request: StructuralObservationRequest, + ) -> StructuralObservation: + return _observation(request.event_timestamp) + + +def _recording(capture_dir: Path): + capture_dir.mkdir() + engine, session_factory = create_db(str(capture_dir / "recording.db")) + session = session_factory() + recording = crud.insert_recording( + session, + { + "timestamp": 100.0, + "monitor_width": 1280, + "monitor_height": 720, + "double_click_interval_seconds": 0.5, + "double_click_distance_pixels": 5, + "platform": "win32", + "task_description": "Structural observation test", + }, + ) + return engine, session, recording + + +def test_windows_uia_observer_returns_exact_available_evidence() -> None: + window = _Wrapper( + automation_id="main-window", + control_type="Window", + name="Orders", + role="Window", + title="Orders - Example", + ) + panel = _Wrapper( + automation_id="order-panel", + control_type="Pane", + name="Order", + role="Pane", + parent=window, + top=window, + ) + target = _Wrapper( + automation_id="submit-order", + control_type="Button", + name="Submit", + role="Button", + process_id=42, + parent=panel, + top=window, + patterns=("invoke",), + ) + duplicate = _Wrapper( + automation_id="submit-order", + control_type="Button", + name="Other name", + role="Button", + top=window, + ) + window._descendants = [target, duplicate] + runtime = SimpleNamespace( + from_point=lambda _x, _y: target, + focused_element=lambda: target, + ) + observer = WindowsUIAStructuralObserver( + runtime=runtime, + process_name_resolver=lambda process_id: ( + "example.exe" if process_id == 42 else None + ), + clock=lambda: 101.25, + ) + + observed = observer.observe( + StructuralObservationRequest( + event_timestamp=101.0, + action_name="click", + x=25, + y=35, + ) + ) + + assert observed is not None + assert observed.element.automation_id == "submit-order" + assert observed.element.supported_patterns == ["Invoke"] + assert observed.process is not None + assert observed.process.process_name == "example.exe" + assert observed.window is not None + assert observed.window.title == "Orders - Example" + assert [ancestor.automation_id for ancestor in observed.ancestry or []] == [ + "order-panel", + "main-window", + ] + assert observed.candidate_count == 2 + assert observed.candidate_context is not None + assert observed.candidate_context.matched_fields == [ + "automation_id", + "control_type", + ] + assert window.descendant_queries == [{"control_type": "Button"}] + assert "framework_id" not in observed.element.model_dump(exclude_none=True) + + +def test_pywinauto_runtime_quantizes_points_and_uses_exact_focused_api() -> None: + calls: list[tuple[str, object]] = [] + desktop = SimpleNamespace( + from_point=lambda x, y: calls.append(("point", (x, y))) or "point-target", + backend=SimpleNamespace( + generic_wrapper_class=lambda info: calls.append(("wrap", info)) + or "focused-target" + ), + ) + iui = SimpleNamespace( + get_focused_element=lambda: calls.append(("focused", None)) or "raw-focused" + ) + + class ElementInfo: + def __init__(self, raw: object) -> None: + calls.append(("info", raw)) + + runtime = _PywinautoRuntime.__new__(_PywinautoRuntime) + runtime._desktop = desktop + runtime._iui = iui + runtime._element_info_type = ElementInfo + + assert runtime.from_point(25.49, -35.5) == "point-target" + assert calls[0] == ("point", (25, -36)) + point = calls[0][1] + assert isinstance(point, tuple) + assert all(isinstance(value, int) for value in point) + + assert runtime.focused_element() == "focused-target" + assert ("focused", None) in calls + assert ("info", "raw-focused") in calls + assert not hasattr(desktop, "get_active") + + +def test_uia_observer_owns_one_balanced_observation_thread() -> None: + lifecycle: list[tuple[str, int]] = [] + observations: list[StructuralObservation | None] = [] + target = _Wrapper( + automation_id="member-id", + control_type="Edit", + name="Member ID", + role="Edit", + ) + + class Apartment: + def __enter__(self): + lifecycle.append(("enter", threading.get_ident())) + return self + + def __exit__(self, _exc_type, _exc_val, _exc_tb) -> None: + lifecycle.append(("exit", threading.get_ident())) + + class Runtime: + def __init__(self) -> None: + lifecycle.append(("runtime", threading.get_ident())) + + def from_point(self, _x: float, _y: float) -> _Wrapper: + lifecycle.append(("point", threading.get_ident())) + return target + + def focused_element(self) -> _Wrapper: + lifecycle.append(("focused", threading.get_ident())) + return target + + def close(self) -> None: + lifecycle.append(("runtime-close", threading.get_ident())) + + observer = WindowsUIAStructuralObserver( + runtime_factory=Runtime, + apartment_factory=Apartment, + ) + def observe_on_delivery_thread() -> None: + observer.open_current_thread() + try: + observations.append( + observer.observe( + StructuralObservationRequest( + event_timestamp=101.0, + action_name="click", + x=25.0, + y=35.0, + ) + ) + ) + observations.append( + observer.observe( + StructuralObservationRequest( + event_timestamp=102.0, + action_name="press", + ) + ) + ) + finally: + observer.close_current_thread() + + thread = threading.Thread(target=observe_on_delivery_thread) + thread.start() + thread.join() + + point, focused = observations + assert point is not None and point.query_kind == "point" + assert focused is not None and focused.query_kind == "focused" + thread_ids = {thread_id for _name, thread_id in lifecycle} + assert len(thread_ids) == 1 + assert [name for name, _thread_id in lifecycle] == [ + "enter", + "runtime", + "point", + "focused", + "runtime-close", + "exit", + ] + + +def test_structural_evidence_persists_through_capture_session( + tmp_path: Path, + monkeypatch, +) -> None: + capture_dir = tmp_path / "capture" + engine, session, recording = _recording(capture_dir) + monkeypatch.setattr( + "openadapt_capture.recorder.window.get_active_element_state", + lambda _x, _y: {}, + ) + monkeypatch.setattr( + "openadapt_capture.recorder.utils.get_timestamp", + lambda: 101.2, + ) + event_queue: queue.Queue = queue.Queue() + perf_queue: queue.Queue = queue.Queue() + + on_click( + event_queue, + None, + 25, + 35, + "left", + True, + timestamp=101.0, + structural_observer=_Observer(), + ) + on_click( + event_queue, + None, + 25, + 35, + "left", + False, + timestamp=101.1, + structural_observer=_Observer(), + ) + write_action_event(session, recording, event_queue.get_nowait(), perf_queue) + write_action_event(session, recording, event_queue.get_nowait(), perf_queue) + session.close() + engine.dispose() + + with CaptureSession.load(capture_dir) as capture: + raw = capture.raw_events() + actions = list(capture.actions()) + + assert raw[0].structural_observation is not None + assert raw[0].structural_observation.element.automation_id == "submit-order" + assert raw[1].structural_observation is None + assert len(actions) == 1 + assert actions[0].structural_observation is not None + assert actions[0].structural_observation.element.name == "Submit" + + +def test_legacy_action_table_is_migrated_without_fabricated_evidence( + tmp_path: Path, +) -> None: + capture_dir = tmp_path / "legacy" + engine, session, recording = _recording(capture_dir) + crud.insert_action_event( + session, + recording, + 101.0, + { + "name": "scroll", + "mouse_x": 25, + "mouse_y": 35, + "mouse_dx": 0, + "mouse_dy": -1, + }, + ) + session.close() + engine.dispose() + + database_path = capture_dir / "recording.db" + connection = sqlite3.connect(database_path) + connection.execute( + "ALTER TABLE action_event DROP COLUMN structural_observation" + ) + connection.commit() + connection.close() + + with CaptureSession.load(capture_dir) as capture: + event = capture.raw_events()[0] + assert event.structural_observation is None + + connection = sqlite3.connect(database_path) + columns = { + row[1] + for row in connection.execute("PRAGMA table_info(action_event)") + } + connection.close() + assert "structural_observation" in columns + + +def test_non_windows_factory_is_headless_safe() -> None: + assert create_structural_observer(platform_name="linux") is None + + +def test_windows_factory_omits_optional_evidence_when_uia_cannot_start( + monkeypatch, +) -> None: + def fail() -> None: + raise RuntimeError("COM unavailable") + + monkeypatch.setattr( + "openadapt_capture.structural_observer.windows.WindowsUIAStructuralObserver", + fail, + ) + assert create_structural_observer(platform_name="win32") is None + + +def test_lazy_uia_start_failure_does_not_abort_native_capture() -> None: + attempts = 0 + + def fail_runtime(): + nonlocal attempts + attempts += 1 + raise RuntimeError("UIA unavailable") + + observer = WindowsUIAStructuralObserver( + runtime_factory=fail_runtime, + apartment_factory=lambda: SimpleNamespace( + __enter__=lambda: None, + __exit__=lambda *_args: None, + ), + ) + request = StructuralObservationRequest( + event_timestamp=101.0, + action_name="click", + x=25.0, + y=35.0, + ) + assert observer.observe(request) is None + assert observer.observe(request) is None + assert attempts == 1 diff --git a/tests/test_windows_uia_live.py b/tests/test_windows_uia_live.py new file mode 100644 index 0000000..1a22aee --- /dev/null +++ b/tests/test_windows_uia_live.py @@ -0,0 +1,211 @@ +"""Windows-only smoke test for the real pywinauto UIA boundary.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import pytest + +from openadapt_capture.structural import ( + StructuralObservation, + StructuralObservationRequest, +) +from openadapt_capture.structural_observer.windows import ( + WindowsUIAStructuralObserver, +) + +pytestmark = pytest.mark.skipif( + sys.platform != "win32", + reason="requires Windows UI Automation", +) + +_WINFORMS_FIXTURE = r""" +Add-Type -AssemblyName System.Windows.Forms +Add-Type -AssemblyName System.Drawing + +$form = New-Object System.Windows.Forms.Form +$form.Text = $env:OPENADAPT_UIA_SMOKE_TITLE +$form.StartPosition = [System.Windows.Forms.FormStartPosition]::Manual +$form.Location = [System.Drawing.Point]::new(180, 140) +$form.Size = [System.Drawing.Size]::new(520, 260) +$form.TopMost = $true + +$label = New-Object System.Windows.Forms.Label +$label.Text = "Member ID" +$label.Location = [System.Drawing.Point]::new(24, 30) +$label.AutoSize = $true + +$textBox = New-Object System.Windows.Forms.TextBox +$textBox.Name = "MemberIdField" +$textBox.Location = [System.Drawing.Point]::new(24, 58) +$textBox.Size = [System.Drawing.Size]::new(300, 28) + +$submitOne = New-Object System.Windows.Forms.Button +$submitOne.Name = "SubmitAction" +$submitOne.Text = "Submit" +$submitOne.Location = [System.Drawing.Point]::new(24, 116) +$submitOne.Size = [System.Drawing.Size]::new(120, 36) + +$submitTwo = New-Object System.Windows.Forms.Button +$submitTwo.Name = "SubmitAction" +$submitTwo.Text = "Submit" +$submitTwo.Location = [System.Drawing.Point]::new(164, 116) +$submitTwo.Size = [System.Drawing.Size]::new(120, 36) + +$form.Controls.AddRange(@($label, $textBox, $submitOne, $submitTwo)) +$form.Add_Shown({ + $form.Activate() + $textBox.Select() + [void]$textBox.Focus() + $form.BeginInvoke([Action]{ + $form.Activate() + $textBox.Select() + [void]$textBox.Focus() + [System.Windows.Forms.Application]::DoEvents() + Start-Sleep -Milliseconds 250 + $buttonPoint = $form.PointToScreen( + ([System.Drawing.Point]::new( + ($submitOne.Left + [int]($submitOne.Width / 2)), + ($submitOne.Top + [int]($submitOne.Height / 2)) + )) + ) + $payload = @{ + button_x = $buttonPoint.X + button_y = $buttonPoint.Y + } | ConvertTo-Json -Compress + [System.IO.File]::WriteAllText( + $env:OPENADAPT_UIA_SMOKE_READY, + $payload, + [System.Text.Encoding]::ASCII + ) + }) +}) + +[System.Windows.Forms.Application]::Run($form) +""" + + +def _wait_for_fixture(path: Path, process: subprocess.Popen[str]) -> dict[str, int]: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if path.exists(): + try: + return json.loads(path.read_text(encoding="ascii")) + except json.JSONDecodeError: + pass + if process.poll() is not None: + stdout, stderr = process.communicate() + raise AssertionError( + f"WinForms fixture exited early ({process.returncode}): " + f"{stdout}\n{stderr}" + ) + time.sleep(0.1) + raise AssertionError("WinForms fixture did not become ready within 30 seconds") + + +def _observe_on_owned_thread( + requests: list[StructuralObservationRequest], +) -> list[StructuralObservation | None]: + observations: list[StructuralObservation | None] = [] + errors: list[BaseException] = [] + + def run() -> None: + observer = WindowsUIAStructuralObserver() + try: + observer.open_current_thread() + observations.extend(observer.observe(request) for request in requests) + except BaseException as exc: + errors.append(exc) + finally: + observer.close_current_thread() + + thread = threading.Thread(target=run, name="capture-uia-smoke") + thread.start() + thread.join(timeout=30) + assert not thread.is_alive(), "UIA observation thread did not terminate" + if errors: + raise errors[0] + return observations + + +def test_real_pywinauto_observes_point_focus_ambiguity_and_reopens( + tmp_path: Path, +) -> None: + """Exercise exact pywinauto APIs and COM ownership without injecting input.""" + + title = f"OpenAdapt UIA smoke {uuid.uuid4()}" + script_path = tmp_path / "uia_fixture.ps1" + ready_path = tmp_path / "ready.json" + script_path.write_text(_WINFORMS_FIXTURE, encoding="utf-8") + env = { + **os.environ, + "OPENADAPT_UIA_SMOKE_TITLE": title, + "OPENADAPT_UIA_SMOKE_READY": str(ready_path), + } + process = subprocess.Popen( + [ + "powershell.exe", + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script_path), + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + try: + point = _wait_for_fixture(ready_path, process) + requests = [ + StructuralObservationRequest( + event_timestamp=time.time(), + action_name="click", + x=float(point["button_x"]), + y=float(point["button_y"]), + ), + StructuralObservationRequest( + event_timestamp=time.time(), + action_name="press", + ), + ] + clicked, focused = _observe_on_owned_thread(requests) + + assert clicked is not None + assert clicked.query_kind == "point" + assert clicked.element.control_type == "Button" + assert clicked.element.name == "Submit" + assert clicked.element.automation_id == "SubmitAction" + assert clicked.candidate_count == 2 + assert clicked.window is not None + assert clicked.window.title == title + + assert focused is not None + assert focused.query_kind == "focused" + assert focused.element.control_type == "Edit" + assert focused.element.automation_id == "MemberIdField" + assert focused.window is not None + assert focused.window.title == title + + # A second service lifecycle in the same process proves the pywinauto + # singleton and COM apartment were released on their owning thread. + reopened = _observe_on_owned_thread(requests[:1]) + assert reopened[0] is not None + assert reopened[0].element.automation_id == "SubmitAction" + finally: + process.terminate() + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10)