Capture Windows UIA evidence with native actions

README.md

@@ -108,6 +108,7 @@ from openadapt_capture import CaptureSession
 with CaptureSession.load("./my-capture") as capture:
     for action in capture.actions():
         print(action.timestamp, action.type, action.x, action.y)
+        print(action.structural_observation)
         frame = action.screenshot
 ```
 
@@ -136,6 +137,27 @@ through a pipe, the selected video encoder, MP4 demuxing/muxing, PNG
 decoding/encoding, the `image2pipe` muxer, and the `select` video filter;
 Desktop provisions and probes that exact closure.
 
+## Native structural observations
+
+On Windows, the recorder can retain a versioned UI Automation observation
+beside the native action that produced it. When UIA exposes the information,
+the observation includes the target's AutomationId, role/control type, name,
+bounds, supported patterns, process/window identity, ancestry, and exact
+candidate cardinality within the top-level window. Unavailable values remain
+absent; capture never infers structural fields from coordinates or pixels.
+
+This evidence is stored in `recording.db`, exposed on raw events and processed
+`Action` objects, and remains optional so existing recordings and non-Windows
+hosts continue to load unchanged. It is enabled by default on Windows and can
+be disabled with
+`Recorder(..., capture_structural_observations=False)`. Applications can inject
+another read-only observer through `Recorder(..., structural_observer=...)`
+using the public `StructuralObserver` protocol.
+
+UIA describes the local Windows accessibility tree. It does not cross an
+RDP/Citrix pixel boundary into the remote application; those demonstrations
+retain window-scoped pixels and coordinates for Flow's remote visual compiler.
+
 ## Window-scoped recording
 
 **Status: implemented and unit-proven on all CI platforms; live-validated
@@ -206,11 +228,11 @@ Capture does not upload a session by default. The sharing command, remote
 transcription, and profiling transfer are explicit opt-in operations. Installing
 the `privacy` extra alone does not automatically scrub a recording.
 
-The current desktop-to-Flow conversion has no field geometry for reliable
-secret redaction and no live UIA locator. `openadapt-flow` therefore refuses its
-desktop `--secret` option, and converted desktop workflows rely on retained
-visual evidence unless a separate structural recording path arms them. Review
-the desktop guide before recording sensitive workflows.
+Structural observations can also contain sensitive control names, window
+titles, and accessibility ancestry. They stay in the same local raw-capture
+boundary. `openadapt-flow` still refuses desktop `--secret` authoring until its
+source-time field-redaction contract can prove that sensitive values were not
+retained. Review the desktop guide before recording sensitive workflows.
 
 The experimental Chrome extension can observe pages across its configured host
 permissions and can emit DOM text and keyboard events to a local WebSocket.
@@ -220,8 +242,8 @@ Treat it as development code; do not deploy it in a sensitive browser profile.
 
 - Native recording requires a visible user session plus the operating system's
   screen-recording and input-monitoring permissions.
-- Desktop capture records pixels and coordinates, not a structural accessibility
-  locator for each demonstrated target.
+- Native Windows capture retains UIA evidence when the application exposes it;
+  opaque remote applications still require Flow's visual and OCR bindings.
 - The Flow adapter rejects unsupported input such as drag, non-left-click, and
   modifier-chord actions instead of silently compiling an incomplete workflow.
 - Browser-extension installation, security hardening, and compiler integration

openadapt_capture/__init__.py

@@ -83,6 +83,20 @@
     PerfStat,
     plot_capture_performance,
 )
+from openadapt_capture.structural import (
+    STRUCTURAL_OBSERVATION_SCHEMA_VERSION,
+    StructuralAncestor,
+    StructuralBounds,
+    StructuralCandidateContext,
+    StructuralElement,
+    StructuralObservation,
+    StructuralObservationRequest,
+    StructuralObserver,
+    StructuralProcessIdentity,
+    StructuralWindowIdentity,
+    create_structural_observer,
+    observe_structural_action,
+)
 
 # Visualization
 from openadapt_capture.visualize import create_demo, create_html
@@ -133,6 +147,19 @@
     "Capture",
     "CaptureSession",
     "Action",
+    # Native structural observation
+    "STRUCTURAL_OBSERVATION_SCHEMA_VERSION",
+    "StructuralAncestor",
+    "StructuralBounds",
+    "StructuralCandidateContext",
+    "StructuralElement",
+    "StructuralObservation",
+    "StructuralObservationRequest",
+    "StructuralObserver",
+    "StructuralProcessIdentity",
+    "StructuralWindowIdentity",
+    "create_structural_observer",
+    "observe_structural_action",
     # Window-scoped capture
     "WindowTarget",
     "TargetWindow",

openadapt_capture/capture.py

@@ -36,13 +36,25 @@
     MouseUpEvent,
 )
 from openadapt_capture.processing import process_events
+from openadapt_capture.structural import StructuralObservation
 
 if TYPE_CHECKING:
     from PIL import Image
 
     from openadapt_capture.browser_events import BrowserEvent
 
 
+def _parse_structural_observation(raw: object) -> StructuralObservation | None:
+    """Load optional structural evidence without breaking legacy recordings."""
+
+    if raw is None:
+        return None
+    try:
+        return StructuralObservation.model_validate(raw)
+    except Exception:
+        return None
+
+
 def _convert_action_event(db_event) -> PydanticActionEvent | None:
     """Convert a SQLAlchemy ActionEvent to a Pydantic event.
 
@@ -52,11 +64,16 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None:
     Returns:
         Pydantic event or None if unrecognized.
     """
-    ts = db_event.timestamp
+    common = {
+        "timestamp": db_event.timestamp,
+        "structural_observation": _parse_structural_observation(
+            getattr(db_event, "structural_observation", None)
+        ),
+    }
 
     if db_event.name == "move":
         return MouseMoveEvent(
-            timestamp=ts,
+            **common,
             x=db_event.mouse_x or 0,
             y=db_event.mouse_y or 0,
         )
@@ -65,14 +82,14 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None:
 
         if db_event.mouse_pressed is True:
             return MouseDownEvent(
-                timestamp=ts,
+                **common,
                 x=db_event.mouse_x or 0,
                 y=db_event.mouse_y or 0,
                 button=button,
             )
         elif db_event.mouse_pressed is False:
             return MouseUpEvent(
-                timestamp=ts,
+                **common,
                 x=db_event.mouse_x or 0,
                 y=db_event.mouse_y or 0,
                 button=button,
@@ -81,15 +98,15 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None:
             return None
     elif db_event.name == "scroll":
         return MouseScrollEvent(
-            timestamp=ts,
+            **common,
             x=db_event.mouse_x or 0,
             y=db_event.mouse_y or 0,
             dx=db_event.mouse_dx or 0,
             dy=db_event.mouse_dy or 0,
         )
     elif db_event.name == "press":
         return KeyDownEvent(
-            timestamp=ts,
+            **common,
             key_name=db_event.key_name,
             key_char=db_event.key_char,
             key_vk=db_event.key_vk,
@@ -99,7 +116,7 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None:
         )
     elif db_event.name == "release":
         return KeyUpEvent(
-            timestamp=ts,
+            **common,
             key_name=db_event.key_name,
             key_char=db_event.key_char,
             key_vk=db_event.key_vk,
@@ -369,6 +386,11 @@ def button(self) -> str | None:
             return btn.value if hasattr(btn, "value") else str(btn)
         return None
 
+    @property
+    def structural_observation(self) -> StructuralObservation | None:
+        """Accessibility evidence captured at this action, when available."""
+        return self.event.structural_observation
+
     @property
     def screenshot(self) -> "Image" | None:
         """Get the screenshot at the time of this action.

openadapt_capture/config.py

@@ -34,6 +34,9 @@ class Settings(BaseSettings):
     # Record and replay (from legacy OpenAdapt config.defaults.json)
     RECORD_WINDOW_DATA: bool = False
     RECORD_READ_ACTIVE_ELEMENT_STATE: bool = False
+    # Retain live native accessibility evidence beside actionable input.
+    # The platform factory is a no-op outside Windows.
+    RECORD_STRUCTURAL_OBSERVATIONS: bool = True
     RECORD_VIDEO: bool = True
     RECORD_AUDIO: bool = False
     RECORD_BROWSER_EVENTS: bool = False
@@ -99,6 +102,7 @@ class Settings(BaseSettings):
     "capture_audio": "RECORD_AUDIO",
     "capture_images": "RECORD_IMAGES",
     "capture_window_data": "RECORD_WINDOW_DATA",
+    "capture_structural_observations": "RECORD_STRUCTURAL_OBSERVATIONS",
     "capture_browser_events": "RECORD_BROWSER_EVENTS",
     "window_owner": "RECORD_WINDOW_OWNER",
     "window_title": "RECORD_WINDOW_TITLE",
@@ -124,6 +128,7 @@ class RecordingConfig:
     capture_audio: bool | None = None
     capture_images: bool | None = None
     capture_window_data: bool | None = None
+    capture_structural_observations: bool | None = None
     capture_browser_events: bool | None = None
     # Window-scoped capture selectors (see window_capture.WindowTarget).
     window_owner: str | None = None

openadapt_capture/db/models.py

@@ -124,6 +124,9 @@ class ActionEvent(Base):
     canonical_key_vk = sa.Column(sa.String)
     parent_id = sa.Column(sa.Integer, sa.ForeignKey("action_event.id"))
     element_state = sa.Column(sa.JSON)
+    # Versioned optional accessibility evidence captured at action time.
+    # Nullable + additive migration keep older recording.db files readable.
+    structural_observation = sa.Column(sa.JSON)
     disabled = sa.Column(sa.Boolean, default=False)
 
     children = sa.orm.relationship("ActionEvent")

openadapt_capture/events.py

@@ -12,6 +12,8 @@
 
 from pydantic import BaseModel, Field
 
+from openadapt_capture.structural import StructuralObservation
+
 
 class EventType(str, Enum):
     """Event type identifiers."""
@@ -62,12 +64,21 @@ class BaseEvent(BaseModel):
     model_config = {"use_enum_values": True}
 
 
+class ActionBaseEvent(BaseEvent):
+    """Base event for native actions with optional structural evidence."""
+
+    structural_observation: StructuralObservation | None = Field(
+        default=None,
+        description="Versioned accessibility evidence observed at action time",
+    )
+
+
 # =============================================================================
 # Mouse Events
 # =============================================================================
 
 
-class MouseMoveEvent(BaseEvent):
+class MouseMoveEvent(ActionBaseEvent):
     """Mouse cursor movement event.
 
     Corresponds to OpenAdapt's ActionEvent with name="move".
@@ -78,7 +89,7 @@ class MouseMoveEvent(BaseEvent):
     y: float = Field(description="Mouse Y position in pixels")
 
 
-class MouseDownEvent(BaseEvent):
+class MouseDownEvent(ActionBaseEvent):
     """Mouse button press event.
 
     Corresponds to OpenAdapt's ActionEvent with name="click" and mouse_pressed=True.
@@ -90,7 +101,7 @@ class MouseDownEvent(BaseEvent):
     button: str = Field(description="Native mouse button name")
 
 
-class MouseUpEvent(BaseEvent):
+class MouseUpEvent(ActionBaseEvent):
     """Mouse button release event.
 
     Corresponds to OpenAdapt's ActionEvent with name="click" and mouse_pressed=False.
@@ -102,7 +113,7 @@ class MouseUpEvent(BaseEvent):
     button: str = Field(description="Native mouse button name")
 
 
-class MouseScrollEvent(BaseEvent):
+class MouseScrollEvent(ActionBaseEvent):
     """Mouse scroll wheel event.
 
     Corresponds to OpenAdapt's ActionEvent with name="scroll".
@@ -120,7 +131,7 @@ class MouseScrollEvent(BaseEvent):
 # =============================================================================
 
 
-class KeyDownEvent(BaseEvent):
+class KeyDownEvent(ActionBaseEvent):
     """Keyboard key press event.
 
     Corresponds to OpenAdapt's ActionEvent with name="press".
@@ -135,7 +146,7 @@ class KeyDownEvent(BaseEvent):
     canonical_key_vk: str | None = Field(default=None, description="Canonical virtual key code")
 
 
-class KeyUpEvent(BaseEvent):
+class KeyUpEvent(ActionBaseEvent):
     """Keyboard key release event.
 
     Corresponds to OpenAdapt's ActionEvent with name="release".
@@ -192,7 +203,7 @@ class AudioChunkEvent(BaseEvent):
 # =============================================================================
 
 
-class MouseClickEvent(BaseEvent):
+class MouseClickEvent(ActionBaseEvent):
     """Combined mouse click event (down + up).
 
     Corresponds to OpenAdapt's ActionEvent with name="singleclick".
@@ -208,7 +219,7 @@ class MouseClickEvent(BaseEvent):
     )
 
 
-class MouseDoubleClickEvent(BaseEvent):
+class MouseDoubleClickEvent(ActionBaseEvent):
     """Double click event.
 
     Corresponds to OpenAdapt's ActionEvent with name="doubleclick".
@@ -224,7 +235,7 @@ class MouseDoubleClickEvent(BaseEvent):
     )
 
 
-class MouseDragEvent(BaseEvent):
+class MouseDragEvent(ActionBaseEvent):
     """Mouse drag event (down + moves + up).
 
     Uses x/y for start position and dx/dy for displacement (like MouseScrollEvent).
@@ -242,7 +253,7 @@ class MouseDragEvent(BaseEvent):
     )
 
 
-class KeyTypeEvent(BaseEvent):
+class KeyTypeEvent(ActionBaseEvent):
     """Sequence of typed characters.
 
     Corresponds to OpenAdapt's ActionEvent with name="type".