From c435f52abca9c2744a3b99658fc36efc233c4839 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 12:58:58 -0700 Subject: [PATCH 01/26] fix: restore capture runtime contract --- .github/workflows/test.yml | 19 +++ openadapt_capture/comparison.py | 4 + openadapt_capture/recorder.py | 221 +++++++++++++++++++------- openadapt_capture/stats.py | 4 + openadapt_capture/window/_macos.py | 95 +++++++---- pyproject.toml | 8 +- scripts/verify_distribution.py | 74 +++++++++ tests/test_highlevel.py | 52 +++++- tests/test_macos_accessibility.py | 85 ++++++++++ tests/test_runtime_import_contract.py | 70 ++++++++ 10 files changed, 541 insertions(+), 91 deletions(-) create mode 100644 scripts/verify_distribution.py create mode 100644 tests/test_macos_accessibility.py create mode 100644 tests/test_runtime_import_contract.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4778b91..804d13f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,3 +121,22 @@ jobs: - name: Run ruff run: uv run ruff check openadapt_capture/ + + package-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + version: "latest" + + - name: Set up Python + run: uv python install 3.12 + + - name: Build wheel and source distribution + run: uv build + + - name: Verify release archive boundaries + run: python scripts/verify_distribution.py dist/* diff --git a/openadapt_capture/comparison.py b/openadapt_capture/comparison.py index b2665db..da9eb69 100644 --- a/openadapt_capture/comparison.py +++ b/openadapt_capture/comparison.py @@ -206,6 +206,10 @@ def plot_comparison( PIL Image if neither output_path nor show, else None. """ try: + import matplotlib + + if not show: + matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: raise ImportError( diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 5ce8e92..be63857 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -43,7 +43,7 @@ from pynput import keyboard, mouse from tqdm import tqdm -from openadapt_capture import platform, plotting, utils, video, window +from openadapt_capture import platform, utils, video, window from openadapt_capture.config import config from openadapt_capture.db import create_db, crud, get_session_for_path from openadapt_capture.db.models import ActionEvent, Recording @@ -154,11 +154,102 @@ def __bool__(self): "browser": True, } NUM_MEMORY_STATS_TO_LOG = 3 +STARTUP_WAIT_POLL_SECONDS = 0.1 +PRE_READY_TASK_JOIN_TIMEOUT_SECONDS = 2.0 stop_sequence_detected = False ws_server_instance = None +def _wait_for_tasks_started( + task_by_name: dict[str, Any], + task_started_events: dict[str, Any], + terminate_processing: Any, +) -> bool: + """Wait for pipeline readiness while honoring shutdown and worker failure. + + Returns ``True`` only when every task has announced readiness. A shutdown + request or a task that exits before setting its readiness event fails the + startup and signals the rest of the pipeline to stop. + """ + expected_starts = len(task_by_name) + logger.info(f"{expected_starts=}") + + while True: + if terminate_processing.is_set(): + logger.info("Recording startup cancelled before all tasks were ready") + return False + + waiting_for = [ + name + for name, event in task_started_events.items() + if not event.is_set() + ] + if not waiting_for: + return True + + stopped_before_ready = [ + name + for name in waiting_for + if name in task_by_name and not task_by_name[name].is_alive() + ] + if stopped_before_ready: + logger.error( + "Recording tasks exited before readiness: " + f"{stopped_before_ready}" + ) + terminate_processing.set() + return False + + logger.info(f"Waiting for tasks to start: {waiting_for}") + logger.info( + f"Started tasks: {expected_starts - len(waiting_for)}/{expected_starts}" + ) + terminate_processing.wait(STARTUP_WAIT_POLL_SECONDS) + + +def _join_tasks( + task_by_name: dict[str, Any], + task_names: list[str], + *, + timeout: float | None = None, +) -> list[str]: + """Join pipeline tasks, bounding teardown when startup never completed.""" + deadline = time.monotonic() + timeout if timeout is not None else None + lingering: list[str] = [] + + for task_name in task_names: + task = task_by_name.get(task_name) + if task is None: + continue + + logger.info(f"joining {task_name=}...") + if deadline is None: + task.join() + else: + task.join(timeout=max(0.0, deadline - time.monotonic())) + + if not task.is_alive(): + continue + + # Processes can be stopped after the shared graceful-shutdown deadline. + # Python cannot forcibly stop threads; recorder-owned threads are daemons + # and all receive terminate_processing before this helper is called. + if isinstance(task, multiprocessing.process.BaseProcess): + logger.warning( + f"terminating {task_name!r} after pre-ready shutdown timeout" + ) + task.terminate() + task.join(timeout=0.5) + + if task.is_alive(): + lingering.append(task_name) + + if lingering: + logger.warning(f"tasks still exiting after bounded shutdown: {lingering}") + return lingering + + def collect_stats(performance_snapshots: list[tracemalloc.Snapshot]) -> None: """Collects and appends performance snapshots using tracemalloc. @@ -1613,6 +1704,7 @@ def record( if config.RECORD_WINDOW_DATA and window_scope is None: window_event_reader = threading.Thread( target=read_window_events, + daemon=True, args=( event_q, terminate_processing, @@ -1628,6 +1720,7 @@ def record( if config.RECORD_BROWSER_EVENTS: browser_event_reader = threading.Thread( target=run_browser_event_server, + daemon=True, args=( event_q, terminate_processing, @@ -1642,6 +1735,7 @@ def record( screen_event_reader = threading.Thread( target=read_screen_events, + daemon=True, args=( event_q, terminate_processing, @@ -1656,6 +1750,7 @@ def record( keyboard_event_reader = threading.Thread( target=read_keyboard_events, + daemon=True, args=( event_q, terminate_processing, @@ -1668,6 +1763,7 @@ def record( mouse_event_reader = threading.Thread( target=read_mouse_events, + daemon=True, args=( event_q, terminate_processing, @@ -1692,6 +1788,7 @@ def record( event_processor = threading.Thread( target=process_events, + daemon=True, args=( event_q, screen_write_q, @@ -1875,35 +1972,31 @@ def record( # TODO: discard events until everything is ready - # Wait for all to signal they've started - expected_starts = len(task_by_name) - logger.info(f"{expected_starts=}") - while True: - started_tasks = sum(event.is_set() for event in task_started_events.values()) - if started_tasks >= expected_starts: - break - waiting_for = [ - task for task, event in task_started_events.items() if not event.is_set() - ] - logger.info(f"Waiting for tasks to start: {waiting_for}") - logger.info(f"Started tasks: {started_tasks}/{expected_starts}") - time.sleep(1) # Sleep to reduce busy waiting - - for _ in range(5): - logger.info("*" * 40) - logger.info("All readers and writers have started. Waiting for input events...") - - if status_pipe: - status_pipe.send({"type": "record.started"}) - global stop_sequence_detected stop_sequence_detected = False - try: - while not (stop_sequence_detected or terminate_processing.is_set()): - time.sleep(1) - terminate_processing.set() - except KeyboardInterrupt: - terminate_processing.set() + startup_ready = _wait_for_tasks_started( + task_by_name, + task_started_events, + terminate_processing, + ) + if startup_ready: + for _ in range(5): + logger.info("*" * 40) + logger.info( + "All readers and writers have started. Waiting for input events..." + ) + + if status_pipe: + status_pipe.send({"type": "record.started"}) + + try: + while not (stop_sequence_detected or terminate_processing.is_set()): + terminate_processing.wait(1) + except KeyboardInterrupt: + terminate_processing.set() + else: + logger.info("Tearing down recording after incomplete startup") + terminate_processing.set() if status_pipe: status_pipe.send({"type": "record.stopping"}) @@ -1912,14 +2005,11 @@ def record( collect_stats(performance_snapshots) log_memory_usage(_tracker, performance_snapshots) - def join_tasks(task_names: list[str]) -> None: - for task_name in task_names: - if task_name in task_by_name: - logger.info(f"joining {task_name=}...") - task = task_by_name[task_name] - task.join() - - join_tasks( + pre_ready_timeout = ( + None if startup_ready else PRE_READY_TASK_JOIN_TIMEOUT_SECONDS + ) + _join_tasks( + task_by_name, [ "window_event_reader", "browser_event_reader", @@ -1933,18 +2023,23 @@ def join_tasks(task_names: list[str]) -> None: "window_event_writer", "video_writer", "audio_recorder", - ] + ], + timeout=pre_ready_timeout, ) terminate_perf_event.set() - join_tasks( + _join_tasks( + task_by_name, [ "perf_stats_writer", "mem_writer", - ] + ], + timeout=pre_ready_timeout, ) - if config.PLOT_PERFORMANCE: + if config.PLOT_PERFORMANCE and startup_ready: + from openadapt_capture import plotting + session = get_session_for_path(db_path) plotting.plot_performance( session, recording, save_dir=capture_dir, @@ -2119,6 +2214,7 @@ def __init__( self._status_recv, self._status_send = multiprocessing.Pipe(duplex=False) self._ready_event = threading.Event() self._stopped_event = threading.Event() + self._ready_or_stopped_event = threading.Event() # Internal self._record_thread: threading.Thread | None = None @@ -2134,8 +2230,10 @@ def _drain_status_pipe(self) -> None: if isinstance(msg, dict): if msg.get("type") == "record.started": self._ready_event.set() + self._ready_or_stopped_event.set() elif msg.get("type") == "record.stopped": self._stopped_event.set() + self._ready_or_stopped_event.set() except (EOFError, OSError): pass @@ -2143,20 +2241,30 @@ def _run_record(self) -> None: """Thread target: apply config overrides, then call record().""" from openadapt_capture.config import config_override - with config_override(self._recording_config): - record( - task_description=self.task_description, - capture_dir=self.capture_dir, - terminate_processing=self._terminate_processing, - terminate_recording=self._terminate_recording, - status_pipe=self._status_send, - num_action_events=self._num_action_events, - num_screen_events=self._num_screen_events, - num_window_events=self._num_window_events, - num_browser_events=self._num_browser_events, - num_video_events=self._num_video_events, - send_profile=self._send_profile, - ) + try: + with config_override(self._recording_config): + record( + task_description=self.task_description, + capture_dir=self.capture_dir, + terminate_processing=self._terminate_processing, + terminate_recording=self._terminate_recording, + status_pipe=self._status_send, + num_action_events=self._num_action_events, + num_screen_events=self._num_screen_events, + num_window_events=self._num_window_events, + num_browser_events=self._num_browser_events, + num_video_events=self._num_video_events, + send_profile=self._send_profile, + ) + except BaseException: + # A setup exception must wake wait_for_ready() and let context-manager + # teardown finish instead of leaving callers blocked for its timeout. + try: + self._status_send.send({"type": "record.stopped"}) + except (BrokenPipeError, EOFError, OSError): + self._stopped_event.set() + self._ready_or_stopped_event.set() + raise def __enter__(self) -> "Recorder": # Start status drain thread @@ -2185,9 +2293,10 @@ def stop(self) -> None: def wait_for_ready(self, timeout: float = 60) -> bool: """Block until all recording threads/processes have started. - Returns True if ready, False if timeout expired. + Returns True if ready, False if startup stopped or the timeout expired. """ - return self._ready_event.wait(timeout=timeout) + self._ready_or_stopped_event.wait(timeout=timeout) + return self._ready_event.is_set() @property def is_recording(self) -> bool: diff --git a/openadapt_capture/stats.py b/openadapt_capture/stats.py index 464749c..7423fc3 100644 --- a/openadapt_capture/stats.py +++ b/openadapt_capture/stats.py @@ -117,6 +117,10 @@ def plot( PIL Image if neither output_path nor show, else None. """ try: + import matplotlib + + if not show: + matplotlib.use("Agg") import matplotlib.pyplot as plt except ImportError: raise ImportError( diff --git a/openadapt_capture/window/_macos.py b/openadapt_capture/window/_macos.py index 415c195..7df0acb 100644 --- a/openadapt_capture/window/_macos.py +++ b/openadapt_capture/window/_macos.py @@ -1,11 +1,7 @@ -"""macOS platform window capture using Quartz/AppKit. - -Copied from legacy OpenAdapt window/_macos.py. Only import paths changed. -""" +"""macOS window and accessibility capture using native PyObjC frameworks.""" import pickle import plistlib -import re import time from pprint import pprint from typing import Any, Literal, Union @@ -14,11 +10,11 @@ import AppKit import ApplicationServices import Foundation - import oa_atomacos import Quartz except ImportError as e: raise ImportError( - f"macOS window capture requires AppKit, Quartz, and oa_atomacos: {e}" + f"macOS window capture requires PyObjC AppKit, Quartz, and " + f"ApplicationServices: {e}" ) from loguru import logger @@ -257,24 +253,10 @@ def deepconvert_objc(object: Any) -> Any | list | dict | Literal[0]: value = {deepconvert_objc(k): deepconvert_objc(v) for k, v in object.items()} elif isinstance(object, strings): value = str(object) - # handle core-foundation class AXValueRef + # Handle Core Foundation AXValueRef without a third-party accessibility + # wrapper. PyObjC returns ``(success, value)`` from AXValueGetValue. elif isinstance(object, ApplicationServices.AXValueRef): - # convert to dict - note: this object is not iterable - # TODO: access directly, e.g. via - # ApplicationServices.AXUIElementCopyAttributeValue - rep = repr(object) - x_value = re.search(r"x:([\d.]+)", rep) - y_value = re.search(r"y:([\d.]+)", rep) - w_value = re.search(r"w:([\d.]+)", rep) - h_value = re.search(r"h:([\d.]+)", rep) - type_value = re.search(r"type\s?=\s?(\w+)", rep) - value = { - "x": float(x_value.group(1)) if x_value else None, - "y": float(y_value.group(1)) if y_value else None, - "w": float(w_value.group(1)) if w_value else None, - "h": float(h_value.group(1)) if h_value else None, - "type": type_value.group(1) if type_value else None, - } + value = _convert_ax_value(object) elif isinstance(object, Foundation.NSURL): value = str(object.absoluteString()) elif isinstance(object, Foundation.__NSCFAttributedString): @@ -294,11 +276,50 @@ def deepconvert_objc(object: Any) -> Any | list | dict | Literal[0]: "github.com/OpenAdaptAI/openadapt-capture/issues/new" ) logger.warning(f"{object=}") - if value: - value = oa_atomacos._converter.Converter().convert_value(value) return value +def _convert_ax_value(value: ApplicationServices.AXValueRef) -> Any: + """Convert a PyObjC AXValue into stable, pickle-safe primitives.""" + value_type = ApplicationServices.AXValueGetType(value) + success, raw_value = ApplicationServices.AXValueGetValue( + value, + value_type, + None, + ) + if not success: + logger.warning(f"Could not convert AXValue of type {value_type}") + return None + + if value_type == ApplicationServices.kAXValueCGPointType: + return { + "x": float(raw_value.x), + "y": float(raw_value.y), + "type": "CGPoint", + } + if value_type == ApplicationServices.kAXValueCGSizeType: + return { + "w": float(raw_value.width), + "h": float(raw_value.height), + "type": "CGSize", + } + if value_type == ApplicationServices.kAXValueCGRectType: + return { + "x": float(raw_value.origin.x), + "y": float(raw_value.origin.y), + "w": float(raw_value.size.width), + "h": float(raw_value.size.height), + "type": "CGRect", + } + if value_type == ApplicationServices.kAXValueCFRangeType: + return { + "location": int(raw_value.location), + "length": int(raw_value.length), + "type": "CFRange", + } + return str(raw_value) + + def get_active_element_state(x: int, y: int) -> dict: """Get the state of the active element at the specified coordinates. @@ -309,11 +330,23 @@ def get_active_element_state(x: int, y: int) -> dict: Returns: dict: A dictionary containing the state of the active element. """ - window_meta = get_active_window_meta() - pid = window_meta["kCGWindowOwnerPID"] - app = oa_atomacos._a11y.AXUIElement.from_pid(pid) - el = app.get_element_at_position(x, y) - state = dump_state(el.ref) + system_wide = ApplicationServices.AXUIElementCreateSystemWide() + error_code, element = ( + ApplicationServices.AXUIElementCopyElementAtPosition( + system_wide, + float(x), + float(y), + None, + ) + ) + if error_code != ApplicationServices.kAXErrorSuccess or element is None: + logger.warning( + "Could not resolve accessibility element at " + f"({x}, {y}): AX error {error_code}" + ) + return {} + + state = dump_state(element) state = deepconvert_objc(state) try: pickle.dumps(state, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/pyproject.toml b/pyproject.toml index 320bd1b..0fce451 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,11 @@ dependencies = [ "pympler>=1.0.0", "tqdm>=4.0.0", "numpy>=1.26.0", - "oa-atomacos>=3.2.0; sys_platform == 'darwin'", + # Recorder generates a performance plot by default, and the package-level + # API imports the plotting module while exposing Recorder. + "matplotlib>=3.10.8", + # Native macOS window/accessibility observation via permissive PyObjC. + "pyobjc-framework-ApplicationServices>=12.2.1; sys_platform == 'darwin'", ] [project.optional-dependencies] @@ -76,7 +80,6 @@ dev = [ "pytest-cov>=4.0.0", "pytest-timeout>=2.0.0", "ruff>=0.1.0", - "matplotlib>=3.5.0", "numpy>=1.21.0", ] @@ -128,7 +131,6 @@ patch_tags = ["fix", "perf"] [dependency-groups] dev = [ - "matplotlib>=3.10.8", "numpy>=2.2.6", "psutil>=7.2.2", "pytest>=9.0.2", diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py new file mode 100644 index 0000000..7b61f98 --- /dev/null +++ b/scripts/verify_distribution.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Verify release archives preserve the MIT package boundary.""" + +from __future__ import annotations + +import argparse +import tarfile +import zipfile +from pathlib import Path + +FORBIDDEN_DEPENDENCIES = ("oa-atomacos",) +FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos",) + + +def _archive_files(path: Path) -> dict[str, bytes]: + if path.suffix == ".whl": + with zipfile.ZipFile(path) as archive: + return { + name: archive.read(name) + for name in archive.namelist() + if not name.endswith("/") + } + if path.name.endswith(".tar.gz"): + with tarfile.open(path, "r:gz") as archive: + return { + member.name: extracted.read() + for member in archive.getmembers() + if member.isfile() + and (extracted := archive.extractfile(member)) is not None + } + raise ValueError(f"Unsupported distribution archive: {path}") + + +def verify_distribution(path: Path) -> None: + files = _archive_files(path) + assert any(Path(name).name == "LICENSE" for name in files), ( + f"{path}: MIT LICENSE file is missing" + ) + + metadata_files = [ + content.decode("utf-8") + for name, content in files.items() + if name.endswith((".dist-info/METADATA", "/PKG-INFO")) + ] + assert metadata_files, f"{path}: package metadata is missing" + metadata = "\n".join(metadata_files).lower() + for dependency in FORBIDDEN_DEPENDENCIES: + assert f"requires-dist: {dependency}" not in metadata, ( + f"{path}: forbidden dependency {dependency!r} is in package metadata" + ) + + python_sources = "\n".join( + content.decode("utf-8") + for name, content in files.items() + if name.endswith(".py") + and "/openadapt_capture/" in f"/{name}" + ) + for token in FORBIDDEN_SOURCE_TOKENS: + assert token not in python_sources, ( + f"{path}: forbidden source token {token!r} is in the archive" + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("archives", nargs="+", type=Path) + args = parser.parse_args() + for archive in args.archives: + verify_distribution(archive) + print(f"verified {archive}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_highlevel.py b/tests/test_highlevel.py index 4669110..cd707bc 100644 --- a/tests/test_highlevel.py +++ b/tests/test_highlevel.py @@ -4,6 +4,7 @@ """ import tempfile +import threading import time from pathlib import Path @@ -14,8 +15,11 @@ # Recorder requires pynput which needs a display server try: - from openadapt_capture.recorder import Recorder + from openadapt_capture import recorder as recorder_module + + Recorder = recorder_module.Recorder except ImportError: + recorder_module = None Recorder = None @@ -136,6 +140,52 @@ def test_recorder_video_frame_count_property(self): rec = Recorder("/tmp/test_never_created") assert rec.video_frame_count == 0 + def test_stop_during_incomplete_startup_returns_promptly( + self, monkeypatch, tmp_path + ): + """A worker that never announces readiness cannot trap context teardown.""" + worker_entered = threading.Event() + + def stalled_record( + *, + terminate_processing, + terminate_recording, + status_pipe, + **_kwargs, + ): + def wait_for_shutdown(): + worker_entered.set() + terminate_processing.wait() + + worker = threading.Thread(target=wait_for_shutdown, daemon=True) + worker.start() + tasks = {"never_ready": worker} + started_events = {"never_ready": threading.Event()} + + assert not recorder_module._wait_for_tasks_started( + tasks, + started_events, + terminate_processing, + ) + assert not recorder_module._join_tasks( + tasks, + ["never_ready"], + timeout=0.5, + ) + status_pipe.send({"type": "record.stopped"}) + terminate_recording.set() + + monkeypatch.setattr(recorder_module, "record", stalled_record) + recorder = Recorder(str(tmp_path / "incomplete-startup")) + + with recorder: + assert worker_entered.wait(timeout=1) + stop_started = time.monotonic() + recorder.stop() + + assert time.monotonic() - stop_started < 1 + assert recorder.wait_for_ready(timeout=0) is False + class TestCapture: """Tests for Capture/CaptureSession class.""" diff --git a/tests/test_macos_accessibility.py b/tests/test_macos_accessibility.py new file mode 100644 index 0000000..87e3608 --- /dev/null +++ b/tests/test_macos_accessibility.py @@ -0,0 +1,85 @@ +"""Tests for the permissive native macOS accessibility implementation.""" + +from __future__ import annotations + +import sys + +import pytest + +pytestmark = pytest.mark.skipif( + sys.platform != "darwin", + reason="PyObjC accessibility APIs are available only on macOS", +) + + +def test_ax_geometry_values_convert_to_primitives() -> None: + import ApplicationServices + + from openadapt_capture.window import _macos + + point = ApplicationServices.AXValueCreate( + ApplicationServices.kAXValueCGPointType, + (1.5, 2.5), + ) + size = ApplicationServices.AXValueCreate( + ApplicationServices.kAXValueCGSizeType, + (3.5, 4.5), + ) + rect = ApplicationServices.AXValueCreate( + ApplicationServices.kAXValueCGRectType, + ((1.0, 2.0), (3.0, 4.0)), + ) + + assert _macos.deepconvert_objc(point) == { + "x": 1.5, + "y": 2.5, + "type": "CGPoint", + } + assert _macos.deepconvert_objc(size) == { + "w": 3.5, + "h": 4.5, + "type": "CGSize", + } + assert _macos.deepconvert_objc(rect) == { + "x": 1.0, + "y": 2.0, + "w": 3.0, + "h": 4.0, + "type": "CGRect", + } + + +def test_element_at_position_uses_native_application_services( + monkeypatch, +) -> None: + from openadapt_capture.window import _macos + + system_wide = object() + element = object() + calls = [] + + monkeypatch.setattr( + _macos.ApplicationServices, + "AXUIElementCreateSystemWide", + lambda: system_wide, + ) + + def copy_element(system, x, y, output): + calls.append((system, x, y, output)) + return _macos.ApplicationServices.kAXErrorSuccess, element + + monkeypatch.setattr( + _macos.ApplicationServices, + "AXUIElementCopyElementAtPosition", + copy_element, + ) + monkeypatch.setattr( + _macos, + "dump_state", + lambda candidate: {"AXRole": "AXButton"} if candidate is element else {}, + ) + + assert _macos.get_active_element_state(120, 240) == { + "AXRole": "AXButton" + } + assert calls == [(system_wide, 120.0, 240.0, None)] diff --git a/tests/test_runtime_import_contract.py b/tests/test_runtime_import_contract.py new file mode 100644 index 0000000..969e18d --- /dev/null +++ b/tests/test_runtime_import_contract.py @@ -0,0 +1,70 @@ +"""Contracts for the installed package's default runtime API.""" + +from __future__ import annotations + +import ast +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _runtime_dependency_names() -> set[str]: + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search( + r"(?ms)^dependencies = \[\n(?P.*?)^\]\n", + pyproject, + ) + assert match is not None + requirements = ast.literal_eval("[" + match.group("requirements") + "]") + return { + re.split(r"[\s\[<>=!~;]", requirement, maxsplit=1)[0].lower() + for requirement in requirements + } + + +def test_recorder_import_dependencies_are_in_the_default_install() -> None: + """The default package API must not rely on development-only packages.""" + assert "matplotlib" in _runtime_dependency_names() + + +def test_macos_accessibility_uses_permissive_runtime_dependencies() -> None: + """The MIT package must not pull a GPL accessibility wrapper.""" + runtime_dependencies = _runtime_dependency_names() + assert "oa-atomacos" not in runtime_dependencies + assert "pyobjc-framework-applicationservices" in runtime_dependencies + + package_sources = "\n".join( + path.read_text(encoding="utf-8") + for path in (ROOT / "openadapt_capture").rglob("*.py") + ) + assert "oa_atomacos" not in package_sources + + +def test_default_install_exposes_recorder() -> None: + """Recorder import never fails because a runtime dependency is undeclared.""" + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import importlib; " + "\ntry:\n" + " module = importlib.import_module('openadapt_capture.recorder')\n" + "except ImportError as exc:\n" + " assert exc.name != 'matplotlib' and " + "'matplotlib' not in str(exc), exc\n" + "else:\n" + " assert module.Recorder is not None\n" + ), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + "Recorder depended on an undeclared default-runtime package.\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) From e5895dfaaa8c97226bce875087fdd7016932c8e5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:02:40 -0700 Subject: [PATCH 02/26] feat: add native input observer contract --- openadapt_capture/input_observer/__init__.py | 31 +++ openadapt_capture/input_observer/base.py | 201 +++++++++++++++++++ openadapt_capture/input_observer/factory.py | 41 ++++ 3 files changed, 273 insertions(+) create mode 100644 openadapt_capture/input_observer/__init__.py create mode 100644 openadapt_capture/input_observer/base.py create mode 100644 openadapt_capture/input_observer/factory.py diff --git a/openadapt_capture/input_observer/__init__.py b/openadapt_capture/input_observer/__init__.py new file mode 100644 index 0000000..a5c8a6e --- /dev/null +++ b/openadapt_capture/input_observer/__init__.py @@ -0,0 +1,31 @@ +"""Permissively licensed native global input observation.""" + +from .base import ( + InputCallback, + InputObserver, + InputObserverError, + InputObserverPermissionError, + InputObserverUnavailableError, + ObservedInput, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + ThreadedInputObserver, +) +from .factory import create_input_observer + +__all__ = [ + "InputCallback", + "InputObserver", + "InputObserverError", + "InputObserverPermissionError", + "InputObserverUnavailableError", + "ObservedInput", + "ObservedKey", + "ObservedMouseButton", + "ObservedMouseMove", + "ObservedMouseScroll", + "ThreadedInputObserver", + "create_input_observer", +] diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py new file mode 100644 index 0000000..11d75c6 --- /dev/null +++ b/openadapt_capture/input_observer/base.py @@ -0,0 +1,201 @@ +"""Platform-neutral contracts for native global input observation.""" + +from __future__ import annotations + +import threading +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Callable, TypeAlias + + +class InputObserverError(RuntimeError): + """Base error for fail-loud input observation.""" + + +class InputObserverUnavailableError(InputObserverError): + """The current platform/session cannot provide complete input observation.""" + + +class InputObserverPermissionError(InputObserverError): + """The operating system denied global input-observation permission.""" + + +@dataclass(frozen=True, slots=True) +class ObservedMouseMove: + """A global mouse-pointer movement in logical screen coordinates.""" + + x: float + y: float + injected: bool = False + + +@dataclass(frozen=True, slots=True) +class ObservedMouseButton: + """A global mouse-button transition.""" + + x: float + y: float + button: str + pressed: bool + injected: bool = False + + +@dataclass(frozen=True, slots=True) +class ObservedMouseScroll: + """A global scroll event using platform-normalized wheel units.""" + + x: float + y: float + dx: float + dy: float + injected: bool = False + + +@dataclass(frozen=True, slots=True) +class ObservedKey: + """A global key transition with both physical and canonical identity.""" + + pressed: bool + key_name: str | None = None + key_char: str | None = None + key_vk: str | None = None + canonical_key_name: str | None = None + canonical_key_char: str | None = None + canonical_key_vk: str | None = None + injected: bool = False + + +ObservedInput: TypeAlias = ( + ObservedMouseMove | ObservedMouseButton | ObservedMouseScroll | ObservedKey +) +InputCallback: TypeAlias = Callable[[ObservedInput], None] + + +class InputObserver(ABC): + """Lifecycle contract for a complete native input observer.""" + + @abstractmethod + def start(self) -> None: + """Start observing, or raise before returning if setup is incomplete.""" + + @abstractmethod + def check_health(self) -> None: + """Raise if the observer failed after startup.""" + + @abstractmethod + def stop(self) -> None: + """Stop and join the observer, surfacing any observer failure.""" + + def __enter__(self) -> "InputObserver": + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> None: + self.stop() + + +class ThreadedInputObserver(InputObserver): + """Shared fail-loud lifecycle for native event-loop implementations.""" + + def __init__( + self, + callback: InputCallback, + *, + observe_keyboard: bool, + observe_mouse: bool, + capture_mouse_moves: bool, + startup_timeout: float = 5.0, + shutdown_timeout: float = 5.0, + ) -> None: + if not observe_keyboard and not observe_mouse: + raise ValueError("at least one of observe_keyboard or observe_mouse is required") + self.callback = callback + self.observe_keyboard = observe_keyboard + self.observe_mouse = observe_mouse + self.capture_mouse_moves = capture_mouse_moves + self.startup_timeout = startup_timeout + self.shutdown_timeout = shutdown_timeout + self._ready = threading.Event() + self._stop_requested = threading.Event() + self._thread: threading.Thread | None = None + self._failure: BaseException | None = None + + @abstractmethod + def _setup(self) -> None: + """Create platform resources on the observer thread.""" + + @abstractmethod + def _run_loop(self) -> None: + """Run until ``_stop_requested`` is set.""" + + @abstractmethod + def _teardown(self) -> None: + """Release any resources created by ``_setup``.""" + + def _wake(self) -> None: + """Wake a blocked event loop during shutdown, when needed.""" + + def _emit(self, event: ObservedInput) -> None: + self.callback(event) + + def _thread_main(self) -> None: + try: + self._setup() + self._ready.set() + self._run_loop() + except BaseException as exc: + self._failure = exc + self._ready.set() + finally: + try: + self._teardown() + except BaseException as exc: + if self._failure is None: + self._failure = exc + + def start(self) -> None: + if self._thread is not None and self._thread.is_alive(): + return + self._ready.clear() + self._stop_requested.clear() + self._failure = None + self._thread = threading.Thread( + target=self._thread_main, + name=f"{type(self).__name__}-event-loop", + daemon=True, + ) + self._thread.start() + if not self._ready.wait(self.startup_timeout): + self._stop_requested.set() + self._wake() + raise InputObserverError( + f"{type(self).__name__} did not become ready within " + f"{self.startup_timeout:.1f}s" + ) + self.check_health() + + def check_health(self) -> None: + if self._failure is not None: + if isinstance(self._failure, InputObserverError): + raise self._failure + raise InputObserverError( + f"{type(self).__name__} failed: {self._failure}" + ) from self._failure + if self._thread is not None and not self._thread.is_alive(): + raise InputObserverError(f"{type(self).__name__} stopped unexpectedly") + + def stop(self) -> None: + thread = self._thread + if thread is None: + return + self._stop_requested.set() + self._wake() + thread.join(self.shutdown_timeout) + if thread.is_alive(): + raise InputObserverError( + f"{type(self).__name__} did not stop within " + f"{self.shutdown_timeout:.1f}s" + ) + self._thread = None + self.check_health() + diff --git a/openadapt_capture/input_observer/factory.py b/openadapt_capture/input_observer/factory.py new file mode 100644 index 0000000..f5c63d5 --- /dev/null +++ b/openadapt_capture/input_observer/factory.py @@ -0,0 +1,41 @@ +"""Factory for the supported native input observers.""" + +from __future__ import annotations + +import sys + +from .base import InputCallback, InputObserver, InputObserverUnavailableError + + +def create_input_observer( + callback: InputCallback, + *, + observe_keyboard: bool = True, + observe_mouse: bool = True, + capture_mouse_moves: bool = True, + platform_name: str | None = None, +) -> InputObserver: + """Create the complete native observer for the current desktop platform.""" + selected = platform_name or sys.platform + kwargs = { + "callback": callback, + "observe_keyboard": observe_keyboard, + "observe_mouse": observe_mouse, + "capture_mouse_moves": capture_mouse_moves, + } + if selected == "darwin": + from .darwin import DarwinInputObserver + + return DarwinInputObserver(**kwargs) + if selected == "win32": + from .windows import WindowsInputObserver + + return WindowsInputObserver(**kwargs) + if selected.startswith("linux"): + from .linux import LinuxXInputObserver + + return LinuxXInputObserver(**kwargs) + raise InputObserverUnavailableError( + f"native input observation is not implemented for platform {selected!r}" + ) + From 334200b95caeb01d1309268a771a532da1733d51 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:13:31 -0700 Subject: [PATCH 03/26] feat: add native macOS input observer --- openadapt_capture/input_observer/darwin.py | 514 +++++++++++++++++++++ tests/test_input_observer_darwin.py | 509 ++++++++++++++++++++ 2 files changed, 1023 insertions(+) create mode 100644 openadapt_capture/input_observer/darwin.py create mode 100644 tests/test_input_observer_darwin.py diff --git a/openadapt_capture/input_observer/darwin.py b/openadapt_capture/input_observer/darwin.py new file mode 100644 index 0000000..e7e4792 --- /dev/null +++ b/openadapt_capture/input_observer/darwin.py @@ -0,0 +1,514 @@ +"""Native macOS global input observation through Quartz CGEventTap.""" + +from __future__ import annotations + +from typing import Any + +from .base import ( + InputCallback, + InputObserverError, + InputObserverPermissionError, + InputObserverUnavailableError, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + ThreadedInputObserver, +) + +_KEY_NAMES = { + 36: "enter", + 48: "tab", + 49: "space", + 51: "backspace", + 53: "esc", + 54: "cmd_r", + 55: "cmd_l", + 56: "shift_l", + 57: "caps_lock", + 58: "alt_l", + 59: "ctrl_l", + 60: "shift_r", + 61: "alt_r", + 62: "ctrl_r", + 63: "fn", + 65: "decimal", + 67: "multiply", + 69: "add", + 71: "clear", + 75: "divide", + 76: "enter", + 78: "subtract", + 81: "equals", + 82: "num0", + 83: "num1", + 84: "num2", + 85: "num3", + 86: "num4", + 87: "num5", + 88: "num6", + 89: "num7", + 91: "num8", + 92: "num9", + 96: "f5", + 97: "f6", + 98: "f7", + 99: "f3", + 100: "f8", + 101: "f9", + 103: "f11", + 105: "f13", + 106: "f16", + 107: "f14", + 109: "f10", + 111: "f12", + 113: "f15", + 114: "insert", + 115: "home", + 116: "page_up", + 117: "delete", + 118: "f4", + 119: "end", + 120: "f2", + 121: "page_down", + 122: "f1", + 123: "left", + 124: "right", + 125: "down", + 126: "up", +} + +_CANONICAL_KEY_NAMES = { + "cmd_l": "cmd", + "cmd_r": "cmd", + "shift_l": "shift", + "shift_r": "shift", + "alt_l": "alt", + "alt_r": "alt", + "ctrl_l": "ctrl", + "ctrl_r": "ctrl", +} + + +class DarwinInputObserver(ThreadedInputObserver): + """Observe macOS keyboard and mouse input without modifying the event stream.""" + + def __init__( + self, + callback: InputCallback, + *, + observe_keyboard: bool, + observe_mouse: bool, + capture_mouse_moves: bool, + startup_timeout: float = 5.0, + shutdown_timeout: float = 5.0, + _quartz: Any | None = None, + _application_services: Any | None = None, + ) -> None: + super().__init__( + callback, + observe_keyboard=observe_keyboard, + observe_mouse=observe_mouse, + capture_mouse_moves=capture_mouse_moves, + startup_timeout=startup_timeout, + shutdown_timeout=shutdown_timeout, + ) + self._quartz = _quartz + self._application_services = _application_services + self._event_tap: Any | None = None + self._cf_run_loop: Any | None = None + self._run_loop_source: Any | None = None + self._tap_callback_ref: Any | None = None + self._pressed_modifier_codes: set[int] = set() + + def _load_frameworks(self) -> None: + if self._quartz is not None and self._application_services is not None: + return + try: + import ApplicationServices + import Quartz + except ImportError as exc: + raise InputObserverUnavailableError( + "macOS input observation requires the permissively licensed " + "PyObjC Quartz and ApplicationServices frameworks" + ) from exc + self._quartz = Quartz + self._application_services = ApplicationServices + + def _setup(self) -> None: + self._load_frameworks() + quartz = self._quartz + + if not self._has_observation_permission(): + raise InputObserverPermissionError( + "macOS denied global input observation. Enable Input Monitoring " + "for OpenAdapt in System Settings > Privacy & Security." + ) + + event_mask = 0 + for event_type in self._observed_event_types(): + event_mask |= quartz.CGEventMaskBit(event_type) + + self._tap_callback_ref = self._event_callback + self._event_tap = quartz.CGEventTapCreate( + quartz.kCGHIDEventTap, + quartz.kCGHeadInsertEventTap, + quartz.kCGEventTapOptionListenOnly, + event_mask, + self._tap_callback_ref, + None, + ) + if self._event_tap is None: + if not self._has_observation_permission(): + raise InputObserverPermissionError( + "macOS Input Monitoring permission was revoked while " + "OpenAdapt was starting" + ) + raise InputObserverUnavailableError( + "macOS could not create a listen-only Quartz event tap" + ) + + self._cf_run_loop = quartz.CFRunLoopGetCurrent() + self._run_loop_source = quartz.CFMachPortCreateRunLoopSource( + None, + self._event_tap, + 0, + ) + if self._run_loop_source is None: + raise InputObserverUnavailableError( + "macOS could not create a run-loop source for the input observer" + ) + quartz.CFRunLoopAddSource( + self._cf_run_loop, + self._run_loop_source, + quartz.kCFRunLoopDefaultMode, + ) + quartz.CGEventTapEnable(self._event_tap, True) + + def _has_observation_permission(self) -> bool: + preflight = getattr( + self._quartz, + "CGPreflightListenEventAccess", + None, + ) + if callable(preflight): + return bool(preflight()) + + # CGPreflightListenEventAccess was introduced after AX event taps. + # On older supported macOS versions, Accessibility trust is the + # fail-loud permission oracle available through ApplicationServices. + accessibility_check = getattr( + self._application_services, + "AXIsProcessTrusted", + None, + ) + if callable(accessibility_check): + return bool(accessibility_check()) + raise InputObserverUnavailableError( + "macOS does not expose an input-observation permission API" + ) + + def _observed_event_types(self) -> tuple[int, ...]: + quartz = self._quartz + event_types: list[int] = [] + if self.observe_keyboard: + event_types.extend( + ( + quartz.kCGEventKeyDown, + quartz.kCGEventKeyUp, + quartz.kCGEventFlagsChanged, + ) + ) + if self.observe_mouse: + event_types.extend( + ( + quartz.kCGEventLeftMouseDown, + quartz.kCGEventLeftMouseUp, + quartz.kCGEventRightMouseDown, + quartz.kCGEventRightMouseUp, + quartz.kCGEventOtherMouseDown, + quartz.kCGEventOtherMouseUp, + quartz.kCGEventScrollWheel, + ) + ) + if self.capture_mouse_moves: + event_types.extend( + ( + quartz.kCGEventMouseMoved, + quartz.kCGEventLeftMouseDragged, + quartz.kCGEventRightMouseDragged, + quartz.kCGEventOtherMouseDragged, + ) + ) + return tuple(event_types) + + def _run_loop(self) -> None: + quartz = self._quartz + while not self._stop_requested.is_set(): + quartz.CFRunLoopRunInMode( + quartz.kCFRunLoopDefaultMode, + 0.1, + False, + ) + + def _wake(self) -> None: + if self._cf_run_loop is not None and self._quartz is not None: + self._quartz.CFRunLoopStop(self._cf_run_loop) + + def _teardown(self) -> None: + quartz = self._quartz + if quartz is None: + return + if self._event_tap is not None: + quartz.CGEventTapEnable(self._event_tap, False) + if self._cf_run_loop is not None and self._run_loop_source is not None: + quartz.CFRunLoopRemoveSource( + self._cf_run_loop, + self._run_loop_source, + quartz.kCFRunLoopDefaultMode, + ) + if self._run_loop_source is not None: + quartz.CFRunLoopSourceInvalidate(self._run_loop_source) + if self._event_tap is not None: + quartz.CFMachPortInvalidate(self._event_tap) + # PyObjC owns the Core Foundation references returned above. Dropping + # the Python references after invalidation releases them safely; an + # extra CFRelease here would risk over-releasing PyObjC-owned objects. + self._run_loop_source = None + self._cf_run_loop = None + self._event_tap = None + self._tap_callback_ref = None + self._pressed_modifier_codes.clear() + + def _event_callback( + self, + _proxy: Any, + event_type: int, + event: Any, + _refcon: Any, + ) -> Any: + quartz = self._quartz + if event_type in ( + quartz.kCGEventTapDisabledByTimeout, + quartz.kCGEventTapDisabledByUserInput, + ): + if self._event_tap is not None: + quartz.CGEventTapEnable(self._event_tap, True) + return event + + try: + self._handle_event(event_type, event) + except BaseException as exc: + self._failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError(f"macOS input callback failed: {exc}") + ) + self._stop_requested.set() + self._wake() + return event + + def _handle_event(self, event_type: int, event: Any) -> None: + quartz = self._quartz + injected = self._is_injected(event) + + if event_type in self._mouse_move_event_types(): + if self.observe_mouse and self.capture_mouse_moves: + x, y = self._location(event) + self._emit(ObservedMouseMove(x=x, y=y, injected=injected)) + return + + button_event = self._mouse_button_event(event_type, event, injected) + if button_event is not None: + self._emit(button_event) + return + + if event_type == quartz.kCGEventScrollWheel and self.observe_mouse: + x, y = self._location(event) + self._emit( + ObservedMouseScroll( + x=x, + y=y, + dx=float( + quartz.CGEventGetIntegerValueField( + event, + quartz.kCGScrollWheelEventDeltaAxis2, + ) + ), + dy=float( + quartz.CGEventGetIntegerValueField( + event, + quartz.kCGScrollWheelEventDeltaAxis1, + ) + ), + injected=injected, + ) + ) + return + + if event_type in (quartz.kCGEventKeyDown, quartz.kCGEventKeyUp): + if self.observe_keyboard: + self._emit( + self._key_event( + event, + pressed=event_type == quartz.kCGEventKeyDown, + injected=injected, + ) + ) + return + + if event_type == quartz.kCGEventFlagsChanged and self.observe_keyboard: + keycode = self._keycode(event) + pressed = self._modifier_pressed(event, keycode) + if pressed: + self._pressed_modifier_codes.add(keycode) + else: + self._pressed_modifier_codes.discard(keycode) + self._emit( + self._key_event( + event, + pressed=pressed, + injected=injected, + include_character=False, + ) + ) + + def _modifier_pressed(self, event: Any, keycode: int) -> bool: + quartz = self._quartz + modifier_masks = { + 54: quartz.kCGEventFlagMaskCommand, + 55: quartz.kCGEventFlagMaskCommand, + 56: quartz.kCGEventFlagMaskShift, + 57: quartz.kCGEventFlagMaskAlphaShift, + 58: quartz.kCGEventFlagMaskAlternate, + 59: quartz.kCGEventFlagMaskControl, + 60: quartz.kCGEventFlagMaskShift, + 61: quartz.kCGEventFlagMaskAlternate, + 62: quartz.kCGEventFlagMaskControl, + 63: quartz.kCGEventFlagMaskSecondaryFn, + } + mask = modifier_masks.get(keycode) + if mask is None: + raise InputObserverError( + f"macOS reported flags-changed for unknown keycode {keycode}" + ) + + flags = int(quartz.CGEventGetFlags(event)) + flag_is_active = bool(flags & mask) + if keycode in self._pressed_modifier_codes: + # This physical key was previously pressed. It is now released + # even when a sibling key sharing the same aggregate flag remains + # down (for example left Shift released while right Shift is held). + return False + # Deriving the initial observation from the OS flag makes a first + # observed release after startup a release rather than a toggle drift. + return flag_is_active + + def _mouse_move_event_types(self) -> tuple[int, ...]: + quartz = self._quartz + return ( + quartz.kCGEventMouseMoved, + quartz.kCGEventLeftMouseDragged, + quartz.kCGEventRightMouseDragged, + quartz.kCGEventOtherMouseDragged, + ) + + def _mouse_button_event( + self, + event_type: int, + event: Any, + injected: bool, + ) -> ObservedMouseButton | None: + if not self.observe_mouse: + return None + quartz = self._quartz + transitions = { + quartz.kCGEventLeftMouseDown: ("left", True), + quartz.kCGEventLeftMouseUp: ("left", False), + quartz.kCGEventRightMouseDown: ("right", True), + quartz.kCGEventRightMouseUp: ("right", False), + quartz.kCGEventOtherMouseDown: (None, True), + quartz.kCGEventOtherMouseUp: (None, False), + } + transition = transitions.get(event_type) + if transition is None: + return None + button, pressed = transition + if button is None: + number = int( + quartz.CGEventGetIntegerValueField( + event, + quartz.kCGMouseEventButtonNumber, + ) + ) + button = "middle" if number == 2 else f"button{number + 1}" + x, y = self._location(event) + return ObservedMouseButton( + x=x, + y=y, + button=button, + pressed=pressed, + injected=injected, + ) + + def _key_event( + self, + event: Any, + *, + pressed: bool, + injected: bool, + include_character: bool = True, + ) -> ObservedKey: + keycode = self._keycode(event) + key_name = _KEY_NAMES.get(keycode) + key_char = ( + self._keyboard_character(event) + if include_character and key_name is None + else None + ) + canonical_name = _CANONICAL_KEY_NAMES.get(key_name, key_name) + canonical_char = key_char.lower() if key_char is not None else None + vk = str(keycode) + return ObservedKey( + pressed=pressed, + key_name=key_name, + key_char=key_char, + key_vk=vk, + canonical_key_name=canonical_name, + canonical_key_char=canonical_char, + canonical_key_vk=vk, + injected=injected, + ) + + def _keyboard_character(self, event: Any) -> str | None: + length, text = self._quartz.CGEventKeyboardGetUnicodeString( + event, + 8, + None, + None, + ) + if length <= 0 or not text: + return None + return str(text)[: int(length)] + + def _keycode(self, event: Any) -> int: + return int( + self._quartz.CGEventGetIntegerValueField( + event, + self._quartz.kCGKeyboardEventKeycode, + ) + ) + + def _location(self, event: Any) -> tuple[float, float]: + point = self._quartz.CGEventGetLocation(event) + return float(point.x), float(point.y) + + def _is_injected(self, event: Any) -> bool: + quartz = self._quartz + source_pid_field = getattr(quartz, "kCGEventSourceUnixProcessID", None) + if source_pid_field is None: + return False + source_pid = int( + quartz.CGEventGetIntegerValueField(event, source_pid_field) + ) + return source_pid > 0 diff --git a/tests/test_input_observer_darwin.py b/tests/test_input_observer_darwin.py new file mode 100644 index 0000000..da40421 --- /dev/null +++ b/tests/test_input_observer_darwin.py @@ -0,0 +1,509 @@ +"""Mocked macOS native-input observer contracts.""" + +from __future__ import annotations + +import time +from types import SimpleNamespace + +import pytest + +from openadapt_capture.input_observer.base import ( + InputObserverError, + InputObserverPermissionError, + InputObserverUnavailableError, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, +) +from openadapt_capture.input_observer.darwin import DarwinInputObserver + + +class FakeEvent: + def __init__( + self, + *, + x: float = 10, + y: float = 20, + fields: dict[int, int] | None = None, + text: str = "", + flags: int = 0, + ) -> None: + self.location = SimpleNamespace(x=x, y=y) + self.fields = fields or {} + self.text = text + self.flags = flags + + +class FakeQuartz: + kCGEventKeyDown = 1 + kCGEventKeyUp = 2 + kCGEventFlagsChanged = 3 + kCGEventLeftMouseDown = 4 + kCGEventLeftMouseUp = 5 + kCGEventRightMouseDown = 6 + kCGEventRightMouseUp = 7 + kCGEventOtherMouseDown = 8 + kCGEventOtherMouseUp = 9 + kCGEventScrollWheel = 10 + kCGEventMouseMoved = 11 + kCGEventLeftMouseDragged = 12 + kCGEventRightMouseDragged = 13 + kCGEventOtherMouseDragged = 14 + kCGEventTapDisabledByTimeout = 100 + kCGEventTapDisabledByUserInput = 101 + + kCGKeyboardEventKeycode = 200 + kCGMouseEventButtonNumber = 201 + kCGScrollWheelEventDeltaAxis1 = 202 + kCGScrollWheelEventDeltaAxis2 = 203 + kCGEventSourceUnixProcessID = 204 + kCGEventFlagMaskAlphaShift = 1 << 0 + kCGEventFlagMaskShift = 1 << 1 + kCGEventFlagMaskControl = 1 << 2 + kCGEventFlagMaskAlternate = 1 << 3 + kCGEventFlagMaskCommand = 1 << 4 + kCGEventFlagMaskSecondaryFn = 1 << 5 + + kCGHIDEventTap = 300 + kCGHeadInsertEventTap = 301 + kCGEventTapOptionListenOnly = 302 + kCFRunLoopDefaultMode = "default" + + def __init__( + self, + *, + permission: bool = True, + create_tap: bool = True, + ) -> None: + self.permission = permission + self.create_tap = create_tap + self.tap = object() + self.loop = object() + self.source = object() + self.callback = None + self.event_mask = None + self.tap_enabled: list[bool] = [] + self.added_sources = [] + self.removed_sources = [] + self.stopped_loops = [] + self.invalidated_sources = [] + self.invalidated_ports = [] + + def CGPreflightListenEventAccess(self): + return self.permission + + @staticmethod + def CGEventMaskBit(event_type): + return 1 << event_type + + def CGEventTapCreate( + self, + tap_location, + placement, + options, + event_mask, + callback, + refcon, + ): + assert tap_location == self.kCGHIDEventTap + assert placement == self.kCGHeadInsertEventTap + assert options == self.kCGEventTapOptionListenOnly + assert refcon is None + self.event_mask = event_mask + self.callback = callback + return self.tap if self.create_tap else None + + def CFRunLoopGetCurrent(self): + return self.loop + + def CFMachPortCreateRunLoopSource(self, allocator, tap, order): + assert allocator is None + assert tap is self.tap + assert order == 0 + return self.source + + def CFRunLoopAddSource(self, loop, source, mode): + self.added_sources.append((loop, source, mode)) + + @staticmethod + def CFRunLoopRunInMode(_mode, seconds, _return_after_source): + time.sleep(min(seconds, 0.002)) + return 0 + + def CFRunLoopStop(self, loop): + self.stopped_loops.append(loop) + + def CFRunLoopRemoveSource(self, loop, source, mode): + self.removed_sources.append((loop, source, mode)) + + def CFRunLoopSourceInvalidate(self, source): + self.invalidated_sources.append(source) + + def CFMachPortInvalidate(self, port): + self.invalidated_ports.append(port) + + def CGEventTapEnable(self, tap, enabled): + assert tap is self.tap + self.tap_enabled.append(enabled) + + @staticmethod + def CGEventGetLocation(event): + return event.location + + @staticmethod + def CGEventGetIntegerValueField(event, field): + return event.fields.get(field, 0) + + @staticmethod + def CGEventGetFlags(event): + return event.flags + + @staticmethod + def CGEventKeyboardGetUnicodeString(event, max_length, _length, _buffer): + text = event.text[:max_length] + return len(text), text + + +class FakeApplicationServices: + def __init__(self, *, trusted: bool = True) -> None: + self.trusted = trusted + + def AXIsProcessTrusted(self): + return self.trusted + + +def make_observer( + quartz: FakeQuartz, + callback, + *, + observe_keyboard: bool = True, + observe_mouse: bool = True, + capture_mouse_moves: bool = True, + application_services=None, +) -> DarwinInputObserver: + return DarwinInputObserver( + callback, + observe_keyboard=observe_keyboard, + observe_mouse=observe_mouse, + capture_mouse_moves=capture_mouse_moves, + startup_timeout=0.5, + shutdown_timeout=0.5, + _quartz=quartz, + _application_services=application_services or FakeApplicationServices(), + ) + + +def test_lifecycle_creates_listen_only_tap_and_releases_run_loop() -> None: + quartz = FakeQuartz() + observer = make_observer(quartz, lambda _event: None) + + observer.start() + observer.check_health() + observer.stop() + + assert quartz.callback is not None + assert quartz.tap_enabled == [True, False] + assert quartz.added_sources == [ + (quartz.loop, quartz.source, quartz.kCFRunLoopDefaultMode) + ] + assert quartz.removed_sources == quartz.added_sources + assert quartz.invalidated_sources == [quartz.source] + assert quartz.invalidated_ports == [quartz.tap] + assert quartz.loop in quartz.stopped_loops + for event_type in observer._observed_event_types(): + assert quartz.event_mask & quartz.CGEventMaskBit(event_type) + + +def test_permission_denial_fails_before_event_tap_creation() -> None: + quartz = FakeQuartz(permission=False) + observer = make_observer(quartz, lambda _event: None) + + with pytest.raises(InputObserverPermissionError, match="Input Monitoring"): + observer.start() + assert quartz.callback is None + + +def test_accessibility_permission_is_fallback_on_older_macos() -> None: + quartz = FakeQuartz() + quartz.CGPreflightListenEventAccess = None + observer = make_observer( + quartz, + lambda _event: None, + application_services=FakeApplicationServices(trusted=False), + ) + + with pytest.raises(InputObserverPermissionError, match="Input Monitoring"): + observer.start() + assert quartz.callback is None + + +def test_event_tap_creation_failure_is_explicit() -> None: + quartz = FakeQuartz(create_tap=False) + observer = make_observer(quartz, lambda _event: None) + + with pytest.raises( + InputObserverUnavailableError, + match="listen-only Quartz event tap", + ): + observer.start() + + +def test_mouse_move_button_and_scroll_normalization() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + physical = {quartz.kCGEventSourceUnixProcessID: 0} + + observer._handle_event( + quartz.kCGEventMouseMoved, + FakeEvent(x=11.5, y=22.5, fields=physical), + ) + observer._handle_event( + quartz.kCGEventLeftMouseDown, + FakeEvent(x=30, y=40, fields=physical), + ) + observer._handle_event( + quartz.kCGEventOtherMouseUp, + FakeEvent( + x=50, + y=60, + fields={ + **physical, + quartz.kCGMouseEventButtonNumber: 2, + }, + ), + ) + observer._handle_event( + quartz.kCGEventScrollWheel, + FakeEvent( + x=70, + y=80, + fields={ + **physical, + quartz.kCGScrollWheelEventDeltaAxis1: 3, + quartz.kCGScrollWheelEventDeltaAxis2: -2, + }, + ), + ) + + assert events == [ + ObservedMouseMove(x=11.5, y=22.5), + ObservedMouseButton( + x=30, + y=40, + button="left", + pressed=True, + ), + ObservedMouseButton( + x=50, + y=60, + button="middle", + pressed=False, + ), + ObservedMouseScroll(x=70, y=80, dx=-2, dy=3), + ] + + +def test_move_filter_does_not_disable_buttons() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer( + quartz, + events.append, + capture_mouse_moves=False, + ) + + observer._handle_event(quartz.kCGEventMouseMoved, FakeEvent()) + observer._handle_event(quartz.kCGEventRightMouseUp, FakeEvent()) + + assert events == [ + ObservedMouseButton( + x=10, + y=20, + button="right", + pressed=False, + ) + ] + assert quartz.kCGEventMouseMoved not in observer._observed_event_types() + + +def test_key_press_release_and_modifier_canonicalization() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + + observer._handle_event( + quartz.kCGEventKeyDown, + FakeEvent( + fields={ + quartz.kCGKeyboardEventKeycode: 0, + quartz.kCGEventSourceUnixProcessID: 123, + }, + text="A", + ), + ) + observer._handle_event( + quartz.kCGEventKeyUp, + FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 0}, + text="a", + ), + ) + shift_press = FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 56}, + flags=quartz.kCGEventFlagMaskShift, + ) + shift_release = FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 56}, + ) + observer._handle_event(quartz.kCGEventFlagsChanged, shift_press) + observer._handle_event(quartz.kCGEventFlagsChanged, shift_release) + + assert events == [ + ObservedKey( + pressed=True, + key_char="A", + key_vk="0", + canonical_key_char="a", + canonical_key_vk="0", + injected=True, + ), + ObservedKey( + pressed=False, + key_char="a", + key_vk="0", + canonical_key_char="a", + canonical_key_vk="0", + ), + ObservedKey( + pressed=True, + key_name="shift_l", + key_vk="56", + canonical_key_name="shift", + canonical_key_vk="56", + ), + ObservedKey( + pressed=False, + key_name="shift_l", + key_vk="56", + canonical_key_name="shift", + canonical_key_vk="56", + ), + ] + + +def test_first_observed_modifier_release_does_not_toggle_to_press() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + + observer._handle_event( + quartz.kCGEventFlagsChanged, + FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 56}, + flags=0, + ), + ) + + assert events == [ + ObservedKey( + pressed=False, + key_name="shift_l", + key_vk="56", + canonical_key_name="shift", + canonical_key_vk="56", + ) + ] + + +def test_caps_lock_state_comes_from_event_flags() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + + observer._handle_event( + quartz.kCGEventFlagsChanged, + FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 57}, + flags=quartz.kCGEventFlagMaskAlphaShift, + ), + ) + observer._handle_event( + quartz.kCGEventFlagsChanged, + FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: 57}, + flags=0, + ), + ) + + assert [(event.key_name, event.pressed) for event in events] == [ + ("caps_lock", True), + ("caps_lock", False), + ] + + +def test_releasing_one_shift_ignores_the_sibling_aggregate_flag() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + + for keycode, flags in [ + (56, quartz.kCGEventFlagMaskShift), + (60, quartz.kCGEventFlagMaskShift), + (56, quartz.kCGEventFlagMaskShift), + (60, 0), + ]: + observer._handle_event( + quartz.kCGEventFlagsChanged, + FakeEvent( + fields={quartz.kCGKeyboardEventKeycode: keycode}, + flags=flags, + ), + ) + + assert [(event.key_name, event.pressed) for event in events] == [ + ("shift_l", True), + ("shift_r", True), + ("shift_l", False), + ("shift_r", False), + ] + + +def test_disabled_tap_is_reenabled_without_emitting_input() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + observer._event_tap = quartz.tap + event = FakeEvent() + + assert ( + observer._event_callback( + None, + quartz.kCGEventTapDisabledByTimeout, + event, + None, + ) + is event + ) + assert quartz.tap_enabled == [True] + assert events == [] + + +def test_callback_failure_becomes_health_failure() -> None: + quartz = FakeQuartz() + + def fail(_event): + raise ValueError("callback boom") + + observer = make_observer(quartz, fail) + event = FakeEvent() + observer._handle_event = lambda _type, _event: observer._emit( # type: ignore[method-assign] + ObservedMouseMove(1, 2) + ) + + observer._event_callback(None, quartz.kCGEventMouseMoved, event, None) + + with pytest.raises(InputObserverError, match="callback boom"): + observer.check_health() + assert observer._stop_requested.is_set() From 1ea551cf1a743184c3218df9becc048f0b1d827e Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:21:32 -0700 Subject: [PATCH 04/26] fix: make input delivery bounded and fail loud --- openadapt_capture/input_observer/__init__.py | 2 + openadapt_capture/input_observer/base.py | 159 +++++++++++++++++-- 2 files changed, 149 insertions(+), 12 deletions(-) diff --git a/openadapt_capture/input_observer/__init__.py b/openadapt_capture/input_observer/__init__.py index a5c8a6e..933488e 100644 --- a/openadapt_capture/input_observer/__init__.py +++ b/openadapt_capture/input_observer/__init__.py @@ -12,6 +12,7 @@ ObservedMouseMove, ObservedMouseScroll, ThreadedInputObserver, + add_exception_note, ) from .factory import create_input_observer @@ -27,5 +28,6 @@ "ObservedMouseMove", "ObservedMouseScroll", "ThreadedInputObserver", + "add_exception_note", "create_input_observer", ] diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index 11d75c6..b2fe076 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -2,6 +2,7 @@ from __future__ import annotations +import queue import threading from abc import ABC, abstractmethod from dataclasses import dataclass @@ -71,6 +72,13 @@ class ObservedKey: InputCallback: TypeAlias = Callable[[ObservedInput], None] +def add_exception_note(error: BaseException, note: str) -> None: + """Attach cleanup context when supported without masking the primary error.""" + add_note = getattr(error, "add_note", None) + if add_note is not None: + add_note(note) + + class InputObserver(ABC): """Lifecycle contract for a complete native input observer.""" @@ -106,6 +114,7 @@ def __init__( capture_mouse_moves: bool, startup_timeout: float = 5.0, shutdown_timeout: float = 5.0, + delivery_queue_size: int = 4096, ) -> None: if not observe_keyboard and not observe_mouse: raise ValueError("at least one of observe_keyboard or observe_mouse is required") @@ -115,10 +124,20 @@ def __init__( self.capture_mouse_moves = capture_mouse_moves self.startup_timeout = startup_timeout self.shutdown_timeout = shutdown_timeout + if delivery_queue_size <= 0: + raise ValueError("delivery_queue_size must be positive") + self.delivery_queue_size = delivery_queue_size self._ready = threading.Event() self._stop_requested = threading.Event() self._thread: threading.Thread | None = None + self._delivery_thread: threading.Thread | None = None + self._delivery_queue: queue.Queue[ObservedInput | object] = queue.Queue( + maxsize=delivery_queue_size + ) + self._delivery_sentinel = object() self._failure: BaseException | None = None + self._failure_lock = threading.Lock() + self._startup_failure: BaseException | None = None @abstractmethod def _setup(self) -> None: @@ -136,7 +155,79 @@ def _wake(self) -> None: """Wake a blocked event loop during shutdown, when needed.""" def _emit(self, event: ObservedInput) -> None: - self.callback(event) + try: + self._delivery_queue.put_nowait(event) + except queue.Full: + failure = InputObserverError( + f"{type(self).__name__} input delivery queue overflowed; " + "recording coverage is incomplete" + ) + self._fail(failure) + raise failure + + def _fail(self, failure: BaseException) -> None: + with self._failure_lock: + if self._failure is None: + self._failure = failure + self._stop_requested.set() + self._wake() + + def _delivery_main(self) -> None: + while True: + item = self._delivery_queue.get() + try: + if item is self._delivery_sentinel: + return + self.callback(item) # type: ignore[arg-type] + except BaseException as exc: + failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError( + f"{type(self).__name__} input consumer failed: {exc}" + ) + ) + self._fail(failure) + return + finally: + self._delivery_queue.task_done() + + def _start_delivery(self) -> None: + self._delivery_queue = queue.Queue(maxsize=self.delivery_queue_size) + self._delivery_thread = threading.Thread( + target=self._delivery_main, + name=f"{type(self).__name__}-delivery", + daemon=True, + ) + self._delivery_thread.start() + + def _stop_delivery(self) -> None: + thread = self._delivery_thread + if thread is None: + return + if thread.is_alive(): + try: + self._delivery_queue.put( + self._delivery_sentinel, + timeout=self.shutdown_timeout, + ) + except queue.Full: + self._fail( + InputObserverError( + f"{type(self).__name__} delivery queue could not drain " + "during shutdown" + ) + ) + thread.join(self.shutdown_timeout) + if thread.is_alive(): + self._fail( + InputObserverError( + f"{type(self).__name__} delivery thread did not stop within " + f"{self.shutdown_timeout:.1f}s" + ) + ) + else: + self._delivery_thread = None def _thread_main(self) -> None: try: @@ -144,7 +235,9 @@ def _thread_main(self) -> None: self._ready.set() self._run_loop() except BaseException as exc: - self._failure = exc + with self._failure_lock: + if self._failure is None: + self._failure = exc self._ready.set() finally: try: @@ -155,10 +248,17 @@ def _thread_main(self) -> None: def start(self) -> None: if self._thread is not None and self._thread.is_alive(): + if self._startup_failure is not None: + raise InputObserverError( + f"{type(self).__name__} still has a live thread from a " + "failed startup; create a new observer after fixing the cause" + ) from self._startup_failure return self._ready.clear() self._stop_requested.clear() self._failure = None + self._startup_failure = None + self._start_delivery() self._thread = threading.Thread( target=self._thread_main, name=f"{type(self).__name__}-event-loop", @@ -166,23 +266,58 @@ def start(self) -> None: ) self._thread.start() if not self._ready.wait(self.startup_timeout): - self._stop_requested.set() - self._wake() - raise InputObserverError( + primary = InputObserverError( f"{type(self).__name__} did not become ready within " f"{self.startup_timeout:.1f}s" ) - self.check_health() + self._abort_start(primary) + try: + self.check_health() + except BaseException as exc: + self._abort_start(exc) + + def _abort_start(self, primary: BaseException) -> None: + """Make a failed ``start`` transactional before surfacing its cause.""" + self._startup_failure = primary + thread = self._thread + if thread is None: + self._stop_delivery() + raise primary + self._stop_requested.set() + try: + self._wake() + except BaseException as cleanup_exc: + add_exception_note( + primary, + f"observer wake during failed startup also failed: {cleanup_exc}" + ) + thread.join(self.shutdown_timeout) + if thread.is_alive(): + add_exception_note( + primary, + f"{type(self).__name__} setup thread did not stop within " + f"{self.shutdown_timeout:.1f}s", + ) + else: + self._thread = None + self._stop_delivery() + raise primary def check_health(self) -> None: - if self._failure is not None: - if isinstance(self._failure, InputObserverError): - raise self._failure + with self._failure_lock: + failure = self._failure + if failure is not None: + if isinstance(failure, InputObserverError): + raise failure raise InputObserverError( - f"{type(self).__name__} failed: {self._failure}" - ) from self._failure + f"{type(self).__name__} failed: {failure}" + ) from failure if self._thread is not None and not self._thread.is_alive(): raise InputObserverError(f"{type(self).__name__} stopped unexpectedly") + if self._delivery_thread is not None and not self._delivery_thread.is_alive(): + raise InputObserverError( + f"{type(self).__name__} delivery stopped unexpectedly" + ) def stop(self) -> None: thread = self._thread @@ -197,5 +332,5 @@ def stop(self) -> None: f"{self.shutdown_timeout:.1f}s" ) self._thread = None + self._stop_delivery() self.check_health() - From 604bad321a0f3467666a9f6036c39f917fad682e Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:22:43 -0700 Subject: [PATCH 05/26] fix: complete observer startup cleanup --- openadapt_capture/input_observer/base.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index b2fe076..7a359f4 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -169,8 +169,15 @@ def _fail(self, failure: BaseException) -> None: with self._failure_lock: if self._failure is None: self._failure = failure + primary = self._failure self._stop_requested.set() - self._wake() + try: + self._wake() + except BaseException as wake_error: + add_exception_note( + primary, + f"waking the failed observer also failed: {wake_error}", + ) def _delivery_main(self) -> None: while True: @@ -264,7 +271,13 @@ def start(self) -> None: name=f"{type(self).__name__}-event-loop", daemon=True, ) - self._thread.start() + try: + self._thread.start() + except BaseException as exc: + self._startup_failure = exc + self._thread = None + self._stop_delivery() + raise if not self._ready.wait(self.startup_timeout): primary = InputObserverError( f"{type(self).__name__} did not become ready within " From 3f3f346e93cf553538bc6dfc22b2c028849c7487 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:29:31 -0700 Subject: [PATCH 06/26] fix: harden native observer lifecycle --- openadapt_capture/input_observer/base.py | 96 +++++++++++++++++++----- 1 file changed, 77 insertions(+), 19 deletions(-) diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index 7a359f4..46a5b37 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -166,10 +166,7 @@ def _emit(self, event: ObservedInput) -> None: raise failure def _fail(self, failure: BaseException) -> None: - with self._failure_lock: - if self._failure is None: - self._failure = failure - primary = self._failure + primary = self._store_failure(failure) self._stop_requested.set() try: self._wake() @@ -179,6 +176,14 @@ def _fail(self, failure: BaseException) -> None: f"waking the failed observer also failed: {wake_error}", ) + def _store_failure(self, failure: BaseException) -> BaseException: + """Record and return the first failure atomically.""" + with self._failure_lock: + if self._failure is None: + self._failure = failure + primary = self._failure + return primary + def _delivery_main(self) -> None: while True: item = self._delivery_queue.get() @@ -200,13 +205,24 @@ def _delivery_main(self) -> None: self._delivery_queue.task_done() def _start_delivery(self) -> None: + if self._delivery_thread is not None: + state = "live" if self._delivery_thread.is_alive() else "stopped" + raise InputObserverError( + f"{type(self).__name__} has a {state} delivery thread from a " + "previous lifecycle; stop it or create a new observer" + ) self._delivery_queue = queue.Queue(maxsize=self.delivery_queue_size) - self._delivery_thread = threading.Thread( + thread = threading.Thread( target=self._delivery_main, name=f"{type(self).__name__}-delivery", daemon=True, ) - self._delivery_thread.start() + self._delivery_thread = thread + try: + thread.start() + except BaseException: + self._delivery_thread = None + raise def _stop_delivery(self) -> None: thread = self._delivery_thread @@ -250,22 +266,41 @@ def _thread_main(self) -> None: try: self._teardown() except BaseException as exc: - if self._failure is None: - self._failure = exc + self._store_failure(exc) def start(self) -> None: - if self._thread is not None and self._thread.is_alive(): - if self._startup_failure is not None: - raise InputObserverError( - f"{type(self).__name__} still has a live thread from a " - "failed startup; create a new observer after fixing the cause" - ) from self._startup_failure - return + if self._thread is not None: + if self._thread.is_alive(): + if self._startup_failure is not None: + raise InputObserverError( + f"{type(self).__name__} still has a live thread from a " + "failed startup; create a new observer after fixing the cause" + ) from self._startup_failure + self.check_health() + return + self._stop_delivery() + self._thread = None + self.check_health() + raise InputObserverError( + f"{type(self).__name__} cannot restart after its event loop " + "stopped unexpectedly; create a new observer" + ) + if self._delivery_thread is not None: + state = "live" if self._delivery_thread.is_alive() else "stopped" + raise InputObserverError( + f"{type(self).__name__} still has a {state} delivery thread " + "from a previous lifecycle; stop it or create a new observer" + ) self._ready.clear() self._stop_requested.clear() self._failure = None self._startup_failure = None - self._start_delivery() + try: + self._start_delivery() + except BaseException as exc: + self._startup_failure = exc + self._stop_delivery() + raise self._thread = threading.Thread( target=self._thread_main, name=f"{type(self).__name__}-event-loop", @@ -335,15 +370,38 @@ def check_health(self) -> None: def stop(self) -> None: thread = self._thread if thread is None: + self._stop_delivery() + self.check_health() return self._stop_requested.set() - self._wake() + primary: BaseException | None = None + try: + self._wake() + except BaseException as wake_error: + primary = wake_error thread.join(self.shutdown_timeout) if thread.is_alive(): - raise InputObserverError( + timeout_error = InputObserverError( f"{type(self).__name__} did not stop within " f"{self.shutdown_timeout:.1f}s" ) - self._thread = None + if primary is None: + primary = timeout_error + else: + add_exception_note( + primary, + f"observer event loop also failed to stop: {timeout_error}", + ) + else: + self._thread = None self._stop_delivery() + if primary is not None: + with self._failure_lock: + cleanup_failure = self._failure + if cleanup_failure is not None: + add_exception_note( + primary, + f"observer shutdown also recorded: {cleanup_failure}", + ) + raise primary self.check_health() From 109a779d09812186ecbfb47928c4f9097d2b0082 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:33:17 -0700 Subject: [PATCH 07/26] fix: replace pynput with native input observers --- .github/workflows/test.yml | 10 +- openadapt_capture/__init__.py | 24 +- openadapt_capture/capture.py | 5 - openadapt_capture/events.py | 10 +- openadapt_capture/input.py | 338 ++++++------- openadapt_capture/input_observer/darwin.py | 25 +- openadapt_capture/input_observer/linux.py | 524 +++++++++++++++++++++ openadapt_capture/recorder.py | 341 ++++++++------ openadapt_capture/visualize/html.py | 2 +- openadapt_capture/window_capture.py | 6 +- pyproject.toml | 3 +- scripts/verify_distribution.py | 4 +- tests/test_highlevel.py | 13 +- tests/test_input_observer.py | 493 +++++++++++++++++++ tests/test_input_observer_darwin.py | 58 ++- tests/test_performance.py | 13 +- tests/test_runtime_import_contract.py | 10 + tests/test_window_capture.py | 12 +- 18 files changed, 1465 insertions(+), 426 deletions(-) create mode 100644 openadapt_capture/input_observer/linux.py create mode 100644 tests/test_input_observer.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 804d13f..1317196 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,12 +38,12 @@ jobs: # Windows is the primary live-recording platform: this job runs the full # unit suite AND the live Recorder integration tests (tests/test_performance.py, - # marked 'slow'), which start the real multiprocessing pipeline (pynput - # listeners -> writer processes -> per-capture SQLite + video) and verify + # marked 'slow'), which start the real multiprocessing pipeline (native + # observers -> writer processes -> per-capture SQLite + video) and verify # clean startup, bounded shutdown/memory, and db creation. The three # listener-dependent tests (roundtrip/reuse/throughput) skip here via # OPENADAPT_CI_NO_INPUT_INJECTION: hosted runners execute jobs in a - # non-interactive session, so injected input never reaches pynput's hooks; + # non-interactive session, so injected input never reaches native hooks; # run those on an interactive Windows desktop. These tests skip everywhere # except Windows, so this job is the org's live-recording CI proof. test-windows: @@ -68,7 +68,7 @@ jobs: - name: Run live recorder integration tests env: - # Non-interactive session: SendInput never reaches pynput's + # Non-interactive session: SendInput never reaches native # low-level hooks, so the event-capture tests skip (see # tests/test_performance.py); pipeline start/stop/db/memory # tests still run the live recorder for real. @@ -79,7 +79,7 @@ jobs: # macOS coverage: runs the unit suite (event processing, storage, video # encode/decode via PyAV, headless-import invariant, Recorder API surface — # CGWindowList-backed screen capture works on GitHub macOS runners). The - # 'slow' live-recording tests stay skipped here: pynput input injection + # 'slow' live-recording tests stay skipped here: synthetic input injection # requires Accessibility permissions a hosted runner cannot grant, and the # multiprocessing 'spawn' writer path is not yet validated on macOS (see # tests/test_performance.py). diff --git a/openadapt_capture/__init__.py b/openadapt_capture/__init__.py index f1b766c..3fca6ff 100644 --- a/openadapt_capture/__init__.py +++ b/openadapt_capture/__init__.py @@ -3,7 +3,12 @@ Platform-agnostic event streams with time-aligned media. """ -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("openadapt-capture") +except PackageNotFoundError: + __version__ = "0+unknown" # High-level APIs (primary interface) from openadapt_capture.capture import Action, Capture, CaptureSession @@ -68,20 +73,9 @@ remove_redundant_mouse_move_events, ) -# Recorder requires pynput which needs a display server (X11/Wayland/macOS/Windows). -# Make it optional so the package is importable in headless environments (CI, servers). -# The recorder must never take a screenshot at import time, but guard against a -# display/screenshot failure (mss ScreenShotError) as well as a missing dependency -# so a headless import degrades to ``Recorder = None`` instead of crashing. -try: - from mss.exception import ScreenShotError as _ScreenShotError -except ImportError: # pragma: no cover - mss is a hard dependency - _ScreenShotError = () # type: ignore[assignment,misc] - -try: - from openadapt_capture.recorder import Recorder -except (ImportError, OSError, _ScreenShotError): - Recorder = None # type: ignore[assignment,misc] +# Recorder imports are display-side-effect free. Platform permissions and +# display availability are checked only when native observers start. +from openadapt_capture.recorder import Recorder # Performance statistics from openadapt_capture.stats import ( diff --git a/openadapt_capture/capture.py b/openadapt_capture/capture.py index 10fc572..0df400e 100644 --- a/openadapt_capture/capture.py +++ b/openadapt_capture/capture.py @@ -30,7 +30,6 @@ KeyDownEvent, KeyTypeEvent, KeyUpEvent, - MouseButton, MouseDownEvent, MouseMoveEvent, MouseScrollEvent, @@ -63,10 +62,6 @@ def _convert_action_event(db_event) -> PydanticActionEvent | None: ) elif db_event.name == "click": button = db_event.mouse_button_name or "left" - try: - button = MouseButton(button) - except ValueError: - button = MouseButton.LEFT if db_event.mouse_pressed is True: return MouseDownEvent( diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index bf526ac..ac5ae5b 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -87,7 +87,7 @@ class MouseDownEvent(BaseEvent): type: Literal[EventType.MOUSE_DOWN] = EventType.MOUSE_DOWN x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButton = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") class MouseUpEvent(BaseEvent): @@ -99,7 +99,7 @@ class MouseUpEvent(BaseEvent): type: Literal[EventType.MOUSE_UP] = EventType.MOUSE_UP x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButton = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") class MouseScrollEvent(BaseEvent): @@ -202,7 +202,7 @@ class MouseClickEvent(BaseEvent): type: Literal[EventType.MOUSE_SINGLECLICK] = EventType.MOUSE_SINGLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButton = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -218,7 +218,7 @@ class MouseDoubleClickEvent(BaseEvent): type: Literal[EventType.MOUSE_DOUBLECLICK] = EventType.MOUSE_DOUBLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButton = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -236,7 +236,7 @@ class MouseDragEvent(BaseEvent): y: float = Field(description="Starting Y position in pixels") dx: float = Field(description="Horizontal displacement (end_x - start_x)") dy: float = Field(description="Vertical displacement (end_y - start_y)") - button: MouseButton = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseMoveEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) diff --git a/openadapt_capture/input.py b/openadapt_capture/input.py index b16f8f6..d2bda4f 100644 --- a/openadapt_capture/input.py +++ b/openadapt_capture/input.py @@ -1,30 +1,28 @@ -"""Input capture for mouse and keyboard events. - -This module provides cross-platform input capture using pynput, -following OpenAdapt's proven implementation. -""" +"""Input capture built on OpenAdapt's native platform observers.""" from __future__ import annotations import threading import time -from typing import TYPE_CHECKING, Any, Callable - -from pynput import keyboard as _pynput_keyboard -from pynput import mouse as _pynput_mouse - -if TYPE_CHECKING: - from pynput import keyboard, mouse +from typing import Any, Callable from openadapt_capture.events import ( KeyDownEvent, KeyUpEvent, - MouseButton, MouseDownEvent, MouseMoveEvent, MouseScrollEvent, MouseUpEvent, ) +from openadapt_capture.input_observer import ( + InputObserver, + ObservedInput, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + create_input_observer, +) def _get_timestamp() -> float: @@ -32,17 +30,67 @@ def _get_timestamp() -> float: return time.time() -def _button_to_mouse_button(button: "mouse.Button") -> MouseButton: - """Convert pynput button to MouseButton enum.""" - name = button.name.lower() - if name == "left": - return MouseButton.LEFT - elif name == "right": - return MouseButton.RIGHT - elif name == "middle": - return MouseButton.MIDDLE - else: - return MouseButton.LEFT # Default fallback +def _to_public_event(observed: ObservedInput): + """Convert a native normalized event to the public Pydantic event model.""" + timestamp = _get_timestamp() + if isinstance(observed, ObservedMouseMove): + return MouseMoveEvent(timestamp=timestamp, x=observed.x, y=observed.y) + if isinstance(observed, ObservedMouseButton): + event_type = MouseDownEvent if observed.pressed else MouseUpEvent + return event_type( + timestamp=timestamp, + x=observed.x, + y=observed.y, + button=observed.button, + ) + if isinstance(observed, ObservedMouseScroll): + return MouseScrollEvent( + timestamp=timestamp, + x=observed.x, + y=observed.y, + dx=observed.dx, + dy=observed.dy, + ) + event_type = KeyDownEvent if observed.pressed else KeyUpEvent + return event_type( + timestamp=timestamp, + key_name=observed.key_name, + key_char=observed.key_char, + key_vk=observed.key_vk, + canonical_key_name=observed.canonical_key_name, + canonical_key_char=observed.canonical_key_char, + canonical_key_vk=observed.canonical_key_vk, + ) + + +class _StopSequenceMatcher: + def __init__( + self, + sequences: list[str] | None, + callback: Callable[[], None] | None, + ) -> None: + self.sequences = [sequence for sequence in (sequences or []) if sequence] + self.callback = callback + self.indices = [0 for _ in self.sequences] + + def observe(self, event: ObservedKey) -> None: + if not event.pressed: + return + candidate = event.canonical_key_char or event.canonical_key_name + if candidate is None: + self.indices = [0 for _ in self.sequences] + return + candidate = candidate.lower() + for index, sequence in enumerate(self.sequences): + expected = sequence[self.indices[index]].lower() + if candidate == expected: + self.indices[index] += 1 + else: + self.indices[index] = 1 if candidate == sequence[0].lower() else 0 + if self.indices[index] == len(sequence): + self.indices[index] = 0 + if self.callback is not None: + self.callback() # ============================================================================= @@ -81,80 +129,40 @@ def __init__( """ self.callback = callback self.capture_moves = capture_moves - self._listener: "mouse.Listener" | None = None + self._observer: InputObserver | None = None self._running = False - def _on_move(self, x: int, y: int) -> None: - """Handle mouse move event.""" - if not self.capture_moves: - return - event = MouseMoveEvent( - timestamp=_get_timestamp(), - x=float(x), - y=float(y), - ) - self.callback(event) - - def _on_click( - self, - x: int, - y: int, - button: "mouse.Button", - pressed: bool, - ) -> None: - """Handle mouse click event.""" - timestamp = _get_timestamp() - mouse_button = _button_to_mouse_button(button) - - if pressed: - event = MouseDownEvent( - timestamp=timestamp, - x=float(x), - y=float(y), - button=mouse_button, - ) - else: - event = MouseUpEvent( - timestamp=timestamp, - x=float(x), - y=float(y), - button=mouse_button, - ) - self.callback(event) - - def _on_scroll(self, x: int, y: int, dx: int, dy: int) -> None: - """Handle mouse scroll event.""" - event = MouseScrollEvent( - timestamp=_get_timestamp(), - x=float(x), - y=float(y), - dx=float(dx), - dy=float(dy), - ) - self.callback(event) + def _on_observed(self, observed: ObservedInput) -> None: + if isinstance( + observed, + (ObservedMouseMove, ObservedMouseButton, ObservedMouseScroll), + ): + self.callback(_to_public_event(observed)) def start(self) -> None: """Start capturing mouse events.""" if self._running: return - - self._listener = _pynput_mouse.Listener( - on_move=self._on_move, - on_click=self._on_click, - on_scroll=self._on_scroll, + observer = create_input_observer( + self._on_observed, + observe_keyboard=False, + observe_mouse=True, + capture_mouse_moves=self.capture_moves, ) - self._listener.start() + observer.start() + self._observer = observer self._running = True def stop(self) -> None: """Stop capturing mouse events.""" if not self._running: return - - if self._listener is not None: - self._listener.stop() - self._listener = None - self._running = False + try: + if self._observer is not None: + self._observer.stop() + finally: + self._observer = None + self._running = False def __enter__(self) -> "MouseListener": """Context manager entry.""" @@ -198,115 +206,28 @@ def __init__( on_stop_sequence: Callback when a stop sequence is detected. """ self.callback = callback - self.stop_sequences = stop_sequences or [] - self.on_stop_sequence = on_stop_sequence - self._listener: "keyboard.Listener" | None = None + self._matcher = _StopSequenceMatcher(stop_sequences, on_stop_sequence) + self._observer: InputObserver | None = None self._running = False - self._stop_sequence_indices = [0 for _ in self.stop_sequences] - - def _extract_key_info( - self, - key: "keyboard.Key | keyboard.KeyCode", - canonical_key: "keyboard.Key | keyboard.KeyCode | None" = None, - ) -> dict[str, str | None]: - """Extract key information from pynput key object.""" - key_name = getattr(key, "name", None) - key_char = getattr(key, "char", None) - key_vk = str(getattr(key, "vk", None)) if hasattr(key, "vk") and key.vk else None - - canonical_key_name = None - canonical_key_char = None - canonical_key_vk = None - - if canonical_key is not None: - canonical_key_name = getattr(canonical_key, "name", None) - canonical_key_char = getattr(canonical_key, "char", None) - canonical_key_vk = ( - str(getattr(canonical_key, "vk", None)) - if hasattr(canonical_key, "vk") and canonical_key.vk - else None - ) - - return { - "key_name": key_name, - "key_char": key_char, - "key_vk": key_vk, - "canonical_key_name": canonical_key_name, - "canonical_key_char": canonical_key_char, - "canonical_key_vk": canonical_key_vk, - } - - def _check_stop_sequence( - self, - key: "keyboard.Key | keyboard.KeyCode", - canonical_key: "keyboard.Key | keyboard.KeyCode | None", - ) -> bool: - """Check if key press completes a stop sequence.""" - if not self.stop_sequences: - return False - - canonical_key_name = getattr(canonical_key, "name", None) if canonical_key else None - - for i, stop_sequence in enumerate(self.stop_sequences): - current_char = stop_sequence[self._stop_sequence_indices[i]] - - # Get canonical representation for comparison - canonical_sequence_key = self._listener.canonical( - _pynput_keyboard.KeyCode.from_char(current_char) - ) - - matches = ( - canonical_key == canonical_sequence_key - or canonical_key_name == current_char - ) - - if matches: - self._stop_sequence_indices[i] += 1 - else: - self._stop_sequence_indices[i] = 0 - - if self._stop_sequence_indices[i] == len(stop_sequence): - return True - - return False - - def _on_press(self, key: "keyboard.Key | keyboard.KeyCode") -> None: - """Handle key press event.""" - canonical_key = self._listener.canonical(key) if self._listener else None - key_info = self._extract_key_info(key, canonical_key) - - event = KeyDownEvent( - timestamp=_get_timestamp(), - **key_info, - ) - self.callback(event) - - # Check stop sequences - if self._check_stop_sequence(key, canonical_key): - if self.on_stop_sequence: - self.on_stop_sequence() - def _on_release(self, key: "keyboard.Key | keyboard.KeyCode") -> None: - """Handle key release event.""" - canonical_key = self._listener.canonical(key) if self._listener else None - key_info = self._extract_key_info(key, canonical_key) - - event = KeyUpEvent( - timestamp=_get_timestamp(), - **key_info, - ) - self.callback(event) + def _on_observed(self, observed: ObservedInput) -> None: + if not isinstance(observed, ObservedKey): + return + self.callback(_to_public_event(observed)) + self._matcher.observe(observed) def start(self) -> None: """Start capturing keyboard events.""" if self._running: return - - self._listener = _pynput_keyboard.Listener( - on_press=self._on_press, - on_release=self._on_release, + observer = create_input_observer( + self._on_observed, + observe_keyboard=True, + observe_mouse=False, + capture_mouse_moves=False, ) - self._listener.start() + observer.start() + self._observer = observer self._running = True def stop(self) -> None: @@ -314,10 +235,12 @@ def stop(self) -> None: if not self._running: return - if self._listener is not None: - self._listener.stop() - self._listener = None - self._running = False + try: + if self._observer is not None: + self._observer.stop() + finally: + self._observer = None + self._running = False def __enter__(self) -> "KeyboardListener": """Context manager entry.""" @@ -365,24 +288,28 @@ def __init__( on_stop_sequence: Callback when a stop sequence is detected. """ self.callback = callback - self._mouse_listener = MouseListener( - callback=callback, - capture_moves=capture_mouse_moves, - ) - self._keyboard_listener = KeyboardListener( - callback=callback, - stop_sequences=stop_sequences, - on_stop_sequence=on_stop_sequence, - ) + self.capture_mouse_moves = capture_mouse_moves + self._matcher = _StopSequenceMatcher(stop_sequences, on_stop_sequence) + self._observer: InputObserver | None = None self._running = False + def _on_observed(self, observed: ObservedInput) -> None: + self.callback(_to_public_event(observed)) + if isinstance(observed, ObservedKey): + self._matcher.observe(observed) + def start(self) -> None: """Start capturing all input events.""" if self._running: return - - self._mouse_listener.start() - self._keyboard_listener.start() + observer = create_input_observer( + self._on_observed, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=self.capture_mouse_moves, + ) + observer.start() + self._observer = observer self._running = True def stop(self) -> None: @@ -390,9 +317,12 @@ def stop(self) -> None: if not self._running: return - self._mouse_listener.stop() - self._keyboard_listener.stop() - self._running = False + try: + if self._observer is not None: + self._observer.stop() + finally: + self._observer = None + self._running = False def __enter__(self) -> "InputListener": """Context manager entry.""" diff --git a/openadapt_capture/input_observer/darwin.py b/openadapt_capture/input_observer/darwin.py index e7e4792..65625b5 100644 --- a/openadapt_capture/input_observer/darwin.py +++ b/openadapt_capture/input_observer/darwin.py @@ -140,10 +140,18 @@ def _setup(self) -> None: quartz = self._quartz if not self._has_observation_permission(): - raise InputObserverPermissionError( - "macOS denied global input observation. Enable Input Monitoring " - "for OpenAdapt in System Settings > Privacy & Security." + request_permission = getattr( + quartz, + "CGRequestListenEventAccess", + None, ) + if callable(request_permission): + request_permission() + if not self._has_observation_permission(): + raise InputObserverPermissionError( + "macOS denied global input observation. Enable Input Monitoring " + "for OpenAdapt in System Settings > Privacy & Security." + ) event_mask = 0 for event_type in self._observed_event_types(): @@ -294,18 +302,23 @@ def _event_callback( ): if self._event_tap is not None: quartz.CGEventTapEnable(self._event_tap, True) + self._fail( + InputObserverError( + "macOS disabled the input event tap; recording coverage is " + "incomplete" + ) + ) return event try: self._handle_event(event_type, event) except BaseException as exc: - self._failure = ( + failure = ( exc if isinstance(exc, InputObserverError) else InputObserverError(f"macOS input callback failed: {exc}") ) - self._stop_requested.set() - self._wake() + self._fail(failure) return event def _handle_event(self, event_type: int, event: Any) -> None: diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py new file mode 100644 index 0000000..c3394d6 --- /dev/null +++ b/openadapt_capture/input_observer/linux.py @@ -0,0 +1,524 @@ +"""Linux global input observation through XInput2 raw events.""" + +from __future__ import annotations + +import ctypes +import ctypes.util +import os +import sys +from typing import Any + +from .base import ( + InputObserverError, + InputObserverPermissionError, + InputObserverUnavailableError, + ObservedInput, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + ThreadedInputObserver, +) + +_GENERIC_EVENT = 35 +_XI_ALL_MASTER_DEVICES = 1 +_XI_RAW_KEY_PRESS = 13 +_XI_RAW_KEY_RELEASE = 14 +_XI_RAW_BUTTON_PRESS = 15 +_XI_RAW_BUTTON_RELEASE = 16 +_XI_RAW_MOTION = 17 + +_SPECIAL_KEY_NAMES = { + "Alt_L": "alt", + "Alt_R": "alt_r", + "BackSpace": "backspace", + "Caps_Lock": "caps_lock", + "Control_L": "ctrl", + "Control_R": "ctrl_r", + "Delete": "delete", + "Down": "down", + "End": "end", + "Escape": "esc", + "Home": "home", + "Left": "left", + "Meta_L": "cmd", + "Meta_R": "cmd_r", + "Page_Down": "page_down", + "Page_Up": "page_up", + "Return": "enter", + "Right": "right", + "Shift_L": "shift", + "Shift_R": "shift_r", + "Super_L": "cmd", + "Super_R": "cmd_r", + "Tab": "tab", + "Up": "up", + "space": "space", +} +_KEYSYM_CHARACTERS = { + "ampersand": "&", + "apostrophe": "'", + "asciicircum": "^", + "asciitilde": "~", + "asterisk": "*", + "at": "@", + "backslash": "\\", + "bar": "|", + "braceleft": "{", + "braceright": "}", + "bracketleft": "[", + "bracketright": "]", + "colon": ":", + "comma": ",", + "dollar": "$", + "equal": "=", + "exclam": "!", + "greater": ">", + "less": "<", + "minus": "-", + "numbersign": "#", + "parenleft": "(", + "parenright": ")", + "percent": "%", + "period": ".", + "plus": "+", + "question": "?", + "quotedbl": '"', + "semicolon": ";", + "slash": "/", + "underscore": "_", +} + + +class _XIEventMask(ctypes.Structure): + _fields_ = [ + ("deviceid", ctypes.c_int), + ("mask_len", ctypes.c_int), + ("mask", ctypes.POINTER(ctypes.c_ubyte)), + ] + + +class _XIButtonState(ctypes.Structure): + _fields_ = [ + ("mask_len", ctypes.c_int), + ("mask", ctypes.POINTER(ctypes.c_ubyte)), + ] + + +class _XIValuatorState(ctypes.Structure): + _fields_ = [ + ("mask_len", ctypes.c_int), + ("mask", ctypes.POINTER(ctypes.c_ubyte)), + ("values", ctypes.POINTER(ctypes.c_double)), + ] + + +class _XGenericEventCookie(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("serial", ctypes.c_ulong), + ("send_event", ctypes.c_int), + ("display", ctypes.c_void_p), + ("extension", ctypes.c_int), + ("evtype", ctypes.c_int), + ("cookie", ctypes.c_uint), + ("data", ctypes.c_void_p), + ] + + +class _XEvent(ctypes.Union): + _fields_ = [ + ("type", ctypes.c_int), + ("xcookie", _XGenericEventCookie), + ("pad", ctypes.c_long * 24), + ] + + +class _XIRawEvent(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int), + ("serial", ctypes.c_ulong), + ("send_event", ctypes.c_int), + ("display", ctypes.c_void_p), + ("extension", ctypes.c_int), + ("evtype", ctypes.c_int), + ("time", ctypes.c_ulong), + ("deviceid", ctypes.c_int), + ("sourceid", ctypes.c_int), + ("detail", ctypes.c_int), + ("root", ctypes.c_ulong), + ("root_x", ctypes.c_double), + ("root_y", ctypes.c_double), + ("flags", ctypes.c_int), + ("buttons", _XIButtonState), + ("valuators", _XIValuatorState), + ("raw_values", ctypes.POINTER(ctypes.c_double)), + ] + + +def normalize_xinput_button_event( + *, + detail: int, + pressed: bool, + x: float, + y: float, + injected: bool = False, +) -> ObservedInput | None: + """Map X11 button numbers to a click or normalized wheel event.""" + if detail in {4, 5, 6, 7}: + if not pressed: + return None + delta_by_button = { + 4: (0.0, 1.0), + 5: (0.0, -1.0), + 6: (-1.0, 0.0), + 7: (1.0, 0.0), + } + dx, dy = delta_by_button[detail] + return ObservedMouseScroll( + x=x, + y=y, + dx=dx, + dy=dy, + injected=injected, + ) + button_by_detail = { + 1: "left", + 2: "middle", + 3: "right", + 8: "x1", + 9: "x2", + } + button = button_by_detail.get(detail) + if button is None and detail > 0: + button = f"button{detail}" + if button is None: + return None + return ObservedMouseButton( + x=x, + y=y, + button=button, + pressed=pressed, + injected=injected, + ) + + +def normalize_xinput_key_event( + *, + keycode: int, + pressed: bool, + keysym_name: str | None, + injected: bool = False, +) -> ObservedKey: + """Normalize an XInput2 keycode and XKB keysym name.""" + key_name = _SPECIAL_KEY_NAMES.get(keysym_name or "") + character = None + if keysym_name: + if len(keysym_name) == 1 and keysym_name.isprintable(): + character = keysym_name + else: + character = _KEYSYM_CHARACTERS.get(keysym_name) + if key_name is None and character is None: + key_name = keysym_name or f"keycode_{keycode}" + virtual_key = str(keycode) + return ObservedKey( + pressed=pressed, + key_name=key_name, + key_char=character, + key_vk=virtual_key, + canonical_key_name=key_name, + canonical_key_char=character.lower() if character else None, + canonical_key_vk=virtual_key, + injected=injected, + ) + + +class LinuxXInputObserver(ThreadedInputObserver): + """Observe complete X11 desktop input with XInput2 raw events.""" + + def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._environ = environ if environ is not None else os.environ + self._x11: Any = None + self._xi: Any = None + self._display: Any = None + self._root = 0 + self._xi_opcode = 0 + self._event_mask_buffer: Any = None + self._shift_keys_down: set[int] = set() + + @staticmethod + def _set_mask(mask: ctypes.Array[ctypes.c_ubyte], event_type: int) -> None: + mask[event_type >> 3] |= 1 << (event_type & 7) + + def _setup(self) -> None: + if not sys.platform.startswith("linux"): + raise InputObserverUnavailableError( + "XInput2 input observation is available only on Linux" + ) + session_type = self._environ.get("XDG_SESSION_TYPE", "").lower() + if session_type == "wayland" or self._environ.get("WAYLAND_DISPLAY"): + raise InputObserverUnavailableError( + "native Wayland does not expose complete global input to XInput2. " + "Run an X11 session; OpenAdapt refuses XWayland-only capture because " + "it would silently omit input from native Wayland applications." + ) + display_name = self._environ.get("DISPLAY") + if not display_name: + raise InputObserverUnavailableError( + "XInput2 requires an X11 desktop and DISPLAY is not set" + ) + + x11_name = ctypes.util.find_library("X11") + xi_name = ctypes.util.find_library("Xi") + if not x11_name or not xi_name: + raise InputObserverUnavailableError( + "XInput2 requires the system libX11 and libXi runtime libraries" + ) + try: + self._x11 = ctypes.CDLL(x11_name) + self._xi = ctypes.CDLL(xi_name) + except OSError as exc: + raise InputObserverUnavailableError( + f"could not load X11 input libraries: {exc}" + ) from exc + self._configure_api() + self._display = self._x11.XOpenDisplay(display_name.encode()) + if not self._display: + raise InputObserverPermissionError( + f"could not open X11 display {display_name!r}; authorize the " + "recording user with the display server and retry" + ) + self._root = int(self._x11.XDefaultRootWindow(self._display)) + + opcode = ctypes.c_int() + first_event = ctypes.c_int() + first_error = ctypes.c_int() + if not self._x11.XQueryExtension( + self._display, + b"XInputExtension", + ctypes.byref(opcode), + ctypes.byref(first_event), + ctypes.byref(first_error), + ): + raise InputObserverUnavailableError( + "the X11 server does not expose the XInput extension" + ) + self._xi_opcode = opcode.value + major = ctypes.c_int(2) + minor = ctypes.c_int(0) + status = self._xi.XIQueryVersion( + self._display, ctypes.byref(major), ctypes.byref(minor) + ) + if status != 0 or major.value < 2: + raise InputObserverUnavailableError( + f"XInput2 is required; server negotiation returned " + f"{major.value}.{minor.value} with status {status}" + ) + + mask = (ctypes.c_ubyte * 4)() + if self.observe_keyboard: + self._set_mask(mask, _XI_RAW_KEY_PRESS) + self._set_mask(mask, _XI_RAW_KEY_RELEASE) + if self.observe_mouse: + self._set_mask(mask, _XI_RAW_BUTTON_PRESS) + self._set_mask(mask, _XI_RAW_BUTTON_RELEASE) + if self.capture_mouse_moves: + self._set_mask(mask, _XI_RAW_MOTION) + event_mask = _XIEventMask( + deviceid=_XI_ALL_MASTER_DEVICES, + mask_len=len(mask), + mask=ctypes.cast(mask, ctypes.POINTER(ctypes.c_ubyte)), + ) + self._event_mask_buffer = mask + if self._xi.XISelectEvents( + self._display, self._root, ctypes.byref(event_mask), 1 + ) != 0: + raise InputObserverPermissionError( + "the X11 server refused XInput2 raw-event selection" + ) + self._x11.XFlush(self._display) + + def _configure_api(self) -> None: + self._x11.XOpenDisplay.argtypes = [ctypes.c_char_p] + self._x11.XOpenDisplay.restype = ctypes.c_void_p + self._x11.XCloseDisplay.argtypes = [ctypes.c_void_p] + self._x11.XDefaultRootWindow.argtypes = [ctypes.c_void_p] + self._x11.XDefaultRootWindow.restype = ctypes.c_ulong + self._x11.XQueryExtension.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ] + self._x11.XPending.argtypes = [ctypes.c_void_p] + self._x11.XPending.restype = ctypes.c_int + self._x11.XNextEvent.argtypes = [ctypes.c_void_p, ctypes.POINTER(_XEvent)] + self._x11.XGetEventData.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(_XGenericEventCookie), + ] + self._x11.XGetEventData.restype = ctypes.c_int + self._x11.XFreeEventData.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(_XGenericEventCookie), + ] + self._x11.XFlush.argtypes = [ctypes.c_void_p] + self._x11.XQueryPointer.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.c_ulong), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_uint), + ] + self._x11.XQueryPointer.restype = ctypes.c_int + self._x11.XkbKeycodeToKeysym.argtypes = [ + ctypes.c_void_p, + ctypes.c_ubyte, + ctypes.c_int, + ctypes.c_int, + ] + self._x11.XkbKeycodeToKeysym.restype = ctypes.c_ulong + self._x11.XKeysymToString.argtypes = [ctypes.c_ulong] + self._x11.XKeysymToString.restype = ctypes.c_char_p + + self._xi.XIQueryVersion.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ] + self._xi.XIQueryVersion.restype = ctypes.c_int + self._xi.XISelectEvents.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ctypes.POINTER(_XIEventMask), + ctypes.c_int, + ] + self._xi.XISelectEvents.restype = ctypes.c_int + + def _query_pointer(self) -> tuple[float, float]: + root_return = ctypes.c_ulong() + child_return = ctypes.c_ulong() + root_x = ctypes.c_int() + root_y = ctypes.c_int() + window_x = ctypes.c_int() + window_y = ctypes.c_int() + mask = ctypes.c_uint() + if not self._x11.XQueryPointer( + self._display, + self._root, + ctypes.byref(root_return), + ctypes.byref(child_return), + ctypes.byref(root_x), + ctypes.byref(root_y), + ctypes.byref(window_x), + ctypes.byref(window_y), + ctypes.byref(mask), + ): + raise InputObserverError( + "XInput2 delivered mouse input but XQueryPointer could not establish " + "its global coordinates" + ) + return float(root_x.value), float(root_y.value) + + def _keysym_name(self, keycode: int, pressed: bool) -> str | None: + unshifted = self._x11.XkbKeycodeToKeysym( + self._display, keycode, 0, 0 + ) + unshifted_name_ptr = self._x11.XKeysymToString(unshifted) if unshifted else None + unshifted_name = ( + unshifted_name_ptr.decode(errors="replace") + if unshifted_name_ptr + else None + ) + is_shift = unshifted_name in {"Shift_L", "Shift_R"} + if is_shift: + if pressed: + self._shift_keys_down.add(keycode) + else: + self._shift_keys_down.discard(keycode) + return unshifted_name + + level = 1 if self._shift_keys_down else 0 + keysym = self._x11.XkbKeycodeToKeysym(self._display, keycode, 0, level) + name_ptr = self._x11.XKeysymToString(keysym) if keysym else None + return name_ptr.decode(errors="replace") if name_ptr else unshifted_name + + def _handle_raw_event(self, raw: _XIRawEvent) -> None: + injected = bool(raw.send_event) + if raw.evtype == _XI_RAW_MOTION: + if self.observe_mouse and self.capture_mouse_moves: + x, y = self._query_pointer() + self._emit(ObservedMouseMove(x=x, y=y, injected=injected)) + return + if raw.evtype in {_XI_RAW_BUTTON_PRESS, _XI_RAW_BUTTON_RELEASE}: + if self.observe_mouse: + x, y = self._query_pointer() + event = normalize_xinput_button_event( + detail=raw.detail, + pressed=raw.evtype == _XI_RAW_BUTTON_PRESS, + x=x, + y=y, + injected=injected, + ) + if event is not None: + self._emit(event) + return + if raw.evtype in {_XI_RAW_KEY_PRESS, _XI_RAW_KEY_RELEASE}: + if self.observe_keyboard: + pressed = raw.evtype == _XI_RAW_KEY_PRESS + self._emit( + normalize_xinput_key_event( + keycode=raw.detail, + pressed=pressed, + keysym_name=self._keysym_name(raw.detail, pressed), + injected=injected, + ) + ) + + def _run_loop(self) -> None: + while not self._stop_requested.is_set(): + while self._x11.XPending(self._display): + event = _XEvent() + self._x11.XNextEvent(self._display, ctypes.byref(event)) + cookie = event.xcookie + if ( + event.type != _GENERIC_EVENT + or cookie.extension != self._xi_opcode + ): + continue + if not self._x11.XGetEventData( + self._display, ctypes.byref(event.xcookie) + ): + raise InputObserverError( + "XInput2 event cookie had no accessible event data" + ) + try: + if event.xcookie.data: + raw = ctypes.cast( + event.xcookie.data, ctypes.POINTER(_XIRawEvent) + ).contents + self._handle_raw_event(raw) + finally: + self._x11.XFreeEventData( + self._display, ctypes.byref(event.xcookie) + ) + self._stop_requested.wait(0.01) + + def _teardown(self) -> None: + if self._display and self._x11 is not None: + self._x11.XCloseDisplay(self._display) + self._display = None + self._event_mask_buffer = None + + +__all__ = [ + "LinuxXInputObserver", + "normalize_xinput_button_event", + "normalize_xinput_key_event", +] diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index be63857..16b6489 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -5,14 +5,13 @@ real desktop workflows. It is the recorder behind OpenAdapt's desktop path (openadapt-flow ``record --backend windows|rdp`` wraps ``Recorder``). -Architecture: a multiprocessing pipeline. pynput listeners feed raw -mouse/keyboard events into synchronized queues; dedicated writer processes +Architecture: a multiprocessing pipeline. Native platform input observers feed +normalized mouse/keyboard events into synchronized queues; dedicated writer processes persist action events, screenshots, video frames, and (optionally) audio and window state into a per-capture SQLite database plus time-aligned media files. Adapted from the original for per-capture databases. Importing this module must never touch the display (no screenshot at module scope — enforced by -tests/test_headless_import.py); pynput itself may be unavailable headless, in -which case the package ``__init__`` degrades ``Recorder`` to ``None``. +tests/test_headless_import.py). Usage: @@ -40,7 +39,6 @@ import psutil from loguru import logger from pympler import tracker -from pynput import keyboard, mouse from tqdm import tqdm from openadapt_capture import platform, utils, video, window @@ -48,6 +46,15 @@ from openadapt_capture.db import create_db, crud, get_session_for_path from openadapt_capture.db.models import ActionEvent, Recording from openadapt_capture.extensions import synchronized_queue as sq +from openadapt_capture.input_observer import ( + ObservedInput, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + add_exception_note, + create_input_observer, +) from openadapt_capture.window_capture import ( WindowCaptureError, WindowCaptureScope, @@ -161,6 +168,21 @@ def __bool__(self): ws_server_instance = None +def _run_task_fail_loud( + task_name: str, + target: Callable[..., None], + args: tuple[Any, ...], + terminate_processing: Any, + task_errors: queue.Queue, +) -> None: + """Propagate reader-thread failures back through the recording boundary.""" + try: + target(*args) + except BaseException as exc: + task_errors.put((task_name, exc)) + terminate_processing.set() + + def _wait_for_tasks_started( task_by_name: dict[str, Any], task_started_events: dict[str, Any], @@ -828,8 +850,8 @@ def trigger_action_event( def on_move( event_q: queue.Queue, window_scope: WindowCaptureScope | None, - x: int, - y: int, + x: float, + y: float, injected: bool = False, ) -> None: """Handles the 'move' event. @@ -856,9 +878,9 @@ def on_move( def on_click( event_q: queue.Queue, window_scope: WindowCaptureScope | None, - x: int, - y: int, - button: mouse.Button, + x: float, + y: float, + button: str, pressed: bool, injected: bool = False, ) -> None: @@ -884,7 +906,7 @@ def on_click( "name": "click", "mouse_x": x, "mouse_y": y, - "mouse_button_name": button.name, + "mouse_button_name": button, "mouse_pressed": pressed, }, window_scope, @@ -894,10 +916,10 @@ def on_click( def on_scroll( event_q: queue.Queue, window_scope: WindowCaptureScope | None, - x: int, - y: int, - dx: int, - dy: int, + x: float, + y: float, + dx: float, + dy: float, injected: bool = False, ) -> None: """Handles the 'scroll' event. @@ -931,36 +953,29 @@ def on_scroll( def handle_key( event_q: queue.Queue, - event_name: str, - key: keyboard.KeyCode, - canonical_key: keyboard.KeyCode, + key: ObservedKey, ) -> None: - """Handles a key event. + """Persist a normalized native key transition. Args: event_q: The event queue to add the key event to. - event_name: The name of the key event. - key: The key code of the key event. - canonical_key: The canonical key code of the key event. + key: Normalized physical/canonical key identity. Returns: None """ - attr_names = [ - "name", - "char", - "vk", - ] - attrs = { - f"key_{attr_name}": getattr(key, attr_name, None) for attr_name in attr_names - } - logger.debug(f"{attrs=}") - canonical_attrs = { - f"canonical_key_{attr_name}": getattr(canonical_key, attr_name, None) - for attr_name in attr_names - } - logger.debug(f"{canonical_attrs=}") - trigger_action_event(event_q, {"name": event_name, **attrs, **canonical_attrs}) + trigger_action_event( + event_q, + { + "name": "press" if key.pressed else "release", + "key_name": key.key_name, + "key_char": key.key_char, + "key_vk": key.key_vk, + "canonical_key_name": key.canonical_key_name, + "canonical_key_char": key.canonical_key_char, + "canonical_key_vk": key.canonical_key_vk, + }, + ) def read_screen_events( @@ -1265,98 +1280,57 @@ def read_keyboard_events( Returns: None """ - # create list of indices for sequence detection - # one index for each stop sequence in config.STOP_SEQUENCES - stop_sequences = config.STOP_SEQUENCES + stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] stop_sequence_indices = [0 for _ in stop_sequences] - def on_press( - event_q: queue.Queue, - key: keyboard.Key | keyboard.KeyCode, - injected: bool = False, - ) -> None: - """Event handler for key press events. - - Args: - event_q (queue.Queue): The event queue for processing key events. - key (keyboard.KeyboardEvent): The key event object representing - the pressed key. - injected (bool): A flag indicating whether the key event was injected. - - Returns: - None - """ - canonical_key = keyboard_listener.canonical(key) - logger.debug(f"{key=} {injected=} {canonical_key=}") - if not injected: - handle_key(event_q, "press", key, canonical_key) + def on_observed(event: ObservedInput) -> None: + if not isinstance(event, ObservedKey) or event.injected: + return + logger.debug(f"{event=}") + handle_key(event_q, event) + if not event.pressed: + return - # stop sequence code nonlocal stop_sequence_indices global stop_sequence_detected - canonical_key_name = getattr(canonical_key, "name", None) - - for i in range(0, len(stop_sequences)): - # check each stop sequence - stop_sequence = stop_sequences[i] - # stop_sequence_indices[i] is the index for this stop sequence - # get canonical KeyCode of current letter in this sequence - canonical_sequence = keyboard_listener.canonical( - keyboard.KeyCode.from_char(stop_sequence[stop_sequence_indices[i]]) - ) - - # Check if the pressed key matches the current key in this sequence - if ( - canonical_key == canonical_sequence - or canonical_key_name == stop_sequence[stop_sequence_indices[i]] - ): - # increment this index - stop_sequence_indices[i] += 1 + candidate = event.canonical_key_char or event.canonical_key_name + if candidate is None: + stop_sequence_indices = [0 for _ in stop_sequences] + return + candidate = candidate.lower() + for index, sequence in enumerate(stop_sequences): + expected = sequence[stop_sequence_indices[index]].lower() + if candidate == expected: + stop_sequence_indices[index] += 1 else: - # Reset index since pressed key doesn't match sequence key - stop_sequence_indices[i] = 0 - - # Check if the entire sequence has been entered correctly - if stop_sequence_indices[i] >= len(stop_sequence): - stop_sequence_indices[i] = 0 + stop_sequence_indices[index] = ( + 1 if candidate == sequence[0].lower() else 0 + ) + if stop_sequence_indices[index] == len(sequence): + stop_sequence_indices[index] = 0 logger.info("Stop sequence entered! Stopping recording now.") stop_sequence_detected = True - def on_release( - event_q: queue.Queue, - key: keyboard.Key | keyboard.KeyCode, - injected: bool = False, - ) -> None: - """Event handler for key release events. - - Args: - event_q (queue.Queue): The event queue for processing key events. - key (keyboard.KeyboardEvent): The key event object representing - the released key. - injected (bool): A flag indicating whether the key event was injected. - - Returns: - None - """ - canonical_key = keyboard_listener.canonical(key) - logger.debug(f"{key=} {injected=} {canonical_key=}") - if not injected: - handle_key(event_q, "release", key, canonical_key) - utils.set_start_time(recording.timestamp) - - keyboard_listener = keyboard.Listener( - on_press=partial(on_press, event_q), - on_release=partial(on_release, event_q), + observer = create_input_observer( + on_observed, + observe_keyboard=True, + observe_mouse=False, + capture_mouse_moves=False, ) - keyboard_listener.start() - - # NOTE: listener may not have actually started by now - # TODO: handle race condition, e.g. by sending synthetic events from main thread - started_event.set() - - terminate_processing.wait() - keyboard_listener.stop() + started = False + try: + observer.start() + started = True + started_event.set() + while not terminate_processing.wait(0.1): + observer.check_health() + except BaseException: + terminate_processing.set() + raise + finally: + if started: + observer.stop() def read_mouse_events( @@ -1381,19 +1355,55 @@ def read_mouse_events( """ utils.set_start_time(recording.timestamp) - mouse_listener = mouse.Listener( - on_move=partial(on_move, event_q, window_scope), - on_click=partial(on_click, event_q, window_scope), - on_scroll=partial(on_scroll, event_q, window_scope), - ) - mouse_listener.start() - - # NOTE: listener may not have actually started by now - # TODO: handle race condition, e.g. by sending synthetic events from main thread - started_event.set() + def on_observed(event: ObservedInput) -> None: + if isinstance(event, ObservedMouseMove): + on_move( + event_q, + window_scope, + event.x, + event.y, + event.injected, + ) + elif isinstance(event, ObservedMouseButton): + on_click( + event_q, + window_scope, + event.x, + event.y, + event.button, + event.pressed, + event.injected, + ) + elif isinstance(event, ObservedMouseScroll): + on_scroll( + event_q, + window_scope, + event.x, + event.y, + event.dx, + event.dy, + event.injected, + ) - terminate_processing.wait() - mouse_listener.stop() + observer = create_input_observer( + on_observed, + observe_keyboard=False, + observe_mouse=True, + capture_mouse_moves=True, + ) + started = False + try: + observer.start() + started = True + started_event.set() + while not terminate_processing.wait(0.1): + observer.check_health() + except BaseException: + terminate_processing.set() + raise + finally: + if started: + observer.stop() def record_audio( @@ -1599,7 +1609,7 @@ def run_server() -> None: server_thread.join() -@logger.catch +@logger.catch(reraise=True) @utils.trace(logger) def record( task_description: str, @@ -1696,6 +1706,7 @@ def record( terminate_processing = multiprocessing.Event() task_by_name = {} task_started_events = {} + task_errors: queue.Queue = queue.Queue() _screen_timing = _ScreenTimingStats() # running stats, no unbounded list # In window-scoped mode the screen reader emits the target window's @@ -1748,28 +1759,42 @@ def record( screen_event_reader.start() task_by_name["screen_event_reader"] = screen_event_reader + keyboard_reader_args = ( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("keyboard_event_reader", threading.Event()), + ) keyboard_event_reader = threading.Thread( - target=read_keyboard_events, + target=_run_task_fail_loud, daemon=True, args=( - event_q, + "keyboard_event_reader", + read_keyboard_events, + keyboard_reader_args, terminate_processing, - recording, - task_started_events.setdefault("keyboard_event_reader", threading.Event()), + task_errors, ), ) keyboard_event_reader.start() task_by_name["keyboard_event_reader"] = keyboard_event_reader + mouse_reader_args = ( + event_q, + terminate_processing, + recording, + task_started_events.setdefault("mouse_event_reader", threading.Event()), + window_scope, + ) mouse_event_reader = threading.Thread( - target=read_mouse_events, + target=_run_task_fail_loud, daemon=True, args=( - event_q, + "mouse_event_reader", + read_mouse_events, + mouse_reader_args, terminate_processing, - recording, - task_started_events.setdefault("mouse_event_reader", threading.Event()), - window_scope, + task_errors, ), ) mouse_event_reader.start() @@ -2037,6 +2062,11 @@ def record( timeout=pre_ready_timeout, ) + if not task_errors.empty(): + task_name, task_error = task_errors.get_nowait() + add_exception_note(task_error, f"recording task {task_name!r} failed") + raise task_error + if config.PLOT_PERFORMANCE and startup_ready: from openadapt_capture import plotting @@ -2220,6 +2250,8 @@ def __init__( self._record_thread: threading.Thread | None = None self._status_thread: threading.Thread | None = None self._capture = None # lazy CaptureSession + self._worker_error: BaseException | None = None + self._worker_error_lock = threading.Lock() def _drain_status_pipe(self) -> None: """Background thread that reads status messages from record().""" @@ -2256,15 +2288,18 @@ def _run_record(self) -> None: num_video_events=self._num_video_events, send_profile=self._send_profile, ) - except BaseException: + except BaseException as exc: # A setup exception must wake wait_for_ready() and let context-manager # teardown finish instead of leaving callers blocked for its timeout. + with self._worker_error_lock: + if self._worker_error is None: + self._worker_error = exc + self._terminate_processing.set() try: self._status_send.send({"type": "record.stopped"}) except (BrokenPipeError, EOFError, OSError): self._stopped_event.set() self._ready_or_stopped_event.set() - raise def __enter__(self) -> "Recorder": # Start status drain thread @@ -2285,10 +2320,28 @@ def __exit__(self, exc_type, exc_val, exc_tb) -> None: self._stopped_event.set() # ensure status thread exits if self._status_thread is not None: self._status_thread.join(timeout=5) + if self._worker_error is not None: + if exc_val is not None: + add_exception_note( + exc_val, + f"the recorder worker also failed: {self._worker_error!r}" + ) + else: + raise self._worker_error def stop(self) -> None: - """Stop recording programmatically.""" + """Stop, join, and surface any recording-worker failure.""" self._terminate_processing.set() + if self._record_thread is not None: + self._record_thread.join() + self.check_health() + + def check_health(self) -> None: + """Raise the first recording-worker error observed by the owner thread.""" + with self._worker_error_lock: + worker_error = self._worker_error + if worker_error is not None: + raise worker_error def wait_for_ready(self, timeout: float = 60) -> bool: """Block until all recording threads/processes have started. @@ -2296,6 +2349,7 @@ def wait_for_ready(self, timeout: float = 60) -> bool: Returns True if ready, False if startup stopped or the timeout expired. """ self._ready_or_stopped_event.wait(timeout=timeout) + self.check_health() return self._ready_event.is_set() @property @@ -2340,6 +2394,7 @@ def capture(self): Returns None if recording has not finished yet. """ + self.check_health() if self._capture is None and not self.is_recording: try: from openadapt_capture.capture import CaptureSession diff --git a/openadapt_capture/visualize/html.py b/openadapt_capture/visualize/html.py index 3687ea7..6f07627 100644 --- a/openadapt_capture/visualize/html.py +++ b/openadapt_capture/visualize/html.py @@ -1108,7 +1108,7 @@ def _generate_html( if (!showOverlay) return; // Scale factors: convert mouse coordinates to display pixels - // Mouse coords from pynput are in LOGICAL space (e.g., 1512x982 on Retina) + // Native observer mouse coords are LOGICAL (e.g., 1512x982 on Retina) // screenWidth/Height we stored are in PHYSICAL space (e.g., 3024x1964) // pixelRatio = physical/logical (e.g., 2.0 for Retina) // diff --git a/openadapt_capture/window_capture.py b/openadapt_capture/window_capture.py index 3abf618..4588dfc 100644 --- a/openadapt_capture/window_capture.py +++ b/openadapt_capture/window_capture.py @@ -15,7 +15,7 @@ ``backends/remote_display.py`` — read that module before changing these): - ``bounds`` is ``(x, y, w, h)`` in **screen points**, top-left origin — the - space pynput/CGEvent global mouse coordinates use. + space native platform observers use for global mouse coordinates. - A captured frame contains the window's own **pixels** (macOS: ``CGWindowListCreateImage`` with ``kCGWindowImageBoundsIgnoreFraming`` — the identical call flow's replay capture path uses). @@ -102,7 +102,7 @@ class TargetWindow: """One resolved on-screen window. ``bounds`` is ``(x, y, w, h)`` in screen points, top-left origin — the - same space as global pynput mouse coordinates. Same field semantics as + same space as native global mouse coordinates. Same field semantics as openadapt-flow's ``WindowInfo``. """ @@ -143,7 +143,7 @@ class WindowCaptureScope: """Thread-safe tracker of the target window's bounds/scale during recording. The screen-reader thread calls :meth:`capture_frame` (which re-resolves the - window each frame — windows move and resize); pynput listener threads call + window each frame — windows move and resize); native observer threads call :meth:`translate` concurrently to convert global input coordinates into the captured frame's pixel space using the freshest bounds. diff --git a/pyproject.toml b/pyproject.toml index 0fce451..6d95507 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ classifiers = [ dependencies = [ "pydantic>=2.0.0", - "pynput>=1.7.6", "av>=12.0.0", "Pillow>=10.1.0", "fire>=0.7.1", @@ -133,5 +132,7 @@ patch_tags = ["fix", "perf"] dev = [ "numpy>=2.2.6", "psutil>=7.2.2", + # Test-only synthetic input driver; excluded from package metadata/runtime. + "pynput>=1.7.6", "pytest>=9.0.2", ] diff --git a/scripts/verify_distribution.py b/scripts/verify_distribution.py index 7b61f98..c5a4e99 100644 --- a/scripts/verify_distribution.py +++ b/scripts/verify_distribution.py @@ -8,8 +8,8 @@ import zipfile from pathlib import Path -FORBIDDEN_DEPENDENCIES = ("oa-atomacos",) -FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos",) +FORBIDDEN_DEPENDENCIES = ("oa-atomacos", "pynput") +FORBIDDEN_SOURCE_TOKENS = ("oa_atomacos", "pynput") def _archive_files(path: Path) -> dict[str, bytes]: diff --git a/tests/test_highlevel.py b/tests/test_highlevel.py index cd707bc..163f72f 100644 --- a/tests/test_highlevel.py +++ b/tests/test_highlevel.py @@ -10,18 +10,10 @@ import pytest +from openadapt_capture import recorder as recorder_module from openadapt_capture.capture import Capture from openadapt_capture.db import create_db, crud - -# Recorder requires pynput which needs a display server -try: - from openadapt_capture import recorder as recorder_module - - Recorder = recorder_module.Recorder -except ImportError: - recorder_module = None - Recorder = None - +from openadapt_capture.recorder import Recorder # Sessions/engines created by _create_test_recording, released by the # temp_capture_dir teardown BEFORE the TemporaryDirectory is removed. Both @@ -70,7 +62,6 @@ def _create_test_recording(capture_dir, task_description="Test task"): return recording, db_path, session -@pytest.mark.skipif(Recorder is None, reason="pynput unavailable (headless)") class TestRecorder: """Tests for Recorder class.""" diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py new file mode 100644 index 0000000..90049ad --- /dev/null +++ b/tests/test_input_observer.py @@ -0,0 +1,493 @@ +"""Platform-neutral and Linux contracts for native input observation.""" + +from __future__ import annotations + +import ctypes +import sys +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from openadapt_capture import recorder as recorder_module +from openadapt_capture.capture import _convert_action_event +from openadapt_capture.events import MouseClickEvent, MouseDownEvent, MouseUpEvent +from openadapt_capture.input_observer import ( + InputObserverError, + InputObserverUnavailableError, + ObservedKey, + ObservedMouseButton, + ObservedMouseScroll, + ThreadedInputObserver, + add_exception_note, + create_input_observer, +) +from openadapt_capture.input_observer.linux import ( + LinuxXInputObserver, + _XIRawEvent, + normalize_xinput_button_event, + normalize_xinput_key_event, +) +from openadapt_capture.processing import merge_consecutive_mouse_click_events + + +class _CooperativeNeverReadyObserver(ThreadedInputObserver): + def __init__(self) -> None: + super().__init__( + lambda event: None, + observe_keyboard=True, + observe_mouse=False, + capture_mouse_moves=False, + startup_timeout=0.02, + shutdown_timeout=0.2, + ) + self.resource_open = False + self.torn_down = threading.Event() + + def _setup(self) -> None: + self.resource_open = True + self._stop_requested.wait() + + def _run_loop(self) -> None: + return + + def _teardown(self) -> None: + self.resource_open = False + self.torn_down.set() + + +class _StubbornNeverReadyObserver(_CooperativeNeverReadyObserver): + def __init__(self) -> None: + super().__init__() + self.shutdown_timeout = 0.01 + self.release_setup = threading.Event() + + def _setup(self) -> None: + self.resource_open = True + self.release_setup.wait() + + +class _ReadyObserver(ThreadedInputObserver): + def __init__(self, callback=lambda _event: None) -> None: + super().__init__( + callback, + observe_keyboard=True, + observe_mouse=False, + capture_mouse_moves=False, + startup_timeout=0.2, + shutdown_timeout=0.02, + ) + self.release_loop = threading.Event() + + def _setup(self) -> None: + return + + def _run_loop(self) -> None: + self.release_loop.wait() + + def _teardown(self) -> None: + return + + def _wake(self) -> None: + self.release_loop.set() + + +def test_failed_startup_joins_and_tears_down_before_raising() -> None: + observer = _CooperativeNeverReadyObserver() + + with pytest.raises(InputObserverError, match="did not become ready"): + observer.start() + + assert observer._thread is None + assert observer._delivery_thread is None + assert observer.torn_down.is_set() + assert not observer.resource_open + + +def test_lingering_failed_startup_cannot_be_reused_as_success() -> None: + observer = _StubbornNeverReadyObserver() + try: + with pytest.raises(InputObserverError, match="did not become ready"): + observer.start() + assert observer._thread is not None + assert observer._thread.is_alive() + assert observer._delivery_thread is None + + with pytest.raises(InputObserverError, match="live thread from a failed startup"): + observer.start() + finally: + observer.release_setup.set() + deadline = time.monotonic() + 1 + while observer._thread is not None and observer._thread.is_alive(): + if time.monotonic() >= deadline: + pytest.fail("stubborn observer thread did not finish after release") + time.sleep(0.01) + observer.stop() + assert observer.torn_down.is_set() + assert not observer.resource_open + + +def test_event_thread_start_failure_cleans_delivery_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = _ReadyObserver() + original_start = threading.Thread.start + + def fail_event_thread(thread: threading.Thread) -> None: + if thread.name.endswith("-event-loop"): + raise RuntimeError("event thread start failed") + original_start(thread) + + monkeypatch.setattr(threading.Thread, "start", fail_event_thread) + + with pytest.raises(RuntimeError, match="event thread start failed"): + observer.start() + + assert observer._thread is None + assert observer._delivery_thread is None + + +def test_delivery_thread_start_failure_is_transactional( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = _ReadyObserver() + + def fail_start(_thread: threading.Thread) -> None: + raise RuntimeError("delivery thread start failed") + + monkeypatch.setattr(threading.Thread, "start", fail_start) + + with pytest.raises(RuntimeError, match="delivery thread start failed"): + observer.start() + + assert observer._thread is None + assert observer._delivery_thread is None + + +def test_first_failure_is_preserved_when_wake_also_fails() -> None: + observer = _ReadyObserver() + primary = InputObserverError("primary observer failure") + + def fail_wake() -> None: + raise RuntimeError("wake failed") + + observer._wake = fail_wake # type: ignore[method-assign] + observer._fail(primary) + + with pytest.raises(InputObserverError, match="primary observer failure") as caught: + observer.check_health() + assert caught.value is primary + assert any("wake failed" in note for note in getattr(primary, "__notes__", [])) + + +def test_idempotent_start_surfaces_live_delivery_consumer_failure() -> None: + callback_entered = threading.Event() + + def fail_consumer(_event) -> None: + callback_entered.set() + raise RuntimeError("consumer failed") + + observer = _ReadyObserver(fail_consumer) + observer.start() + observer._emit(ObservedKey(pressed=True, key_char="a")) + assert callback_entered.wait(timeout=1) + + with pytest.raises(InputObserverError, match="consumer failed"): + observer.start() + with pytest.raises(InputObserverError, match="consumer failed"): + observer.stop() + + +def test_event_shutdown_timeout_still_cleans_delivery_thread() -> None: + class StubbornLoopObserver(_ReadyObserver): + def _wake(self) -> None: + return + + observer = StubbornLoopObserver() + observer.start() + + with pytest.raises(InputObserverError, match="did not stop"): + observer.stop() + + assert observer._thread is not None + assert observer._thread.is_alive() + assert observer._delivery_thread is None + observer.release_loop.set() + observer._thread.join(timeout=1) + observer.stop() + + +def test_repeated_stop_retries_lingering_delivery_cleanup() -> None: + callback_entered = threading.Event() + release_callback = threading.Event() + + def block_consumer(_event) -> None: + callback_entered.set() + release_callback.wait() + + observer = _ReadyObserver(block_consumer) + observer.start() + observer._emit(ObservedKey(pressed=True, key_char="a")) + assert callback_entered.wait(timeout=1) + + with pytest.raises(InputObserverError, match="delivery thread did not stop"): + observer.stop() + assert observer._delivery_thread is not None + assert observer._delivery_thread.is_alive() + + release_callback.set() + observer._delivery_thread.join(timeout=1) + with pytest.raises(InputObserverError, match="delivery thread did not stop"): + observer.stop() + assert observer._delivery_thread is None + + +@pytest.mark.parametrize( + ("detail", "pressed", "expected"), + [ + ( + 1, + True, + ObservedMouseButton( + x=12, + y=34, + button="left", + pressed=True, + ), + ), + ( + 2, + False, + ObservedMouseButton( + x=12, + y=34, + button="middle", + pressed=False, + ), + ), + ( + 4, + True, + ObservedMouseScroll(x=12, y=34, dx=0, dy=1), + ), + ( + 7, + True, + ObservedMouseScroll(x=12, y=34, dx=1, dy=0), + ), + ( + 8, + True, + ObservedMouseButton( + x=12, + y=34, + button="x1", + pressed=True, + ), + ), + ( + 9, + False, + ObservedMouseButton( + x=12, + y=34, + button="x2", + pressed=False, + ), + ), + (4, False, None), + (0, True, None), + ], +) +def test_xinput_button_normalization( + detail: int, + pressed: bool, + expected: ObservedMouseButton | ObservedMouseScroll | None, +) -> None: + assert ( + normalize_xinput_button_event( + detail=detail, + pressed=pressed, + x=12, + y=34, + ) + == expected + ) + + +def test_xinput_key_normalization_preserves_physical_and_canonical_identity() -> None: + letter = normalize_xinput_key_event( + keycode=38, + pressed=True, + keysym_name="A", + ) + assert letter == ObservedKey( + pressed=True, + key_char="A", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + ) + + modifier = normalize_xinput_key_event( + keycode=50, + pressed=False, + keysym_name="Shift_L", + injected=True, + ) + assert modifier.key_name == "shift" + assert modifier.canonical_key_name == "shift" + assert modifier.injected + + +@pytest.mark.parametrize("button", ["x1", "x2", "button4"]) +def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( + button: str, +) -> None: + down = _convert_action_event( + SimpleNamespace( + name="click", + timestamp=1.0, + mouse_x=12.5, + mouse_y=34.5, + mouse_button_name=button, + mouse_pressed=True, + ) + ) + up = _convert_action_event( + SimpleNamespace( + name="click", + timestamp=1.1, + mouse_x=12.5, + mouse_y=34.5, + mouse_button_name=button, + mouse_pressed=False, + ) + ) + + assert isinstance(down, MouseDownEvent) + assert isinstance(up, MouseUpEvent) + assert down.button == button + assert up.button == button + processed = merge_consecutive_mouse_click_events([down, up]) + assert len(processed) == 1 + assert isinstance(processed[0], MouseClickEvent) + assert processed[0].button == button + + +@pytest.mark.skipif( + ctypes.sizeof(ctypes.c_void_p) != 8, + reason="ABI offsets below describe the supported 64-bit Linux runners", +) +def test_xinput_raw_event_matches_64_bit_libxi_abi() -> None: + assert ctypes.sizeof(_XIRawEvent) == 144 + assert _XIRawEvent.root.offset == 64 + assert _XIRawEvent.root_x.offset == 72 + assert _XIRawEvent.root_y.offset == 80 + assert _XIRawEvent.flags.offset == 88 + assert _XIRawEvent.buttons.offset == 96 + assert _XIRawEvent.valuators.offset == 112 + assert _XIRawEvent.raw_values.offset == 136 + + +def test_wayland_refuses_instead_of_silently_observing_only_xwayland( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(sys, "platform", "linux") + observer = LinuxXInputObserver( + lambda event: None, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + environ={"XDG_SESSION_TYPE": "wayland", "DISPLAY": ":0"}, + ) + + with pytest.raises(InputObserverUnavailableError, match="silently omit"): + observer.start() + assert observer._thread is None + + +def test_factory_refuses_unknown_platform() -> None: + with pytest.raises(InputObserverUnavailableError, match="not implemented"): + create_input_observer(lambda event: None, platform_name="plan9") + + +def test_exception_note_helper_is_safe_without_python_311_api() -> None: + class LegacyException: + pass + + add_exception_note(LegacyException(), "cleanup context") # type: ignore[arg-type] + + +def test_record_boundary_reraises_setup_failure( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + def fail_scope(*args, **kwargs): + raise RuntimeError("native observer setup failed") + + monkeypatch.setattr(recorder_module, "build_window_scope", fail_scope) + with pytest.raises(RuntimeError, match="native observer setup failed"): + recorder_module.record("test", capture_dir=str(tmp_path)) + + +def test_post_readiness_worker_failure_reaches_recorder_owner( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + release_failure = threading.Event() + + def fail_after_ready( + *, + terminate_processing, + terminate_recording, + status_pipe, + **kwargs, + ) -> None: + status_pipe.send({"type": "record.started"}) + assert release_failure.wait(timeout=1) + raise RuntimeError("native observer failed after readiness") + + monkeypatch.setattr(recorder_module, "record", fail_after_ready) + recorder = recorder_module.Recorder(str(tmp_path / "late-failure")) + + with pytest.raises( + RuntimeError, + match="native observer failed after readiness", + ): + with recorder: + assert recorder.wait_for_ready(timeout=1) + release_failure.set() + + with pytest.raises( + RuntimeError, + match="native observer failed after readiness", + ): + _ = recorder.capture + + +def test_recorder_worker_error_does_not_mask_outer_exception( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + release_failure = threading.Event() + + def fail_after_ready(*, status_pipe, **kwargs) -> None: + status_pipe.send({"type": "record.started"}) + assert release_failure.wait(timeout=1) + raise RuntimeError("secondary recorder failure") + + monkeypatch.setattr(recorder_module, "record", fail_after_ready) + recorder = recorder_module.Recorder(str(tmp_path / "outer-failure")) + + with pytest.raises(ValueError, match="primary caller failure") as caught: + with recorder: + assert recorder.wait_for_ready(timeout=1) + release_failure.set() + raise ValueError("primary caller failure") + + notes = getattr(caught.value, "__notes__", []) + if notes: + assert any("secondary recorder failure" in note for note in notes) diff --git a/tests/test_input_observer_darwin.py b/tests/test_input_observer_darwin.py index da40421..caef1ff 100644 --- a/tests/test_input_observer_darwin.py +++ b/tests/test_input_observer_darwin.py @@ -74,9 +74,12 @@ def __init__( self, *, permission: bool = True, + grant_on_request: bool = False, create_tap: bool = True, ) -> None: self.permission = permission + self.grant_on_request = grant_on_request + self.permission_requests = 0 self.create_tap = create_tap self.tap = object() self.loop = object() @@ -93,6 +96,12 @@ def __init__( def CGPreflightListenEventAccess(self): return self.permission + def CGRequestListenEventAccess(self): + self.permission_requests += 1 + if self.grant_on_request: + self.permission = True + return self.permission + @staticmethod def CGEventMaskBit(event_type): return 1 << event_type @@ -222,6 +231,18 @@ def test_permission_denial_fails_before_event_tap_creation() -> None: with pytest.raises(InputObserverPermissionError, match="Input Monitoring"): observer.start() assert quartz.callback is None + assert quartz.permission_requests == 1 + + +def test_permission_request_can_complete_explicit_observer_start() -> None: + quartz = FakeQuartz(permission=False, grant_on_request=True) + observer = make_observer(quartz, lambda _event: None) + + observer.start() + observer.stop() + + assert quartz.permission_requests == 1 + assert quartz.callback is not None def test_accessibility_permission_is_fallback_on_older_macos() -> None: @@ -255,6 +276,7 @@ def test_mouse_move_button_and_scroll_normalization() -> None: observer = make_observer(quartz, events.append) physical = {quartz.kCGEventSourceUnixProcessID: 0} + observer.start() observer._handle_event( quartz.kCGEventMouseMoved, FakeEvent(x=11.5, y=22.5, fields=physical), @@ -286,6 +308,7 @@ def test_mouse_move_button_and_scroll_normalization() -> None: }, ), ) + observer.stop() assert events == [ ObservedMouseMove(x=11.5, y=22.5), @@ -314,8 +337,10 @@ def test_move_filter_does_not_disable_buttons() -> None: capture_mouse_moves=False, ) + observer.start() observer._handle_event(quartz.kCGEventMouseMoved, FakeEvent()) observer._handle_event(quartz.kCGEventRightMouseUp, FakeEvent()) + observer.stop() assert events == [ ObservedMouseButton( @@ -333,6 +358,7 @@ def test_key_press_release_and_modifier_canonicalization() -> None: events = [] observer = make_observer(quartz, events.append) + observer.start() observer._handle_event( quartz.kCGEventKeyDown, FakeEvent( @@ -359,6 +385,7 @@ def test_key_press_release_and_modifier_canonicalization() -> None: ) observer._handle_event(quartz.kCGEventFlagsChanged, shift_press) observer._handle_event(quartz.kCGEventFlagsChanged, shift_release) + observer.stop() assert events == [ ObservedKey( @@ -398,6 +425,7 @@ def test_first_observed_modifier_release_does_not_toggle_to_press() -> None: events = [] observer = make_observer(quartz, events.append) + observer.start() observer._handle_event( quartz.kCGEventFlagsChanged, FakeEvent( @@ -405,6 +433,7 @@ def test_first_observed_modifier_release_does_not_toggle_to_press() -> None: flags=0, ), ) + observer.stop() assert events == [ ObservedKey( @@ -422,6 +451,7 @@ def test_caps_lock_state_comes_from_event_flags() -> None: events = [] observer = make_observer(quartz, events.append) + observer.start() observer._handle_event( quartz.kCGEventFlagsChanged, FakeEvent( @@ -436,6 +466,7 @@ def test_caps_lock_state_comes_from_event_flags() -> None: flags=0, ), ) + observer.stop() assert [(event.key_name, event.pressed) for event in events] == [ ("caps_lock", True), @@ -448,6 +479,7 @@ def test_releasing_one_shift_ignores_the_sibling_aggregate_flag() -> None: events = [] observer = make_observer(quartz, events.append) + observer.start() for keycode, flags in [ (56, quartz.kCGEventFlagMaskShift), (60, quartz.kCGEventFlagMaskShift), @@ -461,6 +493,7 @@ def test_releasing_one_shift_ignores_the_sibling_aggregate_flag() -> None: flags=flags, ), ) + observer.stop() assert [(event.key_name, event.pressed) for event in events] == [ ("shift_l", True), @@ -470,13 +503,13 @@ def test_releasing_one_shift_ignores_the_sibling_aggregate_flag() -> None: ] -def test_disabled_tap_is_reenabled_without_emitting_input() -> None: +def test_disabled_tap_fails_loud_instead_of_hiding_incomplete_coverage() -> None: quartz = FakeQuartz() events = [] observer = make_observer(quartz, events.append) - observer._event_tap = quartz.tap event = FakeEvent() + observer.start() assert ( observer._event_callback( None, @@ -486,7 +519,11 @@ def test_disabled_tap_is_reenabled_without_emitting_input() -> None: ) is event ) - assert quartz.tap_enabled == [True] + with pytest.raises(InputObserverError, match="coverage is incomplete"): + observer.check_health() + with pytest.raises(InputObserverError, match="coverage is incomplete"): + observer.stop() + assert quartz.tap_enabled == [True, True, False] assert events == [] @@ -502,8 +539,19 @@ def fail(_event): ObservedMouseMove(1, 2) ) + observer.start() observer._event_callback(None, quartz.kCGEventMouseMoved, event, None) - with pytest.raises(InputObserverError, match="callback boom"): - observer.check_health() + deadline = time.monotonic() + 1 + while True: + try: + observer.check_health() + except InputObserverError as exc: + assert "callback boom" in str(exc) + break + if time.monotonic() >= deadline: + pytest.fail("asynchronous callback failure did not reach observer health") + time.sleep(0.001) assert observer._stop_requested.is_set() + with pytest.raises(InputObserverError, match="callback boom"): + observer.stop() diff --git a/tests/test_performance.py b/tests/test_performance.py index dfd54ba..b4d04ff 100644 --- a/tests/test_performance.py +++ b/tests/test_performance.py @@ -26,25 +26,20 @@ import pytest from openadapt_capture.capture import CaptureSession - -# Recorder requires pynput which needs a display server -try: - from openadapt_capture.recorder import Recorder -except ImportError: - Recorder = None +from openadapt_capture.recorder import Recorder # Skip on non-Windows platforms where the live pipeline is not yet validated _SKIP_REASON = ( "Live recorder integration tests target Windows (the primary recording " "platform, exercised in CI on windows-latest). The multiprocessing " "'spawn' writer path on macOS/Linux is not yet validated end to end; " - "on GitHub macOS runners pynput input injection also needs Accessibility " + "on GitHub macOS runners synthetic input injection also needs Accessibility " "permissions that cannot be granted." ) _ON_WINDOWS = sys.platform == "win32" # GitHub-hosted Windows runners execute jobs in a non-interactive session: -# SendInput-injected events never reach the low-level hooks pynput uses, so +# SendInput-injected events never reach native low-level hooks in this session, so # listener-dependent tests capture zero events there. The CI workflow sets # this flag; the live-pipeline tests that do not depend on captured input # (startup/shutdown, db creation, bounded memory) still run for real. Run the @@ -52,7 +47,7 @@ # interactive self-hosted runner). _NO_INPUT_INJECTION = os.environ.get("OPENADAPT_CI_NO_INPUT_INJECTION") == "1" _INJECTION_SKIP_REASON = ( - "OPENADAPT_CI_NO_INPUT_INJECTION=1: injected input does not reach pynput " + "OPENADAPT_CI_NO_INPUT_INJECTION=1: injected input does not reach native " "hooks in a non-interactive session, so event-capture assertions cannot " "hold (hosted CI runner limitation, not a recorder bug)" ) diff --git a/tests/test_runtime_import_contract.py b/tests/test_runtime_import_contract.py index 969e18d..b46dbd6 100644 --- a/tests/test_runtime_import_contract.py +++ b/tests/test_runtime_import_contract.py @@ -43,6 +43,16 @@ def test_macos_accessibility_uses_permissive_runtime_dependencies() -> None: assert "oa_atomacos" not in package_sources +def test_native_input_observers_do_not_ship_pynput() -> None: + """The default runtime and packaged source stay outside pynput's LGPL path.""" + assert "pynput" not in _runtime_dependency_names() + package_sources = "\n".join( + path.read_text(encoding="utf-8") + for path in (ROOT / "openadapt_capture").rglob("*.py") + ) + assert "pynput" not in package_sources + + def test_default_install_exposes_recorder() -> None: """Recorder import never fails because a runtime dependency is undeclared.""" result = subprocess.run( diff --git a/tests/test_window_capture.py b/tests/test_window_capture.py index 2ee2f5e..a278d41 100644 --- a/tests/test_window_capture.py +++ b/tests/test_window_capture.py @@ -22,6 +22,7 @@ from openadapt_capture.capture import CaptureSession from openadapt_capture.db import create_db, crud +from openadapt_capture.recorder import Recorder from openadapt_capture.window_capture import ( TargetWindow, WindowCaptureError, @@ -31,13 +32,6 @@ translate_point, ) -# Recorder requires pynput which needs a display server -try: - from openadapt_capture.recorder import Recorder -except ImportError: - Recorder = None - - # --------------------------------------------------------------------------- # translate_point: exact inverse of flow's replay mapping # --------------------------------------------------------------------------- @@ -283,7 +277,6 @@ def test_config_override_window_fields(self): assert config.RECORD_WINDOW_OWNER is None assert config.RECORD_WINDOW_TITLE is None - @pytest.mark.skipif(Recorder is None, reason="pynput unavailable (headless)") def test_recorder_accepts_window_param(self): rec = Recorder( "/tmp/test_never_created", @@ -293,13 +286,11 @@ def test_recorder_accepts_window_param(self): assert rec._recording_config.window_owner == "Parallels" assert rec._recording_config.window_title is None - @pytest.mark.skipif(Recorder is None, reason="pynput unavailable (headless)") def test_recorder_without_window_param_records_fullscreen(self): rec = Recorder("/tmp/test_never_created") assert rec._recording_config.window_owner is None assert rec._recording_config.window_title is None - @pytest.mark.skipif(Recorder is None, reason="pynput unavailable (headless)") def test_recorder_rejects_bad_window_spec(self): with pytest.raises(ValueError): Recorder("/tmp/test_never_created", window={"app": "Parallels"}) @@ -310,7 +301,6 @@ def test_recorder_rejects_bad_window_spec(self): # --------------------------------------------------------------------------- -@pytest.mark.skipif(Recorder is None, reason="pynput unavailable (headless)") class TestActionTranslation: """trigger_action_event translates coordinates in window mode.""" From 27d4b3f6561248df93ff103a711fcdba74d762e5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:38:22 -0700 Subject: [PATCH 08/26] fix: preserve input receipt time and callback stops --- openadapt_capture/input_observer/base.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index 46a5b37..594ec7d 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -28,6 +28,7 @@ class ObservedMouseMove: x: float y: float injected: bool = False + timestamp: float | None = None @dataclass(frozen=True, slots=True) @@ -39,6 +40,7 @@ class ObservedMouseButton: button: str pressed: bool injected: bool = False + timestamp: float | None = None @dataclass(frozen=True, slots=True) @@ -50,6 +52,7 @@ class ObservedMouseScroll: dx: float dy: float injected: bool = False + timestamp: float | None = None @dataclass(frozen=True, slots=True) @@ -64,6 +67,7 @@ class ObservedKey: canonical_key_char: str | None = None canonical_key_vk: str | None = None injected: bool = False + timestamp: float | None = None ObservedInput: TypeAlias = ( @@ -135,6 +139,7 @@ def __init__( maxsize=delivery_queue_size ) self._delivery_sentinel = object() + self._delivery_stop_requested = threading.Event() self._failure: BaseException | None = None self._failure_lock = threading.Lock() self._startup_failure: BaseException | None = None @@ -186,7 +191,15 @@ def _store_failure(self, failure: BaseException) -> BaseException: def _delivery_main(self) -> None: while True: - item = self._delivery_queue.get() + if ( + self._delivery_stop_requested.is_set() + and self._delivery_queue.empty() + ): + return + try: + item = self._delivery_queue.get(timeout=0.1) + except queue.Empty: + continue try: if item is self._delivery_sentinel: return @@ -212,6 +225,7 @@ def _start_delivery(self) -> None: "previous lifecycle; stop it or create a new observer" ) self._delivery_queue = queue.Queue(maxsize=self.delivery_queue_size) + self._delivery_stop_requested.clear() thread = threading.Thread( target=self._delivery_main, name=f"{type(self).__name__}-delivery", @@ -228,6 +242,13 @@ def _stop_delivery(self) -> None: thread = self._delivery_thread if thread is None: return + self._delivery_stop_requested.set() + if thread is threading.current_thread(): + # A consumer callback may intentionally stop its listener (for + # example after matching the configured stop sequence). Returning + # lets the callback unwind; the delivery loop then drains events + # already received before exiting without attempting a self-join. + return if thread.is_alive(): try: self._delivery_queue.put( From 07e2e3fc494e7eb1066cbc44f4e98dc2ab001e0e Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:41:19 -0700 Subject: [PATCH 09/26] fix: preserve ordered input receipt timestamps --- openadapt_capture/input.py | 6 +- openadapt_capture/input_observer/darwin.py | 39 ++++- openadapt_capture/recorder.py | 183 ++++++++------------- tests/test_input_observer.py | 150 +++++++++++++++++ tests/test_input_observer_darwin.py | 21 ++- 5 files changed, 274 insertions(+), 125 deletions(-) diff --git a/openadapt_capture/input.py b/openadapt_capture/input.py index d2bda4f..a70b2fa 100644 --- a/openadapt_capture/input.py +++ b/openadapt_capture/input.py @@ -32,7 +32,11 @@ def _get_timestamp() -> float: def _to_public_event(observed: ObservedInput): """Convert a native normalized event to the public Pydantic event model.""" - timestamp = _get_timestamp() + timestamp = ( + observed.timestamp + if observed.timestamp is not None + else _get_timestamp() + ) if isinstance(observed, ObservedMouseMove): return MouseMoveEvent(timestamp=timestamp, x=observed.x, y=observed.y) if isinstance(observed, ObservedMouseButton): diff --git a/openadapt_capture/input_observer/darwin.py b/openadapt_capture/input_observer/darwin.py index 65625b5..09e07f0 100644 --- a/openadapt_capture/input_observer/darwin.py +++ b/openadapt_capture/input_observer/darwin.py @@ -2,6 +2,7 @@ from __future__ import annotations +import time from typing import Any from .base import ( @@ -311,7 +312,7 @@ def _event_callback( return event try: - self._handle_event(event_type, event) + self._handle_event(event_type, event, timestamp=time.time()) except BaseException as exc: failure = ( exc @@ -321,17 +322,39 @@ def _event_callback( self._fail(failure) return event - def _handle_event(self, event_type: int, event: Any) -> None: + def _handle_event( + self, + event_type: int, + event: Any, + *, + timestamp: float | None = None, + ) -> None: quartz = self._quartz injected = self._is_injected(event) + # Production callbacks always pass the receipt time. Keeping ``None`` + # for direct normalization calls makes the pure helper independently + # testable without inventing a second observation moment. + observed_at = timestamp if event_type in self._mouse_move_event_types(): if self.observe_mouse and self.capture_mouse_moves: x, y = self._location(event) - self._emit(ObservedMouseMove(x=x, y=y, injected=injected)) + self._emit( + ObservedMouseMove( + x=x, + y=y, + injected=injected, + timestamp=observed_at, + ) + ) return - button_event = self._mouse_button_event(event_type, event, injected) + button_event = self._mouse_button_event( + event_type, + event, + injected, + timestamp=observed_at, + ) if button_event is not None: self._emit(button_event) return @@ -355,6 +378,7 @@ def _handle_event(self, event_type: int, event: Any) -> None: ) ), injected=injected, + timestamp=observed_at, ) ) return @@ -366,6 +390,7 @@ def _handle_event(self, event_type: int, event: Any) -> None: event, pressed=event_type == quartz.kCGEventKeyDown, injected=injected, + timestamp=observed_at, ) ) return @@ -383,6 +408,7 @@ def _handle_event(self, event_type: int, event: Any) -> None: pressed=pressed, injected=injected, include_character=False, + timestamp=observed_at, ) ) @@ -431,6 +457,8 @@ def _mouse_button_event( event_type: int, event: Any, injected: bool, + *, + timestamp: float | None, ) -> ObservedMouseButton | None: if not self.observe_mouse: return None @@ -462,6 +490,7 @@ def _mouse_button_event( button=button, pressed=pressed, injected=injected, + timestamp=timestamp, ) def _key_event( @@ -471,6 +500,7 @@ def _key_event( pressed: bool, injected: bool, include_character: bool = True, + timestamp: float | None, ) -> ObservedKey: keycode = self._keycode(event) key_name = _KEY_NAMES.get(keycode) @@ -491,6 +521,7 @@ def _key_event( canonical_key_char=canonical_char, canonical_key_vk=vk, injected=injected, + timestamp=timestamp, ) def _keyboard_character(self, event: Any) -> str | None: diff --git a/openadapt_capture/recorder.py b/openadapt_capture/recorder.py index 16b6489..277b7c7 100644 --- a/openadapt_capture/recorder.py +++ b/openadapt_capture/recorder.py @@ -811,6 +811,7 @@ def trigger_action_event( event_q: queue.Queue, action_event_args: dict[str, Any], window_scope: WindowCaptureScope | None = None, + timestamp: float | None = None, ) -> None: """Triggers an action event and adds it to the event queue. @@ -821,6 +822,8 @@ def trigger_action_event( coordinates are translated into the target window's pixel space before being recorded, so recorded coordinates match the captured frames directly. + timestamp: Native event-receipt time. Defaults to the current recording + clock only for legacy/direct callers. Returns: None @@ -844,7 +847,13 @@ def trigger_action_event( return action_event_args["mouse_x"] = wx action_event_args["mouse_y"] = wy - event_q.put(Event(utils.get_timestamp(), "action", action_event_args)) + event_q.put( + Event( + utils.get_timestamp() if timestamp is None else timestamp, + "action", + action_event_args, + ) + ) def on_move( @@ -853,6 +862,7 @@ def on_move( x: float, y: float, injected: bool = False, + timestamp: float | None = None, ) -> None: """Handles the 'move' event. @@ -872,6 +882,7 @@ def on_move( event_q, {"name": "move", "mouse_x": x, "mouse_y": y}, window_scope, + timestamp, ) @@ -883,6 +894,7 @@ def on_click( button: str, pressed: bool, injected: bool = False, + timestamp: float | None = None, ) -> None: """Handles the 'click' event. @@ -910,6 +922,7 @@ def on_click( "mouse_pressed": pressed, }, window_scope, + timestamp, ) @@ -921,6 +934,7 @@ def on_scroll( dx: float, dy: float, injected: bool = False, + timestamp: float | None = None, ) -> None: """Handles the 'scroll' event. @@ -948,6 +962,7 @@ def on_scroll( "mouse_dy": dy, }, window_scope, + timestamp, ) @@ -975,6 +990,7 @@ def handle_key( "canonical_key_char": key.canonical_key_char, "canonical_key_vk": key.canonical_key_vk, }, + timestamp=key.timestamp, ) @@ -1262,30 +1278,55 @@ def create_recording( return recording, db_path -def read_keyboard_events( +def read_input_events( event_q: queue.Queue, terminate_processing: multiprocessing.Event, recording: Recording, started_event: threading.Event, + window_scope: WindowCaptureScope | None = None, ) -> None: - """Reads keyboard events and adds them to the event queue. - - Args: - event_q (queue.Queue): The event queue to add the keyboard events to. - terminate_processing (multiprocessing.Event): The event to signal termination - of event reading. - recording (Recording): The recording object. - started_event: Event to set once started. - - Returns: - None - """ + """Read globally ordered keyboard and mouse events from one native observer.""" stop_sequences = [sequence for sequence in config.STOP_SEQUENCES if sequence] stop_sequence_indices = [0 for _ in stop_sequences] def on_observed(event: ObservedInput) -> None: - if not isinstance(event, ObservedKey) or event.injected: + if isinstance(event, ObservedMouseMove): + on_move( + event_q, + window_scope, + event.x, + event.y, + event.injected, + timestamp=event.timestamp, + ) + return + if isinstance(event, ObservedMouseButton): + on_click( + event_q, + window_scope, + event.x, + event.y, + event.button, + event.pressed, + event.injected, + timestamp=event.timestamp, + ) return + if isinstance(event, ObservedMouseScroll): + on_scroll( + event_q, + window_scope, + event.x, + event.y, + event.dx, + event.dy, + event.injected, + timestamp=event.timestamp, + ) + return + if event.injected: + return + logger.debug(f"{event=}") handle_key(event_q, event) if not event.pressed: @@ -1315,79 +1356,6 @@ def on_observed(event: ObservedInput) -> None: observer = create_input_observer( on_observed, observe_keyboard=True, - observe_mouse=False, - capture_mouse_moves=False, - ) - started = False - try: - observer.start() - started = True - started_event.set() - while not terminate_processing.wait(0.1): - observer.check_health() - except BaseException: - terminate_processing.set() - raise - finally: - if started: - observer.stop() - - -def read_mouse_events( - event_q: queue.Queue, - terminate_processing: multiprocessing.Event, - recording: Recording, - started_event: threading.Event, - window_scope: WindowCaptureScope | None = None, -) -> None: - """Reads mouse events and adds them to the event queue. - - Args: - event_q: The event queue to add the mouse events to. - terminate_processing: The event to signal termination of event reading. - recording: The recording object. - started_event: Event to set once started. - window_scope: Optional window scope; when set, mouse coordinates are - translated into the target window's pixel space. - - Returns: - None - """ - utils.set_start_time(recording.timestamp) - - def on_observed(event: ObservedInput) -> None: - if isinstance(event, ObservedMouseMove): - on_move( - event_q, - window_scope, - event.x, - event.y, - event.injected, - ) - elif isinstance(event, ObservedMouseButton): - on_click( - event_q, - window_scope, - event.x, - event.y, - event.button, - event.pressed, - event.injected, - ) - elif isinstance(event, ObservedMouseScroll): - on_scroll( - event_q, - window_scope, - event.x, - event.y, - event.dx, - event.dy, - event.injected, - ) - - observer = create_input_observer( - on_observed, - observe_keyboard=False, observe_mouse=True, capture_mouse_moves=True, ) @@ -1759,46 +1727,26 @@ def record( screen_event_reader.start() task_by_name["screen_event_reader"] = screen_event_reader - keyboard_reader_args = ( - event_q, - terminate_processing, - recording, - task_started_events.setdefault("keyboard_event_reader", threading.Event()), - ) - keyboard_event_reader = threading.Thread( - target=_run_task_fail_loud, - daemon=True, - args=( - "keyboard_event_reader", - read_keyboard_events, - keyboard_reader_args, - terminate_processing, - task_errors, - ), - ) - keyboard_event_reader.start() - task_by_name["keyboard_event_reader"] = keyboard_event_reader - - mouse_reader_args = ( + input_reader_args = ( event_q, terminate_processing, recording, - task_started_events.setdefault("mouse_event_reader", threading.Event()), + task_started_events.setdefault("input_event_reader", threading.Event()), window_scope, ) - mouse_event_reader = threading.Thread( + input_event_reader = threading.Thread( target=_run_task_fail_loud, daemon=True, args=( - "mouse_event_reader", - read_mouse_events, - mouse_reader_args, + "input_event_reader", + read_input_events, + input_reader_args, terminate_processing, task_errors, ), ) - mouse_event_reader.start() - task_by_name["mouse_event_reader"] = mouse_event_reader + input_event_reader.start() + task_by_name["input_event_reader"] = input_event_reader if num_action_events is None: num_action_events = multiprocessing.Value("i", 0) @@ -2039,8 +1987,7 @@ def record( "window_event_reader", "browser_event_reader", "screen_event_reader", - "keyboard_event_reader", - "mouse_event_reader", + "input_event_reader", "event_processor", "screen_event_writer", "browser_event_writer", diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 90049ad..8aa0f66 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -3,6 +3,7 @@ from __future__ import annotations import ctypes +import queue import sys import threading import time @@ -11,6 +12,7 @@ import pytest +from openadapt_capture import input as input_module from openadapt_capture import recorder as recorder_module from openadapt_capture.capture import _convert_action_event from openadapt_capture.events import MouseClickEvent, MouseDownEvent, MouseUpEvent @@ -244,6 +246,154 @@ def block_consumer(_event) -> None: assert observer._delivery_thread is None +def test_async_delivery_preserves_native_receipt_timestamp_and_order() -> None: + callback_entered = threading.Event() + release_callback = threading.Event() + public_events = [] + + def consume(event) -> None: + if not public_events: + callback_entered.set() + assert release_callback.wait(timeout=1) + public_events.append(input_module._to_public_event(event)) + + observer = _ReadyObserver(consume) + observer.start() + observer._emit( + ObservedKey( + pressed=True, + key_char="a", + canonical_key_char="a", + timestamp=10.25, + ) + ) + assert callback_entered.wait(timeout=1) + observer._emit( + ObservedKey( + pressed=False, + key_char="a", + canonical_key_char="a", + timestamp=10.5, + ) + ) + release_callback.set() + observer.stop() + + assert [event.timestamp for event in public_events] == [10.25, 10.5] + + +def test_stop_sequence_callback_can_stop_listener_from_delivery_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observers: list[_ReadyObserver] = [] + + def create(callback, **_kwargs): + observer = _ReadyObserver(callback) + observers.append(observer) + return observer + + monkeypatch.setattr(input_module, "create_input_observer", create) + listener: input_module.KeyboardListener + listener = input_module.KeyboardListener( + lambda _event: None, + stop_sequences=["q"], + on_stop_sequence=lambda: listener.stop(), + ) + listener.start() + observer = observers[0] + observer._emit( + ObservedKey( + pressed=True, + key_char="q", + canonical_key_char="q", + timestamp=20.0, + ) + ) + + deadline = time.monotonic() + 1 + while observer._delivery_thread is not None and observer._delivery_thread.is_alive(): + if time.monotonic() >= deadline: + pytest.fail("listener delivery thread did not exit after callback stop") + time.sleep(0.001) + + assert not listener._running + assert observer._failure is None + + +def test_recorder_uses_one_observer_and_preserves_cross_device_receipt_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + terminate = threading.Event() + started = threading.Event() + event_q: queue.Queue = queue.Queue() + observed = [ + ObservedMouseButton( + x=1, + y=2, + button="left", + pressed=True, + timestamp=100.1, + ), + ObservedKey( + pressed=True, + key_char="a", + canonical_key_char="a", + timestamp=100.2, + ), + ObservedMouseButton( + x=1, + y=2, + button="left", + pressed=False, + timestamp=100.3, + ), + ] + factory_calls = [] + + class FakeObserver: + def __init__(self, callback) -> None: + self.callback = callback + + def start(self) -> None: + for event in observed: + self.callback(event) + terminate.set() + + def check_health(self) -> None: + return + + def stop(self) -> None: + return + + def create(callback, **kwargs): + factory_calls.append(kwargs) + return FakeObserver(callback) + + monkeypatch.setattr(recorder_module, "create_input_observer", create) + recorder_module.read_input_events( + event_q, + terminate, + SimpleNamespace(timestamp=100.0), + started, + ) + + persisted = [event_q.get_nowait() for _ in range(event_q.qsize())] + assert factory_calls == [ + { + "observe_keyboard": True, + "observe_mouse": True, + "capture_mouse_moves": True, + } + ] + assert started.is_set() + assert [event.timestamp for event in persisted] == [100.1, 100.2, 100.3] + assert [event.data["name"] for event in persisted] == [ + "click", + "press", + "click", + ] + + @pytest.mark.parametrize( ("detail", "pressed", "expected"), [ diff --git a/tests/test_input_observer_darwin.py b/tests/test_input_observer_darwin.py index caef1ff..7643ce4 100644 --- a/tests/test_input_observer_darwin.py +++ b/tests/test_input_observer_darwin.py @@ -353,6 +353,23 @@ def test_move_filter_does_not_disable_buttons() -> None: assert quartz.kCGEventMouseMoved not in observer._observed_event_types() +def test_event_callback_stamps_native_receipt_time_before_async_delivery() -> None: + quartz = FakeQuartz() + events = [] + observer = make_observer(quartz, events.append) + event = FakeEvent() + + observer.start() + before = time.time() + observer._event_callback(None, quartz.kCGEventMouseMoved, event, None) + after = time.time() + observer.stop() + + assert len(events) == 1 + assert events[0].timestamp is not None + assert before <= events[0].timestamp <= after + + def test_key_press_release_and_modifier_canonicalization() -> None: quartz = FakeQuartz() events = [] @@ -535,8 +552,8 @@ def fail(_event): observer = make_observer(quartz, fail) event = FakeEvent() - observer._handle_event = lambda _type, _event: observer._emit( # type: ignore[method-assign] - ObservedMouseMove(1, 2) + observer._handle_event = ( # type: ignore[method-assign] + lambda _type, _event, **_kwargs: observer._emit(ObservedMouseMove(1, 2)) ) observer.start() From 2ca57fefb23eab2201a5f41cb812dd81e801ace0 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:45:18 -0700 Subject: [PATCH 10/26] feat: add nonblocking native Windows input observer --- openadapt_capture/input_observer/windows.py | 901 ++++++++++++++++++++ tests/test_input_observer_windows.py | 741 ++++++++++++++++ 2 files changed, 1642 insertions(+) create mode 100644 openadapt_capture/input_observer/windows.py create mode 100644 tests/test_input_observer_windows.py diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py new file mode 100644 index 0000000..0b0d5ba --- /dev/null +++ b/openadapt_capture/input_observer/windows.py @@ -0,0 +1,901 @@ +"""Native Windows global input observation using permissive stdlib ``ctypes``. + +The implementation uses the documented ``WH_KEYBOARD_LL`` and ``WH_MOUSE_LL`` +hooks. It intentionally exposes observation only: input injection remains a +separate concern, and events marked as injected by Windows are discarded. +""" + +from __future__ import annotations + +import ctypes +import queue +import sys +import threading +import time +from dataclasses import dataclass +from typing import Any, Callable + +from .base import ( + InputCallback, + InputObserverError, + InputObserverPermissionError, + InputObserverUnavailableError, + ObservedInput, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, + ThreadedInputObserver, +) + +# Hook and message constants from winuser.h. +WH_KEYBOARD_LL = 13 +WH_MOUSE_LL = 14 +HC_ACTION = 0 + +WM_QUIT = 0x0012 +WM_KEYDOWN = 0x0100 +WM_KEYUP = 0x0101 +WM_SYSKEYDOWN = 0x0104 +WM_SYSKEYUP = 0x0105 +WM_MOUSEMOVE = 0x0200 +WM_LBUTTONDOWN = 0x0201 +WM_LBUTTONUP = 0x0202 +WM_RBUTTONDOWN = 0x0204 +WM_RBUTTONUP = 0x0205 +WM_MBUTTONDOWN = 0x0207 +WM_MBUTTONUP = 0x0208 +WM_MOUSEWHEEL = 0x020A +WM_XBUTTONDOWN = 0x020B +WM_XBUTTONUP = 0x020C +WM_MOUSEHWHEEL = 0x020E +WM_USER = 0x0400 + +PM_NOREMOVE = 0x0000 +WHEEL_DELTA = 120 +XBUTTON1 = 0x0001 +XBUTTON2 = 0x0002 +TO_UNICODE_NO_STATE_CHANGE = 0x0004 + +LLKHF_EXTENDED = 0x00000001 +LLKHF_LOWER_IL_INJECTED = 0x00000002 +LLKHF_INJECTED = 0x00000010 +LLMHF_INJECTED = 0x00000001 +LLMHF_LOWER_IL_INJECTED = 0x00000002 + +ERROR_ACCESS_DENIED = 5 +ERROR_PRIVILEGE_NOT_HELD = 1314 + +VK_SHIFT = 0x10 +VK_CONTROL = 0x11 +VK_MENU = 0x12 +VK_CAPITAL = 0x14 +VK_LSHIFT = 0xA0 +VK_RSHIFT = 0xA1 +VK_LCONTROL = 0xA2 +VK_RCONTROL = 0xA3 +VK_LMENU = 0xA4 +VK_RMENU = 0xA5 +VK_NUMLOCK = 0x90 +VK_SCROLL = 0x91 + +# Fixed-width aliases keep the structures testable off Windows as well as +# matching the Windows ABI on both 32- and 64-bit Python. +DWORD = ctypes.c_uint32 +LONG = ctypes.c_int32 +ULONG_PTR = ctypes.c_size_t +WPARAM = ctypes.c_size_t +LPARAM = ctypes.c_ssize_t +LRESULT = ctypes.c_ssize_t +HANDLE = ctypes.c_void_p +HWND = ctypes.c_void_p +HHOOK = ctypes.c_void_p +HINSTANCE = ctypes.c_void_p +SHORT = ctypes.c_short + +WINFUNCTYPE = getattr(ctypes, "WINFUNCTYPE", ctypes.CFUNCTYPE) +HOOKPROC = WINFUNCTYPE(LRESULT, ctypes.c_int, WPARAM, LPARAM) + + +class POINT(ctypes.Structure): + """Windows screen coordinate.""" + + _fields_ = [("x", LONG), ("y", LONG)] + + +class KBDLLHOOKSTRUCT(ctypes.Structure): + """Payload delivered to a low-level keyboard hook.""" + + _fields_ = [ + ("vkCode", DWORD), + ("scanCode", DWORD), + ("flags", DWORD), + ("time", DWORD), + ("dwExtraInfo", ULONG_PTR), + ] + + +class MSLLHOOKSTRUCT(ctypes.Structure): + """Payload delivered to a low-level mouse hook.""" + + _fields_ = [ + ("pt", POINT), + ("mouseData", DWORD), + ("flags", DWORD), + ("time", DWORD), + ("dwExtraInfo", ULONG_PTR), + ] + + +class MSG(ctypes.Structure): + """Message-loop payload.""" + + _fields_ = [ + ("hwnd", HWND), + ("message", ctypes.c_uint32), + ("wParam", WPARAM), + ("lParam", LPARAM), + ("time", DWORD), + ("pt", POINT), + ("lPrivate", DWORD), + ] + + +@dataclass(frozen=True, slots=True) +class _RawKeyboardTransition: + """Scalar copy of a keyboard hook payload safe to process off-hook.""" + + pressed: bool + vk_code: int + scan_code: int + flags: int + timestamp: float + + +@dataclass(frozen=True, slots=True) +class _RawMouseTransition: + """Scalar copy of a mouse hook payload safe to process off-hook.""" + + message: int + x: int + y: int + mouse_data: int + flags: int + timestamp: float + + +_RawInputTransition = _RawKeyboardTransition | _RawMouseTransition + + +_PHYSICAL_KEY_NAMES = { + 0x08: "backspace", + 0x09: "tab", + 0x0D: "enter", + 0x10: "shift", + 0x11: "ctrl", + 0x12: "alt", + 0x13: "pause", + 0x14: "caps_lock", + 0x1B: "esc", + 0x20: "space", + 0x21: "page_up", + 0x22: "page_down", + 0x23: "end", + 0x24: "home", + 0x25: "left", + 0x26: "up", + 0x27: "right", + 0x28: "down", + 0x2C: "print_screen", + 0x2D: "insert", + 0x2E: "delete", + 0x5B: "cmd_l", + 0x5C: "cmd_r", + 0x5D: "menu", + 0x70: "f1", + 0x71: "f2", + 0x72: "f3", + 0x73: "f4", + 0x74: "f5", + 0x75: "f6", + 0x76: "f7", + 0x77: "f8", + 0x78: "f9", + 0x79: "f10", + 0x7A: "f11", + 0x7B: "f12", + 0x7C: "f13", + 0x7D: "f14", + 0x7E: "f15", + 0x7F: "f16", + 0x80: "f17", + 0x81: "f18", + 0x82: "f19", + 0x83: "f20", + 0x84: "f21", + 0x85: "f22", + 0x86: "f23", + 0x87: "f24", + 0x90: "num_lock", + 0x91: "scroll_lock", + 0xA0: "shift_l", + 0xA1: "shift_r", + 0xA2: "ctrl_l", + 0xA3: "ctrl_r", + 0xA4: "alt_l", + 0xA5: "alt_r", +} + +_CANONICAL_MODIFIERS = { + 0xA0: ("shift", 0x10), + 0xA1: ("shift", 0x10), + 0xA2: ("ctrl", 0x11), + 0xA3: ("ctrl", 0x11), + 0xA4: ("alt", 0x12), + 0xA5: ("alt", 0x12), + 0x5B: ("cmd", 0x5B), + 0x5C: ("cmd", 0x5B), +} + +_BUTTON_MESSAGES = { + WM_LBUTTONDOWN: ("left", True), + WM_LBUTTONUP: ("left", False), + WM_RBUTTONDOWN: ("right", True), + WM_RBUTTONUP: ("right", False), + WM_MBUTTONDOWN: ("middle", True), + WM_MBUTTONUP: ("middle", False), +} + + +def _set_signature(function: Any, argtypes: list[Any], restype: Any) -> None: + """Set a ctypes signature while allowing plain Python fakes in tests.""" + + try: + function.argtypes = argtypes + function.restype = restype + except (AttributeError, TypeError): + return + + +def _handle_value(handle: object) -> int: + """Return a stable integer value for a ctypes or fake Windows handle.""" + + if handle is None: + return 0 + if isinstance(handle, int): + return handle + value = getattr(handle, "value", None) + return int(value or 0) + + +def _signed_high_word(value: int) -> int: + """Extract a signed 16-bit wheel delta from ``mouseData``.""" + + high_word = (int(value) >> 16) & 0xFFFF + return high_word - 0x10000 if high_word & 0x8000 else high_word + + +def _key_identity(vk_code: int, character: str | None) -> dict[str, str | None]: + """Normalize physical and canonical key identity like pynput's listener.""" + + key_name = _PHYSICAL_KEY_NAMES.get(vk_code) + canonical_name = key_name + canonical_vk = vk_code + canonical_character = character.lower() if character else None + + modifier = _CANONICAL_MODIFIERS.get(vk_code) + if modifier is not None: + canonical_name, canonical_vk = modifier + character = None + canonical_character = None + elif key_name is not None: + character = None + canonical_character = None + + return { + "key_name": key_name, + "key_char": character, + "key_vk": str(vk_code), + "canonical_key_name": canonical_name, + "canonical_key_char": canonical_character, + "canonical_key_vk": str(canonical_vk), + } + + +def _mouse_event( + message: int, + payload: MSLLHOOKSTRUCT, + *, + capture_mouse_moves: bool, + timestamp: float | None = None, +) -> ObservedInput | None: + """Translate a Windows low-level mouse payload to the common contract.""" + + injected = bool(payload.flags & (LLMHF_INJECTED | LLMHF_LOWER_IL_INJECTED)) + if injected: + return None + + x = float(payload.pt.x) + y = float(payload.pt.y) + if message == WM_MOUSEMOVE: + if not capture_mouse_moves: + return None + return ObservedMouseMove(x=x, y=y, timestamp=timestamp) + + button_transition = _BUTTON_MESSAGES.get(message) + if button_transition is not None: + button, pressed = button_transition + return ObservedMouseButton( + x=x, + y=y, + button=button, + pressed=pressed, + timestamp=timestamp, + ) + + if message in {WM_XBUTTONDOWN, WM_XBUTTONUP}: + xbutton = (int(payload.mouseData) >> 16) & 0xFFFF + button = "x1" if xbutton == XBUTTON1 else "x2" if xbutton == XBUTTON2 else None + if button is None: + return None + return ObservedMouseButton( + x=x, + y=y, + button=button, + pressed=message == WM_XBUTTONDOWN, + timestamp=timestamp, + ) + + if message == WM_MOUSEWHEEL: + return ObservedMouseScroll( + x=x, + y=y, + dx=0.0, + dy=_signed_high_word(payload.mouseData) / WHEEL_DELTA, + timestamp=timestamp, + ) + if message == WM_MOUSEHWHEEL: + return ObservedMouseScroll( + x=x, + y=y, + dx=_signed_high_word(payload.mouseData) / WHEEL_DELTA, + dy=0.0, + timestamp=timestamp, + ) + return None + + +class WindowsInputObserver(ThreadedInputObserver): + """Observe global Windows keyboard and mouse input with low-level hooks.""" + + def __init__( + self, + callback: InputCallback, + *, + observe_keyboard: bool, + observe_mouse: bool, + capture_mouse_moves: bool, + startup_timeout: float = 5.0, + shutdown_timeout: float = 5.0, + delivery_queue_size: int = 4096, + translation_queue_size: int | None = None, + _user32: Any | None = None, + _kernel32: Any | None = None, + _clock: Callable[[], float] = time.time, + ) -> None: + super().__init__( + callback, + observe_keyboard=observe_keyboard, + observe_mouse=observe_mouse, + capture_mouse_moves=capture_mouse_moves, + startup_timeout=startup_timeout, + shutdown_timeout=shutdown_timeout, + delivery_queue_size=delivery_queue_size, + ) + self._user32 = _user32 + self._kernel32 = _kernel32 + self._clock = _clock + self._keyboard_hook: object | None = None + self._mouse_hook: object | None = None + self._keyboard_proc: Any | None = None + self._mouse_proc: Any | None = None + self._thread_id: int | None = None + self.translation_queue_size = ( + delivery_queue_size if translation_queue_size is None else translation_queue_size + ) + if self.translation_queue_size <= 0: + raise ValueError("translation_queue_size must be positive") + self._translation_queue: queue.Queue[_RawInputTransition | object] = queue.Queue( + maxsize=self.translation_queue_size + ) + self._translation_sentinel = object() + self._translation_stop_requested = threading.Event() + self._translation_thread: threading.Thread | None = None + self._keyboard_state = (ctypes.c_ubyte * 256)() + self._keys_down: set[int] = set() + + def _load_apis(self) -> None: + if self._user32 is not None and self._kernel32 is not None: + return + if sys.platform != "win32": + raise InputObserverUnavailableError( + "Windows input observation requires a native Windows session" + ) + try: + self._user32 = ctypes.WinDLL("user32", use_last_error=True) + self._kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + except (AttributeError, OSError) as exc: + raise InputObserverUnavailableError( + f"Windows input APIs are unavailable: {exc}" + ) from exc + + def _configure_apis(self) -> None: + user32 = self._user32 + kernel32 = self._kernel32 + assert user32 is not None + assert kernel32 is not None + _set_signature( + user32.SetWindowsHookExW, + [ctypes.c_int, HOOKPROC, HINSTANCE, DWORD], + HHOOK, + ) + _set_signature(user32.UnhookWindowsHookEx, [HHOOK], ctypes.c_int) + _set_signature( + user32.CallNextHookEx, + [HHOOK, ctypes.c_int, WPARAM, LPARAM], + LRESULT, + ) + _set_signature( + user32.GetMessageW, + [ctypes.POINTER(MSG), HWND, ctypes.c_uint32, ctypes.c_uint32], + ctypes.c_int, + ) + _set_signature(user32.TranslateMessage, [ctypes.POINTER(MSG)], ctypes.c_int) + _set_signature(user32.DispatchMessageW, [ctypes.POINTER(MSG)], LRESULT) + _set_signature( + user32.PeekMessageW, + [ + ctypes.POINTER(MSG), + HWND, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_uint32, + ], + ctypes.c_int, + ) + _set_signature( + user32.PostThreadMessageW, + [DWORD, ctypes.c_uint32, WPARAM, LPARAM], + ctypes.c_int, + ) + _set_signature(user32.GetAsyncKeyState, [ctypes.c_int], SHORT) + _set_signature(user32.GetKeyState, [ctypes.c_int], SHORT) + _set_signature(user32.GetForegroundWindow, [], HWND) + _set_signature( + user32.GetWindowThreadProcessId, + [HWND, ctypes.POINTER(DWORD)], + DWORD, + ) + _set_signature( + user32.ToUnicodeEx, + [ + ctypes.c_uint, + ctypes.c_uint, + ctypes.POINTER(ctypes.c_ubyte), + ctypes.POINTER(ctypes.c_wchar), + ctypes.c_int, + ctypes.c_uint, + HANDLE, + ], + ctypes.c_int, + ) + _set_signature(user32.GetKeyboardLayout, [DWORD], HANDLE) + _set_signature(kernel32.GetModuleHandleW, [ctypes.c_wchar_p], HINSTANCE) + _set_signature(kernel32.GetCurrentThreadId, [], DWORD) + + def _last_error(self) -> int: + for api in (self._user32, self._kernel32): + value = getattr(api, "last_error", None) + if value is not None: + return int(value) + get_last_error = getattr(ctypes, "get_last_error", None) + return int(get_last_error()) if get_last_error is not None else 0 + + def _hook_error(self, hook_name: str) -> InputObserverError: + error_code = self._last_error() + detail = f"Windows error {error_code}" if error_code else "unknown Windows error" + message = f"could not install the global {hook_name} hook ({detail})" + if error_code in {ERROR_ACCESS_DENIED, ERROR_PRIVILEGE_NOT_HELD}: + return InputObserverPermissionError(message) + return InputObserverUnavailableError(message) + + def _setup(self) -> None: + self._load_apis() + self._configure_apis() + user32 = self._user32 + kernel32 = self._kernel32 + assert user32 is not None + assert kernel32 is not None + + self._thread_id = int(kernel32.GetCurrentThreadId()) + # Force creation of the thread message queue before another thread can + # request shutdown with PostThreadMessageW. + message = MSG() + user32.PeekMessageW( + ctypes.byref(message), + None, + WM_USER, + WM_USER, + PM_NOREMOVE, + ) + + if self.observe_keyboard: + self._initialize_keyboard_state() + self._start_input_translation() + + self._keyboard_proc = HOOKPROC(self._keyboard_hook_callback) + self._mouse_proc = HOOKPROC(self._mouse_hook_callback) + module = kernel32.GetModuleHandleW(None) + + if self.observe_keyboard: + hook = user32.SetWindowsHookExW( + WH_KEYBOARD_LL, + self._keyboard_proc, + module, + 0, + ) + if not _handle_value(hook): + raise self._hook_error("keyboard") + self._keyboard_hook = hook + + if self.observe_mouse: + hook = user32.SetWindowsHookExW( + WH_MOUSE_LL, + self._mouse_proc, + module, + 0, + ) + if not _handle_value(hook): + raise self._hook_error("mouse") + self._mouse_hook = hook + + def _run_loop(self) -> None: + user32 = self._user32 + assert user32 is not None + message = MSG() + while not self._stop_requested.is_set(): + result = int(user32.GetMessageW(ctypes.byref(message), None, 0, 0)) + if result == 0: + return + if result == -1: + error_code = self._last_error() + raise InputObserverError( + "Windows input-observer message loop failed " f"(Windows error {error_code})" + ) + user32.TranslateMessage(ctypes.byref(message)) + user32.DispatchMessageW(ctypes.byref(message)) + + def _wake(self) -> None: + user32 = self._user32 + thread_id = self._thread_id + if user32 is None or thread_id is None: + return + user32.PostThreadMessageW(thread_id, WM_QUIT, 0, 0) + + def _teardown(self) -> None: + user32 = self._user32 + failures: list[str] = [] + if user32 is not None: + for name, attribute in ( + ("keyboard", "_keyboard_hook"), + ("mouse", "_mouse_hook"), + ): + hook = getattr(self, attribute) + if hook is None: + continue + try: + released = user32.UnhookWindowsHookEx(hook) + except BaseException as exc: + failures.append(f"{name} hook ({exc})") + else: + if not released: + failures.append(f"{name} hook (Windows error {self._last_error()})") + finally: + setattr(self, attribute, None) + self._stop_input_translation(failures) + self._keyboard_proc = None + self._mouse_proc = None + self._thread_id = None + if failures: + raise InputObserverError( + "could not release Windows input observer " + ", ".join(failures) + ) + + def _record_callback_failure(self, exc: BaseException) -> None: + self._fail(exc) + + def _call_next(self, hook: object | None, code: int, wparam: int, lparam: int) -> int: + user32 = self._user32 + assert user32 is not None + return int(user32.CallNextHookEx(hook, code, wparam, lparam)) + + @staticmethod + def _physical_vk(transition: _RawKeyboardTransition) -> int: + """Resolve generic modifier virtual keys to their physical side.""" + + if transition.vk_code == VK_SHIFT: + return VK_RSHIFT if transition.scan_code == 0x36 else VK_LSHIFT + if transition.vk_code == VK_CONTROL: + return VK_RCONTROL if transition.flags & LLKHF_EXTENDED else VK_LCONTROL + if transition.vk_code == VK_MENU: + return VK_RMENU if transition.flags & LLKHF_EXTENDED else VK_LMENU + return transition.vk_code + + def _sync_generic_modifier_state(self) -> None: + groups = { + VK_SHIFT: {VK_SHIFT, VK_LSHIFT, VK_RSHIFT}, + VK_CONTROL: {VK_CONTROL, VK_LCONTROL, VK_RCONTROL}, + VK_MENU: {VK_MENU, VK_LMENU, VK_RMENU}, + } + for generic_vk, members in groups.items(): + pressed = any(member in self._keys_down for member in members) + toggled = self._keyboard_state[generic_vk] & 0x01 + self._keyboard_state[generic_vk] = toggled | (0x80 if pressed else 0) + + def _initialize_keyboard_state(self) -> None: + user32 = self._user32 + assert user32 is not None + self._keyboard_state = (ctypes.c_ubyte * 256)() + self._keys_down.clear() + for vk_code in { + VK_LSHIFT, + VK_RSHIFT, + VK_LCONTROL, + VK_RCONTROL, + VK_LMENU, + VK_RMENU, + }: + if int(user32.GetAsyncKeyState(vk_code)) & 0x8000: + self._keys_down.add(vk_code) + self._keyboard_state[vk_code] |= 0x80 + for vk_code in {VK_CAPITAL, VK_NUMLOCK, VK_SCROLL}: + if int(user32.GetKeyState(vk_code)) & 0x0001: + self._keyboard_state[vk_code] |= 0x01 + self._sync_generic_modifier_state() + + def _apply_keyboard_transition(self, transition: _RawKeyboardTransition) -> int: + physical_vk = self._physical_vk(transition) + was_pressed = physical_vk in self._keys_down + if transition.pressed: + self._keys_down.add(physical_vk) + self._keyboard_state[physical_vk] |= 0x80 + else: + self._keys_down.discard(physical_vk) + self._keyboard_state[physical_vk] &= 0x7F + + if ( + transition.pressed + and not was_pressed + and physical_vk in {VK_CAPITAL, VK_NUMLOCK, VK_SCROLL} + ): + self._keyboard_state[physical_vk] ^= 0x01 + self._sync_generic_modifier_state() + return physical_vk + + def _foreground_keyboard_layout(self) -> object: + user32 = self._user32 + assert user32 is not None + foreground = user32.GetForegroundWindow() + if not _handle_value(foreground): + raise InputObserverUnavailableError( + "Windows could not identify the foreground window for keyboard layout" + ) + thread_id = int(user32.GetWindowThreadProcessId(foreground, None)) + if not thread_id: + raise InputObserverUnavailableError( + "Windows could not identify the foreground input thread" + ) + layout = user32.GetKeyboardLayout(thread_id) + if not _handle_value(layout): + raise InputObserverUnavailableError( + "Windows could not resolve the foreground keyboard layout" + ) + return layout + + def _keyboard_character( + self, + transition: _RawKeyboardTransition, + ) -> str | None: + user32 = self._user32 + assert user32 is not None + buffer = ctypes.create_unicode_buffer(8) + count = int( + user32.ToUnicodeEx( + transition.vk_code, + transition.scan_code, + self._keyboard_state, + buffer, + len(buffer), + TO_UNICODE_NO_STATE_CHANGE, + self._foreground_keyboard_layout(), + ) + ) + if count == 0: + return None + character_count = abs(count) + if ctypes.sizeof(ctypes.c_wchar) == 2: + raw = ctypes.string_at(buffer, character_count * 2) + return raw.decode("utf-16-le") + return "".join(buffer[:character_count]) + + def _translate_keyboard( + self, + transition: _RawKeyboardTransition, + ) -> ObservedKey: + physical_vk = self._apply_keyboard_transition(transition) + character = ( + None if physical_vk in _PHYSICAL_KEY_NAMES else self._keyboard_character(transition) + ) + return ObservedKey( + pressed=transition.pressed, + timestamp=transition.timestamp, + **_key_identity(physical_vk, character), + ) + + def _translate_mouse( + self, + transition: _RawMouseTransition, + ) -> ObservedInput | None: + payload = MSLLHOOKSTRUCT( + pt=POINT(transition.x, transition.y), + mouseData=transition.mouse_data, + flags=transition.flags, + ) + return _mouse_event( + transition.message, + payload, + capture_mouse_moves=self.capture_mouse_moves, + timestamp=transition.timestamp, + ) + + def _input_translation_main(self) -> None: + while True: + if self._translation_stop_requested.is_set() and self._translation_queue.empty(): + return + try: + item = self._translation_queue.get(timeout=0.1) + except queue.Empty: + continue + try: + if item is self._translation_sentinel: + return + assert isinstance( + item, + (_RawKeyboardTransition, _RawMouseTransition), + ) + event = ( + self._translate_keyboard(item) + if isinstance(item, _RawKeyboardTransition) + else self._translate_mouse(item) + ) + if event is not None: + self._emit(event) + except BaseException as exc: + failure = ( + exc + if isinstance(exc, InputObserverError) + else InputObserverError(f"Windows input translation failed: {exc}") + ) + self._fail(failure) + return + finally: + self._translation_queue.task_done() + + def _start_input_translation(self) -> None: + thread = self._translation_thread + if thread is not None and thread.is_alive(): + raise InputObserverError("Windows input translation worker is already running") + self._translation_queue = queue.Queue(maxsize=self.translation_queue_size) + self._translation_stop_requested.clear() + thread = threading.Thread( + target=self._input_translation_main, + name=f"{type(self).__name__}-input-translation", + daemon=True, + ) + self._translation_thread = thread + try: + thread.start() + except BaseException: + self._translation_thread = None + raise + + def _stop_input_translation(self, failures: list[str]) -> None: + thread = self._translation_thread + if thread is None: + return + self._translation_stop_requested.set() + deadline = time.monotonic() + (self.shutdown_timeout * 0.8) + if thread.is_alive(): + try: + self._translation_queue.put_nowait(self._translation_sentinel) + except queue.Full: + pass + thread.join(max(0.0, deadline - time.monotonic())) + if thread.is_alive(): + failures.append( + "input translation worker did not stop within " f"{self.shutdown_timeout:.1f}s" + ) + else: + self._translation_thread = None + + def _enqueue_input(self, transition: _RawInputTransition) -> None: + try: + self._translation_queue.put_nowait(transition) + except queue.Full: + failure = InputObserverError( + "Windows input translation queue overflowed; " "recording coverage is incomplete" + ) + self._fail(failure) + raise failure + + def _keyboard_hook_callback( + self, + code: int, + wparam: int, + lparam: int, + ) -> int: + try: + if code == HC_ACTION and wparam in { + WM_KEYDOWN, + WM_KEYUP, + WM_SYSKEYDOWN, + WM_SYSKEYUP, + }: + timestamp = self._clock() + payload = ctypes.cast( + lparam, + ctypes.POINTER(KBDLLHOOKSTRUCT), + ).contents + injected = bool(payload.flags & (LLKHF_INJECTED | LLKHF_LOWER_IL_INJECTED)) + if not injected: + self._enqueue_input( + _RawKeyboardTransition( + pressed=wparam in {WM_KEYDOWN, WM_SYSKEYDOWN}, + vk_code=int(payload.vkCode), + scan_code=int(payload.scanCode), + flags=int(payload.flags), + timestamp=timestamp, + ) + ) + except BaseException as exc: + self._record_callback_failure(exc) + return self._call_next(self._keyboard_hook, code, wparam, lparam) + + def _mouse_hook_callback( + self, + code: int, + wparam: int, + lparam: int, + ) -> int: + try: + if code == HC_ACTION: + timestamp = self._clock() + payload = ctypes.cast( + lparam, + ctypes.POINTER(MSLLHOOKSTRUCT), + ).contents + injected = bool(payload.flags & (LLMHF_INJECTED | LLMHF_LOWER_IL_INJECTED)) + if not injected and (wparam != WM_MOUSEMOVE or self.capture_mouse_moves): + self._enqueue_input( + _RawMouseTransition( + message=wparam, + x=int(payload.pt.x), + y=int(payload.pt.y), + mouse_data=int(payload.mouseData), + flags=int(payload.flags), + timestamp=timestamp, + ) + ) + except BaseException as exc: + self._record_callback_failure(exc) + return self._call_next(self._mouse_hook, code, wparam, lparam) diff --git a/tests/test_input_observer_windows.py b/tests/test_input_observer_windows.py new file mode 100644 index 0000000..34d13ba --- /dev/null +++ b/tests/test_input_observer_windows.py @@ -0,0 +1,741 @@ +"""Focused contract tests for the stdlib Windows input observer.""" + +from __future__ import annotations + +import ctypes +import threading +import time + +import pytest + +from openadapt_capture.input_observer import ( + InputObserverError, + InputObserverPermissionError, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, + ObservedMouseScroll, +) +from openadapt_capture.input_observer.windows import ( + ERROR_ACCESS_DENIED, + HC_ACTION, + KBDLLHOOKSTRUCT, + LLKHF_INJECTED, + LLMHF_INJECTED, + MSLLHOOKSTRUCT, + POINT, + VK_CAPITAL, + VK_SHIFT, + WM_KEYDOWN, + WM_KEYUP, + WM_LBUTTONDOWN, + WM_MOUSEHWHEEL, + WM_MOUSEMOVE, + WM_MOUSEWHEEL, + WM_XBUTTONUP, + XBUTTON2, + WindowsInputObserver, + _key_identity, + _mouse_event, + _signed_high_word, +) + + +class FakeKernel32: + def GetCurrentThreadId(self) -> int: + return 4242 + + def GetModuleHandleW(self, _name): + return 99 + + +class FakeUser32: + def __init__( + self, + *, + hook_results: list[int] | None = None, + last_error: int = 0, + unhook_failures: set[int] | None = None, + foreground_thread_id: int = 8080, + keyboard_layout: int = 1, + translation_entered: threading.Event | None = None, + translation_release: threading.Event | None = None, + translation_error: BaseException | None = None, + ) -> None: + self.hook_results = list(hook_results or [101, 102]) + self.last_error = last_error + self.unhook_failures = unhook_failures or set() + self.installed: list[int] = [] + self.unhooked: list[int] = [] + self.next_calls: list[tuple[object, int, int, int]] = [] + self.posted = threading.Event() + self.foreground_thread_id = foreground_thread_id + self.keyboard_layout = keyboard_layout + self.translation_entered = translation_entered + self.translation_release = translation_release + self.translation_error = translation_error + self.async_key_state: dict[int, int] = {} + self.toggle_key_state: dict[int, int] = {} + self.layout_thread_ids: list[int] = [] + + def PeekMessageW(self, *_args) -> int: + return 0 + + def SetWindowsHookExW(self, hook_type, _callback, _module, _thread_id): + result = self.hook_results.pop(0) + if result: + self.installed.append(hook_type) + return result + + def UnhookWindowsHookEx(self, hook) -> int: + self.unhooked.append(int(hook)) + return int(hook not in self.unhook_failures) + + def CallNextHookEx(self, hook, code, wparam, lparam) -> int: + self.next_calls.append((hook, code, wparam, lparam)) + return 73 + + def GetMessageW(self, *_args) -> int: + self.posted.wait(timeout=2) + return 0 + + def TranslateMessage(self, *_args) -> int: + return 1 + + def DispatchMessageW(self, *_args) -> int: + return 1 + + def PostThreadMessageW(self, *_args) -> int: + self.posted.set() + return 1 + + def GetAsyncKeyState(self, vk_code) -> int: + return self.async_key_state.get(vk_code, 0) + + def GetKeyState(self, vk_code) -> int: + return self.toggle_key_state.get(vk_code, 0) + + def GetForegroundWindow(self) -> int: + return 9001 + + def GetWindowThreadProcessId(self, hwnd, process_id) -> int: + assert hwnd == 9001 + assert process_id is None + return self.foreground_thread_id + + def ToUnicodeEx( + self, + vk_code, + _scan_code, + keyboard_state, + buffer, + _buffer_size, + flags, + _layout, + ) -> int: + assert flags == 4 + if self.translation_entered is not None: + self.translation_entered.set() + if self.translation_release is not None: + self.translation_release.wait(timeout=2) + if self.translation_error is not None: + raise self.translation_error + character = chr(vk_code) + shift = bool(keyboard_state[VK_SHIFT] & 0x80) + caps_lock = bool(keyboard_state[VK_CAPITAL] & 0x01) + if shift ^ caps_lock: + character = character.upper() + else: + character = character.lower() + buffer[0] = character + return 1 + + def GetKeyboardLayout(self, thread_id): + self.layout_thread_ids.append(thread_id) + return self.keyboard_layout + + +def make_observer( + callback, + *, + user32: FakeUser32 | None = None, + observe_keyboard: bool = True, + observe_mouse: bool = True, + capture_mouse_moves: bool = True, + delivery_queue_size: int = 4096, + translation_queue_size: int | None = None, + clock=lambda: 1234.5, +) -> WindowsInputObserver: + return WindowsInputObserver( + callback, + observe_keyboard=observe_keyboard, + observe_mouse=observe_mouse, + capture_mouse_moves=capture_mouse_moves, + startup_timeout=1, + shutdown_timeout=1, + delivery_queue_size=delivery_queue_size, + translation_queue_size=translation_queue_size, + _user32=user32 or FakeUser32(), + _kernel32=FakeKernel32(), + _clock=clock, + ) + + +def wait_until(predicate, *, timeout: float = 1.0) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.005) + return bool(predicate()) + + +def test_signed_wheel_delta_and_mouse_normalization() -> None: + assert _signed_high_word(120 << 16) == 120 + assert _signed_high_word(((-240) & 0xFFFF) << 16) == -240 + + vertical = MSLLHOOKSTRUCT( + pt=POINT(-10, 25), + mouseData=((-240) & 0xFFFF) << 16, + ) + horizontal = MSLLHOOKSTRUCT( + pt=POINT(30, 40), + mouseData=(120 & 0xFFFF) << 16, + ) + assert _mouse_event( + WM_MOUSEWHEEL, + vertical, + capture_mouse_moves=True, + ) == ObservedMouseScroll(x=-10.0, y=25.0, dx=0.0, dy=-2.0) + assert _mouse_event( + WM_MOUSEHWHEEL, + horizontal, + capture_mouse_moves=True, + ) == ObservedMouseScroll(x=30.0, y=40.0, dx=1.0, dy=0.0) + + +def test_mouse_move_button_xbutton_and_injected_filtering() -> None: + payload = MSLLHOOKSTRUCT(pt=POINT(12, 34)) + assert _mouse_event( + WM_MOUSEMOVE, + payload, + capture_mouse_moves=True, + ) == ObservedMouseMove(x=12.0, y=34.0) + assert ( + _mouse_event( + WM_MOUSEMOVE, + payload, + capture_mouse_moves=False, + ) + is None + ) + assert _mouse_event( + WM_LBUTTONDOWN, + payload, + capture_mouse_moves=True, + ) == ObservedMouseButton(x=12.0, y=34.0, button="left", pressed=True) + + payload.mouseData = XBUTTON2 << 16 + assert _mouse_event( + WM_XBUTTONUP, + payload, + capture_mouse_moves=True, + ) == ObservedMouseButton(x=12.0, y=34.0, button="x2", pressed=False) + + payload.flags = LLMHF_INJECTED + assert ( + _mouse_event( + WM_LBUTTONDOWN, + payload, + capture_mouse_moves=True, + ) + is None + ) + + +def test_key_identity_preserves_physical_and_canonical_semantics() -> None: + assert _key_identity(0x41, "A") == { + "key_name": None, + "key_char": "A", + "key_vk": "65", + "canonical_key_name": None, + "canonical_key_char": "a", + "canonical_key_vk": "65", + } + assert _key_identity(0xA1, None) == { + "key_name": "shift_r", + "key_char": None, + "key_vk": "161", + "canonical_key_name": "shift", + "canonical_key_char": None, + "canonical_key_vk": "16", + } + + +def test_callbacks_emit_normalized_events_filter_injected_and_chain() -> None: + events = [] + user32 = FakeUser32() + observer = make_observer(events.append, user32=user32) + observer.start() + + key_payload = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=30) + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(key_payload), + ) + == 73 + ) + key_payload.flags = LLKHF_INJECTED + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYUP, + ctypes.addressof(key_payload), + ) + + mouse_payload = MSLLHOOKSTRUCT(pt=POINT(5, 7)) + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(mouse_payload), + ) + mouse_payload.flags = LLMHF_INJECTED + observer._mouse_hook_callback( + HC_ACTION, + WM_MOUSEMOVE, + ctypes.addressof(mouse_payload), + ) + + assert wait_until(lambda: len(events) == 2) + assert events == [ + ObservedKey( + pressed=True, + key_char="a", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", + timestamp=1234.5, + ), + ObservedMouseButton( + x=5.0, + y=7.0, + button="left", + pressed=True, + timestamp=1234.5, + ), + ] + assert len(user32.next_calls) == 4 + observer.stop() + + +def test_partial_setup_permission_failure_rolls_back_installed_hook() -> None: + user32 = FakeUser32( + hook_results=[501, 0], + last_error=ERROR_ACCESS_DENIED, + ) + observer = make_observer(lambda _event: None, user32=user32) + + with pytest.raises(InputObserverPermissionError, match="mouse hook"): + observer.start() + + assert user32.installed == [13] + assert user32.unhooked == [501] + assert observer._translation_thread is None + + +def test_threaded_lifecycle_wakes_and_unhooks_every_installed_hook() -> None: + user32 = FakeUser32() + observer = make_observer(lambda _event: None, user32=user32) + + observer.start() + observer.stop() + + assert user32.installed == [13, 14] + assert user32.posted.is_set() + assert user32.unhooked == [101, 102] + assert observer._delivery_thread is None + assert observer._translation_thread is None + + +def test_slow_consumer_never_blocks_hook_callback_or_hook_chain() -> None: + entered = threading.Event() + release = threading.Event() + delivered = [] + user32 = FakeUser32() + + def consume(event) -> None: + entered.set() + release.wait(timeout=2) + delivered.append(event) + + observer = make_observer(consume, user32=user32) + observer.start() + payload = MSLLHOOKSTRUCT(pt=POINT(9, 11)) + + callback_result = [] + callback_returned = threading.Event() + + def invoke_hook() -> None: + callback_result.append( + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + ) + callback_returned.set() + + caller = threading.Thread(target=invoke_hook) + caller.start() + assert callback_returned.wait(timeout=0.5) + caller.join(timeout=0.5) + assert not caller.is_alive() + assert callback_result == [73] + assert entered.wait(timeout=1) + assert not release.is_set() + assert user32.next_calls[-1][1:] == ( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + + release.set() + observer.stop() + assert delivered == [ + ObservedMouseButton( + x=9.0, + y=11.0, + button="left", + pressed=True, + timestamp=1234.5, + ) + ] + + +def test_keyboard_translation_is_off_hook_and_uses_foreground_layout() -> None: + events = [] + user32 = FakeUser32(foreground_thread_id=8765, keyboard_layout=44) + observer = make_observer(events.append, user32=user32) + observer.start() + + shift = KBDLLHOOKSTRUCT(vkCode=VK_SHIFT, scanCode=0x2A) + letter = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(shift), + ) + == 73 + ) + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(letter), + ) + == 73 + ) + + assert wait_until(lambda: len(events) == 2) + user32.foreground_thread_id = 9876 + user32.keyboard_layout = 55 + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYUP, + ctypes.addressof(letter), + ) + == 73 + ) + assert wait_until(lambda: len(events) == 3) + assert events == [ + ObservedKey( + pressed=True, + key_name="shift_l", + key_vk="160", + canonical_key_name="shift", + canonical_key_vk="16", + timestamp=1234.5, + ), + ObservedKey( + pressed=True, + key_char="A", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", + timestamp=1234.5, + ), + ObservedKey( + pressed=False, + key_char="A", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", + timestamp=1234.5, + ), + ] + assert user32.layout_thread_ids == [8765, 9876] + observer.stop() + + +def test_keyboard_state_tracks_toggle_without_repeat_retoggling() -> None: + events = [] + user32 = FakeUser32() + observer = make_observer(events.append, user32=user32) + observer.start() + + caps_lock = KBDLLHOOKSTRUCT(vkCode=VK_CAPITAL, scanCode=0x3A) + letter = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(caps_lock), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(caps_lock), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(letter), + ) + + assert wait_until(lambda: len(events) == 3) + assert events[-1] == ObservedKey( + pressed=True, + key_char="A", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", + timestamp=1234.5, + ) + assert observer._keyboard_state[VK_CAPITAL] & 0x01 + observer.stop() + + +def test_blocked_keyboard_translation_does_not_block_hook_or_reorder_mouse() -> None: + translation_entered = threading.Event() + translation_release = threading.Event() + events = [] + user32 = FakeUser32( + translation_entered=translation_entered, + translation_release=translation_release, + ) + clock_value = [111.25] + observer = make_observer( + events.append, + user32=user32, + clock=lambda: clock_value[0], + ) + observer.start() + key = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + mouse = MSLLHOOKSTRUCT(pt=POINT(21, 22)) + + callback_result = [] + callback_returned = threading.Event() + + def invoke_key_hook() -> None: + callback_result.append( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(key), + ) + ) + callback_returned.set() + + caller = threading.Thread(target=invoke_key_hook) + caller.start() + assert callback_returned.wait(timeout=0.5) + caller.join(timeout=0.5) + assert not caller.is_alive() + assert callback_result == [73] + assert translation_entered.wait(timeout=1) + assert not translation_release.is_set() + + clock_value[0] = 222.5 + assert ( + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(mouse), + ) + == 73 + ) + translation_release.set() + + assert wait_until(lambda: len(events) == 2) + assert events == [ + ObservedKey( + pressed=True, + key_char="a", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", + timestamp=111.25, + ), + ObservedMouseButton( + x=21.0, + y=22.0, + button="left", + pressed=True, + timestamp=222.5, + ), + ] + observer.stop() + + +def test_translation_queue_overflow_fails_loud_and_unhooks() -> None: + translation_entered = threading.Event() + translation_release = threading.Event() + user32 = FakeUser32( + translation_entered=translation_entered, + translation_release=translation_release, + ) + observer = make_observer( + lambda _event: None, + user32=user32, + translation_queue_size=1, + ) + observer.start() + + first = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + second = KBDLLHOOKSTRUCT(vkCode=0x42, scanCode=0x30) + third = KBDLLHOOKSTRUCT(vkCode=0x43, scanCode=0x2E) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(first), + ) + assert translation_entered.wait(timeout=1) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(second), + ) + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(third), + ) + == 73 + ) + + translation_release.set() + with pytest.raises(InputObserverError, match="translation queue overflowed"): + observer.stop() + assert user32.unhooked == [101, 102] + assert observer._translation_thread is None + + +def test_translation_failure_propagates_and_cleans_up() -> None: + user32 = FakeUser32(translation_error=RuntimeError("layout failed")) + observer = make_observer(lambda _event: None, user32=user32) + observer.start() + key = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + + assert ( + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(key), + ) + == 73 + ) + + assert wait_until(lambda: observer._failure is not None) + with pytest.raises(InputObserverError, match="input translation failed"): + observer.stop() + assert user32.unhooked == [101, 102] + assert observer._translation_thread is None + + +def test_delivery_overflow_fails_loud_wakes_and_cleans_up() -> None: + entered = threading.Event() + release = threading.Event() + delivery_count = 0 + user32 = FakeUser32() + + def consume(_event) -> None: + nonlocal delivery_count + delivery_count += 1 + if delivery_count == 1: + entered.set() + release.wait(timeout=2) + + observer = make_observer( + consume, + user32=user32, + delivery_queue_size=1, + translation_queue_size=4096, + ) + observer.start() + payload = MSLLHOOKSTRUCT(pt=POINT(3, 4)) + + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + assert entered.wait(timeout=1) + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + result = observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + + assert result == 73 + assert wait_until(user32.posted.is_set) + release.set() + with pytest.raises(InputObserverError, match="queue overflowed"): + observer.stop() + assert user32.unhooked == [101, 102] + assert observer._delivery_thread is None + + +def test_teardown_attempts_every_hook_and_surfaces_release_failure() -> None: + user32 = FakeUser32(unhook_failures={101}) + observer = make_observer(lambda _event: None, user32=user32) + observer._keyboard_hook = 101 + observer._mouse_hook = 102 + + with pytest.raises(InputObserverError, match="keyboard hook"): + observer._teardown() + + assert user32.unhooked == [101, 102] + assert observer._keyboard_hook is None + assert observer._mouse_hook is None + + +def test_consumer_failure_is_surfaced_after_clean_teardown() -> None: + user32 = FakeUser32() + + def fail(_event) -> None: + raise ValueError("consumer failed") + + observer = make_observer(fail, user32=user32) + observer.start() + payload = MSLLHOOKSTRUCT(pt=POINT(5, 7)) + + observer._mouse_hook_callback( + HC_ACTION, + WM_LBUTTONDOWN, + ctypes.addressof(payload), + ) + + assert wait_until(lambda: observer._failure is not None) + assert user32.posted.is_set() + with pytest.raises(InputObserverError, match="input consumer failed"): + observer.stop() + assert user32.unhooked == [101, 102] + assert observer._delivery_thread is None From 95ed128d54bebf84e170bd0a4ac8974f39e7d185 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:48:07 -0700 Subject: [PATCH 11/26] fix: keep native observer source dependency-neutral --- openadapt_capture/input_observer/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py index 0b0d5ba..12d1e7e 100644 --- a/openadapt_capture/input_observer/windows.py +++ b/openadapt_capture/input_observer/windows.py @@ -276,7 +276,7 @@ def _signed_high_word(value: int) -> int: def _key_identity(vk_code: int, character: str | None) -> dict[str, str | None]: - """Normalize physical and canonical key identity like pynput's listener.""" + """Normalize physical and canonical key identity for the public event model.""" key_name = _PHYSICAL_KEY_NAMES.get(vk_code) canonical_name = key_name From cf3433a5551285a98e722ae342537dffa2ec49cb Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 13:49:19 -0700 Subject: [PATCH 12/26] fix: keep dead-key text unverifiable --- openadapt_capture/input_observer/windows.py | 7 ++++- tests/test_input_observer_windows.py | 29 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py index 12d1e7e..87a27e1 100644 --- a/openadapt_capture/input_observer/windows.py +++ b/openadapt_capture/input_observer/windows.py @@ -722,7 +722,12 @@ def _keyboard_character( ) if count == 0: return None - character_count = abs(count) + if count < 0: + # A dead key is composition state, not committed text. Preserve + # physical identity but do not claim that its spacing accent was + # typed before a later key resolves the composition. + return None + character_count = count if ctypes.sizeof(ctypes.c_wchar) == 2: raw = ctypes.string_at(buffer, character_count * 2) return raw.decode("utf-16-le") diff --git a/tests/test_input_observer_windows.py b/tests/test_input_observer_windows.py index 34d13ba..f82d2d4 100644 --- a/tests/test_input_observer_windows.py +++ b/tests/test_input_observer_windows.py @@ -61,6 +61,7 @@ def __init__( translation_entered: threading.Event | None = None, translation_release: threading.Event | None = None, translation_error: BaseException | None = None, + translation_result: int = 1, ) -> None: self.hook_results = list(hook_results or [101, 102]) self.last_error = last_error @@ -74,6 +75,7 @@ def __init__( self.translation_entered = translation_entered self.translation_release = translation_release self.translation_error = translation_error + self.translation_result = translation_result self.async_key_state: dict[int, int] = {} self.toggle_key_state: dict[int, int] = {} self.layout_thread_ids: list[int] = [] @@ -148,7 +150,7 @@ def ToUnicodeEx( else: character = character.lower() buffer[0] = character - return 1 + return self.translation_result def GetKeyboardLayout(self, thread_id): self.layout_thread_ids.append(thread_id) @@ -517,6 +519,31 @@ def test_keyboard_state_tracks_toggle_without_repeat_retoggling() -> None: observer.stop() +def test_dead_key_preserves_physical_identity_without_claiming_text() -> None: + events = [] + user32 = FakeUser32(translation_result=-1) + observer = make_observer(events.append, user32=user32) + observer.start() + dead_key = KBDLLHOOKSTRUCT(vkCode=0xDE, scanCode=0x28) + + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(dead_key), + ) + + assert wait_until(lambda: len(events) == 1) + assert events == [ + ObservedKey( + pressed=True, + key_vk="222", + canonical_key_vk="222", + timestamp=1234.5, + ) + ] + observer.stop() + + def test_blocked_keyboard_translation_does_not_block_hook_or_reorder_mouse() -> None: translation_entered = threading.Event() translation_release = threading.Event() From f583a84f785f22342888d4f3836900a7fd479762 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:00:35 -0700 Subject: [PATCH 13/26] fix: preserve Linux XKB text semantics --- openadapt_capture/input_observer/factory.py | 1 - openadapt_capture/input_observer/linux.py | 209 +++++++++++++---- tests/test_input_observer.py | 12 +- tests/test_input_observer_linux_xkb.py | 248 ++++++++++++++++++++ 4 files changed, 419 insertions(+), 51 deletions(-) create mode 100644 tests/test_input_observer_linux_xkb.py diff --git a/openadapt_capture/input_observer/factory.py b/openadapt_capture/input_observer/factory.py index f5c63d5..4ae4561 100644 --- a/openadapt_capture/input_observer/factory.py +++ b/openadapt_capture/input_observer/factory.py @@ -38,4 +38,3 @@ def create_input_observer( raise InputObserverUnavailableError( f"native input observation is not implemented for platform {selected!r}" ) - diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index c3394d6..7db41a7 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -6,6 +6,7 @@ import ctypes.util import os import sys +import time from typing import Any from .base import ( @@ -27,6 +28,9 @@ _XI_RAW_BUTTON_PRESS = 15 _XI_RAW_BUTTON_RELEASE = 16 _XI_RAW_MOTION = 17 +_XKB_USE_CORE_KBD = 0x0100 +_XKB_GROUP_SHIFT = 13 +_MAX_PENDING_BATCH = 256 _SPECIAL_KEY_NAMES = { "Alt_L": "alt", @@ -98,13 +102,6 @@ class _XIEventMask(ctypes.Structure): ] -class _XIButtonState(ctypes.Structure): - _fields_ = [ - ("mask_len", ctypes.c_int), - ("mask", ctypes.POINTER(ctypes.c_ubyte)), - ] - - class _XIValuatorState(ctypes.Structure): _fields_ = [ ("mask_len", ctypes.c_int), @@ -146,16 +143,31 @@ class _XIRawEvent(ctypes.Structure): ("deviceid", ctypes.c_int), ("sourceid", ctypes.c_int), ("detail", ctypes.c_int), - ("root", ctypes.c_ulong), - ("root_x", ctypes.c_double), - ("root_y", ctypes.c_double), ("flags", ctypes.c_int), - ("buttons", _XIButtonState), ("valuators", _XIValuatorState), ("raw_values", ctypes.POINTER(ctypes.c_double)), ] +class _XkbStateRec(ctypes.Structure): + _fields_ = [ + ("group", ctypes.c_ubyte), + ("locked_group", ctypes.c_ubyte), + ("base_group", ctypes.c_ushort), + ("latched_group", ctypes.c_ushort), + ("mods", ctypes.c_ubyte), + ("base_mods", ctypes.c_ubyte), + ("latched_mods", ctypes.c_ubyte), + ("locked_mods", ctypes.c_ubyte), + ("compat_state", ctypes.c_ubyte), + ("grab_mods", ctypes.c_ubyte), + ("compat_grab_mods", ctypes.c_ubyte), + ("lookup_mods", ctypes.c_ubyte), + ("compat_lookup_mods", ctypes.c_ubyte), + ("ptr_buttons", ctypes.c_ushort), + ] + + def normalize_xinput_button_event( *, detail: int, @@ -163,6 +175,7 @@ def normalize_xinput_button_event( x: float, y: float, injected: bool = False, + timestamp: float | None = None, ) -> ObservedInput | None: """Map X11 button numbers to a click or normalized wheel event.""" if detail in {4, 5, 6, 7}: @@ -181,6 +194,7 @@ def normalize_xinput_button_event( dx=dx, dy=dy, injected=injected, + timestamp=timestamp, ) button_by_detail = { 1: "left", @@ -200,6 +214,7 @@ def normalize_xinput_button_event( button=button, pressed=pressed, injected=injected, + timestamp=timestamp, ) @@ -209,11 +224,13 @@ def normalize_xinput_key_event( pressed: bool, keysym_name: str | None, injected: bool = False, + character: str | None = None, + derive_character: bool = True, + timestamp: float | None = None, ) -> ObservedKey: """Normalize an XInput2 keycode and XKB keysym name.""" key_name = _SPECIAL_KEY_NAMES.get(keysym_name or "") - character = None - if keysym_name: + if derive_character and keysym_name: if len(keysym_name) == 1 and keysym_name.isprintable(): character = keysym_name else: @@ -230,6 +247,7 @@ def normalize_xinput_key_event( canonical_key_char=character.lower() if character else None, canonical_key_vk=virtual_key, injected=injected, + timestamp=timestamp, ) @@ -241,11 +259,13 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._environ = environ if environ is not None else os.environ self._x11: Any = None self._xi: Any = None + self._xkbcommon: Any = None self._display: Any = None self._root = 0 self._xi_opcode = 0 self._event_mask_buffer: Any = None - self._shift_keys_down: set[int] = set() + self._composition_mode: str | None = None + self._unverifiable_keycodes: set[int] = set() @staticmethod def _set_mask(mask: ctypes.Array[ctypes.c_ubyte], event_type: int) -> None: @@ -271,13 +291,16 @@ def _setup(self) -> None: x11_name = ctypes.util.find_library("X11") xi_name = ctypes.util.find_library("Xi") - if not x11_name or not xi_name: + xkbcommon_name = ctypes.util.find_library("xkbcommon") + if not x11_name or not xi_name or not xkbcommon_name: raise InputObserverUnavailableError( - "XInput2 requires the system libX11 and libXi runtime libraries" + "Linux input observation requires the system libX11, libXi, " + "and permissively licensed libxkbcommon runtime libraries" ) try: self._x11 = ctypes.CDLL(x11_name) self._xi = ctypes.CDLL(xi_name) + self._xkbcommon = ctypes.CDLL(xkbcommon_name) except OSError as exc: raise InputObserverUnavailableError( f"could not load X11 input libraries: {exc}" @@ -377,13 +400,20 @@ def _configure_api(self) -> None: ctypes.POINTER(ctypes.c_uint), ] self._x11.XQueryPointer.restype = ctypes.c_int - self._x11.XkbKeycodeToKeysym.argtypes = [ + self._x11.XkbGetState.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint, + ctypes.POINTER(_XkbStateRec), + ] + self._x11.XkbGetState.restype = ctypes.c_int + self._x11.XkbLookupKeySym.argtypes = [ ctypes.c_void_p, ctypes.c_ubyte, - ctypes.c_int, - ctypes.c_int, + ctypes.c_uint, + ctypes.POINTER(ctypes.c_uint), + ctypes.POINTER(ctypes.c_ulong), ] - self._x11.XkbKeycodeToKeysym.restype = ctypes.c_ulong + self._x11.XkbLookupKeySym.restype = ctypes.c_int self._x11.XKeysymToString.argtypes = [ctypes.c_ulong] self._x11.XKeysymToString.restype = ctypes.c_char_p @@ -400,6 +430,12 @@ def _configure_api(self) -> None: ctypes.c_int, ] self._xi.XISelectEvents.restype = ctypes.c_int + self._xkbcommon.xkb_keysym_to_utf8.argtypes = [ + ctypes.c_uint32, + ctypes.POINTER(ctypes.c_char), + ctypes.c_size_t, + ] + self._xkbcommon.xkb_keysym_to_utf8.restype = ctypes.c_int def _query_pointer(self) -> tuple[float, float]: root_return = ctypes.c_ulong() @@ -426,35 +462,106 @@ def _query_pointer(self) -> tuple[float, float]: ) return float(root_x.value), float(root_y.value) - def _keysym_name(self, keycode: int, pressed: bool) -> str | None: - unshifted = self._x11.XkbKeycodeToKeysym( - self._display, keycode, 0, 0 - ) - unshifted_name_ptr = self._x11.XKeysymToString(unshifted) if unshifted else None - unshifted_name = ( - unshifted_name_ptr.decode(errors="replace") - if unshifted_name_ptr - else None + def _lookup_keysym(self, keycode: int) -> tuple[int, str | None]: + state = _XkbStateRec() + if self._x11.XkbGetState( + self._display, + _XKB_USE_CORE_KBD, + ctypes.byref(state), + ) != 0: + raise InputObserverError( + "XKB could not read the active keyboard group and modifier state" + ) + core_state = int(state.lookup_mods) | (int(state.group) << _XKB_GROUP_SHIFT) + consumed_modifiers = ctypes.c_uint() + keysym = ctypes.c_ulong() + if not self._x11.XkbLookupKeySym( + self._display, + keycode, + core_state, + ctypes.byref(consumed_modifiers), + ctypes.byref(keysym), + ): + raise InputObserverError( + f"XKB could not resolve keycode {keycode} in active group " + f"{int(state.group)}" + ) + name_ptr = self._x11.XKeysymToString(keysym.value) if keysym.value else None + name = name_ptr.decode(errors="replace") if name_ptr else None + return int(keysym.value), name + + def _keysym_character(self, keysym: int) -> str | None: + buffer = ctypes.create_string_buffer(64) + length = int( + self._xkbcommon.xkb_keysym_to_utf8( + keysym, + buffer, + len(buffer), + ) ) - is_shift = unshifted_name in {"Shift_L", "Shift_R"} - if is_shift: + if length <= 1: + return None + return buffer.value.decode("utf-8", errors="strict") + + def _resolved_character( + self, + *, + keycode: int, + keysym: int, + keysym_name: str | None, + pressed: bool, + ) -> str | None: + if not pressed and keycode in self._unverifiable_keycodes: + self._unverifiable_keycodes.discard(keycode) + return None + if keysym_name and ( + keysym_name.startswith("dead_") + or keysym_name in {"Multi_key", "Compose"} + ): if pressed: - self._shift_keys_down.add(keycode) - else: - self._shift_keys_down.discard(keycode) - return unshifted_name + self._composition_mode = ( + "dead" if keysym_name.startswith("dead_") else "multi" + ) + self._unverifiable_keycodes.add(keycode) + return None - level = 1 if self._shift_keys_down else 0 - keysym = self._x11.XkbKeycodeToKeysym(self._display, keycode, 0, level) - name_ptr = self._x11.XKeysymToString(keysym) if keysym else None - return name_ptr.decode(errors="replace") if name_ptr else unshifted_name + special_name = _SPECIAL_KEY_NAMES.get(keysym_name or "") + if self._composition_mode is not None and special_name in { + "enter", + "esc", + "space", + }: + if pressed: + self._composition_mode = None + self._unverifiable_keycodes.add(keycode) + return None + if self._composition_mode is not None and special_name is None: + if pressed: + self._unverifiable_keycodes.add(keycode) + if self._composition_mode == "dead": + # A dead-key sequence resolves (or is rejected) on the + # next non-modifier key. Without the target app's IME + # result, that key's text is deliberately unverifiable. + self._composition_mode = None + return None + if special_name is not None: + return None + return self._keysym_character(keysym) def _handle_raw_event(self, raw: _XIRawEvent) -> None: injected = bool(raw.send_event) + observed_at = time.time() if raw.evtype == _XI_RAW_MOTION: if self.observe_mouse and self.capture_mouse_moves: x, y = self._query_pointer() - self._emit(ObservedMouseMove(x=x, y=y, injected=injected)) + self._emit( + ObservedMouseMove( + x=x, + y=y, + injected=injected, + timestamp=observed_at, + ) + ) return if raw.evtype in {_XI_RAW_BUTTON_PRESS, _XI_RAW_BUTTON_RELEASE}: if self.observe_mouse: @@ -465,6 +572,7 @@ def _handle_raw_event(self, raw: _XIRawEvent) -> None: x=x, y=y, injected=injected, + timestamp=observed_at, ) if event is not None: self._emit(event) @@ -472,18 +580,33 @@ def _handle_raw_event(self, raw: _XIRawEvent) -> None: if raw.evtype in {_XI_RAW_KEY_PRESS, _XI_RAW_KEY_RELEASE}: if self.observe_keyboard: pressed = raw.evtype == _XI_RAW_KEY_PRESS + keysym, keysym_name = self._lookup_keysym(raw.detail) self._emit( normalize_xinput_key_event( keycode=raw.detail, pressed=pressed, - keysym_name=self._keysym_name(raw.detail, pressed), + keysym_name=keysym_name, injected=injected, + character=self._resolved_character( + keycode=raw.detail, + keysym=keysym, + keysym_name=keysym_name, + pressed=pressed, + ), + derive_character=False, + timestamp=observed_at, ) ) def _run_loop(self) -> None: while not self._stop_requested.is_set(): - while self._x11.XPending(self._display): + processed = 0 + while ( + not self._stop_requested.is_set() + and processed < _MAX_PENDING_BATCH + and self._x11.XPending(self._display) + ): + processed += 1 event = _XEvent() self._x11.XNextEvent(self._display, ctypes.byref(event)) cookie = event.xcookie @@ -515,6 +638,8 @@ def _teardown(self) -> None: self._x11.XCloseDisplay(self._display) self._display = None self._event_mask_buffer = None + self._composition_mode = None + self._unverifiable_keycodes.clear() __all__ = [ diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 8aa0f66..c746e42 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -532,14 +532,10 @@ def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( reason="ABI offsets below describe the supported 64-bit Linux runners", ) def test_xinput_raw_event_matches_64_bit_libxi_abi() -> None: - assert ctypes.sizeof(_XIRawEvent) == 144 - assert _XIRawEvent.root.offset == 64 - assert _XIRawEvent.root_x.offset == 72 - assert _XIRawEvent.root_y.offset == 80 - assert _XIRawEvent.flags.offset == 88 - assert _XIRawEvent.buttons.offset == 96 - assert _XIRawEvent.valuators.offset == 112 - assert _XIRawEvent.raw_values.offset == 136 + assert ctypes.sizeof(_XIRawEvent) == 96 + assert _XIRawEvent.flags.offset == 60 + assert _XIRawEvent.valuators.offset == 64 + assert _XIRawEvent.raw_values.offset == 88 def test_wayland_refuses_instead_of_silently_observing_only_xwayland( diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py new file mode 100644 index 0000000..a91878e --- /dev/null +++ b/tests/test_input_observer_linux_xkb.py @@ -0,0 +1,248 @@ +"""Linux XInput2 ABI, XKB text semantics, and shutdown contracts.""" + +from __future__ import annotations + +import ctypes + +from openadapt_capture.input_observer import ObservedKey +from openadapt_capture.input_observer import linux as linux_module +from openadapt_capture.input_observer.linux import ( + LinuxXInputObserver, + _XIRawEvent, + _XkbStateRec, +) + + +class FakeX11: + def __init__( + self, + *, + group: int = 0, + lookup_mods: int = 0, + keysym: int = 0x61, + keysym_name: str = "a", + ) -> None: + self.group = group + self.lookup_mods = lookup_mods + self.keysym = keysym + self.keysym_name = keysym_name + self.lookup_states: list[int] = [] + + def XkbGetState(self, _display, _device, state_pointer) -> int: + state = ctypes.cast( + state_pointer, + ctypes.POINTER(_XkbStateRec), + ).contents + state.group = self.group + state.lookup_mods = self.lookup_mods + return 0 + + def XkbLookupKeySym( + self, + _display, + _keycode, + state, + _consumed_pointer, + keysym_pointer, + ) -> int: + self.lookup_states.append(int(state)) + ctypes.cast( + keysym_pointer, + ctypes.POINTER(ctypes.c_ulong), + ).contents.value = self.keysym + return 1 + + def XKeysymToString(self, _keysym): + return self.keysym_name.encode() + + +class FakeXkbCommon: + def __init__(self, text_by_keysym: dict[int, str]) -> None: + self.text_by_keysym = text_by_keysym + + def xkb_keysym_to_utf8(self, keysym, buffer, size) -> int: + encoded = self.text_by_keysym.get(int(keysym), "").encode() + if not encoded: + return 0 + assert len(encoded) + 1 <= int(size) + ctypes.memmove(buffer, encoded + b"\0", len(encoded) + 1) + return len(encoded) + 1 + + +def make_observer() -> LinuxXInputObserver: + return LinuxXInputObserver( + lambda _event: None, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + environ={"DISPLAY": ":0", "XDG_SESSION_TYPE": "x11"}, + ) + + +def test_xinput_raw_event_matches_primary_header_lp64_abi() -> None: + assert ctypes.sizeof(_XIRawEvent) == 96 + assert _XIRawEvent.flags.offset == 60 + assert _XIRawEvent.valuators.offset == 64 + assert _XIRawEvent.raw_values.offset == 88 + + +def test_xkb_lookup_uses_active_group_caps_and_altgr_state() -> None: + observer = make_observer() + x11 = FakeX11( + group=2, + lookup_mods=0x82, + keysym=0x20AC, + keysym_name="EuroSign", + ) + observer._display = object() + observer._x11 = x11 + observer._xkbcommon = FakeXkbCommon({0x20AC: "€"}) + + keysym, name = observer._lookup_keysym(26) + + assert (keysym, name) == (0x20AC, "EuroSign") + assert x11.lookup_states == [0x82 | (2 << linux_module._XKB_GROUP_SHIFT)] + assert ( + observer._resolved_character( + keycode=26, + keysym=keysym, + keysym_name=name, + pressed=True, + ) + == "€" + ) + + +def test_dead_key_and_resolving_key_are_text_unverifiable() -> None: + observer = make_observer() + observer._xkbcommon = FakeXkbCommon({0x65: "e", 0x78: "x"}) + + assert ( + observer._resolved_character( + keycode=48, + keysym=0xFE51, + keysym_name="dead_acute", + pressed=True, + ) + is None + ) + assert ( + observer._resolved_character( + keycode=26, + keysym=0x65, + keysym_name="e", + pressed=True, + ) + is None + ) + assert ( + observer._resolved_character( + keycode=26, + keysym=0x65, + keysym_name="e", + pressed=False, + ) + is None + ) + assert ( + observer._resolved_character( + keycode=53, + keysym=0x78, + keysym_name="x", + pressed=True, + ) + == "x" + ) + + +def test_multi_key_sequence_never_guesses_application_compose_text() -> None: + observer = make_observer() + observer._xkbcommon = FakeXkbCommon({0x61: "a", 0x62: "b", 0x78: "x"}) + + for keycode, keysym, name in [ + (65, 0xFF20, "Multi_key"), + (38, 0x61, "a"), + (56, 0x62, "b"), + ]: + assert ( + observer._resolved_character( + keycode=keycode, + keysym=keysym, + keysym_name=name, + pressed=True, + ) + is None + ) + + observer._resolved_character( + keycode=36, + keysym=0xFF0D, + keysym_name="Return", + pressed=True, + ) + assert ( + observer._resolved_character( + keycode=53, + keysym=0x78, + keysym_name="x", + pressed=True, + ) + == "x" + ) + + +def test_raw_event_stamps_receipt_time_before_delivery( + monkeypatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._lookup_keysym = lambda _keycode: (0x61, "a") # type: ignore[method-assign] + observer._xkbcommon = FakeXkbCommon({0x61: "a"}) + monkeypatch.setattr(linux_module.time, "time", lambda: 321.5) + raw = _XIRawEvent( + evtype=linux_module._XI_RAW_KEY_PRESS, + detail=38, + ) + + observer._handle_raw_event(raw) + + assert events == [ + ObservedKey( + pressed=True, + key_char="a", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + timestamp=321.5, + ) + ] + + +def test_continuous_pending_events_cannot_prevent_stop() -> None: + observer = make_observer() + + class AlwaysPendingX11: + def __init__(self) -> None: + self.next_count = 0 + + def XPending(self, _display) -> int: + return 1 + + def XNextEvent(self, _display, event_pointer) -> None: + self.next_count += 1 + event = ctypes.cast( + event_pointer, + ctypes.POINTER(linux_module._XEvent), + ).contents + event.type = 0 + if self.next_count == 5: + observer._stop_requested.set() + + x11 = AlwaysPendingX11() + observer._x11 = x11 + observer._display = object() + + observer._run_loop() + + assert x11.next_count == 5 From 093f7c0255b8a5442902e60b3c536756bb4eb2a1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:02:53 -0700 Subject: [PATCH 14/26] fix: bound Linux compose state --- openadapt_capture/input_observer/linux.py | 127 +++++++++++++++++++++- tests/test_input_observer_linux_xkb.py | 70 +++++++++++- 2 files changed, 188 insertions(+), 9 deletions(-) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 7db41a7..edfb422 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -31,6 +31,9 @@ _XKB_USE_CORE_KBD = 0x0100 _XKB_GROUP_SHIFT = 13 _MAX_PENDING_BATCH = 256 +_XKB_COMPOSE_COMPOSING = 1 +_XKB_COMPOSE_COMPOSED = 2 +_XKB_COMPOSE_CANCELLED = 3 _SPECIAL_KEY_NAMES = { "Alt_L": "alt", @@ -265,7 +268,11 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._xi_opcode = 0 self._event_mask_buffer: Any = None self._composition_mode: str | None = None + self._composition_remaining = 0 self._unverifiable_keycodes: set[int] = set() + self._compose_context: Any = None + self._compose_table: Any = None + self._compose_state: Any = None @staticmethod def _set_mask(mask: ctypes.Array[ctypes.c_ubyte], event_type: int) -> None: @@ -306,6 +313,7 @@ def _setup(self) -> None: f"could not load X11 input libraries: {exc}" ) from exc self._configure_api() + self._initialize_compose() self._display = self._x11.XOpenDisplay(display_name.encode()) if not self._display: raise InputObserverPermissionError( @@ -436,6 +444,92 @@ def _configure_api(self) -> None: ctypes.c_size_t, ] self._xkbcommon.xkb_keysym_to_utf8.restype = ctypes.c_int + self._xkbcommon.xkb_context_new.argtypes = [ctypes.c_int] + self._xkbcommon.xkb_context_new.restype = ctypes.c_void_p + self._xkbcommon.xkb_context_unref.argtypes = [ctypes.c_void_p] + self._xkbcommon.xkb_compose_table_new_from_locale.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_int, + ] + self._xkbcommon.xkb_compose_table_new_from_locale.restype = ctypes.c_void_p + self._xkbcommon.xkb_compose_table_unref.argtypes = [ctypes.c_void_p] + self._xkbcommon.xkb_compose_state_new.argtypes = [ + ctypes.c_void_p, + ctypes.c_int, + ] + self._xkbcommon.xkb_compose_state_new.restype = ctypes.c_void_p + self._xkbcommon.xkb_compose_state_unref.argtypes = [ctypes.c_void_p] + self._xkbcommon.xkb_compose_state_feed.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint32, + ] + self._xkbcommon.xkb_compose_state_feed.restype = ctypes.c_int + self._xkbcommon.xkb_compose_state_get_status.argtypes = [ctypes.c_void_p] + self._xkbcommon.xkb_compose_state_get_status.restype = ctypes.c_int + self._xkbcommon.xkb_compose_state_get_utf8.argtypes = [ + ctypes.c_void_p, + ctypes.POINTER(ctypes.c_char), + ctypes.c_size_t, + ] + self._xkbcommon.xkb_compose_state_get_utf8.restype = ctypes.c_int + self._xkbcommon.xkb_compose_state_reset.argtypes = [ctypes.c_void_p] + + def _initialize_compose(self) -> None: + locale_name = ( + self._environ.get("LC_ALL") + or self._environ.get("LC_CTYPE") + or self._environ.get("LANG") + or "C" + ) + self._compose_context = self._xkbcommon.xkb_context_new(0) + if not self._compose_context: + raise InputObserverUnavailableError( + "libxkbcommon could not create a compose context" + ) + self._compose_table = self._xkbcommon.xkb_compose_table_new_from_locale( + self._compose_context, + locale_name.encode(), + 0, + ) + if not self._compose_table: + raise InputObserverUnavailableError( + f"libxkbcommon could not load compose rules for locale {locale_name!r}" + ) + self._compose_state = self._xkbcommon.xkb_compose_state_new( + self._compose_table, + 0, + ) + if not self._compose_state: + raise InputObserverUnavailableError( + "libxkbcommon could not create compose state" + ) + + def _compose_character(self, keysym: int) -> tuple[bool, str | None]: + """Return whether compose consumed this press and any committed text.""" + self._xkbcommon.xkb_compose_state_feed(self._compose_state, keysym) + status = int( + self._xkbcommon.xkb_compose_state_get_status(self._compose_state) + ) + if status == _XKB_COMPOSE_COMPOSING: + return True, None + if status == _XKB_COMPOSE_COMPOSED: + buffer = ctypes.create_string_buffer(64) + length = int( + self._xkbcommon.xkb_compose_state_get_utf8( + self._compose_state, + buffer, + len(buffer), + ) + ) + self._xkbcommon.xkb_compose_state_reset(self._compose_state) + if length <= 0: + return True, None + return True, buffer.value.decode("utf-8", errors="strict") + if status == _XKB_COMPOSE_CANCELLED: + self._xkbcommon.xkb_compose_state_reset(self._compose_state) + return True, None + return False, None def _query_pointer(self) -> tuple[float, float]: root_return = ctypes.c_ulong() @@ -514,6 +608,14 @@ def _resolved_character( if not pressed and keycode in self._unverifiable_keycodes: self._unverifiable_keycodes.discard(keycode) return None + if pressed and self._compose_state is not None: + consumed, composed_text = self._compose_character(keysym) + if consumed: + self._unverifiable_keycodes.add(keycode) + return composed_text + + # Pure-unit and defensive fallback when no compose state exists. + # Production setup fails unless locale-aware compose state was created. if keysym_name and ( keysym_name.startswith("dead_") or keysym_name in {"Multi_key", "Compose"} @@ -522,6 +624,9 @@ def _resolved_character( self._composition_mode = ( "dead" if keysym_name.startswith("dead_") else "multi" ) + self._composition_remaining = ( + 1 if self._composition_mode == "dead" else 2 + ) self._unverifiable_keycodes.add(keycode) return None @@ -533,16 +638,19 @@ def _resolved_character( }: if pressed: self._composition_mode = None + self._composition_remaining = 0 self._unverifiable_keycodes.add(keycode) return None if self._composition_mode is not None and special_name is None: if pressed: self._unverifiable_keycodes.add(keycode) - if self._composition_mode == "dead": - # A dead-key sequence resolves (or is rejected) on the - # next non-modifier key. Without the target app's IME - # result, that key's text is deliberately unverifiable. + self._composition_remaining -= 1 + if self._composition_remaining <= 0: + # This fallback is only reachable without initialized + # production compose state. It never guesses sequence + # text and bounds how long later unrelated text is hidden. self._composition_mode = None + self._composition_remaining = 0 return None if special_name is not None: return None @@ -636,9 +744,20 @@ def _run_loop(self) -> None: def _teardown(self) -> None: if self._display and self._x11 is not None: self._x11.XCloseDisplay(self._display) + if self._xkbcommon is not None: + if self._compose_state: + self._xkbcommon.xkb_compose_state_unref(self._compose_state) + if self._compose_table: + self._xkbcommon.xkb_compose_table_unref(self._compose_table) + if self._compose_context: + self._xkbcommon.xkb_context_unref(self._compose_context) + self._compose_state = None + self._compose_table = None + self._compose_context = None self._display = None self._event_mask_buffer = None self._composition_mode = None + self._composition_remaining = 0 self._unverifiable_keycodes.clear() diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index a91878e..2b90fa7 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -69,6 +69,36 @@ def xkb_keysym_to_utf8(self, keysym, buffer, size) -> int: return len(encoded) + 1 +class FakeComposeXkbCommon(FakeXkbCommon): + def __init__(self) -> None: + super().__init__({0x65: "e", 0x78: "x"}) + self.statuses = iter( + [ + linux_module._XKB_COMPOSE_COMPOSING, + linux_module._XKB_COMPOSE_COMPOSED, + 0, + ] + ) + self.status = 0 + self.reset_count = 0 + + def xkb_compose_state_feed(self, _state, _keysym) -> int: + self.status = next(self.statuses) + return 1 + + def xkb_compose_state_get_status(self, _state) -> int: + return self.status + + def xkb_compose_state_get_utf8(self, _state, buffer, size) -> int: + encoded = "é".encode() + assert len(encoded) + 1 <= int(size) + ctypes.memmove(buffer, encoded + b"\0", len(encoded) + 1) + return len(encoded) + + def xkb_compose_state_reset(self, _state) -> None: + self.reset_count += 1 + + def make_observer() -> LinuxXInputObserver: return LinuxXInputObserver( lambda _event: None, @@ -174,12 +204,42 @@ def test_multi_key_sequence_never_guesses_application_compose_text() -> None: is None ) - observer._resolved_character( - keycode=36, - keysym=0xFF0D, - keysym_name="Return", - pressed=True, + assert ( + observer._resolved_character( + keycode=53, + keysym=0x78, + keysym_name="x", + pressed=True, + ) + == "x" + ) + + +def test_locale_compose_state_commits_exact_text_then_resumes_normal_text() -> None: + observer = make_observer() + xkbcommon = FakeComposeXkbCommon() + observer._xkbcommon = xkbcommon + observer._compose_state = object() + + assert ( + observer._resolved_character( + keycode=48, + keysym=0xFE51, + keysym_name="dead_acute", + pressed=True, + ) + is None + ) + assert ( + observer._resolved_character( + keycode=26, + keysym=0x65, + keysym_name="e", + pressed=True, + ) + == "é" ) + assert xkbcommon.reset_count == 1 assert ( observer._resolved_character( keycode=53, From e13eb8eef85d2343e52276331dc5d5349de0e1a6 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:11:21 -0700 Subject: [PATCH 15/26] test: scope platform-specific compatibility guards --- tests/test_input_observer.py | 10 +++++++--- tests/test_input_observer_linux_xkb.py | 6 ++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index c746e42..5097f57 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -181,7 +181,11 @@ def fail_wake() -> None: with pytest.raises(InputObserverError, match="primary observer failure") as caught: observer.check_health() assert caught.value is primary - assert any("wake failed" in note for note in getattr(primary, "__notes__", [])) + notes = getattr(primary, "__notes__", []) + if hasattr(primary, "add_note"): + assert any("wake failed" in note for note in notes) + else: + assert notes == [] def test_idempotent_start_surfaces_live_delivery_consumer_failure() -> None: @@ -528,8 +532,8 @@ def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( @pytest.mark.skipif( - ctypes.sizeof(ctypes.c_void_p) != 8, - reason="ABI offsets below describe the supported 64-bit Linux runners", + ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_ulong) != 8, + reason="ABI offsets below describe Linux's 64-bit LP64 data model", ) def test_xinput_raw_event_matches_64_bit_libxi_abi() -> None: assert ctypes.sizeof(_XIRawEvent) == 96 diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 2b90fa7..8411854 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -4,6 +4,8 @@ import ctypes +import pytest + from openadapt_capture.input_observer import ObservedKey from openadapt_capture.input_observer import linux as linux_module from openadapt_capture.input_observer.linux import ( @@ -109,6 +111,10 @@ def make_observer() -> LinuxXInputObserver: ) +@pytest.mark.skipif( + ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_ulong) != 8, + reason="ABI offsets below describe Linux's 64-bit LP64 data model", +) def test_xinput_raw_event_matches_primary_header_lp64_abi() -> None: assert ctypes.sizeof(_XIRawEvent) == 96 assert _XIRawEvent.flags.offset == 60 From dd6a78186e8248fcd6705366c6ac69b439e70743 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:16:39 -0700 Subject: [PATCH 16/26] fix: initialize Xlib and preserve mouse button API --- openadapt_capture/events.py | 28 +++- openadapt_capture/input.py | 2 + openadapt_capture/input_observer/factory.py | 4 + openadapt_capture/platform/linux.py | 5 + openadapt_capture/utils.py | 3 + openadapt_capture/x11_threads.py | 56 ++++++++ tests/test_events.py | 15 ++ tests/test_input_observer.py | 45 +++++- tests/test_x11_threads.py | 147 ++++++++++++++++++++ 9 files changed, 297 insertions(+), 8 deletions(-) create mode 100644 openadapt_capture/x11_threads.py create mode 100644 tests/test_x11_threads.py diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index ac5ae5b..3c59c9e 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -8,9 +8,9 @@ from __future__ import annotations from enum import Enum -from typing import Literal +from typing import Annotated, Literal -from pydantic import BaseModel, Field +from pydantic import AfterValidator, BaseModel, Field class EventType(str, Enum): @@ -49,6 +49,20 @@ class MouseButton(str, Enum): MIDDLE = "middle" +def _normalize_mouse_button(value: MouseButton | str) -> MouseButton | str: + """Preserve the established enum while allowing native auxiliary buttons.""" + try: + return MouseButton(value) + except ValueError: + return value + + +MouseButtonValue = Annotated[ + MouseButton | str, + AfterValidator(_normalize_mouse_button), +] + + class BaseEvent(BaseModel): """Base class for all events. @@ -87,7 +101,7 @@ class MouseDownEvent(BaseEvent): type: Literal[EventType.MOUSE_DOWN] = EventType.MOUSE_DOWN x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: str = Field(description="Native mouse button name") + button: MouseButtonValue = Field(description="Mouse button name") class MouseUpEvent(BaseEvent): @@ -99,7 +113,7 @@ class MouseUpEvent(BaseEvent): type: Literal[EventType.MOUSE_UP] = EventType.MOUSE_UP x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: str = Field(description="Native mouse button name") + button: MouseButtonValue = Field(description="Mouse button name") class MouseScrollEvent(BaseEvent): @@ -202,7 +216,7 @@ class MouseClickEvent(BaseEvent): type: Literal[EventType.MOUSE_SINGLECLICK] = EventType.MOUSE_SINGLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: str = Field(description="Native mouse button name") + button: MouseButtonValue = Field(description="Mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -218,7 +232,7 @@ class MouseDoubleClickEvent(BaseEvent): type: Literal[EventType.MOUSE_DOUBLECLICK] = EventType.MOUSE_DOUBLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: str = Field(description="Native mouse button name") + button: MouseButtonValue = Field(description="Mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -236,7 +250,7 @@ class MouseDragEvent(BaseEvent): y: float = Field(description="Starting Y position in pixels") dx: float = Field(description="Horizontal displacement (end_x - start_x)") dy: float = Field(description="Vertical displacement (end_y - start_y)") - button: str = Field(description="Native mouse button name") + button: MouseButtonValue = Field(description="Mouse button name") children: list[MouseDownEvent | MouseMoveEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) diff --git a/openadapt_capture/input.py b/openadapt_capture/input.py index a70b2fa..0fac068 100644 --- a/openadapt_capture/input.py +++ b/openadapt_capture/input.py @@ -23,6 +23,7 @@ ObservedMouseScroll, create_input_observer, ) +from openadapt_capture.x11_threads import ensure_xlib_thread_support def _get_timestamp() -> float: @@ -418,6 +419,7 @@ def start(self) -> None: if self._running: return + ensure_xlib_thread_support() self._stop_event.clear() self._thread = threading.Thread(target=self._capture_loop, daemon=True) self._thread.start() diff --git a/openadapt_capture/input_observer/factory.py b/openadapt_capture/input_observer/factory.py index 4ae4561..de0c205 100644 --- a/openadapt_capture/input_observer/factory.py +++ b/openadapt_capture/input_observer/factory.py @@ -32,6 +32,10 @@ def create_input_observer( return WindowsInputObserver(**kwargs) if selected.startswith("linux"): + if sys.platform.startswith("linux"): + from openadapt_capture.x11_threads import ensure_xlib_thread_support + + ensure_xlib_thread_support() from .linux import LinuxXInputObserver return LinuxXInputObserver(**kwargs) diff --git a/openadapt_capture/platform/linux.py b/openadapt_capture/platform/linux.py index 78f5e09..f0fd2fb 100644 --- a/openadapt_capture/platform/linux.py +++ b/openadapt_capture/platform/linux.py @@ -10,6 +10,8 @@ import sys +from openadapt_capture.x11_threads import ensure_xlib_thread_support + if not sys.platform.startswith("linux"): raise ImportError("This module is only available on Linux") @@ -43,6 +45,7 @@ def get_screen_dimensions() -> tuple[int, int]: Returns: Tuple of (width, height) in physical pixels. """ + ensure_xlib_thread_support() try: from PIL import ImageGrab screenshot = ImageGrab.grab() @@ -109,6 +112,8 @@ def get_display_pixel_ratio() -> float: """ import os + ensure_xlib_thread_support() + # Check GNOME scaling factor try: import subprocess diff --git a/openadapt_capture/utils.py b/openadapt_capture/utils.py index b523c9a..05fbc26 100644 --- a/openadapt_capture/utils.py +++ b/openadapt_capture/utils.py @@ -15,6 +15,8 @@ from loguru import logger from PIL import Image +from openadapt_capture.x11_threads import ensure_xlib_thread_support + if sys.platform == "win32": import mss.windows @@ -39,6 +41,7 @@ def get_process_local_sct() -> mss.mss: """Retrieve or create the `mss` instance for the current thread.""" if not hasattr(_process_local, "sct"): + ensure_xlib_thread_support() _process_local.sct = mss.mss() return _process_local.sct diff --git a/openadapt_capture/x11_threads.py b/openadapt_capture/x11_threads.py new file mode 100644 index 0000000..22312d6 --- /dev/null +++ b/openadapt_capture/x11_threads.py @@ -0,0 +1,56 @@ +"""Process-wide Xlib threading initialization for Linux capture paths.""" + +from __future__ import annotations + +import ctypes +import ctypes.util +import sys +import threading + + +class X11ThreadInitializationError(RuntimeError): + """Raised when Xlib cannot be made safe for concurrent capture threads.""" + + +_initialization_lock = threading.Lock() +_initialized = False + + +def ensure_xlib_thread_support() -> None: + """Call ``XInitThreads`` before this process uses any Xlib-backed capture. + + Both MSS and the native Linux input observer use Xlib from different + threads. Xlib requires ``XInitThreads`` before the first other Xlib call; + continuing after a missing or rejected initialization could corrupt + concurrent display access, so Linux callers fail loudly. + """ + + if not sys.platform.startswith("linux"): + return + + global _initialized + with _initialization_lock: + if _initialized: + return + + x11_name = ctypes.util.find_library("X11") + if not x11_name: + raise X11ThreadInitializationError( + "Linux capture requires libX11 with XInitThreads support" + ) + try: + x11 = ctypes.CDLL(x11_name) + xinit_threads = x11.XInitThreads + xinit_threads.argtypes = [] + xinit_threads.restype = ctypes.c_int + initialized = int(xinit_threads()) + except (AttributeError, OSError) as exc: + raise X11ThreadInitializationError( + f"could not initialize Xlib threading through {x11_name!r}: {exc}" + ) from exc + + if initialized == 0: + raise X11ThreadInitializationError( + "XInitThreads rejected process-wide Xlib thread initialization" + ) + _initialized = True diff --git a/tests/test_events.py b/tests/test_events.py index b6d9fdd..1e4e167 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -49,6 +49,7 @@ def test_mouse_down_event(self): ) assert event.type == EventType.MOUSE_DOWN assert event.button == MouseButton.LEFT + assert isinstance(event.button, MouseButton) def test_mouse_up_event(self): """Test MouseUpEvent creation.""" @@ -60,6 +61,20 @@ def test_mouse_up_event(self): ) assert event.type == EventType.MOUSE_UP assert event.button == MouseButton.RIGHT + assert isinstance(event.button, MouseButton) + + def test_auxiliary_mouse_button_remains_a_native_string(self): + """Unknown native buttons survive without weakening standard enums.""" + event = MouseDownEvent( + timestamp=1234567890.123, + x=100.0, + y=200.0, + button="x1", + ) + assert event.button == "x1" + assert isinstance(event.button, str) + assert not isinstance(event.button, MouseButton) + assert event.model_dump_json().find('"button":"x1"') != -1 def test_mouse_scroll_event(self): """Test MouseScrollEvent creation.""" diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 5097f57..572762a 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -15,7 +15,12 @@ from openadapt_capture import input as input_module from openadapt_capture import recorder as recorder_module from openadapt_capture.capture import _convert_action_event -from openadapt_capture.events import MouseClickEvent, MouseDownEvent, MouseUpEvent +from openadapt_capture.events import ( + MouseButton, + MouseClickEvent, + MouseDownEvent, + MouseUpEvent, +) from openadapt_capture.input_observer import ( InputObserverError, InputObserverUnavailableError, @@ -525,10 +530,48 @@ def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( assert isinstance(up, MouseUpEvent) assert down.button == button assert up.button == button + assert not isinstance(down.button, MouseButton) + assert not isinstance(up.button, MouseButton) processed = merge_consecutive_mouse_click_events([down, up]) assert len(processed) == 1 assert isinstance(processed[0], MouseClickEvent) assert processed[0].button == button + assert not isinstance(processed[0].button, MouseButton) + + +@pytest.mark.parametrize("button", list(MouseButton)) +def test_standard_mouse_button_keeps_public_enum_through_processing( + button: MouseButton, +) -> None: + down = _convert_action_event( + SimpleNamespace( + name="click", + timestamp=1.0, + mouse_x=12.5, + mouse_y=34.5, + mouse_button_name=button.value, + mouse_pressed=True, + ) + ) + up = _convert_action_event( + SimpleNamespace( + name="click", + timestamp=1.1, + mouse_x=12.5, + mouse_y=34.5, + mouse_button_name=button.value, + mouse_pressed=False, + ) + ) + + assert isinstance(down, MouseDownEvent) + assert isinstance(up, MouseUpEvent) + assert down.button is button + assert up.button is button + processed = merge_consecutive_mouse_click_events([down, up]) + assert len(processed) == 1 + assert isinstance(processed[0], MouseClickEvent) + assert processed[0].button is button @pytest.mark.skipif( diff --git a/tests/test_x11_threads.py b/tests/test_x11_threads.py new file mode 100644 index 0000000..bbb8e3f --- /dev/null +++ b/tests/test_x11_threads.py @@ -0,0 +1,147 @@ +"""Xlib must be initialized before concurrent Linux capture starts.""" + +from __future__ import annotations + +import ctypes +import threading + +import pytest + +from openadapt_capture import input as input_module +from openadapt_capture import utils, x11_threads +from openadapt_capture.input_observer import factory +from openadapt_capture.x11_threads import ( + X11ThreadInitializationError, + ensure_xlib_thread_support, +) + + +class _FakeXInitThreads: + def __init__(self, result: int = 1) -> None: + self.result = result + self.calls = 0 + self.argtypes = None + self.restype = None + + def __call__(self) -> int: + self.calls += 1 + return self.result + + +class _FakeX11: + def __init__(self, result: int = 1) -> None: + self.XInitThreads = _FakeXInitThreads(result) + + +def _linux_x11( + monkeypatch: pytest.MonkeyPatch, + *, + result: int = 1, +) -> _FakeX11: + fake = _FakeX11(result) + monkeypatch.setattr(x11_threads.sys, "platform", "linux") + monkeypatch.setattr(x11_threads, "_initialized", False) + monkeypatch.setattr(x11_threads.ctypes.util, "find_library", lambda _name: "libX11") + monkeypatch.setattr(x11_threads.ctypes, "CDLL", lambda _name: fake) + return fake + + +def test_xinit_threads_is_configured_once_per_process( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fake = _linux_x11(monkeypatch) + + ensure_xlib_thread_support() + ensure_xlib_thread_support() + + assert fake.XInitThreads.calls == 1 + assert fake.XInitThreads.argtypes == [] + assert fake.XInitThreads.restype is ctypes.c_int + + +def test_missing_x11_fails_loudly(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(x11_threads.sys, "platform", "linux") + monkeypatch.setattr(x11_threads, "_initialized", False) + monkeypatch.setattr(x11_threads.ctypes.util, "find_library", lambda _name: None) + + with pytest.raises(X11ThreadInitializationError, match="requires libX11"): + ensure_xlib_thread_support() + + +def test_rejected_xinit_threads_fails_loudly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _linux_x11(monkeypatch, result=0) + + with pytest.raises(X11ThreadInitializationError, match="rejected"): + ensure_xlib_thread_support() + + +def test_non_linux_initialization_is_a_side_effect_free_noop( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(x11_threads.sys, "platform", "darwin") + monkeypatch.setattr( + x11_threads.ctypes.util, + "find_library", + lambda _name: pytest.fail("non-Linux code loaded X11"), + ) + + ensure_xlib_thread_support() + + +def test_mss_is_constructed_only_after_xlib_initialization( + monkeypatch: pytest.MonkeyPatch, +) -> None: + order: list[str] = [] + fake_sct = object() + monkeypatch.setattr(utils, "_process_local", threading.local()) + monkeypatch.setattr( + utils, + "ensure_xlib_thread_support", + lambda: order.append("XInitThreads"), + ) + monkeypatch.setattr( + utils.mss, + "mss", + lambda: order.append("mss") or fake_sct, + ) + + assert utils.get_process_local_sct() is fake_sct + assert order == ["XInitThreads", "mss"] + + +def test_screen_capturer_surfaces_xlib_initialization_failure_before_thread( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def fail() -> None: + raise X11ThreadInitializationError("XInitThreads failed") + + monkeypatch.setattr(input_module, "ensure_xlib_thread_support", fail) + capturer = input_module.ScreenCapturer(lambda _image, _timestamp: None) + + with pytest.raises(X11ThreadInitializationError, match="failed"): + capturer.start() + + assert capturer._thread is None + assert not capturer._running + + +def test_linux_input_factory_initializes_xlib_before_observer_construction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[str] = [] + monkeypatch.setattr(factory.sys, "platform", "linux") + monkeypatch.setattr( + x11_threads, + "ensure_xlib_thread_support", + lambda: calls.append("XInitThreads"), + ) + + observer = factory.create_input_observer( + lambda _event: None, + platform_name="linux", + ) + + assert calls == ["XInitThreads"] + assert observer is not None From b44ffecb7ef6a47cb68ba9033539dab46c96c16a Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:17:44 -0700 Subject: [PATCH 17/26] fix: bind Windows text to event-time layout --- openadapt_capture/input_observer/windows.py | 58 ++++++-- tests/test_input_observer_windows.py | 147 +++++++++++++++++--- 2 files changed, 173 insertions(+), 32 deletions(-) diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py index 87a27e1..58ad15f 100644 --- a/openadapt_capture/input_observer/windows.py +++ b/openadapt_capture/input_observer/windows.py @@ -149,6 +149,7 @@ class _RawKeyboardTransition: vk_code: int scan_code: int flags: int + keyboard_layout: int timestamp: float @@ -413,6 +414,8 @@ def __init__( self._translation_thread: threading.Thread | None = None self._keyboard_state = (ctypes.c_ubyte * 256)() self._keys_down: set[int] = set() + self._dead_key_pending = False + self._unverifiable_text_keys: set[int] = set() def _load_apis(self) -> None: if self._user32 is not None and self._kernel32 is not None: @@ -647,6 +650,8 @@ def _initialize_keyboard_state(self) -> None: assert user32 is not None self._keyboard_state = (ctypes.c_ubyte * 256)() self._keys_down.clear() + self._dead_key_pending = False + self._unverifiable_text_keys.clear() for vk_code in { VK_LSHIFT, VK_RSHIFT, @@ -682,7 +687,7 @@ def _apply_keyboard_transition(self, transition: _RawKeyboardTransition) -> int: self._sync_generic_modifier_state() return physical_vk - def _foreground_keyboard_layout(self) -> object: + def _foreground_keyboard_layout(self) -> int: user32 = self._user32 assert user32 is not None foreground = user32.GetForegroundWindow() @@ -700,12 +705,12 @@ def _foreground_keyboard_layout(self) -> object: raise InputObserverUnavailableError( "Windows could not resolve the foreground keyboard layout" ) - return layout + return _handle_value(layout) def _keyboard_character( self, transition: _RawKeyboardTransition, - ) -> str | None: + ) -> tuple[str | None, bool]: user32 = self._user32 assert user32 is not None buffer = ctypes.create_unicode_buffer(8) @@ -717,21 +722,52 @@ def _keyboard_character( buffer, len(buffer), TO_UNICODE_NO_STATE_CHANGE, - self._foreground_keyboard_layout(), + transition.keyboard_layout, ) ) if count == 0: - return None + return None, False if count < 0: # A dead key is composition state, not committed text. Preserve # physical identity but do not claim that its spacing accent was # typed before a later key resolves the composition. - return None + return None, True character_count = count if ctypes.sizeof(ctypes.c_wchar) == 2: raw = ctypes.string_at(buffer, character_count * 2) - return raw.decode("utf-16-le") - return "".join(buffer[:character_count]) + return raw.decode("utf-16-le"), False + return "".join(buffer[:character_count]), False + + def _translated_character( + self, + transition: _RawKeyboardTransition, + physical_vk: int, + ) -> str | None: + """Return committed text without guessing across dead-key composition. + + ``TO_UNICODE_NO_STATE_CHANGE`` prevents the observer from mutating the + application's keyboard state, but it also means Windows cannot give us + a trustworthy committed character for the key that resolves a dead-key + sequence. Preserve both physical events and suppress their text until + the resolving key is released; the following normal key then recovers. + """ + + if not transition.pressed and physical_vk in self._unverifiable_text_keys: + self._unverifiable_text_keys.discard(physical_vk) + return None + + character, is_dead_key = self._keyboard_character(transition) + if is_dead_key: + self._dead_key_pending = True + self._unverifiable_text_keys.add(physical_vk) + return None + + if transition.pressed and self._dead_key_pending: + self._dead_key_pending = False + self._unverifiable_text_keys.add(physical_vk) + return None + + return character def _translate_keyboard( self, @@ -739,7 +775,9 @@ def _translate_keyboard( ) -> ObservedKey: physical_vk = self._apply_keyboard_transition(transition) character = ( - None if physical_vk in _PHYSICAL_KEY_NAMES else self._keyboard_character(transition) + None + if physical_vk in _PHYSICAL_KEY_NAMES + else self._translated_character(transition, physical_vk) ) return ObservedKey( pressed=transition.pressed, @@ -863,12 +901,14 @@ def _keyboard_hook_callback( ).contents injected = bool(payload.flags & (LLKHF_INJECTED | LLKHF_LOWER_IL_INJECTED)) if not injected: + keyboard_layout = self._foreground_keyboard_layout() self._enqueue_input( _RawKeyboardTransition( pressed=wparam in {WM_KEYDOWN, WM_SYSKEYDOWN}, vk_code=int(payload.vkCode), scan_code=int(payload.scanCode), flags=int(payload.flags), + keyboard_layout=keyboard_layout, timestamp=timestamp, ) ) diff --git a/tests/test_input_observer_windows.py b/tests/test_input_observer_windows.py index f82d2d4..99523ed 100644 --- a/tests/test_input_observer_windows.py +++ b/tests/test_input_observer_windows.py @@ -62,6 +62,7 @@ def __init__( translation_release: threading.Event | None = None, translation_error: BaseException | None = None, translation_result: int = 1, + translation_script: list[tuple[int, str | None]] | None = None, ) -> None: self.hook_results = list(hook_results or [101, 102]) self.last_error = last_error @@ -76,9 +77,11 @@ def __init__( self.translation_release = translation_release self.translation_error = translation_error self.translation_result = translation_result + self.translation_script = list(translation_script or []) self.async_key_state: dict[int, int] = {} self.toggle_key_state: dict[int, int] = {} self.layout_thread_ids: list[int] = [] + self.translation_layouts: list[int] = [] def PeekMessageW(self, *_args) -> int: return 0 @@ -136,12 +139,18 @@ def ToUnicodeEx( _layout, ) -> int: assert flags == 4 + self.translation_layouts.append(int(_layout)) if self.translation_entered is not None: self.translation_entered.set() if self.translation_release is not None: self.translation_release.wait(timeout=2) if self.translation_error is not None: raise self.translation_error + if self.translation_script: + result, scripted_character = self.translation_script.pop(0) + if scripted_character is not None: + buffer[0] = scripted_character + return result character = chr(vk_code) shift = bool(keyboard_state[VK_SHIFT] & 0x80) caps_lock = bool(keyboard_state[VK_CAPITAL] & 0x01) @@ -415,39 +424,50 @@ def invoke_hook() -> None: ] -def test_keyboard_translation_is_off_hook_and_uses_foreground_layout() -> None: +def test_keyboard_translation_uses_layout_snapshotted_at_hook_receipt() -> None: + translation_entered = threading.Event() + translation_release = threading.Event() events = [] - user32 = FakeUser32(foreground_thread_id=8765, keyboard_layout=44) + user32 = FakeUser32( + foreground_thread_id=8765, + keyboard_layout=44, + translation_entered=translation_entered, + translation_release=translation_release, + ) observer = make_observer(events.append, user32=user32) observer.start() - shift = KBDLLHOOKSTRUCT(vkCode=VK_SHIFT, scanCode=0x2A) - letter = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + first = KBDLLHOOKSTRUCT(vkCode=0x41, scanCode=0x1E) + second = KBDLLHOOKSTRUCT(vkCode=0x42, scanCode=0x30) assert ( observer._keyboard_hook_callback( HC_ACTION, WM_KEYDOWN, - ctypes.addressof(shift), + ctypes.addressof(first), ) == 73 ) + assert translation_entered.wait(timeout=1) assert ( observer._keyboard_hook_callback( HC_ACTION, WM_KEYDOWN, - ctypes.addressof(letter), + ctypes.addressof(second), ) == 73 ) - assert wait_until(lambda: len(events) == 2) user32.foreground_thread_id = 9876 user32.keyboard_layout = 55 + translation_release.set() + assert wait_until(lambda: len(events) == 2) + + third = KBDLLHOOKSTRUCT(vkCode=0x43, scanCode=0x2E) assert ( observer._keyboard_hook_callback( HC_ACTION, - WM_KEYUP, - ctypes.addressof(letter), + WM_KEYDOWN, + ctypes.addressof(third), ) == 73 ) @@ -455,30 +475,31 @@ def test_keyboard_translation_is_off_hook_and_uses_foreground_layout() -> None: assert events == [ ObservedKey( pressed=True, - key_name="shift_l", - key_vk="160", - canonical_key_name="shift", - canonical_key_vk="16", + key_char="a", + key_vk="65", + canonical_key_char="a", + canonical_key_vk="65", timestamp=1234.5, ), ObservedKey( pressed=True, - key_char="A", - key_vk="65", - canonical_key_char="a", - canonical_key_vk="65", + key_char="b", + key_vk="66", + canonical_key_char="b", + canonical_key_vk="66", timestamp=1234.5, ), ObservedKey( - pressed=False, - key_char="A", - key_vk="65", - canonical_key_char="a", - canonical_key_vk="65", + pressed=True, + key_char="c", + key_vk="67", + canonical_key_char="c", + canonical_key_vk="67", timestamp=1234.5, ), ] - assert user32.layout_thread_ids == [8765, 9876] + assert user32.layout_thread_ids == [8765, 8765, 9876] + assert user32.translation_layouts == [44, 44, 55] observer.stop() @@ -544,6 +565,86 @@ def test_dead_key_preserves_physical_identity_without_claiming_text() -> None: observer.stop() +def test_dead_key_resolution_is_unverifiable_then_normal_text_recovers() -> None: + events = [] + user32 = FakeUser32( + translation_script=[ + (-1, "´"), + (1, "e"), + (1, "x"), + ] + ) + observer = make_observer(events.append, user32=user32) + observer.start() + dead_key = KBDLLHOOKSTRUCT(vkCode=0xDE, scanCode=0x28) + resolving_key = KBDLLHOOKSTRUCT(vkCode=0x45, scanCode=0x12) + normal_key = KBDLLHOOKSTRUCT(vkCode=0x58, scanCode=0x2D) + + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(dead_key), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYUP, + ctypes.addressof(dead_key), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(resolving_key), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYUP, + ctypes.addressof(resolving_key), + ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(normal_key), + ) + + assert wait_until(lambda: len(events) == 5) + assert events == [ + ObservedKey( + pressed=True, + key_vk="222", + canonical_key_vk="222", + timestamp=1234.5, + ), + ObservedKey( + pressed=False, + key_vk="222", + canonical_key_vk="222", + timestamp=1234.5, + ), + ObservedKey( + pressed=True, + key_vk="69", + canonical_key_vk="69", + timestamp=1234.5, + ), + ObservedKey( + pressed=False, + key_vk="69", + canonical_key_vk="69", + timestamp=1234.5, + ), + ObservedKey( + pressed=True, + key_char="x", + key_vk="88", + canonical_key_char="x", + canonical_key_vk="88", + timestamp=1234.5, + ), + ] + assert user32.translation_script == [] + observer.stop() + + def test_blocked_keyboard_translation_does_not_block_hook_or_reorder_mouse() -> None: translation_entered = threading.Event() translation_release = threading.Event() From 1eec8184399031ba947e23d5dd9ce20b6eb17ef7 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:17:54 -0700 Subject: [PATCH 18/26] fix: preserve published mouse button runtime type --- openadapt_capture/events.py | 28 ++++--------- openadapt_capture/input_observer/windows.py | 5 ++- tests/test_events.py | 15 ------- tests/test_input_observer.py | 45 +-------------------- tests/test_input_observer_windows.py | 13 +++++- 5 files changed, 23 insertions(+), 83 deletions(-) diff --git a/openadapt_capture/events.py b/openadapt_capture/events.py index 3c59c9e..ac5ae5b 100644 --- a/openadapt_capture/events.py +++ b/openadapt_capture/events.py @@ -8,9 +8,9 @@ from __future__ import annotations from enum import Enum -from typing import Annotated, Literal +from typing import Literal -from pydantic import AfterValidator, BaseModel, Field +from pydantic import BaseModel, Field class EventType(str, Enum): @@ -49,20 +49,6 @@ class MouseButton(str, Enum): MIDDLE = "middle" -def _normalize_mouse_button(value: MouseButton | str) -> MouseButton | str: - """Preserve the established enum while allowing native auxiliary buttons.""" - try: - return MouseButton(value) - except ValueError: - return value - - -MouseButtonValue = Annotated[ - MouseButton | str, - AfterValidator(_normalize_mouse_button), -] - - class BaseEvent(BaseModel): """Base class for all events. @@ -101,7 +87,7 @@ class MouseDownEvent(BaseEvent): type: Literal[EventType.MOUSE_DOWN] = EventType.MOUSE_DOWN x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButtonValue = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") class MouseUpEvent(BaseEvent): @@ -113,7 +99,7 @@ class MouseUpEvent(BaseEvent): type: Literal[EventType.MOUSE_UP] = EventType.MOUSE_UP x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButtonValue = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") class MouseScrollEvent(BaseEvent): @@ -216,7 +202,7 @@ class MouseClickEvent(BaseEvent): type: Literal[EventType.MOUSE_SINGLECLICK] = EventType.MOUSE_SINGLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButtonValue = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -232,7 +218,7 @@ class MouseDoubleClickEvent(BaseEvent): type: Literal[EventType.MOUSE_DOUBLECLICK] = EventType.MOUSE_DOUBLECLICK x: float = Field(description="Mouse X position in pixels") y: float = Field(description="Mouse Y position in pixels") - button: MouseButtonValue = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) @@ -250,7 +236,7 @@ class MouseDragEvent(BaseEvent): y: float = Field(description="Starting Y position in pixels") dx: float = Field(description="Horizontal displacement (end_x - start_x)") dy: float = Field(description="Vertical displacement (end_y - start_y)") - button: MouseButtonValue = Field(description="Mouse button name") + button: str = Field(description="Native mouse button name") children: list[MouseDownEvent | MouseMoveEvent | MouseUpEvent] = Field( default_factory=list, description="Child events that were merged" ) diff --git a/openadapt_capture/input_observer/windows.py b/openadapt_capture/input_observer/windows.py index 58ad15f..11a4851 100644 --- a/openadapt_capture/input_observer/windows.py +++ b/openadapt_capture/input_observer/windows.py @@ -752,8 +752,9 @@ def _translated_character( the resolving key is released; the following normal key then recovers. """ - if not transition.pressed and physical_vk in self._unverifiable_text_keys: - self._unverifiable_text_keys.discard(physical_vk) + if physical_vk in self._unverifiable_text_keys: + if not transition.pressed: + self._unverifiable_text_keys.discard(physical_vk) return None character, is_dead_key = self._keyboard_character(transition) diff --git a/tests/test_events.py b/tests/test_events.py index 1e4e167..b6d9fdd 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -49,7 +49,6 @@ def test_mouse_down_event(self): ) assert event.type == EventType.MOUSE_DOWN assert event.button == MouseButton.LEFT - assert isinstance(event.button, MouseButton) def test_mouse_up_event(self): """Test MouseUpEvent creation.""" @@ -61,20 +60,6 @@ def test_mouse_up_event(self): ) assert event.type == EventType.MOUSE_UP assert event.button == MouseButton.RIGHT - assert isinstance(event.button, MouseButton) - - def test_auxiliary_mouse_button_remains_a_native_string(self): - """Unknown native buttons survive without weakening standard enums.""" - event = MouseDownEvent( - timestamp=1234567890.123, - x=100.0, - y=200.0, - button="x1", - ) - assert event.button == "x1" - assert isinstance(event.button, str) - assert not isinstance(event.button, MouseButton) - assert event.model_dump_json().find('"button":"x1"') != -1 def test_mouse_scroll_event(self): """Test MouseScrollEvent creation.""" diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 572762a..5097f57 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -15,12 +15,7 @@ from openadapt_capture import input as input_module from openadapt_capture import recorder as recorder_module from openadapt_capture.capture import _convert_action_event -from openadapt_capture.events import ( - MouseButton, - MouseClickEvent, - MouseDownEvent, - MouseUpEvent, -) +from openadapt_capture.events import MouseClickEvent, MouseDownEvent, MouseUpEvent from openadapt_capture.input_observer import ( InputObserverError, InputObserverUnavailableError, @@ -530,48 +525,10 @@ def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( assert isinstance(up, MouseUpEvent) assert down.button == button assert up.button == button - assert not isinstance(down.button, MouseButton) - assert not isinstance(up.button, MouseButton) processed = merge_consecutive_mouse_click_events([down, up]) assert len(processed) == 1 assert isinstance(processed[0], MouseClickEvent) assert processed[0].button == button - assert not isinstance(processed[0].button, MouseButton) - - -@pytest.mark.parametrize("button", list(MouseButton)) -def test_standard_mouse_button_keeps_public_enum_through_processing( - button: MouseButton, -) -> None: - down = _convert_action_event( - SimpleNamespace( - name="click", - timestamp=1.0, - mouse_x=12.5, - mouse_y=34.5, - mouse_button_name=button.value, - mouse_pressed=True, - ) - ) - up = _convert_action_event( - SimpleNamespace( - name="click", - timestamp=1.1, - mouse_x=12.5, - mouse_y=34.5, - mouse_button_name=button.value, - mouse_pressed=False, - ) - ) - - assert isinstance(down, MouseDownEvent) - assert isinstance(up, MouseUpEvent) - assert down.button is button - assert up.button is button - processed = merge_consecutive_mouse_click_events([down, up]) - assert len(processed) == 1 - assert isinstance(processed[0], MouseClickEvent) - assert processed[0].button is button @pytest.mark.skipif( diff --git a/tests/test_input_observer_windows.py b/tests/test_input_observer_windows.py index 99523ed..24c7a36 100644 --- a/tests/test_input_observer_windows.py +++ b/tests/test_input_observer_windows.py @@ -595,6 +595,11 @@ def test_dead_key_resolution_is_unverifiable_then_normal_text_recovers() -> None WM_KEYDOWN, ctypes.addressof(resolving_key), ) + observer._keyboard_hook_callback( + HC_ACTION, + WM_KEYDOWN, + ctypes.addressof(resolving_key), + ) observer._keyboard_hook_callback( HC_ACTION, WM_KEYUP, @@ -606,7 +611,7 @@ def test_dead_key_resolution_is_unverifiable_then_normal_text_recovers() -> None ctypes.addressof(normal_key), ) - assert wait_until(lambda: len(events) == 5) + assert wait_until(lambda: len(events) == 6) assert events == [ ObservedKey( pressed=True, @@ -626,6 +631,12 @@ def test_dead_key_resolution_is_unverifiable_then_normal_text_recovers() -> None canonical_key_vk="69", timestamp=1234.5, ), + ObservedKey( + pressed=True, + key_vk="69", + canonical_key_vk="69", + timestamp=1234.5, + ), ObservedKey( pressed=False, key_vk="69", From e722cd531532dbc9a71effe181797cc152610378 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:27:17 -0700 Subject: [PATCH 19/26] fix: bind Linux input to event-time state --- openadapt_capture/input_observer/linux.py | 923 +++++++++++++++------- tests/test_input_observer_linux_xkb.py | 582 ++++++++++++-- 2 files changed, 1142 insertions(+), 363 deletions(-) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index edfb422..613fe04 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -1,4 +1,15 @@ -"""Linux global input observation through XInput2 raw events.""" +"""Fail-closed Linux global input observation through the X RECORD extension. + +The RECORD specification deliberately leaves most fields in an intercepted +``device_event`` undefined: key/button events guarantee only ``time`` and +``detail``; motion additionally guarantees root coordinates. This observer +therefore never samples current X state after an event. It preserves the +globally ordered device stream and enriches key/button events only from the +matching delivered core event, whose wire record carries event-time state and +coordinates. Missing enrichment degrades keyboard text to physical identity +and makes pointer actions fail closed unless an exact earlier pointer position +is known. +""" from __future__ import annotations @@ -7,6 +18,7 @@ import os import sys import time +from dataclasses import dataclass from typing import Any from .base import ( @@ -21,16 +33,29 @@ ThreadedInputObserver, ) -_GENERIC_EVENT = 35 -_XI_ALL_MASTER_DEVICES = 1 -_XI_RAW_KEY_PRESS = 13 -_XI_RAW_KEY_RELEASE = 14 -_XI_RAW_BUTTON_PRESS = 15 -_XI_RAW_BUTTON_RELEASE = 16 -_XI_RAW_MOTION = 17 -_XKB_USE_CORE_KBD = 0x0100 -_XKB_GROUP_SHIFT = 13 -_MAX_PENDING_BATCH = 256 +_KEY_PRESS = 2 +_KEY_RELEASE = 3 +_BUTTON_PRESS = 4 +_BUTTON_RELEASE = 5 +_MOTION_NOTIFY = 6 +_CORE_DEVICE_EVENT_TYPES = { + _KEY_PRESS, + _KEY_RELEASE, + _BUTTON_PRESS, + _BUTTON_RELEASE, + _MOTION_NOTIFY, +} + +_XRECORD_FROM_SERVER = 0 +_XRECORD_START_OF_DATA = 4 +_XRECORD_END_OF_DATA = 5 +_XRECORD_ALL_CLIENTS = 3 +_XRECORD_FROM_SERVER_TIME = 1 << 0 +_X_QUERY_POINTER = 38 +_CORE_EVENT_BYTES = 32 +_CORRELATION_TIMEOUT_SECONDS = 0.100 +_LOOP_INTERVAL_SECONDS = 0.010 + _XKB_COMPOSE_COMPOSING = 1 _XKB_COMPOSE_COMPOSED = 2 _XKB_COMPOSE_CANCELLED = 3 @@ -97,78 +122,134 @@ } -class _XIEventMask(ctypes.Structure): +class _XRecordRange8(ctypes.Structure): _fields_ = [ - ("deviceid", ctypes.c_int), - ("mask_len", ctypes.c_int), - ("mask", ctypes.POINTER(ctypes.c_ubyte)), + ("first", ctypes.c_ubyte), + ("last", ctypes.c_ubyte), ] -class _XIValuatorState(ctypes.Structure): +class _XRecordRange16(ctypes.Structure): _fields_ = [ - ("mask_len", ctypes.c_int), - ("mask", ctypes.POINTER(ctypes.c_ubyte)), - ("values", ctypes.POINTER(ctypes.c_double)), + ("first", ctypes.c_ushort), + ("last", ctypes.c_ushort), ] -class _XGenericEventCookie(ctypes.Structure): +class _XRecordExtRange(ctypes.Structure): _fields_ = [ - ("type", ctypes.c_int), - ("serial", ctypes.c_ulong), - ("send_event", ctypes.c_int), - ("display", ctypes.c_void_p), - ("extension", ctypes.c_int), - ("evtype", ctypes.c_int), - ("cookie", ctypes.c_uint), - ("data", ctypes.c_void_p), + ("ext_major", _XRecordRange8), + ("ext_minor", _XRecordRange16), ] -class _XEvent(ctypes.Union): +class _XRecordRange(ctypes.Structure): + """ABI mirror of ``X11/extensions/record.h::XRecordRange``.""" + _fields_ = [ - ("type", ctypes.c_int), - ("xcookie", _XGenericEventCookie), - ("pad", ctypes.c_long * 24), + ("core_requests", _XRecordRange8), + ("core_replies", _XRecordRange8), + ("ext_requests", _XRecordExtRange), + ("ext_replies", _XRecordExtRange), + ("delivered_events", _XRecordRange8), + ("device_events", _XRecordRange8), + ("errors", _XRecordRange8), + ("client_started", ctypes.c_int), + ("client_died", ctypes.c_int), ] -class _XIRawEvent(ctypes.Structure): +class _XRecordInterceptData(ctypes.Structure): + """ABI mirror of ``XRecordInterceptData`` from ``record.h``.""" + _fields_ = [ - ("type", ctypes.c_int), - ("serial", ctypes.c_ulong), - ("send_event", ctypes.c_int), - ("display", ctypes.c_void_p), - ("extension", ctypes.c_int), - ("evtype", ctypes.c_int), - ("time", ctypes.c_ulong), - ("deviceid", ctypes.c_int), - ("sourceid", ctypes.c_int), - ("detail", ctypes.c_int), - ("flags", ctypes.c_int), - ("valuators", _XIValuatorState), - ("raw_values", ctypes.POINTER(ctypes.c_double)), + ("id_base", ctypes.c_ulong), + ("server_time", ctypes.c_ulong), + ("client_seq", ctypes.c_ulong), + ("category", ctypes.c_int), + ("client_swapped", ctypes.c_int), + ("data", ctypes.POINTER(ctypes.c_ubyte)), + ("data_len", ctypes.c_ulong), ] -class _XkbStateRec(ctypes.Structure): - _fields_ = [ - ("group", ctypes.c_ubyte), - ("locked_group", ctypes.c_ubyte), - ("base_group", ctypes.c_ushort), - ("latched_group", ctypes.c_ushort), - ("mods", ctypes.c_ubyte), - ("base_mods", ctypes.c_ubyte), - ("latched_mods", ctypes.c_ubyte), - ("locked_mods", ctypes.c_ubyte), - ("compat_state", ctypes.c_ubyte), - ("grab_mods", ctypes.c_ubyte), - ("compat_grab_mods", ctypes.c_ubyte), - ("lookup_mods", ctypes.c_ubyte), - ("compat_lookup_mods", ctypes.c_ubyte), - ("ptr_buttons", ctypes.c_ushort), - ] +_XRecordInterceptProc = ctypes.CFUNCTYPE( + None, + ctypes.c_void_p, + ctypes.POINTER(_XRecordInterceptData), +) + + +@dataclass(frozen=True, slots=True) +class _CoreWireEvent: + event_type: int + detail: int + injected: bool + time: int + root: int + event: int + child: int + root_x: int + root_y: int + event_x: int + event_y: int + state: int + + +@dataclass(frozen=True, slots=True) +class _DeliveredCandidate: + state: int + root_x: float + root_y: float + injected: bool + id_base: int + + +@dataclass(slots=True) +class _PendingDeviceEvent: + event_type: int + detail: int + server_time: int + injected: bool + receipt_timestamp: float + deadline: float + candidate: _DeliveredCandidate | None = None + + +def _event_byteorder(*, client_swapped: bool) -> str: + native_little = sys.byteorder == "little" + event_little = native_little != client_swapped + return "little" if event_little else "big" + + +def _decode_core_event( + data: bytes, + *, + client_swapped: bool, +) -> _CoreWireEvent: + """Decode one 32-byte core xEvent in the recorded client's byte order.""" + if len(data) != _CORE_EVENT_BYTES: + raise InputObserverError( + f"X RECORD core event was {len(data)} bytes, expected " + f"{_CORE_EVENT_BYTES}" + ) + byteorder = _event_byteorder(client_swapped=client_swapped) + encoded_type = data[0] + event_type = encoded_type & 0x7F + return _CoreWireEvent( + event_type=event_type, + detail=data[1], + injected=bool(encoded_type & 0x80), + time=int.from_bytes(data[4:8], byteorder), + root=int.from_bytes(data[8:12], byteorder), + event=int.from_bytes(data[12:16], byteorder), + child=int.from_bytes(data[16:20], byteorder), + root_x=int.from_bytes(data[20:22], byteorder, signed=True), + root_y=int.from_bytes(data[22:24], byteorder, signed=True), + event_x=int.from_bytes(data[24:26], byteorder, signed=True), + event_y=int.from_bytes(data[26:28], byteorder, signed=True), + state=int.from_bytes(data[28:30], byteorder), + ) def normalize_xinput_button_event( @@ -180,7 +261,7 @@ def normalize_xinput_button_event( injected: bool = False, timestamp: float | None = None, ) -> ObservedInput | None: - """Map X11 button numbers to a click or normalized wheel event.""" + """Map X11 core button numbers to a click or normalized wheel event.""" if detail in {4, 5, 6, 7}: if not pressed: return None @@ -231,7 +312,7 @@ def normalize_xinput_key_event( derive_character: bool = True, timestamp: float | None = None, ) -> ObservedKey: - """Normalize an XInput2 keycode and XKB keysym name.""" + """Normalize an X11 keycode and XKB keysym name.""" key_name = _SPECIAL_KEY_NAMES.get(keysym_name or "") if derive_character and keysym_name: if len(keysym_name) == 1 and keysym_name.isprintable(): @@ -255,18 +336,31 @@ def normalize_xinput_key_event( class LinuxXInputObserver(ThreadedInputObserver): - """Observe complete X11 desktop input with XInput2 raw events.""" + """Observe complete X11 input without post-event state sampling.""" def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> None: super().__init__(*args, **kwargs) self._environ = environ if environ is not None else os.environ self._x11: Any = None - self._xi: Any = None + self._xtst: Any = None self._xkbcommon: Any = None - self._display: Any = None + self._control_display: Any = None + self._data_display: Any = None + self._record_range: Any = None + self._record_context = 0 + self._record_enabled = False + self._record_started = False + self._accepting_events = False + self._waiting_baseline_marker = False + self._baseline_marker_seen = False + self._control_id_base = 0 self._root = 0 - self._xi_opcode = 0 - self._event_mask_buffer: Any = None + self._record_callback: Any = None + self._record_callback_failure: BaseException | None = None + self._pending: _PendingDeviceEvent | None = None + self._delivered_correlation_uncertain = False + self._last_pointer: tuple[float, float] | None = None + self._text_state_uncertain = False self._composition_mode: str | None = None self._composition_remaining = 0 self._unverifiable_keycodes: set[int] = set() @@ -274,128 +368,173 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._compose_table: Any = None self._compose_state: Any = None - @staticmethod - def _set_mask(mask: ctypes.Array[ctypes.c_ubyte], event_type: int) -> None: - mask[event_type >> 3] |= 1 << (event_type & 7) - def _setup(self) -> None: if not sys.platform.startswith("linux"): raise InputObserverUnavailableError( - "XInput2 input observation is available only on Linux" + "X RECORD input observation is available only on Linux" ) session_type = self._environ.get("XDG_SESSION_TYPE", "").lower() if session_type == "wayland" or self._environ.get("WAYLAND_DISPLAY"): raise InputObserverUnavailableError( - "native Wayland does not expose complete global input to XInput2. " + "native Wayland does not expose complete global input to X RECORD. " "Run an X11 session; OpenAdapt refuses XWayland-only capture because " "it would silently omit input from native Wayland applications." ) display_name = self._environ.get("DISPLAY") if not display_name: raise InputObserverUnavailableError( - "XInput2 requires an X11 desktop and DISPLAY is not set" + "X RECORD requires an X11 desktop and DISPLAY is not set" ) x11_name = ctypes.util.find_library("X11") - xi_name = ctypes.util.find_library("Xi") + xtst_name = ctypes.util.find_library("Xtst") xkbcommon_name = ctypes.util.find_library("xkbcommon") - if not x11_name or not xi_name or not xkbcommon_name: + if not x11_name or not xtst_name or not xkbcommon_name: raise InputObserverUnavailableError( - "Linux input observation requires the system libX11, libXi, " - "and permissively licensed libxkbcommon runtime libraries" + "Linux input observation requires the system libX11, libXtst " + "(X RECORD), and permissively licensed libxkbcommon runtime libraries" ) try: self._x11 = ctypes.CDLL(x11_name) - self._xi = ctypes.CDLL(xi_name) + self._xtst = ctypes.CDLL(xtst_name) self._xkbcommon = ctypes.CDLL(xkbcommon_name) except OSError as exc: raise InputObserverUnavailableError( f"could not load X11 input libraries: {exc}" ) from exc + self._configure_api() self._initialize_compose() - self._display = self._x11.XOpenDisplay(display_name.encode()) - if not self._display: + encoded_display = display_name.encode() + self._control_display = self._x11.XOpenDisplay(encoded_display) + if not self._control_display: raise InputObserverPermissionError( f"could not open X11 display {display_name!r}; authorize the " "recording user with the display server and retry" ) - self._root = int(self._x11.XDefaultRootWindow(self._display)) - - opcode = ctypes.c_int() - first_event = ctypes.c_int() - first_error = ctypes.c_int() - if not self._x11.XQueryExtension( - self._display, - b"XInputExtension", - ctypes.byref(opcode), - ctypes.byref(first_event), - ctypes.byref(first_error), - ): - raise InputObserverUnavailableError( - "the X11 server does not expose the XInput extension" + self._data_display = self._x11.XOpenDisplay(encoded_display) + if not self._data_display: + raise InputObserverPermissionError( + f"could not open the X RECORD data connection to {display_name!r}" + ) + self._root = int(self._x11.XDefaultRootWindow(self._control_display)) + probe_resource = int( + self._x11.XCreatePixmap( + self._control_display, + self._root, + 1, + 1, + 1, ) - self._xi_opcode = opcode.value - major = ctypes.c_int(2) - minor = ctypes.c_int(0) - status = self._xi.XIQueryVersion( - self._display, ctypes.byref(major), ctypes.byref(minor) ) - if status != 0 or major.value < 2: + if not probe_resource: + raise InputObserverError( + "X11 could not allocate a resource ID for the recording boundary" + ) + id_base_mask = int(self._xtst.XRecordIdBaseMask(self._control_display)) + self._control_id_base = probe_resource & id_base_mask + self._x11.XFreePixmap(self._control_display, probe_resource) + + major = ctypes.c_int() + minor = ctypes.c_int() + if not self._xtst.XRecordQueryVersion( + self._control_display, + ctypes.byref(major), + ctypes.byref(minor), + ): raise InputObserverUnavailableError( - f"XInput2 is required; server negotiation returned " - f"{major.value}.{minor.value} with status {status}" + "the X11 server does not expose the X RECORD extension" ) - mask = (ctypes.c_ubyte * 4)() - if self.observe_keyboard: - self._set_mask(mask, _XI_RAW_KEY_PRESS) - self._set_mask(mask, _XI_RAW_KEY_RELEASE) - if self.observe_mouse: - self._set_mask(mask, _XI_RAW_BUTTON_PRESS) - self._set_mask(mask, _XI_RAW_BUTTON_RELEASE) - if self.capture_mouse_moves: - self._set_mask(mask, _XI_RAW_MOTION) - event_mask = _XIEventMask( - deviceid=_XI_ALL_MASTER_DEVICES, - mask_len=len(mask), - mask=ctypes.cast(mask, ctypes.POINTER(ctypes.c_ubyte)), + record_range = self._xtst.XRecordAllocRange() + if not record_range: + raise InputObserverUnavailableError( + "libXtst could not allocate an X RECORD event range" + ) + self._record_range = record_range + # QueryPointer replies serve only as an ordered setup marker. Input + # acceptance begins inside the callback that observes our control + # connection's reply, after the exact baseline position is stored. + record_range.contents.core_replies = _XRecordRange8( + _X_QUERY_POINTER, + _X_QUERY_POINTER, + ) + record_range.contents.delivered_events = _XRecordRange8( + _KEY_PRESS, + _MOTION_NOTIFY, + ) + record_range.contents.device_events = _XRecordRange8( + _KEY_PRESS, + _MOTION_NOTIFY, ) - self._event_mask_buffer = mask - if self._xi.XISelectEvents( - self._display, self._root, ctypes.byref(event_mask), 1 - ) != 0: + client_specs = (ctypes.c_ulong * 1)(_XRECORD_ALL_CLIENTS) + ranges = (ctypes.POINTER(_XRecordRange) * 1)(record_range) + context = self._xtst.XRecordCreateContext( + self._control_display, + _XRECORD_FROM_SERVER_TIME, + client_specs, + 1, + ranges, + 1, + ) + if not context: + raise InputObserverPermissionError( + "the X11 server refused creation of a global X RECORD context" + ) + self._record_context = int(context) + self._record_callback = _XRecordInterceptProc(self._record_intercept) + if not self._xtst.XRecordEnableContextAsync( + self._data_display, + self._record_context, + self._record_callback, + None, + ): raise InputObserverPermissionError( - "the X11 server refused XInput2 raw-event selection" + "the X11 server refused to enable the global X RECORD context" + ) + self._record_enabled = True + self._raise_callback_failure() + if not self._record_started: + raise InputObserverError( + "XRecordEnableContextAsync returned before StartOfData; " + "the event stream cannot be trusted" + ) + self._waiting_baseline_marker = True + self._query_pointer_baseline() + marker_deadline = time.monotonic() + self.startup_timeout + while not self._baseline_marker_seen: + self._xtst.XRecordProcessReplies(self._data_display) + self._raise_callback_failure() + if time.monotonic() >= marker_deadline: + raise InputObserverError( + "X RECORD did not deliver the ordered pointer-baseline marker " + f"within {self.startup_timeout:.1f}s" + ) + self._stop_requested.wait(0.001) + if not self._accepting_events: + raise InputObserverError( + "X RECORD observed the pointer-baseline marker without arming input" ) - self._x11.XFlush(self._display) def _configure_api(self) -> None: self._x11.XOpenDisplay.argtypes = [ctypes.c_char_p] self._x11.XOpenDisplay.restype = ctypes.c_void_p self._x11.XCloseDisplay.argtypes = [ctypes.c_void_p] + self._x11.XCloseDisplay.restype = ctypes.c_int self._x11.XDefaultRootWindow.argtypes = [ctypes.c_void_p] self._x11.XDefaultRootWindow.restype = ctypes.c_ulong - self._x11.XQueryExtension.argtypes = [ + self._x11.XCreatePixmap.argtypes = [ ctypes.c_void_p, - ctypes.c_char_p, - ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_int), - ctypes.POINTER(ctypes.c_int), - ] - self._x11.XPending.argtypes = [ctypes.c_void_p] - self._x11.XPending.restype = ctypes.c_int - self._x11.XNextEvent.argtypes = [ctypes.c_void_p, ctypes.POINTER(_XEvent)] - self._x11.XGetEventData.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(_XGenericEventCookie), - ] - self._x11.XGetEventData.restype = ctypes.c_int - self._x11.XFreeEventData.argtypes = [ - ctypes.c_void_p, - ctypes.POINTER(_XGenericEventCookie), + ctypes.c_ulong, + ctypes.c_uint, + ctypes.c_uint, + ctypes.c_uint, ] - self._x11.XFlush.argtypes = [ctypes.c_void_p] + self._x11.XCreatePixmap.restype = ctypes.c_ulong + self._x11.XFreePixmap.argtypes = [ctypes.c_void_p, ctypes.c_ulong] + self._x11.XFreePixmap.restype = ctypes.c_int + self._x11.XFree.argtypes = [ctypes.c_void_p] + self._x11.XFree.restype = ctypes.c_int self._x11.XQueryPointer.argtypes = [ ctypes.c_void_p, ctypes.c_ulong, @@ -408,12 +547,6 @@ def _configure_api(self) -> None: ctypes.POINTER(ctypes.c_uint), ] self._x11.XQueryPointer.restype = ctypes.c_int - self._x11.XkbGetState.argtypes = [ - ctypes.c_void_p, - ctypes.c_uint, - ctypes.POINTER(_XkbStateRec), - ] - self._x11.XkbGetState.restype = ctypes.c_int self._x11.XkbLookupKeySym.argtypes = [ ctypes.c_void_p, ctypes.c_ubyte, @@ -425,19 +558,49 @@ def _configure_api(self) -> None: self._x11.XKeysymToString.argtypes = [ctypes.c_ulong] self._x11.XKeysymToString.restype = ctypes.c_char_p - self._xi.XIQueryVersion.argtypes = [ + self._xtst.XRecordQueryVersion.argtypes = [ ctypes.c_void_p, ctypes.POINTER(ctypes.c_int), ctypes.POINTER(ctypes.c_int), ] - self._xi.XIQueryVersion.restype = ctypes.c_int - self._xi.XISelectEvents.argtypes = [ + self._xtst.XRecordQueryVersion.restype = ctypes.c_int + self._xtst.XRecordIdBaseMask.argtypes = [ctypes.c_void_p] + self._xtst.XRecordIdBaseMask.restype = ctypes.c_ulong + self._xtst.XRecordAllocRange.argtypes = [] + self._xtst.XRecordAllocRange.restype = ctypes.POINTER(_XRecordRange) + self._xtst.XRecordCreateContext.argtypes = [ ctypes.c_void_p, - ctypes.c_ulong, - ctypes.POINTER(_XIEventMask), ctypes.c_int, + ctypes.POINTER(ctypes.c_ulong), + ctypes.c_int, + ctypes.POINTER(ctypes.POINTER(_XRecordRange)), + ctypes.c_int, + ] + self._xtst.XRecordCreateContext.restype = ctypes.c_ulong + self._xtst.XRecordEnableContextAsync.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + _XRecordInterceptProc, + ctypes.c_void_p, + ] + self._xtst.XRecordEnableContextAsync.restype = ctypes.c_int + self._xtst.XRecordProcessReplies.argtypes = [ctypes.c_void_p] + self._xtst.XRecordProcessReplies.restype = None + self._xtst.XRecordFreeData.argtypes = [ + ctypes.POINTER(_XRecordInterceptData) + ] + self._xtst.XRecordFreeData.restype = None + self._xtst.XRecordDisableContext.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, ] - self._xi.XISelectEvents.restype = ctypes.c_int + self._xtst.XRecordDisableContext.restype = ctypes.c_int + self._xtst.XRecordFreeContext.argtypes = [ + ctypes.c_void_p, + ctypes.c_ulong, + ] + self._xtst.XRecordFreeContext.restype = ctypes.c_int + self._xkbcommon.xkb_keysym_to_utf8.argtypes = [ ctypes.c_uint32, ctypes.POINTER(ctypes.c_char), @@ -505,33 +668,67 @@ def _initialize_compose(self) -> None: "libxkbcommon could not create compose state" ) - def _compose_character(self, keysym: int) -> tuple[bool, str | None]: - """Return whether compose consumed this press and any committed text.""" - self._xkbcommon.xkb_compose_state_feed(self._compose_state, keysym) - status = int( - self._xkbcommon.xkb_compose_state_get_status(self._compose_state) - ) - if status == _XKB_COMPOSE_COMPOSING: - return True, None - if status == _XKB_COMPOSE_COMPOSED: - buffer = ctypes.create_string_buffer(64) - length = int( - self._xkbcommon.xkb_compose_state_get_utf8( - self._compose_state, - buffer, - len(buffer), + def _record_intercept( + self, + _closure: ctypes.c_void_p, + recorded_data_pointer: ctypes.POINTER(_XRecordInterceptData), + ) -> None: + """Copy and process one RECORD datum without leaking across the callback.""" + try: + recorded = recorded_data_pointer.contents + if recorded.category == _XRECORD_START_OF_DATA: + self._record_started = True + return + if recorded.category == _XRECORD_END_OF_DATA: + return + if recorded.category != _XRECORD_FROM_SERVER: + return + byte_length = int(recorded.data_len) * 4 + if byte_length != _CORE_EVENT_BYTES or not recorded.data: + raise InputObserverError( + "X RECORD returned a malformed core input event: " + f"{byte_length} bytes" ) + payload = ctypes.string_at(recorded.data, byte_length) + if ( + self._waiting_baseline_marker + and int(recorded.id_base) == self._control_id_base + and payload[0] == 1 + ): + self._waiting_baseline_marker = False + self._baseline_marker_seen = True + self._accepting_events = True + return + event = _decode_core_event( + payload, + client_swapped=bool(recorded.client_swapped), ) - self._xkbcommon.xkb_compose_state_reset(self._compose_state) - if length <= 0: - return True, None - return True, buffer.value.decode("utf-8", errors="strict") - if status == _XKB_COMPOSE_CANCELLED: - self._xkbcommon.xkb_compose_state_reset(self._compose_state) - return True, None - return False, None + if event.event_type not in _CORE_DEVICE_EVENT_TYPES: + # QueryPointer replies from other clients are included by the + # marker range but cannot arm this observer. + return + if not self._accepting_events: + # Establishing the baseline is a deliberate cut: every event + # before our recorded QueryPointer reply is discarded. + return + if int(recorded.id_base) == 0: + if recorded.client_swapped: + raise InputObserverError( + "X RECORD marked a device event as byte-swapped; " + "device-event byte order must match the recording client" + ) + self._handle_device_event(event) + else: + self._handle_delivered_event(event, id_base=int(recorded.id_base)) + except BaseException as exc: + if self._record_callback_failure is None: + self._record_callback_failure = exc + self._stop_requested.set() + finally: + self._xtst.XRecordFreeData(recorded_data_pointer) - def _query_pointer(self) -> tuple[float, float]: + def _query_pointer_baseline(self) -> None: + """Sample the pointer once, before the recorded acceptance boundary.""" root_return = ctypes.c_ulong() child_return = ctypes.c_ulong() root_x = ctypes.c_int() @@ -540,7 +737,7 @@ def _query_pointer(self) -> tuple[float, float]: window_y = ctypes.c_int() mask = ctypes.c_uint() if not self._x11.XQueryPointer( - self._display, + self._control_display, self._root, ctypes.byref(root_return), ctypes.byref(child_return), @@ -551,39 +748,216 @@ def _query_pointer(self) -> tuple[float, float]: ctypes.byref(mask), ): raise InputObserverError( - "XInput2 delivered mouse input but XQueryPointer could not establish " - "its global coordinates" + "X11 could not establish the exact pointer position at the " + "recording boundary" ) - return float(root_x.value), float(root_y.value) - - def _lookup_keysym(self, keycode: int) -> tuple[int, str | None]: - state = _XkbStateRec() - if self._x11.XkbGetState( - self._display, - _XKB_USE_CORE_KBD, - ctypes.byref(state), - ) != 0: + self._last_pointer = (float(root_x.value), float(root_y.value)) + + def _handle_device_event(self, event: _CoreWireEvent) -> None: + """Accept the next event in the global device stream.""" + self._finalize_pending() + observed_at = time.time() + if event.event_type == _MOTION_NOTIFY: + if not self.observe_mouse: + return + position = (float(event.root_x), float(event.root_y)) + self._last_pointer = position + if self.capture_mouse_moves: + self._emit( + ObservedMouseMove( + x=position[0], + y=position[1], + injected=event.injected, + timestamp=observed_at, + ) + ) + return + if event.event_type in {_KEY_PRESS, _KEY_RELEASE}: + if not self.observe_keyboard: + return + elif event.event_type in {_BUTTON_PRESS, _BUTTON_RELEASE}: + if not self.observe_mouse: + return + else: # pragma: no cover - guarded by the callback range check + return + self._pending = _PendingDeviceEvent( + event_type=event.event_type, + detail=event.detail, + server_time=event.time, + injected=event.injected, + receipt_timestamp=observed_at, + deadline=time.monotonic() + _CORRELATION_TIMEOUT_SECONDS, + ) + + def _handle_delivered_event( + self, + event: _CoreWireEvent, + *, + id_base: int, + ) -> None: + """Attach exact delivered-event state to the matching device event.""" + pending = self._pending + if pending is None or self._delivered_correlation_uncertain: + return + if ( + event.event_type != pending.event_type + or event.time != pending.server_time + or event.detail != pending.detail + ): + return + candidate = _DeliveredCandidate( + state=event.state, + root_x=float(event.root_x), + root_y=float(event.root_y), + injected=event.injected, + id_base=id_base, + ) + current = pending.candidate + if current is not None and ( + current.state != candidate.state + or current.root_x != candidate.root_x + or current.root_y != candidate.root_y + or current.injected != candidate.injected + ): + raise InputObserverError( + "X RECORD delivered conflicting event-time state for the same " + f"device event from clients {current.id_base:#x} and {id_base:#x}" + ) + if current is None: + pending.candidate = candidate + + def _finalize_pending(self) -> None: + pending = self._pending + if pending is None: + return + self._pending = None + candidate = pending.candidate + if candidate is None: + # RECORD guarantees delivered copies precede the next device event + # only in the absence of grabs. Once one copy is late or absent, + # a later identical (type, time, detail) tuple could be stale. + # Permanently stop correlating rather than risk attaching it to a + # newer physical event. + self._delivered_correlation_uncertain = True + injected = pending.injected or ( + candidate.injected if candidate is not None else False + ) + if pending.event_type in {_KEY_PRESS, _KEY_RELEASE}: + pressed = pending.event_type == _KEY_PRESS + if candidate is None: + self._text_state_uncertain = True + if self._compose_state is not None: + self._xkbcommon.xkb_compose_state_reset(self._compose_state) + self._emit( + normalize_xinput_key_event( + keycode=pending.detail, + pressed=pressed, + keysym_name=None, + injected=injected, + character=None, + derive_character=False, + timestamp=pending.receipt_timestamp, + ) + ) + return + keysym, keysym_name = self._lookup_keysym( + pending.detail, + candidate.state, + ) + self._emit( + normalize_xinput_key_event( + keycode=pending.detail, + pressed=pressed, + keysym_name=keysym_name, + injected=injected, + character=self._resolved_character( + keycode=pending.detail, + keysym=keysym, + keysym_name=keysym_name, + pressed=pressed, + ), + derive_character=False, + timestamp=pending.receipt_timestamp, + ) + ) + return + + if candidate is not None: + position = (candidate.root_x, candidate.root_y) + self._last_pointer = position + else: + position = self._last_pointer + if position is None: raise InputObserverError( - "XKB could not read the active keyboard group and modifier state" + "X RECORD delivered a button event without event-time coordinates " + "before any exact pointer position was observed" ) - core_state = int(state.lookup_mods) | (int(state.group) << _XKB_GROUP_SHIFT) + event = normalize_xinput_button_event( + detail=pending.detail, + pressed=pending.event_type == _BUTTON_PRESS, + x=position[0], + y=position[1], + injected=injected, + timestamp=pending.receipt_timestamp, + ) + if event is not None: + self._emit(event) + + def _finalize_expired_pending(self) -> None: + pending = self._pending + if pending is not None and time.monotonic() >= pending.deadline: + self._finalize_pending() + + def _lookup_keysym( + self, + keycode: int, + event_state: int, + ) -> tuple[int, str | None]: + """Resolve a key using the state carried by its delivered wire event.""" consumed_modifiers = ctypes.c_uint() keysym = ctypes.c_ulong() if not self._x11.XkbLookupKeySym( - self._display, + self._control_display, keycode, - core_state, + event_state, ctypes.byref(consumed_modifiers), ctypes.byref(keysym), ): raise InputObserverError( - f"XKB could not resolve keycode {keycode} in active group " - f"{int(state.group)}" + f"XKB could not resolve keycode {keycode} from its event-time state" ) - name_ptr = self._x11.XKeysymToString(keysym.value) if keysym.value else None - name = name_ptr.decode(errors="replace") if name_ptr else None + name_pointer = ( + self._x11.XKeysymToString(keysym.value) if keysym.value else None + ) + name = name_pointer.decode(errors="replace") if name_pointer else None return int(keysym.value), name + def _compose_character(self, keysym: int) -> tuple[bool, str | None]: + """Return whether compose consumed this press and any committed text.""" + self._xkbcommon.xkb_compose_state_feed(self._compose_state, keysym) + status = int( + self._xkbcommon.xkb_compose_state_get_status(self._compose_state) + ) + if status == _XKB_COMPOSE_COMPOSING: + return True, None + if status == _XKB_COMPOSE_COMPOSED: + buffer = ctypes.create_string_buffer(64) + length = int( + self._xkbcommon.xkb_compose_state_get_utf8( + self._compose_state, + buffer, + len(buffer), + ) + ) + self._xkbcommon.xkb_compose_state_reset(self._compose_state) + if length <= 0: + return True, None + return True, buffer.value.decode("utf-8", errors="strict") + if status == _XKB_COMPOSE_CANCELLED: + self._xkbcommon.xkb_compose_state_reset(self._compose_state) + return True, None + return False, None + def _keysym_character(self, keysym: int) -> str | None: buffer = ctypes.create_string_buffer(64) length = int( @@ -605,6 +979,8 @@ def _resolved_character( keysym_name: str | None, pressed: bool, ) -> str | None: + if self._text_state_uncertain: + return None if not pressed and keycode in self._unverifiable_keycodes: self._unverifiable_keycodes.discard(keycode) return None @@ -646,9 +1022,6 @@ def _resolved_character( self._unverifiable_keycodes.add(keycode) self._composition_remaining -= 1 if self._composition_remaining <= 0: - # This fallback is only reachable without initialized - # production compose state. It never guesses sequence - # text and bounds how long later unrelated text is hidden. self._composition_mode = None self._composition_remaining = 0 return None @@ -656,94 +1029,58 @@ def _resolved_character( return None return self._keysym_character(keysym) - def _handle_raw_event(self, raw: _XIRawEvent) -> None: - injected = bool(raw.send_event) - observed_at = time.time() - if raw.evtype == _XI_RAW_MOTION: - if self.observe_mouse and self.capture_mouse_moves: - x, y = self._query_pointer() - self._emit( - ObservedMouseMove( - x=x, - y=y, - injected=injected, - timestamp=observed_at, - ) - ) - return - if raw.evtype in {_XI_RAW_BUTTON_PRESS, _XI_RAW_BUTTON_RELEASE}: - if self.observe_mouse: - x, y = self._query_pointer() - event = normalize_xinput_button_event( - detail=raw.detail, - pressed=raw.evtype == _XI_RAW_BUTTON_PRESS, - x=x, - y=y, - injected=injected, - timestamp=observed_at, - ) - if event is not None: - self._emit(event) - return - if raw.evtype in {_XI_RAW_KEY_PRESS, _XI_RAW_KEY_RELEASE}: - if self.observe_keyboard: - pressed = raw.evtype == _XI_RAW_KEY_PRESS - keysym, keysym_name = self._lookup_keysym(raw.detail) - self._emit( - normalize_xinput_key_event( - keycode=raw.detail, - pressed=pressed, - keysym_name=keysym_name, - injected=injected, - character=self._resolved_character( - keycode=raw.detail, - keysym=keysym, - keysym_name=keysym_name, - pressed=pressed, - ), - derive_character=False, - timestamp=observed_at, - ) - ) + def _raise_callback_failure(self) -> None: + failure = self._record_callback_failure + if failure is not None: + raise failure def _run_loop(self) -> None: while not self._stop_requested.is_set(): - processed = 0 - while ( - not self._stop_requested.is_set() - and processed < _MAX_PENDING_BATCH - and self._x11.XPending(self._display) - ): - processed += 1 - event = _XEvent() - self._x11.XNextEvent(self._display, ctypes.byref(event)) - cookie = event.xcookie - if ( - event.type != _GENERIC_EVENT - or cookie.extension != self._xi_opcode - ): - continue - if not self._x11.XGetEventData( - self._display, ctypes.byref(event.xcookie) - ): - raise InputObserverError( - "XInput2 event cookie had no accessible event data" - ) - try: - if event.xcookie.data: - raw = ctypes.cast( - event.xcookie.data, ctypes.POINTER(_XIRawEvent) - ).contents - self._handle_raw_event(raw) - finally: - self._x11.XFreeEventData( - self._display, ctypes.byref(event.xcookie) - ) - self._stop_requested.wait(0.01) + self._xtst.XRecordProcessReplies(self._data_display) + self._raise_callback_failure() + self._finalize_expired_pending() + self._stop_requested.wait(_LOOP_INTERVAL_SECONDS) + self._raise_callback_failure() + self._finalize_pending() def _teardown(self) -> None: - if self._display and self._x11 is not None: - self._x11.XCloseDisplay(self._display) + cleanup_failure: InputObserverError | None = None + if ( + self._record_enabled + and self._record_context + and self._control_display + and self._xtst is not None + ): + if not self._xtst.XRecordDisableContext( + self._control_display, + self._record_context, + ): + cleanup_failure = InputObserverError( + "X RECORD context could not be disabled cleanly" + ) + self._record_enabled = False + if ( + self._record_context + and self._control_display + and self._xtst is not None + ): + if not self._xtst.XRecordFreeContext( + self._control_display, + self._record_context, + ) and cleanup_failure is None: + cleanup_failure = InputObserverError( + "X RECORD context could not be freed cleanly" + ) + self._record_context = 0 + if self._record_range is not None and self._x11 is not None: + self._x11.XFree(self._record_range) + self._record_range = None + if self._data_display and self._x11 is not None: + self._x11.XCloseDisplay(self._data_display) + if self._control_display and self._x11 is not None: + self._x11.XCloseDisplay(self._control_display) + self._data_display = None + self._control_display = None if self._xkbcommon is not None: if self._compose_state: self._xkbcommon.xkb_compose_state_unref(self._compose_state) @@ -754,11 +1091,23 @@ def _teardown(self) -> None: self._compose_state = None self._compose_table = None self._compose_context = None - self._display = None - self._event_mask_buffer = None + self._record_callback = None + self._record_callback_failure = None + self._record_started = False + self._accepting_events = False + self._waiting_baseline_marker = False + self._baseline_marker_seen = False + self._control_id_base = 0 + self._root = 0 + self._pending = None + self._delivered_correlation_uncertain = False + self._last_pointer = None + self._text_state_uncertain = False self._composition_mode = None self._composition_remaining = 0 self._unverifiable_keycodes.clear() + if cleanup_failure is not None: + raise cleanup_failure __all__ = [ diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 8411854..233016c 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -1,17 +1,25 @@ -"""Linux XInput2 ABI, XKB text semantics, and shutdown contracts.""" +"""Linux X RECORD ABI, event-time state, and fail-closed correlation tests.""" from __future__ import annotations import ctypes +import sys import pytest -from openadapt_capture.input_observer import ObservedKey +from openadapt_capture.input_observer import ( + InputObserverError, + ObservedKey, + ObservedMouseButton, + ObservedMouseMove, +) from openadapt_capture.input_observer import linux as linux_module from openadapt_capture.input_observer.linux import ( LinuxXInputObserver, - _XIRawEvent, - _XkbStateRec, + _CoreWireEvent, + _decode_core_event, + _XRecordInterceptData, + _XRecordRange, ) @@ -19,26 +27,13 @@ class FakeX11: def __init__( self, *, - group: int = 0, - lookup_mods: int = 0, keysym: int = 0x61, keysym_name: str = "a", ) -> None: - self.group = group - self.lookup_mods = lookup_mods self.keysym = keysym self.keysym_name = keysym_name self.lookup_states: list[int] = [] - def XkbGetState(self, _display, _device, state_pointer) -> int: - state = ctypes.cast( - state_pointer, - ctypes.POINTER(_XkbStateRec), - ).contents - state.group = self.group - state.lookup_mods = self.lookup_mods - return 0 - def XkbLookupKeySym( self, _display, @@ -61,6 +56,7 @@ def XKeysymToString(self, _keysym): class FakeXkbCommon: def __init__(self, text_by_keysym: dict[int, str]) -> None: self.text_by_keysym = text_by_keysym + self.reset_count = 0 def xkb_keysym_to_utf8(self, keysym, buffer, size) -> int: encoded = self.text_by_keysym.get(int(keysym), "").encode() @@ -70,6 +66,9 @@ def xkb_keysym_to_utf8(self, keysym, buffer, size) -> int: ctypes.memmove(buffer, encoded + b"\0", len(encoded) + 1) return len(encoded) + 1 + def xkb_compose_state_reset(self, _state) -> None: + self.reset_count += 1 + class FakeComposeXkbCommon(FakeXkbCommon): def __init__(self) -> None: @@ -82,7 +81,6 @@ def __init__(self) -> None: ] ) self.status = 0 - self.reset_count = 0 def xkb_compose_state_feed(self, _state, _keysym) -> int: self.status = next(self.statuses) @@ -97,9 +95,6 @@ def xkb_compose_state_get_utf8(self, _state, buffer, size) -> int: ctypes.memmove(buffer, encoded + b"\0", len(encoded) + 1) return len(encoded) - def xkb_compose_state_reset(self, _state) -> None: - self.reset_count += 1 - def make_observer() -> LinuxXInputObserver: return LinuxXInputObserver( @@ -111,42 +106,126 @@ def make_observer() -> LinuxXInputObserver: ) +def wire_event( + *, + event_type: int, + detail: int, + event_time: int, + state: int = 0, + root_x: int = 0, + root_y: int = 0, + swapped: bool = False, +) -> bytes: + byteorder = ( + "big" + if (sys.byteorder == "little") == swapped + else "little" + ) + data = bytearray(32) + data[0] = event_type + data[1] = detail + data[4:8] = event_time.to_bytes(4, byteorder) + data[8:12] = (0x1234).to_bytes(4, byteorder) + data[20:22] = root_x.to_bytes(2, byteorder, signed=True) + data[22:24] = root_y.to_bytes(2, byteorder, signed=True) + data[28:30] = state.to_bytes(2, byteorder) + return bytes(data) + + @pytest.mark.skipif( ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_ulong) != 8, - reason="ABI offsets below describe Linux's 64-bit LP64 data model", + reason="ABI offsets below describe X11's 64-bit LP64 data model", ) -def test_xinput_raw_event_matches_primary_header_lp64_abi() -> None: - assert ctypes.sizeof(_XIRawEvent) == 96 - assert _XIRawEvent.flags.offset == 60 - assert _XIRawEvent.valuators.offset == 64 - assert _XIRawEvent.raw_values.offset == 88 +def test_xrecord_structures_match_primary_header_lp64_abi() -> None: + assert ctypes.sizeof(_XRecordRange) == 32 + assert _XRecordRange.delivered_events.offset == 16 + assert _XRecordRange.device_events.offset == 18 + assert _XRecordRange.client_started.offset == 24 + assert ctypes.sizeof(_XRecordInterceptData) == 48 + assert _XRecordInterceptData.category.offset == 24 + assert _XRecordInterceptData.data.offset == 32 + assert _XRecordInterceptData.data_len.offset == 40 + + +@pytest.mark.parametrize("swapped", [False, True]) +def test_core_wire_event_decodes_recorded_client_byte_order(swapped: bool) -> None: + decoded = _decode_core_event( + wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=0x11223344, + state=0x4282, + root_x=-120, + root_y=735, + swapped=swapped, + ), + client_swapped=swapped, + ) + assert decoded.event_type == linux_module._KEY_PRESS + assert decoded.detail == 38 + assert decoded.time == 0x11223344 + assert decoded.state == 0x4282 + assert (decoded.root_x, decoded.root_y) == (-120, 735) -def test_xkb_lookup_uses_active_group_caps_and_altgr_state() -> None: + +def test_control_reply_marker_discards_pre_boundary_input_then_arms( + monkeypatch: pytest.MonkeyPatch, +) -> None: observer = make_observer() - x11 = FakeX11( - group=2, - lookup_mods=0x82, - keysym=0x20AC, - keysym_name="EuroSign", + observer._waiting_baseline_marker = True + observer._control_id_base = 0x400000 + freed: list[int] = [] + + class FakeXtst: + def XRecordFreeData(self, pointer) -> None: + freed.append(ctypes.addressof(pointer.contents)) + + observer._xtst = FakeXtst() + monkeypatch.setattr(linux_module.time, "time", lambda: 10.0) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 20.0) + + def intercept(payload: bytes, *, id_base: int) -> None: + buffer = (ctypes.c_ubyte * len(payload)).from_buffer_copy(payload) + recorded = _XRecordInterceptData( + id_base=id_base, + category=linux_module._XRECORD_FROM_SERVER, + data=ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ubyte)), + data_len=len(payload) // 4, + ) + observer._record_intercept(None, ctypes.pointer(recorded)) + + key = wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=7, ) - observer._display = object() + intercept(key, id_base=0) + assert observer._pending is None + assert not observer._accepting_events + + marker = bytearray(32) + marker[0] = 1 + intercept(bytes(marker), id_base=0x400000) + assert observer._baseline_marker_seen + assert observer._accepting_events + + intercept(key, id_base=0) + assert observer._pending is not None + assert observer._pending.detail == 38 + assert len(freed) == 3 + + +def test_xkb_lookup_uses_state_from_delivered_event() -> None: + observer = make_observer() + x11 = FakeX11(keysym=0x20AC, keysym_name="EuroSign") + observer._control_display = object() observer._x11 = x11 - observer._xkbcommon = FakeXkbCommon({0x20AC: "€"}) - keysym, name = observer._lookup_keysym(26) + keysym, name = observer._lookup_keysym(26, 0x4282) assert (keysym, name) == (0x20AC, "EuroSign") - assert x11.lookup_states == [0x82 | (2 << linux_module._XKB_GROUP_SHIFT)] - assert ( - observer._resolved_character( - keycode=26, - keysym=keysym, - keysym_name=name, - pressed=True, - ) - == "€" - ) + assert x11.lookup_states == [0x4282] def test_dead_key_and_resolving_key_are_text_unverifiable() -> None: @@ -257,58 +336,409 @@ def test_locale_compose_state_commits_exact_text_then_resumes_normal_text() -> N ) -def test_raw_event_stamps_receipt_time_before_delivery( - monkeypatch, +def test_device_and_delivered_key_correlate_with_event_time_state( + monkeypatch: pytest.MonkeyPatch, ) -> None: events = [] observer = make_observer() observer._emit = events.append # type: ignore[method-assign] - observer._lookup_keysym = lambda _keycode: (0x61, "a") # type: ignore[method-assign] - observer._xkbcommon = FakeXkbCommon({0x61: "a"}) + x11 = FakeX11(keysym=0x20AC, keysym_name="EuroSign") + observer._x11 = x11 + observer._control_display = object() + observer._xkbcommon = FakeXkbCommon({0x20AC: "€"}) monkeypatch.setattr(linux_module.time, "time", lambda: 321.5) - raw = _XIRawEvent( - evtype=linux_module._XI_RAW_KEY_PRESS, - detail=38, + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 100.0) + + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 26, + False, + 900, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) + observer._handle_delivered_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 26, + False, + 900, + 0, + 0, + 0, + 50, + 60, + 0, + 0, + 0x4282, + ), + id_base=0x400000, + ) + observer._finalize_pending() + + assert events == [ + ObservedKey( + pressed=True, + key_char="€", + key_vk="26", + canonical_key_char="€", + canonical_key_vk="26", + timestamp=321.5, + ) + ] + assert x11.lookup_states == [0x4282] + + +def test_unmatched_key_keeps_physical_identity_and_suppresses_later_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._xkbcommon = FakeXkbCommon({0x61: "a"}) + observer._compose_state = object() + monkeypatch.setattr(linux_module.time, "time", lambda: 10.5) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 20.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) ) - observer._handle_raw_event(raw) + observer._finalize_pending() assert events == [ ObservedKey( pressed=True, - key_char="a", + key_name="keycode_38", key_vk="38", - canonical_key_char="a", + canonical_key_name="keycode_38", canonical_key_vk="38", - timestamp=321.5, + timestamp=10.5, ) ] + assert observer._text_state_uncertain + assert ( + observer._resolved_character( + keycode=38, + keysym=0x61, + keysym_name="a", + pressed=True, + ) + is None + ) -def test_continuous_pending_events_cannot_prevent_stop() -> None: +def test_motion_uses_event_time_root_coordinates_without_pointer_query( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + monkeypatch.setattr(linux_module.time, "time", lambda: 44.0) + + observer._handle_device_event( + _CoreWireEvent( + linux_module._MOTION_NOTIFY, + 0, + False, + 2, + 0, + 0, + 0, + -120, + 735, + 0, + 0, + 0, + ) + ) - class AlwaysPendingX11: - def __init__(self) -> None: - self.next_count = 0 + assert observer._last_pointer == (-120.0, 735.0) + assert events == [ + ObservedMouseMove(x=-120.0, y=735.0, timestamp=44.0) + ] - def XPending(self, _display) -> int: + +def test_unmatched_button_uses_last_exact_recorded_pointer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._last_pointer = (12.0, 34.0) + monkeypatch.setattr(linux_module.time, "time", lambda: 70.0) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 80.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._BUTTON_PRESS, + 1, + False, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) + + observer._finalize_pending() + + assert events == [ + ObservedMouseButton( + x=12.0, + y=34.0, + button="left", + pressed=True, + timestamp=70.0, + ) + ] + + +def test_stationary_first_click_uses_pre_ready_baseline_without_resampling( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._control_display = object() + observer._root = 99 + + class BaselineX11: + def __init__(self) -> None: + self.query_count = 0 + + def XQueryPointer( + self, + _display, + _root, + _root_return, + _child_return, + root_x_pointer, + root_y_pointer, + _window_x_pointer, + _window_y_pointer, + _mask_pointer, + ) -> int: + self.query_count += 1 + ctypes.cast(root_x_pointer, ctypes.POINTER(ctypes.c_int)).contents.value = 41 + ctypes.cast(root_y_pointer, ctypes.POINTER(ctypes.c_int)).contents.value = 73 return 1 - def XNextEvent(self, _display, event_pointer) -> None: - self.next_count += 1 - event = ctypes.cast( - event_pointer, - ctypes.POINTER(linux_module._XEvent), - ).contents - event.type = 0 - if self.next_count == 5: - observer._stop_requested.set() - - x11 = AlwaysPendingX11() + x11 = BaselineX11() observer._x11 = x11 - observer._display = object() + observer._query_pointer_baseline() + observer._accepting_events = True + monkeypatch.setattr(linux_module.time, "time", lambda: 123.0) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 80.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._BUTTON_PRESS, + 1, + False, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) + observer._finalize_pending() + + assert x11.query_count == 1 + assert events == [ + ObservedMouseButton( + x=41.0, + y=73.0, + button="left", + pressed=True, + timestamp=123.0, + ) + ] + + +def test_unmatched_button_without_exact_position_fails_loud( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = make_observer() + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 80.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._BUTTON_PRESS, + 1, + False, + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) - observer._run_loop() + with pytest.raises(InputObserverError, match="without event-time coordinates"): + observer._finalize_pending() + + +def test_conflicting_delivered_candidates_fail_instead_of_choosing_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = make_observer() + monkeypatch.setattr(linux_module.time, "monotonic", lambda: 1.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) + observer._handle_delivered_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 1, + ), + id_base=0x200000, + ) + + with pytest.raises(InputObserverError, match="conflicting event-time state"): + observer._handle_delivered_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 5, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 2, + ), + id_base=0x400000, + ) + + +def test_new_device_finalizes_prior_event_before_it() -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._last_pointer = (10.0, 20.0) + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 10, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) + observer._handle_device_event( + _CoreWireEvent( + linux_module._MOTION_NOTIFY, + 0, + False, + 11, + 0, + 0, + 0, + 30, + 40, + 0, + 0, + 0, + ) + ) + + assert isinstance(events[0], ObservedKey) + assert isinstance(events[1], ObservedMouseMove) + assert (events[1].x, events[1].y) == (30.0, 40.0) + + +def test_idle_timeout_finalizes_unmatched_event( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + moments = iter([1.0, 1.05, 1.11]) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: next(moments)) + observer._handle_device_event( + _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 12, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ) + ) - assert x11.next_count == 5 + observer._finalize_expired_pending() + assert events == [] + observer._finalize_expired_pending() + assert len(events) == 1 + assert isinstance(events[0], ObservedKey) From ac5e395505af7ce0e58448e5e4690222af5a8cdb Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:28:17 -0700 Subject: [PATCH 20/26] test: remove obsolete Linux raw-event ABI contract --- tests/test_input_observer.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 5097f57..2ad6147 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -2,7 +2,6 @@ from __future__ import annotations -import ctypes import queue import sys import threading @@ -28,7 +27,6 @@ ) from openadapt_capture.input_observer.linux import ( LinuxXInputObserver, - _XIRawEvent, normalize_xinput_button_event, normalize_xinput_key_event, ) @@ -531,17 +529,6 @@ def test_auxiliary_mouse_button_survives_storage_conversion_and_processing( assert processed[0].button == button -@pytest.mark.skipif( - ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_ulong) != 8, - reason="ABI offsets below describe Linux's 64-bit LP64 data model", -) -def test_xinput_raw_event_matches_64_bit_libxi_abi() -> None: - assert ctypes.sizeof(_XIRawEvent) == 96 - assert _XIRawEvent.flags.offset == 60 - assert _XIRawEvent.valuators.offset == 64 - assert _XIRawEvent.raw_values.offset == 88 - - def test_wayland_refuses_instead_of_silently_observing_only_xwayland( monkeypatch: pytest.MonkeyPatch, ) -> None: From 16769552fa9cc209cbfa5cb87c3c6350349631c0 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:33:28 -0700 Subject: [PATCH 21/26] test: document X RECORD event ordering --- openadapt_capture/input_observer/linux.py | 18 +++-- tests/test_input_observer_linux_xkb.py | 87 +++++++++++++++++++++++ 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 613fe04..8a1d33e 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -1,11 +1,13 @@ """Fail-closed Linux global input observation through the X RECORD extension. -The RECORD specification deliberately leaves most fields in an intercepted -``device_event`` undefined: key/button events guarantee only ``time`` and -``detail``; motion additionally guarantees root coordinates. This observer -therefore never samples current X state after an event. It preserves the -globally ordered device stream and enriches key/button events only from the -matching delivered core event, whose wire record carries event-time state and +The normative RECORD protocol guarantees exactly one selected ``device_event`` +for every generated device event, whether or not any client receives it, in +device generation order. Each delivered copy is a separate record after that +device event. Device key/button records deliberately guarantee only ``time`` +and ``detail``; device motion additionally guarantees root coordinates. This +observer therefore never samples current X state after an event. It preserves +the global device stream and enriches key/button events only from matching +delivered core events, whose wire records carry event-time state and coordinates. Missing enrichment degrades keyboard text to physical identity and makes pointer actions fail closed unless an exact earlier pointer position is known. @@ -712,6 +714,10 @@ def _record_intercept( # before our recorded QueryPointer reply is discarded. return if int(recorded.id_base) == 0: + # Normative RECORD semantics make this the one global record + # for every generated device event. Zero does not mean the + # event was necessarily undelivered; any delivered copies + # follow separately with their target client's id-base. if recorded.client_swapped: raise InputObserverError( "X RECORD marked a device event as byte-swapped; " diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 233016c..d59a690 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -397,6 +397,93 @@ def test_device_and_delivered_key_correlate_with_event_time_state( assert x11.lookup_states == [0x4282] +def test_normative_device_then_duplicate_deliveries_then_next_device_order( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + x11 = FakeX11(keysym=0x61, keysym_name="a") + observer._x11 = x11 + observer._control_display = object() + observer._xkbcommon = FakeXkbCommon({0x61: "a"}) + wall_times = iter([100.0, 101.0]) + monotonic_times = iter([200.0, 201.0]) + monkeypatch.setattr(linux_module.time, "time", lambda: next(wall_times)) + monkeypatch.setattr( + linux_module.time, + "monotonic", + lambda: next(monotonic_times), + ) + + press = _CoreWireEvent( + linux_module._KEY_PRESS, + 38, + False, + 900, + 0, + 0, + 0, + 50, + 60, + 0, + 0, + 0x0001, + ) + observer._handle_device_event(press) + observer._handle_delivered_event(press, id_base=0x200000) + observer._handle_delivered_event(press, id_base=0x400000) + assert events == [] + + release = _CoreWireEvent( + linux_module._KEY_RELEASE, + 38, + False, + 901, + 0, + 0, + 0, + 50, + 60, + 0, + 0, + 0x0001, + ) + observer._handle_device_event(release) + assert len(events) == 1 + assert events[0] == ObservedKey( + pressed=True, + key_char="a", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + timestamp=100.0, + ) + + observer._handle_delivered_event(release, id_base=0x200000) + observer._finalize_pending() + + assert events == [ + ObservedKey( + pressed=True, + key_char="a", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + timestamp=100.0, + ), + ObservedKey( + pressed=False, + key_char="a", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + timestamp=101.0, + ), + ] + assert x11.lookup_states == [0x0001, 0x0001] + + def test_unmatched_key_keeps_physical_identity_and_suppresses_later_text( monkeypatch: pytest.MonkeyPatch, ) -> None: From 7444be1ebe500e0a4063068db1700da56c006330 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:36:35 -0700 Subject: [PATCH 22/26] ci: gate capture publication on green main --- .github/workflows/release.yml | 134 +++++++++++++++++++++++++++++----- 1 file changed, 115 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 130e620..716e494 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,60 +1,155 @@ name: Release and PyPI Publish +# Publication is a deliberate operation from protected main. Ordinary merges +# run Tests only; this workflow refuses to version or publish until that exact +# main commit has completed the full test.yml workflow (including the package +# archive contract). +# +# Requires ADMIN_TOKEN (GitHub PAT with repo scope) to push the release commit +# and tag through branch protection. PyPI publication uses Trusted Publishing. + on: - push: - branches: - - main + workflow_dispatch: + inputs: + operation: + description: "Release operation" + type: choice + required: true + default: semantic-release + options: + - semantic-release + +concurrency: + group: release + cancel-in-progress: false jobs: - release: + semantic-release: + name: Version, tag, and publish the green main release train + if: >- + github.event_name == 'workflow_dispatch' && + inputs.operation == 'semantic-release' && + github.ref == 'refs/heads/main' runs-on: ubuntu-latest - concurrency: release permissions: + actions: read id-token: write contents: write + issues: write steps: - - name: Checkout repository - uses: actions/checkout@v7 + - name: Checkout protected main at the dispatched commit + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 token: ${{ secrets.ADMIN_TOKEN }} - - name: Check if should skip + - name: Refuse a release commit replay id: check_skip + shell: bash + run: | + set -euo pipefail + author="$(git log -1 --pretty=format:'%an')" + subject="$(git log -1 --pretty=format:'%s')" + if [ "${author}" = "semantic-release" ] || printf '%s' "${subject}" | grep -q '^chore: release'; then + echo "skip=true" >> "${GITHUB_OUTPUT}" + fi + + - name: Wait for exact-head Tests workflow + if: steps.check_skip.outputs.skip != 'true' + env: + GH_TOKEN: ${{ github.token }} + shell: bash run: | - if [ "$(git log -1 --pretty=format:'%an')" = "semantic-release" ]; then - echo "skip=true" >> $GITHUB_OUTPUT + set -euo pipefail + deadline=$((SECONDS + 2700)) + while [ "${SECONDS}" -lt "${deadline}" ]; do + response="$( + gh api --method GET \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/test.yml/runs" \ + --raw-field head_sha="${GITHUB_SHA}" \ + --raw-field event=push \ + --raw-field per_page=20 + )" + status="$( + printf '%s' "${response}" | + jq -r --arg sha "${GITHUB_SHA}" \ + '[.workflow_runs[] + | select(.head_sha == $sha and .event == "push")] + | sort_by(.created_at) | last | .status // "missing"' + )" + conclusion="$( + printf '%s' "${response}" | + jq -r --arg sha "${GITHUB_SHA}" \ + '[.workflow_runs[] + | select(.head_sha == $sha and .event == "push")] + | sort_by(.created_at) | last | .conclusion // "pending"' + )" + echo "test.yml for ${GITHUB_SHA}: ${status}/${conclusion}" + if [ "${status}" = "completed" ] && [ "${conclusion}" != "success" ]; then + echo "Refusing to publish because test.yml concluded ${conclusion}." + exit 1 + fi + if [ "${status}" = "completed" ] && [ "${conclusion}" = "success" ]; then + break + fi + sleep 10 + done + if [ "${status}" != "completed" ] || [ "${conclusion}" != "success" ]; then + echo "Refusing to publish: exact-head test.yml did not succeed within 45 minutes." + exit 1 + fi + + - name: Require dispatched head to remain current protected main + if: steps.check_skip.outputs.skip != 'true' + shell: bash + run: | + set -euo pipefail + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + current_main="$(git rev-parse refs/remotes/origin/main)" + if [ "${current_main}" != "${GITHUB_SHA}" ]; then + echo "Refusing stale release dispatch: main is ${current_main}, dispatched head is ${GITHUB_SHA}." + exit 1 fi - name: Set up Python if: steps.check_skip.outputs.skip != 'true' - uses: actions/setup-python@v5 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: - python-version: '3.12' + python-version: "3.12" - - name: Install uv + - name: Install exact uv if: steps.check_skip.outputs.skip != 'true' - uses: astral-sh/setup-uv@v7 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + with: + version: "0.11.29" - name: Python Semantic Release if: steps.check_skip.outputs.skip != 'true' id: release - uses: python-semantic-release/python-semantic-release@v10.6.1 + uses: python-semantic-release/python-semantic-release@39dd2052f2ce8282a5d932c31d58a2ca06d2550e # v10.6.1 with: github_token: ${{ secrets.ADMIN_TOKEN }} - - name: Build package + # Do not trust the semantic-release action's transient build directory. + # Rebuild the versioned tree and re-run the exact archive boundary guard + # before either external publication channel sees an artifact. + - name: Rebuild and verify release artifacts if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true' - run: uv build + shell: bash + run: | + set -euo pipefail + rm -rf dist + uv build --wheel --sdist + python scripts/verify_distribution.py dist/* - name: Publish to PyPI if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true' - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 - name: Publish to GitHub Releases if: steps.check_skip.outputs.skip != 'true' && steps.release.outputs.released == 'true' - uses: python-semantic-release/publish-action@v9.15.2 + uses: python-semantic-release/publish-action@5a5718ce47b892ef699f2972dae122297771d641 # v10.6.1 with: github_token: ${{ secrets.ADMIN_TOKEN }} @@ -64,6 +159,7 @@ jobs: if: failure() env: GH_TOKEN: ${{ secrets.ADMIN_TOKEN }} + shell: bash run: | TITLE="Release workflow failed on main" BODY="The release workflow failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} From 02461fec6e525e5f1afbcc2d58621d813b3c506f Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:45:53 -0700 Subject: [PATCH 23/26] fix: drain Linux recording tail on shutdown --- openadapt_capture/input_observer/linux.py | 153 +++++++++++-- tests/test_input_observer_linux_xkb.py | 266 ++++++++++++++++++++++ 2 files changed, 398 insertions(+), 21 deletions(-) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 8a1d33e..37c14f1 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from typing import Any +from openadapt_capture.x11_threads import ensure_xlib_thread_support + from .base import ( InputObserverError, InputObserverPermissionError, @@ -33,6 +35,7 @@ ObservedMouseMove, ObservedMouseScroll, ThreadedInputObserver, + add_exception_note, ) _KEY_PRESS = 2 @@ -352,6 +355,7 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._record_context = 0 self._record_enabled = False self._record_started = False + self._record_ended = False self._accepting_events = False self._waiting_baseline_marker = False self._baseline_marker_seen = False @@ -388,6 +392,12 @@ def _setup(self) -> None: "X RECORD requires an X11 desktop and DISPLAY is not set" ) + # The factory normally performs this process-wide initialization, but + # the observer is also a public class and may be constructed directly. + # XInitThreads must precede every other Xlib call or library-backed + # capture may be unsafe once screen and input threads run concurrently. + ensure_xlib_thread_support() + x11_name = ctypes.util.find_library("X11") xtst_name = ctypes.util.find_library("Xtst") xkbcommon_name = ctypes.util.find_library("xkbcommon") @@ -682,6 +692,12 @@ def _record_intercept( self._record_started = True return if recorded.category == _XRECORD_END_OF_DATA: + self._record_ended = True + return + if self._record_callback_failure is not None: + # After coverage has failed, drain only to the terminal marker. + # Emitting later records would make a failed recording appear + # partially usable. return if recorded.category != _XRECORD_FROM_SERVER: return @@ -1047,22 +1063,80 @@ def _run_loop(self) -> None: self._finalize_expired_pending() self._stop_requested.wait(_LOOP_INTERVAL_SECONDS) self._raise_callback_failure() + + def _drain_record_tail(self) -> None: + """Drain DisableContext's complete tail through EndOfData, or refuse.""" + cleanup_reserve = min( + 0.5, + max( + 2 * _LOOP_INTERVAL_SECONDS, + self.shutdown_timeout * 0.1, + ), + ) + drain_budget = max(0.0, self.shutdown_timeout - cleanup_reserve) + deadline = time.monotonic() + drain_budget + while not self._record_ended: + self._xtst.XRecordProcessReplies(self._data_display) + if self._record_ended: + break + if time.monotonic() >= deadline: + timeout = InputObserverError( + "X RECORD did not deliver EndOfData within the bounded " + f"{drain_budget:.3f}s shutdown drain" + ) + if self._record_callback_failure is not None: + add_exception_note( + self._record_callback_failure, + str(timeout), + ) + raise self._record_callback_failure + raise timeout + time.sleep( + min( + _LOOP_INTERVAL_SECONDS, + max(0.0, deadline - time.monotonic()), + ) + ) + self._raise_callback_failure() + # RECORD guarantees DisableContext flushes all complete protocol + # elements before EndOfData. Only now is the last device event known + # to have received every possible delivered copy. self._finalize_pending() def _teardown(self) -> None: - cleanup_failure: InputObserverError | None = None + cleanup_failures: list[BaseException] = [] + + def attempt(label: str, operation): + try: + return operation() + except BaseException as exc: + add_exception_note(exc, label) + cleanup_failures.append(exc) + return None + if ( self._record_enabled and self._record_context and self._control_display and self._xtst is not None ): - if not self._xtst.XRecordDisableContext( - self._control_display, - self._record_context, - ): - cleanup_failure = InputObserverError( - "X RECORD context could not be disabled cleanly" + disabled = attempt( + "while disabling the X RECORD context", + lambda: self._xtst.XRecordDisableContext( + self._control_display, + self._record_context, + ), + ) + if disabled: + attempt( + "while draining the X RECORD shutdown tail", + self._drain_record_tail, + ) + elif disabled is not None: + cleanup_failures.append( + InputObserverError( + "X RECORD context could not be disabled cleanly" + ) ) self._record_enabled = False if ( @@ -1070,36 +1144,67 @@ def _teardown(self) -> None: and self._control_display and self._xtst is not None ): - if not self._xtst.XRecordFreeContext( - self._control_display, - self._record_context, - ) and cleanup_failure is None: - cleanup_failure = InputObserverError( - "X RECORD context could not be freed cleanly" + freed_context = attempt( + "while freeing the X RECORD context", + lambda: self._xtst.XRecordFreeContext( + self._control_display, + self._record_context, + ), + ) + if freed_context is not None and not freed_context: + cleanup_failures.append( + InputObserverError( + "X RECORD context could not be freed cleanly" + ) ) self._record_context = 0 if self._record_range is not None and self._x11 is not None: - self._x11.XFree(self._record_range) + attempt( + "while freeing the X RECORD range", + lambda: self._x11.XFree(self._record_range), + ) self._record_range = None if self._data_display and self._x11 is not None: - self._x11.XCloseDisplay(self._data_display) + attempt( + "while closing the X RECORD data display", + lambda: self._x11.XCloseDisplay(self._data_display), + ) if self._control_display and self._x11 is not None: - self._x11.XCloseDisplay(self._control_display) + attempt( + "while closing the X RECORD control display", + lambda: self._x11.XCloseDisplay(self._control_display), + ) self._data_display = None self._control_display = None if self._xkbcommon is not None: if self._compose_state: - self._xkbcommon.xkb_compose_state_unref(self._compose_state) + attempt( + "while releasing the XKB compose state", + lambda: self._xkbcommon.xkb_compose_state_unref( + self._compose_state + ), + ) if self._compose_table: - self._xkbcommon.xkb_compose_table_unref(self._compose_table) + attempt( + "while releasing the XKB compose table", + lambda: self._xkbcommon.xkb_compose_table_unref( + self._compose_table + ), + ) if self._compose_context: - self._xkbcommon.xkb_context_unref(self._compose_context) + attempt( + "while releasing the XKB context", + lambda: self._xkbcommon.xkb_context_unref( + self._compose_context + ), + ) self._compose_state = None self._compose_table = None self._compose_context = None self._record_callback = None self._record_callback_failure = None self._record_started = False + self._record_ended = False self._accepting_events = False self._waiting_baseline_marker = False self._baseline_marker_seen = False @@ -1112,8 +1217,14 @@ def _teardown(self) -> None: self._composition_mode = None self._composition_remaining = 0 self._unverifiable_keycodes.clear() - if cleanup_failure is not None: - raise cleanup_failure + if cleanup_failures: + primary = cleanup_failures[0] + for secondary in cleanup_failures[1:]: + add_exception_note( + primary, + f"additional teardown failure: {secondary}", + ) + raise primary __all__ = [ diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index d59a690..11349f1 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -132,6 +132,52 @@ def wire_event( return bytes(data) +def intercept_record( + observer: LinuxXInputObserver, + *, + category: int, + payload: bytes = b"", + id_base: int = 0, + client_swapped: bool = False, +) -> None: + buffer = (ctypes.c_ubyte * len(payload)).from_buffer_copy(payload) + recorded = _XRecordInterceptData( + id_base=id_base, + category=category, + client_swapped=client_swapped, + data=ctypes.cast(buffer, ctypes.POINTER(ctypes.c_ubyte)), + data_len=len(payload) // 4, + ) + observer._record_intercept(None, ctypes.pointer(recorded)) + + +def test_direct_setup_initializes_xlib_threads_before_loading_x11( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = make_observer() + monkeypatch.setattr(linux_module.sys, "platform", "linux") + + def reject_xlib_threading() -> None: + raise RuntimeError("XInitThreads refused") + + monkeypatch.setattr( + linux_module, + "ensure_xlib_thread_support", + reject_xlib_threading, + ) + monkeypatch.setattr( + linux_module.ctypes.util, + "find_library", + lambda _name: pytest.fail("loaded Xlib before XInitThreads succeeded"), + ) + + with pytest.raises(RuntimeError, match="XInitThreads refused"): + observer._setup() + + assert observer._x11 is None + assert observer._control_display is None + + @pytest.mark.skipif( ctypes.sizeof(ctypes.c_void_p) != 8 or ctypes.sizeof(ctypes.c_ulong) != 8, reason="ABI offsets below describe X11's 64-bit LP64 data model", @@ -829,3 +875,223 @@ def test_idle_timeout_finalizes_unmatched_event( observer._finalize_expired_pending() assert len(events) == 1 assert isinstance(events[0], ObservedKey) + + +def test_shutdown_drain_emits_buffered_tail_through_end_of_data( + monkeypatch: pytest.MonkeyPatch, +) -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._accepting_events = True + observer._x11 = FakeX11(keysym=0x61, keysym_name="a") + observer._control_display = object() + observer._xkbcommon = FakeXkbCommon({0x61: "a"}) + monkeypatch.setattr(linux_module.time, "time", lambda: 51.0) + + class TailXtst: + def __init__(self) -> None: + self.process_count = 0 + self.free_count = 0 + + def XRecordProcessReplies(self, _display) -> None: + self.process_count += 1 + device = wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=500, + ) + delivered = wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=500, + state=1, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=device, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=delivered, + id_base=0x400000, + ) + intercept_record( + observer, + category=linux_module._XRECORD_END_OF_DATA, + ) + + def XRecordFreeData(self, _pointer) -> None: + self.free_count += 1 + + xtst = TailXtst() + observer._xtst = xtst + + observer._drain_record_tail() + + assert observer._record_ended + assert xtst.process_count == 1 + assert xtst.free_count == 3 + assert events == [ + ObservedKey( + pressed=True, + key_char="a", + key_vk="38", + canonical_key_char="a", + canonical_key_vk="38", + timestamp=51.0, + ) + ] + + +def test_shutdown_drain_times_out_without_end_of_data( + monkeypatch: pytest.MonkeyPatch, +) -> None: + observer = make_observer() + observer.shutdown_timeout = 0.05 + process_count = 0 + + class NeverEndsXtst: + def XRecordProcessReplies(self, _display) -> None: + nonlocal process_count + process_count += 1 + + observer._xtst = NeverEndsXtst() + moments = iter([10.0, 10.1]) + monkeypatch.setattr(linux_module.time, "monotonic", lambda: next(moments)) + + with pytest.raises(InputObserverError, match="did not deliver EndOfData"): + observer._drain_record_tail() + + assert process_count == 1 + + +def test_teardown_drains_then_releases_every_native_resource() -> None: + observer = make_observer() + observer.shutdown_timeout = 0.1 + control_display = object() + data_display = object() + record_range = object() + compose_state = object() + compose_table = object() + compose_context = object() + observer._record_enabled = True + observer._record_context = 7 + observer._control_display = control_display + observer._data_display = data_display + observer._record_range = record_range + observer._compose_state = compose_state + observer._compose_table = compose_table + observer._compose_context = compose_context + + class TeardownXtst: + def __init__(self) -> None: + self.calls: list[str] = [] + self.free_data_count = 0 + + def XRecordDisableContext(self, display, context) -> int: + assert display is control_display + assert context == 7 + self.calls.append("disable") + return 1 + + def XRecordProcessReplies(self, display) -> None: + assert display is data_display + self.calls.append("process") + intercept_record( + observer, + category=linux_module._XRECORD_END_OF_DATA, + ) + + def XRecordFreeData(self, _pointer) -> None: + self.free_data_count += 1 + + def XRecordFreeContext(self, display, context) -> int: + assert display is control_display + assert context == 7 + self.calls.append("free_context") + return 1 + + class TeardownX11: + def __init__(self) -> None: + self.freed = [] + self.closed = [] + + def XFree(self, value) -> int: + self.freed.append(value) + return 1 + + def XCloseDisplay(self, value) -> int: + self.closed.append(value) + return 0 + + class TeardownXkb: + def __init__(self) -> None: + self.released = [] + + def xkb_compose_state_unref(self, value) -> None: + self.released.append(("state", value)) + + def xkb_compose_table_unref(self, value) -> None: + self.released.append(("table", value)) + + def xkb_context_unref(self, value) -> None: + self.released.append(("context", value)) + + xtst = TeardownXtst() + x11 = TeardownX11() + xkb = TeardownXkb() + observer._xtst = xtst + observer._x11 = x11 + observer._xkbcommon = xkb + + observer._teardown() + + assert xtst.calls == ["disable", "process", "free_context"] + assert xtst.free_data_count == 1 + assert x11.freed == [record_range] + assert x11.closed == [data_display, control_display] + assert xkb.released == [ + ("state", compose_state), + ("table", compose_table), + ("context", compose_context), + ] + assert observer._record_context == 0 + assert observer._data_display is None + assert observer._control_display is None + + +def test_setup_failure_without_enabled_context_skips_shutdown_drain() -> None: + observer = make_observer() + observer._record_enabled = False + observer._record_context = 7 + observer._control_display = object() + observer._data_display = object() + + class SetupFailureXtst: + def __init__(self) -> None: + self.process_count = 0 + self.free_context_count = 0 + + def XRecordProcessReplies(self, _display) -> None: + self.process_count += 1 + + def XRecordFreeContext(self, _display, _context) -> int: + self.free_context_count += 1 + return 1 + + class SetupFailureX11: + def XCloseDisplay(self, _display) -> int: + return 0 + + xtst = SetupFailureXtst() + observer._xtst = xtst + observer._x11 = SetupFailureX11() + observer._xkbcommon = None + + observer._teardown() + + assert xtst.process_count == 0 + assert xtst.free_context_count == 1 From b638e3675d21d50fc27a34aee871d16586793142 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:51:47 -0700 Subject: [PATCH 24/26] fix: keep failed Linux capture starts transactional --- openadapt_capture/input_observer/linux.py | 9 +++ tests/test_input_observer_linux_xkb.py | 81 +++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 37c14f1..7ddb28d 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -356,6 +356,7 @@ def __init__(self, *args, environ: dict[str, str] | None = None, **kwargs) -> No self._record_enabled = False self._record_started = False self._record_ended = False + self._setup_complete = False self._accepting_events = False self._waiting_baseline_marker = False self._baseline_marker_seen = False @@ -527,6 +528,7 @@ def _setup(self) -> None: raise InputObserverError( "X RECORD observed the pointer-baseline marker without arming input" ) + self._setup_complete = True def _configure_api(self) -> None: self._x11.XOpenDisplay.argtypes = [ctypes.c_char_p] @@ -1114,6 +1116,12 @@ def attempt(label: str, operation): cleanup_failures.append(exc) return None + if self._record_enabled and not self._setup_complete: + # A startup timeout can leave the ordered QueryPointer marker in + # the server tail. Teardown must free that marker and subsequent + # records without letting a failed start arm event acceptance. + self._waiting_baseline_marker = False + self._accepting_events = False if ( self._record_enabled and self._record_context @@ -1205,6 +1213,7 @@ def attempt(label: str, operation): self._record_callback_failure = None self._record_started = False self._record_ended = False + self._setup_complete = False self._accepting_events = False self._waiting_baseline_marker = False self._baseline_marker_seen = False diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 11349f1..6fb7aa5 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -1095,3 +1095,84 @@ def XCloseDisplay(self, _display) -> int: assert xtst.process_count == 0 assert xtst.free_context_count == 1 + + +def test_failed_start_drain_never_arms_or_emits_buffered_input() -> None: + events = [] + observer = make_observer() + observer._emit = events.append # type: ignore[method-assign] + observer._record_enabled = True + observer._record_context = 7 + observer._control_display = object() + observer._data_display = object() + observer._control_id_base = 0x400000 + observer._waiting_baseline_marker = True + observer._accepting_events = False + observer._setup_complete = False + + class FailedStartXtst: + def __init__(self) -> None: + self.free_data_count = 0 + + def XRecordDisableContext(self, _display, _context) -> int: + return 1 + + def XRecordProcessReplies(self, _display) -> None: + marker = bytearray(32) + marker[0] = 1 + device = wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=500, + ) + delivered = wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=500, + state=1, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=bytes(marker), + id_base=observer._control_id_base, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=device, + ) + intercept_record( + observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=delivered, + id_base=0x800000, + ) + intercept_record( + observer, + category=linux_module._XRECORD_END_OF_DATA, + ) + + def XRecordFreeData(self, _pointer) -> None: + self.free_data_count += 1 + + def XRecordFreeContext(self, _display, _context) -> int: + return 1 + + class FailedStartX11: + def XCloseDisplay(self, _display) -> int: + return 0 + + xtst = FailedStartXtst() + observer._xtst = xtst + observer._x11 = FailedStartX11() + observer._xkbcommon = None + + observer._teardown() + + assert events == [] + assert xtst.free_data_count == 4 + assert not observer._setup_complete + assert not observer._baseline_marker_seen + assert not observer._waiting_baseline_marker + assert not observer._accepting_events From f463fc008a57d33d963c2a7de9956cd620b8bc6d Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 14:58:18 -0700 Subject: [PATCH 25/26] fix: cancel Linux input startup transactionally --- openadapt_capture/input_observer/linux.py | 67 ++++++--- tests/test_input_observer_linux_xkb.py | 173 ++++++++++++++++++++++ 2 files changed, 222 insertions(+), 18 deletions(-) diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index 7ddb28d..c689a94 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -514,21 +514,46 @@ def _setup(self) -> None: ) self._waiting_baseline_marker = True self._query_pointer_baseline() - marker_deadline = time.monotonic() + self.startup_timeout + self._wait_for_baseline_marker() + self._raise_if_setup_cancelled() + if not self._accepting_events: + raise InputObserverError( + "X RECORD observed the pointer-baseline marker without arming input" + ) + self._setup_complete = True + + def _raise_if_setup_cancelled(self) -> None: + """Keep an outer readiness timeout from becoming a partial start.""" + if ( + not self._stop_requested.is_set() + and self._startup_failure is None + ): + return + self._waiting_baseline_marker = False + self._accepting_events = False + raise InputObserverError( + "X RECORD setup was cancelled before the input boundary was ready" + ) + + def _wait_for_baseline_marker( + self, + *, + timeout: float | None = None, + ) -> None: + """Wait interruptibly for the ordered QueryPointer boundary.""" + marker_timeout = self.startup_timeout if timeout is None else timeout + marker_deadline = time.monotonic() + marker_timeout while not self._baseline_marker_seen: + self._raise_if_setup_cancelled() self._xtst.XRecordProcessReplies(self._data_display) + self._raise_if_setup_cancelled() self._raise_callback_failure() if time.monotonic() >= marker_deadline: raise InputObserverError( "X RECORD did not deliver the ordered pointer-baseline marker " - f"within {self.startup_timeout:.1f}s" + f"within {marker_timeout:.1f}s" ) self._stop_requested.wait(0.001) - if not self._accepting_events: - raise InputObserverError( - "X RECORD observed the pointer-baseline marker without arming input" - ) - self._setup_complete = True def _configure_api(self) -> None: self._x11.XOpenDisplay.argtypes = [ctypes.c_char_p] @@ -716,6 +741,12 @@ def _record_intercept( and payload[0] == 1 ): self._waiting_baseline_marker = False + if ( + self._stop_requested.is_set() + or self._startup_failure is not None + ): + self._accepting_events = False + return self._baseline_marker_seen = True self._accepting_events = True return @@ -1068,14 +1099,10 @@ def _run_loop(self) -> None: def _drain_record_tail(self) -> None: """Drain DisableContext's complete tail through EndOfData, or refuse.""" - cleanup_reserve = min( - 0.5, - max( - 2 * _LOOP_INTERVAL_SECONDS, - self.shutdown_timeout * 0.1, - ), - ) - drain_budget = max(0.0, self.shutdown_timeout - cleanup_reserve) + # This drain runs inside the outer observer thread's join window. Use + # at most 80% so native frees, display closes, delivery shutdown, and + # scheduler jitter retain a meaningful cleanup budget. + drain_budget = max(0.0, self.shutdown_timeout * 0.8) deadline = time.monotonic() + drain_budget while not self._record_ended: self._xtst.XRecordProcessReplies(self._data_display) @@ -1116,10 +1143,14 @@ def attempt(label: str, operation): cleanup_failures.append(exc) return None - if self._record_enabled and not self._setup_complete: + if self._record_enabled and ( + not self._setup_complete or self._startup_failure is not None + ): # A startup timeout can leave the ordered QueryPointer marker in - # the server tail. Teardown must free that marker and subsequent - # records without letting a failed start arm event acceptance. + # the server tail, including after _setup completes but before the + # base lifecycle publishes readiness. Teardown must free that + # marker and subsequent records without letting a failed start + # arm or retain event acceptance. self._waiting_baseline_marker = False self._accepting_events = False if ( diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 6fb7aa5..2524d0d 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -4,6 +4,7 @@ import ctypes import sys +import time import pytest @@ -1176,3 +1177,175 @@ def XCloseDisplay(self, _display) -> int: assert not observer._baseline_marker_seen assert not observer._waiting_baseline_marker assert not observer._accepting_events + + +def exercise_cancelled_start_lifecycle( + *, + include_buffered_tail: bool, + enter_marker_wait_after_cancel: bool = False, + complete_setup_before_cancel: bool = False, +) -> tuple[LinuxXInputObserver, list, object, float]: + events = [] + + class LifecycleX11: + def __init__(self) -> None: + self.close_count = 0 + + def XCloseDisplay(self, _display) -> int: + self.close_count += 1 + return 0 + + class LifecycleXtst: + def __init__(self) -> None: + self.observer: LinuxXInputObserver | None = None + self.disabled = False + self.tail_sent = False + self.process_before_disable = 0 + self.free_data_count = 0 + self.free_context_count = 0 + + def XRecordDisableContext(self, _display, _context) -> int: + self.disabled = True + return 1 + + def XRecordProcessReplies(self, _display) -> None: + if not self.disabled: + self.process_before_disable += 1 + return + if self.tail_sent: + return + self.tail_sent = True + assert self.observer is not None + if include_buffered_tail: + marker = bytearray(32) + marker[0] = 1 + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=bytes(marker), + id_base=self.observer._control_id_base, + ) + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=700, + ), + ) + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=700, + state=1, + ), + id_base=0x800000, + ) + intercept_record( + self.observer, + category=linux_module._XRECORD_END_OF_DATA, + ) + + def XRecordFreeData(self, _pointer) -> None: + self.free_data_count += 1 + + def XRecordFreeContext(self, _display, _context) -> int: + self.free_context_count += 1 + return 1 + + x11 = LifecycleX11() + xtst = LifecycleXtst() + + class ControlledLifecycleObserver(LinuxXInputObserver): + def _setup(self) -> None: + self._x11 = x11 + self._xtst = xtst + self._xkbcommon = None + self._control_display = object() + self._data_display = object() + self._record_context = 11 + self._record_enabled = True + self._record_started = True + self._control_id_base = 0x400000 + self._waiting_baseline_marker = True + if complete_setup_before_cancel: + self._waiting_baseline_marker = False + self._baseline_marker_seen = True + self._accepting_events = True + self._setup_complete = True + assert self._stop_requested.wait(self.shutdown_timeout) + return + if enter_marker_wait_after_cancel: + assert self._stop_requested.wait(self.shutdown_timeout) + self._wait_for_baseline_marker(timeout=self.shutdown_timeout * 2) + self._raise_if_setup_cancelled() + if not self._accepting_events: + raise InputObserverError( + "test boundary was observed without arming input" + ) + self._setup_complete = True + + observer = ControlledLifecycleObserver( + events.append, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + startup_timeout=0.02, + shutdown_timeout=0.2, + environ={"DISPLAY": ":0", "XDG_SESSION_TYPE": "x11"}, + ) + xtst.observer = observer + started_at = time.monotonic() + with pytest.raises(InputObserverError, match="did not become ready"): + observer.start() + elapsed = time.monotonic() - started_at + + assert x11.close_count == 2 + return observer, events, xtst, elapsed + + +def test_start_timeout_discards_delayed_marker_and_buffered_input_tail() -> None: + observer, events, xtst, elapsed = exercise_cancelled_start_lifecycle( + include_buffered_tail=True, + ) + + assert events == [] + assert xtst.tail_sent + assert xtst.free_data_count == 4 + assert xtst.free_context_count == 1 + assert observer._thread is None + assert observer._delivery_thread is None + assert elapsed < observer.startup_timeout + observer.shutdown_timeout + 0.1 + + +def test_start_cancellation_before_marker_wait_returns_without_polling() -> None: + observer, events, xtst, elapsed = exercise_cancelled_start_lifecycle( + include_buffered_tail=False, + enter_marker_wait_after_cancel=True, + ) + + assert events == [] + assert xtst.tail_sent + assert xtst.process_before_disable == 0 + assert xtst.free_data_count == 1 + assert xtst.free_context_count == 1 + assert observer._thread is None + assert elapsed < observer.startup_timeout + observer.shutdown_timeout + 0.1 + + +def test_timeout_after_setup_before_ready_still_suppresses_tail() -> None: + observer, events, xtst, elapsed = exercise_cancelled_start_lifecycle( + include_buffered_tail=True, + complete_setup_before_cancel=True, + ) + + assert events == [] + assert xtst.tail_sent + assert xtst.free_data_count == 4 + assert xtst.free_context_count == 1 + assert observer._thread is None + assert elapsed < observer.startup_timeout + observer.shutdown_timeout + 0.1 From 6e1e867e248ab2e68e453e06f23040c92ea8b325 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 23 Jul 2026 15:08:33 -0700 Subject: [PATCH 26/26] fix: commit input delivery after startup --- openadapt_capture/input_observer/base.py | 98 ++++++++++++-- openadapt_capture/input_observer/linux.py | 9 ++ tests/test_input_observer.py | 83 ++++++++++++ tests/test_input_observer_linux_xkb.py | 151 ++++++++++++++++++++++ 4 files changed, 333 insertions(+), 8 deletions(-) diff --git a/openadapt_capture/input_observer/base.py b/openadapt_capture/input_observer/base.py index 594ec7d..88a819d 100644 --- a/openadapt_capture/input_observer/base.py +++ b/openadapt_capture/input_observer/base.py @@ -140,6 +140,9 @@ def __init__( ) self._delivery_sentinel = object() self._delivery_stop_requested = threading.Event() + self._delivery_decided = threading.Event() + self._delivery_state_lock = threading.Lock() + self._delivery_state = "inactive" self._failure: BaseException | None = None self._failure_lock = threading.Lock() self._startup_failure: BaseException | None = None @@ -160,13 +163,23 @@ def _wake(self) -> None: """Wake a blocked event loop during shutdown, when needed.""" def _emit(self, event: ObservedInput) -> None: - try: - self._delivery_queue.put_nowait(event) - except queue.Full: - failure = InputObserverError( - f"{type(self).__name__} input delivery queue overflowed; " - "recording coverage is incomplete" - ) + # Keep the state check and enqueue atomic with startup cancellation. + # Otherwise an abort could drain the queue and exit its delivery thread + # between these two operations, leaving a late setup event stranded. + failure: InputObserverError | None = None + with self._delivery_state_lock: + if self._delivery_state in {"aborted", "inactive"}: + return + try: + self._delivery_queue.put_nowait(event) + except queue.Full: + failure = InputObserverError( + f"{type(self).__name__} input delivery queue overflowed; " + "recording coverage is incomplete" + ) + if failure is not None: + # Wake hooks are platform-defined and may re-enter lifecycle code; + # never call them while holding the delivery-state lock. self._fail(failure) raise failure @@ -190,6 +203,23 @@ def _store_failure(self, failure: BaseException) -> BaseException: return primary def _delivery_main(self) -> None: + # Native setup may need to process events before it can prove that the + # observation boundary is ready. Preserve those events, but do not make + # them externally visible until the parent start() transaction commits. + self._delivery_decided.wait() + with self._delivery_state_lock: + delivery_state = self._delivery_state + if delivery_state == "aborted": + self._discard_delivery_queue() + return + if delivery_state != "committed": + self._fail( + InputObserverError( + f"{type(self).__name__} delivery started without a valid " + "startup decision" + ) + ) + return while True: if ( self._delivery_stop_requested.is_set() @@ -217,6 +247,39 @@ def _delivery_main(self) -> None: finally: self._delivery_queue.task_done() + def _discard_delivery_queue(self) -> None: + """Discard every event buffered by a startup that did not commit.""" + while True: + try: + self._delivery_queue.get_nowait() + except queue.Empty: + return + else: + self._delivery_queue.task_done() + + def _abort_delivery_start(self) -> None: + """Prevent setup-time events from escaping a failed start transaction.""" + with self._delivery_state_lock: + if self._delivery_state == "pending": + self._delivery_state = "aborted" + self._delivery_decided.set() + + def _commit_delivery_start(self) -> None: + """Publish setup-time events after readiness and health are proven.""" + with self._delivery_state_lock: + if self._delivery_state != "pending": + raise InputObserverError( + f"{type(self).__name__} cannot commit input delivery from " + f"state {self._delivery_state!r}" + ) + self._delivery_state = "committed" + self._delivery_decided.set() + + def _delivery_start_was_aborted(self) -> bool: + """Return whether the current startup transaction was cancelled.""" + with self._delivery_state_lock: + return self._delivery_state == "aborted" + def _start_delivery(self) -> None: if self._delivery_thread is not None: state = "live" if self._delivery_thread.is_alive() else "stopped" @@ -226,6 +289,9 @@ def _start_delivery(self) -> None: ) self._delivery_queue = queue.Queue(maxsize=self.delivery_queue_size) self._delivery_stop_requested.clear() + self._delivery_decided.clear() + with self._delivery_state_lock: + self._delivery_state = "pending" thread = threading.Thread( target=self._delivery_main, name=f"{type(self).__name__}-delivery", @@ -235,6 +301,8 @@ def _start_delivery(self) -> None: try: thread.start() except BaseException: + with self._delivery_state_lock: + self._delivery_state = "inactive" self._delivery_thread = None raise @@ -242,6 +310,7 @@ def _stop_delivery(self) -> None: thread = self._delivery_thread if thread is None: return + self._abort_delivery_start() self._delivery_stop_requested.set() if thread is threading.current_thread(): # A consumer callback may intentionally stop its listener (for @@ -249,7 +318,9 @@ def _stop_delivery(self) -> None: # lets the callback unwind; the delivery loop then drains events # already received before exiting without attempting a self-join. return - if thread.is_alive(): + with self._delivery_state_lock: + delivery_state = self._delivery_state + if thread.is_alive() and delivery_state == "committed": try: self._delivery_queue.put( self._delivery_sentinel, @@ -263,6 +334,10 @@ def _stop_delivery(self) -> None: ) ) thread.join(self.shutdown_timeout) + elif thread.is_alive(): + # A rejected start wakes the delivery thread through + # _delivery_decided and discards its buffered events. + thread.join(self.shutdown_timeout) if thread.is_alive(): self._fail( InputObserverError( @@ -272,6 +347,8 @@ def _stop_delivery(self) -> None: ) else: self._delivery_thread = None + with self._delivery_state_lock: + self._delivery_state = "inactive" def _thread_main(self) -> None: try: @@ -344,10 +421,15 @@ def start(self) -> None: self.check_health() except BaseException as exc: self._abort_start(exc) + try: + self._commit_delivery_start() + except BaseException as exc: + self._abort_start(exc) def _abort_start(self, primary: BaseException) -> None: """Make a failed ``start`` transactional before surfacing its cause.""" self._startup_failure = primary + self._abort_delivery_start() thread = self._thread if thread is None: self._stop_delivery() diff --git a/openadapt_capture/input_observer/linux.py b/openadapt_capture/input_observer/linux.py index c689a94..ca9bdb0 100644 --- a/openadapt_capture/input_observer/linux.py +++ b/openadapt_capture/input_observer/linux.py @@ -750,6 +750,15 @@ def _record_intercept( self._baseline_marker_seen = True self._accepting_events = True return + if self._delivery_start_was_aborted(): + # XRecordProcessReplies may have copied a complete native batch + # before the outer start() deadline fired. Once that transaction + # is cancelled, do not process the remaining non-marker records + # from the already-armed batch. A normal committed shutdown still + # drains its tail through EndOfData below. + self._waiting_baseline_marker = False + self._accepting_events = False + return event = _decode_core_event( payload, client_swapped=bool(recorded.client_swapped), diff --git a/tests/test_input_observer.py b/tests/test_input_observer.py index 2ad6147..069e3ab 100644 --- a/tests/test_input_observer.py +++ b/tests/test_input_observer.py @@ -94,6 +94,89 @@ def _wake(self) -> None: self.release_loop.set() +class _SetupEmittingObserver(_ReadyObserver): + def __init__( + self, + callback, + *, + fail_setup: bool = False, + delivery_queue_size: int = 4096, + ) -> None: + self.callback_invoked = threading.Event() + + def observe_delivery(event) -> None: + self.callback_invoked.set() + callback(event) + + super().__init__(observe_delivery) + self.delivery_queue_size = delivery_queue_size + self.fail_setup = fail_setup + self.health_checked = False + self.callback_observed_during_setup = False + self.events = [ + ObservedKey(pressed=True, key_char="a", timestamp=1.0), + ObservedKey(pressed=False, key_char="a", timestamp=2.0), + ] + + def _setup(self) -> None: + for event in self.events: + self._emit(event) + self.callback_observed_during_setup = self.callback_invoked.wait(timeout=0.05) + if self.fail_setup: + raise RuntimeError("setup failed after receiving input") + + def check_health(self) -> None: + self.health_checked = True + super().check_health() + + +def test_setup_events_are_delivered_in_order_only_after_start_commits() -> None: + delivered = [] + observer = _SetupEmittingObserver(delivered.append) + + observer.start() + deadline = time.monotonic() + 1 + while len(delivered) < len(observer.events): + if time.monotonic() >= deadline: + pytest.fail("committed setup events were not delivered") + time.sleep(0.001) + observer.stop() + + assert observer.health_checked + assert not observer.callback_observed_during_setup + assert delivered == observer.events + + +def test_setup_events_are_discarded_when_start_fails() -> None: + delivered = [] + observer = _SetupEmittingObserver(delivered.append, fail_setup=True) + + with pytest.raises(InputObserverError, match="setup failed after receiving input"): + observer.start() + + assert observer.health_checked + assert delivered == [] + assert observer._delivery_queue.empty() + assert observer._thread is None + assert observer._delivery_thread is None + + +def test_setup_delivery_queue_overflow_still_fails_loud() -> None: + delivered = [] + observer = _SetupEmittingObserver( + delivered.append, + delivery_queue_size=1, + ) + + with pytest.raises(InputObserverError, match="delivery queue overflowed"): + observer.start() + + assert delivered == [] + assert observer._delivery_queue.empty() + assert observer._thread is None + assert observer._delivery_thread is None + + def test_failed_startup_joins_and_tears_down_before_raising() -> None: observer = _CooperativeNeverReadyObserver() diff --git a/tests/test_input_observer_linux_xkb.py b/tests/test_input_observer_linux_xkb.py index 2524d0d..c49f366 100644 --- a/tests/test_input_observer_linux_xkb.py +++ b/tests/test_input_observer_linux_xkb.py @@ -1349,3 +1349,154 @@ def test_timeout_after_setup_before_ready_still_suppresses_tail() -> None: assert xtst.free_context_count == 1 assert observer._thread is None assert elapsed < observer.startup_timeout + observer.shutdown_timeout + 0.1 + + +def test_outer_timeout_cancels_already_armed_record_batch_transactionally() -> None: + events = [] + + class LifecycleX11: + def __init__(self) -> None: + self.close_count = 0 + + def XCloseDisplay(self, _display) -> int: + self.close_count += 1 + return 0 + + class CancelledBatchObserver(LinuxXInputObserver): + def __init__(self) -> None: + super().__init__( + events.append, + observe_keyboard=True, + observe_mouse=True, + capture_mouse_moves=True, + startup_timeout=0.02, + shutdown_timeout=0.2, + environ={"DISPLAY": ":0", "XDG_SESSION_TYPE": "x11"}, + ) + self.device_records_processed = 0 + self.delivered_records_processed = 0 + + def _handle_device_event(self, event: _CoreWireEvent) -> None: + self.device_records_processed += 1 + super()._handle_device_event(event) + + def _handle_delivered_event( + self, + event: _CoreWireEvent, + *, + id_base: int, + ) -> None: + self.delivered_records_processed += 1 + super()._handle_delivered_event(event, id_base=id_base) + + def _setup(self) -> None: + self._x11 = x11 + self._xtst = xtst + self._xkbcommon = None + self._control_display = object() + self._data_display = object() + self._record_context = 11 + self._record_enabled = True + self._record_started = True + self._control_id_base = 0x400000 + self._waiting_baseline_marker = True + self._wait_for_baseline_marker(timeout=self.shutdown_timeout * 2) + self._raise_if_setup_cancelled() + self._setup_complete = True + + class LifecycleXtst: + def __init__(self) -> None: + self.observer: CancelledBatchObserver | None = None + self.disabled = False + self.pre_disable_batch_sent = False + self.end_sent = False + self.free_data_count = 0 + self.free_context_count = 0 + + def XRecordDisableContext(self, _display, _context) -> int: + self.disabled = True + return 1 + + def XRecordProcessReplies(self, _display) -> None: + assert self.observer is not None + if not self.disabled and not self.pre_disable_batch_sent: + self.pre_disable_batch_sent = True + marker = bytearray(32) + marker[0] = 1 + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=bytes(marker), + id_base=self.observer._control_id_base, + ) + assert self.observer._accepting_events + # Model a native batch already copied by ProcessReplies while + # the parent start() deadline expires between its marker and + # remaining records. + assert self.observer._stop_requested.wait(timeout=1) + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=800, + ), + ) + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._KEY_PRESS, + detail=38, + event_time=800, + state=1, + ), + id_base=0x800000, + ) + intercept_record( + self.observer, + category=linux_module._XRECORD_FROM_SERVER, + payload=wire_event( + event_type=linux_module._MOTION_NOTIFY, + detail=0, + event_time=801, + root_x=30, + root_y=40, + ), + ) + assert self.observer.device_records_processed == 0 + assert self.observer.delivered_records_processed == 0 + return + if self.disabled and not self.end_sent: + self.end_sent = True + intercept_record( + self.observer, + category=linux_module._XRECORD_END_OF_DATA, + ) + + def XRecordFreeData(self, _pointer) -> None: + self.free_data_count += 1 + + def XRecordFreeContext(self, _display, _context) -> int: + self.free_context_count += 1 + return 1 + + x11 = LifecycleX11() + xtst = LifecycleXtst() + observer = CancelledBatchObserver() + xtst.observer = observer + + with pytest.raises(InputObserverError, match="did not become ready"): + observer.start() + + assert events == [] + assert observer.device_records_processed == 0 + assert observer.delivered_records_processed == 0 + assert xtst.pre_disable_batch_sent + assert xtst.end_sent + assert xtst.free_data_count == 5 + assert xtst.free_context_count == 1 + assert x11.close_count == 2 + assert observer._thread is None + assert observer._delivery_thread is None