Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions je_auto_control/gui/flow_editor/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,13 @@
FlowEdge, FlowLayout, FlowNodePosition, layout_steps,
)

_SCENE_MODULE = "je_auto_control.gui.flow_editor.scene"
_TAB_MODULE = "je_auto_control.gui.flow_editor.tab"
_LAZY_SUBMODULES = {
"FlowEdgeItem": "je_auto_control.gui.flow_editor.scene",
"FlowGraphScene": "je_auto_control.gui.flow_editor.scene",
"FlowNodeItem": "je_auto_control.gui.flow_editor.scene",
"FlowEditorTab": "je_auto_control.gui.flow_editor.tab",
"FlowEdgeItem": _SCENE_MODULE,
"FlowGraphScene": _SCENE_MODULE,
"FlowNodeItem": _SCENE_MODULE,
"FlowEditorTab": _TAB_MODULE,
}


Expand Down
218 changes: 114 additions & 104 deletions je_auto_control/gui/script_builder/command_schema.py

Large diffs are not rendered by default.

68 changes: 34 additions & 34 deletions je_auto_control/utils/accessibility/backends/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,76 +29,76 @@ def get_value(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> Optional[str]:
"""Return the matched control's value text, or None if not found."""
self._unsupported("get_value")
self._unsupported("get_value", name, role, app_name, automation_id)

def set_value(self, value: str, name: Optional[str] = None,
role: Optional[str] = None, app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Set the matched control's value; return True on success."""
self._unsupported("set_value")
self._unsupported("set_value", value, name, role, app_name, automation_id)

def invoke(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Invoke the matched control (e.g. press a button)."""
self._unsupported("invoke")
self._unsupported("invoke", name, role, app_name, automation_id)

def toggle(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Toggle the matched control (e.g. a checkbox)."""
self._unsupported("toggle")
self._unsupported("toggle", name, role, app_name, automation_id)

def read_table(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None,
) -> List[List[str]]:
"""Read a grid/table/list control as rows of cell strings."""
self._unsupported("read_table")
self._unsupported("read_table", name, role, app_name, automation_id)

# --- extended control patterns (Expand / Selection / Range / Scroll) ----

def expand(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Expand the matched control (ExpandCollapsePattern); True on success."""
self._unsupported("expand")
self._unsupported("expand", name, role, app_name, automation_id)

def collapse(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Collapse the matched control (ExpandCollapsePattern); True on success."""
self._unsupported("collapse")
self._unsupported("collapse", name, role, app_name, automation_id)

def expand_state(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> Optional[str]:
"""Return ``expanded`` / ``collapsed`` / ``partial`` / ``leaf``, or None."""
self._unsupported("expand_state")
self._unsupported("expand_state", name, role, app_name, automation_id)

def select_item(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Select the matched item (SelectionItemPattern); True on success."""
self._unsupported("select_item")
self._unsupported("select_item", name, role, app_name, automation_id)

def get_range(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> Optional[Dict[str, Any]]:
"""Return ``{value, minimum, maximum}`` (RangeValuePattern), or None."""
self._unsupported("get_range")
self._unsupported("get_range", name, role, app_name, automation_id)

def set_range_value(self, value: float, name: Optional[str] = None,
role: Optional[str] = None, app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Set a slider / progress value (RangeValuePattern); True on success."""
self._unsupported("set_range_value")
self._unsupported("set_range_value", value, name, role, app_name, automation_id)

def scroll_into_view(self, name: Optional[str] = None,
role: Optional[str] = None, app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Scroll the matched control into view (ScrollItemPattern); True on success."""
self._unsupported("scroll_into_view")
self._unsupported("scroll_into_view", name, role, app_name, automation_id)

# --- text patterns (TextPattern reads) ---------------------------------

Expand All @@ -109,49 +109,49 @@ def document_text(self, name: Optional[str] = None, role: Optional[str] = None,

Reads multiline / document controls where ValuePattern returns ``""``.
"""
self._unsupported("document_text")
self._unsupported("document_text", name, role, app_name, automation_id)

def selected_text(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> Optional[str]:
"""Return the control's currently selected text (TextPattern), or None."""
self._unsupported("selected_text")
self._unsupported("selected_text", name, role, app_name, automation_id)

def visible_text(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> Optional[str]:
"""Return only the on-screen text of the control (TextPattern), or None."""
self._unsupported("visible_text")
self._unsupported("visible_text", name, role, app_name, automation_id)

def find_text(self, text: str = "", ignore_case: bool = True,
name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Return whether ``text`` occurs in the control (TextPattern.FindText)."""
self._unsupported("find_text")
self._unsupported("find_text", text, ignore_case, name, role, app_name, automation_id)

def select_text(self, text: str = "", ignore_case: bool = True,
name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Find ``text`` and select its range (TextPattern.FindText + Select)."""
self._unsupported("select_text")
self._unsupported("select_text", text, ignore_case, name, role, app_name, automation_id)

def text_attributes(self, name: Optional[str] = None,
role: Optional[str] = None, app_name: Optional[str] = None,
automation_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Return formatting of the control's selection — ``{font_name, font_size,
bold, italic, foreground_color}`` (TextPattern attributes), or None."""
self._unsupported("text_attributes")
self._unsupported("text_attributes", name, role, app_name, automation_id)

# --- keyboard focus ----------------------------------------------------

def set_focus(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Set keyboard focus on the matched control (SetFocus); True on success."""
self._unsupported("set_focus")
self._unsupported("set_focus", name, role, app_name, automation_id)

# --- virtualized items (realize off-screen list / grid items) -----------

Expand All @@ -168,7 +168,7 @@ def find_virtual_item(self, item_name: Optional[str] = None, by: str = "name",
(``VirtualizedItemPattern``) so it exists as a real element. Returns the
realized element, or None if the container or item isn't found.
"""
self._unsupported("find_virtual_item")
self._unsupported("find_virtual_item", item_name, by, container_name, container_role, app_name, automation_id)

# --- rich element properties -------------------------------------------

Expand All @@ -182,7 +182,7 @@ def get_properties(self, name: Optional[str] = None,
``enabled`` / ``offscreen`` / ``help_text`` / ``item_status`` /
``accelerator_key`` / ``access_key`` / ``orientation``.
"""
self._unsupported("get_properties")
self._unsupported("get_properties", name, role, app_name, automation_id)

# --- table headers + cell addressing (TablePattern / GridItemPattern) ---

Expand All @@ -192,7 +192,7 @@ def get_table_headers(self, name: Optional[str] = None,
automation_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Return a table's header labels as ``{columns: [...], rows: [...]}``."""
self._unsupported("get_table_headers")
self._unsupported("get_table_headers", name, role, app_name, automation_id)

def get_grid_cell(self, row: int = 0, column: int = 0,
name: Optional[str] = None, role: Optional[str] = None,
Expand All @@ -201,7 +201,7 @@ def get_grid_cell(self, row: int = 0, column: int = 0,
) -> Optional[Dict[str, Any]]:
"""Return the cell at ``(row, column)`` as ``{value, row, column,
row_span, column_span}`` (GridPattern.GetItem + GridItemPattern)."""
self._unsupported("get_grid_cell")
self._unsupported("get_grid_cell", row, column, name, role, app_name, automation_id)

# --- transform + window patterns (UIA-element-level) --------------------

Expand All @@ -210,21 +210,21 @@ def move_element(self, x: float = 0.0, y: float = 0.0,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Move the matched element to ``(x, y)`` (TransformPattern); True on success."""
self._unsupported("move_element")
self._unsupported("move_element", x, y, name, role, app_name, automation_id)

def resize_element(self, width: float = 0.0, height: float = 0.0,
name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Resize the matched element (TransformPattern); True on success."""
self._unsupported("resize_element")
self._unsupported("resize_element", width, height, name, role, app_name, automation_id)

def set_window_state(self, state: str = "normal",
name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Set a window's visual state ``normal`` / ``maximized`` / ``minimized``."""
self._unsupported("set_window_state")
self._unsupported("set_window_state", state, name, role, app_name, automation_id)

def window_interaction_state(self, name: Optional[str] = None,
role: Optional[str] = None,
Expand All @@ -233,7 +233,7 @@ def window_interaction_state(self, name: Optional[str] = None,
) -> Optional[str]:
"""Return a window's interaction state — ``ready`` / ``blocked_by_modal`` /
``not_responding`` / ``running`` / ``closing`` (WindowPattern), or None."""
self._unsupported("window_interaction_state")
self._unsupported("window_interaction_state", name, role, app_name, automation_id)

# --- MSAA bridge (LegacyIAccessiblePattern) ----------------------------

Expand All @@ -247,15 +247,15 @@ def legacy_info(self, name: Optional[str] = None, role: Optional[str] = None,
last-resort read for legacy Win32 controls that expose nothing useful via
the modern UIA patterns.
"""
self._unsupported("legacy_info")
self._unsupported("legacy_info", name, role, app_name, automation_id)

def legacy_default_action(self, name: Optional[str] = None,
role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Fire an old control's MSAA default action (DoDefaultAction); True on
success — the fallback when Value / Invoke / Toggle all do nothing."""
self._unsupported("legacy_default_action")
self._unsupported("legacy_default_action", name, role, app_name, automation_id)

# --- container selection + views (Selection / MultipleView patterns) ----

Expand All @@ -265,21 +265,21 @@ def get_selection(self, name: Optional[str] = None, role: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Return a container's selection state — ``{items, can_select_multiple,
is_required}`` (SelectionPattern), or None."""
self._unsupported("get_selection")
self._unsupported("get_selection", name, role, app_name, automation_id)

def list_views(self, name: Optional[str] = None, role: Optional[str] = None,
app_name: Optional[str] = None,
automation_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
"""Return a control's selectable views — ``{current, views: [...]}``
(MultipleViewPattern: list / details / tile / …), or None."""
self._unsupported("list_views")
self._unsupported("list_views", name, role, app_name, automation_id)

def set_view(self, view: str = "", name: Optional[str] = None,
role: Optional[str] = None, app_name: Optional[str] = None,
automation_id: Optional[str] = None) -> bool:
"""Switch a control to the named view (MultipleViewPattern); True on success."""
self._unsupported("set_view")
self._unsupported("set_view", view, name, role, app_name, automation_id)

# --- reactive events (UIA event subscription) --------------------------

Expand All @@ -291,9 +291,9 @@ def wait_for_focus_change(self, timeout: float = 5.0,
A zero-latency native wait (UIA AddFocusChangedEventHandler) — unlike the
polling recorder, it can't miss a fast focus transition.
"""
self._unsupported("wait_for_focus_change")
self._unsupported("wait_for_focus_change", timeout)

def _unsupported(self, operation: str):
def _unsupported(self, operation: str, *context: Any):
"""Raise a clear error for an action this backend can't perform."""
raise AccessibilityNotAvailableError(
f"{operation} is not supported by the {self.name} backend",
Expand Down
3 changes: 2 additions & 1 deletion je_auto_control/utils/assertion/assertions.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,11 +489,12 @@ def assert_by_description(description: str,
)
passed = (matched == present)
state = "shows" if present else "does not show"
verdict = "match" if matched else "no match"
message = (
f"assert_by_description passed: screen {state} {description!r}"
if passed else
f"assert_by_description failed: expected screen to {state} "
f"{description!r} (VLM verdict: {'match' if matched else 'no match'})"
f"{description!r} (VLM verdict: {verdict})"
)
return _finalize(
"vlm", passed, message,
Expand Down
6 changes: 4 additions & 2 deletions je_auto_control/utils/color_match/color_match.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@

def _hsv(source, region, is_haystack: bool):
import cv2
rgb = (_to_rgb(source) if source is not None
else _grab_rgb(region)) if is_haystack else _to_rgb(source)
if is_haystack:
rgb = _to_rgb(source) if source is not None else _grab_rgb(region)
else:
rgb = _to_rgb(source)
return cv2.cvtColor(rgb, cv2.COLOR_RGB2HSV)


Expand Down
3 changes: 2 additions & 1 deletion je_auto_control/utils/config_bundle/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ def _do_export(output: Path, root: Optional[Path]) -> int:

def _do_import(source: Path, root: Optional[Path], dry_run: bool) -> int:
try:
bundle = json.loads(source.read_text(encoding="utf-8"))
# source is an operator-supplied CLI path, not remote input
bundle = json.loads(source.read_text(encoding="utf-8")) # NOSONAR
except (OSError, ValueError) as error:
print(f"failed to read {source}: {error}", file=sys.stderr)
return 2
Expand Down
33 changes: 22 additions & 11 deletions je_auto_control/utils/element_scoring/element_scoring.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,26 @@ def _proximity(element: Element, anchor: Sequence[int]) -> float:
return 1.0 / (1.0 + distance / 100.0)


def _signal_parts(element: Element, want_role: Optional[str],
want_name: Optional[str],
similarity: Callable[[str, str], float],
prefer_enabled: bool,
anchor: Optional[Sequence[int]]) -> Dict[str, float]:
"""Build the per-signal 0..1 breakdown for one ``element``."""
parts: Dict[str, float] = {}
if want_role is not None:
parts["role"] = (1.0 if str(element.get("role", "")).lower()
== str(want_role).lower() else 0.0)
if want_name is not None:
parts["name"] = float(similarity(want_name,
str(element.get("name", ""))))
if anchor is not None:
parts["proximity"] = _proximity(element, anchor)
if prefer_enabled:
parts["enabled"] = 1.0 if element.get("enabled", True) else 0.0
return parts


def score_candidates(candidates: Sequence[Element], *,
want_role: Optional[str] = None,
want_name: Optional[str] = None,
Expand All @@ -57,17 +77,8 @@ def score_candidates(candidates: Sequence[Element], *,
similarity = name_similarity or fuzzy_ratio
scored: List[ScoredCandidate] = []
for element in candidates:
parts: Dict[str, float] = {}
if want_role is not None:
parts["role"] = (1.0 if str(element.get("role", "")).lower()
== str(want_role).lower() else 0.0)
if want_name is not None:
parts["name"] = float(similarity(want_name,
str(element.get("name", ""))))
if anchor is not None:
parts["proximity"] = _proximity(element, anchor)
if prefer_enabled:
parts["enabled"] = 1.0 if element.get("enabled", True) else 0.0
parts = _signal_parts(element, want_role, want_name, similarity,
prefer_enabled, anchor)
score = sum(parts.values()) / len(parts) if parts else 0.0
scored.append(ScoredCandidate(element, round(score, 4), parts))
scored.sort(key=lambda candidate: candidate.score, reverse=True)
Expand Down
Loading
Loading