From d798e9d0bbb83d82572a5e8e8e9014bd38312180 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Fri, 26 Jun 2026 07:48:39 +0800 Subject: [PATCH] Add marks_layout: non-overlapping Set-of-Marks labels with readable colour set_of_marks draws each numbered label at a fixed offset, so on dense UIs the numbers pile up and a dark label on a dark element vanishes. place_labels does greedy non-overlap placement over a candidate ring, staying in bounds; label_color picks black/white by better WCAG contrast (reusing a11y_audit.contrast_ratio). Pure geometry, fully testable. --- WHATS_NEW.md | 6 + .../doc/new_features/v215_features_doc.rst | 43 +++++++ .../Zh/doc/new_features/v215_features_doc.rst | 37 ++++++ je_auto_control/__init__.py | 3 + .../gui/script_builder/command_schema.py | 20 +++ .../utils/executor/action_executor.py | 19 +++ .../utils/marks_layout/__init__.py | 6 + .../utils/marks_layout/marks_layout.py | 118 ++++++++++++++++++ .../utils/mcp_server/tools/_factories.py | 27 ++++ .../utils/mcp_server/tools/_handlers.py | 10 ++ .../headless/test_marks_layout_batch.py | 95 ++++++++++++++ 11 files changed, 384 insertions(+) create mode 100644 docs/source/Eng/doc/new_features/v215_features_doc.rst create mode 100644 docs/source/Zh/doc/new_features/v215_features_doc.rst create mode 100644 je_auto_control/utils/marks_layout/__init__.py create mode 100644 je_auto_control/utils/marks_layout/marks_layout.py create mode 100644 test/unit_test/headless/test_marks_layout_batch.py diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 7e97b9ed..0f537fc4 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -2,6 +2,12 @@ ## What's new (2026-06-26) +### Set-of-Marks Label Layout (No Overlap, Readable Colour) + +Number every element without the labels piling up or vanishing into the background. Full reference: [`docs/source/Eng/doc/new_features/v215_features_doc.rst`](docs/source/Eng/doc/new_features/v215_features_doc.rst). + +- **`place_labels` / `label_color`** (`AC_place_labels`, `AC_label_color`): Set-of-Marks draws each numbered label at a fixed offset, so on dense UIs the numbers pile on top of each other and a dark label on a dark element vanishes. `place_labels` is greedy non-overlap placement — for each mark it tries a ring of candidate positions around its box (above/below/inside, left/right aligned) and takes the first that stays in bounds and clears every already-placed label; `label_color` picks black or white by whichever has the better WCAG contrast against the element background (reusing `a11y_audit.contrast_ratio`). Pure standard library, deterministic, fully testable without rendering. Second feature of the ROUND-15 perception lane. No `PySide6`. + ### Colour-Vision-Deficiency Simulation + Collision Check Check whether your red/green status colours are distinguishable to colour-blind users. Full reference: [`docs/source/Eng/doc/new_features/v214_features_doc.rst`](docs/source/Eng/doc/new_features/v214_features_doc.rst). diff --git a/docs/source/Eng/doc/new_features/v215_features_doc.rst b/docs/source/Eng/doc/new_features/v215_features_doc.rst new file mode 100644 index 00000000..4529494d --- /dev/null +++ b/docs/source/Eng/doc/new_features/v215_features_doc.rst @@ -0,0 +1,43 @@ +Set-of-Marks Label Layout (No Overlap, Readable Colour) +======================================================= + +Set-of-Marks overlays a numbered label on every element so a vision model can +say "click 7". ``set_of_marks`` draws each label at a fixed offset, so on dense +UIs the numbers pile on top of each other (unreadable) and a dark label on a +dark element vanishes. ``marks_layout`` fixes both with pure geometry. + +* :func:`place_labels` — greedy non-overlap placement: for each mark, try a ring + of candidate positions around its box (above, below, inside; left/right + aligned) and take the first that stays in bounds and clears every + already-placed label. +* :func:`label_color` — pick the label text colour (black or white) with the + better WCAG contrast against the element's background. + +Pure standard library; reuses :func:`a11y_audit.contrast_ratio`. Fully testable +without rendering. Imports no ``PySide6``. + +Headless API +------------ + +.. code-block:: python + + from je_auto_control import mark_elements, place_labels, label_color + + marks = mark_elements(elements) # [{id, bbox, ...}] + layout = place_labels(marks, bounds=(1920, 1080)) + # [{'id': 1, 'label': [x, y, 22, 16], 'anchor': [bx, by]}, ...] + + label_color((30, 30, 30)) # {'rgb': [255, 255, 255], 'contrast': ...} + +Feed the ``label`` boxes from :func:`place_labels` to your renderer instead of a +naive fixed offset, and pick each number's colour with :func:`label_color` so it +stays legible on its background. ``place_labels`` is deterministic and ordered by +the input marks, so the same screen always numbers the same way. + +Executor commands +----------------- + +``AC_place_labels`` (``marks`` JSON list + ``label_width`` / ``label_height`` / +``bounds`` ``[w, h]`` → ``{labels}``) and ``AC_label_color`` (``background`` +``[r, g, b]`` → ``{rgb, contrast}``). They are the matching read-only ``ac_*`` +MCP tools and Script Builder commands under **Image**. diff --git a/docs/source/Zh/doc/new_features/v215_features_doc.rst b/docs/source/Zh/doc/new_features/v215_features_doc.rst new file mode 100644 index 00000000..d5daad5d --- /dev/null +++ b/docs/source/Zh/doc/new_features/v215_features_doc.rst @@ -0,0 +1,37 @@ +Set-of-Marks 標籤佈局(不重疊、可讀顏色) +========================================= + +Set-of-Marks 在每個元素上疊一個編號標籤,讓視覺模型能說「點 7」。``set_of_marks`` 以固定偏移繪製 +每個標籤,故在密集 UI 上數字會互相疊壓(難以辨讀),而深色標籤在深色元素上會消失。``marks_layout`` +以純幾何修正兩者。 + +* :func:`place_labels` ——貪婪式不重疊放置:對每個 mark,在其方框周圍嘗試一圈候選位置 + (上、下、內;左/右對齊),取第一個仍在邊界內且不與任何已放置標籤重疊者。 +* :func:`label_color` ——挑選標籤文字顏色(黑或白),取對元素背景 WCAG 對比較佳者。 + +純標準函式庫;重用 :func:`a11y_audit.contrast_ratio`。無需繪製即可完整測試。不匯入 ``PySide6``。 + +無頭 API +-------- + +.. code-block:: python + + from je_auto_control import mark_elements, place_labels, label_color + + marks = mark_elements(elements) # [{id, bbox, ...}] + layout = place_labels(marks, bounds=(1920, 1080)) + # [{'id': 1, 'label': [x, y, 22, 16], 'anchor': [bx, by]}, ...] + + label_color((30, 30, 30)) # {'rgb': [255, 255, 255], 'contrast': ...} + +把 :func:`place_labels` 產生的 ``label`` 方框餵給你的繪製器(取代固定偏移),並用 :func:`label_color` +挑選每個編號的顏色,使其在背景上維持可讀。``place_labels`` 是確定性的且依輸入 marks 排序, +故同一畫面總是以相同方式編號。 + +執行器指令 +---------- + +``AC_place_labels``(``marks`` JSON 清單加上 ``label_width`` / ``label_height`` / +``bounds`` ``[w, h]`` → ``{labels}``)與 ``AC_label_color``(``background`` +``[r, g, b]`` → ``{rgb, contrast}``)。皆以對應的唯讀 ``ac_*`` MCP 工具及 Script Builder 指令 +(位於 **Image** 分類下)形式提供。 diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index 9930a9e4..0b171abe 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -135,6 +135,8 @@ from je_auto_control.utils.cvd_simulate import ( color_distance, colors_collide, simulate_cvd, ) +# Lay out Set-of-Marks labels without overlap + readable colour +from je_auto_control.utils.marks_layout import label_color, place_labels # Rich clipboard formats — RTF + CSV/TSV codecs and Windows get / set from je_auto_control.utils.clipboard_rich_formats import ( build_rtf, csv_to_rows, get_clipboard_csv, get_clipboard_rtf, rows_to_csv, @@ -1760,6 +1762,7 @@ def start_autocontrol_gui(*args, **kwargs): "ensure_state", "ensure_toggle", "wait_until_app_idle", "idle_point", "simulate_cvd", "colors_collide", "color_distance", + "place_labels", "label_color", "build_rtf", "rtf_to_text", "rows_to_csv", "csv_to_rows", "set_clipboard_rtf", "get_clipboard_rtf", "set_clipboard_csv", "get_clipboard_csv", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index 996c0594..90af29b2 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -4547,6 +4547,26 @@ def _add_work_queue_specs(specs: List[CommandSpec]) -> None: ), description="Whether two colours become confusable under a CVD type.", )) + specs.append(CommandSpec( + "AC_place_labels", "Image", "Place Mark Labels", + fields=( + FieldSpec("marks", FieldType.STRING, + placeholder="JSON list of {id, bbox}"), + FieldSpec("label_width", FieldType.INT, optional=True, default=22), + FieldSpec("label_height", FieldType.INT, optional=True, + default=16), + FieldSpec("bounds", FieldType.STRING, optional=True, + placeholder="[width, height]"), + ), + description="Lay out non-overlapping Set-of-Marks label boxes.", + )) + specs.append(CommandSpec( + "AC_label_color", "Image", "Label Colour for Background", + fields=( + FieldSpec("background", FieldType.STRING, placeholder="[r, g, b]"), + ), + description="Higher-contrast label colour (black/white) for a background.", + )) specs.append(CommandSpec( "AC_normalize_ext", "Shell", "Normalize Extension", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index 42a80840..01afd1ba 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -2863,6 +2863,23 @@ def _colors_collide(left: Any, right: Any, kind: Any = "deuteranopia", threshold=float(threshold)) +def _place_labels(marks: Any, label_width: Any = 22, label_height: Any = 16, + bounds: Any = None) -> Dict[str, Any]: + """Adapter: lay out non-overlapping Set-of-Marks label boxes (pure).""" + from je_auto_control.utils.marks_layout import place_labels + items = _coerce_list(marks) if marks else [] + limit = _coerce_list(bounds) if bounds else None + labels = place_labels(items, label_width=int(label_width), + label_height=int(label_height), bounds=limit) + return {"labels": labels} + + +def _label_color(background: Any) -> Dict[str, Any]: + """Adapter: the higher-contrast label colour for a background (pure).""" + from je_auto_control.utils.marks_layout import label_color + return label_color(_coerce_rgb(background)) + + def _normalize_ext(target: str) -> Dict[str, Any]: """Adapter: the lowercased extension of a path / bare ext (pure).""" from je_auto_control.utils.file_assoc import normalize_ext @@ -6896,6 +6913,8 @@ def __init__(self): "AC_idle_point": _idle_point, "AC_simulate_cvd": _simulate_cvd, "AC_colors_collide": _colors_collide, + "AC_place_labels": _place_labels, + "AC_label_color": _label_color, "AC_normalize_ext": _normalize_ext, "AC_file_association": _file_association, "AC_get_control_text": _get_control_text, diff --git a/je_auto_control/utils/marks_layout/__init__.py b/je_auto_control/utils/marks_layout/__init__.py new file mode 100644 index 00000000..5a742b5d --- /dev/null +++ b/je_auto_control/utils/marks_layout/__init__.py @@ -0,0 +1,6 @@ +"""Place Set-of-Marks labels without overlap, with readable label colours.""" +from je_auto_control.utils.marks_layout.marks_layout import ( + label_color, place_labels, +) + +__all__ = ["place_labels", "label_color"] diff --git a/je_auto_control/utils/marks_layout/marks_layout.py b/je_auto_control/utils/marks_layout/marks_layout.py new file mode 100644 index 00000000..bfcfb48d --- /dev/null +++ b/je_auto_control/utils/marks_layout/marks_layout.py @@ -0,0 +1,118 @@ +"""Place Set-of-Marks labels so they don't overlap, with readable label colours. + +Set-of-Marks overlays a numbered label on every element so a vision model can +say "click 7". ``set_of_marks`` draws each label at a fixed offset, so on dense +UIs the numbers pile on top of each other (unreadable) and a dark label on a +dark element vanishes. ``marks_layout`` fixes both with pure geometry: + +* :func:`place_labels` — greedy non-overlap placement: for each mark, try a ring + of candidate positions around its box and take the first that stays in bounds + and clears every already-placed label. +* :func:`label_color` — pick the label text colour (black or white) with the + better WCAG contrast against the element's background. + +Pure standard library; reuses :func:`a11y_audit.contrast_ratio`. Fully testable +without rendering. Imports no ``PySide6``. +""" +from typing import Any, Dict, List, Optional, Sequence, Tuple + +Rect = Tuple[int, int, int, int] + +_BLACK = (0, 0, 0) +_WHITE = (255, 255, 255) + + +def _overlap(first: Rect, second: Rect) -> bool: + """Whether two ``(x, y, w, h)`` rectangles overlap (pure).""" + ax, ay, aw, ah = first + bx, by, bw, bh = second + return not (ax + aw <= bx or bx + bw <= ax + or ay + ah <= by or by + bh <= ay) + + +def _in_bounds(rect: Rect, bounds: Tuple[int, int]) -> bool: + """Whether ``rect`` fits inside ``(width, height)`` (pure).""" + x, y, w, h = rect + return x >= 0 and y >= 0 and x + w <= int(bounds[0]) \ + and y + h <= int(bounds[1]) + + +def _candidates(bbox: Sequence[int], label_w: int, + label_h: int) -> List[Tuple[int, int]]: + """Candidate label top-left positions around an anchor box (pure).""" + bx, by, bw, bh = (int(bbox[0]), int(bbox[1]), int(bbox[2]), int(bbox[3])) + right = bx + bw - label_w + below = by + bh + return [ + (bx, by - label_h), # above, left-aligned (default SoM spot) + (right, by - label_h), # above, right-aligned + (bx, below), # below, left-aligned + (right, below), # below, right-aligned + (bx, by), # inside, top-left + (right, by), # inside, top-right + ] + + +def _clamp_to_bounds(rect: Rect, bounds: Tuple[int, int]) -> Rect: + """Shift ``rect`` to fit inside ``(width, height)`` (pure fallback).""" + x, y, w, h = rect + x = max(0, min(int(bounds[0]) - w, x)) + y = max(0, min(int(bounds[1]) - h, y)) + return (x, y, w, h) + + +def _pick_position(bbox: Sequence[int], label_w: int, label_h: int, + bounds: Optional[Tuple[int, int]], + placed: List[Rect]) -> Rect: + """Pick the first candidate that is in bounds and clears placed labels.""" + fallback: Optional[Rect] = None + for cx, cy in _candidates(bbox, label_w, label_h): + rect = (cx, cy, label_w, label_h) + if fallback is None: + fallback = rect + if bounds is not None and not _in_bounds(rect, bounds): + continue + if any(_overlap(rect, other) for other in placed): + continue + return rect + if bounds is not None and fallback is not None: + return _clamp_to_bounds(fallback, bounds) + return fallback if fallback is not None else (0, 0, label_w, label_h) + + +def place_labels(marks: Sequence[Dict[str, Any]], *, label_width: int = 22, + label_height: int = 16, + bounds: Optional[Sequence[int]] = None + ) -> List[Dict[str, Any]]: + """Lay out non-overlapping label boxes for ``marks`` (pure). + + ``marks`` is the :func:`set_of_marks.mark_elements` output (each has an + ``id`` and ``bbox`` ``[x, y, w, h]``). ``bounds`` is the ``(width, height)`` + the labels must stay within. Returns ``[{id, label, anchor}]`` where + ``label`` is the placed ``[x, y, w, h]`` box. + """ + size = (int(label_width), int(label_height)) + limit = (int(bounds[0]), int(bounds[1])) if bounds else None + placed: List[Rect] = [] + results: List[Dict[str, Any]] = [] + for mark in marks: + bbox = [int(value) for value in mark["bbox"][:4]] + rect = _pick_position(bbox, size[0], size[1], limit, placed) + placed.append(rect) + results.append({"id": mark.get("id"), "label": list(rect), + "anchor": [bbox[0], bbox[1]]}) + return results + + +def label_color(background: Sequence[float]) -> Dict[str, Any]: + """Pick the higher-contrast label text colour for ``background`` (pure). + + Returns ``{rgb, contrast}`` — black or white, whichever has the better WCAG + contrast ratio against the element background colour. + """ + from je_auto_control.utils.a11y_audit import contrast_ratio + black_contrast = contrast_ratio(background, _BLACK) + white_contrast = contrast_ratio(background, _WHITE) + if white_contrast >= black_contrast: + return {"rgb": list(_WHITE), "contrast": round(white_contrast, 3)} + return {"rgb": list(_BLACK), "contrast": round(black_contrast, 3)} diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index c57aba83..5c0431be 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -4030,6 +4030,33 @@ def img_histogram_tools() -> List[MCPTool]: handler=h.colors_collide, annotations=READ_ONLY, ), + MCPTool( + name="ac_place_labels", + description=("Lay out non-overlapping Set-of-Marks label boxes for " + "'marks' (each {id, bbox:[x,y,w,h]}). 'bounds' is " + "[width, height] to stay within. Pure. Returns " + "{labels:[{id, label:[x,y,w,h], anchor}]}."), + input_schema=schema({"marks": {"type": "array", + "items": {"type": "object"}}, + "label_width": {"type": "integer"}, + "label_height": {"type": "integer"}, + "bounds": {"type": "array", + "items": {"type": "integer"}}}, + required=["marks"]), + handler=h.place_labels, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_label_color", + description=("The higher-contrast label text colour (black or " + "white) for a 'background' [r,g,b], by WCAG contrast. " + "Returns {rgb, contrast}."), + input_schema=schema({"background": {"type": "array", + "items": {"type": "integer"}}}, + required=["background"]), + handler=h.label_color, + annotations=READ_ONLY, + ), ] diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 2d400a6d..980b3197 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -744,6 +744,16 @@ def colors_collide(left, right, kind="deuteranopia", severity=1.0, return _colors_collide(left, right, kind, severity, threshold) +def place_labels(marks, label_width=22, label_height=16, bounds=None): + from je_auto_control.utils.executor.action_executor import _place_labels + return _place_labels(marks, label_width, label_height, bounds) + + +def label_color(background): + from je_auto_control.utils.executor.action_executor import _label_color + return _label_color(background) + + def normalize_ext(target): from je_auto_control.utils.executor.action_executor import _normalize_ext return _normalize_ext(target) diff --git a/test/unit_test/headless/test_marks_layout_batch.py b/test/unit_test/headless/test_marks_layout_batch.py new file mode 100644 index 00000000..c38c4907 --- /dev/null +++ b/test/unit_test/headless/test_marks_layout_batch.py @@ -0,0 +1,95 @@ +"""Headless tests for marks_layout (pure label placement + colour).""" +import je_auto_control as ac +from je_auto_control.utils.marks_layout import label_color, place_labels + + +def _rects_overlap(a, b): + ax, ay, aw, ah = a + bx, by, bw, bh = b + return not (ax + aw <= bx or bx + bw <= ax + or ay + ah <= by or by + bh <= ay) + + +# --- place_labels --------------------------------------------------------- + +def test_place_labels_returns_one_per_mark(): + marks = [{"id": 1, "bbox": [100, 100, 40, 20]}, + {"id": 2, "bbox": [300, 300, 40, 20]}] + labels = place_labels(marks) + assert [item["id"] for item in labels] == [1, 2] + assert all(len(item["label"]) == 4 for item in labels) + + +def test_place_labels_no_overlap_on_stacked_marks(): + # three marks at the exact same spot would collide if placed naively; + # the candidate ring de-collides them + marks = [{"id": i, "bbox": [200, 200, 30, 18]} for i in range(1, 4)] + labels = place_labels(marks, bounds=[1920, 1080]) + boxes = [tuple(item["label"]) for item in labels] + for i in range(len(boxes)): + for j in range(i + 1, len(boxes)): + assert not _rects_overlap(boxes[i], boxes[j]) + + +def test_place_labels_stays_in_bounds(): + # a mark at the top-left corner can't put its label above the screen + marks = [{"id": 1, "bbox": [0, 0, 40, 20]}] + labels = place_labels(marks, label_width=22, label_height=16, + bounds=[800, 600]) + x, y, w, h = labels[0]["label"] + assert x >= 0 and y >= 0 + assert x + w <= 800 and y + h <= 600 + + +def test_place_labels_default_above_when_room(): + marks = [{"id": 1, "bbox": [100, 100, 40, 20]}] + labels = place_labels(marks, label_height=16, bounds=[800, 600]) + # default candidate is directly above the box (y = 100 - 16 = 84) + assert labels[0]["label"][1] == 84 + + +def test_place_labels_empty(): + assert place_labels([]) == [] + + +# --- label_color ---------------------------------------------------------- + +def test_label_color_white_on_dark(): + result = label_color((20, 20, 20)) + assert result["rgb"] == [255, 255, 255] + assert result["contrast"] > 1.0 + + +def test_label_color_black_on_light(): + result = label_color((240, 240, 240)) + assert result["rgb"] == [0, 0, 0] + + +# --- wiring --------------------------------------------------------------- + +def test_executor_paths(): + from je_auto_control.utils.executor.action_executor import ( + _label_color, _place_labels, + ) + out = _place_labels('[{"id": 1, "bbox": [10, 10, 30, 20]}]', 22, 16, + "[800, 600]") + assert out["labels"][0]["id"] == 1 + assert _label_color([10, 10, 10])["rgb"] == [255, 255, 255] + + +def test_wiring(): + known = set(ac.executor.known_commands()) + assert {"AC_place_labels", "AC_label_color"} <= known + from je_auto_control.utils.mcp_server.tools import ( + build_default_tool_registry, + ) + names = {t.name for t in build_default_tool_registry()} + assert {"ac_place_labels", "ac_label_color"} <= names + from je_auto_control.gui.script_builder.command_schema import _build_specs + specs = {s.command for s in _build_specs()} + assert {"AC_place_labels", "AC_label_color"} <= specs + + +def test_facade_exports(): + for name in ("place_labels", "label_color"): + assert hasattr(ac, name) and name in ac.__all__