diff --git a/docs/02_using_agents.md b/docs/02_using_agents.md index c2c8745c..8ab14665 100644 --- a/docs/02_using_agents.md +++ b/docs/02_using_agents.md @@ -34,6 +34,41 @@ Requires the `android` dependency installed (`pip install askui[android]`) and a **Default tools:** `screenshot`, `tap`, `type`, `swipe`, `drag_and_drop`, `key_tap_event`, `key_combination`, `shell`, `select_device_by_serial_number`, `select_display_by_unique_id`, `get_connected_devices_serial_numbers`, `get_connected_displays_infos`, `get_current_connected_device_infos` +### Selecting a display + +By default the agent drives the first detected display. On multi-display hardware (e.g. automotive head units) you can pin a specific display with the `display` parameter: + +```python +from askui import AndroidAgent +from askui.tools.android.agent_os import AndroidDisplay + +# Pin one display by its exact ids (bypasses auto-detection). +# The ids below are placeholders — read your device's real values from +# `adb shell dumpsys display` (see the command at the end of this section). +with AndroidAgent( + device="emulator-5554", + display=AndroidDisplay(unique_display_id=1234567890123456789, display_name="secondary", display_id=2), + display_allow_switching=False, +) as agent: + agent.act("Open settings") +``` + +`display` accepts: + +- **`AndroidDisplay`** — pins that exact display and bypasses auto-detection, so you control the ids used for shell commands. `display_id` is the logical id passed to `input` (tap/swipe/type) as `-d `; `unique_display_id` is the physical id passed to `screencap` as `-d `. Get both from the device: `adb shell dumpsys display` (look for the `mViewports` line, which maps `displayId ↔ uniqueId`). +- **`list[AndroidDisplay]`** — the authoritative set of selectable displays. The first is active, and the agent may switch among them at runtime with correct ids. +- **`int`** — select by index, **`str`** — select by name (both via auto-detection). + +Either id may be `None`, which omits the `-d` flag so that command targets adb's **default display** (display 0). Use `display_id=None` only when your target screen is the default display; on a multi-display setup, pass the real logical id instead. + +Set `display_allow_switching=False` to remove the runtime display/device selection tools, so a pinned `display` cannot be changed mid-run by the model. + +You can find the ids for a device with (the quotes keep the pipe inside the device shell, so this works the same on PowerShell, cmd, and bash): + +```bash +adb shell "dumpsys display | grep -iE 'mViewports|uniqueId'" +``` + ## WebVisionAgent For web browser automation using Playwright. Extends `ComputerAgent` with web-specific tools like navigation, URL handling, and page title retrieval. diff --git a/src/askui/android_agent.py b/src/askui/android_agent.py index 1125c6b1..956655e7 100644 --- a/src/askui/android_agent.py +++ b/src/askui/android_agent.py @@ -15,7 +15,7 @@ from askui.models.shared.tools import Tool from askui.models.shared.truncation_strategies import TruncationStrategy from askui.prompts.act_prompts import create_android_agent_prompt -from askui.tools.android.agent_os import ANDROID_KEY +from askui.tools.android.agent_os import ANDROID_KEY, AndroidDisplay from askui.tools.android.agent_os_facade import AndroidAgentOsFacade from askui.tools.android.ppadb_agent_os import PpadbAgentOs from askui.tools.android.tools import ( @@ -50,6 +50,8 @@ class AndroidAgent(Agent): Args: device (str | int, optional): The Android device to connect to. Can be either a serial number (as a `str`) or an index (as an `int`) representing the position in the `adb devices` list. Index `0` refers to the first device. Defaults to `0`. + display (AndroidDisplay | list[AndroidDisplay] | int | str | None, optional): Which display to drive. An `AndroidDisplay` pins that exact display (bypassing auto-detection, so you control the input/screencap `-d` ids). A `list[AndroidDisplay]` becomes the authoritative set of selectable displays — the first is active and the model may switch among them at runtime with correct ids. An `int` selects by index and a `str` by name, both via auto-detection. `None` (default) auto-detects and selects the first display. + display_allow_switching (bool, optional): When `False`, the runtime display/device selection tools are removed so a pinned `display` cannot be changed mid-run. Defaults to `True`. reporters (list[Reporter] | None, optional): List of reporter instances for logging and reporting. If `None`, an empty list is used. settings (AgentSettings | None, optional): Provider-based model settings. If `None`, uses the default AskUI model stack. retry (Retry, optional): The retry instance to use for retrying failed actions. Defaults to `ConfigurableRetry` with exponential backoff. Currently only supported for `locate()` method. @@ -81,6 +83,8 @@ class AndroidAgent(Agent): def __init__( self, device: str | int = 0, + display: "AndroidDisplay | list[AndroidDisplay] | int | str | None" = None, + display_allow_switching: bool = True, reporters: list[Reporter] | None = None, settings: AgentSettings | None = None, retry: Retry | None = None, @@ -90,11 +94,16 @@ def __init__( secrets: list[Secret] | None = None, ) -> None: reporter = CompositeReporter(reporters=reporters) - self.os = PpadbAgentOs(device_identifier=device, reporter=reporter) + self.os = PpadbAgentOs( + device_identifier=device, display=display, reporter=reporter + ) + default_tools = self._apply_display_tool_policy( + self.get_default_tools(), display_allow_switching + ) super().__init__( reporter=reporter, retry=retry, - tools=self.get_default_tools() + (act_tools or []), + tools=default_tools + (act_tools or []), agent_os=self.os, settings=settings, callbacks=callbacks, @@ -362,6 +371,30 @@ def set_device_by_serial_number( ) self.os.set_device_by_serial_number(device_sn) + @staticmethod + def _apply_display_tool_policy( + tools: list[Tool], display_allow_switching: bool + ) -> list[Tool]: + """Drop the display/device *mutating* tools when switching is disabled. + + The read-only display-info tools are kept. Device selection is included + because selecting a device resets the active display to index 0, which + would undo a pinned `display`. + """ + if display_allow_switching: + return tools + return [ + t + for t in tools + if not isinstance( + t, + ( + AndroidSelectDisplayByUniqueIDTool, + AndroidSelectDeviceBySerialNumberTool, + ), + ) + ] + @staticmethod def get_default_tools() -> list[Tool]: return [ diff --git a/src/askui/tools/android/agent_os.py b/src/askui/tools/android/agent_os.py index d7fe7e04..2eb927e2 100644 --- a/src/askui/tools/android/agent_os.py +++ b/src/askui/tools/android/agent_os.py @@ -201,12 +201,32 @@ class AndroidDisplay: + """A selectable Android display. + + ``display_id`` is the logical DisplayManager id consumed by ``input`` + (tap/swipe/type/keyevent). ``unique_display_id`` is the physical display id + consumed by ``screencap``. Either may be ``None``, in which case the + corresponding shell command runs with no ``-d`` flag and therefore targets + adb's default display (display 0). + + Passing ``None`` for ``display_id`` is only correct when the target screen + IS the default display; on a multi-display setup where the agent switches + displays, provide the real logical id instead (a ``None`` input flag ignores + which display is currently active). Likewise, a display with + ``unique_display_id=None`` cannot be selected by the model at runtime — the + ``select_display_by_unique_id`` tool needs a concrete unique id — so use it + only for a pinned, non-switchable display. + """ + def __init__( - self, unique_display_id: int, display_name: str, display_id: int + self, + unique_display_id: int | None, + display_name: str, + display_id: int | None, ) -> None: - self.unique_display_id: int = unique_display_id + self.unique_display_id: int | None = unique_display_id self.display_name: str = display_name - self.display_id: int = display_id + self.display_id: int | None = display_id def __repr__(self) -> str: return ( @@ -214,40 +234,49 @@ def __repr__(self) -> str: f"display_name={self.display_name}, display_id={self.display_id})" ) + def __str__(self) -> str: + # Model-facing rendering (used by the display-info tools). Only surface a + # unique id the model can hand back to select_display_by_unique_id; a + # None id means "default display" and is not selectable. + if self.unique_display_id is None: + return f"display '{self.display_name}' (default display)" + return ( + f"display '{self.display_name}' " + f"(unique_display_id={self.unique_display_id})" + ) + def get_display_id_flag(self) -> str: """ - Returns the display ID flag for shell commands. + Returns the display ID flag for input/tap/swipe/type shell commands. Returns: - str: The display ID flag in the format `-d {display_id}`. + str: ``-d {display_id}``, or an empty string when ``display_id`` is + ``None`` (input then targets adb's default display). """ - return f"-d {self.display_id}" + return "" if self.display_id is None else f"-d {self.display_id}" def get_display_unique_id_flag(self) -> str: """ - Returns the display unique ID flag for shell screencap command. + Returns the display unique ID flag for the screencap shell command. Returns: - str: The display unique ID flag in the format `-d {unique_display_id}`. + str: ``-d {unique_display_id}``, or an empty string when + ``unique_display_id`` is ``None`` (screencap then targets the default + display). """ - return f"-d {self.unique_display_id}" + return "" if self.unique_display_id is None else f"-d {self.unique_display_id}" class SingleAndroidDisplay(AndroidDisplay): """ Single display when there is only one display connected. + + Both ids are ``None`` so input and screencap run without a ``-d`` flag + (adb's default display). """ def __init__(self, display_name: str) -> None: - super().__init__(0, display_name, 0) - - # In case of a single display, the display id flag is not needed - def get_display_id_flag(self) -> str: - return "" - - # In case of a single display, the display unique id flag is not needed - def get_display_unique_id_flag(self) -> str: - return "" + super().__init__(None, display_name, None) class UnknownAndroidDisplay(SingleAndroidDisplay): diff --git a/src/askui/tools/android/ppadb_agent_os.py b/src/askui/tools/android/ppadb_agent_os.py index 517ed4e1..c70efa95 100644 --- a/src/askui/tools/android/ppadb_agent_os.py +++ b/src/askui/tools/android/ppadb_agent_os.py @@ -41,7 +41,10 @@ class PpadbAgentOs(AndroidAgentOs): _UIAUTOMATOR_DUMP_PATH: str = "/data/local/tmp/askui_window_dump.xml" def __init__( - self, reporter: Reporter = NULL_REPORTER, device_identifier: str | int = 0 + self, + reporter: Reporter = NULL_REPORTER, + device_identifier: str | int = 0, + display: "AndroidDisplay | list[AndroidDisplay] | int | str | None" = None, ) -> None: self._client: Optional[AdbClient] = None self._device: Optional[AndroidDevice] = None @@ -50,6 +53,21 @@ def __init__( self._selected_display: Optional[AndroidDisplay] = None self._reporter: Reporter = reporter self._device_identifier: str | int = device_identifier + self._display_config = display + # When the caller supplies explicit AndroidDisplay(s), they become the + # authoritative display list — get_connected_displays() returns them + # verbatim instead of the SurfaceFlinger auto-detection, so both the + # initial selection AND every runtime set_display_by_* (incl. the model's + # select_display_by_unique_id tool) resolve against the caller's correct + # ids. int/str selectors keep using auto-detection. + self._display_override: Optional[list[AndroidDisplay]] = None + if isinstance(display, AndroidDisplay): + self._display_override = [display] + elif isinstance(display, list): + if not display: + msg = "display list must not be empty" + raise AndroidAgentOsError(msg) + self._display_override = list(display) def connect_adb_client(self) -> None: if self._client is not None: @@ -77,6 +95,15 @@ def connect(self) -> None: self.set_device_by_serial_number(self._device_identifier) else: self.set_device_by_index(self._device_identifier) + # Device selection defaults the active display to index 0. With an + # override list that already resolves to the first supplied display; for + # int/str selectors, apply the caller's explicit choice on top. + if isinstance(self._display_config, bool): + pass # bool is an int subclass; never treat True/False as an index + elif isinstance(self._display_config, int): + self.set_display_by_index(self._display_config) + elif isinstance(self._display_config, str): + self.set_display_by_name(self._display_config) device: AndroidDevice = self._get_selected_device() device.wait_boot_complete() @@ -97,6 +124,8 @@ def _set_display(self, display: AndroidDisplay) -> None: ) def get_connected_displays(self) -> list[AndroidDisplay]: + if self._display_override is not None: + return list(self._display_override) device: AndroidDevice = self._get_selected_device() displays: list[AndroidDisplay] = [] output: str = device.shell( diff --git a/tests/unit/tools/android/__init__.py b/tests/unit/tools/android/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/tools/android/test_display_selection.py b/tests/unit/tools/android/test_display_selection.py new file mode 100644 index 00000000..204fdfb9 --- /dev/null +++ b/tests/unit/tools/android/test_display_selection.py @@ -0,0 +1,134 @@ +"""Unit tests for AndroidDisplay flag emission and PpadbAgentOs display pinning.""" + +import pytest + +from askui.android_agent import AndroidAgent +from askui.tools.android.agent_os import ( + AndroidDisplay, + SingleAndroidDisplay, + UnknownAndroidDisplay, +) +from askui.tools.android.android_agent_os_error import AndroidAgentOsError +from askui.tools.android.ppadb_agent_os import PpadbAgentOs +from askui.tools.android.tools import ( + AndroidGetConnectedDisplaysInfosTool, + AndroidSelectDeviceBySerialNumberTool, + AndroidSelectDisplayByUniqueIDTool, +) + +# --- AndroidDisplay flag emission ------------------------------------------ + + +def test_both_ids_emit_flags() -> None: + d = AndroidDisplay(1000000000000000002, "secondary", 2) + assert d.get_display_id_flag() == "-d 2" + assert d.get_display_unique_id_flag() == "-d 1000000000000000002" + + +def test_none_display_id_omits_input_flag_but_keeps_screencap() -> None: + # primary case: input runs with no -d (default display), screencap keeps -d. + d = AndroidDisplay(1000000000000000001, "primary", None) + assert d.get_display_id_flag() == "" + assert d.get_display_unique_id_flag() == "-d 1000000000000000001" + + +def test_none_unique_id_omits_screencap_flag() -> None: + d = AndroidDisplay(None, "primary", 0) + assert d.get_display_id_flag() == "-d 0" + assert d.get_display_unique_id_flag() == "" + + +def test_single_display_omits_both_flags() -> None: + d = SingleAndroidDisplay("primary") + assert d.get_display_id_flag() == "" + assert d.get_display_unique_id_flag() == "" + + +def test_unknown_display_omits_both_flags() -> None: + d = UnknownAndroidDisplay() + assert d.get_display_id_flag() == "" + assert d.get_display_unique_id_flag() == "" + + +def test_str_hides_none_ids_from_model() -> None: + # The model selects displays by unique id; a None id must not be presented + # as a selectable value. + with_id = AndroidDisplay(1000000000000000002, "secondary", 2) + assert "unique_display_id=1000000000000000002" in str(with_id) + + without_id = AndroidDisplay(None, "primary", None) + assert "None" not in str(without_id) + assert "default display" in str(without_id) + + +# --- PpadbAgentOs display override ----------------------------------------- + + +def test_single_display_becomes_override_list() -> None: + d = AndroidDisplay(1000000000000000001, "primary", None) + os = PpadbAgentOs(display=d) + assert os.get_connected_displays() == [d] + + +def test_display_list_is_authoritative() -> None: + d1 = AndroidDisplay(1000000000000000001, "primary", 0) + d2 = AndroidDisplay(1000000000000000002, "secondary", 2) + os = PpadbAgentOs(display=[d1, d2]) + assert os.get_connected_displays() == [d1, d2] + + +def test_empty_display_list_rejected() -> None: + with pytest.raises(AndroidAgentOsError): + PpadbAgentOs(display=[]) + + +def test_set_display_by_unique_id_resolves_against_override() -> None: + d1 = AndroidDisplay(1000000000000000001, "primary", 0) + d2 = AndroidDisplay(1000000000000000002, "secondary", 2) + os = PpadbAgentOs(display=[d1, d2]) + + os.set_display_by_unique_id(1000000000000000002) + assert os._selected_display is d2 + + +def test_set_display_by_unique_id_unknown_raises() -> None: + d1 = AndroidDisplay(1000000000000000001, "primary", 0) + os = PpadbAgentOs(display=[d1]) + with pytest.raises(AndroidAgentOsError): + os.set_display_by_unique_id(999999) + + +def test_no_override_when_selector_is_int_or_str() -> None: + assert PpadbAgentOs(display=0)._display_override is None + assert PpadbAgentOs(display="primary")._display_override is None + assert PpadbAgentOs()._display_override is None + + +# --- AndroidAgent display-tool policy -------------------------------------- + + +def test_default_tools_include_display_selectors() -> None: + tools = AndroidAgent.get_default_tools() + types = {type(t) for t in tools} + assert AndroidSelectDisplayByUniqueIDTool in types + assert AndroidSelectDeviceBySerialNumberTool in types + + +def test_disallowing_switching_drops_mutating_display_tools() -> None: + tools = AndroidAgent.get_default_tools() + filtered = AndroidAgent._apply_display_tool_policy( + tools, display_allow_switching=False + ) + types = {type(t) for t in filtered} + assert AndroidSelectDisplayByUniqueIDTool not in types + assert AndroidSelectDeviceBySerialNumberTool not in types + # read-only info tool stays + assert AndroidGetConnectedDisplaysInfosTool in types + + +def test_allowing_switching_keeps_all_tools() -> None: + tools = AndroidAgent.get_default_tools() + filtered = AndroidAgent._apply_display_tool_policy( + tools, display_allow_switching=True + ) + assert filtered == tools