cua_backend.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. """Cua-driver backend (macOS, Windows, Linux): MCP over stdio to `cua-driver`. The async `mcp` SDK runs on a
  2. background loop (``cua_backend_session``); the same tool surface works on all three platforms, and per-host gaps
  3. (no DISPLAY, missing AT-SPI, TCC) surface via `hermes computer-use doctor` instead of failing silently. Install
  4. with `hermes computer-use install`. The macOS path uses private SkyLight SPIs that can break on OS updates.
  5. Siblings: ``cua_backend_driver`` (binary/contract), ``cua_backend_capture`` + ``cua_backend_input``
  6. (mixins), ``cua_backend_parse``, ``cua_backend_session`` (bridge + session + CLI fallback), ``cua_backend_daemon``
  7. (private daemon + macOS app identity). Siblings look this module's config/policy helpers up lazily."""
  8. from __future__ import annotations
  9. import contextlib
  10. import logging
  11. import os
  12. import subprocess
  13. import sys
  14. import uuid
  15. from pathlib import PureWindowsPath
  16. from typing import Any, Dict, List, Optional, Tuple
  17. from hermes_cli._subprocess_compat import windows_hide_flags
  18. from hermes_platform.host.runtime import is_wsl
  19. from tools.computer_use.backend import ActionResult, ComputerUseBackend
  20. from tools.computer_use.cua_backend_capture import _CaptureMixin
  21. from tools.computer_use.cua_backend_daemon import _EmbeddedCuaDaemon
  22. from tools.computer_use.cua_backend_driver import (
  23. _CUA_DRIVER_CMD_ENV, cua_driver_binary_available, cua_driver_runtime_contract_status,
  24. resolve_cua_driver_cmd)
  25. from tools.computer_use.cua_backend_input import _InputMixin
  26. from tools.computer_use.cua_backend_parse import _action_result_from
  27. from tools.computer_use.cua_backend_session import _AsyncBridge, _CuaDriverSession
  28. logger = logging.getLogger(__name__)
  29. # cua-driver's anonymous PostHog telemetry gate ("0" disables; absent => ON upstream).
  30. _CUA_TELEMETRY_ENV_VAR = "CUA_DRIVER_RS_TELEMETRY_ENABLED"
  31. _CUA_NATIVE_WAYLAND_ENV_VAR = "CUA_DRIVER_RS_ENABLE_WAYLAND"
  32. def _computer_use_cfg() -> dict[str, Any]:
  33. """The ``computer_use`` config block, or ``{}`` when config is unreadable."""
  34. with contextlib.suppress(Exception):
  35. from hermes_cli.config import load_config
  36. return (load_config() or {}).get("computer_use") or {}
  37. return {}
  38. def _cua_no_overlay() -> bool:
  39. """Pass ``--no-overlay``? ``computer_use.no_overlay`` overrides; else off on macOS (cursor-overlay redraw
  40. loop can peg a core after a session), headless Linux / WSL2 / containers, and Linux X11 (the overlay is a
  41. fullscreen always-on-top all-workspaces window with no compositor-owned lifecycle, so an unclean session
  42. end can leave it wedged over every app); on for Windows and Linux Wayland (compositor owns the surface).
  43. Explicit ``True`` / ``False`` overrides auto-detection. See #28152, #47032.
  44. """
  45. val = _computer_use_cfg().get("no_overlay")
  46. if val is not None or sys.platform != "linux":
  47. return bool(val) if val is not None else sys.platform == "darwin"
  48. wsl = is_wsl()
  49. return wsl or not os.environ.get("DISPLAY") or (
  50. # Linux/X11: the cursor overlay is a fullscreen, always-on-top, all-workspaces X11 window
  51. # (save-unders path). An unclean session end (agent interrupted mid-capture, stale target window)
  52. # can leave it stuck above every app on every workspace, wedging desktop input until the app
  53. # restarts — the same failure class as the HUD window on Mutter/X11 (#83473). There is no
  54. # compositor-owned surface to tear down with the client connection, so default the overlay off on
  55. # X11 too; set computer_use.no_overlay: false to keep the cursor. Wayland keeps it: the compositor
  56. # owns the overlay surface lifecycle there.
  57. os.environ.get("XDG_SESSION_TYPE") != "wayland" and not os.environ.get("WAYLAND_DISPLAY"))
  58. def _cua_telemetry_disabled() -> bool:
  59. """True unless ``computer_use.cua_telemetry`` opts in (unreadable config fails SAFE toward disabling)."""
  60. return not bool(_computer_use_cfg().get("cua_telemetry", False))
  61. def _cua_configured_permission_mode() -> str:
  62. """``computer_use.permission_mode``: ``standard`` (default) or ``bounded``; unknown values fall closed to
  63. ``standard``. ``unrestricted`` is deliberately NOT a config value — it stays tied to the per-session YOLO
  64. toggle so a stale config line can never silently bypass approvals."""
  65. raw = str(_computer_use_cfg().get("permission_mode", "standard") or "").strip().lower()
  66. return "bounded" if raw == "bounded" else "standard"
  67. # ``computer_use.ax_max_elements``: bound on the DRIVER's accessibility-tree walk per capture. The
  68. # visible-element cap in tool.py (_DEFAULT_MAX_ELEMENTS) trims the RESPONSE only, so without this an
  69. # unbounded walk pays for nodes the model never sees; 0 disables the bound (driver default: 2,000
  70. # elements / depth 25).
  71. _DEFAULT_AX_MAX_ELEMENTS = 200
  72. def _cua_configured_ax_max_elements() -> int:
  73. """Bound on ``get_window_state``'s AX walk; 0 = no bound (driver default). Unreadable config fails to
  74. the default, never to unbounded. Measured on macOS (cua-driver 0.28.2, M-series): a 1,444-node Chrome
  75. window 540 ms -> 83 ms and a 456-node Finder window 6.9 s -> 0.6 s at 200, on the CLI transport. The
  76. bounded result is a prefix of the unbounded walk, so the elements the model sees are unchanged. Raise
  77. it toward 400 to keep the full first 100 visible elements on a pathological tree (~1.4 s on Finder);
  78. cost grows with the bound, and a bound in the thousands buys nothing the response cap keeps. This
  79. bounds the nodes COLLECTED, not the walk's wall clock: a target whose accessibility surface exceeds the
  80. driver's own 20 s walk timeout still fails at every bound (and every depth bound), so a timeout error is
  81. never an argument for a smaller or larger value here."""
  82. raw = _computer_use_cfg().get("ax_max_elements", _DEFAULT_AX_MAX_ELEMENTS)
  83. try:
  84. return max(0, int(raw))
  85. except (TypeError, ValueError):
  86. return _DEFAULT_AX_MAX_ELEMENTS
  87. def _manifest_is_mode_independent(path: str) -> bool:
  88. """True when this manifest may accompany any permission mode: v1/v2 declare ``mode: bounded`` and abort
  89. startup under an unrestricted runtime; v3 has no mode and is the ceiling the driver accepts alongside any
  90. mode. Unreadable / unparseable -> False (forwarding one would turn a working session into a hard startup
  91. failure; bounded forwards unconditionally anyway)."""
  92. try:
  93. import hermes_yaml as yaml
  94. with open(path, "r", encoding="utf-8-sig") as handle:
  95. parsed = yaml.safe_load(handle)
  96. except Exception:
  97. logger.debug("could not read capability manifest %s", path, exc_info=True)
  98. return False
  99. version = parsed.get("version") if isinstance(parsed, dict) else None
  100. return isinstance(version, int) and not isinstance(version, bool) and version >= 3
  101. def _computer_use_max_image_dimension() -> Optional[int]:
  102. """``computer_use.max_image_dimension`` longest-edge cap (default 1456 = aux-vision downscale); ``0``/negative -> None."""
  103. try:
  104. dim = int(_computer_use_cfg().get("max_image_dimension", 1456))
  105. except (TypeError, ValueError):
  106. dim = 1456
  107. return dim if dim > 0 else None
  108. def desktop_identity(env: Optional[dict[str, str]] = None) -> str:
  109. """The screen a backend spawned from ``env`` acts on: its DISPLAY (``''`` when none). Recorded next to the
  110. cached backend so a Bot Desktop that starts (or restarts on another number) AFTER the backend was cached is
  111. noticed — the cached cua-driver still points at the old seat or at no display at all."""
  112. return str((cua_driver_child_env(env) if env is None else env).get("DISPLAY") or "")
  113. def backend_display_stale(recorded: str, current: str) -> bool:
  114. """True when a cached backend's recorded display identity no longer matches the one a fresh spawn would get."""
  115. return (recorded or "") != (current or "")
  116. def cua_driver_child_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
  117. """Env for spawning cua-driver: ``base_env`` (default ``os.environ``) plus ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0``
  118. unless the user opted in, plus the native-Wayland bridge (``computer_use.native_wayland`` config opt-in, only when
  119. the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway
  120. runtimes share one policy."""
  121. env = dict(os.environ if base_env is None else base_env)
  122. # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so
  123. # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display.
  124. from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env
  125. env = _bot_desktop_env(env)
  126. if _cua_telemetry_disabled():
  127. env[_CUA_TELEMETRY_ENV_VAR] = "0"
  128. if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)):
  129. env[_CUA_NATIVE_WAYLAND_ENV_VAR] = "1"
  130. return env
  131. def sandbox_mcp_invocation() -> Optional[tuple[tuple[str, list[str]], dict[str, str]]]:
  132. """``((command, args), child_env)`` spawning ``cua-driver mcp`` INSIDE the terminal backend when the Bot
  133. Desktop is placed there (the driver in the sandbox image drives the sandbox's own screen); None on a
  134. gateway-hosted desktop, where the local driver is used. Placement is the authority: a ``terminal``
  135. placement gets its screen started here and a ``refused`` one raises — the host driver is never the
  136. fallback for a sandbox whose screen is down."""
  137. from tools.bot_desktop import placement, runtime as _bd_runtime
  138. if _bd_runtime.tool_placement() == placement.GATEWAY:
  139. return None
  140. published = _bd_runtime.published_env()
  141. if not published.get("DISPLAY"):
  142. raise RuntimeError("the screen inside the terminal backend's sandbox is gone; start it again")
  143. from tools.bot_desktop import sandbox_host
  144. env = _bd_runtime._sandbox_env(create=True)
  145. if env is None:
  146. raise RuntimeError("the terminal backend's sandbox is not running, so there is nowhere to run cua-driver")
  147. command, args = sandbox_host.cua_mcp_invocation(env, _bd_runtime._profile_name(),
  148. {**published, _CUA_TELEMETRY_ENV_VAR: "0"})
  149. _bd_runtime.touch_activity()
  150. return (command, args), {"PATH": os.environ.get("PATH", "")}
  151. def sanitized_cua_driver_env() -> dict[str, str]:
  152. """``cua_driver_child_env()`` with Hermes provider secrets stripped — cua-driver is a third-party binary and must
  153. never inherit API keys. Falls back to the unsanitized telemetry env if the sanitizer can't import."""
  154. env = cua_driver_child_env()
  155. with contextlib.suppress(Exception):
  156. # cua-driver is a third-party binary — never hand it provider API keys via inherited env (same
  157. # policy as the manifest probe and MCP spawn; #53503/#55709/#58889 lineage).
  158. from tools.environments.local import _sanitize_subprocess_env
  159. return _sanitize_subprocess_env(env)
  160. return env
  161. def _run_quiet(argv: list[str], *, timeout: float, swallow: Any = (), **kw: Any) -> Any:
  162. """``subprocess.run`` for short probe verbs: text mode, stdin=DEVNULL unless overridden (older drivers fall into a
  163. stdin-reading mode on unknown verbs; EOF makes them exit fast instead of blocking until the timeout), output
  164. captured unless the caller redirects it. Exceptions in ``swallow`` return None; others raise."""
  165. kw.setdefault("stdin", subprocess.DEVNULL)
  166. kw.setdefault("encoding", "utf-8")
  167. kw.setdefault("errors", "replace")
  168. "stdout" in kw or kw.setdefault("capture_output", True)
  169. try:
  170. return subprocess.run(argv, text=True, timeout=timeout, stdin=kw.pop("stdin"), encoding=kw.pop("encoding"), # noqa: PLW1510 -- forwarding wrapper: callers pass subprocess kwargs via **kwargs (check may arrive through it); an explicit check=False would raise TypeError
  171. errors=kw.pop("errors"), **kw)
  172. except swallow:
  173. return None
  174. def _run_driver(driver_cmd: str, *args: str, timeout: float, swallow: Any = ()) -> Any:
  175. """Run a short cua-driver verb with the sanitized env and hidden window."""
  176. return _run_quiet([driver_cmd, *args], timeout=timeout, swallow=swallow, encoding="utf-8",
  177. errors="replace", creationflags=windows_hide_flags(), env=sanitized_cua_driver_env())
  178. def cua_daemon_listening(driver_cmd: str, socket_path: Optional[str] = None, *, timeout: float = 3.0) -> Optional[bool]:
  179. """Socket-level liveness of a ``cua-driver serve`` daemon: ``cua-driver status`` connects to the daemon
  180. socket (the driver's default, or ``socket_path``) and exits 0 only when a daemon answers. False when the
  181. CLI reports the daemon is not running, None when the probe itself failed (unknown). Never raises.
  182. The binary-level runtime contract (``manifest``) cannot see this — a dead daemon looks healthy there (#114748)."""
  183. args = ("status", "--socket", socket_path) if socket_path else ("status",)
  184. proc = _run_driver(driver_cmd, *args, timeout=timeout, swallow=(OSError, subprocess.SubprocessError))
  185. if proc is None:
  186. return None
  187. if proc.returncode == 0:
  188. return True
  189. return False if "not running" in f"{proc.stdout}\n{proc.stderr}".lower() else None
  190. def _linux_session_locked() -> Optional[bool]:
  191. """Is the graphical session locked? (Linux; best-effort.) A locked KDE/GNOME session freezes renderers and
  192. half-disables the AX tree, so discovery legitimately returns nothing — which otherwise reads as a driver bug.
  193. True/False when loginctl answers, None when unavailable (non-Linux, no systemd-logind, probe failure)."""
  194. # Auto-detect: macOS overlay can peg a core indefinitely after a computer_use session (#47032). Prefer
  195. # off until the driver teardown is solid; set computer_use.no_overlay: false to keep the cursor.
  196. if sys.platform != "linux":
  197. return None
  198. try:
  199. proc = _run_quiet(["loginctl", "list-sessions", "--no-legend"], timeout=2.0)
  200. seats = [line.split()[0] for line in proc.stdout.splitlines() if len(line.split()) >= 2 and "seat" in line]
  201. if proc.returncode != 0 or not seats:
  202. return None
  203. return not any("LockedHint=no" in _run_quiet(["loginctl", "show-session", s, "-p", "LockedHint"], timeout=2.0).stdout
  204. for s in seats)
  205. except Exception:
  206. return None
  207. def _empty_discovery_reason() -> str:
  208. """One-line diagnosis for 'window discovery found nothing'."""
  209. if _linux_session_locked() is True:
  210. return ("the desktop session is LOCKED (loginctl LockedHint=yes) — unlock the screen; "
  211. "a locked compositor hides windows and freezes app renderers")
  212. if sys.platform == "linux" and not os.environ.get("DISPLAY"):
  213. return "no DISPLAY is set — X11/XWayland is not reachable from this process"
  214. if sys.platform == "darwin": # headless Mac / asleep panel: ScreenCaptureKit has 0 shareable displays while TCC looks fine
  215. return ("window discovery returned no windows; on macOS this usually means no shareable display (headless Mac or "
  216. "panel asleep) — wake the display or attach a monitor/HDMI dummy, then run `hermes computer-use doctor`")
  217. return "window discovery returned no windows; run `hermes computer-use doctor` (display reachability, AX capability)"
  218. class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
  219. """Default computer-use backend. Cross-platform via cua-driver MCP."""
  220. def __init__(self, permission_mode: str = "standard") -> None:
  221. if permission_mode not in {"standard", "bounded", "unrestricted"}:
  222. raise ValueError(f"unsupported cua-driver permission mode: {permission_mode}")
  223. self.permission_mode = permission_mode
  224. self._embedded_daemon: Optional[_EmbeddedCuaDaemon] = None
  225. if permission_mode != "standard":
  226. # Manifest: mandatory for bounded (the daemon validates it), optional for unrestricted where it still
  227. # caps what an approval-bypassed run may touch.
  228. raw = _computer_use_cfg().get("capability_manifest")
  229. # Resolve at daemon launch, after start() reconciles the PM pin.
  230. self._embedded_daemon = _EmbeddedCuaDaemon(
  231. "", permission_mode,
  232. capability_manifest=raw.strip() if isinstance(raw, str) and raw.strip() else None)
  233. self._bridge = _AsyncBridge()
  234. self._session = _CuaDriverSession(self._bridge, self._embedded_daemon)
  235. # Sticky target (set by capture()/focus_app(), used by actions): `_active_pid`, `_active_window_id`, `_last_app`,
  236. # `_last_target` (exact identity for capture_after — Linux app names may be generic, e.g. several unrelated Qt
  237. # windows all say Qt6Application), `_snapshot_tokens` (element_index -> element_token, attached to actions so
  238. # cua-driver reports "stale" instead of silently re-resolving).
  239. self._clear_active_target()
  240. # Public session label (one per Hermes run) sent as `session` on every call: owns the cursor color and
  241. # gives config/recording state a stable owner across transport restarts. Part of the 0.20 runtime contract.
  242. self._session_id: str = f"hermes-{uuid.uuid4().hex[:12]}"
  243. self._session.set_transport_reset_callback(self._handle_transport_reset)
  244. def _handle_transport_reset(self) -> None:
  245. """Invalidate every capability minted by the replaced transport."""
  246. self._clear_active_target()
  247. def start(self) -> None:
  248. # Driver inside the terminal backend: the sandbox image pins its own cua-driver; the host
  249. # binary (if any) is not the one that will run, so neither its acquisition nor its contract
  250. # matters. On the host, runtime acquisition is on-demand, never the explicit install command
  251. # (which may elevate for host setup and bypass the lazy-install gate).
  252. if sandbox_mcp_invocation() is not None:
  253. contract = {"ready": True}
  254. else:
  255. if not os.environ.get(_CUA_DRIVER_CMD_ENV, "").strip():
  256. from pm import ensure
  257. ensure("cua-driver")
  258. contract = cua_driver_runtime_contract_status()
  259. if not contract.get("ready"):
  260. raise RuntimeError(f"cua-driver is not ready: {contract.get('reason') or 'runtime contract is incomplete'}. "
  261. + ("Update the binary selected by HERMES_CUA_DRIVER_CMD or remove that override."
  262. if os.environ.get(_CUA_DRIVER_CMD_ENV, "").strip() else "Run `hermes computer-use install` to repair it."))
  263. # The MCP client SDK (`mcp`) is an optional dependency (the
  264. # `computer-use` / `mcp` extras), not part of Hermes' minimal core.
  265. # Lazy-install it on first use — the same pattern every other optional
  266. # backend uses — so users never hit an opaque `No module named 'mcp'`
  267. # at invoke time. Auto-install is gated by `security.allow_lazy_installs`
  268. # (default on); when it's disabled or fails, ensure_import() raises
  269. # InstallError explaining why PM could not enable the SDK,
  270. # which surfaces via the backend-unavailable path in tool.py.
  271. from pm import ensure_import
  272. ensure_import("computer-use")
  273. # A just-installed package may not be importable until the import
  274. # machinery's caches are refreshed within this process.
  275. import importlib
  276. importlib.invalidate_caches()
  277. with contextlib.ExitStack() as rollback: # a failed start stops the private daemon, then re-raises
  278. if self._embedded_daemon is not None:
  279. rollback.callback(self._embedded_daemon.stop) and self._embedded_daemon.start()
  280. self._session.start()
  281. rollback.pop_all()
  282. # Declare this run's identity. Non-fatal: cua-driver accepts anonymous calls (cursor won't render), so degrade.
  283. self._best_effort("start_session failed (continuing anonymous)",
  284. self._session.call_tool, "start_session", {"session": self._session_id})
  285. # Post-handshake tuning guards on `_started`: before the handshake flips it, call_tool would re-enter
  286. # session.start() (stubbed start() recurses).
  287. if self._session._started:
  288. max_dim = _computer_use_max_image_dimension()
  289. if max_dim: # smaller screenshots cost less over the daemon socket and per turn
  290. self._best_effort("set_config(max_image_dimension) failed",
  291. self.set_config, max_image_dimension=max_dim)
  292. if _cua_no_overlay(): # belt-and-suspenders when --no-overlay is unsupported or ignored
  293. self._best_effort("set_agent_cursor_enabled failed",
  294. self.set_agent_cursor_enabled, False, cursor_id=self._session_id)
  295. def stop(self) -> None:
  296. # Best-effort end_session so the driver cleans per-session state (cursor overlay, recording ownership,
  297. # config overrides); the connection drop below releases daemon-side state regardless.
  298. if self._session._started:
  299. self._best_effort("end_session failed (continuing teardown)",
  300. self._session.call_tool, "end_session", {"session": self._session_id})
  301. with contextlib.ExitStack() as teardown: # every step runs even if one raised (LIFO: session, bridge, daemon)
  302. self._embedded_daemon is None or teardown.callback(self._embedded_daemon.stop)
  303. teardown.callback(self._bridge.stop)
  304. teardown.callback(self._session.stop)
  305. @staticmethod
  306. def _best_effort(what: str, fn, *args: Any, **kwargs: Any) -> None:
  307. """Run a non-fatal driver call, logging (debug) instead of raising."""
  308. try:
  309. fn(*args, **kwargs)
  310. except Exception as e:
  311. logger.debug("cua-driver %s: %s", what, e)
  312. def is_available(self) -> bool:
  313. return sys.platform in ("darwin", "win32", "linux") and cua_driver_binary_available() # other Unix-likes untested E2E
  314. def _clear_active_target(self) -> None:
  315. """Forget a capture/focus target so a failed lookup cannot misroute input."""
  316. self._active_pid = self._active_window_id = self._last_app = self._last_target = None
  317. # Surface 6 of NousResearch/hermes-agent#47072: per-snapshot `element_index -> element_token` map
  318. # populated on capture(). Action tools (click/scroll/set_value/...) attach the matching token
  319. # alongside `element_index` so cua-driver detects "stale" explicitly instead of silently
  320. # re-resolving to a different element. Cleared whenever a fresh capture overwrites the snapshot
  321. # context.
  322. self._snapshot_tokens: dict[int, str] = {}
  323. def _set_active_target(self, target: dict[str, Any]) -> None:
  324. self._active_pid = target["pid"]
  325. self._active_window_id = target["window_id"]
  326. self._snapshot_tokens = {} # prior snapshot's tokens: disarm before any capture so an exception can't pair them
  327. self._last_target = {"pid": self._active_pid, "window_id": self._active_window_id}
  328. def launch_app(self, *, bundle_id: Optional[str] = None, name: Optional[str] = None,
  329. urls: Optional[list[str]] = None, additional_arguments: Optional[list[str]] = None,
  330. creates_new_application_instance: bool = False) -> dict[str, Any]:
  331. """Idempotent launch returning ``{pid, bundle_id, name, windows[]}``. ``creates_new_application_instance=True``
  332. forces a fresh instance so concurrent runs touching the same app get isolated windows."""
  333. if not bundle_id and not name:
  334. raise ValueError("launch_app requires either bundle_id or name")
  335. args: dict[str, Any] = {"session": self._session_id, **{k: v for k, v in (
  336. ("bundle_id", bundle_id), ("name", name), ("urls", urls and list(urls)),
  337. ("additional_arguments", additional_arguments and list(additional_arguments)),
  338. ("creates_new_application_instance", creates_new_application_instance or None)) if v}}
  339. out = self._session.call_tool("launch_app", args)
  340. return out["structuredContent"] or {"data": out["data"]}
  341. def bring_to_front(self, *, pid: int, window_id: Optional[int] = None) -> ActionResult:
  342. """Activate a window so subsequent foreground-dispatched input lands on it."""
  343. args: dict[str, Any] = {"pid": int(pid), **({} if window_id is None else {"window_id": int(window_id)})}
  344. # Strict live schema with no session property: a standalone native focus op, not a session-scoped input action.
  345. return self._action("bring_to_front", args, inject_session=False)
  346. def set_agent_cursor_enabled(self, enabled: bool, *, cursor_id: Optional[str] = None) -> ActionResult:
  347. """Toggle the agent cursor overlay's visibility for this run."""
  348. return self._action("set_agent_cursor_enabled",
  349. {"enabled": bool(enabled), **({"cursor_id": cursor_id} if cursor_id else {})})
  350. def set_config(self, **config) -> ActionResult:
  351. """Set cua-driver config keys (e.g. ``max_image_dimension``); unknown keys pass through — cua-driver validates."""
  352. return self._action("set_config", dict(config))
  353. def call_tool(self, name: str, args: Optional[dict[str, Any]] = None, *, timeout: float = 30.0) -> dict[str, Any]:
  354. """Generic escape hatch: call any cua-driver MCP tool by name. ``session`` is injected via setdefault, so
  355. this is the supported path for tools the wrapper does not type-wrap (preferred over ``self._session.call_tool``)."""
  356. payload = dict(args) if args else {}
  357. payload.setdefault("session", self._session_id)
  358. return self._session.call_tool(name, payload, timeout=timeout)
  359. def _action(self, name: str, args: dict[str, Any], *, inject_session: bool = True) -> ActionResult:
  360. # Attach the snapshot's `element_token` to an `element_index` call so a superseded snapshot yields an explicit
  361. # 'stale' error. Two ways to establish support, the live input schema first: cua-driver 0.21+ stopped
  362. # publishing per-tool `capabilities[]` while still accepting `element_token` in its schema, and it REFUSES a
  363. # bare `element_index` (`snapshot_id_required`) — gating on the capability alone broke EVERY element click and
  364. # left only pixel clicks working. The capability check stays so older drivers that shipped the vocabulary keep
  365. # working; drivers advertising neither (`additionalProperties: false`) must never see the property.
  366. idx = args.get("element_index")
  367. token = self._snapshot_tokens.get(idx) if isinstance(idx, int) else None
  368. if token and (self._session.supports_input_property(name, "element_token")
  369. or self._session.supports_capability("accessibility.element_tokens", tool=name)):
  370. args["element_token"] = token
  371. if inject_session: # setdefault preserves any explicit session a caller already supplied
  372. args.setdefault("session", self._session_id)
  373. try:
  374. out = self._session.call_tool(name, args)
  375. except Exception as e:
  376. logger.exception("cua-driver %s call failed", name)
  377. return ActionResult(ok=False, action=name, message=f"cua-driver error: {e}")
  378. data = out["data"]
  379. structured = out.get("structuredContent") or {}
  380. message = (str(data.get("message", "")) if isinstance(data, dict) else data if isinstance(data, str) else "") \
  381. or (str(structured.get("message", "")) if isinstance(structured, dict) else "")
  382. # Merge data + structuredContent into meta, structured winning on overlap (canonical verdict surface).
  383. meta = {k: v for part in (data, structured) if isinstance(part, dict) for k, v in part.items()}
  384. return _action_result_from(name, not out["isError"], message, meta, structured,
  385. requested_delivery=args.get("delivery_mode"))