| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430 |
- """Cua-driver backend (macOS, Windows, Linux): MCP over stdio to `cua-driver`. The async `mcp` SDK runs on a
- background loop (``cua_backend_session``); the same tool surface works on all three platforms, and per-host gaps
- (no DISPLAY, missing AT-SPI, TCC) surface via `hermes computer-use doctor` instead of failing silently. Install
- with `hermes computer-use install`. The macOS path uses private SkyLight SPIs that can break on OS updates.
- Siblings: ``cua_backend_driver`` (binary/contract), ``cua_backend_capture`` + ``cua_backend_input``
- (mixins), ``cua_backend_parse``, ``cua_backend_session`` (bridge + session + CLI fallback), ``cua_backend_daemon``
- (private daemon + macOS app identity). Siblings look this module's config/policy helpers up lazily."""
- from __future__ import annotations
- import contextlib
- import logging
- import os
- import subprocess
- import sys
- import uuid
- from pathlib import PureWindowsPath
- from typing import Any, Dict, List, Optional, Tuple
- from hermes_cli._subprocess_compat import windows_hide_flags
- from hermes_platform.host.runtime import is_wsl
- from tools.computer_use.backend import ActionResult, ComputerUseBackend
- from tools.computer_use.cua_backend_capture import _CaptureMixin
- from tools.computer_use.cua_backend_daemon import _EmbeddedCuaDaemon
- from tools.computer_use.cua_backend_driver import (
- _CUA_DRIVER_CMD_ENV, cua_driver_binary_available, cua_driver_runtime_contract_status,
- resolve_cua_driver_cmd)
- from tools.computer_use.cua_backend_input import _InputMixin
- from tools.computer_use.cua_backend_parse import _action_result_from
- from tools.computer_use.cua_backend_session import _AsyncBridge, _CuaDriverSession
- logger = logging.getLogger(__name__)
- # cua-driver's anonymous PostHog telemetry gate ("0" disables; absent => ON upstream).
- _CUA_TELEMETRY_ENV_VAR = "CUA_DRIVER_RS_TELEMETRY_ENABLED"
- _CUA_NATIVE_WAYLAND_ENV_VAR = "CUA_DRIVER_RS_ENABLE_WAYLAND"
- def _computer_use_cfg() -> dict[str, Any]:
- """The ``computer_use`` config block, or ``{}`` when config is unreadable."""
- with contextlib.suppress(Exception):
- from hermes_cli.config import load_config
- return (load_config() or {}).get("computer_use") or {}
- return {}
- def _cua_no_overlay() -> bool:
- """Pass ``--no-overlay``? ``computer_use.no_overlay`` overrides; else off on macOS (cursor-overlay redraw
- loop can peg a core after a session), headless Linux / WSL2 / containers, and Linux X11 (the overlay is a
- fullscreen always-on-top all-workspaces window with no compositor-owned lifecycle, so an unclean session
- end can leave it wedged over every app); on for Windows and Linux Wayland (compositor owns the surface).
- Explicit ``True`` / ``False`` overrides auto-detection. See #28152, #47032.
- """
- val = _computer_use_cfg().get("no_overlay")
- if val is not None or sys.platform != "linux":
- return bool(val) if val is not None else sys.platform == "darwin"
- wsl = is_wsl()
- return wsl or not os.environ.get("DISPLAY") or (
- # Linux/X11: the cursor overlay is a fullscreen, always-on-top, all-workspaces X11 window
- # (save-unders path). An unclean session end (agent interrupted mid-capture, stale target window)
- # can leave it stuck above every app on every workspace, wedging desktop input until the app
- # restarts — the same failure class as the HUD window on Mutter/X11 (#83473). There is no
- # compositor-owned surface to tear down with the client connection, so default the overlay off on
- # X11 too; set computer_use.no_overlay: false to keep the cursor. Wayland keeps it: the compositor
- # owns the overlay surface lifecycle there.
- os.environ.get("XDG_SESSION_TYPE") != "wayland" and not os.environ.get("WAYLAND_DISPLAY"))
- def _cua_telemetry_disabled() -> bool:
- """True unless ``computer_use.cua_telemetry`` opts in (unreadable config fails SAFE toward disabling)."""
- return not bool(_computer_use_cfg().get("cua_telemetry", False))
- def _cua_configured_permission_mode() -> str:
- """``computer_use.permission_mode``: ``standard`` (default) or ``bounded``; unknown values fall closed to
- ``standard``. ``unrestricted`` is deliberately NOT a config value — it stays tied to the per-session YOLO
- toggle so a stale config line can never silently bypass approvals."""
- raw = str(_computer_use_cfg().get("permission_mode", "standard") or "").strip().lower()
- return "bounded" if raw == "bounded" else "standard"
- # ``computer_use.ax_max_elements``: bound on the DRIVER's accessibility-tree walk per capture. The
- # visible-element cap in tool.py (_DEFAULT_MAX_ELEMENTS) trims the RESPONSE only, so without this an
- # unbounded walk pays for nodes the model never sees; 0 disables the bound (driver default: 2,000
- # elements / depth 25).
- _DEFAULT_AX_MAX_ELEMENTS = 200
- def _cua_configured_ax_max_elements() -> int:
- """Bound on ``get_window_state``'s AX walk; 0 = no bound (driver default). Unreadable config fails to
- the default, never to unbounded. Measured on macOS (cua-driver 0.28.2, M-series): a 1,444-node Chrome
- window 540 ms -> 83 ms and a 456-node Finder window 6.9 s -> 0.6 s at 200, on the CLI transport. The
- bounded result is a prefix of the unbounded walk, so the elements the model sees are unchanged. Raise
- it toward 400 to keep the full first 100 visible elements on a pathological tree (~1.4 s on Finder);
- cost grows with the bound, and a bound in the thousands buys nothing the response cap keeps. This
- bounds the nodes COLLECTED, not the walk's wall clock: a target whose accessibility surface exceeds the
- driver's own 20 s walk timeout still fails at every bound (and every depth bound), so a timeout error is
- never an argument for a smaller or larger value here."""
- raw = _computer_use_cfg().get("ax_max_elements", _DEFAULT_AX_MAX_ELEMENTS)
- try:
- return max(0, int(raw))
- except (TypeError, ValueError):
- return _DEFAULT_AX_MAX_ELEMENTS
- def _manifest_is_mode_independent(path: str) -> bool:
- """True when this manifest may accompany any permission mode: v1/v2 declare ``mode: bounded`` and abort
- startup under an unrestricted runtime; v3 has no mode and is the ceiling the driver accepts alongside any
- mode. Unreadable / unparseable -> False (forwarding one would turn a working session into a hard startup
- failure; bounded forwards unconditionally anyway)."""
- try:
- import hermes_yaml as yaml
- with open(path, "r", encoding="utf-8-sig") as handle:
- parsed = yaml.safe_load(handle)
- except Exception:
- logger.debug("could not read capability manifest %s", path, exc_info=True)
- return False
- version = parsed.get("version") if isinstance(parsed, dict) else None
- return isinstance(version, int) and not isinstance(version, bool) and version >= 3
- def _computer_use_max_image_dimension() -> Optional[int]:
- """``computer_use.max_image_dimension`` longest-edge cap (default 1456 = aux-vision downscale); ``0``/negative -> None."""
- try:
- dim = int(_computer_use_cfg().get("max_image_dimension", 1456))
- except (TypeError, ValueError):
- dim = 1456
- return dim if dim > 0 else None
- def desktop_identity(env: Optional[dict[str, str]] = None) -> str:
- """The screen a backend spawned from ``env`` acts on: its DISPLAY (``''`` when none). Recorded next to the
- cached backend so a Bot Desktop that starts (or restarts on another number) AFTER the backend was cached is
- noticed — the cached cua-driver still points at the old seat or at no display at all."""
- return str((cua_driver_child_env(env) if env is None else env).get("DISPLAY") or "")
- def backend_display_stale(recorded: str, current: str) -> bool:
- """True when a cached backend's recorded display identity no longer matches the one a fresh spawn would get."""
- return (recorded or "") != (current or "")
- def cua_driver_child_env(base_env: Optional[dict[str, str]] = None) -> dict[str, str]:
- """Env for spawning cua-driver: ``base_env`` (default ``os.environ``) plus ``CUA_DRIVER_RS_TELEMETRY_ENABLED=0``
- unless the user opted in, plus the native-Wayland bridge (``computer_use.native_wayland`` config opt-in, only when
- the child has a Wayland display). Used by every spawn site (MCP, status, doctor, install) so CLI and gateway
- runtimes share one policy."""
- env = dict(os.environ if base_env is None else base_env)
- # A running Bot Desktop for this profile owns the agent's screen: DISPLAY/XAUTHORITY/DBUS point there so
- # cua-driver never acts on a seat the human is sitting at (#90374 class) and headless hosts get a display.
- from tools.bot_desktop.runtime import desktop_env as _bot_desktop_env
- env = _bot_desktop_env(env)
- if _cua_telemetry_disabled():
- env[_CUA_TELEMETRY_ENV_VAR] = "0"
- if sys.platform == "linux" and env.get("WAYLAND_DISPLAY") and bool(_computer_use_cfg().get("native_wayland", False)):
- env[_CUA_NATIVE_WAYLAND_ENV_VAR] = "1"
- return env
- def sandbox_mcp_invocation() -> Optional[tuple[tuple[str, list[str]], dict[str, str]]]:
- """``((command, args), child_env)`` spawning ``cua-driver mcp`` INSIDE the terminal backend when the Bot
- Desktop is placed there (the driver in the sandbox image drives the sandbox's own screen); None on a
- gateway-hosted desktop, where the local driver is used. Placement is the authority: a ``terminal``
- placement gets its screen started here and a ``refused`` one raises — the host driver is never the
- fallback for a sandbox whose screen is down."""
- from tools.bot_desktop import placement, runtime as _bd_runtime
- if _bd_runtime.tool_placement() == placement.GATEWAY:
- return None
- published = _bd_runtime.published_env()
- if not published.get("DISPLAY"):
- raise RuntimeError("the screen inside the terminal backend's sandbox is gone; start it again")
- from tools.bot_desktop import sandbox_host
- env = _bd_runtime._sandbox_env(create=True)
- if env is None:
- raise RuntimeError("the terminal backend's sandbox is not running, so there is nowhere to run cua-driver")
- command, args = sandbox_host.cua_mcp_invocation(env, _bd_runtime._profile_name(),
- {**published, _CUA_TELEMETRY_ENV_VAR: "0"})
- _bd_runtime.touch_activity()
- return (command, args), {"PATH": os.environ.get("PATH", "")}
- def sanitized_cua_driver_env() -> dict[str, str]:
- """``cua_driver_child_env()`` with Hermes provider secrets stripped — cua-driver is a third-party binary and must
- never inherit API keys. Falls back to the unsanitized telemetry env if the sanitizer can't import."""
- env = cua_driver_child_env()
- with contextlib.suppress(Exception):
- # cua-driver is a third-party binary — never hand it provider API keys via inherited env (same
- # policy as the manifest probe and MCP spawn; #53503/#55709/#58889 lineage).
- from tools.environments.local import _sanitize_subprocess_env
- return _sanitize_subprocess_env(env)
- return env
- def _run_quiet(argv: list[str], *, timeout: float, swallow: Any = (), **kw: Any) -> Any:
- """``subprocess.run`` for short probe verbs: text mode, stdin=DEVNULL unless overridden (older drivers fall into a
- stdin-reading mode on unknown verbs; EOF makes them exit fast instead of blocking until the timeout), output
- captured unless the caller redirects it. Exceptions in ``swallow`` return None; others raise."""
- kw.setdefault("stdin", subprocess.DEVNULL)
- kw.setdefault("encoding", "utf-8")
- kw.setdefault("errors", "replace")
- "stdout" in kw or kw.setdefault("capture_output", True)
- try:
- return subprocess.run(argv, text=True, timeout=timeout, stdin=kw.pop("stdin"), encoding=kw.pop("encoding"), # noqa: PLW1510 -- forwarding wrapper: callers pass subprocess kwargs via **kwargs (check may arrive through it); an explicit check=False would raise TypeError
- errors=kw.pop("errors"), **kw)
- except swallow:
- return None
- def _run_driver(driver_cmd: str, *args: str, timeout: float, swallow: Any = ()) -> Any:
- """Run a short cua-driver verb with the sanitized env and hidden window."""
- return _run_quiet([driver_cmd, *args], timeout=timeout, swallow=swallow, encoding="utf-8",
- errors="replace", creationflags=windows_hide_flags(), env=sanitized_cua_driver_env())
- def cua_daemon_listening(driver_cmd: str, socket_path: Optional[str] = None, *, timeout: float = 3.0) -> Optional[bool]:
- """Socket-level liveness of a ``cua-driver serve`` daemon: ``cua-driver status`` connects to the daemon
- socket (the driver's default, or ``socket_path``) and exits 0 only when a daemon answers. False when the
- CLI reports the daemon is not running, None when the probe itself failed (unknown). Never raises.
- The binary-level runtime contract (``manifest``) cannot see this — a dead daemon looks healthy there (#114748)."""
- args = ("status", "--socket", socket_path) if socket_path else ("status",)
- proc = _run_driver(driver_cmd, *args, timeout=timeout, swallow=(OSError, subprocess.SubprocessError))
- if proc is None:
- return None
- if proc.returncode == 0:
- return True
- return False if "not running" in f"{proc.stdout}\n{proc.stderr}".lower() else None
- def _linux_session_locked() -> Optional[bool]:
- """Is the graphical session locked? (Linux; best-effort.) A locked KDE/GNOME session freezes renderers and
- half-disables the AX tree, so discovery legitimately returns nothing — which otherwise reads as a driver bug.
- True/False when loginctl answers, None when unavailable (non-Linux, no systemd-logind, probe failure)."""
- # Auto-detect: macOS overlay can peg a core indefinitely after a computer_use session (#47032). Prefer
- # off until the driver teardown is solid; set computer_use.no_overlay: false to keep the cursor.
- if sys.platform != "linux":
- return None
- try:
- proc = _run_quiet(["loginctl", "list-sessions", "--no-legend"], timeout=2.0)
- seats = [line.split()[0] for line in proc.stdout.splitlines() if len(line.split()) >= 2 and "seat" in line]
- if proc.returncode != 0 or not seats:
- return None
- return not any("LockedHint=no" in _run_quiet(["loginctl", "show-session", s, "-p", "LockedHint"], timeout=2.0).stdout
- for s in seats)
- except Exception:
- return None
- def _empty_discovery_reason() -> str:
- """One-line diagnosis for 'window discovery found nothing'."""
- if _linux_session_locked() is True:
- return ("the desktop session is LOCKED (loginctl LockedHint=yes) — unlock the screen; "
- "a locked compositor hides windows and freezes app renderers")
- if sys.platform == "linux" and not os.environ.get("DISPLAY"):
- return "no DISPLAY is set — X11/XWayland is not reachable from this process"
- if sys.platform == "darwin": # headless Mac / asleep panel: ScreenCaptureKit has 0 shareable displays while TCC looks fine
- return ("window discovery returned no windows; on macOS this usually means no shareable display (headless Mac or "
- "panel asleep) — wake the display or attach a monitor/HDMI dummy, then run `hermes computer-use doctor`")
- return "window discovery returned no windows; run `hermes computer-use doctor` (display reachability, AX capability)"
- class CuaDriverBackend(_CaptureMixin, _InputMixin, ComputerUseBackend):
- """Default computer-use backend. Cross-platform via cua-driver MCP."""
- def __init__(self, permission_mode: str = "standard") -> None:
- if permission_mode not in {"standard", "bounded", "unrestricted"}:
- raise ValueError(f"unsupported cua-driver permission mode: {permission_mode}")
- self.permission_mode = permission_mode
- self._embedded_daemon: Optional[_EmbeddedCuaDaemon] = None
- if permission_mode != "standard":
- # Manifest: mandatory for bounded (the daemon validates it), optional for unrestricted where it still
- # caps what an approval-bypassed run may touch.
- raw = _computer_use_cfg().get("capability_manifest")
- # Resolve at daemon launch, after start() reconciles the PM pin.
- self._embedded_daemon = _EmbeddedCuaDaemon(
- "", permission_mode,
- capability_manifest=raw.strip() if isinstance(raw, str) and raw.strip() else None)
- self._bridge = _AsyncBridge()
- self._session = _CuaDriverSession(self._bridge, self._embedded_daemon)
- # Sticky target (set by capture()/focus_app(), used by actions): `_active_pid`, `_active_window_id`, `_last_app`,
- # `_last_target` (exact identity for capture_after — Linux app names may be generic, e.g. several unrelated Qt
- # windows all say Qt6Application), `_snapshot_tokens` (element_index -> element_token, attached to actions so
- # cua-driver reports "stale" instead of silently re-resolving).
- self._clear_active_target()
- # Public session label (one per Hermes run) sent as `session` on every call: owns the cursor color and
- # gives config/recording state a stable owner across transport restarts. Part of the 0.20 runtime contract.
- self._session_id: str = f"hermes-{uuid.uuid4().hex[:12]}"
- self._session.set_transport_reset_callback(self._handle_transport_reset)
- def _handle_transport_reset(self) -> None:
- """Invalidate every capability minted by the replaced transport."""
- self._clear_active_target()
- def start(self) -> None:
- # Driver inside the terminal backend: the sandbox image pins its own cua-driver; the host
- # binary (if any) is not the one that will run, so neither its acquisition nor its contract
- # matters. On the host, runtime acquisition is on-demand, never the explicit install command
- # (which may elevate for host setup and bypass the lazy-install gate).
- if sandbox_mcp_invocation() is not None:
- contract = {"ready": True}
- else:
- if not os.environ.get(_CUA_DRIVER_CMD_ENV, "").strip():
- from pm import ensure
- ensure("cua-driver")
- contract = cua_driver_runtime_contract_status()
- if not contract.get("ready"):
- raise RuntimeError(f"cua-driver is not ready: {contract.get('reason') or 'runtime contract is incomplete'}. "
- + ("Update the binary selected by HERMES_CUA_DRIVER_CMD or remove that override."
- if os.environ.get(_CUA_DRIVER_CMD_ENV, "").strip() else "Run `hermes computer-use install` to repair it."))
- # The MCP client SDK (`mcp`) is an optional dependency (the
- # `computer-use` / `mcp` extras), not part of Hermes' minimal core.
- # Lazy-install it on first use — the same pattern every other optional
- # backend uses — so users never hit an opaque `No module named 'mcp'`
- # at invoke time. Auto-install is gated by `security.allow_lazy_installs`
- # (default on); when it's disabled or fails, ensure_import() raises
- # InstallError explaining why PM could not enable the SDK,
- # which surfaces via the backend-unavailable path in tool.py.
- from pm import ensure_import
- ensure_import("computer-use")
- # A just-installed package may not be importable until the import
- # machinery's caches are refreshed within this process.
- import importlib
- importlib.invalidate_caches()
- with contextlib.ExitStack() as rollback: # a failed start stops the private daemon, then re-raises
- if self._embedded_daemon is not None:
- rollback.callback(self._embedded_daemon.stop) and self._embedded_daemon.start()
- self._session.start()
- rollback.pop_all()
- # Declare this run's identity. Non-fatal: cua-driver accepts anonymous calls (cursor won't render), so degrade.
- self._best_effort("start_session failed (continuing anonymous)",
- self._session.call_tool, "start_session", {"session": self._session_id})
- # Post-handshake tuning guards on `_started`: before the handshake flips it, call_tool would re-enter
- # session.start() (stubbed start() recurses).
- if self._session._started:
- max_dim = _computer_use_max_image_dimension()
- if max_dim: # smaller screenshots cost less over the daemon socket and per turn
- self._best_effort("set_config(max_image_dimension) failed",
- self.set_config, max_image_dimension=max_dim)
- if _cua_no_overlay(): # belt-and-suspenders when --no-overlay is unsupported or ignored
- self._best_effort("set_agent_cursor_enabled failed",
- self.set_agent_cursor_enabled, False, cursor_id=self._session_id)
- def stop(self) -> None:
- # Best-effort end_session so the driver cleans per-session state (cursor overlay, recording ownership,
- # config overrides); the connection drop below releases daemon-side state regardless.
- if self._session._started:
- self._best_effort("end_session failed (continuing teardown)",
- self._session.call_tool, "end_session", {"session": self._session_id})
- with contextlib.ExitStack() as teardown: # every step runs even if one raised (LIFO: session, bridge, daemon)
- self._embedded_daemon is None or teardown.callback(self._embedded_daemon.stop)
- teardown.callback(self._bridge.stop)
- teardown.callback(self._session.stop)
- @staticmethod
- def _best_effort(what: str, fn, *args: Any, **kwargs: Any) -> None:
- """Run a non-fatal driver call, logging (debug) instead of raising."""
- try:
- fn(*args, **kwargs)
- except Exception as e:
- logger.debug("cua-driver %s: %s", what, e)
- def is_available(self) -> bool:
- return sys.platform in ("darwin", "win32", "linux") and cua_driver_binary_available() # other Unix-likes untested E2E
- def _clear_active_target(self) -> None:
- """Forget a capture/focus target so a failed lookup cannot misroute input."""
- self._active_pid = self._active_window_id = self._last_app = self._last_target = None
- # Surface 6 of NousResearch/hermes-agent#47072: per-snapshot `element_index -> element_token` map
- # populated on capture(). Action tools (click/scroll/set_value/...) attach the matching token
- # alongside `element_index` so cua-driver detects "stale" explicitly instead of silently
- # re-resolving to a different element. Cleared whenever a fresh capture overwrites the snapshot
- # context.
- self._snapshot_tokens: dict[int, str] = {}
- def _set_active_target(self, target: dict[str, Any]) -> None:
- self._active_pid = target["pid"]
- self._active_window_id = target["window_id"]
- self._snapshot_tokens = {} # prior snapshot's tokens: disarm before any capture so an exception can't pair them
- self._last_target = {"pid": self._active_pid, "window_id": self._active_window_id}
- def launch_app(self, *, bundle_id: Optional[str] = None, name: Optional[str] = None,
- urls: Optional[list[str]] = None, additional_arguments: Optional[list[str]] = None,
- creates_new_application_instance: bool = False) -> dict[str, Any]:
- """Idempotent launch returning ``{pid, bundle_id, name, windows[]}``. ``creates_new_application_instance=True``
- forces a fresh instance so concurrent runs touching the same app get isolated windows."""
- if not bundle_id and not name:
- raise ValueError("launch_app requires either bundle_id or name")
- args: dict[str, Any] = {"session": self._session_id, **{k: v for k, v in (
- ("bundle_id", bundle_id), ("name", name), ("urls", urls and list(urls)),
- ("additional_arguments", additional_arguments and list(additional_arguments)),
- ("creates_new_application_instance", creates_new_application_instance or None)) if v}}
- out = self._session.call_tool("launch_app", args)
- return out["structuredContent"] or {"data": out["data"]}
- def bring_to_front(self, *, pid: int, window_id: Optional[int] = None) -> ActionResult:
- """Activate a window so subsequent foreground-dispatched input lands on it."""
- args: dict[str, Any] = {"pid": int(pid), **({} if window_id is None else {"window_id": int(window_id)})}
- # Strict live schema with no session property: a standalone native focus op, not a session-scoped input action.
- return self._action("bring_to_front", args, inject_session=False)
- def set_agent_cursor_enabled(self, enabled: bool, *, cursor_id: Optional[str] = None) -> ActionResult:
- """Toggle the agent cursor overlay's visibility for this run."""
- return self._action("set_agent_cursor_enabled",
- {"enabled": bool(enabled), **({"cursor_id": cursor_id} if cursor_id else {})})
- def set_config(self, **config) -> ActionResult:
- """Set cua-driver config keys (e.g. ``max_image_dimension``); unknown keys pass through — cua-driver validates."""
- return self._action("set_config", dict(config))
- def call_tool(self, name: str, args: Optional[dict[str, Any]] = None, *, timeout: float = 30.0) -> dict[str, Any]:
- """Generic escape hatch: call any cua-driver MCP tool by name. ``session`` is injected via setdefault, so
- this is the supported path for tools the wrapper does not type-wrap (preferred over ``self._session.call_tool``)."""
- payload = dict(args) if args else {}
- payload.setdefault("session", self._session_id)
- return self._session.call_tool(name, payload, timeout=timeout)
- def _action(self, name: str, args: dict[str, Any], *, inject_session: bool = True) -> ActionResult:
- # Attach the snapshot's `element_token` to an `element_index` call so a superseded snapshot yields an explicit
- # 'stale' error. Two ways to establish support, the live input schema first: cua-driver 0.21+ stopped
- # publishing per-tool `capabilities[]` while still accepting `element_token` in its schema, and it REFUSES a
- # bare `element_index` (`snapshot_id_required`) — gating on the capability alone broke EVERY element click and
- # left only pixel clicks working. The capability check stays so older drivers that shipped the vocabulary keep
- # working; drivers advertising neither (`additionalProperties: false`) must never see the property.
- idx = args.get("element_index")
- token = self._snapshot_tokens.get(idx) if isinstance(idx, int) else None
- if token and (self._session.supports_input_property(name, "element_token")
- or self._session.supports_capability("accessibility.element_tokens", tool=name)):
- args["element_token"] = token
- if inject_session: # setdefault preserves any explicit session a caller already supplied
- args.setdefault("session", self._session_id)
- try:
- out = self._session.call_tool(name, args)
- except Exception as e:
- logger.exception("cua-driver %s call failed", name)
- return ActionResult(ok=False, action=name, message=f"cua-driver error: {e}")
- data = out["data"]
- structured = out.get("structuredContent") or {}
- message = (str(data.get("message", "")) if isinstance(data, dict) else data if isinstance(data, str) else "") \
- or (str(structured.get("message", "")) if isinstance(structured, dict) else "")
- # Merge data + structuredContent into meta, structured winning on overlap (canonical verdict surface).
- meta = {k: v for part in (data, structured) if isinstance(part, dict) for k, v in part.items()}
- return _action_result_from(name, not out["isError"], message, meta, structured,
- requested_delivery=args.get("delivery_mode"))
|