| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168 |
- """Abstract backend interface for computer use. Any implementation (cua-driver over MCP, a plugin
- driver, the test noop) returns the shapes below. All methods are synchronous; async is handled inside
- the backend implementation if needed. :class:`ComputerUseProvider` is the factory a provider plugin
- registers (see :mod:`plugins.computer_use`); ``computer_use.backend`` selects exactly one."""
- from __future__ import annotations
- import time
- from abc import ABC, abstractmethod
- from dataclasses import dataclass, field
- from typing import Any, Dict, List, Optional, Tuple
- from agent.provider_base import ProviderBase
- @dataclass
- class UIElement:
- """One interactable element on the current screen."""
- index: int # 1-based SOM index
- role: str # AX role (AXButton, AXTextField, ...)
- label: str = "" # AXTitle / AXDescription / AXValue snippet
- bounds: tuple[int, int, int, int] = (0, 0, 0, 0) # x, y, w, h (logical px)
- app: str = "" # owning bundle ID or app name
- pid: int = 0 # owning process PID
- window_id: int = 0 # SkyLight / CG window ID
- attributes: dict[str, Any] = field(default_factory=dict)
- # Opaque per-snapshot handle from cua-driver, passed alongside `index` for explicit stale-detection: a
- # stale token errors instead of silently re-resolving to a different element. None on older drivers.
- # None for pre-#1961 drivers that didn't carry the field.
- element_token: Optional[str] = None
- @dataclass
- class CaptureResult:
- """Result of a screen capture call. mode="vision" → png_b64 only; mode="ax" → elements
- only; mode="som" (default) → both: the PNG already carries numbered overlays drawn by
- the backend and `elements` holds the matching index → element mapping."""
- mode: str
- width: int # screenshot width (logical px, pre-Anthropic-scale)
- height: int
- png_b64: Optional[str] = None
- elements: list[UIElement] = field(default_factory=list)
- app: str = "" # target app/window the elements were captured for
- window_title: str = ""
- png_bytes_len: int = 0 # raw bytes sent to Anthropic, for token estimation
- # MIME type of `png_b64` when the backend supplied it (cua-driver-rs emits `mimeType` on every image
- # part). None → consumers fall back to base64-prefix sniffing (older drivers).
- # See #1961, #47072.
- image_mime_type: Optional[str] = None
- # Guidance appended to the summary by capture lanes that intentionally return no elements (e.g.
- # full-screen composited grabs) to point the model at an interactive lane.
- note: str = ""
- # ``max_elements`` the backend asked the driver's AX walk to stop at (0 = unbounded / not applicable);
- # ``len(elements) >= ax_max_elements > 0`` means the tree may be truncated.
- ax_max_elements: int = 0
- @dataclass
- class ActionResult:
- """Result of any action (click / type / scroll / drag / key / wait). ``ok`` is
- tool/transport success only — NOT the semantic verdict; read ``effect`` / ``escalation``
- (cua-driver's structured verdict) to pick the next rung of the verify → escalate ladder.
- Structured fields are optional and additive: an older driver that omits
- ``structuredContent`` leaves them ``None``, behavior unchanged.
- Beyond the transport-level ``ok`` flag, this carries cua-driver's structured action verdict so the model
- can follow the documented verify → escalate ladder (NousResearch/hermes-agent#67052).
- """
- ok: bool
- action: str
- message: str = "" # human-readable summary
- capture: Optional[CaptureResult] = None # trailing screenshot, when requested / always-on
- meta: dict[str, Any] = field(default_factory=dict) # debugging / telemetry extras
- verified: Optional[bool] = None # AX read-back: True confirmed, False unconfirmed, None n/a
- effect: Optional[str] = None # "confirmed" | "unverifiable" | "suspected_noop"
- # {"recommended": "px"|"foreground"|"page", "reason": str} — only when driver recommends climbing
- escalation: Optional[dict[str, Any]] = None
- path: Optional[str] = None # delivery rung that ran (e.g. "ax", "x11_pixel", "cgevent_fg")
- degraded: Optional[bool] = None # AX walk found no actionable elements (act by px instead)
- delivery_mode: Optional[str] = None # the delivery_mode the caller requested, echoed back
- code: Optional[str] = None # refusal code, e.g. "background_unavailable", "desktop_scope_disabled"
- class ComputerUseBackend(ABC):
- """Lifecycle: `start()` before first use, `stop()` at shutdown. Pointer/keyboard actions
- take ``delivery_mode`` (background (default) | foreground) and ``bring_to_front``;
- ``button`` is left | right | middle; ``modifiers`` a list of key names. ``element`` args
- are 1-based SOM indices from a prior capture. `direction` is up | down | left | right and
- `amount` is wheel ticks; `keys` is a combo such as 'cmd+s', 'ctrl+alt+t', 'return'."""
- @abstractmethod
- def start(self) -> None: ...
- @abstractmethod
- def stop(self) -> None: ...
- @abstractmethod
- def is_available(self) -> bool: ... # usable on this host right now (check_fn gating, setup wizard)
- @abstractmethod
- def capture(self, mode: str = "som", app: Optional[str] = None, pid: Optional[int] = None,
- window_id: Optional[int] = None) -> CaptureResult: ...
- @abstractmethod
- def click(self, *, element: Optional[int] = None, x: Optional[int] = None, y: Optional[int] = None,
- button: str = "left", click_count: int = 1, modifiers: Optional[list[str]] = None,
- delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
- @abstractmethod
- def drag(self, *, from_element: Optional[int] = None, to_element: Optional[int] = None,
- from_xy: Optional[tuple[int, int]] = None, to_xy: Optional[tuple[int, int]] = None,
- button: str = "left", modifiers: Optional[list[str]] = None,
- delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
- @abstractmethod
- def scroll(self, *, direction: str, amount: int = 3, element: Optional[int] = None,
- x: Optional[int] = None, y: Optional[int] = None, modifiers: Optional[list[str]] = None,
- delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
- @abstractmethod
- def type_text(self, text: str, *, delivery_mode: Optional[str] = None,
- bring_to_front: bool = False) -> ActionResult: ...
- @abstractmethod
- def key(self, keys: str, *, delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
- @abstractmethod
- def list_apps(self) -> list[dict[str, Any]]: ... # running apps with bundle IDs, PIDs, window counts
- def list_windows(self) -> list[dict[str, Any]]:
- """Visible native windows with PID and window identifiers. Optional compatibility hook: backends that
- predate window discovery stay instantiable and report none."""
- return []
- @abstractmethod
- def focus_app(self, app: str, raise_window: bool = False) -> ActionResult: ... # route input to `app` (name / bundle ID)
- @abstractmethod
- def set_value(self, value: str, element: Optional[int] = None) -> ActionResult: ... # e.g. AXPopUpButton selection
- def wait(self, seconds: float) -> ActionResult: # default implementation
- time.sleep(max(0.0, min(seconds, 30.0)))
- return ActionResult(ok=True, action="wait", message=f"waited {seconds:.2f}s")
- class ComputerUseProvider(ProviderBase):
- """Factory for one computer_use driver, registered via ``ctx.register_computer_use_provider`` from a provider
- plugin whose directory name is the ``computer_use.backend`` value (built-in: ``plugins/computer_use/cua``). Only
- the selected provider is imported and instantiated. The model-facing tool schema is identical for every backend:
- an action the driver cannot perform returns ``ActionResult(ok=False, code="unsupported_action", message=...)``
- rather than changing the schema."""
- @abstractmethod
- def create_backend(self, *, permission_mode: str) -> ComputerUseBackend:
- """A fresh, unstarted backend for one Hermes session (the tool calls ``start()``/``stop()``).
- ``permission_mode`` is ``standard`` | ``bounded`` | ``unrestricted`` (approval bypass active)."""
- def is_available(self) -> bool:
- """Usable on this host right now (the tool's ``check_fn``). Cheap: no network, no spawning."""
- return True
- def doctor(self) -> Optional[int]:
- """``hermes computer-use doctor`` for this backend: print findings, return the exit code.
- ``None`` (default) means the backend ships no doctor."""
- return None
|