backend.py 8.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. """Abstract backend interface for computer use. Any implementation (cua-driver over MCP, a plugin
  2. driver, the test noop) returns the shapes below. All methods are synchronous; async is handled inside
  3. the backend implementation if needed. :class:`ComputerUseProvider` is the factory a provider plugin
  4. registers (see :mod:`plugins.computer_use`); ``computer_use.backend`` selects exactly one."""
  5. from __future__ import annotations
  6. import time
  7. from abc import ABC, abstractmethod
  8. from dataclasses import dataclass, field
  9. from typing import Any, Dict, List, Optional, Tuple
  10. from agent.provider_base import ProviderBase
  11. @dataclass
  12. class UIElement:
  13. """One interactable element on the current screen."""
  14. index: int # 1-based SOM index
  15. role: str # AX role (AXButton, AXTextField, ...)
  16. label: str = "" # AXTitle / AXDescription / AXValue snippet
  17. bounds: tuple[int, int, int, int] = (0, 0, 0, 0) # x, y, w, h (logical px)
  18. app: str = "" # owning bundle ID or app name
  19. pid: int = 0 # owning process PID
  20. window_id: int = 0 # SkyLight / CG window ID
  21. attributes: dict[str, Any] = field(default_factory=dict)
  22. # Opaque per-snapshot handle from cua-driver, passed alongside `index` for explicit stale-detection: a
  23. # stale token errors instead of silently re-resolving to a different element. None on older drivers.
  24. # None for pre-#1961 drivers that didn't carry the field.
  25. element_token: Optional[str] = None
  26. @dataclass
  27. class CaptureResult:
  28. """Result of a screen capture call. mode="vision" → png_b64 only; mode="ax" → elements
  29. only; mode="som" (default) → both: the PNG already carries numbered overlays drawn by
  30. the backend and `elements` holds the matching index → element mapping."""
  31. mode: str
  32. width: int # screenshot width (logical px, pre-Anthropic-scale)
  33. height: int
  34. png_b64: Optional[str] = None
  35. elements: list[UIElement] = field(default_factory=list)
  36. app: str = "" # target app/window the elements were captured for
  37. window_title: str = ""
  38. png_bytes_len: int = 0 # raw bytes sent to Anthropic, for token estimation
  39. # MIME type of `png_b64` when the backend supplied it (cua-driver-rs emits `mimeType` on every image
  40. # part). None → consumers fall back to base64-prefix sniffing (older drivers).
  41. # See #1961, #47072.
  42. image_mime_type: Optional[str] = None
  43. # Guidance appended to the summary by capture lanes that intentionally return no elements (e.g.
  44. # full-screen composited grabs) to point the model at an interactive lane.
  45. note: str = ""
  46. # ``max_elements`` the backend asked the driver's AX walk to stop at (0 = unbounded / not applicable);
  47. # ``len(elements) >= ax_max_elements > 0`` means the tree may be truncated.
  48. ax_max_elements: int = 0
  49. @dataclass
  50. class ActionResult:
  51. """Result of any action (click / type / scroll / drag / key / wait). ``ok`` is
  52. tool/transport success only — NOT the semantic verdict; read ``effect`` / ``escalation``
  53. (cua-driver's structured verdict) to pick the next rung of the verify → escalate ladder.
  54. Structured fields are optional and additive: an older driver that omits
  55. ``structuredContent`` leaves them ``None``, behavior unchanged.
  56. Beyond the transport-level ``ok`` flag, this carries cua-driver's structured action verdict so the model
  57. can follow the documented verify → escalate ladder (NousResearch/hermes-agent#67052).
  58. """
  59. ok: bool
  60. action: str
  61. message: str = "" # human-readable summary
  62. capture: Optional[CaptureResult] = None # trailing screenshot, when requested / always-on
  63. meta: dict[str, Any] = field(default_factory=dict) # debugging / telemetry extras
  64. verified: Optional[bool] = None # AX read-back: True confirmed, False unconfirmed, None n/a
  65. effect: Optional[str] = None # "confirmed" | "unverifiable" | "suspected_noop"
  66. # {"recommended": "px"|"foreground"|"page", "reason": str} — only when driver recommends climbing
  67. escalation: Optional[dict[str, Any]] = None
  68. path: Optional[str] = None # delivery rung that ran (e.g. "ax", "x11_pixel", "cgevent_fg")
  69. degraded: Optional[bool] = None # AX walk found no actionable elements (act by px instead)
  70. delivery_mode: Optional[str] = None # the delivery_mode the caller requested, echoed back
  71. code: Optional[str] = None # refusal code, e.g. "background_unavailable", "desktop_scope_disabled"
  72. class ComputerUseBackend(ABC):
  73. """Lifecycle: `start()` before first use, `stop()` at shutdown. Pointer/keyboard actions
  74. take ``delivery_mode`` (background (default) | foreground) and ``bring_to_front``;
  75. ``button`` is left | right | middle; ``modifiers`` a list of key names. ``element`` args
  76. are 1-based SOM indices from a prior capture. `direction` is up | down | left | right and
  77. `amount` is wheel ticks; `keys` is a combo such as 'cmd+s', 'ctrl+alt+t', 'return'."""
  78. @abstractmethod
  79. def start(self) -> None: ...
  80. @abstractmethod
  81. def stop(self) -> None: ...
  82. @abstractmethod
  83. def is_available(self) -> bool: ... # usable on this host right now (check_fn gating, setup wizard)
  84. @abstractmethod
  85. def capture(self, mode: str = "som", app: Optional[str] = None, pid: Optional[int] = None,
  86. window_id: Optional[int] = None) -> CaptureResult: ...
  87. @abstractmethod
  88. def click(self, *, element: Optional[int] = None, x: Optional[int] = None, y: Optional[int] = None,
  89. button: str = "left", click_count: int = 1, modifiers: Optional[list[str]] = None,
  90. delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
  91. @abstractmethod
  92. def drag(self, *, from_element: Optional[int] = None, to_element: Optional[int] = None,
  93. from_xy: Optional[tuple[int, int]] = None, to_xy: Optional[tuple[int, int]] = None,
  94. button: str = "left", modifiers: Optional[list[str]] = None,
  95. delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
  96. @abstractmethod
  97. def scroll(self, *, direction: str, amount: int = 3, element: Optional[int] = None,
  98. x: Optional[int] = None, y: Optional[int] = None, modifiers: Optional[list[str]] = None,
  99. delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
  100. @abstractmethod
  101. def type_text(self, text: str, *, delivery_mode: Optional[str] = None,
  102. bring_to_front: bool = False) -> ActionResult: ...
  103. @abstractmethod
  104. def key(self, keys: str, *, delivery_mode: Optional[str] = None, bring_to_front: bool = False) -> ActionResult: ...
  105. @abstractmethod
  106. def list_apps(self) -> list[dict[str, Any]]: ... # running apps with bundle IDs, PIDs, window counts
  107. def list_windows(self) -> list[dict[str, Any]]:
  108. """Visible native windows with PID and window identifiers. Optional compatibility hook: backends that
  109. predate window discovery stay instantiable and report none."""
  110. return []
  111. @abstractmethod
  112. def focus_app(self, app: str, raise_window: bool = False) -> ActionResult: ... # route input to `app` (name / bundle ID)
  113. @abstractmethod
  114. def set_value(self, value: str, element: Optional[int] = None) -> ActionResult: ... # e.g. AXPopUpButton selection
  115. def wait(self, seconds: float) -> ActionResult: # default implementation
  116. time.sleep(max(0.0, min(seconds, 30.0)))
  117. return ActionResult(ok=True, action="wait", message=f"waited {seconds:.2f}s")
  118. class ComputerUseProvider(ProviderBase):
  119. """Factory for one computer_use driver, registered via ``ctx.register_computer_use_provider`` from a provider
  120. plugin whose directory name is the ``computer_use.backend`` value (built-in: ``plugins/computer_use/cua``). Only
  121. the selected provider is imported and instantiated. The model-facing tool schema is identical for every backend:
  122. an action the driver cannot perform returns ``ActionResult(ok=False, code="unsupported_action", message=...)``
  123. rather than changing the schema."""
  124. @abstractmethod
  125. def create_backend(self, *, permission_mode: str) -> ComputerUseBackend:
  126. """A fresh, unstarted backend for one Hermes session (the tool calls ``start()``/``stop()``).
  127. ``permission_mode`` is ``standard`` | ``bounded`` | ``unrestricted`` (approval bypass active)."""
  128. def is_available(self) -> bool:
  129. """Usable on this host right now (the tool's ``check_fn``). Cheap: no network, no spawning."""
  130. return True
  131. def doctor(self) -> Optional[int]:
  132. """``hermes computer-use doctor`` for this backend: print findings, return the exit code.
  133. ``None`` (default) means the backend ships no doctor."""
  134. return None