tool.py 63 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981
  1. """`computer_use` tool entry point: any-model desktop control (macOS/Windows/Linux) via cua-driver.
  2. Return contract: text-only results are a JSON string; captures / `capture_after=True` return
  3. ``{"_multimodal": True, "content": [text, image_url], "text_summary": <fallback>}`` (run_agent.py /
  4. the Anthropic adapter turn it into provider-specific image tool content)."""
  5. from __future__ import annotations
  6. import atexit
  7. import base64
  8. import contextlib
  9. import hashlib
  10. import json
  11. import logging
  12. import os
  13. import re
  14. import sys
  15. import threading
  16. import uuid
  17. from collections import namedtuple
  18. from functools import partial
  19. from types import SimpleNamespace
  20. from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
  21. from hermes_dir_acl import make_user_dir
  22. from agent.image_eviction_policy import image_dimensions_from_bytes
  23. from tools.computer_use.backend import ActionResult, CaptureResult, ComputerUseBackend, UIElement
  24. logger = logging.getLogger(__name__)
  25. # ── Approval & safety ───────────────────────────────────────────────────────
  26. # Optional computer_use-specific prompt handed to the shared gate as its explicit ``approval_callback``; when None the
  27. # gate resolves the per-thread CLI callback (``tools.terminal_tool.set_approval_callback``) like every other tool, so
  28. # in-tree hosts never call this. Same contract as that callback: ``cb(command, description, **kw)`` ->
  29. # "once" | "session" | "always" | "deny" | "timeout".
  30. _approval_callback = None
  31. def set_approval_callback(cb) -> None:
  32. global _approval_callback
  33. _approval_callback = cb
  34. # Hard-blocked regardless of approval level (e.g. logout kills the session Hermes runs in). Alt is
  35. # canonicalized to option, so the Windows variants are blocked before any backend sees them.
  36. # See #4562.
  37. _BLOCKED_KEY_COMBOS = {
  38. frozenset({"cmd", "shift", "backspace"}), frozenset({"cmd", "option", "backspace"}), # empty trash / force delete
  39. frozenset({"cmd", "ctrl", "q"}), frozenset({"cmd", "shift", "q"}), # lock screen / log out
  40. frozenset({"cmd", "option", "shift", "q"}), frozenset({"win", "l"}), # force log out / lock
  41. frozenset({"ctrl", "option", "delete"}), frozenset({"ctrl", "option", "del"}), frozenset({"option", "f4"}),
  42. }
  43. _KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option",
  44. "windows": "win", "super": "win", "meta": "win"}
  45. _BLOCKED_TYPE_PATTERNS = [re.compile(p, re.IGNORECASE) for p in ( # dangerous shell patterns for `type` (last one: fork bomb)
  46. r"curl\s+[^|]*\|\s*bash", r"curl\s+[^|]*\|\s*sh", r"wget\s+[^|]*\|\s*bash",
  47. r"\bsudo\s+rm\s+-[rf]", r"\brm\s+-rf\s+/\s*$", r":\s*\(\)\s*\{\s*:\|:\s*&\s*\}")]
  48. def _canon_key_combo(keys: str) -> frozenset:
  49. # Split on "+" AND "-": cua-driver accepts hyphenated combos, so "ctrl-alt-delete" would bypass otherwise.
  50. return frozenset(_KEY_ALIASES.get(p, p) for p in (q.strip().lower() for q in re.split(r"\s*[+\-]\s*", keys)) if p)
  51. def _reject_unsafe(action: str, args: dict[str, Any]) -> Optional[str]:
  52. """JSON error for hard-blocked input, else None. Runs BEFORE the approval prompt."""
  53. if action == "type" and (pat := next((p.pattern for p in _BLOCKED_TYPE_PATTERNS if p.search(args.get("text", ""))), None)):
  54. return json.dumps({"error": f"blocked pattern in type text: {pat!r}",
  55. "hint": "Dangerous shell patterns cannot be typed via computer_use."})
  56. if action == "key" and (blocked := next((b for b in _BLOCKED_KEY_COMBOS
  57. if b.issubset(_canon_key_combo(args.get("keys", "")))), None)) is not None:
  58. return json.dumps({"error": f"blocked key combo: {sorted(blocked)}", "hint": "Destructive system shortcuts are hard-blocked."})
  59. if args.get("bring_to_front") and args.get("delivery_mode") != "foreground":
  60. return json.dumps({"error": "bring_to_front requires delivery_mode='foreground'",
  61. "code": "bring_to_front_requires_foreground"})
  62. return None
  63. def _input_target_mismatch(backend, requested_app: str) -> Optional[str]:
  64. """Current sticky-target app when it provably differs from *requested_app*: both known and neither a substring
  65. of the other ('Google-chrome' vs 'chrome'). Unknown target -> None (fail open; the verify ladder catches it)."""
  66. last_app = getattr(backend, "_last_app", None)
  67. current, wanted = (last_app or "").strip().lower(), requested_app.strip().lower()
  68. return None if not current or not wanted or wanted in current or current in wanted else last_app
  69. # ── Backend selection — env-swappable for tests ─────────────────────────────
  70. # Per-Hermes-session cached backends (own cua-driver session, native target, refs, grant namespace).
  71. _backend_lock = threading.Lock()
  72. _backend: Optional[ComputerUseBackend] = None # backward-compatible empty-session injection hook (older tests)
  73. _backends: dict[str, ComputerUseBackend] = {}
  74. _backend_call_locks: dict[str, threading.RLock] = {}
  75. _backend_permission_modes: dict[str, str] = {}
  76. _backend_displays: dict[str, str] = {} # DISPLAY the cached backend was spawned against (Bot Desktop rebind)
  77. # Approval grants live in the shared store (``tools.approval``: session set + permanent allowlist), keyed by the
  78. # gate's session key, so a computer_use "always" is one allowlist entry like any terminal pattern. Only the
  79. # once-per-session escalation warning is tracked here.
  80. _approval_lock = threading.Lock()
  81. _escalation_warned: set = set() # sids already warned that a bypass widened the driver mode
  82. # Screenshot dedup: a tight capture→act→capture loop on a static screen resends the same ~1200px image every step.
  83. # Every delivered frame is hashed (sha256 over mime + base64 payload); when the next capture of the SAME target
  84. # (app, window) in the SAME session is byte-identical, the tool returns the normal text metadata (element index
  85. # included) plus an explicit "screen unchanged" note and omits the image block. Append-only — no prior transcript
  86. # message is rewritten, so prompt-cache prefixes stay intact. Staleness is bounded by a consecutive-omission streak
  87. # cap: full pixels are re-delivered before compaction (which keeps only the newest image-bearing tool results)
  88. # could evict the image the note refers to. State is per session; sessionless calls never dedup.
  89. _screenshot_dedup_lock = threading.Lock()
  90. _last_screenshot_state: dict[str, dict[str, Any]] = {} # session_id -> {"digest", "target": (app, window), "streak"}
  91. _SCREENSHOT_DEDUP_MAX_STREAK = 2
  92. def _screenshot_dedup_check(session_id: str, digest: str, target: tuple[str, str]) -> bool:
  93. """True when this capture should be delivered WITHOUT its image: the previous frame for this session had identical
  94. bytes for the same target and the omission streak is below _SCREENSHOT_DEDUP_MAX_STREAK. Any miss (new pixels,
  95. new target, streak exhausted, first capture) resets the stored state to this digest so the image goes out."""
  96. with _screenshot_dedup_lock:
  97. state = _last_screenshot_state.get(session_id)
  98. if (state is not None and state.get("digest") == digest and state.get("target") == target
  99. and int(state.get("streak", 0)) < _SCREENSHOT_DEDUP_MAX_STREAK):
  100. state["streak"] = int(state.get("streak", 0)) + 1
  101. return True
  102. _last_screenshot_state[session_id] = {"digest": digest, "target": target, "streak": 0}
  103. return False
  104. def _reset_screenshot_dedup(session_id: Optional[str] = None) -> None:
  105. """Forget dedup state (all sessions, or one scoped key)."""
  106. with _screenshot_dedup_lock:
  107. if session_id is None:
  108. _last_screenshot_state.clear()
  109. else:
  110. _last_screenshot_state.pop(session_id, None)
  111. def reset_screenshot_dedup(session_id: str) -> None:
  112. """Compaction boundary hook (mirrors ``reset_file_dedup``): the summary may have dropped the frame an
  113. "unchanged" note would point at, so the next capture of this session must deliver pixels again."""
  114. _reset_screenshot_dedup(_scoped_sid(session_id))
  115. def _cua_permission_mode(session_id: str) -> str:
  116. """Map Hermes's approval bypass onto Cua's immutable mode; fails closed. Both identity namespaces are consulted
  117. (DB ``session_id`` and gateway ``session_key`` contextvar) or a gateway ``/yolo`` would be invisible here.
  118. Warns once per session that ``-z``/``--yolo`` swapped the driver onto a private ``unrestricted`` daemon, dropping
  119. the configured ceiling: deliberate (``unrestricted`` is not a config value) but easy to trigger by accident."""
  120. # Configured cua mode (standard | bounded; "standard" if unresolvable). bounded needs a
  121. # computer_use.capability_manifest — the backend fails loudly without it.
  122. configured = "standard"
  123. with contextlib.suppress(Exception):
  124. from tools.computer_use.cua_backend import _cua_configured_permission_mode
  125. configured = _cua_configured_permission_mode()
  126. with contextlib.suppress(Exception):
  127. from tools.approval import is_approval_bypass_active_for_session
  128. from tools.approval_context import get_current_session_key
  129. if is_approval_bypass_active_for_session(session_id) or (
  130. bool(key := get_current_session_key(default="")) and is_approval_bypass_active_for_session(key)):
  131. with _approval_lock:
  132. warn = (key := str(session_id or "")) not in _escalation_warned
  133. _escalation_warned.add(key)
  134. if warn:
  135. logger.warning(
  136. "computer_use: approval bypass (--yolo / -z) escalated the cua-driver permission mode from the "
  137. "configured '%s' to 'unrestricted' for this session. Runtime approval prompts are disabled and the "
  138. "driver's residual ceilings no longer apply. Drop the bypass flag to keep '%s', or declare a "
  139. "version-3 computer_use.capability_manifest to keep a ceiling on bypassed runs.", configured, configured)
  140. return "unrestricted"
  141. return configured
  142. def _new_backend(permission_mode: str) -> ComputerUseBackend:
  143. from plugins.computer_use import get_active_provider
  144. return get_active_provider().create_backend(permission_mode=permission_mode)
  145. def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str) -> ComputerUseBackend:
  146. """Record a backend in the session caches (the empty session also mirrors it onto the ``_backend`` hook).
  147. Caller holds ``_backend_lock``."""
  148. global _backend
  149. from tools.computer_use.cua_backend import desktop_identity
  150. _backends[sid], _backend_permission_modes[sid] = backend, permission_mode
  151. _backend_displays[sid] = desktop_identity()
  152. _backend_call_locks[sid] = threading.RLock()
  153. _backend = backend if sid == "" else _backend
  154. return backend
  155. def _detach_locked(sid: str) -> tuple[Optional[ComputerUseBackend], Optional[threading.RLock]]:
  156. """Remove one session's cache entries, plus the ``_backend`` injection hook when it aliases the empty session
  157. (older callers/tests may populate only the hook). Caller holds ``_backend_lock``."""
  158. global _backend
  159. _backend_permission_modes.pop(sid, None), _backend_displays.pop(sid, None)
  160. backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None)
  161. if sid == "":
  162. backend = _backend if backend is None else backend
  163. _backend = None if _backend is backend else _backend
  164. return backend, call_lock
  165. def _stop_backend(backend: ComputerUseBackend, call_lock: Optional[threading.RLock], on_error: Callable[[Exception], None]) -> None:
  166. """Stop under the session call lock (if any) so an in-flight action finishes first. Never called under
  167. ``_backend_lock`` (unrelated sessions stay free). ``on_error`` absorbs the failure (never raises)."""
  168. try:
  169. with call_lock if call_lock is not None else contextlib.nullcontext():
  170. backend.stop()
  171. except Exception as e:
  172. on_error(e)
  173. def _scoped_sid(session_id: str) -> str:
  174. """Cache key for one Hermes session's backend. Outside a served-profile scope it is the bare id
  175. (legacy keys byte-identical); under a multiplexed turn the routed profile's home key is appended
  176. so two profiles that share a session id (or a DISPLAY) never share one cua-driver (#110032).
  177. Every cache path — lookup, install, release — goes through this, so release finds what lookup made."""
  178. from hermes_constants import get_hermes_home_override, hermes_home_key
  179. sid = str(session_id or "")
  180. return sid if get_hermes_home_override() is None else f"{sid}@{hermes_home_key()}"
  181. def _get_backend(session_id: str = "") -> ComputerUseBackend:
  182. bare_sid, sid = str(session_id or ""), _scoped_sid(session_id)
  183. while True:
  184. with _backend_lock:
  185. # Mode resolved under the cache lock; YOLO mutation never holds the approval lock while releasing it.
  186. permission_mode = _cua_permission_mode(bare_sid) # approval state is keyed by the Hermes session id
  187. if sid == "" and _backend is not None and sid not in _backends:
  188. _install_backend(sid, _backend, permission_mode) # fold the injection hook into the cache
  189. if (cached := _backends.get(sid)) is None:
  190. backend = _new_backend(permission_mode)
  191. backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it
  192. return _install_backend(sid, backend, permission_mode)
  193. from tools.computer_use.cua_backend import backend_display_stale, desktop_identity
  194. if (_backend_permission_modes.get(sid, "standard") == permission_mode
  195. and not backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())):
  196. return cached
  197. # Cua's mode and DISPLAY are fixed at daemon startup: a /yolo toggle, or a Bot Desktop that started
  198. # (or moved) after this backend was cached, replaces only this session's backend.
  199. _, stale_lock = _detach_locked(sid) # stopped outside the cache lock; the loop re-reads the mode first
  200. _stop_backend(cached, stale_lock, lambda e: None)
  201. @contextlib.contextmanager
  202. def _backend_for_call(session_id: str = "") -> Iterator[ComputerUseBackend]:
  203. """Hold a live backend through dispatch, retrying admission but never an action.
  204. A display/mode change or release can retire a backend before lock lookup or
  205. while a caller waits. Revalidate AFTER acquiring its profile-scoped call
  206. lock; teardown needs that same lock. Never wait under the global cache lock.
  207. """
  208. from tools.computer_use.cua_backend import backend_display_stale, desktop_identity
  209. sid = _scoped_sid(session_id)
  210. while True:
  211. backend = _get_backend(session_id=session_id)
  212. with _backend_lock:
  213. if _backends.get(sid) is not backend:
  214. continue
  215. call_lock = _backend_call_locks[sid]
  216. with call_lock:
  217. with _backend_lock:
  218. if (_backends.get(sid) is not backend
  219. or _backend_call_locks.get(sid) is not call_lock):
  220. continue
  221. # A queued call can outlive a display/config change even when
  222. # no other caller has replaced the cached backend yet.
  223. if (_backend_permission_modes.get(sid) != _cua_permission_mode(str(session_id or ""))
  224. or backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())):
  225. continue
  226. yield backend
  227. return
  228. def release_computer_use_session(session_id: str) -> bool:
  229. """Release one session-owned backend (lifecycle seam for hosts/plugins); idempotent, True iff one was released.
  230. Cache entries are removed BEFORE stopping so new lookups cannot retain the stale target/ref namespace. Approval
  231. grants are not touched here: they live in the shared store and die with ``tools.approval.clear_session``."""
  232. sid = _scoped_sid(session_id)
  233. _reset_screenshot_dedup(sid) # the next capture of a re-created session must deliver pixels
  234. with _backend_lock:
  235. backend, call_lock = _detach_locked(sid)
  236. if backend is None:
  237. return False
  238. _stop_backend(backend, call_lock,
  239. lambda e: logger.debug("computer_use backend release failed for session %s", sid, exc_info=True))
  240. return True
  241. @atexit.register
  242. def _shutdown_backend_atexit() -> None:
  243. """Stop all cached backends so cua-driver subprocesses don't outlive us. atexit only, no signal handlers: a
  244. ``SystemExit`` from a prompt_toolkit key binding corrupts its coroutine state and makes the process unkillable.
  245. Never raises. Drops the global lock before stop(): teardown budgets 5s and must not block spawns.
  246. Each session backend holds a long-lived ``cua-driver`` subprocess, so without this a driver can survive
  247. the Hermes process that spawned it (#28152 item 3). #69903 kept the orphan from burning a core by
  248. disabling the cursor overlay; the process itself still lingered.
  249. """
  250. global _backend
  251. with _backend_lock:
  252. unique = {id(b): (b, _backend_call_locks.get(sid)) for sid, b in _backends.items()}
  253. if _backend is not None:
  254. unique.setdefault(id(_backend), (_backend, _backend_call_locks.get("")))
  255. _backend = None
  256. _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear(), _backend_displays.clear()
  257. with _approval_lock:
  258. _escalation_warned.clear()
  259. for backend, call_lock in unique.values():
  260. _stop_backend(backend, call_lock, lambda e: logger.debug("cua-driver atexit teardown failed: %s", e))
  261. def reset_backend_for_tests() -> None: # pragma: no cover — tear down the cached backend and per-session state
  262. _shutdown_backend_atexit()
  263. _reset_screenshot_dedup()
  264. def _noop_stub(name: str, *params: str, result: Any = None):
  265. # Recording stub: positional args are folded in under *params* (declared params default to None). ``result`` may
  266. # be a factory of the recorded kwargs; None -> a trivial ok ActionResult.
  267. def method(self, *pos, **kw):
  268. self.calls.append((name, call := {**dict.fromkeys(params), **dict(zip(params, pos)), **kw}))
  269. return result(call) if callable(result) else ActionResult(ok=True, action=name) if result is None else result
  270. return method
  271. class _NoopBackend(ComputerUseBackend): # pragma: no cover
  272. """Test stub (tests patch ``_new_backend`` to return it). Records ``(name, kwargs)`` calls; returns trivial results."""
  273. def __init__(self) -> None: self.calls: list[tuple[str, dict[str, Any]]] = []
  274. start = stop = lambda self: None
  275. def is_available(self) -> bool: return True
  276. capture = _noop_stub("capture", "mode", "app", "pid", "window_id", result=lambda kw: CaptureResult(
  277. mode=kw["mode"] or "som", width=1024, height=768, png_b64=None, elements=[], app=kw["app"] or "", window_title=""))
  278. click, drag, scroll = _noop_stub("click"), _noop_stub("drag"), _noop_stub("scroll")
  279. type_text, key, set_value = _noop_stub("type", "text"), _noop_stub("key", "keys"), _noop_stub("set_value", "value", "element")
  280. list_apps, list_windows = _noop_stub("list_apps", result=[]), _noop_stub("list_windows", result=[])
  281. focus_app = _noop_stub("focus_app", "app", "raise_window")
  282. # ── Dispatch ────────────────────────────────────────────────────────────────
  283. def handle_computer_use(args: dict[str, Any], **kwargs) -> Any:
  284. """Main entry point (tools.registry): a JSON string (text-only) or a dict marked `_multimodal`. Order: hard
  285. blocks (_reject_unsafe) -> approval scopes (destructive action, then 'bring_to_front' — persistent focus is a
  286. separate visible side effect with its own scope) -> backend -> dispatch under the session call lock."""
  287. action = (args.get("action") or "").strip().lower()
  288. if not action:
  289. return json.dumps({"error": "missing `action`"})
  290. session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key
  291. # Bot Desktop lease: while a human drives the screen every action, capture included, is refused.
  292. from tools.bot_desktop import lease as _bd_lease
  293. from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started
  294. def _refused(e: Exception) -> str:
  295. return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)})
  296. try:
  297. admitted = _bd_lease.assert_agent_may_act()
  298. except _bd_lease.HumanHasControl as e:
  299. return _refused(e)
  300. _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY
  301. if (err := _reject_unsafe(action, args)) is not None:
  302. return err
  303. scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + (
  304. ["bring_to_front"] if args.get("bring_to_front") or (action == "focus_app" and args.get("raise_window")) else [])
  305. for scope in scopes:
  306. if (err := _request_approval(scope, args)) is not None:
  307. return err
  308. # Acquire separately so startup errors retain the install hint; the stack
  309. # releases the admitted call lock on every dispatch return or exception.
  310. call = contextlib.ExitStack()
  311. try:
  312. backend = call.enter_context(_backend_for_call(session_id))
  313. except Exception as e:
  314. from plugins.computer_use import DEFAULT_BACKEND, configured_backend_name
  315. hint = ({"hint": "If the cua-driver binary is missing, run `hermes computer-use install`. If a Python "
  316. "dependency is missing, the error above shows the exact install command."}
  317. if configured_backend_name() == DEFAULT_BACKEND else {})
  318. return json.dumps({"error": f"computer_use backend unavailable: {e}", **hint})
  319. try:
  320. with call:
  321. # Re-check under the dispatch lock: approval, backend start-up and lock waits above can take
  322. # seconds, and a human may have taken over meanwhile. A result produced after such a flip is
  323. # discarded too — it may picture what they typed.
  324. try:
  325. _bd_lease.assert_agent_may_act()
  326. except _bd_lease.HumanHasControl as e:
  327. return _refused(e)
  328. def _fence() -> None:
  329. # Any lease transition since admission voids the frame — including a full take-over /
  330. # hand-back cycle that already finished: it still belongs to the human's turn. Capture
  331. # paths call this BEFORE the frame is persisted, spilled or sent to auxiliary vision.
  332. if _bd_lease.get().epoch != admitted.epoch:
  333. raise _bd_lease.HumanHasControl(
  334. "A human took over this desktop while the action ran; its result was discarded. "
  335. "Tell the user what you need and re-capture once they hand back.")
  336. _fence() # input actions never receive fence=; refuse before the device op starts
  337. result = _dispatch(backend, action, args, fence=_fence, session_id=session_id or None)
  338. _fence()
  339. return result
  340. except _bd_lease.HumanHasControl as e:
  341. return _refused(e)
  342. except Exception as e:
  343. logger.exception("computer_use %s failed", action)
  344. return json.dumps({"error": f"{action} failed: {e}"})
  345. def _request_approval(action: str, args: dict[str, Any]) -> Optional[str]:
  346. """None if approved, else a JSON error string. The decision (yolo bypass, session/permanent grants, CLI prompt,
  347. gateway pending, cron/unattended policy, fail-closed with nobody to ask) is ``tools.approval``'s shared gate,
  348. so a computer_use grant is one store entry like any terminal pattern. Scope key ``cua:<action>:<mode>``:
  349. foreground delivery is a visible focus change, so a background ``session`` grant must NOT cover it (#67052).
  350. """
  351. from tools.approval import _run_approval_gate
  352. mode = "foreground" if args.get("delivery_mode") == "foreground" else "background"
  353. description = f"Allow computer_use to perform `{action}`?"
  354. result = _run_approval_gate(
  355. pattern_key=f"cua:{action}:{mode}", description=description,
  356. display_target=f"computer_use: {_summarize_action(action, args)}", approval_callback=_approval_callback,
  357. subject=f"computer_use `{action}` requires approval", noun="desktop actions",
  358. advice="Find an alternative approach that avoids driving the desktop.",
  359. autoapprove_log_prefix="computer_use action in non-interactive non-gateway context",
  360. fail_closed_when_no_human=True,
  361. no_human_block_message=(f"BLOCKED: computer_use `{action}` requires approval but no interactive user or "
  362. "gateway is present to approve it."),
  363. )
  364. if result.get("approved"):
  365. return None
  366. return json.dumps({"error": result.get("message") or "denied by user", "action": action})
  367. def _summarize_action(action: str, args: dict[str, Any]) -> str:
  368. fg = " [FOREGROUND — briefly raises the window / changes focus]" if args.get("delivery_mode") == "foreground" else ""
  369. return _ACTIONS.get(action, _ActionSpec(None)).summarize(action, args, fg)
  370. # --- handlers: (backend, action, args, **delivery) -> ActionResult (_dispatch applies the follow-up capture) or a
  371. # final str/dict result. `delivery` = delivery_mode + bring_to_front; only input actions use it.
  372. def _xy(args: dict[str, Any]) -> dict[str, Any]:
  373. """Click semantics: a coordinate only counts when its x is set (a bare y is not a point)."""
  374. return dict(x=coord[0], y=coord[1]) if (coord := args.get("coordinate")) and coord[0] is not None else dict(x=None, y=None)
  375. def _scroll_xy(args: dict[str, Any]) -> dict[str, Any]:
  376. """Scroll semantics: axes are independent — ``coordinate=[null, 100]`` scrolls at y=100 with x unset."""
  377. coord = args.get("coordinate") or (None, None)
  378. return dict(x=coord[0] if coord and coord[0] is not None else None,
  379. y=coord[1] if coord and coord[1] is not None else None)
  380. def _do_click(backend, action, args, button=None, count=1, **delivery):
  381. return backend.click(element=args.get("element"), **_xy(args), button=button or args.get("button") or "left",
  382. click_count=count, modifiers=args.get("modifiers"), **delivery)
  383. def _do_drag(backend, action, args, **delivery):
  384. src, dst = args.get("from_coordinate"), args.get("to_coordinate")
  385. if (args.get("from_element") is None or args.get("to_element") is None) and not (src and dst):
  386. return json.dumps({"error": "drag requires from_coordinate/to_coordinate or from_element/to_element"})
  387. return backend.drag(from_element=args.get("from_element"), to_element=args.get("to_element"),
  388. from_xy=tuple(src) if src else None, to_xy=tuple(dst) if dst else None,
  389. button=args.get("button", "left"), modifiers=args.get("modifiers"), **delivery)
  390. def _do_scroll(backend, action, args, **delivery):
  391. return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)),
  392. element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery)
  393. def _do_capture(backend, action, args, fence=lambda: None, session_id=None, **_):
  394. if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}:
  395. return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"})
  396. # pid/window_id forwarded only when given so older backends keep their defaults.
  397. cap = backend.capture(mode=mode, app=args.get("app"), **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None})
  398. fence()
  399. return _capture_response(cap, session_id=session_id)
  400. def _do_listing(backend, action, args, key, **_):
  401. return json.dumps({key: (items := getattr(backend, action)()), "count": len(items)})
  402. def _summarize_click(action: str, args: dict[str, Any], fg: str) -> str:
  403. where = (f" element #{args['element']}" if args.get("element") is not None
  404. else f" at {tuple(args['coordinate'])}" if args.get("coordinate") else "")
  405. return f"{action}{where}{fg}"
  406. # One `action`. ``input``: native input to the backend's sticky target (gets delivery kwargs + the `app=` mismatch
  407. # guard). ``destructive``: mutates user-visible state -> approval prompt (the rest only read).
  408. # ``summarize(action, args, fg_suffix)`` renders the one-line approval prompt.
  409. _ActionSpec = namedtuple("_ActionSpec", "handler input destructive summarize",
  410. defaults=(False, False, lambda a, args, fg: a + fg))
  411. _input = partial(_ActionSpec, input=True, destructive=True)
  412. _ACTIONS: dict[str, _ActionSpec] = {
  413. "click": _input(_do_click, summarize=_summarize_click),
  414. "double_click": _input(partial(_do_click, count=2), summarize=_summarize_click),
  415. "right_click": _input(partial(_do_click, button="right"), summarize=_summarize_click),
  416. "middle_click": _input(partial(_do_click, button="middle"), summarize=_summarize_click),
  417. "drag": _input(_do_drag, summarize=lambda a, args, fg: (f"drag {args.get('from_element') or args.get('from_coordinate')} → "
  418. f"{args.get('to_element') or args.get('to_coordinate')}{fg}")),
  419. "scroll": _input(_do_scroll, summarize=lambda a, args, fg: f"scroll {args.get('direction', '?')} x{args.get('amount', 3)}{fg}"),
  420. "type": _input(lambda backend, action, args, **delivery: backend.type_text(args.get("text", ""), **delivery),
  421. summarize=lambda a, args, fg: f"type {args.get('text', '')[:60]!r}" + ("..." if len(args.get("text", "")) > 60 else "") + fg),
  422. "key": _input(lambda backend, action, args, **delivery: backend.key(args.get("keys", ""), **delivery),
  423. summarize=lambda a, args, fg: f"key {args.get('keys', '')!r}{fg}"),
  424. "set_value": _input(lambda backend, action, args, **_: (
  425. json.dumps({"error": "set_value requires `value`"}) if args.get("value") is None
  426. else backend.set_value(value=str(args["value"]), element=args.get("element")))),
  427. "focus_app": _ActionSpec(lambda backend, action, args, **_: (
  428. json.dumps({"error": "focus_app requires `app`"}) if not args.get("app")
  429. else backend.focus_app(args["app"], raise_window=bool(args.get("raise_window")))), destructive=True,
  430. summarize=lambda a, args, fg: f"focus {args.get('app', '')!r}" + (" (raise)" if args.get("raise_window") else "")),
  431. "capture": _ActionSpec(_do_capture),
  432. "wait": _ActionSpec(lambda backend, action, args, **_: _text_response(backend.wait(float(args.get("seconds", 1.0))))),
  433. "list_apps": _ActionSpec(partial(_do_listing, key="apps")),
  434. "list_windows": _ActionSpec(partial(_do_listing, key="windows")),
  435. }
  436. # Native input actions deliver to the backend's sticky target; `app=` is NOT a targeting parameter (guard in _dispatch).
  437. _INPUT_ACTIONS = frozenset(a for a, s in _ACTIONS.items() if s.input)
  438. # Unknown actions are never aliased (no repairing bad model output), but the nearest real action is named as guidance.
  439. _ACTION_SUGGESTIONS = {
  440. "hotkey": "key", "press_key": "key", "keypress": "key", "key_combo": "key", "shortcut": "key", "type_text": "type",
  441. "input_text": "type", "screenshot": "capture", "get_window_state": "capture", "left_click": "click", "mouse_click": "click",
  442. }
  443. def _dispatch(backend: ComputerUseBackend, action: str, args: dict[str, Any], fence: Callable[[], None] = lambda: None,
  444. session_id: Optional[str] = None) -> Any:
  445. """``fence`` raises when the screen lease moved since admission; capture paths call it as soon as the
  446. frame is in hand, before anything derived from it leaves the process (including the screenshot dedup
  447. cache keyed by ``session_id``)."""
  448. spec = _ACTIONS.get(action)
  449. if spec is None:
  450. return json.dumps({"error": f"unknown action {action!r}" + (f" — did you mean {hint!r}? See the action enum in the tool schema."
  451. if (hint := _ACTION_SUGGESTIONS.get(str(action))) else "")})
  452. # app= guard: input goes to the sticky target from the last capture/focus_app and the backend drops app=
  453. # silently — refuse a clear mismatch rather than type into the wrong window while reporting ok:true.
  454. if (spec.input and isinstance(requested_app := args.get("app"), str) and requested_app.strip()
  455. and (mismatch := _input_target_mismatch(backend, requested_app)) is not None):
  456. return json.dumps({"ok": False, "action": action, "code": "input_target_mismatch", "error": (
  457. f"{action} would go to the current target {mismatch!r}, not {requested_app.strip()!r} "
  458. "— input actions always hit the sticky target from the last capture/focus_app. "
  459. f"Call capture(app={requested_app.strip()!r}) or focus_app first, then retry.")})
  460. # delivery_mode / bring_to_front thread through every input action (background → foreground ladder); input
  461. # handlers forward their kwargs to the backend verbatim, so the lease fence and the dedup session key ride
  462. # only on read handlers (delivery kwargs would leak into backend input calls, and vice versa).
  463. res = spec.handler(backend, action, args, **(dict(delivery_mode=args.get("delivery_mode"), bring_to_front=bool(args.get("bring_to_front")))
  464. if spec.input else dict(fence=fence, session_id=session_id)))
  465. return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")), fence,
  466. session_id=session_id)
  467. # ── Response shaping ────────────────────────────────────────────────────────
  468. def _classify_action_result(res: ActionResult) -> dict[str, Any]:
  469. """Next ladder step from semantic evidence, in precedence order. Escalation is advisory: it never overrides
  470. a confirmed effect nor licenses repeating input."""
  471. if res.effect == "confirmed" or res.verified is True:
  472. return {"decision": "done"}
  473. if res.effect == "unverifiable":
  474. return {"decision": "verify_fresh_state", "hint": ("Input was delivered but not confirmed. Re-capture and check the "
  475. "result BEFORE any retry — do not repeat the input on an escalation recommendation alone.")}
  476. if res.effect == "suspected_noop" or not res.ok or res.code is not None:
  477. meta = res.meta if isinstance(res.meta, dict) else {}
  478. delivered, requested = meta.get("delivered_chars"), meta.get("requested_chars")
  479. if (
  480. res.action in ("type", "type_text")
  481. and res.code == "type_text_incomplete"
  482. and isinstance(delivered, int)
  483. and isinstance(requested, int)
  484. and requested > 0
  485. and delivered <= 0
  486. ):
  487. # Zero delivery on the driver's own partial-delivery verdict: the field swallowed every
  488. # synthetic keystroke (trusted-event checks on web inputs do this). Neither rung of the
  489. # delivery ladder can fix a target that drops events at the source — the AX set_value
  490. # path writes the value directly and bypasses event filtering. The code gate matters:
  491. # `type_text_synthesis_budget_exceeded` also reports delivered 0, but that is the bounded
  492. # synthesis budget declining to emit at all, and the driver's own `chunk` recommendation
  493. # stays the right next step there.
  494. return {
  495. "decision": "escalate",
  496. "recommended": "set_value",
  497. "hint": (
  498. "0 characters landed: this input drops synthetic keystrokes, so no delivery rung will fix it. "
  499. "Climb to the set_value action on the field's element index instead — it is an action, not a "
  500. "delivery mode, and sets the value through the accessibility API, bypassing event filtering. "
  501. "Re-capture first if the index is stale."
  502. ),
  503. }
  504. return {"decision": "escalate", **({"recommended": res.escalation.get("recommended")}
  505. if isinstance(res.escalation, dict) else {}), "hint": (
  506. "The input likely did not land. Climb one rung following `recommended`: 'px' → re-issue by coordinate; "
  507. "'foreground' (or a failed pixel click) → re-issue with delivery_mode='foreground' (separate approval). "
  508. "Do not predict the rung from the app being Electron/Chromium — react to this signal.")}
  509. return {"decision": "verify_fresh_state", # transport success without semantic proof is not proof of effect
  510. "hint": "Transport succeeded but the effect is unproven. Re-capture and confirm before continuing."}
  511. def _present(**fields: Any) -> dict[str, Any]:
  512. return {k: v for k, v in fields.items() if v} # only the truthy optional fields, in the given order
  513. def _action_payload(res: ActionResult) -> dict[str, Any]:
  514. # cua-driver's structured verdict fields only when returned (None = old driver). ok is transport success;
  515. # effect/escalation are the semantic verdict.
  516. return {"ok": res.ok, "action": res.action, **_present(message=res.message),
  517. **{k: v for k in ("verified", "effect", "escalation", "path", "degraded", "delivery_mode", "code")
  518. if (v := getattr(res, k)) is not None}, **_present(meta=res.meta),
  519. "verdict": _classify_action_result(res)}
  520. def _text_response(res: ActionResult) -> str:
  521. return json.dumps(_action_payload(res))
  522. # AX `elements` cap: dense UIs publish 500+ nodes (one capture would exhaust context); the full tree spills to a file.
  523. _DEFAULT_MAX_ELEMENTS = 100
  524. # Some providers reject images below 8x8 before the model sees the result; such captures fall back to text.
  525. _MIN_PROVIDER_IMAGE_DIMENSION = 8
  526. # Some AX trees (Discord/Slack via UIA, Electron chat clients) expose ENTIRE message bodies as labels; uncapped
  527. # they blew the tool-result budget and leaked private chat text. Labels identify a control, not text extraction.
  528. _MAX_ELEMENT_LABEL_CHARS = 120
  529. # Bounded cache trails: every dense capture can spill, and CLI-only sessions never run the gateway's media cleanup.
  530. _MAX_SPILL_FILES = _MAX_CAPTURE_FILES = 20
  531. def _capture_image_format(cap: CaptureResult) -> tuple[str, str]:
  532. # (MIME, file extension): cua-driver's explicit MIME type, else sniff the base64 prefix (JPEG starts with /9j/,
  533. # PNG with iVBOR). The extension matches the on-disk bytes for MIME sniffing.
  534. mime = cap.image_mime_type or ("image/jpeg" if (cap.png_b64 or "").startswith("/9j/") else "image/png")
  535. return mime, (".jpg" if mime.lower() == "image/jpeg" else ".png")
  536. def _bounds_unknown(bounds) -> bool:
  537. # No real geometry: KDE/Qt apps report [0, 0, 0, 0] for elements clickable by index; serializing that as a
  538. # rect invites coordinate=[0, 0] clicks.
  539. with contextlib.suppress(TypeError, ValueError):
  540. return all(int(v) == 0 for v in bounds)
  541. return False
  542. def _element_to_dict(e: UIElement) -> dict[str, Any]:
  543. # A zero rect is "geometry unknown", not a position — null it so no coordinate= is ever derived from it (the index still works).
  544. return {"index": e.index, "role": e.role, "label": e.label[:_MAX_ELEMENT_LABEL_CHARS],
  545. "bounds": None if _bounds_unknown(e.bounds) else list(e.bounds), "app": e.app,
  546. **({"label_truncated": True} if len(e.label) > _MAX_ELEMENT_LABEL_CHARS else {})}
  547. def _format_elements(elements: list[UIElement], max_lines: int = 40) -> list[str]:
  548. out = [f" #{e.index} {e.role} {e.label.replace(chr(10), ' ')[:60]!r} "
  549. + ("@ bounds-unknown (click by element index)" if _bounds_unknown(e.bounds) else f"@ {e.bounds}")
  550. + (f" [{e.app}]" if e.app else "") for e in elements[:max_lines]]
  551. return out + ([f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)"] if len(elements) > max_lines else [])
  552. def _bounds_hints(elements: list[UIElement], image_width: int, image_height: int) -> tuple[Optional[float], Optional[str]]:
  553. """(scale, note) when element bounds live in a different coordinate space than the screenshot, else (None, None).
  554. On HiDPI displays AX bounds are native while the screenshot is downscaled, so coordinate= clicks read off the
  555. screenshot miss by the scale factor. 5% slack: window chrome can hang a few px past the captured frame without
  556. implying a different space. Scale heuristic: larger axis ratio wins."""
  557. if not elements or image_width <= 0 or image_height <= 0:
  558. return None, None
  559. max_x = max_y = 0
  560. for e in elements:
  561. try:
  562. x, y, w, h = e.bounds
  563. except (TypeError, ValueError):
  564. continue
  565. max_x, max_y = max(max_x, int(x) + int(w)), max(max_y, int(y) + int(h))
  566. if max_x <= image_width * 1.05 and max_y <= image_height * 1.05:
  567. return None, None
  568. note = (f"element bounds are in native desktop coordinates (extend to ~{max_x}x{max_y}), "
  569. f"NOT screenshot pixels ({image_width}x{image_height}). coordinate= clicks expect the native "
  570. "space — derive click points from element bounds, or scale screenshot positions up accordingly")
  571. return round(max(max_x / image_width, max_y / image_height), 2), note
  572. _bounds_scale = lambda elements, image_width, image_height: _bounds_hints(elements, image_width, image_height)[0]
  573. _bounds_space_note = lambda elements, image_width, image_height: _bounds_hints(elements, image_width, image_height)[1]
  574. def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace:
  575. """One capture's derived facts, computed once for every response branch: ``visible`` is the capped element list,
  576. ``dims_omitted`` an image below the provider minimum."""
  577. visible, dims = cap.elements[:max_elements], None
  578. with contextlib.suppress(Exception): # (width, height) of the inline PNG/JPEG screenshot, else the backend's
  579. dims = image_dimensions_from_bytes(base64.b64decode(cap.png_b64, validate=False)) if cap.png_b64 else None
  580. width, height = dims or (cap.width, cap.height)
  581. scale, note = _bounds_hints(visible, width, height)
  582. # Capped labels / capped element array: spill the complete tree for on-demand reads.
  583. lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible)
  584. too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION
  585. has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small
  586. # The driver's own AX walk may have stopped at the ``max_elements`` the backend sent: then the spill file is
  587. # NOT the full tree, and the hint must not promise one.
  588. ax_capped = len(cap.elements) >= cap.ax_max_elements > 0
  589. return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height,
  590. truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note,
  591. ax_capped=cap.ax_max_elements if ax_capped else 0,
  592. elements_file=_spill_elements_to_file(cap) if lost_detail else None,
  593. screenshot_path=_persist_capture_image(cap) if has_image else None,
  594. dims_omitted=dims if too_small else None, has_image=has_image)
  595. def _capture_summary_lines(v: SimpleNamespace) -> list[str]:
  596. """Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the
  597. summary names indices the model can't find."""
  598. notes = (
  599. v.bounds_note and v.bounds_note + (f"; estimated scale ~{v.bounds_scale}x (screenshot position x "
  600. f"{v.bounds_scale} ≈ native coordinate)" if v.bounds_scale else ""),
  601. v.screenshot_path and f"shareable screenshot saved to {v.screenshot_path}",
  602. v.cap.note,
  603. v.elements_file and (f"{'' if v.ax_capped else 'full '}element tree with untruncated labels "
  604. f"saved to {v.elements_file} — read_file/search_files it if you need dropped label "
  605. "text or elements beyond the cap"),
  606. v.ax_capped and (f"accessibility walk capped at {v.ax_capped} elements; pass app= to narrow"),
  607. )
  608. return [
  609. f"capture mode={v.cap.mode} {v.width}x{v.height}"
  610. + (f" app={v.cap.app}" if v.cap.app else "") + (f" window={v.cap.window_title!r}" if v.cap.window_title else ""),
  611. f"{v.total} interactable element(s):",
  612. *(f" ({note})" for note in notes if note),
  613. *_format_elements(v.visible),
  614. *([f" (screenshot omitted: {v.dims_omitted[0]}x{v.dims_omitted[1]} is below the "
  615. f"{_MIN_PROVIDER_IMAGE_DIMENSION}x{_MIN_PROVIDER_IMAGE_DIMENSION} provider minimum)"] if v.dims_omitted else []),
  616. ]
  617. def _text_capture_payload(v: SimpleNamespace, summary: str, extra: Optional[dict[str, Any]] = None) -> str:
  618. """JSON text payload shared by the AX, vision-unavailable and aux-vision branches. Key order is contract:
  619. fixed fields, ``extra`` branch markers, then set optionals."""
  620. return json.dumps({
  621. "mode": v.cap.mode, "width": v.width, "height": v.height, "app": v.cap.app, "window_title": v.cap.window_title,
  622. "elements": [_element_to_dict(e) for e in v.visible], "total_elements": v.total, "summary": summary,
  623. **(extra or {}),
  624. **_present(truncated_elements=v.truncated, elements_file=v.elements_file, screenshot_path=v.screenshot_path,
  625. bounds_scale=v.bounds_scale),
  626. })
  627. def _capture_digest(cap: CaptureResult) -> str:
  628. return hashlib.sha256((str(cap.image_mime_type or "") + ":").encode("utf-8")
  629. + (cap.png_b64 or "").encode("ascii", "ignore")).hexdigest()
  630. def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEMENTS,
  631. session_id: Optional[str] = None) -> Any:
  632. v = _capture_view(cap, max_elements)
  633. lines = _capture_summary_lines(v)
  634. summary, extra = "\n".join(lines), None # multimodal/aux paths use this; text paths append notes and rebuild
  635. if v.has_image and session_id and _screenshot_dedup_check(
  636. _scoped_sid(session_id), _capture_digest(cap), (str(cap.app or ""), str(cap.window_title or ""))):
  637. # Unchanged frame: same pixels for the same target in this session — no image (and no aux-vision call);
  638. # the text metadata is fresh and the note says which earlier result still applies.
  639. lines.append(" (screen unchanged since the previous capture — image omitted to save context; the previous "
  640. "capture's screenshot/analysis still shows the current state. Element indices below are fresh "
  641. "and remain the preferred way to act.)")
  642. extra = {"screen_unchanged": True}
  643. elif v.has_image:
  644. # Hand the screenshot to auxiliary.vision (text-only result) when the main model may not consume images
  645. # natively; returning the multimodal envelope unconditionally tripped HTTP 404/400 at the provider.
  646. if not _should_route_through_aux_vision(): # envelope carrying the screenshot (not the elements array, so no truncation note)
  647. return {
  648. "_multimodal": True,
  649. "content": [{"type": "text", "text": summary},
  650. {"type": "image_url", "image_url": {"url": f"data:{_capture_image_format(cap)[0]};base64,{cap.png_b64}"}}],
  651. "text_summary": summary,
  652. "meta": {"mode": cap.mode, "width": v.width, "height": v.height, "elements": v.total, "png_bytes": cap.png_bytes_len,
  653. **_present(screenshot_path=v.screenshot_path, elements_file=v.elements_file, bounds_scale=v.bounds_scale)},
  654. }
  655. # Decide whether to hand the screenshot to the auxiliary.vision pipeline (text-only result) or keep
  656. # the multimodal envelope (main model handles vision natively). Issue #24015: previously the
  657. # multimodal envelope was returned unconditionally, so non-vision main models tripped HTTP 404 / 400
  658. # at the provider boundary even when auxiliary.vision was explicitly configured to handle this.
  659. routed = _route_capture_through_aux_vision(
  660. cap, summary, visible_elements=v.visible, truncated_elements=v.truncated,
  661. elements_file=v.elements_file, screenshot_path=v.screenshot_path)
  662. if routed is not None:
  663. return routed
  664. # Aux routing requested but failed (vision node down, empty analysis...): the multimodal envelope could
  665. # now break with a provider error, so degrade to text.
  666. lines.append(" (vision unavailable: the auxiliary vision model could not be reached; screenshot "
  667. "omitted. Element-index actions still work — drive via the element list above.)")
  668. extra = {"vision_unavailable": True}
  669. if v.truncated: # text paths carry the `elements` array, so the truncation note applies
  670. lines.append(f" (response truncated to {len(v.visible)} of {v.total} elements; the full tree is in "
  671. "elements_file — read_file/search_files it, or pass app= to narrow scope)")
  672. return _text_capture_payload(v, "\n".join(lines), extra)
  673. def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool,
  674. fence: Callable[[], None] = lambda: None, session_id: Optional[str] = None) -> Any:
  675. # No follow-up capture after a failed action: a normal-looking screenshot would suggest success.
  676. if not do_capture or not res.ok:
  677. return _text_response(res)
  678. try:
  679. # Recapture the exact window when known: on Linux several unrelated windows may share an app name, so
  680. # app-only recapture can switch targets.
  681. exact = {k: (getattr(backend, "_last_target", None) or {}).get(k) for k in ("pid", "window_id")}
  682. cap = backend.capture(mode=_capture_after_mode(), **(exact if None not in exact.values()
  683. else {"app": getattr(backend, "_last_app", None)}))
  684. except Exception as e:
  685. logger.warning("follow-up capture failed: %s", e)
  686. return _text_response(res)
  687. fence() # a frame taken after a takeover never reaches the dedup cache nor the model
  688. resp, payload = _capture_response(cap, session_id=session_id), _action_payload(res)
  689. if isinstance(resp, dict) and resp.get("_multimodal"):
  690. # Keep the evidence/verdict contract visible alongside the image — it governs whether input may repeat.
  691. resp["content"][0]["text"] = resp["text_summary"] = json.dumps(payload) + "\n\n" + resp["text_summary"]
  692. resp["action_result"] = payload
  693. return resp
  694. return json.dumps({**json.loads(resp), **payload}) # text capture: merge the action payload in
  695. # ── Cache files (screenshots, element spills, vision temps) ─────────────────
  696. def _secure_dir_policy(cache_dir) -> None:
  697. """Create/reconcile a Hermes media-cache dir owner-only (0700), except managed.
  698. A capture is as sensitive as the screen it came from — an open password
  699. manager, a private chat, a bank tab. The umask-derived 0755 these dirs
  700. used to get made every local account able to list (and read) those
  701. frames whenever ``HERMES_HOME`` itself is traversable, which is exactly
  702. what the documented ``HERMES_HOME_MODE=0701`` web-server escape hatch
  703. arranges. Delegates to the same house policy as every Hermes secret dir —
  704. ``hermes_cli.config._secure_dir`` — with the mode passed to ``mkdir`` so
  705. there is no window between mkdir and chmod; managed/NixOS installs keep
  706. their group-share design (the mode is left to the configured umask/setgid
  707. because these lazily-created dirs are not covered by the module's
  708. ``systemd.tmpfiles`` rules). Best-effort; never breaks a capture.
  709. """
  710. try:
  711. managed = False
  712. try:
  713. from hermes_cli.config import is_managed
  714. managed = bool(is_managed())
  715. except Exception: # pragma: no cover - defensive
  716. pass
  717. if managed:
  718. cache_dir.mkdir(parents=True, exist_ok=True)
  719. return
  720. make_user_dir(cache_dir, parents=True, exist_ok=True, mode=0o700)
  721. try:
  722. from hermes_cli.config import _secure_dir
  723. _secure_dir(cache_dir)
  724. except Exception as exc: # pragma: no cover - defensive
  725. logger.debug("computer_use: cache dir chmod skipped: %s", exc)
  726. except Exception as exc: # pragma: no cover - defensive
  727. logger.debug("computer_use: cache dir creation failed for %s: %s", cache_dir, exc)
  728. cache_dir.mkdir(parents=True, exist_ok=True)
  729. def _write_private_bytes(path, data: bytes) -> None:
  730. """Write ``data`` to ``path`` with owner-only (0600) permissions.
  731. ``Path.write_bytes`` would land 0644 under a default umask. POSIX mode
  732. bits are advisory on Windows (``os.chmod`` there only toggles the
  733. read-only flag), so this is a best-effort narrowing that falls back to a
  734. plain write rather than failing the capture.
  735. """
  736. import os as _os_priv
  737. flags = _os_priv.O_WRONLY | _os_priv.O_CREAT | _os_priv.O_TRUNC
  738. try:
  739. fd = _os_priv.open(str(path), flags, 0o600)
  740. except OSError:
  741. path.write_bytes(data)
  742. return
  743. with _os_priv.fdopen(fd, "wb") as handle:
  744. handle.write(data)
  745. def _cache_file(subdir: str, legacy: str, name: str, pattern: str = "", cap: int = 0):
  746. """Path for a new file under ``$HERMES_HOME/<subdir>`` (dir created owner-only, per #77579).
  747. With ``pattern``/``cap``, first unlinks the oldest matching files so at most ``cap - 1`` remain
  748. (best-effort)."""
  749. from hermes_constants import get_hermes_dir # lazy so tests can patch get_hermes_dir
  750. cache_dir = get_hermes_dir(subdir, legacy)
  751. _secure_dir_policy(cache_dir)
  752. with contextlib.suppress(Exception):
  753. files = sorted(cache_dir.glob(pattern), key=lambda p: p.stat().st_mtime) if pattern else []
  754. for stale in files[: max(0, len(files) - (cap - 1))]:
  755. stale.unlink(missing_ok=True)
  756. return cache_dir / name
  757. def _write_cache_file(what: str, subdir: str, legacy: str, name: str, pattern: str, cap: int,
  758. write: Callable[[Any], None]) -> Optional[str]:
  759. """Bounded cache write via ``write(path)``; the path, or None on any failure — an unwritable cache must never
  760. break control (a capture keeps working without its spill/screenshot copy)."""
  761. try:
  762. write(path := _cache_file(subdir, legacy, name, pattern, cap))
  763. return str(path)
  764. except Exception as exc: # pragma: no cover - defensive
  765. logger.debug("computer_use: %s failed: %s", what, exc)
  766. return None
  767. def _persist_capture_image(cap: CaptureResult) -> Optional[str]:
  768. """Copy of the capture in Hermes' media cache so attachment surfaces can deliver it (None without an image)."""
  769. return _write_cache_file(
  770. "screenshot persistence", "cache/images", "image_cache", f"computer_use_{uuid.uuid4().hex}{_capture_image_format(cap)[1]}",
  771. "computer_use_*.*", _MAX_CAPTURE_FILES,
  772. lambda p: _write_private_bytes(p, base64.b64decode(cap.png_b64, validate=False)),
  773. ) if cap.png_b64 else None
  774. def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]:
  775. """FULL element tree (untruncated labels) in a cache file — the read_file/search_files escape hatch for capped text."""
  776. payload = {"app": cap.app, "window_title": cap.window_title, "total_elements": len(cap.elements),
  777. "elements": [{"index": e.index, "role": e.role, "label": e.label, "bounds": list(e.bounds), "app": e.app}
  778. for e in cap.elements]}
  779. return _write_cache_file("element spill", "cache/computer_use", "computer_use_cache", f"elements_{uuid.uuid4().hex}.json",
  780. "elements_*.json", _MAX_SPILL_FILES,
  781. lambda p: p.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8"))
  782. # ── auxiliary.vision routing for captured screenshots ───────────────────────
  783. _MAX_VISION_DIM = 1456 # longest image side handed to the aux vision model: full-resolution desktop captures tokenize heavily
  784. # and can overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting vision latency.
  785. def _shrink_capture_for_vision(raw: bytes, ext: str, max_dim: int = _MAX_VISION_DIM) -> tuple[bytes, Optional[str]]:
  786. """Downscale encoded image bytes so the longest side is <= max_dim -> ``(bytes, scale_note)``. note is None when
  787. unchanged (fits, or Pillow unavailable/failed), else it tells the vision model the factor so reported
  788. coordinates map back to the real screen instead of being silently wrong."""
  789. try:
  790. from io import BytesIO
  791. from PIL import Image
  792. img = Image.open(BytesIO(raw))
  793. if max(img.size) <= max_dim:
  794. return raw, None
  795. (orig_w, orig_h), out = img.size, BytesIO()
  796. img.thumbnail((max_dim, max_dim))
  797. new_w, new_h = img.size
  798. img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
  799. fx, fy = (orig_w / new_w if new_w else 1.0), (orig_h / new_h if new_h else 1.0)
  800. factor_clause = (f"multiply any coordinates you report by {fx:.2f} to map back to the real screen." if f"{fx:.2f}" == f"{fy:.2f}"
  801. else f"multiply any x coordinates you report by {fx:.2f} and any y coordinates by {fy:.2f} to map back to the real screen.")
  802. return out.getvalue(), f"Screenshot downscaled from {orig_w}x{orig_h} to {new_w}x{new_h} for vision; {factor_clause}"
  803. except Exception as exc:
  804. logger.debug("computer_use: vision downscale skipped: %s", exc)
  805. return raw, None
  806. def _should_route_through_aux_vision() -> bool:
  807. """True when ``_capture_response`` should hand the PNG to aux vision. Any failure returns False (fail open) so a
  808. broken config never silently drops the screenshot for vision-capable main models. Decided per capture (the
  809. config read is the signature-cached ``load_config_readonly``), so ``/model``, a profile switch or an
  810. ``image_input_mode`` edit applies to the next screenshot instead of a stale verdict."""
  811. stage = "import"
  812. try:
  813. from agent.auxiliary_client import _read_main_model, _read_main_provider
  814. from hermes_cli.config import load_config_readonly
  815. from tools.vision_tools import _native_tool_result_images
  816. stage = "decision"
  817. provider, model = _read_main_provider() or "", _read_main_model() or ""
  818. # The shared native-tool-result gate (vision_analyze, browser screenshots, MCP images use it too), so a
  819. # screenshot takes the same lane whichever tool produced it; anything it cannot vouch for goes to aux.
  820. return not _native_tool_result_images(provider, model, load_config_readonly())
  821. except Exception as exc: # pragma: no cover - defensive
  822. logger.debug("computer_use: aux-vision routing %s failed: %s", stage, exc)
  823. return False
  824. def _capture_after_mode() -> str:
  825. """Mode for ``capture_after`` follow-ups. Default ``som`` (screenshot)."""
  826. with contextlib.suppress(Exception):
  827. from hermes_cli.config import load_config
  828. mode = str(((load_config() or {}).get("computer_use") or {}).get("capture_after_mode", "som") or "som")
  829. return mode if (mode := mode.strip().lower()) in {"som", "vision", "ax"} else "som"
  830. return "som"
  831. _VISION_PROMPT = ("Describe what is visible in this desktop application screenshot in concise but specific terms. Mention "
  832. "the app name and window title if visible, the overall layout, any labelled buttons, menus or text fields, "
  833. "and any prominent text content the user would need to know about. Do not invent details that are not "
  834. "actually visible.\n\nAX/SOM index for cross-reference:\n")
  835. def _route_capture_through_aux_vision(cap: CaptureResult, summary: str, *, visible_elements: Optional[list[UIElement]] = None,
  836. truncated_elements: int = 0, elements_file: Optional[str] = None,
  837. screenshot_path: Optional[str] = None) -> Optional[str]:
  838. """Pre-analyse the capture via ``vision_analyze_tool`` (temp file under ``$HERMES_HOME/cache/vision/``) and merge
  839. the description with the AX/SOM summary into one text payload. JSON, or None on any failure."""
  840. if not cap.png_b64:
  841. return None
  842. problem, temp_image_path = "aux-vision import failed", None
  843. try:
  844. from model_tools import _run_async
  845. from tools.vision_tools import vision_analyze_tool
  846. problem = "failed to decode capture base64"
  847. raw = base64.b64decode(cap.png_b64, validate=False)
  848. problem = None # from here on failures are loud (warning)
  849. ext = _capture_image_format(cap)[1]
  850. temp_image_path = _cache_file("cache/vision", "temp_vision_images", f"computer_use_{uuid.uuid4().hex}{ext}")
  851. raw, scale_note = _shrink_capture_for_vision(raw, ext)
  852. _write_private_bytes(temp_image_path, raw)
  853. prompt = _VISION_PROMPT + summary + (f"\n\nNote: {scale_note}" if scale_note else "")
  854. result_json = _run_async(vision_analyze_tool(str(temp_image_path), prompt))
  855. except Exception as exc:
  856. if problem:
  857. logger.debug("computer_use: %s: %s", problem, exc)
  858. else:
  859. logger.warning("computer_use: auxiliary.vision pre-analysis failed (%s); "
  860. "returning to caller without aux analysis", exc)
  861. return None
  862. finally:
  863. if temp_image_path is not None:
  864. with contextlib.suppress(Exception):
  865. os.unlink(str(temp_image_path))
  866. # The ``analysis`` field of vision_analyze_tool's JSON result; raw text when it isn't JSON; empty -> no merge.
  867. analysis_text = result_json.strip() if isinstance(result_json, str) else ""
  868. with contextlib.suppress(TypeError, json.JSONDecodeError):
  869. parsed = json.loads(analysis_text)
  870. analysis_text = str(parsed.get("analysis") or "").strip() if isinstance(parsed, dict) else ""
  871. if not analysis_text:
  872. return None
  873. # Same element cap as every other capture branch; dumping cap.elements in full would bypass max_elements
  874. # exactly for non-vision main models. Dimensions are the backend's on this branch.
  875. view = SimpleNamespace(cap=cap, visible=cap.elements if visible_elements is None else visible_elements,
  876. total=len(cap.elements), width=cap.width, height=cap.height, truncated=truncated_elements,
  877. elements_file=elements_file, screenshot_path=screenshot_path, bounds_scale=None)
  878. return _text_capture_payload(view, summary, {"vision_analysis": analysis_text,
  879. "vision_analysis_routed_via": "auxiliary.vision"})
  880. # ── Availability check (used by the tool registry check_fn) ─────────────────
  881. def check_computer_use_requirements() -> bool:
  882. """The selected ``computer_use.backend``'s ``is_available()``. An unregistered name stays True so the call
  883. surfaces the "not registered (available: ...)" error instead of the tool silently vanishing."""
  884. from plugins.computer_use import get_active_provider
  885. try:
  886. return get_active_provider().is_available()
  887. except LookupError:
  888. return True
  889. def get_computer_use_schema() -> dict[str, Any]:
  890. from tools.computer_use.schema import COMPUTER_USE_SCHEMA
  891. return COMPUTER_USE_SCHEMA