| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981 |
- """`computer_use` tool entry point: any-model desktop control (macOS/Windows/Linux) via cua-driver.
- Return contract: text-only results are a JSON string; captures / `capture_after=True` return
- ``{"_multimodal": True, "content": [text, image_url], "text_summary": <fallback>}`` (run_agent.py /
- the Anthropic adapter turn it into provider-specific image tool content)."""
- from __future__ import annotations
- import atexit
- import base64
- import contextlib
- import hashlib
- import json
- import logging
- import os
- import re
- import sys
- import threading
- import uuid
- from collections import namedtuple
- from functools import partial
- from types import SimpleNamespace
- from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple
- from hermes_dir_acl import make_user_dir
- from agent.image_eviction_policy import image_dimensions_from_bytes
- from tools.computer_use.backend import ActionResult, CaptureResult, ComputerUseBackend, UIElement
- logger = logging.getLogger(__name__)
- # ── Approval & safety ───────────────────────────────────────────────────────
- # Optional computer_use-specific prompt handed to the shared gate as its explicit ``approval_callback``; when None the
- # gate resolves the per-thread CLI callback (``tools.terminal_tool.set_approval_callback``) like every other tool, so
- # in-tree hosts never call this. Same contract as that callback: ``cb(command, description, **kw)`` ->
- # "once" | "session" | "always" | "deny" | "timeout".
- _approval_callback = None
- def set_approval_callback(cb) -> None:
- global _approval_callback
- _approval_callback = cb
- # Hard-blocked regardless of approval level (e.g. logout kills the session Hermes runs in). Alt is
- # canonicalized to option, so the Windows variants are blocked before any backend sees them.
- # See #4562.
- _BLOCKED_KEY_COMBOS = {
- frozenset({"cmd", "shift", "backspace"}), frozenset({"cmd", "option", "backspace"}), # empty trash / force delete
- frozenset({"cmd", "ctrl", "q"}), frozenset({"cmd", "shift", "q"}), # lock screen / log out
- frozenset({"cmd", "option", "shift", "q"}), frozenset({"win", "l"}), # force log out / lock
- frozenset({"ctrl", "option", "delete"}), frozenset({"ctrl", "option", "del"}), frozenset({"option", "f4"}),
- }
- _KEY_ALIASES = {"command": "cmd", "control": "ctrl", "alt": "option", "⌘": "cmd", "⌥": "option",
- "windows": "win", "super": "win", "meta": "win"}
- _BLOCKED_TYPE_PATTERNS = [re.compile(p, re.IGNORECASE) for p in ( # dangerous shell patterns for `type` (last one: fork bomb)
- r"curl\s+[^|]*\|\s*bash", r"curl\s+[^|]*\|\s*sh", r"wget\s+[^|]*\|\s*bash",
- r"\bsudo\s+rm\s+-[rf]", r"\brm\s+-rf\s+/\s*$", r":\s*\(\)\s*\{\s*:\|:\s*&\s*\}")]
- def _canon_key_combo(keys: str) -> frozenset:
- # Split on "+" AND "-": cua-driver accepts hyphenated combos, so "ctrl-alt-delete" would bypass otherwise.
- return frozenset(_KEY_ALIASES.get(p, p) for p in (q.strip().lower() for q in re.split(r"\s*[+\-]\s*", keys)) if p)
- def _reject_unsafe(action: str, args: dict[str, Any]) -> Optional[str]:
- """JSON error for hard-blocked input, else None. Runs BEFORE the approval prompt."""
- if action == "type" and (pat := next((p.pattern for p in _BLOCKED_TYPE_PATTERNS if p.search(args.get("text", ""))), None)):
- return json.dumps({"error": f"blocked pattern in type text: {pat!r}",
- "hint": "Dangerous shell patterns cannot be typed via computer_use."})
- if action == "key" and (blocked := next((b for b in _BLOCKED_KEY_COMBOS
- if b.issubset(_canon_key_combo(args.get("keys", "")))), None)) is not None:
- return json.dumps({"error": f"blocked key combo: {sorted(blocked)}", "hint": "Destructive system shortcuts are hard-blocked."})
- if args.get("bring_to_front") and args.get("delivery_mode") != "foreground":
- return json.dumps({"error": "bring_to_front requires delivery_mode='foreground'",
- "code": "bring_to_front_requires_foreground"})
- return None
- def _input_target_mismatch(backend, requested_app: str) -> Optional[str]:
- """Current sticky-target app when it provably differs from *requested_app*: both known and neither a substring
- of the other ('Google-chrome' vs 'chrome'). Unknown target -> None (fail open; the verify ladder catches it)."""
- last_app = getattr(backend, "_last_app", None)
- current, wanted = (last_app or "").strip().lower(), requested_app.strip().lower()
- return None if not current or not wanted or wanted in current or current in wanted else last_app
- # ── Backend selection — env-swappable for tests ─────────────────────────────
- # Per-Hermes-session cached backends (own cua-driver session, native target, refs, grant namespace).
- _backend_lock = threading.Lock()
- _backend: Optional[ComputerUseBackend] = None # backward-compatible empty-session injection hook (older tests)
- _backends: dict[str, ComputerUseBackend] = {}
- _backend_call_locks: dict[str, threading.RLock] = {}
- _backend_permission_modes: dict[str, str] = {}
- _backend_displays: dict[str, str] = {} # DISPLAY the cached backend was spawned against (Bot Desktop rebind)
- # Approval grants live in the shared store (``tools.approval``: session set + permanent allowlist), keyed by the
- # gate's session key, so a computer_use "always" is one allowlist entry like any terminal pattern. Only the
- # once-per-session escalation warning is tracked here.
- _approval_lock = threading.Lock()
- _escalation_warned: set = set() # sids already warned that a bypass widened the driver mode
- # Screenshot dedup: a tight capture→act→capture loop on a static screen resends the same ~1200px image every step.
- # Every delivered frame is hashed (sha256 over mime + base64 payload); when the next capture of the SAME target
- # (app, window) in the SAME session is byte-identical, the tool returns the normal text metadata (element index
- # included) plus an explicit "screen unchanged" note and omits the image block. Append-only — no prior transcript
- # message is rewritten, so prompt-cache prefixes stay intact. Staleness is bounded by a consecutive-omission streak
- # cap: full pixels are re-delivered before compaction (which keeps only the newest image-bearing tool results)
- # could evict the image the note refers to. State is per session; sessionless calls never dedup.
- _screenshot_dedup_lock = threading.Lock()
- _last_screenshot_state: dict[str, dict[str, Any]] = {} # session_id -> {"digest", "target": (app, window), "streak"}
- _SCREENSHOT_DEDUP_MAX_STREAK = 2
- def _screenshot_dedup_check(session_id: str, digest: str, target: tuple[str, str]) -> bool:
- """True when this capture should be delivered WITHOUT its image: the previous frame for this session had identical
- bytes for the same target and the omission streak is below _SCREENSHOT_DEDUP_MAX_STREAK. Any miss (new pixels,
- new target, streak exhausted, first capture) resets the stored state to this digest so the image goes out."""
- with _screenshot_dedup_lock:
- state = _last_screenshot_state.get(session_id)
- if (state is not None and state.get("digest") == digest and state.get("target") == target
- and int(state.get("streak", 0)) < _SCREENSHOT_DEDUP_MAX_STREAK):
- state["streak"] = int(state.get("streak", 0)) + 1
- return True
- _last_screenshot_state[session_id] = {"digest": digest, "target": target, "streak": 0}
- return False
- def _reset_screenshot_dedup(session_id: Optional[str] = None) -> None:
- """Forget dedup state (all sessions, or one scoped key)."""
- with _screenshot_dedup_lock:
- if session_id is None:
- _last_screenshot_state.clear()
- else:
- _last_screenshot_state.pop(session_id, None)
- def reset_screenshot_dedup(session_id: str) -> None:
- """Compaction boundary hook (mirrors ``reset_file_dedup``): the summary may have dropped the frame an
- "unchanged" note would point at, so the next capture of this session must deliver pixels again."""
- _reset_screenshot_dedup(_scoped_sid(session_id))
- def _cua_permission_mode(session_id: str) -> str:
- """Map Hermes's approval bypass onto Cua's immutable mode; fails closed. Both identity namespaces are consulted
- (DB ``session_id`` and gateway ``session_key`` contextvar) or a gateway ``/yolo`` would be invisible here.
- Warns once per session that ``-z``/``--yolo`` swapped the driver onto a private ``unrestricted`` daemon, dropping
- the configured ceiling: deliberate (``unrestricted`` is not a config value) but easy to trigger by accident."""
- # Configured cua mode (standard | bounded; "standard" if unresolvable). bounded needs a
- # computer_use.capability_manifest — the backend fails loudly without it.
- configured = "standard"
- with contextlib.suppress(Exception):
- from tools.computer_use.cua_backend import _cua_configured_permission_mode
- configured = _cua_configured_permission_mode()
- with contextlib.suppress(Exception):
- from tools.approval import is_approval_bypass_active_for_session
- from tools.approval_context import get_current_session_key
- if is_approval_bypass_active_for_session(session_id) or (
- bool(key := get_current_session_key(default="")) and is_approval_bypass_active_for_session(key)):
- with _approval_lock:
- warn = (key := str(session_id or "")) not in _escalation_warned
- _escalation_warned.add(key)
- if warn:
- logger.warning(
- "computer_use: approval bypass (--yolo / -z) escalated the cua-driver permission mode from the "
- "configured '%s' to 'unrestricted' for this session. Runtime approval prompts are disabled and the "
- "driver's residual ceilings no longer apply. Drop the bypass flag to keep '%s', or declare a "
- "version-3 computer_use.capability_manifest to keep a ceiling on bypassed runs.", configured, configured)
- return "unrestricted"
- return configured
- def _new_backend(permission_mode: str) -> ComputerUseBackend:
- from plugins.computer_use import get_active_provider
- return get_active_provider().create_backend(permission_mode=permission_mode)
- def _install_backend(sid: str, backend: ComputerUseBackend, permission_mode: str) -> ComputerUseBackend:
- """Record a backend in the session caches (the empty session also mirrors it onto the ``_backend`` hook).
- Caller holds ``_backend_lock``."""
- global _backend
- from tools.computer_use.cua_backend import desktop_identity
- _backends[sid], _backend_permission_modes[sid] = backend, permission_mode
- _backend_displays[sid] = desktop_identity()
- _backend_call_locks[sid] = threading.RLock()
- _backend = backend if sid == "" else _backend
- return backend
- def _detach_locked(sid: str) -> tuple[Optional[ComputerUseBackend], Optional[threading.RLock]]:
- """Remove one session's cache entries, plus the ``_backend`` injection hook when it aliases the empty session
- (older callers/tests may populate only the hook). Caller holds ``_backend_lock``."""
- global _backend
- _backend_permission_modes.pop(sid, None), _backend_displays.pop(sid, None)
- backend, call_lock = _backends.pop(sid, None), _backend_call_locks.pop(sid, None)
- if sid == "":
- backend = _backend if backend is None else backend
- _backend = None if _backend is backend else _backend
- return backend, call_lock
- def _stop_backend(backend: ComputerUseBackend, call_lock: Optional[threading.RLock], on_error: Callable[[Exception], None]) -> None:
- """Stop under the session call lock (if any) so an in-flight action finishes first. Never called under
- ``_backend_lock`` (unrelated sessions stay free). ``on_error`` absorbs the failure (never raises)."""
- try:
- with call_lock if call_lock is not None else contextlib.nullcontext():
- backend.stop()
- except Exception as e:
- on_error(e)
- def _scoped_sid(session_id: str) -> str:
- """Cache key for one Hermes session's backend. Outside a served-profile scope it is the bare id
- (legacy keys byte-identical); under a multiplexed turn the routed profile's home key is appended
- so two profiles that share a session id (or a DISPLAY) never share one cua-driver (#110032).
- Every cache path — lookup, install, release — goes through this, so release finds what lookup made."""
- from hermes_constants import get_hermes_home_override, hermes_home_key
- sid = str(session_id or "")
- return sid if get_hermes_home_override() is None else f"{sid}@{hermes_home_key()}"
- def _get_backend(session_id: str = "") -> ComputerUseBackend:
- bare_sid, sid = str(session_id or ""), _scoped_sid(session_id)
- while True:
- with _backend_lock:
- # Mode resolved under the cache lock; YOLO mutation never holds the approval lock while releasing it.
- permission_mode = _cua_permission_mode(bare_sid) # approval state is keyed by the Hermes session id
- if sid == "" and _backend is not None and sid not in _backends:
- _install_backend(sid, _backend, permission_mode) # fold the injection hook into the cache
- if (cached := _backends.get(sid)) is None:
- backend = _new_backend(permission_mode)
- backend.start() # under the cache lock: one backend per session; a concurrent toggle releases it
- return _install_backend(sid, backend, permission_mode)
- from tools.computer_use.cua_backend import backend_display_stale, desktop_identity
- if (_backend_permission_modes.get(sid, "standard") == permission_mode
- and not backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())):
- return cached
- # Cua's mode and DISPLAY are fixed at daemon startup: a /yolo toggle, or a Bot Desktop that started
- # (or moved) after this backend was cached, replaces only this session's backend.
- _, stale_lock = _detach_locked(sid) # stopped outside the cache lock; the loop re-reads the mode first
- _stop_backend(cached, stale_lock, lambda e: None)
- @contextlib.contextmanager
- def _backend_for_call(session_id: str = "") -> Iterator[ComputerUseBackend]:
- """Hold a live backend through dispatch, retrying admission but never an action.
- A display/mode change or release can retire a backend before lock lookup or
- while a caller waits. Revalidate AFTER acquiring its profile-scoped call
- lock; teardown needs that same lock. Never wait under the global cache lock.
- """
- from tools.computer_use.cua_backend import backend_display_stale, desktop_identity
- sid = _scoped_sid(session_id)
- while True:
- backend = _get_backend(session_id=session_id)
- with _backend_lock:
- if _backends.get(sid) is not backend:
- continue
- call_lock = _backend_call_locks[sid]
- with call_lock:
- with _backend_lock:
- if (_backends.get(sid) is not backend
- or _backend_call_locks.get(sid) is not call_lock):
- continue
- # A queued call can outlive a display/config change even when
- # no other caller has replaced the cached backend yet.
- if (_backend_permission_modes.get(sid) != _cua_permission_mode(str(session_id or ""))
- or backend_display_stale(_backend_displays.get(sid, ""), desktop_identity())):
- continue
- yield backend
- return
- def release_computer_use_session(session_id: str) -> bool:
- """Release one session-owned backend (lifecycle seam for hosts/plugins); idempotent, True iff one was released.
- Cache entries are removed BEFORE stopping so new lookups cannot retain the stale target/ref namespace. Approval
- grants are not touched here: they live in the shared store and die with ``tools.approval.clear_session``."""
- sid = _scoped_sid(session_id)
- _reset_screenshot_dedup(sid) # the next capture of a re-created session must deliver pixels
- with _backend_lock:
- backend, call_lock = _detach_locked(sid)
- if backend is None:
- return False
- _stop_backend(backend, call_lock,
- lambda e: logger.debug("computer_use backend release failed for session %s", sid, exc_info=True))
- return True
- @atexit.register
- def _shutdown_backend_atexit() -> None:
- """Stop all cached backends so cua-driver subprocesses don't outlive us. atexit only, no signal handlers: a
- ``SystemExit`` from a prompt_toolkit key binding corrupts its coroutine state and makes the process unkillable.
- Never raises. Drops the global lock before stop(): teardown budgets 5s and must not block spawns.
- Each session backend holds a long-lived ``cua-driver`` subprocess, so without this a driver can survive
- the Hermes process that spawned it (#28152 item 3). #69903 kept the orphan from burning a core by
- disabling the cursor overlay; the process itself still lingered.
- """
- global _backend
- with _backend_lock:
- unique = {id(b): (b, _backend_call_locks.get(sid)) for sid, b in _backends.items()}
- if _backend is not None:
- unique.setdefault(id(_backend), (_backend, _backend_call_locks.get("")))
- _backend = None
- _backends.clear(), _backend_call_locks.clear(), _backend_permission_modes.clear(), _backend_displays.clear()
- with _approval_lock:
- _escalation_warned.clear()
- for backend, call_lock in unique.values():
- _stop_backend(backend, call_lock, lambda e: logger.debug("cua-driver atexit teardown failed: %s", e))
- def reset_backend_for_tests() -> None: # pragma: no cover — tear down the cached backend and per-session state
- _shutdown_backend_atexit()
- _reset_screenshot_dedup()
- def _noop_stub(name: str, *params: str, result: Any = None):
- # Recording stub: positional args are folded in under *params* (declared params default to None). ``result`` may
- # be a factory of the recorded kwargs; None -> a trivial ok ActionResult.
- def method(self, *pos, **kw):
- self.calls.append((name, call := {**dict.fromkeys(params), **dict(zip(params, pos)), **kw}))
- return result(call) if callable(result) else ActionResult(ok=True, action=name) if result is None else result
- return method
- class _NoopBackend(ComputerUseBackend): # pragma: no cover
- """Test stub (tests patch ``_new_backend`` to return it). Records ``(name, kwargs)`` calls; returns trivial results."""
- def __init__(self) -> None: self.calls: list[tuple[str, dict[str, Any]]] = []
- start = stop = lambda self: None
- def is_available(self) -> bool: return True
- capture = _noop_stub("capture", "mode", "app", "pid", "window_id", result=lambda kw: CaptureResult(
- mode=kw["mode"] or "som", width=1024, height=768, png_b64=None, elements=[], app=kw["app"] or "", window_title=""))
- click, drag, scroll = _noop_stub("click"), _noop_stub("drag"), _noop_stub("scroll")
- type_text, key, set_value = _noop_stub("type", "text"), _noop_stub("key", "keys"), _noop_stub("set_value", "value", "element")
- list_apps, list_windows = _noop_stub("list_apps", result=[]), _noop_stub("list_windows", result=[])
- focus_app = _noop_stub("focus_app", "app", "raise_window")
- # ── Dispatch ────────────────────────────────────────────────────────────────
- def handle_computer_use(args: dict[str, Any], **kwargs) -> Any:
- """Main entry point (tools.registry): a JSON string (text-only) or a dict marked `_multimodal`. Order: hard
- blocks (_reject_unsafe) -> approval scopes (destructive action, then 'bring_to_front' — persistent focus is a
- separate visible side effect with its own scope) -> backend -> dispatch under the session call lock."""
- action = (args.get("action") or "").strip().lower()
- if not action:
- return json.dumps({"error": "missing `action`"})
- session_id = str(kwargs.get("session_id") or "") # approval-state / daemon-mode isolation key
- # Bot Desktop lease: while a human drives the screen every action, capture included, is refused.
- from tools.bot_desktop import lease as _bd_lease
- from tools.bot_desktop.runtime import ensure_started_for_tool as _bd_ensure_started
- def _refused(e: Exception) -> str:
- return json.dumps({"ok": False, "action": action, "code": "human_has_control", "error": str(e)})
- try:
- admitted = _bd_lease.assert_agent_may_act()
- except _bd_lease.HumanHasControl as e:
- return _refused(e)
- _bd_ensure_started() # headless gateway: bring the profile's screen up before the backend probes DISPLAY
- if (err := _reject_unsafe(action, args)) is not None:
- return err
- scopes = ([action] if action in _ACTIONS and _ACTIONS[action].destructive else []) + (
- ["bring_to_front"] if args.get("bring_to_front") or (action == "focus_app" and args.get("raise_window")) else [])
- for scope in scopes:
- if (err := _request_approval(scope, args)) is not None:
- return err
- # Acquire separately so startup errors retain the install hint; the stack
- # releases the admitted call lock on every dispatch return or exception.
- call = contextlib.ExitStack()
- try:
- backend = call.enter_context(_backend_for_call(session_id))
- except Exception as e:
- from plugins.computer_use import DEFAULT_BACKEND, configured_backend_name
- hint = ({"hint": "If the cua-driver binary is missing, run `hermes computer-use install`. If a Python "
- "dependency is missing, the error above shows the exact install command."}
- if configured_backend_name() == DEFAULT_BACKEND else {})
- return json.dumps({"error": f"computer_use backend unavailable: {e}", **hint})
- try:
- with call:
- # Re-check under the dispatch lock: approval, backend start-up and lock waits above can take
- # seconds, and a human may have taken over meanwhile. A result produced after such a flip is
- # discarded too — it may picture what they typed.
- try:
- _bd_lease.assert_agent_may_act()
- except _bd_lease.HumanHasControl as e:
- return _refused(e)
- def _fence() -> None:
- # Any lease transition since admission voids the frame — including a full take-over /
- # hand-back cycle that already finished: it still belongs to the human's turn. Capture
- # paths call this BEFORE the frame is persisted, spilled or sent to auxiliary vision.
- if _bd_lease.get().epoch != admitted.epoch:
- raise _bd_lease.HumanHasControl(
- "A human took over this desktop while the action ran; its result was discarded. "
- "Tell the user what you need and re-capture once they hand back.")
- _fence() # input actions never receive fence=; refuse before the device op starts
- result = _dispatch(backend, action, args, fence=_fence, session_id=session_id or None)
- _fence()
- return result
- except _bd_lease.HumanHasControl as e:
- return _refused(e)
- except Exception as e:
- logger.exception("computer_use %s failed", action)
- return json.dumps({"error": f"{action} failed: {e}"})
- def _request_approval(action: str, args: dict[str, Any]) -> Optional[str]:
- """None if approved, else a JSON error string. The decision (yolo bypass, session/permanent grants, CLI prompt,
- gateway pending, cron/unattended policy, fail-closed with nobody to ask) is ``tools.approval``'s shared gate,
- so a computer_use grant is one store entry like any terminal pattern. Scope key ``cua:<action>:<mode>``:
- foreground delivery is a visible focus change, so a background ``session`` grant must NOT cover it (#67052).
- """
- from tools.approval import _run_approval_gate
- mode = "foreground" if args.get("delivery_mode") == "foreground" else "background"
- description = f"Allow computer_use to perform `{action}`?"
- result = _run_approval_gate(
- pattern_key=f"cua:{action}:{mode}", description=description,
- display_target=f"computer_use: {_summarize_action(action, args)}", approval_callback=_approval_callback,
- subject=f"computer_use `{action}` requires approval", noun="desktop actions",
- advice="Find an alternative approach that avoids driving the desktop.",
- autoapprove_log_prefix="computer_use action in non-interactive non-gateway context",
- fail_closed_when_no_human=True,
- no_human_block_message=(f"BLOCKED: computer_use `{action}` requires approval but no interactive user or "
- "gateway is present to approve it."),
- )
- if result.get("approved"):
- return None
- return json.dumps({"error": result.get("message") or "denied by user", "action": action})
- def _summarize_action(action: str, args: dict[str, Any]) -> str:
- fg = " [FOREGROUND — briefly raises the window / changes focus]" if args.get("delivery_mode") == "foreground" else ""
- return _ACTIONS.get(action, _ActionSpec(None)).summarize(action, args, fg)
- # --- handlers: (backend, action, args, **delivery) -> ActionResult (_dispatch applies the follow-up capture) or a
- # final str/dict result. `delivery` = delivery_mode + bring_to_front; only input actions use it.
- def _xy(args: dict[str, Any]) -> dict[str, Any]:
- """Click semantics: a coordinate only counts when its x is set (a bare y is not a point)."""
- return dict(x=coord[0], y=coord[1]) if (coord := args.get("coordinate")) and coord[0] is not None else dict(x=None, y=None)
- def _scroll_xy(args: dict[str, Any]) -> dict[str, Any]:
- """Scroll semantics: axes are independent — ``coordinate=[null, 100]`` scrolls at y=100 with x unset."""
- coord = args.get("coordinate") or (None, None)
- return dict(x=coord[0] if coord and coord[0] is not None else None,
- y=coord[1] if coord and coord[1] is not None else None)
- def _do_click(backend, action, args, button=None, count=1, **delivery):
- return backend.click(element=args.get("element"), **_xy(args), button=button or args.get("button") or "left",
- click_count=count, modifiers=args.get("modifiers"), **delivery)
- def _do_drag(backend, action, args, **delivery):
- src, dst = args.get("from_coordinate"), args.get("to_coordinate")
- if (args.get("from_element") is None or args.get("to_element") is None) and not (src and dst):
- return json.dumps({"error": "drag requires from_coordinate/to_coordinate or from_element/to_element"})
- return backend.drag(from_element=args.get("from_element"), to_element=args.get("to_element"),
- from_xy=tuple(src) if src else None, to_xy=tuple(dst) if dst else None,
- button=args.get("button", "left"), modifiers=args.get("modifiers"), **delivery)
- def _do_scroll(backend, action, args, **delivery):
- return backend.scroll(direction=args.get("direction", "down"), amount=int(args.get("amount", 3)),
- element=args.get("element"), **_scroll_xy(args), modifiers=args.get("modifiers"), **delivery)
- def _do_capture(backend, action, args, fence=lambda: None, session_id=None, **_):
- if (mode := str(args.get("mode", "som"))) not in {"som", "vision", "ax"}:
- return json.dumps({"error": f"bad mode {mode!r}; use som|vision|ax"})
- # pid/window_id forwarded only when given so older backends keep their defaults.
- cap = backend.capture(mode=mode, app=args.get("app"), **{k: args[k] for k in ("pid", "window_id") if args.get(k) is not None})
- fence()
- return _capture_response(cap, session_id=session_id)
- def _do_listing(backend, action, args, key, **_):
- return json.dumps({key: (items := getattr(backend, action)()), "count": len(items)})
- def _summarize_click(action: str, args: dict[str, Any], fg: str) -> str:
- where = (f" element #{args['element']}" if args.get("element") is not None
- else f" at {tuple(args['coordinate'])}" if args.get("coordinate") else "")
- return f"{action}{where}{fg}"
- # One `action`. ``input``: native input to the backend's sticky target (gets delivery kwargs + the `app=` mismatch
- # guard). ``destructive``: mutates user-visible state -> approval prompt (the rest only read).
- # ``summarize(action, args, fg_suffix)`` renders the one-line approval prompt.
- _ActionSpec = namedtuple("_ActionSpec", "handler input destructive summarize",
- defaults=(False, False, lambda a, args, fg: a + fg))
- _input = partial(_ActionSpec, input=True, destructive=True)
- _ACTIONS: dict[str, _ActionSpec] = {
- "click": _input(_do_click, summarize=_summarize_click),
- "double_click": _input(partial(_do_click, count=2), summarize=_summarize_click),
- "right_click": _input(partial(_do_click, button="right"), summarize=_summarize_click),
- "middle_click": _input(partial(_do_click, button="middle"), summarize=_summarize_click),
- "drag": _input(_do_drag, summarize=lambda a, args, fg: (f"drag {args.get('from_element') or args.get('from_coordinate')} → "
- f"{args.get('to_element') or args.get('to_coordinate')}{fg}")),
- "scroll": _input(_do_scroll, summarize=lambda a, args, fg: f"scroll {args.get('direction', '?')} x{args.get('amount', 3)}{fg}"),
- "type": _input(lambda backend, action, args, **delivery: backend.type_text(args.get("text", ""), **delivery),
- summarize=lambda a, args, fg: f"type {args.get('text', '')[:60]!r}" + ("..." if len(args.get("text", "")) > 60 else "") + fg),
- "key": _input(lambda backend, action, args, **delivery: backend.key(args.get("keys", ""), **delivery),
- summarize=lambda a, args, fg: f"key {args.get('keys', '')!r}{fg}"),
- "set_value": _input(lambda backend, action, args, **_: (
- json.dumps({"error": "set_value requires `value`"}) if args.get("value") is None
- else backend.set_value(value=str(args["value"]), element=args.get("element")))),
- "focus_app": _ActionSpec(lambda backend, action, args, **_: (
- json.dumps({"error": "focus_app requires `app`"}) if not args.get("app")
- else backend.focus_app(args["app"], raise_window=bool(args.get("raise_window")))), destructive=True,
- summarize=lambda a, args, fg: f"focus {args.get('app', '')!r}" + (" (raise)" if args.get("raise_window") else "")),
- "capture": _ActionSpec(_do_capture),
- "wait": _ActionSpec(lambda backend, action, args, **_: _text_response(backend.wait(float(args.get("seconds", 1.0))))),
- "list_apps": _ActionSpec(partial(_do_listing, key="apps")),
- "list_windows": _ActionSpec(partial(_do_listing, key="windows")),
- }
- # Native input actions deliver to the backend's sticky target; `app=` is NOT a targeting parameter (guard in _dispatch).
- _INPUT_ACTIONS = frozenset(a for a, s in _ACTIONS.items() if s.input)
- # Unknown actions are never aliased (no repairing bad model output), but the nearest real action is named as guidance.
- _ACTION_SUGGESTIONS = {
- "hotkey": "key", "press_key": "key", "keypress": "key", "key_combo": "key", "shortcut": "key", "type_text": "type",
- "input_text": "type", "screenshot": "capture", "get_window_state": "capture", "left_click": "click", "mouse_click": "click",
- }
- def _dispatch(backend: ComputerUseBackend, action: str, args: dict[str, Any], fence: Callable[[], None] = lambda: None,
- session_id: Optional[str] = None) -> Any:
- """``fence`` raises when the screen lease moved since admission; capture paths call it as soon as the
- frame is in hand, before anything derived from it leaves the process (including the screenshot dedup
- cache keyed by ``session_id``)."""
- spec = _ACTIONS.get(action)
- if spec is None:
- return json.dumps({"error": f"unknown action {action!r}" + (f" — did you mean {hint!r}? See the action enum in the tool schema."
- if (hint := _ACTION_SUGGESTIONS.get(str(action))) else "")})
- # app= guard: input goes to the sticky target from the last capture/focus_app and the backend drops app=
- # silently — refuse a clear mismatch rather than type into the wrong window while reporting ok:true.
- if (spec.input and isinstance(requested_app := args.get("app"), str) and requested_app.strip()
- and (mismatch := _input_target_mismatch(backend, requested_app)) is not None):
- return json.dumps({"ok": False, "action": action, "code": "input_target_mismatch", "error": (
- f"{action} would go to the current target {mismatch!r}, not {requested_app.strip()!r} "
- "— input actions always hit the sticky target from the last capture/focus_app. "
- f"Call capture(app={requested_app.strip()!r}) or focus_app first, then retry.")})
- # delivery_mode / bring_to_front thread through every input action (background → foreground ladder); input
- # handlers forward their kwargs to the backend verbatim, so the lease fence and the dedup session key ride
- # only on read handlers (delivery kwargs would leak into backend input calls, and vice versa).
- res = spec.handler(backend, action, args, **(dict(delivery_mode=args.get("delivery_mode"), bring_to_front=bool(args.get("bring_to_front")))
- if spec.input else dict(fence=fence, session_id=session_id)))
- return res if isinstance(res, (str, dict)) else _maybe_follow_capture(backend, res, bool(args.get("capture_after")), fence,
- session_id=session_id)
- # ── Response shaping ────────────────────────────────────────────────────────
- def _classify_action_result(res: ActionResult) -> dict[str, Any]:
- """Next ladder step from semantic evidence, in precedence order. Escalation is advisory: it never overrides
- a confirmed effect nor licenses repeating input."""
- if res.effect == "confirmed" or res.verified is True:
- return {"decision": "done"}
- if res.effect == "unverifiable":
- return {"decision": "verify_fresh_state", "hint": ("Input was delivered but not confirmed. Re-capture and check the "
- "result BEFORE any retry — do not repeat the input on an escalation recommendation alone.")}
- if res.effect == "suspected_noop" or not res.ok or res.code is not None:
- meta = res.meta if isinstance(res.meta, dict) else {}
- delivered, requested = meta.get("delivered_chars"), meta.get("requested_chars")
- if (
- res.action in ("type", "type_text")
- and res.code == "type_text_incomplete"
- and isinstance(delivered, int)
- and isinstance(requested, int)
- and requested > 0
- and delivered <= 0
- ):
- # Zero delivery on the driver's own partial-delivery verdict: the field swallowed every
- # synthetic keystroke (trusted-event checks on web inputs do this). Neither rung of the
- # delivery ladder can fix a target that drops events at the source — the AX set_value
- # path writes the value directly and bypasses event filtering. The code gate matters:
- # `type_text_synthesis_budget_exceeded` also reports delivered 0, but that is the bounded
- # synthesis budget declining to emit at all, and the driver's own `chunk` recommendation
- # stays the right next step there.
- return {
- "decision": "escalate",
- "recommended": "set_value",
- "hint": (
- "0 characters landed: this input drops synthetic keystrokes, so no delivery rung will fix it. "
- "Climb to the set_value action on the field's element index instead — it is an action, not a "
- "delivery mode, and sets the value through the accessibility API, bypassing event filtering. "
- "Re-capture first if the index is stale."
- ),
- }
- return {"decision": "escalate", **({"recommended": res.escalation.get("recommended")}
- if isinstance(res.escalation, dict) else {}), "hint": (
- "The input likely did not land. Climb one rung following `recommended`: 'px' → re-issue by coordinate; "
- "'foreground' (or a failed pixel click) → re-issue with delivery_mode='foreground' (separate approval). "
- "Do not predict the rung from the app being Electron/Chromium — react to this signal.")}
- return {"decision": "verify_fresh_state", # transport success without semantic proof is not proof of effect
- "hint": "Transport succeeded but the effect is unproven. Re-capture and confirm before continuing."}
- def _present(**fields: Any) -> dict[str, Any]:
- return {k: v for k, v in fields.items() if v} # only the truthy optional fields, in the given order
- def _action_payload(res: ActionResult) -> dict[str, Any]:
- # cua-driver's structured verdict fields only when returned (None = old driver). ok is transport success;
- # effect/escalation are the semantic verdict.
- return {"ok": res.ok, "action": res.action, **_present(message=res.message),
- **{k: v for k in ("verified", "effect", "escalation", "path", "degraded", "delivery_mode", "code")
- if (v := getattr(res, k)) is not None}, **_present(meta=res.meta),
- "verdict": _classify_action_result(res)}
- def _text_response(res: ActionResult) -> str:
- return json.dumps(_action_payload(res))
- # AX `elements` cap: dense UIs publish 500+ nodes (one capture would exhaust context); the full tree spills to a file.
- _DEFAULT_MAX_ELEMENTS = 100
- # Some providers reject images below 8x8 before the model sees the result; such captures fall back to text.
- _MIN_PROVIDER_IMAGE_DIMENSION = 8
- # Some AX trees (Discord/Slack via UIA, Electron chat clients) expose ENTIRE message bodies as labels; uncapped
- # they blew the tool-result budget and leaked private chat text. Labels identify a control, not text extraction.
- _MAX_ELEMENT_LABEL_CHARS = 120
- # Bounded cache trails: every dense capture can spill, and CLI-only sessions never run the gateway's media cleanup.
- _MAX_SPILL_FILES = _MAX_CAPTURE_FILES = 20
- def _capture_image_format(cap: CaptureResult) -> tuple[str, str]:
- # (MIME, file extension): cua-driver's explicit MIME type, else sniff the base64 prefix (JPEG starts with /9j/,
- # PNG with iVBOR). The extension matches the on-disk bytes for MIME sniffing.
- mime = cap.image_mime_type or ("image/jpeg" if (cap.png_b64 or "").startswith("/9j/") else "image/png")
- return mime, (".jpg" if mime.lower() == "image/jpeg" else ".png")
- def _bounds_unknown(bounds) -> bool:
- # No real geometry: KDE/Qt apps report [0, 0, 0, 0] for elements clickable by index; serializing that as a
- # rect invites coordinate=[0, 0] clicks.
- with contextlib.suppress(TypeError, ValueError):
- return all(int(v) == 0 for v in bounds)
- return False
- def _element_to_dict(e: UIElement) -> dict[str, Any]:
- # A zero rect is "geometry unknown", not a position — null it so no coordinate= is ever derived from it (the index still works).
- return {"index": e.index, "role": e.role, "label": e.label[:_MAX_ELEMENT_LABEL_CHARS],
- "bounds": None if _bounds_unknown(e.bounds) else list(e.bounds), "app": e.app,
- **({"label_truncated": True} if len(e.label) > _MAX_ELEMENT_LABEL_CHARS else {})}
- def _format_elements(elements: list[UIElement], max_lines: int = 40) -> list[str]:
- out = [f" #{e.index} {e.role} {e.label.replace(chr(10), ' ')[:60]!r} "
- + ("@ bounds-unknown (click by element index)" if _bounds_unknown(e.bounds) else f"@ {e.bounds}")
- + (f" [{e.app}]" if e.app else "") for e in elements[:max_lines]]
- return out + ([f" ... +{len(elements) - max_lines} more (call capture with app= to narrow)"] if len(elements) > max_lines else [])
- def _bounds_hints(elements: list[UIElement], image_width: int, image_height: int) -> tuple[Optional[float], Optional[str]]:
- """(scale, note) when element bounds live in a different coordinate space than the screenshot, else (None, None).
- On HiDPI displays AX bounds are native while the screenshot is downscaled, so coordinate= clicks read off the
- screenshot miss by the scale factor. 5% slack: window chrome can hang a few px past the captured frame without
- implying a different space. Scale heuristic: larger axis ratio wins."""
- if not elements or image_width <= 0 or image_height <= 0:
- return None, None
- max_x = max_y = 0
- for e in elements:
- try:
- x, y, w, h = e.bounds
- except (TypeError, ValueError):
- continue
- max_x, max_y = max(max_x, int(x) + int(w)), max(max_y, int(y) + int(h))
- if max_x <= image_width * 1.05 and max_y <= image_height * 1.05:
- return None, None
- note = (f"element bounds are in native desktop coordinates (extend to ~{max_x}x{max_y}), "
- f"NOT screenshot pixels ({image_width}x{image_height}). coordinate= clicks expect the native "
- "space — derive click points from element bounds, or scale screenshot positions up accordingly")
- return round(max(max_x / image_width, max_y / image_height), 2), note
- _bounds_scale = lambda elements, image_width, image_height: _bounds_hints(elements, image_width, image_height)[0]
- _bounds_space_note = lambda elements, image_width, image_height: _bounds_hints(elements, image_width, image_height)[1]
- def _capture_view(cap: CaptureResult, max_elements: int) -> SimpleNamespace:
- """One capture's derived facts, computed once for every response branch: ``visible`` is the capped element list,
- ``dims_omitted`` an image below the provider minimum."""
- visible, dims = cap.elements[:max_elements], None
- with contextlib.suppress(Exception): # (width, height) of the inline PNG/JPEG screenshot, else the backend's
- dims = image_dimensions_from_bytes(base64.b64decode(cap.png_b64, validate=False)) if cap.png_b64 else None
- width, height = dims or (cap.width, cap.height)
- scale, note = _bounds_hints(visible, width, height)
- # Capped labels / capped element array: spill the complete tree for on-demand reads.
- lost_detail = len(cap.elements) > len(visible) or any(len(e.label) > _MAX_ELEMENT_LABEL_CHARS for e in visible)
- too_small = bool(dims) and min(dims) < _MIN_PROVIDER_IMAGE_DIMENSION
- has_image = bool(cap.png_b64) and cap.mode != "ax" and not too_small
- # The driver's own AX walk may have stopped at the ``max_elements`` the backend sent: then the spill file is
- # NOT the full tree, and the hint must not promise one.
- ax_capped = len(cap.elements) >= cap.ax_max_elements > 0
- return SimpleNamespace(cap=cap, visible=visible, total=len(cap.elements), width=width, height=height,
- truncated=len(cap.elements) - len(visible), bounds_scale=scale, bounds_note=note,
- ax_capped=cap.ax_max_elements if ax_capped else 0,
- elements_file=_spill_elements_to_file(cap) if lost_detail else None,
- screenshot_path=_persist_capture_image(cap) if has_image else None,
- dims_omitted=dims if too_small else None, has_image=has_image)
- def _capture_summary_lines(v: SimpleNamespace) -> list[str]:
- """Human-readable capture summary; line ORDER is contract. Lists only what `elements` surfaces, otherwise the
- summary names indices the model can't find."""
- notes = (
- v.bounds_note and v.bounds_note + (f"; estimated scale ~{v.bounds_scale}x (screenshot position x "
- f"{v.bounds_scale} ≈ native coordinate)" if v.bounds_scale else ""),
- v.screenshot_path and f"shareable screenshot saved to {v.screenshot_path}",
- v.cap.note,
- v.elements_file and (f"{'' if v.ax_capped else 'full '}element tree with untruncated labels "
- f"saved to {v.elements_file} — read_file/search_files it if you need dropped label "
- "text or elements beyond the cap"),
- v.ax_capped and (f"accessibility walk capped at {v.ax_capped} elements; pass app= to narrow"),
- )
- return [
- f"capture mode={v.cap.mode} {v.width}x{v.height}"
- + (f" app={v.cap.app}" if v.cap.app else "") + (f" window={v.cap.window_title!r}" if v.cap.window_title else ""),
- f"{v.total} interactable element(s):",
- *(f" ({note})" for note in notes if note),
- *_format_elements(v.visible),
- *([f" (screenshot omitted: {v.dims_omitted[0]}x{v.dims_omitted[1]} is below the "
- f"{_MIN_PROVIDER_IMAGE_DIMENSION}x{_MIN_PROVIDER_IMAGE_DIMENSION} provider minimum)"] if v.dims_omitted else []),
- ]
- def _text_capture_payload(v: SimpleNamespace, summary: str, extra: Optional[dict[str, Any]] = None) -> str:
- """JSON text payload shared by the AX, vision-unavailable and aux-vision branches. Key order is contract:
- fixed fields, ``extra`` branch markers, then set optionals."""
- return json.dumps({
- "mode": v.cap.mode, "width": v.width, "height": v.height, "app": v.cap.app, "window_title": v.cap.window_title,
- "elements": [_element_to_dict(e) for e in v.visible], "total_elements": v.total, "summary": summary,
- **(extra or {}),
- **_present(truncated_elements=v.truncated, elements_file=v.elements_file, screenshot_path=v.screenshot_path,
- bounds_scale=v.bounds_scale),
- })
- def _capture_digest(cap: CaptureResult) -> str:
- return hashlib.sha256((str(cap.image_mime_type or "") + ":").encode("utf-8")
- + (cap.png_b64 or "").encode("ascii", "ignore")).hexdigest()
- def _capture_response(cap: CaptureResult, max_elements: int = _DEFAULT_MAX_ELEMENTS,
- session_id: Optional[str] = None) -> Any:
- v = _capture_view(cap, max_elements)
- lines = _capture_summary_lines(v)
- summary, extra = "\n".join(lines), None # multimodal/aux paths use this; text paths append notes and rebuild
- if v.has_image and session_id and _screenshot_dedup_check(
- _scoped_sid(session_id), _capture_digest(cap), (str(cap.app or ""), str(cap.window_title or ""))):
- # Unchanged frame: same pixels for the same target in this session — no image (and no aux-vision call);
- # the text metadata is fresh and the note says which earlier result still applies.
- lines.append(" (screen unchanged since the previous capture — image omitted to save context; the previous "
- "capture's screenshot/analysis still shows the current state. Element indices below are fresh "
- "and remain the preferred way to act.)")
- extra = {"screen_unchanged": True}
- elif v.has_image:
- # Hand the screenshot to auxiliary.vision (text-only result) when the main model may not consume images
- # natively; returning the multimodal envelope unconditionally tripped HTTP 404/400 at the provider.
- if not _should_route_through_aux_vision(): # envelope carrying the screenshot (not the elements array, so no truncation note)
- return {
- "_multimodal": True,
- "content": [{"type": "text", "text": summary},
- {"type": "image_url", "image_url": {"url": f"data:{_capture_image_format(cap)[0]};base64,{cap.png_b64}"}}],
- "text_summary": summary,
- "meta": {"mode": cap.mode, "width": v.width, "height": v.height, "elements": v.total, "png_bytes": cap.png_bytes_len,
- **_present(screenshot_path=v.screenshot_path, elements_file=v.elements_file, bounds_scale=v.bounds_scale)},
- }
- # Decide whether to hand the screenshot to the auxiliary.vision pipeline (text-only result) or keep
- # the multimodal envelope (main model handles vision natively). Issue #24015: previously the
- # multimodal envelope was returned unconditionally, so non-vision main models tripped HTTP 404 / 400
- # at the provider boundary even when auxiliary.vision was explicitly configured to handle this.
- routed = _route_capture_through_aux_vision(
- cap, summary, visible_elements=v.visible, truncated_elements=v.truncated,
- elements_file=v.elements_file, screenshot_path=v.screenshot_path)
- if routed is not None:
- return routed
- # Aux routing requested but failed (vision node down, empty analysis...): the multimodal envelope could
- # now break with a provider error, so degrade to text.
- lines.append(" (vision unavailable: the auxiliary vision model could not be reached; screenshot "
- "omitted. Element-index actions still work — drive via the element list above.)")
- extra = {"vision_unavailable": True}
- if v.truncated: # text paths carry the `elements` array, so the truncation note applies
- lines.append(f" (response truncated to {len(v.visible)} of {v.total} elements; the full tree is in "
- "elements_file — read_file/search_files it, or pass app= to narrow scope)")
- return _text_capture_payload(v, "\n".join(lines), extra)
- def _maybe_follow_capture(backend: ComputerUseBackend, res: ActionResult, do_capture: bool,
- fence: Callable[[], None] = lambda: None, session_id: Optional[str] = None) -> Any:
- # No follow-up capture after a failed action: a normal-looking screenshot would suggest success.
- if not do_capture or not res.ok:
- return _text_response(res)
- try:
- # Recapture the exact window when known: on Linux several unrelated windows may share an app name, so
- # app-only recapture can switch targets.
- exact = {k: (getattr(backend, "_last_target", None) or {}).get(k) for k in ("pid", "window_id")}
- cap = backend.capture(mode=_capture_after_mode(), **(exact if None not in exact.values()
- else {"app": getattr(backend, "_last_app", None)}))
- except Exception as e:
- logger.warning("follow-up capture failed: %s", e)
- return _text_response(res)
- fence() # a frame taken after a takeover never reaches the dedup cache nor the model
- resp, payload = _capture_response(cap, session_id=session_id), _action_payload(res)
- if isinstance(resp, dict) and resp.get("_multimodal"):
- # Keep the evidence/verdict contract visible alongside the image — it governs whether input may repeat.
- resp["content"][0]["text"] = resp["text_summary"] = json.dumps(payload) + "\n\n" + resp["text_summary"]
- resp["action_result"] = payload
- return resp
- return json.dumps({**json.loads(resp), **payload}) # text capture: merge the action payload in
- # ── Cache files (screenshots, element spills, vision temps) ─────────────────
- def _secure_dir_policy(cache_dir) -> None:
- """Create/reconcile a Hermes media-cache dir owner-only (0700), except managed.
- A capture is as sensitive as the screen it came from — an open password
- manager, a private chat, a bank tab. The umask-derived 0755 these dirs
- used to get made every local account able to list (and read) those
- frames whenever ``HERMES_HOME`` itself is traversable, which is exactly
- what the documented ``HERMES_HOME_MODE=0701`` web-server escape hatch
- arranges. Delegates to the same house policy as every Hermes secret dir —
- ``hermes_cli.config._secure_dir`` — with the mode passed to ``mkdir`` so
- there is no window between mkdir and chmod; managed/NixOS installs keep
- their group-share design (the mode is left to the configured umask/setgid
- because these lazily-created dirs are not covered by the module's
- ``systemd.tmpfiles`` rules). Best-effort; never breaks a capture.
- """
- try:
- managed = False
- try:
- from hermes_cli.config import is_managed
- managed = bool(is_managed())
- except Exception: # pragma: no cover - defensive
- pass
- if managed:
- cache_dir.mkdir(parents=True, exist_ok=True)
- return
- make_user_dir(cache_dir, parents=True, exist_ok=True, mode=0o700)
- try:
- from hermes_cli.config import _secure_dir
- _secure_dir(cache_dir)
- except Exception as exc: # pragma: no cover - defensive
- logger.debug("computer_use: cache dir chmod skipped: %s", exc)
- except Exception as exc: # pragma: no cover - defensive
- logger.debug("computer_use: cache dir creation failed for %s: %s", cache_dir, exc)
- cache_dir.mkdir(parents=True, exist_ok=True)
- def _write_private_bytes(path, data: bytes) -> None:
- """Write ``data`` to ``path`` with owner-only (0600) permissions.
- ``Path.write_bytes`` would land 0644 under a default umask. POSIX mode
- bits are advisory on Windows (``os.chmod`` there only toggles the
- read-only flag), so this is a best-effort narrowing that falls back to a
- plain write rather than failing the capture.
- """
- import os as _os_priv
- flags = _os_priv.O_WRONLY | _os_priv.O_CREAT | _os_priv.O_TRUNC
- try:
- fd = _os_priv.open(str(path), flags, 0o600)
- except OSError:
- path.write_bytes(data)
- return
- with _os_priv.fdopen(fd, "wb") as handle:
- handle.write(data)
- def _cache_file(subdir: str, legacy: str, name: str, pattern: str = "", cap: int = 0):
- """Path for a new file under ``$HERMES_HOME/<subdir>`` (dir created owner-only, per #77579).
- With ``pattern``/``cap``, first unlinks the oldest matching files so at most ``cap - 1`` remain
- (best-effort)."""
- from hermes_constants import get_hermes_dir # lazy so tests can patch get_hermes_dir
- cache_dir = get_hermes_dir(subdir, legacy)
- _secure_dir_policy(cache_dir)
- with contextlib.suppress(Exception):
- files = sorted(cache_dir.glob(pattern), key=lambda p: p.stat().st_mtime) if pattern else []
- for stale in files[: max(0, len(files) - (cap - 1))]:
- stale.unlink(missing_ok=True)
- return cache_dir / name
- def _write_cache_file(what: str, subdir: str, legacy: str, name: str, pattern: str, cap: int,
- write: Callable[[Any], None]) -> Optional[str]:
- """Bounded cache write via ``write(path)``; the path, or None on any failure — an unwritable cache must never
- break control (a capture keeps working without its spill/screenshot copy)."""
- try:
- write(path := _cache_file(subdir, legacy, name, pattern, cap))
- return str(path)
- except Exception as exc: # pragma: no cover - defensive
- logger.debug("computer_use: %s failed: %s", what, exc)
- return None
- def _persist_capture_image(cap: CaptureResult) -> Optional[str]:
- """Copy of the capture in Hermes' media cache so attachment surfaces can deliver it (None without an image)."""
- return _write_cache_file(
- "screenshot persistence", "cache/images", "image_cache", f"computer_use_{uuid.uuid4().hex}{_capture_image_format(cap)[1]}",
- "computer_use_*.*", _MAX_CAPTURE_FILES,
- lambda p: _write_private_bytes(p, base64.b64decode(cap.png_b64, validate=False)),
- ) if cap.png_b64 else None
- def _spill_elements_to_file(cap: CaptureResult) -> Optional[str]:
- """FULL element tree (untruncated labels) in a cache file — the read_file/search_files escape hatch for capped text."""
- payload = {"app": cap.app, "window_title": cap.window_title, "total_elements": len(cap.elements),
- "elements": [{"index": e.index, "role": e.role, "label": e.label, "bounds": list(e.bounds), "app": e.app}
- for e in cap.elements]}
- return _write_cache_file("element spill", "cache/computer_use", "computer_use_cache", f"elements_{uuid.uuid4().hex}.json",
- "elements_*.json", _MAX_SPILL_FILES,
- lambda p: p.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8"))
- # ── auxiliary.vision routing for captured screenshots ───────────────────────
- _MAX_VISION_DIM = 1456 # longest image side handed to the aux vision model: full-resolution desktop captures tokenize heavily
- # and can overflow small local-model context windows; ~1456px keeps SOM badges legible while cutting vision latency.
- def _shrink_capture_for_vision(raw: bytes, ext: str, max_dim: int = _MAX_VISION_DIM) -> tuple[bytes, Optional[str]]:
- """Downscale encoded image bytes so the longest side is <= max_dim -> ``(bytes, scale_note)``. note is None when
- unchanged (fits, or Pillow unavailable/failed), else it tells the vision model the factor so reported
- coordinates map back to the real screen instead of being silently wrong."""
- try:
- from io import BytesIO
- from PIL import Image
- img = Image.open(BytesIO(raw))
- if max(img.size) <= max_dim:
- return raw, None
- (orig_w, orig_h), out = img.size, BytesIO()
- img.thumbnail((max_dim, max_dim))
- new_w, new_h = img.size
- img.save(out, format="JPEG" if ext == ".jpg" else "PNG")
- fx, fy = (orig_w / new_w if new_w else 1.0), (orig_h / new_h if new_h else 1.0)
- factor_clause = (f"multiply any coordinates you report by {fx:.2f} to map back to the real screen." if f"{fx:.2f}" == f"{fy:.2f}"
- else f"multiply any x coordinates you report by {fx:.2f} and any y coordinates by {fy:.2f} to map back to the real screen.")
- return out.getvalue(), f"Screenshot downscaled from {orig_w}x{orig_h} to {new_w}x{new_h} for vision; {factor_clause}"
- except Exception as exc:
- logger.debug("computer_use: vision downscale skipped: %s", exc)
- return raw, None
- def _should_route_through_aux_vision() -> bool:
- """True when ``_capture_response`` should hand the PNG to aux vision. Any failure returns False (fail open) so a
- broken config never silently drops the screenshot for vision-capable main models. Decided per capture (the
- config read is the signature-cached ``load_config_readonly``), so ``/model``, a profile switch or an
- ``image_input_mode`` edit applies to the next screenshot instead of a stale verdict."""
- stage = "import"
- try:
- from agent.auxiliary_client import _read_main_model, _read_main_provider
- from hermes_cli.config import load_config_readonly
- from tools.vision_tools import _native_tool_result_images
- stage = "decision"
- provider, model = _read_main_provider() or "", _read_main_model() or ""
- # The shared native-tool-result gate (vision_analyze, browser screenshots, MCP images use it too), so a
- # screenshot takes the same lane whichever tool produced it; anything it cannot vouch for goes to aux.
- return not _native_tool_result_images(provider, model, load_config_readonly())
- except Exception as exc: # pragma: no cover - defensive
- logger.debug("computer_use: aux-vision routing %s failed: %s", stage, exc)
- return False
- def _capture_after_mode() -> str:
- """Mode for ``capture_after`` follow-ups. Default ``som`` (screenshot)."""
- with contextlib.suppress(Exception):
- from hermes_cli.config import load_config
- mode = str(((load_config() or {}).get("computer_use") or {}).get("capture_after_mode", "som") or "som")
- return mode if (mode := mode.strip().lower()) in {"som", "vision", "ax"} else "som"
- return "som"
- _VISION_PROMPT = ("Describe what is visible in this desktop application screenshot in concise but specific terms. Mention "
- "the app name and window title if visible, the overall layout, any labelled buttons, menus or text fields, "
- "and any prominent text content the user would need to know about. Do not invent details that are not "
- "actually visible.\n\nAX/SOM index for cross-reference:\n")
- def _route_capture_through_aux_vision(cap: CaptureResult, summary: str, *, visible_elements: Optional[list[UIElement]] = None,
- truncated_elements: int = 0, elements_file: Optional[str] = None,
- screenshot_path: Optional[str] = None) -> Optional[str]:
- """Pre-analyse the capture via ``vision_analyze_tool`` (temp file under ``$HERMES_HOME/cache/vision/``) and merge
- the description with the AX/SOM summary into one text payload. JSON, or None on any failure."""
- if not cap.png_b64:
- return None
- problem, temp_image_path = "aux-vision import failed", None
- try:
- from model_tools import _run_async
- from tools.vision_tools import vision_analyze_tool
- problem = "failed to decode capture base64"
- raw = base64.b64decode(cap.png_b64, validate=False)
- problem = None # from here on failures are loud (warning)
- ext = _capture_image_format(cap)[1]
- temp_image_path = _cache_file("cache/vision", "temp_vision_images", f"computer_use_{uuid.uuid4().hex}{ext}")
- raw, scale_note = _shrink_capture_for_vision(raw, ext)
- _write_private_bytes(temp_image_path, raw)
- prompt = _VISION_PROMPT + summary + (f"\n\nNote: {scale_note}" if scale_note else "")
- result_json = _run_async(vision_analyze_tool(str(temp_image_path), prompt))
- except Exception as exc:
- if problem:
- logger.debug("computer_use: %s: %s", problem, exc)
- else:
- logger.warning("computer_use: auxiliary.vision pre-analysis failed (%s); "
- "returning to caller without aux analysis", exc)
- return None
- finally:
- if temp_image_path is not None:
- with contextlib.suppress(Exception):
- os.unlink(str(temp_image_path))
- # The ``analysis`` field of vision_analyze_tool's JSON result; raw text when it isn't JSON; empty -> no merge.
- analysis_text = result_json.strip() if isinstance(result_json, str) else ""
- with contextlib.suppress(TypeError, json.JSONDecodeError):
- parsed = json.loads(analysis_text)
- analysis_text = str(parsed.get("analysis") or "").strip() if isinstance(parsed, dict) else ""
- if not analysis_text:
- return None
- # Same element cap as every other capture branch; dumping cap.elements in full would bypass max_elements
- # exactly for non-vision main models. Dimensions are the backend's on this branch.
- view = SimpleNamespace(cap=cap, visible=cap.elements if visible_elements is None else visible_elements,
- total=len(cap.elements), width=cap.width, height=cap.height, truncated=truncated_elements,
- elements_file=elements_file, screenshot_path=screenshot_path, bounds_scale=None)
- return _text_capture_payload(view, summary, {"vision_analysis": analysis_text,
- "vision_analysis_routed_via": "auxiliary.vision"})
- # ── Availability check (used by the tool registry check_fn) ─────────────────
- def check_computer_use_requirements() -> bool:
- """The selected ``computer_use.backend``'s ``is_available()``. An unregistered name stays True so the call
- surfaces the "not registered (available: ...)" error instead of the tool silently vanishing."""
- from plugins.computer_use import get_active_provider
- try:
- return get_active_provider().is_available()
- except LookupError:
- return True
- def get_computer_use_schema() -> dict[str, Any]:
- from tools.computer_use.schema import COMPUTER_USE_SCHEMA
- return COMPUTER_USE_SCHEMA
|