cua_backend_session.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521
  1. """cua-driver MCP session plumbing: the asyncio bridge thread and the lazily-started, self-healing
  2. ``_CuaDriverSession`` (MCP transport with a ``cua-driver call`` CLI fallback). Config/policy
  3. helpers are looked up lazily through the facade."""
  4. from __future__ import annotations
  5. import asyncio
  6. import base64
  7. import concurrent.futures
  8. import contextlib
  9. import json
  10. import logging
  11. import os
  12. import threading
  13. from typing import Any, Dict, List, Optional
  14. from hermes_cli._subprocess_compat import windows_hide_flags
  15. from tools.computer_use import cua_backend_driver as _driver
  16. from tools.computer_use.cua_backend_parse import _extract_tool_result, _mcp_field, _tool_envelope
  17. logger = logging.getLogger("tools.computer_use.cua_backend")
  18. class _AsyncBridge:
  19. """Runs one asyncio loop on a daemon thread; marshals coroutines from the caller."""
  20. def __init__(self) -> None:
  21. self._loop: Optional[asyncio.AbstractEventLoop] = None
  22. self._thread: Optional[threading.Thread] = None
  23. self._ready = threading.Event()
  24. def start(self) -> None:
  25. if self._thread and self._thread.is_alive():
  26. return
  27. self._ready.clear()
  28. def _run() -> None:
  29. self._loop = asyncio.new_event_loop()
  30. asyncio.set_event_loop(self._loop)
  31. self._ready.set()
  32. try:
  33. self._loop.run_forever()
  34. finally:
  35. with contextlib.suppress(Exception):
  36. self._loop.close()
  37. self._thread = threading.Thread(target=_run, daemon=True, name="cua-driver-loop")
  38. self._thread.start()
  39. if not self._ready.wait(timeout=5.0):
  40. raise RuntimeError("cua-driver asyncio bridge failed to start")
  41. def run(self, coro, timeout: Optional[float] = 30.0) -> Any:
  42. from agent.async_utils import safe_schedule_threadsafe
  43. alive = self._loop is not None and self._thread is not None and self._thread.is_alive()
  44. fut = safe_schedule_threadsafe(coro, self._loop) if alive else None # closes the coroutine on failure
  45. if fut is None:
  46. if asyncio.iscoroutine(coro):
  47. coro.close() # no-op when safe_schedule_threadsafe already closed it
  48. raise RuntimeError("cua-driver bridge not started")
  49. return fut.result(timeout=timeout)
  50. def stop(self) -> None:
  51. if self._loop and self._loop.is_running():
  52. self._loop.call_soon_threadsafe(self._loop.stop)
  53. if self._thread:
  54. self._thread.join(timeout=2.0)
  55. self._thread = self._loop = None
  56. # Fail-closed messages for calls whose effect on the remote screen is unknown. The action MAY have landed, so it
  57. # is never replayed; the caller decides after taking fresh state.
  58. _UNKNOWN_OUTCOME_MESSAGES = {
  59. "transport_outcome_unknown": (
  60. "cua-driver transport failed during {name}; the action outcome is unknown, so Hermes "
  61. "did not replay it. Take fresh state before deciding whether to act again."),
  62. "timeout_outcome_unknown": (
  63. "cua-driver MCP call {name} timed out; the action outcome is unknown and may still have "
  64. "taken effect on the remote screen. The session has been marked suspect and will be "
  65. "recreated before the next computer-use call. Take fresh state before deciding "
  66. "whether to act again."),
  67. }
  68. def _outcome_unknown(name: str, exc: Exception, code: str) -> dict[str, Any]:
  69. """Fail-closed ``isError`` result for *code* (see ``_UNKNOWN_OUTCOME_MESSAGES``)."""
  70. message = _UNKNOWN_OUTCOME_MESSAGES[code].format(name=name)
  71. return _tool_envelope(message, [], {"ok": False, "code": code, "message": message, "operation": name,
  72. "next_step": "fresh_state", "detail": str(exc)}, True, [])
  73. def _tool_field(obj: Any, *names: str) -> Any:
  74. """``_mcp_field`` plus the ``model_extra`` fallback some MCP SDKs (Pydantic v2) forward custom fields via."""
  75. value = _mcp_field(obj, names[0], names[-1])
  76. return (getattr(obj, "model_extra", None) or {}).get(names[-1]) if value is None else value
  77. _CLI_ATTEMPTS = 4 # CLI fallback transport retries (backoff 0.5s doubling)
  78. def _cli_run_json(cmd: list[str], env: dict[str, str], name: str, timeout: float) -> Any:
  79. """Run ``cua-driver call`` with backoff until it prints JSON; return the parsed value. "daemon is not running"
  80. is PERMANENT for this invocation (the CLI needs the machine-wide daemon socket, which Linux installs typically
  81. never start) -> fail fast, no ~3.5s backoff."""
  82. import subprocess as _subprocess
  83. import time as _time
  84. backoff, last_err = 0.5, ""
  85. for attempt in range(_CLI_ATTEMPTS):
  86. try:
  87. proc = _subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace",
  88. timeout=max(15.0, timeout), creationflags=windows_hide_flags(), env=env,
  89. stdin=_subprocess.DEVNULL, check=False)
  90. except Exception as e: # pragma: no cover - subprocess spawn failure
  91. raise RuntimeError(f"cua-driver CLI fallback for {name} failed to spawn: {e}") from e
  92. out, err = (proc.stdout or "").strip(), proc.stderr or ""
  93. last_err = out[:200] or err[:200]
  94. if "daemon is not running" in out or "daemon is not running" in err:
  95. raise RuntimeError(f"cua-driver CLI fallback for {name} unavailable: the "
  96. "machine-wide cua-driver daemon is not running (the "
  97. "CLI transport requires it; the MCP runtime does not).")
  98. start = min((i for i in (out.find("{"), out.find("[")) if i != -1), default=-1)
  99. with contextlib.suppress(json.JSONDecodeError):
  100. if start != -1:
  101. return json.loads(out[start:])
  102. if attempt < _CLI_ATTEMPTS - 1: # no JSON (EAGAIN warning / empty) — retry with backoff
  103. logger.warning("cua-driver CLI fallback for %s got no JSON (attempt %d/%d); "
  104. "retrying in %.1fs", name, attempt + 1, _CLI_ATTEMPTS, backoff)
  105. _time.sleep(backoff)
  106. backoff *= 2
  107. raise RuntimeError(f"cua-driver CLI fallback for {name} returned no JSON after "
  108. f"{_CLI_ATTEMPTS} attempts: {last_err}")
  109. def _cli_result(parsed: Any, shot_file: Optional[str]) -> dict[str, Any]:
  110. """Remap a ``cua-driver call`` JSON body into the ``_extract_tool_result`` shape (no ``image_mime_types`` key)."""
  111. if not isinstance(parsed, dict):
  112. return _tool_envelope(None, [], None, False)
  113. # In-band logical failures with exit 0 must still fail closed.
  114. is_error = parsed.get("isError") is True or parsed.get("is_error") is True
  115. shot = parsed.get("screenshot_png_b64")
  116. # Otherwise the screenshot was routed to a file (ours or the daemon's choice).
  117. fpath = parsed.get("screenshot_file_path") or shot_file
  118. if not shot and fpath and os.path.exists(fpath):
  119. try:
  120. with open(fpath, "rb") as fh:
  121. shot = base64.b64encode(fh.read()).decode("ascii")
  122. except Exception as e:
  123. logger.debug("cua-driver CLI fallback: failed reading %s: %s", fpath, e)
  124. data: Any = parsed.get("tree_markdown")
  125. if data is not None and parsed.get("element_count") is not None:
  126. data = f"{parsed['element_count']} elements\n{data}"
  127. return _tool_envelope(data, [shot] if shot else [], parsed, is_error)
  128. def _logical_error_text(result: dict[str, Any]) -> str:
  129. """Flatten a logical MCP error into text for narrow classification."""
  130. chunks: list[str] = []
  131. for value in (result.get("data"), result.get("structuredContent")):
  132. if value is None:
  133. continue
  134. try:
  135. chunks.append(value if isinstance(value, str) else json.dumps(value, sort_keys=True))
  136. except (TypeError, ValueError):
  137. chunks.append(str(value))
  138. return "\n".join(chunks)
  139. def _is_ended_session_result(result: Any) -> bool:
  140. """Recognise cua-driver's explicit recoverable ended-session result."""
  141. if not isinstance(result, dict) or result.get("isError") is not True:
  142. return False
  143. message = _logical_error_text(result).lower()
  144. return ("session" in message and "start_session" in message
  145. and ("has ended" in message or "session ended" in message))
  146. class _CuaDriverSession:
  147. """Holds the mcp ClientSession. Spawned lazily; re-entered on drop. Lifecycle ownership: one long-running
  148. coroutine (`_lifecycle_coro`) opens the stdio_client + ClientSession contexts, populates capabilities, sets
  149. `_ready_event`, waits on `_shutdown_event`, then closes the contexts — enter and exit in the SAME task, as
  150. anyio's cancel-scope invariant requires (each `bridge.run(coro)` is a NEW task). Tool calls run in short-lived
  151. tasks touching only the session object."""
  152. # Handshake calls issued BY start()/stop() — exempt from call_tool's auto-restart guard, or start() would recurse.
  153. _LIFECYCLE_CALLS = frozenset({"start_session", "end_session"})
  154. # Idempotent reads, safe to replay after a broken transport. Mutations stay out: a lost response does not
  155. # prove they failed.
  156. _TRANSPORT_REPLAY_SAFE_TOOLS = frozenset({"get_cursor_position", "get_displays", "get_screen_size",
  157. "get_window_state", "list_apps", "list_windows"})
  158. # A timed-out MCP session is wedged for later calls, so it is recreated before the next non-lifecycle
  159. # call_tool. Class-level default: tests that bypass __init__ see healthy.
  160. # See #74799.
  161. _timeout_suspect = False
  162. def __init__(self, bridge: _AsyncBridge, embedded_daemon: Optional[Any] = None) -> None:
  163. self._bridge, self._embedded_daemon, self._session = bridge, embedded_daemon, None
  164. self._lock, self._started = threading.Lock(), False
  165. # Per-tool capability-token sets from `tools/list` (read via supports_capability). Raw input schemas are
  166. # the source of truth for action properties: 0.9-era drivers advertise delivery_mode in inputSchema
  167. # without the ``input.delivery_mode`` token.
  168. # Keys are tool names (e.g. "click", "get_window_state"); values are sets of capability strings
  169. # (e.g. "accessibility.element_tokens", "input.keyboard.type.terminal_safe"). Empty until the
  170. # session starts; consumers should call `supports_capability` rather than reading directly. See
  171. # #47072.
  172. self._capabilities: dict[str, set] = {}
  173. self._tool_schemas: dict[str, dict[str, Any]] = {}
  174. self._capability_version, self._ready_event = "", threading.Event()
  175. self._shutdown_event: Optional[asyncio.Event] = None # created on bridge loop
  176. self._lifecycle_future = None # concurrent.futures.Future
  177. self._setup_error: Optional[BaseException] = None
  178. # Declared via start_session; revives an ended-session rejection non-re-entrantly.
  179. # Stable driver-side identity declared through start_session. Used to revive a logical ended-session
  180. # rejection without recursive call_tool re-entry or backend-owned state (#71166).
  181. self._declared_session_id: Optional[str] = None
  182. self._transport_generation, self._transport_reset_callback = 0, None
  183. async def _lifecycle_coro(self) -> None:
  184. """Owns the stdio MCP contexts: open, signal ready, block on shutdown, clean up — all in one task."""
  185. import time as _time
  186. from mcp import ClientSession, StdioServerParameters
  187. from mcp.client.stdio import stdio_client
  188. from tools.computer_use import cua_backend as _cb
  189. from tools.environments.local import _sanitize_subprocess_env
  190. self._shutdown_event = asyncio.Event() # built on the loop's own thread
  191. _t0 = _time.monotonic()
  192. # Phase marker: the ready-timeout error reports HOW FAR a wedged startup got.
  193. # Phase marker surfaced by the ready-timeout error (issue #57025): when startup wedges, the caller
  194. # reports HOW FAR it got instead of an opaque "never reached ready".
  195. self._startup_phase = "binary-check"
  196. try:
  197. driver_cmd = _driver.resolve_cua_driver_cmd()
  198. if not driver_cmd and _cb.sandbox_mcp_invocation() is None:
  199. raise RuntimeError(_driver.cua_driver_install_hint())
  200. self._startup_phase = "manifest-discovery"
  201. daemon = self._embedded_daemon
  202. (command, args), child_env = (
  203. (daemon.proxy_invocation(), daemon.child_env()) if daemon is not None
  204. else _cb.sandbox_mcp_invocation() or (_driver._resolve_mcp_invocation(driver_cmd), _cb.cua_driver_child_env()))
  205. _t_manifest = _time.monotonic()
  206. # Telemetry policy first (default: disabled), then strip Hermes secrets.
  207. params = StdioServerParameters(command=command, args=args, env=_sanitize_subprocess_env(child_env))
  208. async with stdio_client(params) as (read, write):
  209. self._startup_phase = "mcp-initialize"
  210. async with ClientSession(read, write) as session:
  211. await session.initialize()
  212. _t_init = _time.monotonic()
  213. # Capabilities BEFORE exposing the session: the first call sees them.
  214. self._startup_phase = "capability-discovery"
  215. await self._populate_capabilities(session)
  216. self._session, self._startup_phase = session, "ready"
  217. self._ready_event.set()
  218. logger.info("cua-driver session ready in %.1fs (manifest=%.1fs, mcp_init=%.1fs)",
  219. _time.monotonic() - _t0, _t_manifest - _t0, _t_init - _t_manifest)
  220. await self._shutdown_event.wait()
  221. except BaseException as e:
  222. # Ordinary errors and anyio CancelledError alike: start() surfaces this.
  223. self._setup_error = e
  224. self._ready_event.set()
  225. raise
  226. finally:
  227. # A session that dies for ANY reason must be re-enterable: the next call sees _started False and
  228. # rebuilds. Atomic bool write — stop() may hold _lock.
  229. self._session, self._started = None, False
  230. # Reset _started so a session that dies for ANY reason (MCP connection drop, driver crash, unexpected
  231. # coro exit) is re-enterable: the next start()/call sees _started False and rebuilds the session instead
  232. # of hanging forever on a dead one via _require_started(). On the normal stop() path this is a harmless
  233. # idempotent no-op (stop() already set it False). A plain bool write is atomic in CPython, so this is
  234. # safe from the bridge-loop thread without taking self._lock (which stop() may hold while awaiting this
  235. # coro's future). See #55048 Bug 1.
  236. async def _populate_capabilities(self, session: Any) -> None:
  237. """Cache per-tool capability sets, input schemas and capability_version from tools/list. Soft
  238. prerequisite: on failure the map stays empty (capability False)."""
  239. self._capabilities, self._tool_schemas, self._capability_version = {}, {}, ""
  240. try:
  241. tools_list = await session.list_tools()
  242. for tool in getattr(tools_list, "tools", []) or []:
  243. tool_name = getattr(tool, "name", None)
  244. if not isinstance(tool_name, str):
  245. continue
  246. caps, schema = _tool_field(tool, "capabilities"), _tool_field(tool, "input_schema", "inputSchema")
  247. self._capabilities[tool_name] = (
  248. {c for c in caps if isinstance(c, str)} if isinstance(caps, list) else set())
  249. self._tool_schemas[tool_name] = dict(schema) if isinstance(schema, dict) else {}
  250. # capability_version is a sibling of `tools` in tools/list (NOT in initialize).
  251. cv = _tool_field(tools_list, "capability_version")
  252. if isinstance(cv, str):
  253. self._capability_version = cv
  254. except Exception as e:
  255. logger.debug("cua-driver tools/list capability discovery failed: %s", e)
  256. def start(self) -> None:
  257. with self._lock:
  258. if not self._started:
  259. self._bridge.start()
  260. self._start_lifecycle_locked()
  261. self._started = True
  262. def _start_lifecycle_locked(self) -> None:
  263. """Spawn the lifecycle owner and wait for ready. Caller holds self._lock."""
  264. self._ready_event = threading.Event()
  265. self._setup_error = self._shutdown_event = None
  266. # The future tracks the WHOLE lifecycle; readiness is signalled via _ready_event.
  267. loop = self._bridge._loop
  268. if loop is None:
  269. raise RuntimeError("cua-driver bridge not started")
  270. self._lifecycle_future = asyncio.run_coroutine_threadsafe(self._lifecycle_coro(), loop)
  271. if not self._ready_event.wait(timeout=30.0):
  272. self._signal_shutdown_locked()
  273. # Surface which startup phase wedged (issue #57025) — "doctor passes but the wrapper times out"
  274. # reports are undiagnosable from a bare "never reached ready".
  275. from hermes_constants import display_hermes_home
  276. raise RuntimeError(
  277. f"cua-driver session never reached ready (timeout 30s; stuck in phase: "
  278. f"{getattr(self, '_startup_phase', 'unknown')}). Run `hermes computer-use doctor` and check "
  279. f"{display_hermes_home()}/logs/agent.log for the phase timings.")
  280. if self._setup_error is not None:
  281. raise RuntimeError(f"cua-driver session setup failed: {self._setup_error}") from self._setup_error
  282. self._transport_generation += 1
  283. if self._transport_generation > 1:
  284. self._notify_transport_reset()
  285. def stop(self) -> None:
  286. with self._lock:
  287. if self._started:
  288. self._started = False
  289. self._stop_lifecycle_locked()
  290. def set_transport_reset_callback(self, callback: Any) -> None:
  291. """Register a synchronous cache invalidation hook for transport swaps."""
  292. self._transport_reset_callback = callback
  293. def _notify_transport_reset(self) -> None:
  294. try:
  295. if (callback := getattr(self, "_transport_reset_callback", None)) is not None:
  296. callback()
  297. except Exception as exc:
  298. logger.debug("cua-driver transport reset callback failed: %s", exc)
  299. def _stop_lifecycle_locked(self) -> None:
  300. self._signal_shutdown_locked()
  301. fut, self._lifecycle_future = self._lifecycle_future, None
  302. try:
  303. if fut is not None:
  304. fut.result(timeout=5.0)
  305. except concurrent.futures.TimeoutError:
  306. logger.warning("cua-driver session shutdown timed out (5s)")
  307. except Exception as e:
  308. logger.warning("cua-driver shutdown error: %s", e)
  309. def _signal_shutdown_locked(self) -> None:
  310. """Set the asyncio shutdown event from the caller's thread."""
  311. loop, event = self._bridge._loop, self._shutdown_event
  312. if loop is not None and event is not None and loop.is_running():
  313. with contextlib.suppress(RuntimeError): # loop closed — nothing to signal
  314. loop.call_soon_threadsafe(event.set)
  315. async def _call_tool_async(self, name: str, args: dict[str, Any]) -> dict[str, Any]:
  316. return _extract_tool_result(await self._session.call_tool(name, args))
  317. # ── Capability detection ─────────────────────────────────────────
  318. # See #47072.
  319. def supports_capability(self, capability: str, tool: Optional[str] = None) -> bool:
  320. """Driver advertises *capability* for *tool* (or ANY tool). False before start.
  321. capability token (trycua/cua#1961 capability vocabulary).
  322. """
  323. caps = [self._capabilities.get(tool, set())] if tool is not None else self._capabilities.values()
  324. return any(capability in c for c in caps)
  325. def _has_tool(self, name: str) -> bool:
  326. """``tools/list`` advertised *name*. Routes capture() (PNG capture moved into ``get_window_state``).
  327. False before discovery — callers treat that as "unknown"."""
  328. return name in self._capabilities
  329. def supports_input_property(self, tool: str, property_name: str) -> bool:
  330. """Live tools/list schema accepts *property_name* (fails closed; no version guessing)."""
  331. schema = getattr(self, "_tool_schemas", {}).get(tool, {})
  332. properties = schema.get("properties") if isinstance(schema, dict) else None
  333. return isinstance(properties, dict) and property_name in properties
  334. @property
  335. def capabilities_discovered(self) -> bool:
  336. """tools/list populated the map; when False ``_has_tool`` is untrustworthy."""
  337. return bool(self._capabilities)
  338. @property
  339. def capability_version(self) -> str:
  340. """Driver-advertised capability vocabulary version ("" on old builds)."""
  341. return self._capability_version
  342. # ── Error classification (instance-patchable seams; result-shape checks live at module level) ──
  343. @staticmethod
  344. def _is_closed_session_error(exc: Exception) -> bool:
  345. """True for MCP/stdio failures that are recoverable by reconnecting."""
  346. name, module = exc.__class__.__name__, getattr(exc.__class__, "__module__", "")
  347. return (name in {"ClosedResourceError", "BrokenResourceError", "EndOfStream"}
  348. or (module.startswith("anyio") and "Resource" in name)
  349. or isinstance(exc, (BrokenPipeError, EOFError)))
  350. @staticmethod
  351. def _is_transient_daemon_error(exc: Exception) -> bool:
  352. """Daemon-proxy EAGAIN congestion: on macOS the ``cua-driver mcp`` bridge uses a non-blocking unix socket
  353. and heavy ops (``get_window_state``) fail with ``os error 35`` when its buffer is full. A retry succeeds,
  354. so back off / fall back instead of surfacing an empty 0x0 capture."""
  355. msg = str(exc)
  356. return any(needle in msg for needle in ("Resource temporarily unavailable", "os error 35",
  357. "daemon transport error", "daemon proxy"))
  358. # ── Recovery ─────────────────────────────────────────────────────
  359. def _redeclare_session(self, timeout: float, failure_msg: str) -> bool:
  360. """start_session with the declared id; log *failure_msg* and return False on rejection."""
  361. session_id = self._declared_session_id
  362. result = self._bridge.run(self._call_tool_async("start_session", {"session": session_id}), timeout=timeout)
  363. if result.get("isError") is True:
  364. logger.warning(failure_msg, session_id, _logical_error_text(result))
  365. return result.get("isError") is not True
  366. def _recreate_session(self, name: str, timeout: float, log_msg: str, *, restart: bool = True,
  367. clear_timeout_suspect: bool = False) -> None:
  368. """Log *log_msg* (``%s`` = *name*), then either start() a dead session or (``restart``) tear
  369. down and rebuild the MCP lifecycle under ``_lock`` with capabilities repopulated from scratch;
  370. finally re-attach the declared public label inside the replacement private lifecycle."""
  371. logger.warning(log_msg, name)
  372. if not restart:
  373. self.start()
  374. else:
  375. with self._lock:
  376. try:
  377. if self._started:
  378. self._stop_lifecycle_locked()
  379. except Exception as e:
  380. logger.debug("cua-driver session cleanup before reconnect failed: %s", e)
  381. self._started = False
  382. self._capabilities, self._tool_schemas, self._capability_version = {}, {}, ""
  383. self._start_lifecycle_locked()
  384. self._started = True
  385. if clear_timeout_suspect:
  386. self._timeout_suspect = False
  387. if getattr(self, "_declared_session_id", None):
  388. self._redeclare_session(timeout, "cua-driver public session label %s could not be restored: %s")
  389. def _call_tool_via_cli(self, name: str, args: dict[str, Any], timeout: float) -> dict[str, Any]:
  390. """Fallback transport: ``cua-driver call <tool> <json>`` subprocess. The MCP stdio bridge can persistently
  391. fail heavy calls (``get_window_state``) with EAGAIN while the plain CLI, on its own daemon socket, keeps
  392. working. Output is remapped to the ``_extract_tool_result`` shape. ``get_window_state`` routes its
  393. screenshot to a temp file (``screenshot_out_file``) so the daemon returns a tiny JSON body, not the
  394. multi-megabyte base64 blob that congests the socket; ``_cli_result`` reads it back."""
  395. import tempfile as _tempfile
  396. from tools.computer_use import cua_backend as _cb
  397. from tools.environments.local import _sanitize_subprocess_env
  398. call_args, shot_file = dict(args), None
  399. if name == "get_window_state" and "screenshot_out_file" not in call_args:
  400. fd, shot_file = _tempfile.mkstemp(prefix="cua_shot_", suffix=".png")
  401. os.close(fd)
  402. call_args["screenshot_out_file"] = shot_file
  403. driver_command = _driver.resolve_cua_driver_cmd()
  404. if not driver_command:
  405. raise RuntimeError(_driver.cua_driver_install_hint())
  406. child_env, socket_args = _cb.cua_driver_child_env(), []
  407. daemon = getattr(self, "_embedded_daemon", None)
  408. if daemon is not None:
  409. driver_command, child_env = daemon.proxy_invocation()[0], daemon.child_env()
  410. socket_args = ["--socket", daemon.socket_path]
  411. cmd = [driver_command, "call", name, json.dumps(call_args), *socket_args]
  412. try:
  413. return _cli_result(_cli_run_json(cmd, _sanitize_subprocess_env(child_env), name, timeout), shot_file)
  414. finally:
  415. if shot_file and os.path.exists(shot_file):
  416. with contextlib.suppress(OSError):
  417. os.remove(shot_file)
  418. def call_tool(self, name: str, args: dict[str, Any], timeout: float = 30.0) -> dict[str, Any]:
  419. if name not in self._LIFECYCLE_CALLS:
  420. # A prior MCP timeout marks the session suspect (possibly wedged): recreate it so one timeout never
  421. # poisons the run. Healthy sessions are never restarted here.
  422. if self._timeout_suspect:
  423. self._recreate_session(
  424. name, timeout, "cua-driver session suspect after earlier MCP timeout; recreating before %s",
  425. clear_timeout_suspect=True)
  426. # A prior session may have died (MCP drop / driver crash) and reset _started.
  427. if not self._started:
  428. self._recreate_session(
  429. name, timeout, "cua-driver session not active on %s; (re)starting before call", restart=False)
  430. if not self._started:
  431. raise RuntimeError("cua-driver session not started")
  432. try:
  433. result = self._bridge.run(self._call_tool_async(name, args), timeout=timeout)
  434. except concurrent.futures.TimeoutError as e:
  435. # Fail closed: the action may have landed, so never replay it.
  436. # MCP deadline hit (#74799): the session is suspect and must be recreated before the next call.
  437. # Fail closed — the action may have taken effect on the remote screen, so never replay it here;
  438. # surface the uncertainty instead (#74799).
  439. self._timeout_suspect = True
  440. logger.warning("cua-driver MCP timed out on %s; marking session suspect "
  441. "for recreation before the next call", name)
  442. return _outcome_unknown(name, e, "timeout_outcome_unknown")
  443. except Exception as e:
  444. if self._is_transient_daemon_error(e):
  445. if name not in self._TRANSPORT_REPLAY_SAFE_TOOLS:
  446. self._notify_transport_reset()
  447. return _outcome_unknown(name, e, "transport_outcome_unknown")
  448. logger.warning("cua-driver MCP transport failed on %s (%s); "
  449. "falling back to CLI transport", name, e)
  450. return self._call_tool_via_cli(name, args, timeout)
  451. if not self._is_closed_session_error(e):
  452. raise
  453. self._recreate_session(name, timeout, "cua-driver MCP session closed during %s; reconnecting once")
  454. if name not in self._TRANSPORT_REPLAY_SAFE_TOOLS:
  455. return _outcome_unknown(name, e, "transport_outcome_unknown")
  456. result = self._bridge.run(self._call_tool_async(name, args), timeout=timeout)
  457. # Remember only a SUCCESSFULLY declared identity: no stale recovery state.
  458. declared_id, ok = args.get("session"), result.get("isError") is not True
  459. if name == "start_session" and ok and isinstance(declared_id, str) and declared_id:
  460. self._declared_session_id = declared_id
  461. if _is_ended_session_result(result):
  462. # Revive the stable session and replay the rejected call once; a 2nd rejection surfaces as-is.
  463. # Never re-runs lifecycle calls -> an end_session result is final.
  464. session_id = self._declared_session_id
  465. if session_id and name not in self._LIFECYCLE_CALLS:
  466. logger.warning("cua-driver session %s ended during %s; reviving and retrying once", session_id, name)
  467. if self._redeclare_session(timeout, "cua-driver session %s could not be revived: %s"):
  468. result = self._bridge.run(self._call_tool_async(name, args), timeout=timeout)
  469. elif name == "end_session" and ok and declared_id == self._declared_session_id:
  470. self._declared_session_id = None
  471. return result