sandbox_host.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. """Bot Desktop hosted INSIDE the configured terminal backend (docker / ssh / a plugin backend with streams).
  2. Same launcher, same lease, same pane: only WHERE Xvnc + Xfce run changes. ``runtime`` decides between the
  3. gateway host and this module via ``placement()``; callers keep using ``runtime.start/stop/status/desktop_env``.
  4. Layout inside the sandbox, under ``<sandbox tmp>/hermes-bot-desktop/<profile>/``: ``launcher.sh`` (copied in
  5. at start), ``env`` (published DISPLAY/XAUTHORITY/DBUS), ``rfb.sock`` (Xvnc's unix socket), ``launcher.pid``.
  6. Host-side state under ``<HERMES_HOME>/bot-desktop/``: ``sandbox.json`` = ``{"env_key": ..., "display": ...}``
  7. so status() knows which terminal environment owns the screen without re-deriving it.
  8. The Desktop pane never sees a socket path: ``open_rfb_stream()`` returns a Popen whose stdin/stdout ARE the
  9. RFB byte stream, relayed by a 15-line Python script inside the sandbox (python3 is in every sandbox image;
  10. socat is not). cua-driver runs the same way: ``cua_mcp_argv()`` is the docker/ssh prefix + ``cua-driver mcp``.
  11. """
  12. from __future__ import annotations
  13. import base64
  14. import json
  15. import logging
  16. import shlex
  17. import subprocess
  18. import time
  19. from pathlib import Path
  20. from typing import Any, Dict, Optional
  21. from agent.deadline import poll_until
  22. from hermes_constants import get_hermes_home
  23. from tools.environments import streams
  24. logger = logging.getLogger(__name__)
  25. # The user desktop processes run as inside the sandbox image. nikolaik (and hermes-sandbox:desktop on top of
  26. # it) ships uid 1000 `pn`; a root Xvnc/Chromium/cua-driver is the wrong shape (Chromium refuses the sandbox,
  27. # AT-SPI wants a user session). Falls back to the exec default when the image has no such user.
  28. DESKTOP_USER = "pn"
  29. _REQUIRED = ("Xvnc", "xfwm4", "xfce4-panel", "xfdesktop", "xfsettingsd", "dbus-run-session", "xauth", "xdpyinfo", "xprop")
  30. SANDBOX_IMAGE_HINT = "nousresearch/hermes-sandbox:desktop"
  31. _RELAY = (
  32. "import os,socket,sys,threading\n"
  33. "s=socket.socket(socket.AF_UNIX);s.connect(sys.argv[1])\n"
  34. "def up():\n"
  35. " while True:\n"
  36. " d=os.read(0,65536)\n"
  37. " if not d: break\n"
  38. " s.sendall(d)\n"
  39. " s.shutdown(socket.SHUT_WR)\n"
  40. "threading.Thread(target=up,daemon=True).start()\n"
  41. "while True:\n"
  42. " d=s.recv(65536)\n"
  43. " if not d: break\n"
  44. " os.write(1,d)\n"
  45. )
  46. def plugin_image_advice(backend: Optional[str]) -> Optional[str]:
  47. """How to get the desktop stack into a plugin backend's sandbox, or None for a built-in backend: a plugin
  48. picks its image through its own settings, so there is no ``terminal.<plugin>_image`` key to name."""
  49. from tools.terminal_tool_config import _BUILTIN_BACKENDS
  50. if not backend or backend in _BUILTIN_BACKENDS:
  51. return None
  52. return (f"give the {backend} plugin's sandbox the desktop stack through the plugin's own image setting "
  53. f"({SANDBOX_IMAGE_HINT} has it)")
  54. def _marker() -> Path:
  55. return get_hermes_home() / "bot-desktop" / "sandbox.json"
  56. def _read_marker() -> dict[str, Any]:
  57. try:
  58. return json.loads(_marker().read_text(encoding="utf-8-sig"))
  59. except (OSError, ValueError):
  60. return {}
  61. def _owner_identity(env: Any) -> dict[str, Any]:
  62. """What the marker records about the sandbox hosting the screen, so a restarted gateway (empty terminal
  63. registry) can tell a still-running sandbox from one that was removed."""
  64. ident: dict[str, Any] = {"backend": type(env).__name__}
  65. container = getattr(env, "_container_id", None)
  66. if container:
  67. ident["container"] = container
  68. ident["docker"] = getattr(env, "_docker_exe", "docker")
  69. return ident
  70. _ALIVE_CACHE: dict[str, tuple[float, bool]] = {}
  71. def marker_sandbox_alive(marker: dict[str, Any]) -> bool:
  72. """Whether the sandbox a marker names still exists. Docker: ``docker inspect`` on the recorded container
  73. (cached a few seconds; this sits on the browser's per-command path). Other backends (ssh host, apptainer
  74. instance) cannot be probed from here without their environment object: the re-attach's own
  75. ``published_env`` probe decides, so they read as alive."""
  76. container = marker.get("container")
  77. if not container:
  78. return True
  79. now = time.monotonic()
  80. cached = _ALIVE_CACHE.get(container)
  81. if cached and now - cached[0] < 5.0:
  82. return cached[1]
  83. try:
  84. proc = subprocess.run([marker.get("docker") or "docker", "inspect", "-f", "{{.State.Running}}", container],
  85. capture_output=True, text=True, timeout=15, check=False, stdin=subprocess.DEVNULL)
  86. alive = proc.returncode == 0 and proc.stdout.strip() == "true"
  87. except (OSError, subprocess.TimeoutExpired):
  88. alive = False
  89. _ALIVE_CACHE[container] = (now, alive)
  90. return alive
  91. def _remote_dir(env: Any, profile: str) -> str:
  92. return f"{streams.sandbox_temp_dir(env).rstrip('/')}/hermes-bot-desktop/{profile}"
  93. def desktop_home(env: Any) -> str:
  94. """``$HOME`` of the user the desktop and the browser run as inside ``env`` (cached on the env). This
  95. is where state that must outlive a container stop goes: ``/tmp`` in the Docker backend is a 512 MB
  96. tmpfs that is emptied on every stop and shared with the sandbox's scratch, so a browser profile
  97. (logins, cookies) there would vanish and grow into that cap."""
  98. cached = getattr(env, "_bd_desktop_home", None)
  99. if cached:
  100. return cached
  101. proc = streams.run_in(env, ["bash", "-c", 'printf %s "$HOME"'], user=_user_for(env), timeout=15)
  102. home = proc.stdout.decode("utf-8", "replace").strip() if proc.returncode == 0 else ""
  103. if not home or not home.startswith("/"):
  104. home = f"/home/{DESKTOP_USER}" if _user_for(env) else "/root"
  105. env._bd_desktop_home = home
  106. return home
  107. def browser_profile_dir(env: Any) -> str:
  108. """The ONE ``--user-data-dir`` the agent's browser and the dock's Browser icon share inside the
  109. sandbox (same rule as the host: two profiles would mean the human logs into a jar the bot never sees)."""
  110. return f"{desktop_home(env).rstrip('/')}/.hermes/bot-desktop/browser-profile"
  111. _CHROMIUM_PROBE = (
  112. 'for d in "${PLAYWRIGHT_BROWSERS_PATH:-}" /opt/playwright "$HOME/.cache/ms-playwright"; do'
  113. ' [ -n "$d" ] || continue;'
  114. ' for c in "$d"/chromium-*/chrome-linux/chrome "$d"/chromium-*/chrome-linux64/chrome; do'
  115. ' [ -x "$c" ] && { printf %s "$c"; exit 0; }; done; done; exit 1'
  116. )
  117. def chromium_executable(env: Any) -> Optional[str]:
  118. """Path of the Playwright Chromium the agent's browser uses inside ``env`` (the dock's Browser icon must run
  119. the same binary), or None when the image has none. Cached on the env."""
  120. cached = getattr(env, "_bd_chromium", "unset")
  121. if cached != "unset":
  122. return cached
  123. proc = streams.run_in(env, ["bash", "-c", _CHROMIUM_PROBE], user=_user_for(env), timeout=20)
  124. exe = proc.stdout.decode("utf-8", "replace").strip() if proc.returncode == 0 else ""
  125. env._bd_chromium = exe or None
  126. return env._bd_chromium
  127. _GRAB_SCRIPT = """
  128. import io, sys
  129. from PIL import ImageGrab
  130. img = ImageGrab.grab(xdisplay=sys.argv[1])
  131. img.thumbnail((int(sys.argv[2]), int(sys.argv[3])))
  132. buf = io.BytesIO()
  133. img.convert("RGB").save(buf, "JPEG", quality=int(sys.argv[4]), optimize=True)
  134. sys.stdout.buffer.write(buf.getvalue())
  135. """
  136. def grab_jpeg(env: Any, published: dict[str, str], max_size: tuple[int, int], quality: int) -> Optional[bytes]:
  137. """One JPEG of the sandbox screen, grabbed INSIDE the sandbox (the X socket and its cookie live there).
  138. Needs Pillow in the image (the published desktop image ships it); None when the grab fails."""
  139. display = published.get("DISPLAY")
  140. if not display:
  141. return None
  142. child_env = {k: v for k, v in published.items() if k in ("DISPLAY", "XAUTHORITY")}
  143. try:
  144. proc = streams.run_in(env, ["python3", "-c", _GRAB_SCRIPT, display, str(max_size[0]), str(max_size[1]),
  145. str(quality)], child_env=child_env, user=_user_for(env), timeout=20)
  146. except (OSError, subprocess.SubprocessError) as exc:
  147. logger.debug("sandbox screen grab failed: %s", exc)
  148. return None
  149. if proc.returncode != 0 or not proc.stdout.startswith(b"\xff\xd8"):
  150. logger.debug("sandbox screen grab failed: %s", proc.stderr.decode("utf-8", "replace")[-300:])
  151. return None
  152. return proc.stdout
  153. def _user_for(env: Any) -> Optional[str]:
  154. """DESKTOP_USER when the sandbox has it, else None (exec default). Cached on the env object."""
  155. cached = getattr(env, "_bd_desktop_user", "unset")
  156. if cached != "unset":
  157. return cached
  158. probe = streams.run_in(env, ["id", "-u", DESKTOP_USER], timeout=15)
  159. user = DESKTOP_USER if probe.returncode == 0 else None
  160. env._bd_desktop_user = user
  161. return user
  162. def missing_binaries(env: Any) -> list[str]:
  163. script = "for b in " + " ".join(_REQUIRED) + '; do command -v "$b" >/dev/null 2>&1 || echo "$b"; done'
  164. proc = streams.run_in(env, ["bash", "-c", script], user=_user_for(env), timeout=20)
  165. return [b for b in proc.stdout.decode("utf-8", "replace").split() if b]
  166. def _published(env: Any, rdir: str) -> dict[str, str]:
  167. proc = streams.run_in(env, ["bash", "-c", f"kill -0 $(cat {shlex.quote(rdir)}/launcher.pid 2>/dev/null) 2>/dev/null "
  168. f"&& test -S {shlex.quote(rdir)}/rfb.sock && cat {shlex.quote(rdir)}/env"],
  169. user=_user_for(env), timeout=15)
  170. if proc.returncode != 0:
  171. return {}
  172. out: dict[str, str] = {}
  173. for line in proc.stdout.decode("utf-8", "replace").splitlines():
  174. k, sep, v = line.partition("=")
  175. if sep:
  176. out[k.strip()] = v.strip()
  177. return out
  178. def published_env(env: Any, profile: str) -> dict[str, str]:
  179. return _published(env, _remote_dir(env, profile))
  180. def start(env: Any, profile: str, *, geometry: str, wait_seconds: float = 20.0,
  181. browser_exec: Optional[str] = None, browser_exec_line: Optional[str] = None) -> dict[str, str]:
  182. """Bring the screen up inside ``env`` (idempotent); returns the published env. Raises RuntimeError naming
  183. the blocker."""
  184. rdir = _remote_dir(env, profile)
  185. live = _published(env, rdir)
  186. if live.get("DISPLAY"):
  187. # Adopting a screen the sandbox kept while this host's state was lost (fresh HERMES_HOME, a stop()
  188. # whose kill missed): without the marker, status/thumbnail/stop would not know it is ours.
  189. return _record(env, rdir, profile, live)
  190. stop(env, profile) # a dead launcher may have left Xvnc holding :20; the relaunch needs it gone
  191. missing = missing_binaries(env)
  192. if missing:
  193. advice = plugin_image_advice(getattr(env, "_hermes_backend_name", None)) or (
  194. f"use {SANDBOX_IMAGE_HINT} (the default sandbox base plus the desktop stack) as "
  195. f"terminal.docker_image / modal_image")
  196. raise RuntimeError(
  197. f"Bot Desktop needs {', '.join(missing)} inside the terminal backend's sandbox. The configured image "
  198. f"does not have them; {advice}, or set bot_desktop.placement: gateway.")
  199. user = _user_for(env)
  200. launcher = (Path(__file__).with_name("launcher.sh").read_bytes())
  201. wallpaper = Path(__file__).with_name("wallpaper.png").read_bytes()
  202. seed = (
  203. f"set -e; rm -rf {shlex.quote(rdir)}; mkdir -p {shlex.quote(rdir)}; cd {shlex.quote(rdir)};"
  204. f" echo {shlex.quote(base64.b64encode(launcher).decode())} | base64 -d > launcher.sh;"
  205. f" echo {shlex.quote(base64.b64encode(wallpaper).decode())} | base64 -d > wallpaper.png;"
  206. " chmod 0700 . launcher.sh"
  207. )
  208. proc = streams.run_in(env, ["bash", "-c", seed], user=user, timeout=30)
  209. if proc.returncode != 0:
  210. raise RuntimeError(f"could not seed the sandbox desktop dir: {proc.stderr.decode('utf-8', 'replace')[-500:]}")
  211. num = 20 # one profile per sandbox; the launcher's stale-lock logic handles a leftover :20
  212. child_env = {
  213. "HERMES_BD_PROFILE": profile, "HERMES_BD_DISPLAY_NUM": str(num), "HERMES_BD_SOCKET": f"{rdir}/rfb.sock",
  214. "HERMES_BD_XAUTH": f"{rdir}/Xauthority", "HERMES_BD_ENV_FILE": f"{rdir}/env",
  215. "HERMES_BD_CONFIG_HOME": f"{rdir}/xdg", "HERMES_BD_GEOMETRY": geometry, "HERMES_BD_WALLPAPER": f"{rdir}/wallpaper.png",
  216. }
  217. if browser_exec and browser_exec_line:
  218. child_env["HERMES_BD_BROWSER_EXEC"] = browser_exec
  219. child_env["HERMES_BD_BROWSER_EXEC_LINE"] = browser_exec_line
  220. # The launcher must outlive this exec (docker exec / ssh) and be its own session so stop() can take the
  221. # whole group (Xvnc, dbus, Xfce) with one kill. `setsid -f` forks, so `$!` would be the wrong pid: the
  222. # launcher records ITS OWN pid (= its session id) before doing anything else.
  223. spawn = (f"cd {shlex.quote(rdir)} && setsid -f bash -c "
  224. f"'echo $$ > launcher.pid; exec bash launcher.sh' > launcher.log 2>&1 < /dev/null; "
  225. f"for _ in 1 2 3 4 5 6 7 8 9 10; do [ -s launcher.pid ] && break; sleep 0.1; done; [ -s launcher.pid ]")
  226. proc = streams.run_in(env, ["bash", "-c", spawn], child_env=child_env, user=user, timeout=30)
  227. if proc.returncode != 0:
  228. raise RuntimeError(f"sandbox desktop launcher did not start: {proc.stderr.decode('utf-8', 'replace')[-500:]}")
  229. def _live_display() -> dict[str, str] | None:
  230. live = _published(env, rdir)
  231. return live if live.get("DISPLAY") else None
  232. live = poll_until(_live_display, wait_seconds, 0.25)
  233. if live:
  234. logger.info("Bot Desktop for profile %s up inside %s on %s", profile, type(env).__name__, live["DISPLAY"])
  235. return _record(env, rdir, profile, live)
  236. tail = streams.run_in(env, ["tail", "-c", "2000", f"{rdir}/launcher.log"], user=user, timeout=10).stdout
  237. stop(env, profile)
  238. raise RuntimeError(f"sandbox desktop did not publish its display within {wait_seconds:.0f}s:\n"
  239. f"{tail.decode('utf-8', 'replace')}")
  240. def _record(env: Any, rdir: str, profile: str, live: dict[str, str]) -> dict[str, str]:
  241. _marker().parent.mkdir(parents=True, exist_ok=True)
  242. _marker().write_text(json.dumps({"display": live["DISPLAY"], "dir": rdir, "profile": profile,
  243. **_owner_identity(env)}), encoding="utf-8")
  244. return live
  245. def stop(env: Any, profile: str) -> bool:
  246. """Kill the launcher's session (Xvnc, dbus, Xfce, anything the desktop spawned) and, as a backstop, every
  247. process still holding this profile's rfb.sock or Xauthority path (a launcher that lost its pid file left
  248. an Xvnc that 'Server is already active for display 20' on the next start). True when something was live."""
  249. rdir = _remote_dir(env, profile)
  250. q = shlex.quote(rdir)
  251. script = f"""
  252. live=0
  253. p=$(cat {q}/launcher.pid 2>/dev/null)
  254. if [ -n "$p" ] && kill -0 "$p" 2>/dev/null; then live=1; kill -TERM -- -"$p" 2>/dev/null || kill -TERM "$p" 2>/dev/null; fi
  255. for pid in $(pgrep -f -- {shlex.quote(rdir + "/")} 2>/dev/null); do [ "$pid" != "$$" ] && {{ live=1; kill -TERM "$pid" 2>/dev/null; }}; done
  256. sleep 0.5
  257. for pid in $(pgrep -f -- {shlex.quote(rdir + "/")} 2>/dev/null); do [ "$pid" != "$$" ] && kill -KILL "$pid" 2>/dev/null; done
  258. rm -f {q}/env {q}/launcher.pid {q}/rfb.sock
  259. [ "$live" = 1 ]
  260. """
  261. proc = streams.run_in(env, ["bash", "-c", script], user=_user_for(env), timeout=30)
  262. _marker().unlink(missing_ok=True)
  263. return proc.returncode == 0
  264. def open_rfb_stream(env: Any, profile: str) -> subprocess.Popen:
  265. """Popen whose stdin/stdout carry the RFB bytes of the sandbox's Xvnc."""
  266. rdir = _remote_dir(env, profile)
  267. return streams.open_stream(env, ["python3", "-c", _RELAY, f"{rdir}/rfb.sock"], user=_user_for(env))
  268. def cua_mcp_invocation(env: Any, profile: str, published: dict[str, str]) -> tuple[str, list[str]]:
  269. """``(command, args)`` for ``StdioServerParameters``: the backend's exec prefix running ``cua-driver mcp`` on
  270. the sandbox display."""
  271. argv = streams.remote_command(env, ["cua-driver", "mcp", "--no-overlay"], child_env=published,
  272. user=_user_for(env), interactive=True)
  273. if argv is None:
  274. raise RuntimeError(f"{type(env).__name__} cannot host cua-driver")
  275. return argv[0], argv[1:]