browser_session_sandbox.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265
  1. """The bot's Chromium INSIDE the terminal backend's sandbox (Bot Desktop placed there).
  2. Same browser a human takes over in the pane: the sandbox image's Playwright Chromium on the profile the
  3. dock's Browser icon uses, on the sandbox screen's DISPLAY. Its DevTools port is the sandbox's loopback,
  4. unreachable from this host (Docker bridge / ssh remote), so it is forwarded over the sandbox's exec
  5. stream; Chromium's DevTools accepts any loopback ``Host`` header, port included.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import shlex
  10. import subprocess
  11. import threading
  12. import time
  13. from typing import Any, Optional
  14. logger = logging.getLogger(__name__)
  15. _LAUNCH_TIMEOUT_S = 60.0
  16. # Sandbox Chromium this process launched, per shared key: ``(sandbox env, pid, start time, user-data-dir)``.
  17. # Every session in that container attaches to it, whichever profile it serves, so it is stopped only when the
  18. # LAST of them tears down (release_session_backend's in-use check on the shared ``launcher_key``); the env is
  19. # kept so a teardown outside the owner's scope hits the right sandbox. The start time (``/proc/<pid>/stat``
  20. # field 22, read inside the sandbox at launch) and the ``--user-data-dir`` token are the identity a stop
  21. # re-checks there before signalling: a crashed Chromium's pid may name another job by then.
  22. _launched: dict[str, tuple[Any, int, Optional[str], str]] = {}
  23. _launched_lock = threading.Lock()
  24. # Per shared key: held across "is a browser running?" + launch + claim, and across the stop, so two sessions
  25. # first using one container never both see no browser and both launch.
  26. _key_locks: dict[str, threading.Lock] = {}
  27. # Endpoints a liveness probe confirmed: ``cdp_url -> (id(env), sandbox port, forward failures then, when)``. A
  28. # browser call re-probes after a connection through its forward failed, the sandbox was replaced, or the
  29. # confirmation is older than _CONFIRMED_TTL_S (a relaunch on the same port that nothing connected to yet
  30. # leaves no failure behind), so a healthy call spends no exec in the sandbox.
  31. _confirmed: dict[str, tuple[int, int, int, float]] = {}
  32. _CONFIRMED_TTL_S = 30.0
  33. # Exit code of the liveness script when the sandbox answers "not running": no DevTools port file, or nothing
  34. # listens on its port. Any other failure (docker/ssh transport error, timeout) says nothing about Chromium.
  35. _GONE_RC = 3
  36. GONE = "gone"
  37. def _sandbox():
  38. from tools.bot_desktop import runtime as _bd_runtime
  39. env = _bd_runtime._sandbox_env(create=True)
  40. if env is None:
  41. raise RuntimeError("the terminal backend's sandbox is not running, so there is nowhere to run the browser")
  42. published = _bd_runtime.published_env()
  43. if not published.get("DISPLAY"):
  44. # The sandbox died under a live screen: fail here rather than run headless in a dead sandbox or,
  45. # worse, fall back to the host.
  46. raise RuntimeError("the screen inside the terminal backend's sandbox is gone; start it again")
  47. return env, published
  48. def _read_devtools(env, user, udd: str) -> Optional[tuple[int, str]]:
  49. from tools.environments import streams
  50. proc = streams.run_in(env, ["cat", f"{udd}/DevToolsActivePort"], user=user, timeout=10)
  51. lines = proc.stdout.decode("utf-8", "replace").splitlines() if proc.returncode == 0 else []
  52. if len(lines) >= 2 and lines[0].strip().isdigit() and lines[1].startswith("/devtools/browser/"):
  53. return int(lines[0].strip()), lines[1].strip()
  54. return None
  55. def _live_devtools(env, user, udd: str):
  56. """One exec: ``(port, path)`` of the Chromium on ``udd`` while its DevTools port answers; ``GONE`` when the
  57. sandbox says it is not running (a crashed Chromium leaves a stale ``DevToolsActivePort`` behind); None
  58. when the exec itself failed or timed out, which proves nothing either way."""
  59. from tools.environments import streams
  60. active = shlex.quote(f"{udd}/DevToolsActivePort")
  61. script = (f'[ -s {active} ] || exit {_GONE_RC}; p=$(head -n1 {active}); '
  62. f'( exec 3<>"/dev/tcp/127.0.0.1/$p" ) 2>/dev/null || exit {_GONE_RC}; cat {active}')
  63. try:
  64. proc = streams.run_in(env, ["bash", "-c", script], user=user, timeout=10)
  65. except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
  66. logger.debug("sandbox browser liveness probe failed: %s", exc)
  67. return None
  68. if proc.returncode == _GONE_RC:
  69. return GONE
  70. lines = proc.stdout.decode("utf-8", "replace").splitlines() if proc.returncode == 0 else []
  71. if len(lines) >= 2 and lines[0].strip().isdigit() and lines[1].startswith("/devtools/browser/"):
  72. return int(lines[0].strip()), lines[1].strip()
  73. logger.debug("sandbox browser liveness probe inconclusive (exit %s)", proc.returncode)
  74. return None
  75. def _shared_key(env, udd: str) -> str:
  76. """The launcher key every session on this container's Chromium shares. Keyed by the container, never by
  77. profile: profiles on one ``terminal.docker_shared_container_key`` share one browser, and each
  78. ``container_persistent: false`` session has its own. The env object is the container's identity in this
  79. process (its port forwards keep it alive, so the id is never reused while a session names it)."""
  80. return f"sandbox:{id(env):x}:{udd}"
  81. # ``$p``'s start time (clock ticks since boot): field 22 of /proc/<pid>/stat, counted after the ``(comm)``
  82. # field, which may itself hold spaces and parentheses. Empty when the sandbox has no /proc.
  83. _START_TIME_OF_P = 's=$(cat /proc/"$p"/stat 2>/dev/null) && s=${s##*")"} && set -- $s && printf %s "${20}"'
  84. # Exit code of the stop script when the pid is gone or no longer the browser this process launched.
  85. _NOT_OURS_RC = 4
  86. # Signal "$1" only while it is still the launched browser: same start time ("$2"), and its argv carries the
  87. # launch's exact --user-data-dir token ("$3"). Check and signal in one exec inside the sandbox.
  88. _STOP_SCRIPT = (
  89. f'p=$1; t=$({_START_TIME_OF_P}); [ -n "$t" ] && [ "$t" = "$2" ] || exit {_NOT_OURS_RC}; '
  90. f'ok=; while IFS= read -r -d "" a; do [ "$a" = "$3" ] && ok=1; done < /proc/"$p"/cmdline; '
  91. f'[ -n "$ok" ] || exit {_NOT_OURS_RC}; kill -TERM "$p"'
  92. )
  93. def _start_chromium(env, user, exe: str, udd: str, child_env: dict) -> tuple[int, Optional[str]]:
  94. """Launch detached in the sandbox; returns its sandbox pid and start time (None when unreadable)."""
  95. from tools.bot_desktop.browser import SHARED_PROFILE_ARGS
  96. from tools.browser_launcher import CHROMIUM_SANDBOX_BYPASS_ARGS, build_chromium_argv
  97. from tools.environments import streams
  98. # Container: no user namespaces for Chromium's own sandbox, so the bypass flags always apply (this
  99. # host's facts say nothing about the sandbox); the shared-profile flags match the dock's icon.
  100. argv = build_chromium_argv(exe, udd, packaged=False, headed=True, sandbox_bypass=False,
  101. extra_args=(*SHARED_PROFILE_ARGS, *CHROMIUM_SANDBOX_BYPASS_ARGS))
  102. argv = list(dict.fromkeys(argv))
  103. # setsid execs in place (a background child of a non-interactive shell leads no process group), so $! is
  104. # Chromium itself, and its start time is fixed from the fork on.
  105. script = (f"mkdir -p {shlex.quote(udd)} && rm -f {shlex.quote(udd)}/DevToolsActivePort && "
  106. f"(setsid {shlex.join(argv)} >/dev/null 2>&1 & p=$!; echo \"$p $({_START_TIME_OF_P})\")")
  107. proc = streams.run_in(env, ["bash", "-c", script], child_env=child_env, user=user, timeout=30)
  108. last = proc.stdout.decode("utf-8", "replace").strip().splitlines()[-1:] if proc.returncode == 0 else []
  109. words = last[0].split() if last else []
  110. if not words or not words[0].isdigit():
  111. raise RuntimeError(f"Chromium could not be started in the sandbox: {proc.stderr.decode('utf-8', 'replace')[-500:]}")
  112. start = words[1] if len(words) > 1 and words[1].isdigit() else None
  113. return int(words[0]), start
  114. def _wait_for_port(env, user, udd: str) -> tuple[int, str]:
  115. deadline = time.monotonic() + _LAUNCH_TIMEOUT_S
  116. while time.monotonic() < deadline:
  117. try:
  118. found = _read_devtools(env, user, udd)
  119. except subprocess.TimeoutExpired:
  120. found = None # one slow exec is another not-ready answer; the deadline decides readiness
  121. if found:
  122. return found
  123. time.sleep(0.5)
  124. raise RuntimeError("Chromium in the sandbox did not expose a DevTools endpoint in time")
  125. def _key_lock(key: str) -> threading.Lock:
  126. with _launched_lock:
  127. return _key_locks.setdefault(key, threading.Lock())
  128. def _spawn_owned(env, user, udd: str, key: str, published: dict) -> int:
  129. """Launch the sandbox Chromium and own it under ``key`` from the moment it exists, before its port is
  130. up, so stop_sandbox_browser can always stop it; returns its sandbox pid."""
  131. from tools.bot_desktop import sandbox_host
  132. from tools.environments import streams
  133. exe = sandbox_host.chromium_executable(env)
  134. if not exe:
  135. raise RuntimeError("the sandbox image has no Chromium to run the browser")
  136. child_env = {**published, "TMPDIR": streams.sandbox_temp_dir(env)}
  137. pid, start = _start_chromium(env, user, exe, udd, child_env)
  138. with _launched_lock:
  139. _launched[key] = (env, pid, start, udd)
  140. return pid
  141. def create_sandbox_session(cache_key: str) -> dict[str, Any]:
  142. """Session on the sandbox Chromium (attach to the running one, else launch it)."""
  143. from tools.bot_desktop import sandbox_host
  144. from tools.browser_session import _session_record
  145. from tools.environments import streams
  146. env, published = _sandbox()
  147. user = sandbox_host._user_for(env)
  148. udd = sandbox_host.browser_profile_dir(env)
  149. key = _shared_key(env, udd)
  150. with _key_lock(key):
  151. found = _live_devtools(env, user, udd)
  152. if found is None:
  153. # Launching blind would wipe a live browser's DevToolsActivePort and lose to its singleton.
  154. raise RuntimeError("the terminal backend's sandbox did not answer whether its browser is running; "
  155. "try again")
  156. spawned: Optional[int] = None
  157. try:
  158. if found is GONE:
  159. spawned = _spawn_owned(env, user, udd, key, published)
  160. found = _wait_for_port(env, user, udd)
  161. local_port = streams.forward_port(env, found[0], user=user)
  162. except BaseException:
  163. if spawned is not None: # a timeout, a failed forward or an interrupt must not leave it running unowned
  164. _stop_launched(key)
  165. raise
  166. info = _session_record("sb", f"ws://127.0.0.1:{local_port}{found[1]}", {"local": True, "sandbox": True})
  167. _confirm(info["cdp_url"], env, found[0])
  168. info.update(shared_profile=True, launcher_key=key)
  169. logger.info("Created sandbox browser session %s for %s", info["session_name"], cache_key)
  170. return info
  171. def _confirm(cdp_url: str, env, port: int) -> None:
  172. from tools.environments import streams
  173. _confirmed[cdp_url] = (id(env), port, streams.forward_failures(env, port), time.monotonic())
  174. def sandbox_browser_alive(session_info: dict[str, Any]) -> bool:
  175. """The sandbox Chromium still serves this session's endpoint through the current port forward: False once
  176. the sandbox answers that it died, was relaunched (new DevTools path) or the sandbox was replaced (new
  177. forward). Re-probed only after a connection through the forward failed or the last confirmation expired;
  178. a probe that cannot reach the sandbox keeps the session."""
  179. from tools.bot_desktop import runtime as _bd_runtime, sandbox_host
  180. from tools.environments import streams
  181. env = _bd_runtime._sandbox_env(create=False)
  182. if env is None:
  183. return False
  184. cdp_url = session_info.get("cdp_url") or ""
  185. confirmed = _confirmed.get(cdp_url)
  186. if (confirmed and confirmed[0] == id(env) and streams.forward_failures(env, confirmed[1]) == confirmed[2]
  187. and time.monotonic() - confirmed[3] < _CONFIRMED_TTL_S):
  188. return True
  189. try:
  190. user = sandbox_host._user_for(env)
  191. udd = sandbox_host.browser_profile_dir(env)
  192. except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
  193. logger.debug("sandbox browser liveness probe failed: %s", exc)
  194. return True
  195. found = _live_devtools(env, user, udd)
  196. if found is None:
  197. return True
  198. if found is GONE or cdp_url != f"ws://127.0.0.1:{streams.forward_port(env, found[0], user=user)}{found[1]}":
  199. _confirmed.pop(cdp_url, None)
  200. return False
  201. _confirm(cdp_url, env, found[0])
  202. return True
  203. def stop_sandbox_browser(session_info: dict[str, Any]) -> None:
  204. """SIGTERM the sandbox Chromium this process launched on the session's shared key (best effort; the
  205. sandbox's lifetime bounds it). The caller guarantees no other live session uses it."""
  206. _confirmed.pop(session_info.get("cdp_url") or "", None)
  207. key = session_info.get("launcher_key") or ""
  208. with _key_lock(key):
  209. _stop_launched(key)
  210. def _stop_launched(key: str) -> None:
  211. with _launched_lock:
  212. launched = _launched.pop(key, None)
  213. if launched is None:
  214. return
  215. from tools.bot_desktop import sandbox_host
  216. from tools.environments import streams
  217. env, pid, start, udd = launched
  218. if start is None:
  219. logger.warning("browser: leaving sandbox Chromium pid %s running: its start time could not be read at "
  220. "launch, so the pid cannot be proven to still be that browser", pid)
  221. return
  222. argv = ["bash", "-c", _STOP_SCRIPT, "hermes-sandbox-browser-stop", str(int(pid)), start, f"--user-data-dir={udd}"]
  223. try:
  224. proc = streams.run_in(env, argv, user=sandbox_host._user_for(env), timeout=15)
  225. except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
  226. logger.debug("sandbox browser stop for pid %s failed: %s", pid, exc)
  227. return
  228. if proc.returncode == _NOT_OURS_RC:
  229. logger.info("browser: not signalling sandbox pid %s: it is gone or no longer the Chromium launched on %s",
  230. pid, udd)
  231. elif proc.returncode != 0:
  232. logger.debug("sandbox browser stop for pid %s exited %s", pid, proc.returncode)