browser_launcher.py 54 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146
  1. """Launch, track and stop the Chromium processes ``browser_exec`` drives.
  2. One browser per launcher key (the session layer's cache key, or ``profile:<dir>`` for a persistent
  3. profile shared by every key). Chromium picks its own DevTools port (``--remote-debugging-port=0``)
  4. and writes it to ``<user-data-dir>/DevToolsActivePort``; the CDP URL comes from that file, so there
  5. is no port race. Every owned launch leaves a record in a private (0700) launcher state dir so a later
  6. Hermes process can reap the Chromium of an owner that crashed, after proving the pid is still that browser:
  7. a throwaway profile's in its home (``<HERMES_HOME>/cache/browser/chromium-records``), a persistent profile's
  8. in that profile's own dir under :func:`_shared_profiles_root` (per OS user: every home pinning the dir shares
  9. it). A shared-profile Chromium other Hermes processes attached to is stopped only by its last holder
  10. (``tools/browser_launcher_holders.py``).
  11. Stop is always parent-first (``ProcessRegistry._terminate_host_pid``): Chromium reaps its own
  12. zygotes and renderers on SIGTERM, and killing a zygote first turns a clean exit into a crash dump.
  13. """
  14. from __future__ import annotations
  15. import contextlib
  16. import hashlib
  17. import json
  18. import logging
  19. import os
  20. import re
  21. import shutil
  22. import socket
  23. import stat
  24. import subprocess
  25. import sys
  26. import threading
  27. import time
  28. from dataclasses import dataclass
  29. from pathlib import Path
  30. from typing import Any, Callable, Optional
  31. logger = logging.getLogger(__name__)
  32. # THE Chromium startup flags for a host where its sandbox cannot work. The Bot Desktop dock's Browser
  33. # icon (same binary, same profile) uses the same list (``tools.bot_desktop.browser.dock_argv``), or the
  34. # human's click dies while the agent's launch works.
  35. CHROMIUM_SANDBOX_BYPASS_ARGS: tuple[str, ...] = ("--no-sandbox", "--disable-dev-shm-usage")
  36. # Base flags, each kept for a reason (the minimum for an unattended, disposable automation browser):
  37. # --no-first-run / --no-default-browser-check no welcome tab or default-browser bar on a fresh profile
  38. # --disable-background-networking no field-trial / safe-browsing / update pings per launch
  39. # --disable-component-update no component downloads into a throwaway profile
  40. # --disable-default-apps / --disable-sync nothing installed or signed in behind the agent's back
  41. # --disable-features=Translate no translate bubble covering the page
  42. _BASE_FLAGS: tuple[str, ...] = (
  43. "--no-first-run", "--no-default-browser-check", "--disable-background-networking",
  44. "--disable-component-update", "--disable-default-apps", "--disable-sync", "--disable-features=Translate",
  45. )
  46. # Only for the launcher-resolved packaged Chromium: a keyring unlock prompt (Linux desktops) or the macOS
  47. # keychain dialog would block an unattended launch. Never for a real or shared profile, whose cookies are
  48. # encrypted with the OS store (``--password-store=basic`` would sign the copy out).
  49. _PACKAGED_ONLY_FLAGS: tuple[str, ...] = ("--password-store=basic",)
  50. _MACOS_PACKAGED_ONLY_FLAGS: tuple[str, ...] = ("--use-mock-keychain",)
  51. _DEVTOOLS_FILE = "DevToolsActivePort"
  52. _LAUNCH_LOG = "hermes-chromium.log"
  53. _MIN_FIRST_LAUNCH_TIMEOUT_S = 120
  54. _UNRESPONSIVE_WAIT_S = 10 # how long a live browser whose endpoint went silent (paused, swapped out) is waited for
  55. _POLL_INTERVAL_S = 0.1
  56. _STDERR_TAIL_CHARS = 1500
  57. @dataclass
  58. class LaunchedBrowser:
  59. """A Chromium this process launched (``owned``) or attached to (a running instance on a shared
  60. profile, never killed by Hermes)."""
  61. key: str
  62. pid: Optional[int]
  63. start_time: Optional[int]
  64. cdp_url: str
  65. http_url: str
  66. user_data_dir: str
  67. owned: bool
  68. ephemeral: bool
  69. home_key: str
  70. # File name of the launch record its leases are keyed by, when that is not this process's own name for
  71. # it: an attached browser keeps its launcher's record, whichever PID namespace wrote it.
  72. record_name: Optional[str] = None
  73. # Held from an environment this process cannot verify (another PID namespace: a container's browser seen
  74. # from the host, or the reverse): ``pid`` is that namespace's number, names nobody here and is never
  75. # signalled; liveness is its DevTools endpoint.
  76. foreign: bool = False
  77. _browsers: dict[str, LaunchedBrowser] = {}
  78. _procs: dict[str, subprocess.Popen] = {}
  79. _launching: dict[int, LaunchedBrowser] = {} # pid -> a spawned Chromium still in its readiness wait
  80. _key_locks: dict[str, threading.Lock] = {}
  81. _lock = threading.RLock()
  82. _autoinstall_attempted = False
  83. # Profile homes (``hermes_home_key``) this process launched or attached a browser for: the reaper reads
  84. # each one's records, since one process serves many profiles.
  85. _served_homes: set[str] = set()
  86. # ── Host facts ───────────────────────────────────────────────────────────────────────────────────────
  87. def apparmor_restricts_unprivileged_userns() -> bool:
  88. """Ubuntu 23.10+ default: unprivileged user namespaces are denied, so a Chromium whose
  89. ``chrome_sandbox`` helper is not setuid (Playwright's bundle) dies with 'No usable sandbox'."""
  90. try:
  91. with open("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", encoding="utf-8") as f:
  92. return f.read().strip() == "1"
  93. except OSError:
  94. return False
  95. def running_in_docker() -> bool:
  96. """Best-effort detection of whether we're inside a Docker container."""
  97. if os.path.exists("/.dockerenv"):
  98. return True
  99. try:
  100. with open("/proc/1/cgroup", "rt", encoding="utf-8") as fp:
  101. return "docker" in fp.read()
  102. except OSError:
  103. return False
  104. def needs_chromium_sandbox_bypass() -> bool:
  105. """True when Chromium needs --no-sandbox to start reliably (root, Docker, AppArmor userns)."""
  106. if hasattr(os, "geteuid") and os.geteuid() == 0:
  107. return True
  108. if running_in_docker():
  109. return True
  110. return apparmor_restricts_unprivileged_userns()
  111. def _chromium_missing_hint() -> str:
  112. from hermes_cli.browser_runtime import chromium_missing_hint
  113. return chromium_missing_hint()
  114. def _resolve_executable() -> Optional[str]:
  115. from hermes_cli.browser_runtime import chromium_executable
  116. exe = chromium_executable()
  117. if not exe:
  118. return None
  119. if os.path.isfile(exe):
  120. return exe
  121. from hermes_platform.resolver import locate_command
  122. found = locate_command(exe).command # a bare command name pinned in browser.executable_path
  123. return found[0] if found else None
  124. def _explicit_executable_configured(chromium_executable) -> bool:
  125. """An operator-pinned browser binary (even a missing one) is their choice: never download over it."""
  126. return chromium_executable(allow_override=True) != chromium_executable(allow_override=False)
  127. def _maybe_autoinstall_chromium() -> bool:
  128. """One-shot ``pm.ensure("chromium")`` (a failed ~170 MB download must not retry per call). Docker
  129. supplies the binary, Termux has no PM Chromium, and an explicit executable is the user's choice."""
  130. global _autoinstall_attempted
  131. if _autoinstall_attempted:
  132. return False
  133. _autoinstall_attempted = True
  134. from hermes_cli.browser_runtime import chromium_executable
  135. from hermes_constants import is_termux
  136. if running_in_docker() or is_termux() or _explicit_executable_configured(chromium_executable):
  137. return False
  138. from pm import InstallError, ensure, lazy_installs_allowed
  139. if not lazy_installs_allowed():
  140. return False
  141. logger.info("browser: installing PM's pinned Chromium")
  142. try:
  143. ensure("chromium")
  144. except (InstallError, OSError) as exc:
  145. logger.warning("browser: Chromium auto-install failed: %s", exc)
  146. return False
  147. return True
  148. def chromium_ready() -> tuple[bool, str]:
  149. """``(True, "")`` when a Chromium is on disk (after at most one PM auto-install per process), else
  150. ``(False, actionable hint)``."""
  151. if _resolve_executable():
  152. return True, ""
  153. if _maybe_autoinstall_chromium() and _resolve_executable():
  154. return True, ""
  155. return False, _chromium_missing_hint()
  156. # ── Launch ───────────────────────────────────────────────────────────────────────────────────────────
  157. def _default_user_data_dir(key: str, home_key: str) -> str:
  158. """Per process too: two Hermes processes on one profile use the same keys (``browser-exec-default``,
  159. named sessions), and each launch wipes its throwaway dir first. The name carries the owner's pid and
  160. start time: a process that dies before its launch record exists leaves the dir to the reaper
  161. (:func:`_reclaim_throwaway_dirs`)."""
  162. from tools.browser_launcher_holders import ns_tag
  163. digest = hashlib.sha256(f"{home_key}\0{key}".encode()).hexdigest()[:16]
  164. pid = os.getpid()
  165. started = _start_time(pid)
  166. owner = f"{pid}-{started}" if started is not None else str(pid)
  167. return str(_ephemeral_root(home_key) / f"{digest}-{owner}{ns_tag()}")
  168. def _launch_timeout(timeout: Optional[float]) -> float:
  169. if timeout is not None:
  170. return float(timeout)
  171. from tools.browser_session_cloud import command_timeout
  172. return float(max(command_timeout(), _MIN_FIRST_LAUNCH_TIMEOUT_S))
  173. def _has_display(env: dict) -> bool:
  174. return bool(env.get("DISPLAY") or env.get("WAYLAND_DISPLAY"))
  175. def _resolve_headed(headed: Optional[bool], env: dict) -> bool:
  176. """Headless unless ``browser.headed``; a display-less Linux host stays headless (a headed launch
  177. would die there). Windows obeys the same rule: headless unless the user opted in (#64867)."""
  178. if headed is None:
  179. from tools.browser_session_cloud import is_headed_mode
  180. headed = is_headed_mode()
  181. if not headed:
  182. return False
  183. return _has_display(env) or not sys.platform.startswith("linux")
  184. def _configured_chromium_args() -> list[str]:
  185. from tools.browser_session_cloud import chromium_args
  186. return chromium_args()
  187. def build_chromium_argv(executable: str, user_data_dir: str, *, packaged: bool, headed: bool,
  188. sandbox_bypass: bool, extra_args=()) -> list[str]:
  189. """The launch argv (pure in its inputs + host sandbox facts + ``browser.chromium_args``).
  190. ``sandbox_bypass`` lets the host policy (root, Docker, AppArmor userns) add
  191. ``CHROMIUM_SANDBOX_BYPASS_ARGS``; it is False for the user's own browser binary, which is never
  192. started without its sandbox."""
  193. argv = [executable, "--remote-debugging-port=0", f"--user-data-dir={user_data_dir}", *_BASE_FLAGS]
  194. if packaged:
  195. argv += _PACKAGED_ONLY_FLAGS
  196. if sys.platform == "darwin":
  197. argv += _MACOS_PACKAGED_ONLY_FLAGS
  198. if not headed:
  199. argv.append("--headless=new")
  200. configured = _configured_chromium_args()
  201. if sandbox_bypass and needs_chromium_sandbox_bypass():
  202. argv += [flag for flag in CHROMIUM_SANDBOX_BYPASS_ARGS if flag not in configured]
  203. argv += [str(a) for a in extra_args]
  204. argv += configured
  205. argv.append("about:blank")
  206. return argv
  207. def _spawn_kwargs() -> dict[str, Any]:
  208. if os.name == "nt":
  209. from hermes_cli._subprocess_compat import windows_hide_flags
  210. return {"creationflags": windows_hide_flags() | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0)}
  211. # detached-spawn: ok — a shared-profile Chromium outlives its launcher for its other holders (browser_launcher_holders); a crashed owner's throwaway one is reaped from its launch record by the next Hermes
  212. return {"start_new_session": True}
  213. def _read_devtools_file(user_data_dir: str) -> Optional[tuple[int, str]]:
  214. """``(port, "/devtools/browser/<id>")`` once Chromium wrote both lines, else None."""
  215. try:
  216. with open(os.path.join(user_data_dir, _DEVTOOLS_FILE), encoding="utf-8-sig") as fh:
  217. port, path = fh.readline().strip(), fh.readline().strip()
  218. except OSError:
  219. return None
  220. if not port.isdigit() or not path.startswith("/devtools/browser/"):
  221. return None
  222. return int(port), path
  223. def _log_tail(user_data_dir: str) -> str:
  224. try:
  225. raw = Path(user_data_dir, _LAUNCH_LOG).read_bytes()[-_STDERR_TAIL_CHARS:]
  226. except OSError:
  227. return ""
  228. return raw.decode("utf-8", errors="replace").strip()
  229. def _wait_for_devtools(proc: subprocess.Popen, user_data_dir: str, deadline: float) -> tuple[Optional[tuple[int, str]], Optional[str]]:
  230. while True:
  231. found = _read_devtools_file(user_data_dir)
  232. if found:
  233. return found, None
  234. rc = proc.poll()
  235. if rc is not None:
  236. tail = _log_tail(user_data_dir)
  237. hint = (" Chromium's sandbox could not start; add --no-sandbox to browser.chromium_args."
  238. if "sandbox" in tail.lower() else "")
  239. return None, f"Chromium exited with code {rc} during startup.{hint}" + (f"\n{tail}" if tail else "")
  240. if time.monotonic() >= deadline:
  241. return None, "Chromium did not expose a DevTools endpoint in time."
  242. time.sleep(_POLL_INTERVAL_S)
  243. def _singleton_pid(user_data_dir: str) -> Optional[int]:
  244. """Pid of the Chromium holding ``user_data_dir``: the ``SingletonLock`` symlink (POSIX), else (Windows
  245. has no symlink, only a locked ``lockfile``) the browser process whose argv names the dir."""
  246. owner = _singleton_owner(user_data_dir)
  247. if owner is None:
  248. return _pid_naming_dir(user_data_dir) if os.name == "nt" else None
  249. return owner[1]
  250. def _singleton_owner(user_data_dir: str) -> Optional[tuple[str, int]]:
  251. """``(hostname, pid)`` Chromium wrote into ``user_data_dir``'s ``SingletonLock`` symlink (POSIX), else None."""
  252. try:
  253. target = os.readlink(os.path.join(user_data_dir, "SingletonLock"))
  254. except OSError:
  255. return None
  256. host, _, pid_text = target.rpartition("-")
  257. return (host, int(pid_text)) if pid_text.isdigit() else None
  258. def _foreign_holder(user_data_dir: str) -> Optional[int]:
  259. """The ``SingletonLock`` pid of a Chromium on ``user_data_dir`` that runs in an environment this process cannot
  260. verify, else None. Foreign: the lock is not a live process here whose argv names the dir, and it was written
  261. under another hostname (another host or container; Chromium too refuses such a lock) or names the browser
  262. of a launch record from another PID namespace. A lock of this hostname with no such record is stale (this
  263. environment's crashed browser): Chromium itself takes it over, and so does a launch."""
  264. owner = _singleton_owner(user_data_dir)
  265. if owner is None:
  266. return None
  267. host, pid = owner
  268. if _pid_alive(pid) and _cmdline_names_dir(pid, user_data_dir):
  269. return None
  270. if host != socket.gethostname() or _foreign_record_names(user_data_dir, pid):
  271. return pid
  272. return None
  273. def _foreign_record_names(user_data_dir: str, pid: int) -> bool:
  274. """A launch record from another PID namespace names ``pid`` as shared profile ``user_data_dir``'s browser: the one
  275. serving the profile's DevTools endpoint now, or one not yet up there (its record has no endpoint). An earlier
  276. launch there that reached another endpoint is not that browser, whatever pid number it had."""
  277. from tools.browser_launcher_holders import same_namespace
  278. state = _shared_state_dir(user_data_dir)
  279. if state is None:
  280. return False
  281. found = _read_devtools_file(user_data_dir)
  282. for name in _record_names(state, user_data_dir, pid):
  283. record = _load_record(state / name)
  284. if (record is not None and record.get("pid") == pid and not same_namespace(record)
  285. and _same_profile(record.get("user_data_dir"), user_data_dir)
  286. and record.get("devtools") in (None, found and found[1])):
  287. return True
  288. return False
  289. def _record_names(records: Path, user_data_dir: str, pid) -> list[str]:
  290. """Names of the launch records in ``records`` for profile ``user_data_dir``'s browser ``pid``, whichever PID
  291. namespace wrote them (each carries its writer's ``ns_tag``), sorted."""
  292. stem = f"{_profile_digest(user_data_dir)[:16]}-{pid}"
  293. pattern = re.compile(rf"^{re.escape(stem)}(-n[0-9a-f]{{12}})?\.json$")
  294. return sorted(p.name for p in records.glob(f"{stem}*.json") if pattern.match(p.name))
  295. def _pid_naming_dir(user_data_dir: str) -> Optional[int]:
  296. """The browser (not a ``--type=`` helper) process whose argv carries ``--user-data-dir=<dir>``."""
  297. import psutil
  298. for proc in psutil.process_iter(["pid"]):
  299. try:
  300. argv = proc.cmdline()
  301. except (psutil.Error, OSError):
  302. continue
  303. if not any(tok.startswith("--type=") for tok in argv) and _cmdline_names_dir(proc.pid, user_data_dir):
  304. return proc.pid
  305. return None
  306. def _browser_running_on(user_data_dir: str) -> bool:
  307. """A live Chromium still uses ``user_data_dir`` (by argv: a ``SingletonLock`` outlives a crashed browser)."""
  308. return _pid_naming_dir(user_data_dir) is not None
  309. def _devtools_endpoint_live(port: int, path: str) -> bool:
  310. """The port answers ``/json/version`` with the browser id the DevToolsActivePort file names (the file
  311. outlives a crashed Chromium and its port can be recycled by another CDP server)."""
  312. try:
  313. import requests
  314. from agent.proxy_bypass import loopback_request_kwargs
  315. url = f"http://127.0.0.1:{port}/json/version"
  316. ws = str(requests.get(url, timeout=2, **loopback_request_kwargs(url)).json().get("webSocketDebuggerUrl") or "")
  317. except Exception: # health: allow BLE001 -- any probe failure means "not a live browser on this port"
  318. return False
  319. return ws.endswith(path)
  320. def running_instance(user_data_dir: str) -> Optional[tuple[int, str, int]]:
  321. """``(port, browser_path, pid)`` of a Chromium already running on ``user_data_dir`` (a human-started
  322. Bot Desktop dock instance, a surviving real-profile browser), else None. A second launch on that dir
  323. would be forwarded into it by Chromium's singleton and exit without a DevTools endpoint."""
  324. pid = _singleton_pid(user_data_dir)
  325. found = _read_devtools_file(user_data_dir)
  326. if pid is None or found is None or not _pid_alive(pid):
  327. return None
  328. port, path = found
  329. return (port, path, pid) if _devtools_endpoint_live(port, path) else None
  330. def _pid_alive(pid: int) -> bool:
  331. from gateway.status import _pid_exists
  332. return pid > 0 and _pid_exists(pid)
  333. def _start_time(pid: int) -> Optional[int]:
  334. from gateway.status import get_process_start_time
  335. return get_process_start_time(pid)
  336. def _make_browser(key: str, port: int, path: str, *, pid, start_time, user_data_dir: str, owned: bool,
  337. ephemeral: bool, home_key: str) -> LaunchedBrowser:
  338. return LaunchedBrowser(key=key, pid=pid, start_time=start_time, cdp_url=f"ws://127.0.0.1:{port}{path}",
  339. http_url=f"http://127.0.0.1:{port}", user_data_dir=user_data_dir, owned=owned,
  340. ephemeral=ephemeral, home_key=home_key)
  341. def _prepare_dir(user_data_dir: str, ephemeral: bool) -> None:
  342. if ephemeral:
  343. Path(user_data_dir).parent.mkdir(mode=0o700, parents=True, exist_ok=True) # the private ephemeral root
  344. shutil.rmtree(user_data_dir, ignore_errors=True) # fresh profile per launch
  345. os.makedirs(user_data_dir, mode=0o700, exist_ok=True)
  346. try:
  347. os.unlink(os.path.join(user_data_dir, _DEVTOOLS_FILE)) # a stale port confuses the readiness poll
  348. except OSError:
  349. pass
  350. def _spawn(argv: list[str], env: dict, user_data_dir: str) -> subprocess.Popen:
  351. log_path = os.path.join(user_data_dir, _LAUNCH_LOG)
  352. with open(log_path, "wb") as log:
  353. return subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=log, stderr=log, env=env,
  354. **_spawn_kwargs())
  355. def _attach_running(key: str, user_data_dir: str, home_key: str) -> Optional[LaunchedBrowser]:
  356. """The live Chromium already on shared profile ``user_data_dir``, leased (caller holds its lock)."""
  357. running = running_instance(user_data_dir)
  358. if running is None:
  359. return None
  360. port, path, pid = running
  361. logger.info("browser: attaching to the Chromium already running on %s (pid %s)", user_data_dir, pid)
  362. browser = _make_browser(key, port, path, pid=pid, start_time=_start_time(pid), user_data_dir=user_data_dir,
  363. owned=False, ephemeral=False, home_key=home_key)
  364. records = _state_dir(browser)
  365. if records is not None:
  366. from tools import browser_launcher_holders as holders
  367. _bind_launch_record(records, browser)
  368. _adopt_record(browser)
  369. holders.write_lease(_record_path(browser), browser) # its launcher's exit must not stop it under us
  370. return browser
  371. def _bind_launch_record(records: Path, browser: LaunchedBrowser) -> None:
  372. """Lease ``browser`` under the launch record that describes THIS launch of it (:func:`_record_is`), written in
  373. ANY PID namespace: its launcher (a container gateway a host CLI attached to) finds its holders only there.
  374. Records of earlier containers and boots are kept, so the pid number alone can name another browser's record.
  375. This namespace's record wins; with none (a browser a human started) the lease goes under this namespace's
  376. name. A browser of another environment takes its record's start time: its full identity from then on."""
  377. own = _record_name(browser)
  378. names = sorted(_record_names(records, browser.user_data_dir, browser.pid), key=lambda name: name != own)
  379. for name in names:
  380. record = _load_record(records / name)
  381. if record is not None and _record_is(record, browser):
  382. browser.record_name = name
  383. if browser.foreign:
  384. browser.start_time = record.get("start_time")
  385. return
  386. def _attach_foreign(key: str, user_data_dir: str, home_key: str, pid: int) -> Optional[LaunchedBrowser]:
  387. """The Chromium another environment holds on ``user_data_dir``, leased, when its DevTools endpoint answers here
  388. (caller holds the profile's lock); None when it does not. Never adopted: its record is not ours to take."""
  389. found = _read_devtools_file(user_data_dir)
  390. if found is None or not _devtools_endpoint_live(*found):
  391. return None
  392. logger.info("browser: attaching to the Chromium another environment runs on %s (its pid %s there)",
  393. user_data_dir, pid)
  394. browser = _make_browser(key, found[0], found[1], pid=pid, start_time=None, user_data_dir=user_data_dir,
  395. owned=False, ephemeral=False, home_key=home_key)
  396. browser.foreign = True
  397. records = _state_dir(browser)
  398. if records is not None:
  399. from tools import browser_launcher_holders as holders
  400. _bind_launch_record(records, browser)
  401. holders.write_lease(_record_path(browser), browser) # its launcher's exit must not stop it under us
  402. return browser
  403. def _live_local_holder(user_data_dir: str) -> Optional[int]:
  404. """Pid of the live Chromium of this environment holding ``user_data_dir`` (its ``SingletonLock`` names a live
  405. process whose argv carries the dir), else None: no browser, or a stale lock a launch may take over."""
  406. pid = _singleton_pid(user_data_dir)
  407. return pid if pid is not None and _pid_alive(pid) and _cmdline_names_dir(pid, user_data_dir) else None
  408. def _attach_local(key: str, user_data_dir: str, home_key: str,
  409. timeout: Optional[float]) -> tuple[Optional[LaunchedBrowser], Optional[str]]:
  410. """Attach to this environment's live Chromium on ``user_data_dir``; ``(None, None)`` when none holds it (a
  411. launch may start one). A live holder whose endpoint does not answer (paused, hung) is waited for, then
  412. refused: a launch over it would only wipe its ``DevToolsActivePort`` and lose it for every later attach."""
  413. attached = _attach_running(key, user_data_dir, home_key)
  414. pid = _live_local_holder(user_data_dir) if attached is None else None
  415. if pid is None:
  416. return attached, None
  417. deadline = time.monotonic() + min(_UNRESPONSIVE_WAIT_S, _launch_timeout(timeout))
  418. delay = 0.25
  419. while attached is None and _live_local_holder(user_data_dir) == pid:
  420. if time.monotonic() >= deadline:
  421. return None, (f"The browser profile {user_data_dir} is in use by Chromium pid {pid}, which is not "
  422. "responding on its DevTools endpoint (paused, hung, or started without remote debugging). "
  423. "Hermes will not start a second browser on it; try again once it responds, or stop it.")
  424. time.sleep(min(delay, max(0.0, deadline - time.monotonic())))
  425. delay *= 2
  426. attached = _attach_running(key, user_data_dir, home_key)
  427. return attached, None
  428. def _foreign_unreachable(user_data_dir: str) -> str:
  429. return (f"The browser profile {user_data_dir} is in use by a Chromium in another environment (a container or "
  430. "the host) that this process can neither reach nor verify, so Hermes will not start a second browser "
  431. "on it. Stop that browser, or set browser.user_data_dir to another directory for this environment; if "
  432. f"no browser runs there any more, delete {os.path.join(user_data_dir, 'SingletonLock')}.")
  433. def _start_shared(key: str, executable: str, udd: str, home_key: str, prepare,
  434. **launch) -> tuple[Optional[LaunchedBrowser], Optional[str]]:
  435. """Attach to, or launch, shared profile ``udd``'s one Chromium as ONE step under the profile's cross-process
  436. lock: of two processes starting it at once the later waits, then attaches (a second launch would wipe the
  437. first's DevToolsActivePort and lose Chromium's singleton), and an owner's stop cannot fall between the live
  438. probe and the lease. A first launch holds the lock for its whole readiness wait; the reaper skips a busy
  439. profile instead of waiting (:func:`_reap_shared_profiles`). A browser another environment holds is attached
  440. through its endpoint, or refused; never launched over."""
  441. from tools import browser_launcher_holders as holders
  442. state = _shared_state_dir(udd)
  443. with holders.records_lock(state) if state is not None else contextlib.nullcontext():
  444. foreign = _foreign_holder(udd)
  445. if foreign is not None:
  446. attached = _attach_foreign(key, udd, home_key, foreign)
  447. if attached is None:
  448. return None, _foreign_unreachable(udd)
  449. else:
  450. attached, refused = _attach_local(key, udd, home_key, launch.get("timeout"))
  451. if refused:
  452. return None, refused
  453. if attached is None:
  454. failure = prepare() if prepare is not None else None
  455. if failure:
  456. return None, failure
  457. return _launch_owned(key, executable, udd, ephemeral=False, home_key=home_key, **launch)
  458. with _lock: # registered under the lock too: another key's stop must see this key as its heir
  459. _browsers[key] = attached
  460. return attached, None
  461. def launch_chromium(key: str, *, executable: Optional[str] = None, user_data_dir: Optional[str] = None,
  462. ephemeral: bool = True, headed: Optional[bool] = None, extra_args=(),
  463. env: Optional[dict] = None, timeout: Optional[float] = None, sandbox_bypass: bool = True,
  464. prepare: Optional[Callable[[], Optional[str]]] = None,
  465. ) -> tuple[Optional[LaunchedBrowser], Optional[str]]:
  466. """Start (or reuse) the Chromium for ``key``; ``(browser, None)`` or ``(None, error)``.
  467. ``executable=None`` resolves the packaged Chromium (``chromium_ready``); an explicit executable (the
  468. Bot Desktop's, the user's real browser) is used as given. ``user_data_dir=None`` is a fresh
  469. ephemeral profile under ``<HERMES_HOME>/cache/browser/chromium/``; a persistent dir that already has
  470. a live Chromium on it is ATTACHED (``owned=False``, never killed). ``sandbox_bypass=False`` (the
  471. user's real browser) never gets the host's sandbox-bypass flags. ``prepare`` (a persistent dir only) runs
  472. right before a fresh launch, inside the same cross-process step, never when attaching: a profile copy
  473. written there can neither race another process's copy nor land under a browser that just came up; a
  474. message it returns is the launch's error, as is."""
  475. from hermes_constants import hermes_home_key
  476. # Loaded now, not first at teardown: stopping the browser from an atexit hook must not be this module's
  477. # first import (Python refuses new atexit registrations during shutdown, orphaning the browser). An
  478. # attacher needs it too: it is the one that stops a shared browser when it is the last holder.
  479. import tools.process_registry
  480. with _key_lock(key): # one launch per key; other keys launch concurrently
  481. existing = get_browser(key)
  482. if existing is not None:
  483. return existing, None
  484. home_key = hermes_home_key()
  485. with _lock:
  486. _served_homes.add(home_key)
  487. packaged = executable is None
  488. if packaged:
  489. ok, hint = chromium_ready()
  490. if not ok:
  491. return None, hint
  492. executable = _resolve_executable()
  493. udd = user_data_dir or _default_user_data_dir(key, home_key)
  494. launch = dict(packaged=packaged, headed=headed, extra_args=extra_args, env=env, timeout=timeout,
  495. sandbox_bypass=sandbox_bypass)
  496. if ephemeral: # a per-process dir: nobody else starts or attaches to it
  497. return _launch_owned(key, executable, udd, ephemeral=True, home_key=home_key, **launch)
  498. return _start_shared(key, executable, udd, home_key, prepare, **launch)
  499. def _key_lock(key: str) -> threading.Lock:
  500. with _lock:
  501. return _key_locks.setdefault(key, threading.Lock())
  502. def _launch_owned(key: str, executable: str, udd: str, *, ephemeral: bool, packaged: bool, headed,
  503. extra_args, env, timeout, home_key: str, sandbox_bypass: bool,
  504. ) -> tuple[Optional[LaunchedBrowser], Optional[str]]:
  505. if env is None:
  506. from tools.browser_session import build_browser_env
  507. env = build_browser_env()
  508. try:
  509. _prepare_dir(udd, ephemeral)
  510. except OSError as exc:
  511. return None, f"Could not create the browser profile directory {udd}: {exc}"
  512. argv = build_chromium_argv(executable, udd, packaged=packaged, headed=_resolve_headed(headed, env),
  513. sandbox_bypass=sandbox_bypass, extra_args=extra_args)
  514. try:
  515. proc = _spawn(argv, env, udd)
  516. except (OSError, subprocess.SubprocessError) as exc:
  517. return None, f"Failed to launch Chromium ({executable}): {exc}"
  518. start_time = _start_time(proc.pid)
  519. starting = _make_browser(key, 0, "", pid=proc.pid, start_time=start_time, user_data_dir=udd, owned=True,
  520. ephemeral=ephemeral, home_key=home_key)
  521. # Recorded before the readiness wait (a first launch may take 120 s): if this process dies in it, the
  522. # record is what lets the next reaper stop this Chromium. Tracked as launching so this process's own
  523. # reaper leaves it alone meanwhile.
  524. with _lock:
  525. _launching[proc.pid] = starting
  526. _write_record(starting)
  527. try:
  528. found, err = _wait_for_devtools(proc, udd, time.monotonic() + _launch_timeout(timeout))
  529. except BaseException: # interrupted (Ctrl-C, shutdown): nobody else knows this browser exists
  530. _abandon_launch(proc, starting)
  531. raise
  532. if found is None:
  533. _abandon_launch(proc, starting)
  534. return None, err
  535. browser = _make_browser(key, found[0], found[1], pid=proc.pid, start_time=start_time, user_data_dir=udd,
  536. owned=True, ephemeral=ephemeral, home_key=home_key)
  537. _write_record(browser) # now with its DevTools browser id: how another environment tells this launch apart
  538. with _lock:
  539. _launching.pop(proc.pid, None)
  540. _browsers[key] = browser
  541. _procs[key] = proc
  542. logger.info("browser: launched Chromium pid %s for %s (port %s)", proc.pid, key, found[0])
  543. return browser, None
  544. def _abandon_launch(proc: subprocess.Popen, starting: LaunchedBrowser) -> None:
  545. with _lock:
  546. _launching.pop(proc.pid, None)
  547. if proc.poll() is None: # one that already exited is reaped by poll(); its pid is no longer ours
  548. _terminate(proc.pid, starting.start_time, proc)
  549. _remove_record(starting)
  550. if starting.ephemeral:
  551. shutil.rmtree(starting.user_data_dir, ignore_errors=True)
  552. # ── Lookup / stop ────────────────────────────────────────────────────────────────────────────────────
  553. def _alive(browser: LaunchedBrowser) -> bool:
  554. proc = _procs.get(browser.key)
  555. if proc is not None and browser.owned:
  556. return proc.poll() is None
  557. if browser.pid is None:
  558. return False
  559. if not _pid_alive(browser.pid):
  560. return False
  561. return browser.start_time is None or _start_time(browser.pid) == browser.start_time
  562. def get_browser(key: str) -> Optional[LaunchedBrowser]:
  563. """The live browser for ``key``, or None (a dead one is dropped and its leftovers removed)."""
  564. with _lock:
  565. browser = _browsers.get(key)
  566. if browser is None:
  567. return None
  568. if not browser.foreign:
  569. if _alive(browser):
  570. return browser
  571. logger.info("browser: Chromium for %s (pid %s) is gone", key, browser.pid)
  572. _forget(key, browser)
  573. return None
  574. if _foreign_alive(browser): # an HTTP probe: never under the process-wide lock
  575. return browser
  576. with _lock:
  577. if _browsers.get(key) is browser:
  578. logger.info("browser: the Chromium another environment ran for %s is gone", key)
  579. _forget(key, browser)
  580. return None
  581. def _foreign_alive(browser: LaunchedBrowser) -> bool:
  582. """A browser another environment holds is alive while its profile still names this endpoint and it answers."""
  583. found = _read_devtools_file(browser.user_data_dir)
  584. return (found is not None and browser.cdp_url == f"ws://127.0.0.1:{found[0]}{found[1]}"
  585. and _devtools_endpoint_live(*found))
  586. def _heir_key(browser: LaunchedBrowser) -> Optional[str]:
  587. """Another key of this process on the same shared-profile Chromium (one dir reached by two paths, e.g. a
  588. symlink): it inherits this process's hold, so dropping ``browser``'s key must neither drop the lease nor
  589. stop the browser."""
  590. if browser.ephemeral:
  591. return None
  592. with _lock:
  593. return next((k for k, b in _browsers.items() if b is not browser and not b.ephemeral
  594. and _same_browser(b, browser)), None)
  595. def _same_browser(a: LaunchedBrowser, b: LaunchedBrowser) -> bool:
  596. """One launch: same profile (realpath), pid, start time and launch record (whose name carries the PID namespace
  597. and boot its pid belongs to); a browser of another environment also by the DevTools browser id it serves (two
  598. containers can give two browsers one pid number, neither with a start time readable here)."""
  599. return (a.foreign == b.foreign and a.pid == b.pid and a.start_time == b.start_time
  600. and _record_name(a) == _record_name(b) and _same_profile(a.user_data_dir, b.user_data_dir)
  601. and (not a.foreign or a.cdp_url == b.cdp_url))
  602. def _forget(key: str, browser: LaunchedBrowser, *, drop_record: bool = True) -> None:
  603. proc = None
  604. heir = _heir_key(browser)
  605. if _browsers.get(key) is browser: # a replacement registered since is not this browser's to drop
  606. _browsers.pop(key)
  607. proc = _procs.pop(key, None)
  608. if proc is not None and heir is not None:
  609. _procs.setdefault(heir, proc) # the heir stops (and reaps) it when it is the last one out
  610. proc = None
  611. if proc is not None:
  612. try:
  613. proc.wait(timeout=0) # reap the zombie
  614. except subprocess.TimeoutExpired:
  615. pass
  616. if not browser.ephemeral and heir is None:
  617. record = _record_path(browser)
  618. if record is not None:
  619. from tools.browser_launcher_holders import drop_lease
  620. drop_lease(record)
  621. if browser.owned and drop_record:
  622. _remove_record(browser)
  623. if browser.ephemeral:
  624. shutil.rmtree(browser.user_data_dir, ignore_errors=True)
  625. def _terminate(pid: Optional[int], start_time: Optional[int], proc: Optional[subprocess.Popen]) -> None:
  626. """Parent-first teardown (tools/AGENTS.md): SIGTERM the browser alone, then its snapshot survivors."""
  627. if not pid:
  628. return
  629. from tools.process_registry import ProcessRegistry
  630. try:
  631. ProcessRegistry._terminate_host_pid(pid, start_time)
  632. except OSError as exc:
  633. logger.debug("browser: terminating Chromium pid %s failed: %s", pid, exc)
  634. if proc is not None:
  635. try:
  636. proc.wait(timeout=5)
  637. except subprocess.TimeoutExpired:
  638. proc.kill()
  639. def stop_browser(key: str) -> None:
  640. """Let go of ``key``'s browser. A throwaway one this process launched is stopped. A shared-profile one is
  641. stopped only by its LAST holder across Hermes processes (:func:`_last_holder_out`); one a human started is
  642. never stopped. Holds ``key``'s launch lock: a relaunch of ``key`` waits for this stop instead of
  643. registering a replacement on the dir it is about to delete."""
  644. with _key_lock(key):
  645. _stop_locked(key)
  646. def _stop_locked(key: str) -> None:
  647. with _lock:
  648. browser = _browsers.get(key)
  649. proc = _procs.get(key)
  650. if browser is None:
  651. return
  652. record = None if browser.ephemeral else _record_path(browser)
  653. if record is None:
  654. if browser.owned:
  655. _terminate(browser.pid, browser.start_time, proc)
  656. with _lock:
  657. _forget(key, browser)
  658. return
  659. from tools.browser_launcher_holders import records_lock
  660. with records_lock(record.parent): # the decision and the forget are one step: another key may be the heir
  661. if _heir_key(browser) is None and _last_holder_out(record, browser):
  662. _terminate(browser.pid, browser.start_time, proc)
  663. record.unlink(missing_ok=True)
  664. with _lock:
  665. _forget(key, browser, drop_record=False)
  666. def _last_holder_out(record: Path, browser: LaunchedBrowser) -> bool:
  667. """Drop this process's hold on shared-profile ``browser`` (caller holds the records lock). True when it
  668. must stop it now: no other live Hermes process holds it, and Hermes launched it (a launch record names
  669. it). Otherwise a record this process holds, or whose owner is gone, goes to a live holder."""
  670. from tools import browser_launcher_holders as holders
  671. holders.drop_lease(record)
  672. if browser.foreign: # its pid names nobody here: stopped (or reaped) only from its own environment
  673. return False
  674. others = holders.other_live_holders(record, browser.start_time)
  675. data = _load_record(record) if record.exists() else None
  676. if data is not None and not _record_is(data, browser):
  677. data = None # an earlier browser on a recycled pid
  678. held_elsewhere = data is not None and not holders.held_here(data, "owner_pid") and _owner_alive(data)
  679. heir = holders.successor(others)
  680. if heir is not None and data is not None and not held_elsewhere:
  681. holders.hand_over(record, data, heir)
  682. if others or held_elsewhere:
  683. return False
  684. return browser.owned or data is not None
  685. def stop_all_browsers() -> None:
  686. """Stop every browser this process launched. Idempotent."""
  687. with _lock:
  688. keys = list(_browsers)
  689. for key in keys:
  690. stop_browser(key)
  691. # ── Per-profile records + orphan reap ────────────────────────────────────────────────────────────────
  692. def _ephemeral_root(home: str) -> Path:
  693. """Where ``home``'s throwaway profiles live; the reaper deletes nothing outside it."""
  694. return Path(home) / "cache" / "browser" / "chromium"
  695. def _records_dir(home: str) -> Path:
  696. """``home``'s throwaway-profile records (not the process's home): a crashed owner's Chromium is reaped by the
  697. next Hermes process serving that profile. Read only after ``_private_dir`` vouches for it."""
  698. return Path(home) / "cache" / "browser" / "chromium-records"
  699. def _is_private_dir(path: Path) -> bool:
  700. """A real directory (not a symlink) owned by this user and closed to group/other writes. Windows has no
  701. uid and synthesizes the mode bits, so there only the directory check applies."""
  702. try:
  703. st = path.lstat()
  704. except OSError:
  705. return False
  706. if not stat.S_ISDIR(st.st_mode):
  707. return False
  708. return not hasattr(os, "getuid") or (st.st_uid == os.getuid() and not st.st_mode & 0o022)
  709. def _private_dir(path: Path) -> Optional[Path]:
  710. """``path``, created 0700 when missing; None (logged) when an existing one is not private to this user."""
  711. path.parent.mkdir(parents=True, exist_ok=True)
  712. try:
  713. path.mkdir(mode=0o700)
  714. except FileExistsError:
  715. pass
  716. if _is_private_dir(path):
  717. return path
  718. logger.debug("browser: not using %s for launcher state: not a private directory of this user", path)
  719. return None
  720. def _shared_profiles_root() -> Path:
  721. """Launcher state of persistent (shared) profiles, per OS user rather than per HERMES_HOME: two homes pinning
  722. one ``browser.user_data_dir`` drive its one Chromium, so they must see one launch record, one lease set and
  723. one lock. The host rendezvous root the gateway's own cross-profile locks use."""
  724. from gateway.status import _get_lock_dir
  725. return _get_lock_dir() / "browser-shared-profiles"
  726. def _profile_digest(user_data_dir: str) -> str:
  727. """One profile dir, however it is spelled (a symlink, a trailing slash)."""
  728. return hashlib.sha256(os.path.realpath(user_data_dir).encode()).hexdigest()
  729. def _same_profile(recorded, user_data_dir: str) -> bool:
  730. return isinstance(recorded, str) and os.path.realpath(recorded) == os.path.realpath(user_data_dir)
  731. def _devtools_id(browser: LaunchedBrowser) -> Optional[str]:
  732. """The ``/devtools/browser/<id>`` path of ``browser``'s endpoint (new per launch); None before it is up."""
  733. at = browser.cdp_url.find("/devtools/browser/")
  734. return browser.cdp_url[at:] if at >= 0 else None
  735. def _record_is(record: dict, browser: LaunchedBrowser) -> bool:
  736. """``record`` describes this launch of ``browser``: same pid and profile, and its start time; for a browser of
  737. another environment (its pid is another namespace's number and its start time unreadable here), a record from
  738. another namespace naming the DevTools browser id its endpoint serves."""
  739. from tools.browser_launcher_holders import same_namespace
  740. if record.get("pid") != browser.pid or not _same_profile(record.get("user_data_dir"), browser.user_data_dir):
  741. return False
  742. if browser.foreign:
  743. return not same_namespace(record) and record.get("devtools") is not None and (
  744. record.get("devtools") == _devtools_id(browser))
  745. return record.get("start_time") == browser.start_time
  746. def _shared_state_dir(user_data_dir: str) -> Optional[Path]:
  747. """Shared profile ``user_data_dir``'s own private dir (its launch record, leases and lock)."""
  748. root = _private_dir(_shared_profiles_root())
  749. return _private_dir(root / _profile_digest(user_data_dir)[:32]) if root is not None else None
  750. def _state_dir(browser: LaunchedBrowser) -> Optional[Path]:
  751. """The private dir holding ``browser``'s launch record, leases and lock: its home's records dir for a
  752. throwaway profile, else the shared profile's own dir under :func:`_shared_profiles_root`."""
  753. if browser.ephemeral:
  754. return _private_dir(_records_dir(browser.home_key))
  755. return _shared_state_dir(browser.user_data_dir)
  756. def _record_path(browser: LaunchedBrowser) -> Optional[Path]:
  757. records = _state_dir(browser)
  758. if records is None:
  759. return None
  760. return records / _record_name(browser)
  761. def _record_name(browser: LaunchedBrowser) -> str:
  762. """The launch record ``browser``'s leases are keyed by: the one it was bound to, else this namespace's name."""
  763. from tools.browser_launcher_holders import ns_tag
  764. return browser.record_name or f"{_profile_digest(browser.user_data_dir)[:16]}-{browser.pid}{ns_tag()}.json"
  765. def _write_record(browser: LaunchedBrowser) -> None:
  766. from tools import browser_launcher_holders as holders
  767. from utils import atomic_json_write
  768. record = {"owner_pid": os.getpid(), "owner_start_time": _start_time(os.getpid()), "pid": browser.pid,
  769. "start_time": browser.start_time, "user_data_dir": browser.user_data_dir,
  770. "home_key": browser.home_key, "ephemeral": browser.ephemeral, "ns": holders.ns_identity(),
  771. "devtools": _devtools_id(browser)}
  772. try:
  773. path = _record_path(browser)
  774. if path is not None:
  775. atomic_json_write(path, record, indent=None)
  776. except OSError as exc:
  777. logger.debug("browser: could not write launch record for pid %s: %s", browser.pid, exc)
  778. def _remove_record(browser: LaunchedBrowser) -> None:
  779. try:
  780. path = _record_path(browser)
  781. if path is not None:
  782. path.unlink(missing_ok=True)
  783. except OSError as exc:
  784. logger.debug("browser: could not remove launch record for pid %s: %s", browser.pid, exc)
  785. def _owner_alive(record: dict) -> bool:
  786. """The record's owner is alive, or cannot be proven gone (it was written in another PID namespace)."""
  787. from tools.browser_launcher_holders import same_namespace
  788. if not same_namespace(record):
  789. return True
  790. owner = record.get("owner_pid")
  791. if not isinstance(owner, int) or not _pid_alive(owner):
  792. return False
  793. expected = record.get("owner_start_time")
  794. return expected is None or _start_time(owner) == expected
  795. def _cmdline_names_dir(pid: int, user_data_dir: str) -> bool:
  796. """True only when ``pid``'s argv carries ``--user-data-dir=`` naming ``user_data_dir`` (however either is
  797. spelled: a symlink, a trailing slash)."""
  798. try:
  799. import psutil
  800. argv = psutil.Process(pid).cmdline()
  801. except Exception: # health: allow BLE001 -- unreadable identity fails closed (no kill)
  802. return False
  803. return any(tok.startswith("--user-data-dir=") and _same_profile(tok.split("=", 1)[1], user_data_dir)
  804. for tok in argv)
  805. def _inside_ephemeral_root(user_data_dir: str, home: Optional[str]) -> bool:
  806. if home is None: # a shared-profile state dir holds no throwaway profiles
  807. return False
  808. try:
  809. rel = Path(user_data_dir).resolve().relative_to(_ephemeral_root(home).resolve())
  810. except (OSError, ValueError):
  811. return False
  812. return bool(rel.parts)
  813. def _is_recorded_browser(record: dict, home: Optional[str]) -> bool:
  814. pid, udd = record.get("pid"), record.get("user_data_dir")
  815. if not isinstance(pid, int) or not isinstance(udd, str) or not _pid_alive(pid):
  816. return False
  817. if record.get("ephemeral") and not _inside_ephemeral_root(udd, home):
  818. logger.debug("browser: ignoring launch record for pid %s: %s is not a launcher profile dir", pid, udd)
  819. return False
  820. expected = record.get("start_time")
  821. if expected is None or _start_time(pid) != expected:
  822. return False # no recorded fingerprint, or a recycled pid: never signal on argv alone
  823. if not _cmdline_names_dir(pid, udd):
  824. logger.warning("browser: refusing to reap pid %s: its command line does not name %s", pid, udd)
  825. return False
  826. return True
  827. def _tracked_browser(record: dict) -> Optional[LaunchedBrowser]:
  828. with _lock: # a browser another environment holds has a pid of THAT namespace: it tracks no record here
  829. return next((b for b in (*_browsers.values(), *_launching.values())
  830. if not b.foreign and _record_is(record, b)), None)
  831. def _adopt_record(browser: LaunchedBrowser) -> None:
  832. """A crashed owner's record still names the browser this process now uses (attached after the crash):
  833. take the record over, or every reaper (this process's included) would kill it as an orphan. Only the owner
  834. changes: the launch as recorded (its ``--user-data-dir`` spelling, DevTools id) stays what verifies it."""
  835. from utils import atomic_json_write
  836. path = _record_path(browser)
  837. if path is None or not path.exists():
  838. return
  839. record = _load_record(path)
  840. if record is not None and _record_is(record, browser) and not _owner_alive(record):
  841. me = os.getpid()
  842. atomic_json_write(path, {**record, "owner_pid": me, "owner_start_time": _start_time(me)}, indent=None)
  843. def _load_record(path: Path) -> Optional[dict]:
  844. """The record, or None (a corrupt one is removed)."""
  845. try:
  846. record = json.loads(path.read_text(encoding="utf-8-sig"))
  847. if isinstance(record, dict):
  848. return record
  849. except (OSError, ValueError):
  850. pass
  851. path.unlink(missing_ok=True)
  852. return None
  853. def _reap_record(path: Path, home: Optional[str]) -> bool:
  854. """Caller holds the records lock."""
  855. from tools import browser_launcher_holders as holders
  856. record = _load_record(path)
  857. if record is None or not holders.same_namespace(record): # another namespace's pids name nobody here
  858. return False
  859. tracked = _tracked_browser(record)
  860. if tracked is not None: # in use here, whoever launched it
  861. _adopt_record(tracked)
  862. return False
  863. if record.get("owner_pid") != os.getpid() and _owner_alive(record):
  864. return False
  865. others = holders.other_live_holders(path, record.get("start_time"))
  866. if others: # another live Hermes process still drives it
  867. heir = holders.successor(others)
  868. if heir is not None:
  869. holders.hand_over(path, record, heir)
  870. return False
  871. reaped = _is_recorded_browser(record, home)
  872. if reaped:
  873. _terminate(record["pid"], record.get("start_time"), None)
  874. logger.info("browser: reaped orphaned Chromium pid %s (%s)", record["pid"], record["user_data_dir"])
  875. udd = record.get("user_data_dir")
  876. if (record.get("ephemeral") and isinstance(udd, str) and _inside_ephemeral_root(udd, home)
  877. and (reaped or not _browser_running_on(udd))):
  878. shutil.rmtree(udd, ignore_errors=True) # its owner is gone: no launch reuses a per-process dir
  879. path.unlink(missing_ok=True)
  880. return reaped
  881. def _reap_home(home: str) -> int:
  882. return _reap_state_dir(_records_dir(home), home)
  883. def _reap_shared_profiles() -> int:
  884. """Every shared profile's records, whichever home launched its browser."""
  885. root = _shared_profiles_root()
  886. if not os.path.lexists(root) or not _is_private_dir(root):
  887. return 0
  888. try:
  889. states = sorted(root.iterdir())
  890. except OSError as exc:
  891. logger.debug("browser: shared-profile launcher state unavailable in %s: %s", root, exc)
  892. return 0
  893. return sum(_reap_state_dir(state, None) for state in states)
  894. def _reap_state_dir(records: Path, home: Optional[str]) -> int:
  895. """Reap one launcher state dir (``home``'s records, or a shared profile's dir when ``home`` is None). A
  896. shared profile whose lock is taken (a first launch holds it for up to its whole readiness wait) is left
  897. to the next pass."""
  898. if not os.path.lexists(records):
  899. return 0
  900. if not _is_private_dir(records):
  901. logger.debug("browser: not reaping from %s: not a private directory of this user", records)
  902. return 0
  903. from tools import browser_launcher_holders as holders
  904. with holders.records_lock(records, blocking=home is not None) as locked:
  905. if not locked:
  906. logger.debug("browser: %s is busy (a launch or stop holds it); reaping it next pass", records)
  907. return 0
  908. try:
  909. paths = sorted(records.glob("*.json"))
  910. except OSError as exc:
  911. logger.debug("browser: launch records unavailable in %s: %s", records, exc)
  912. return 0
  913. reaped = sum(1 for path in paths if _reap_record(path, home))
  914. holders.drop_dead_leases(records)
  915. return reaped
  916. _THROWAWAY_DIR = re.compile(r"^[0-9a-f]{16}-(\d+)(?:-(\d+))?(-n[0-9a-f]{12})?$") # _default_user_data_dir's names
  917. def _reclaim_throwaway_dirs(home: str) -> None:
  918. """Remove ``home``'s throwaway profiles whose owning process is gone (pid and start time in the name): one
  919. that died before writing its launch record left nothing else that would. The dir of a live owner, or of an
  920. owner in another PID namespace or boot (its name's ``ns_tag`` differs or is missing), is never touched; a
  921. name without a start time is reclaimed only once its pid is gone."""
  922. root = _ephemeral_root(home)
  923. if not os.path.lexists(root) or not _is_private_dir(root):
  924. return
  925. from tools.browser_launcher_holders import ns_tag, process_alive
  926. try:
  927. entries = list(root.iterdir())
  928. except OSError as exc:
  929. logger.debug("browser: throwaway profiles unavailable in %s: %s", root, exc)
  930. return
  931. for entry in entries:
  932. match = _THROWAWAY_DIR.match(entry.name)
  933. if match is None or (match.group(3) or "") != ns_tag() or int(match.group(1)) == os.getpid():
  934. continue # not a launcher dir, another PID namespace's (its owner cannot be checked here), or ours
  935. if process_alive(int(match.group(1)), int(match.group(2)) if match.group(2) else None):
  936. continue
  937. if _browser_running_on(str(entry)): # its record was lost or could not prove the pid: never pull the dir
  938. logger.debug("browser: keeping the throwaway profile %s: a Chromium still runs on it", entry)
  939. continue
  940. shutil.rmtree(entry, ignore_errors=True) # a symlink is refused, never followed
  941. logger.info("browser: removed the throwaway profile %s of a Hermes process that is gone", entry)
  942. def _served_home_keys(homes) -> set[str]:
  943. """The launch profile, every profile this process launched a browser for, and ``homes``."""
  944. from hermes_constants import get_process_hermes_home, hermes_home_key
  945. with _lock:
  946. keys = set(_served_homes)
  947. keys.add(hermes_home_key(get_process_hermes_home()))
  948. keys.update(hermes_home_key(home) for home in homes)
  949. return keys
  950. def reap_orphaned_browsers(homes=()) -> int:
  951. """Stop Chromium processes whose owning Hermes process is gone, in every profile this process serves
  952. (``_served_home_keys``; the janitor and atexit run outside any profile scope, so each home's records
  953. are read by path); returns the count. A pid is signalled only after its start time AND its
  954. ``--user-data-dir`` argv token match the record. Throwaway profiles of owners that are gone go too."""
  955. reaped = _reap_shared_profiles()
  956. for home in sorted(_served_home_keys(homes)):
  957. reaped += _reap_home(home)
  958. _reclaim_throwaway_dirs(home)
  959. return reaped