browser_session.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388
  1. """Browser sessions for ``browser_exec``: one session per cache key, created in a fixed precedence and
  2. always carrying a ``cdp_url``.
  3. Precedence (``BU_CDP_*`` operator env is honoured earlier, by ``browser_exec``):
  4. CDP override (live ``/browser connect`` > ``BROWSER_CDP_URL`` > ``browser.cdp_url``) > hybrid ``::local``
  5. sidecar (private URL, never the real profile) > cloud provider (``create_session``; on failure a LOCAL
  6. session marked ``fallback_from_cloud``) > local: real profile (consented) > Lightpanda
  7. (``browser.engine: lightpanda``) > the packaged Chromium via ``tools.browser_launcher``.
  8. State lives here; sibling modules (``browser_session_*``) late-import this module inside functions.
  9. """
  10. from __future__ import annotations
  11. import atexit
  12. import logging
  13. import threading
  14. import uuid
  15. from typing import Any, Callable, Optional
  16. from hermes_cli.config import DEFAULT_CONFIG
  17. from hermes_constants import hermes_home_key
  18. from utils import env_int
  19. logger = logging.getLogger(__name__)
  20. LOCAL_SUFFIX = "::local"
  21. # Inactivity timeout: config.yaml is authoritative; BROWSER_INACTIVITY_TIMEOUT stays a legacy env fallback.
  22. DEFAULT_SESSION_INACTIVITY_TIMEOUT = int(DEFAULT_CONFIG.get("browser", {}).get("inactivity_timeout", 120))
  23. def _get_session_inactivity_timeout() -> int:
  24. from tools.browser_session_cloud import inactivity_timeout
  25. return inactivity_timeout(env_int("BROWSER_INACTIVITY_TIMEOUT", DEFAULT_SESSION_INACTIVITY_TIMEOUT))
  26. BROWSER_SESSION_INACTIVITY_TIMEOUT = _get_session_inactivity_timeout()
  27. # Orphan reap cadence: a startup-only reap never recovers a leak that appears in a long-lived process.
  28. BROWSER_ORPHAN_REAP_INTERVAL = 300
  29. MAX_INACTIVITY_CLEANUP_FAILURES = 3
  30. _active_sessions: dict[str, dict[str, Any]] = {}
  31. _session_last_activity: dict[str, float] = {}
  32. # Owner Hermes home per session: the janitor is one process-global thread, so each teardown re-enters the
  33. # OWNING profile's scope (#86402).
  34. _session_owner_homes: dict[str, str] = {}
  35. _cleanup_failures: dict[str, int] = {} # consecutive janitor failures (#100738)
  36. _cleanup_lock = threading.Lock() # protects every dict above
  37. _cleanup_thread: Optional[threading.Thread] = None
  38. _cleanup_running = False
  39. _cleanup_done = False
  40. # A session built while its profile published a new live endpoint is closed and rebuilt; bounded so a
  41. # flapping connect/disconnect loop cannot keep a call building browsers forever.
  42. _STALE_ENDPOINT_RETRIES = 2
  43. # ── Env ──────────────────────────────────────────────────────────────────────────────────────────────
  44. def build_browser_env() -> dict:
  45. """Env for every browser child (Chromium, ``lightpanda serve``, the harness CLI base): the served
  46. profile's child env WITHOUT credentials, the Bot Desktop DISPLAY when its screen is up, a loopback
  47. NO_PROXY (a macOS system proxy would capture the local CDP dial, #110565), and a short TMPDIR
  48. (Chrome's SingletonSocket overflows the AF_UNIX path cap under a deep scratch dir)."""
  49. from agent.proxy_bypass import add_loopback_no_proxy
  50. from hermes_constants import socket_safe_tmpdir
  51. from tools.bot_desktop.runtime import desktop_env
  52. from tools.environments.local import served_profile_child_env
  53. env = add_loopback_no_proxy(desktop_env(served_profile_child_env(inherit_credentials=False)))
  54. env["TMPDIR"] = socket_safe_tmpdir()
  55. return env
  56. # ── Bot Desktop fence ────────────────────────────────────────────────────────────────────────────────
  57. def shares_bot_desktop_browser(session_info: dict[str, Any]) -> bool:
  58. """Every LOCAL session is a browser Hermes launched with this profile's Bot Desktop DISPLAY, so it is the
  59. screen a human who took over is typing into. Cloud / user-supplied CDP sessions are another browser. A
  60. human lease with the screen already gone still fences (computer_use does the same)."""
  61. if not (session_info.get("features") or {}).get("local"):
  62. return False
  63. from tools.bot_desktop import lease as _bd_lease, runtime as _bd_runtime
  64. return bool(_bd_runtime.published_env().get("DISPLAY")) or _bd_lease.human_holds()
  65. def human_holds_shared_browser(session_info: dict[str, Any]) -> bool:
  66. """True while a human holds the Bot Desktop lease over the browser ``session_info`` shares with them; the
  67. janitor treats that as activity (reaping it mid-login kills the human's session, #110064)."""
  68. if not shares_bot_desktop_browser(session_info):
  69. return False
  70. from tools.bot_desktop import lease as _bd_lease
  71. return _bd_lease.human_holds()
  72. def run_fenced(session_info: dict[str, Any], fn: Callable[[], dict[str, Any]]) -> dict[str, Any]:
  73. """Run ``fn`` under the Bot Desktop lease fence when ``session_info`` is the bot's LOCAL browser.
  74. While a human holds the lease every action AND read against it is refused (the page may show their
  75. credential); the fence brackets the whole run so a takeover mid-command also voids the result.
  76. """
  77. if not shares_bot_desktop_browser(session_info):
  78. return fn()
  79. from tools.bot_desktop import lease as _bd_lease
  80. try:
  81. admitted = _bd_lease.assert_agent_may_act()
  82. except _bd_lease.HumanHasControl as e:
  83. return {"success": False, "error": str(e), "code": "human_has_control"}
  84. result = fn()
  85. if _bd_lease.get().epoch != admitted.epoch:
  86. return {"success": False, "code": "human_has_control",
  87. "error": "A human took over the bot's screen while this browser command ran; its result was "
  88. "discarded. Tell the user what you need; retry once they hand back."}
  89. return result
  90. def ensure_screen_for_headed_chromium() -> None:
  91. """Tool-call boundary for ``bot_desktop.auto_start``: a headed Chromium is about to start, so a headless
  92. profile gets its screen first. Headless browsing and Lightpanda never bring one up."""
  93. from tools.browser_session_cloud import is_headed_mode
  94. if is_headed_mode():
  95. from tools.bot_desktop.runtime import ensure_started_for_tool
  96. ensure_started_for_tool()
  97. # ── Session records ──────────────────────────────────────────────────────────────────────────────────
  98. def _session_record(prefix: str, cdp_url: Optional[str], features: dict[str, Any]) -> dict[str, Any]:
  99. """Fresh session dict with a random ``<prefix>_<hex10>`` session name (log / Lightpanda identity)."""
  100. return {"session_name": f"{prefix}_{uuid.uuid4().hex[:10]}", "bb_session_id": None,
  101. "cdp_url": cdp_url, "features": features}
  102. def is_local_sidecar_key(session_key: str) -> bool:
  103. return session_key.endswith(LOCAL_SUFFIX)
  104. def owner_task_id(session_key: str) -> str:
  105. """The task a session key belongs to: strips the ``::local`` sidecar suffix and the ``@<home>`` served
  106. profile tag browser_exec adds (per-turn cleanup is called with the bare task id)."""
  107. key = session_key[: -len(LOCAL_SUFFIX)] if is_local_sidecar_key(session_key) else session_key
  108. return key.rsplit("@", 1)[0] if "@" in key else key
  109. def real_profile_note(cache_key: Optional[str]) -> Optional[str]:
  110. """The real-profile fallback note on ``cache_key``'s existing session; never creates or touches one."""
  111. with _cleanup_lock:
  112. return (_active_sessions.get(cache_key or "default") or {}).get("real_profile_unavailable")
  113. def session_for_task(task_id: Optional[str]) -> dict[str, Any]:
  114. """The active session serving ``task_id`` (its exact key, else one whose owner is that task), or {}."""
  115. task = task_id or "default"
  116. with _cleanup_lock:
  117. if task in _active_sessions:
  118. return _active_sessions[task]
  119. return next((info for key, info in _active_sessions.items()
  120. if owner_task_id(key) == task and not is_local_sidecar_key(key)), {})
  121. # ── Session creation ─────────────────────────────────────────────────────────────────────────────────
  122. def _bot_desktop_target() -> Optional[tuple[str, dict[str, Any]]]:
  123. """``(launcher key, launch kwargs)`` for the shared Bot Desktop profile while its host screen is up: the
  124. agent's Chromium and the dock's Browser icon are THE SAME browser (binary + user-data-dir), and a dock
  125. instance a human already started is attached, never relaunched."""
  126. from tools.bot_desktop import runtime as _bd_runtime
  127. if not _bd_runtime.published_env().get("DISPLAY"):
  128. return None
  129. from tools.bot_desktop import browser as _bd_browser
  130. udd = str(_bd_browser.profile_dir())
  131. return f"profile:{udd}", {"executable": _bd_browser.executable(), "user_data_dir": udd, "ephemeral": False,
  132. "extra_args": _bd_browser.SHARED_PROFILE_ARGS}
  133. def _launch_target(cache_key: str) -> tuple[str, dict[str, Any], bool]:
  134. """``(launcher key, launch kwargs, shared)``. A persistent profile (Bot Desktop, ``browser.user_data_dir``)
  135. is keyed by its dir so every cache key shares the one Chromium Chromium's singleton allows on it."""
  136. bot_desktop = _bot_desktop_target()
  137. if bot_desktop is not None:
  138. return bot_desktop[0], bot_desktop[1], True
  139. from tools.browser_session_cloud import user_data_dir
  140. udd = user_data_dir()
  141. if udd:
  142. return f"profile:{udd}", {"user_data_dir": udd, "ephemeral": False}, True
  143. return cache_key, {}, False
  144. def _create_chromium_session(cache_key: str) -> dict[str, Any]:
  145. from tools.bot_desktop import placement, runtime
  146. try:
  147. # Policy first: a refused placement raises, a terminal placement starts its sandbox screen; neither
  148. # ever falls back to a host browser.
  149. where = runtime.tool_placement()
  150. except RuntimeError as exc:
  151. raise RuntimeError(f"The browser cannot run here: {exc}") from exc
  152. if where == placement.TERMINAL:
  153. from tools.browser_session_sandbox import create_sandbox_session
  154. return create_sandbox_session(cache_key)
  155. from tools.browser_launcher import launch_chromium
  156. ensure_screen_for_headed_chromium() # first: an auto-started screen selects the Bot Desktop profile
  157. launcher_key, kwargs, shared = _launch_target(cache_key)
  158. browser, err = launch_chromium(launcher_key, **kwargs)
  159. if browser is None:
  160. raise RuntimeError(err or "the local browser could not be started")
  161. info = _session_record("h", browser.cdp_url, {"local": True})
  162. info.update(launcher_key=launcher_key, shared_profile=shared)
  163. logger.info("Created local browser session %s for %s (Chromium pid %s)", info["session_name"], cache_key, browser.pid)
  164. return info
  165. def _create_lightpanda_session(cache_key: str) -> dict[str, Any]:
  166. from tools.browser_lightpanda import launch_lightpanda
  167. from tools.browser_session_cloud import is_local_backend
  168. info = _session_record("lp", None, {"local": True, "lightpanda": True})
  169. server, err = launch_lightpanda(info["session_name"], block_private_networks=not is_local_backend())
  170. if err:
  171. raise RuntimeError(err)
  172. info["cdp_url"] = server.cdp_url
  173. logger.info("Created Lightpanda session %s (port %s) for %s", info["session_name"], server.port, cache_key)
  174. return info
  175. def _create_local_session(cache_key: str, allow_real_profile: bool = True) -> dict[str, Any]:
  176. """Local session. The ONE place real profile vs fresh browser is decided: when the consented profile
  177. can't load, the fresh browser carries ``real_profile_unavailable`` so the first result tells the model it
  178. is signed out. The hybrid sidecar passes ``allow_real_profile=False``: the user's cookie jar must not
  179. reach an arbitrary internal host the model chose."""
  180. fallback_note = None
  181. if allow_real_profile:
  182. from tools.browser_session_real_profile import real_profile_cdp, real_profile_fallback_note, real_profile_key
  183. cdp_url, err = real_profile_cdp()
  184. if err:
  185. logger.warning("real-profile unavailable for %s, using a fresh browser: %s", cache_key, err)
  186. fallback_note = real_profile_fallback_note(err)
  187. elif cdp_url:
  188. info = _session_record("rp", cdp_url, {"local": True, "real_profile": True})
  189. info.update(launcher_key=real_profile_key(), shared_profile=True)
  190. logger.info("Created real-profile local session %s for %s", info["session_name"], cache_key)
  191. return info
  192. from tools.browser_lightpanda import using_lightpanda_engine
  193. info = _create_lightpanda_session(cache_key) if using_lightpanda_engine() else _create_chromium_session(cache_key)
  194. if fallback_note:
  195. info["real_profile_unavailable"] = fallback_note
  196. return info
  197. def _create_cloud_session_or_fallback(cache_key: str, provider) -> dict[str, Any]:
  198. """Cloud session; on failure (429, quota, outage) a LOCAL launcher session marked ``fallback_from_cloud``.
  199. ``cdp_url`` is resolved here because some providers return an HTTP discovery URL."""
  200. from tools.browser_session_cdp import resolve_cdp_override
  201. try:
  202. session_info = provider.create_session(cache_key)
  203. if not session_info or not isinstance(session_info, dict):
  204. raise ValueError(f"Cloud provider returned invalid session: {session_info!r}")
  205. except Exception as e: # health: allow BLE001 -- any provider failure falls back to a local browser
  206. provider_name = type(provider).__name__
  207. logger.warning("Cloud provider %s failed (%s); falling back to local Chromium for %s",
  208. provider_name, e, cache_key, exc_info=True)
  209. try:
  210. local = _create_local_session(cache_key)
  211. except Exception as local_error: # health: allow BLE001 -- report both failures together
  212. raise RuntimeError(f"Cloud provider {provider_name} failed ({e}) and local "
  213. f"fallback also failed ({local_error})") from e
  214. return {**local, "fallback_from_cloud": True, "fallback_reason": str(e), "fallback_provider": provider_name}
  215. session_info = dict(session_info)
  216. if session_info.get("cdp_url"):
  217. session_info["cdp_url"] = resolve_cdp_override(str(session_info["cdp_url"]))
  218. return session_info
  219. def _create_session_for_key(cache_key: str, force_local: bool) -> dict[str, Any]:
  220. """Fresh session (runs OUTSIDE the lock: cloud mode makes a network call, a launch takes seconds)."""
  221. from tools.browser_session_cdp import get_cdp_override
  222. from tools.browser_session_cloud import get_cloud_provider
  223. override = get_cdp_override()
  224. if override and not force_local:
  225. logger.info("Created CDP browser session for %s", cache_key)
  226. return _session_record("cdp", override, {"cdp_override": True})
  227. if force_local:
  228. return _create_local_session(cache_key, allow_real_profile=False)
  229. provider = get_cloud_provider()
  230. if provider is None:
  231. return _create_local_session(cache_key)
  232. return _create_cloud_session_or_fallback(cache_key, provider)
  233. def _backend_dead(session_info: dict[str, Any]) -> bool:
  234. """True when a local session's process is gone (Chromium crashed or was killed, ``lightpanda serve``
  235. exited): the next call builds a fresh one instead of handing out a dead endpoint.
  236. A launcher key is shared by every cache key on one persistent profile, so a live browser on the key is
  237. not enough: when another key relaunched it after a crash, this record still names the dead endpoint."""
  238. features = session_info.get("features") or {}
  239. if features.get("lightpanda"):
  240. from tools.browser_lightpanda import get_server
  241. server = get_server(session_info.get("session_name", ""))
  242. return server is None or not server.is_alive()
  243. if features.get("sandbox"):
  244. from tools.browser_session_sandbox import sandbox_browser_alive
  245. return not sandbox_browser_alive(session_info)
  246. launcher_key = session_info.get("launcher_key")
  247. if launcher_key:
  248. from tools.browser_launcher import get_browser
  249. browser = get_browser(launcher_key)
  250. return browser is None or browser.cdp_url != session_info.get("cdp_url")
  251. return False
  252. def _reuse_or_evict(cache_key: str) -> Optional[dict[str, Any]]:
  253. from tools import browser_session_lifecycle as _lifecycle
  254. with _cleanup_lock:
  255. existing = _active_sessions.get(cache_key)
  256. if existing is None:
  257. return None
  258. if not _lifecycle.session_has_expired(existing) and not _backend_dead(existing):
  259. return existing
  260. logger.info("Replacing expired or dead browser session for %s", cache_key)
  261. _lifecycle.cleanup_single_session(cache_key)
  262. _lifecycle.update_session_activity(cache_key) # teardown dropped the entry; track the replacement
  263. with _cleanup_lock:
  264. replacement = _active_sessions.get(cache_key) # another thread may already have rebuilt it
  265. return replacement if replacement is not None and replacement is not existing else None
  266. def _publish(cache_key: str, session_info: dict[str, Any], home_key: str) -> dict[str, Any]:
  267. session_info = dict(session_info)
  268. session_info.setdefault("session_key", cache_key)
  269. session_info.setdefault("owner_task_id", owner_task_id(cache_key))
  270. session_info.setdefault("home_key", home_key) # the profile reaped on /browser connect
  271. _active_sessions[cache_key] = session_info
  272. return session_info
  273. def get_session_info(cache_key: Optional[str] = None) -> dict[str, Any]:
  274. """Get or create the session for ``cache_key`` (thread-safe); starts the inactivity janitor and touches
  275. activity. A ``::local`` key forces a local browser even with a cloud provider configured."""
  276. from tools import browser_session_lifecycle as _lifecycle
  277. from tools.browser_session_cdp import ensure_cdp_supervisor, live_cdp_generation
  278. key = cache_key or "default"
  279. _lifecycle.start_cleanup_thread()
  280. _lifecycle.update_session_activity(key)
  281. reused = _reuse_or_evict(key)
  282. if reused is not None:
  283. return reused
  284. force_local = is_local_sidecar_key(key)
  285. home_key = hermes_home_key()
  286. for attempt in range(_STALE_ENDPOINT_RETRIES + 1):
  287. generation = live_cdp_generation(home_key)
  288. session_info = _create_session_for_key(key, force_local)
  289. with _cleanup_lock:
  290. if key in _active_sessions: # created concurrently during the launch: don't leak ours
  291. winner = _active_sessions[key]
  292. stale_ours = session_info
  293. session_info = None
  294. else:
  295. winner = stale_ours = None
  296. # A /browser connect publish while ours was built may mean it resolved the old endpoint:
  297. # never publish it (a concurrent caller would get a browser being closed); rebuild.
  298. if live_cdp_generation(home_key) == generation or attempt == _STALE_ENDPOINT_RETRIES:
  299. session_info = _publish(key, session_info, home_key)
  300. break
  301. if winner is not None:
  302. _lifecycle.release_session_backend(key, stale_ours)
  303. return winner
  304. logger.info("Browser endpoint changed during session creation for %s; recreating", key)
  305. _lifecycle.release_session_backend(key, session_info)
  306. # Cloud / CDP-override sessions get the supervisor here (dialogs, frame tree, vault fill). Local
  307. # browsers are attached by browser_exec per call under the task id.
  308. if not (session_info.get("features") or {}).get("local"):
  309. ensure_cdp_supervisor(key)
  310. return session_info
  311. def _atexit_cleanup() -> None:
  312. """atexit only — NO SIGINT/SIGTERM handlers calling sys.exit(): a SystemExit raised inside a
  313. prompt_toolkit key-binding callback corrupts the coroutine state and makes the process unkillable."""
  314. try:
  315. from tools.browser_session_lifecycle import emergency_cleanup_all_sessions, stop_cleanup_thread
  316. except ImportError: # half-updated tree mid-`hermes update` (#112437): nothing resolvable to clean
  317. return
  318. emergency_cleanup_all_sessions()
  319. stop_cleanup_thread()
  320. atexit.register(_atexit_cleanup)