browser_session_lifecycle.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370
  1. """Browser session lifecycle: inactivity janitor, orphan reap, per-session teardown, ``cleanup_*`` entry
  2. points and the atexit sweep.
  3. Teardown per session kind (CDP supervisor stopped first, so our WebSocket closes before the endpoint):
  4. cloud → ``provider.close_session``; Lightpanda → ``stop_lightpanda``; launcher Chromium → ``stop_browser``
  5. (a shared persistent profile only once no other session uses it); sandbox → SIGTERM inside the sandbox
  6. once no other session uses it;
  7. real profile → shared per profile home, terminated at exit / on that profile's consent-off; CDP override → nothing (the user's browser).
  8. """
  9. from __future__ import annotations
  10. import contextlib
  11. import logging
  12. import threading
  13. import time
  14. from datetime import UTC, datetime
  15. from pathlib import Path
  16. from typing import Any, Optional
  17. from hermes_constants import get_hermes_home, reset_hermes_home_override, set_hermes_home_override
  18. logger = logging.getLogger(__name__)
  19. def _bs():
  20. from tools import browser_session
  21. return browser_session
  22. def _session_expiry_timestamp(session_info: dict[str, Any]) -> Optional[float]:
  23. """Provider-authoritative expiry as epoch seconds; None when absent or malformed."""
  24. value = session_info.get("expires_at")
  25. if isinstance(value, (int, float)) and not isinstance(value, bool):
  26. return float(value)
  27. if not isinstance(value, str) or not value.strip():
  28. return None
  29. normalized = value.strip()
  30. if normalized.endswith(("Z", "z")):
  31. normalized = f"{normalized[:-1]}+00:00"
  32. try:
  33. parsed = datetime.fromisoformat(normalized)
  34. except ValueError:
  35. logger.warning("Ignoring invalid cloud browser session expiry timestamp")
  36. return None
  37. if parsed.tzinfo is None:
  38. parsed = parsed.replace(tzinfo=UTC)
  39. return parsed.timestamp()
  40. def session_has_expired(session_info: dict[str, Any], *, now: Optional[float] = None) -> bool:
  41. """Whether a cached session crossed its provider deadline."""
  42. expires_at = _session_expiry_timestamp(session_info)
  43. return expires_at is not None and (time.time() if now is None else now) >= expires_at
  44. def _best_effort(label: str, fn) -> None:
  45. """Run ``fn()``; log and swallow any exception — teardown must never abort."""
  46. try:
  47. fn()
  48. except Exception as e: # health: allow BLE001 -- teardown boundary: one failure must not stop the rest
  49. logger.debug("%s failed: %s", label, e)
  50. def update_session_activity(cache_key: str) -> None:
  51. """Touch the activity timestamp and record the owning Hermes home on first sight (#86402)."""
  52. bs = _bs()
  53. with bs._cleanup_lock:
  54. bs._session_last_activity[cache_key] = time.time()
  55. bs._session_owner_homes.setdefault(cache_key, str(get_hermes_home()))
  56. @contextlib.contextmanager
  57. def _session_owner_scope(cache_key: str):
  58. """Run under the Hermes home + secret scope owning ``cache_key``'s session (no-op when unrecorded); the
  59. janitor is process-global, so each teardown re-enters ITS profile, never ``os.environ``."""
  60. owner_home = _bs()._session_owner_homes.get(cache_key)
  61. if owner_home is None:
  62. yield
  63. return
  64. from agent.secret_scope import build_profile_secret_scope, reset_secret_scope, set_secret_scope
  65. from hermes_cli.env_loader import hydrate_profile_secret_sources
  66. home_token = set_hermes_home_override(owner_home)
  67. try:
  68. hydrate_profile_secret_sources(Path(owner_home))
  69. secret_token = set_secret_scope(build_profile_secret_scope(Path(owner_home)), profile_home=owner_home)
  70. try:
  71. yield
  72. finally:
  73. reset_secret_scope(secret_token)
  74. finally:
  75. reset_hermes_home_override(home_token)
  76. def _forget_session_tracking(cache_key: str, *, activity: bool = True, session: bool = False) -> None:
  77. bs = _bs()
  78. with bs._cleanup_lock:
  79. if session:
  80. bs._active_sessions.pop(cache_key, None)
  81. if activity:
  82. bs._session_last_activity.pop(cache_key, None)
  83. bs._session_owner_homes.pop(cache_key, None)
  84. bs._cleanup_failures.pop(cache_key, None)
  85. # ── Teardown ─────────────────────────────────────────────────────────────────────────────────────────
  86. def _launcher_key_in_use(launcher_key: str, session_info: dict[str, Any]) -> bool:
  87. bs = _bs()
  88. with bs._cleanup_lock:
  89. return any(info is not session_info and info.get("launcher_key") == launcher_key
  90. for info in bs._active_sessions.values())
  91. def _close_cloud_session(session_info: dict[str, Any]) -> None:
  92. bb_session_id = session_info.get("bb_session_id")
  93. if not bb_session_id:
  94. return
  95. from tools.browser_session_cloud import get_cloud_provider
  96. provider = get_cloud_provider()
  97. if provider is None:
  98. return
  99. try:
  100. provider.close_session(bb_session_id)
  101. except Exception as e: # health: allow BLE001 -- a provider error must never abort the rest of the teardown
  102. logger.warning("Could not close cloud browser session: %s", e)
  103. def release_session_backend(cache_key: str, session_info: dict[str, Any]) -> None:
  104. """Release ``session_info``'s browser BY OBJECT (an unpublished session releases through this alone)."""
  105. _close_cloud_session(session_info)
  106. features = session_info.get("features") or {}
  107. if features.get("lightpanda"):
  108. from tools.browser_lightpanda import stop_lightpanda
  109. stop_lightpanda(session_info.get("session_name", ""))
  110. launcher_key = session_info.get("launcher_key")
  111. if not launcher_key or _launcher_key_in_use(launcher_key, session_info):
  112. return # nothing launched for it, or a shared browser another live session still uses
  113. if features.get("sandbox"):
  114. from tools.browser_session_sandbox import stop_sandbox_browser
  115. stop_sandbox_browser(session_info)
  116. elif not features.get("real_profile"): # outlives its sessions: stopped at exit / on consent-off
  117. from tools.browser_launcher import stop_browser
  118. stop_browser(launcher_key)
  119. def cleanup_single_session(cache_key: str) -> None:
  120. """Tear down one session by its exact key."""
  121. from tools.browser_session_cdp import stop_cdp_supervisor
  122. stop_cdp_supervisor(cache_key) # close our WebSocket BEFORE the backend tears the endpoint down
  123. bs = _bs()
  124. with bs._cleanup_lock:
  125. session_info = bs._active_sessions.pop(cache_key, None)
  126. _forget_session_tracking(cache_key)
  127. if session_info:
  128. release_session_backend(cache_key, session_info)
  129. logger.debug("Closed browser session %s", cache_key)
  130. def cleanup_browser(task_id: Optional[str] = None) -> None:
  131. """Close the sessions serving ``task_id``: its exact key, its hybrid ``::local`` sidecar, and the
  132. served-profile-tagged keys browser_exec builds for it (``<task>@<home>``). Per-turn cleanup calls this
  133. with the bare task id."""
  134. task = task_id or "default"
  135. bs = _bs()
  136. with bs._cleanup_lock:
  137. if bs.is_local_sidecar_key(task):
  138. keys = [task] if task in bs._active_sessions else []
  139. else:
  140. keys = [key for key in bs._active_sessions if key == task or bs.owner_task_id(key) == task]
  141. for key in keys:
  142. cleanup_single_session(key)
  143. if not keys:
  144. _forget_session_tracking(task)
  145. def cleanup_profile_browsers(home_key: str) -> None:
  146. """Close one profile home's sessions, CDP supervisors and harness daemons, and drop its cached config
  147. (``/browser connect`` on a multiplexed gateway). Other profiles' browsers stay."""
  148. bs = _bs()
  149. with bs._cleanup_lock:
  150. keys = [key for key, info in bs._active_sessions.items() if info.get("home_key") == home_key]
  151. for key in keys:
  152. cleanup_single_session(key)
  153. def _stop_supervisors():
  154. from tools.browser_supervisor import SUPERVISOR_REGISTRY
  155. SUPERVISOR_REGISTRY.stop_home(home_key)
  156. _best_effort("CDP supervisor stop", _stop_supervisors)
  157. def _stop_harness():
  158. from tools.browser_use_cli import stop_harness_daemons
  159. stop_harness_daemons(home_key)
  160. _best_effort("Browser Use harness daemon stop", _stop_harness)
  161. from tools.browser_session_cloud import reset_caches
  162. reset_caches(home_key)
  163. def _cleanup_in_owner_scope(cache_key: str) -> None:
  164. with _session_owner_scope(cache_key):
  165. cleanup_single_session(cache_key)
  166. def cleanup_all_browsers() -> None:
  167. """Close every session (gateway shutdown), stop every browser this process launched, reset caches. Each
  168. session is closed under its owning profile, as the janitor does: a cloud session belongs to that profile's
  169. provider and credentials."""
  170. bs = _bs()
  171. with bs._cleanup_lock:
  172. keys = list(bs._active_sessions)
  173. for key in keys:
  174. _best_effort(f"browser cleanup for {key}", lambda key=key: _cleanup_in_owner_scope(key))
  175. def _stop_supervisors():
  176. from tools.browser_supervisor import SUPERVISOR_REGISTRY
  177. SUPERVISOR_REGISTRY.stop_all()
  178. _best_effort("CDP supervisor stop", _stop_supervisors)
  179. def _stop_harness():
  180. from tools.browser_use_cli import stop_harness_daemons
  181. stop_harness_daemons()
  182. _best_effort("Browser Use harness daemon stop", _stop_harness)
  183. def _stop_launched():
  184. from tools.browser_launcher import stop_all_browsers
  185. stop_all_browsers() # incl. the shared real-profile / persistent-profile browsers
  186. _best_effort("Chromium stop", _stop_launched)
  187. def _stop_lightpanda():
  188. from tools.browser_lightpanda import stop_all_lightpanda
  189. stop_all_lightpanda()
  190. _best_effort("Lightpanda stop", _stop_lightpanda)
  191. from tools.browser_session_cloud import reset_caches
  192. reset_caches()
  193. def emergency_cleanup_all_sessions() -> None:
  194. """atexit: close this process's browsers, then reap orphans left by crashed Hermes processes (every
  195. clean exit sweeps accumulated orphans). Idempotent."""
  196. bs = _bs()
  197. if bs._cleanup_done:
  198. return
  199. bs._cleanup_done = True
  200. if bs._active_sessions:
  201. logger.info("Emergency cleanup: closing %s active session(s)...", len(bs._active_sessions))
  202. _best_effort("Browser cleanup on exit", cleanup_all_browsers)
  203. with bs._cleanup_lock:
  204. bs._active_sessions.clear()
  205. bs._session_last_activity.clear()
  206. bs._session_owner_homes.clear()
  207. bs._cleanup_failures.clear()
  208. _best_effort("Orphan reap on exit", reap_orphaned_browser_sessions)
  209. def reap_orphaned_browser_sessions() -> int:
  210. """Stop Chromium / ``lightpanda serve`` processes whose owning Hermes process is gone (Chromium: in every
  211. profile this process serves, incl. each live session's owning home)."""
  212. from tools.browser_launcher import reap_orphaned_browsers
  213. from tools.browser_lightpanda import reap_orphaned_lightpanda
  214. bs = _bs()
  215. with bs._cleanup_lock:
  216. session_homes = list(bs._session_owner_homes.values())
  217. reaped = 0
  218. for name, reap in (("lightpanda", reap_orphaned_lightpanda),
  219. ("chromium", lambda: reap_orphaned_browsers(session_homes))):
  220. try:
  221. reaped += reap()
  222. except OSError as exc:
  223. logger.debug("orphan reap (%s) failed: %s", name, exc)
  224. if reaped:
  225. logger.info("Reaped %d orphaned browser process(es) from previous run(s)", reaped)
  226. return reaped
  227. # ── Inactivity janitor ───────────────────────────────────────────────────────────────────────────────
  228. def _human_holds_shared_browser(cache_key: str) -> bool:
  229. bs = _bs()
  230. with bs._cleanup_lock:
  231. session_info = bs._active_sessions.get(cache_key)
  232. return bool(session_info) and bs.human_holds_shared_browser(session_info)
  233. def _force_reap(cache_key: str) -> None:
  234. """Janitor last resort after repeated failures (#100738): drop the record, release the backend."""
  235. from tools.browser_session_cdp import stop_cdp_supervisor
  236. stop_cdp_supervisor(cache_key)
  237. bs = _bs()
  238. with bs._cleanup_lock:
  239. session_info = bs._active_sessions.get(cache_key)
  240. _forget_session_tracking(cache_key, session=True) # untracked even if the release below fails
  241. if session_info:
  242. release_session_backend(cache_key, session_info)
  243. def _cleanup_one_inactive(cache_key: str, elapsed: int) -> None:
  244. bs = _bs()
  245. logger.info("Cleaning up inactive session %s (inactive for %ss)", cache_key, elapsed)
  246. try:
  247. with _session_owner_scope(cache_key):
  248. cleanup_single_session(cache_key)
  249. _forget_session_tracking(cache_key)
  250. except Exception as e: # health: allow BLE001 -- counted; force-reaped after MAX_INACTIVITY_CLEANUP_FAILURES
  251. with bs._cleanup_lock:
  252. failures = bs._cleanup_failures[cache_key] = bs._cleanup_failures.get(cache_key, 0) + 1
  253. if failures < bs.MAX_INACTIVITY_CLEANUP_FAILURES:
  254. logger.warning("Error cleaning up inactive session %s (attempt %d/%d): %s",
  255. cache_key, failures, bs.MAX_INACTIVITY_CLEANUP_FAILURES, e)
  256. return
  257. logger.error("Browser cleanup failed %d times for %s; force-reaping: %s", failures, cache_key, e)
  258. try:
  259. with _session_owner_scope(cache_key):
  260. _force_reap(cache_key)
  261. finally:
  262. _forget_session_tracking(cache_key, activity=False)
  263. def cleanup_inactive_sessions() -> None:
  264. """Close sessions idle past ``BROWSER_SESSION_INACTIVITY_TIMEOUT`` (janitor thread), each under its owner
  265. profile's scope. A human holding the Bot Desktop lease counts as activity."""
  266. bs = _bs()
  267. now = time.time()
  268. with bs._cleanup_lock:
  269. idle = [(key, int(now - last)) for key, last in bs._session_last_activity.items()
  270. if now - last > bs.BROWSER_SESSION_INACTIVITY_TIMEOUT]
  271. for key, elapsed in idle:
  272. with _session_owner_scope(key):
  273. if _human_holds_shared_browser(key):
  274. update_session_activity(key)
  275. continue
  276. _cleanup_one_inactive(key, elapsed)
  277. def _janitor() -> None:
  278. """Every 30 s: close idle sessions; reap orphans at startup AND every BROWSER_ORPHAN_REAP_INTERVAL."""
  279. bs = _bs()
  280. reap_every = max(1, round(bs.BROWSER_ORPHAN_REAP_INTERVAL / 30))
  281. cycle = 0
  282. while bs._cleanup_running:
  283. if cycle % reap_every == 0:
  284. _best_effort("Orphan reap", reap_orphaned_browser_sessions)
  285. cycle += 1
  286. _best_effort("Inactive browser cleanup", cleanup_inactive_sessions)
  287. for _ in range(30): # 1 s granularity so stop is quick
  288. if not bs._cleanup_running:
  289. break
  290. time.sleep(1)
  291. def start_cleanup_thread() -> None:
  292. bs = _bs()
  293. with bs._cleanup_lock:
  294. if bs._cleanup_thread is None or not bs._cleanup_thread.is_alive():
  295. bs._cleanup_running = True
  296. # health: allow HX012 -- process-wide janitor: each teardown binds its own session owner scope
  297. bs._cleanup_thread = threading.Thread(target=_janitor, daemon=True, name="browser-cleanup")
  298. bs._cleanup_thread.start()
  299. logger.info("Started browser inactivity janitor (timeout: %ss)", bs.BROWSER_SESSION_INACTIVITY_TIMEOUT)
  300. def stop_cleanup_thread() -> None:
  301. bs = _bs()
  302. bs._cleanup_running = False
  303. thread = bs._cleanup_thread
  304. if thread is not None:
  305. # A second Ctrl+C during the timed join lands here; the janitor is a daemon thread.
  306. try:
  307. thread.join(timeout=5)
  308. except (SystemExit, KeyboardInterrupt):
  309. pass