browser_lightpanda.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. """Lightpanda local engine.
  2. With ``browser.engine: lightpanda``, each browser session spawns one ``lightpanda serve`` and
  3. ``browser_exec`` drives its CDP endpoint (``BU_CDP_URL``). ``tools.browser_session`` owns the
  4. session cache, inactivity reaper and atexit sweep.
  5. """
  6. import functools
  7. import json
  8. import logging
  9. import os
  10. import shutil
  11. import socket
  12. import subprocess
  13. import threading
  14. import time
  15. from dataclasses import dataclass
  16. from pathlib import Path
  17. from typing import Dict, Optional, Tuple
  18. from utils import atomic_json_write
  19. logger = logging.getLogger(__name__)
  20. LIGHTPANDA_INSTALL_URL = "https://lightpanda.io/docs/run-locally/installation/one-liner"
  21. LIGHTPANDA_INSTALL_HINT = f"Install Lightpanda from {LIGHTPANDA_INSTALL_URL} and make sure " "`lightpanda` is on PATH"
  22. _READY_TIMEOUT_S = 10.0
  23. _POLL_INTERVAL_S = 0.1
  24. _STDERR_TAIL_LIMIT = 2000
  25. _servers: "dict[str, LightpandaServer]" = {}
  26. _servers_lock = threading.Lock()
  27. @dataclass
  28. class LightpandaServer:
  29. session_name: str
  30. port: int
  31. proc: subprocess.Popen
  32. log_path: str
  33. start_time: Optional[int] = None
  34. @property
  35. def cdp_url(self) -> str:
  36. return f"http://127.0.0.1:{self.port}" # HTTP discovery URL; the harness resolves /json/version itself
  37. def is_alive(self) -> bool:
  38. return self.proc.poll() is None
  39. def _home_candidates() -> list:
  40. home = Path.home()
  41. candidates = [home / ".lightpanda" / "lightpanda", home / ".local" / "bin" / "lightpanda"]
  42. try:
  43. from hermes_constants import get_hermes_home
  44. candidates.append(Path(get_hermes_home()) / "bin" / "lightpanda")
  45. except Exception as e: # pragma: no cover - defensive
  46. logger.debug("hermes home unavailable for lightpanda lookup: %s", e)
  47. return candidates
  48. # System PATH fallbacks for a minimal service PATH (systemd units, launchd), Homebrew included.
  49. _SANE_PATH_DIRS = ("/opt/homebrew/bin", "/opt/homebrew/sbin", "/usr/local/sbin", "/usr/local/bin",
  50. "/usr/sbin", "/usr/bin", "/sbin", "/bin")
  51. def merge_sane_path(existing_path: str = "") -> str:
  52. """``existing_path`` with the standard system dirs that exist appended (existing order kept)."""
  53. parts = [p for p in (existing_path or "").split(os.pathsep) if p]
  54. return os.pathsep.join(parts + [d for d in _SANE_PATH_DIRS if d not in parts and os.path.isdir(d)])
  55. def find_lightpanda_binary() -> Optional[str]:
  56. """Return the lightpanda executable, or None. Order: PATH (plus the standard system dirs), then the
  57. installer's locations, then ``$HERMES_HOME/bin``. No Windows build."""
  58. if os.name == "nt":
  59. logger.debug("Lightpanda has no Windows build")
  60. return None
  61. return shutil.which("lightpanda", path=merge_sane_path(os.environ.get("PATH", ""))) or next(
  62. (str(c) for c in _home_candidates() if c.is_file() and os.access(c, os.X_OK)), None)
  63. def _pick_free_loopback_port() -> int:
  64. with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
  65. sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  66. sock.bind(("127.0.0.1", 0))
  67. return int(sock.getsockname()[1])
  68. def _state_dir() -> Path:
  69. from hermes_constants import get_hermes_home
  70. path = Path(get_hermes_home()) / "cache" / "browser-use" / "lightpanda"
  71. path.mkdir(parents=True, exist_ok=True)
  72. return path
  73. def _http_cache_dir() -> Path:
  74. """Filesystem HTTP cache shared by every Lightpanda this Hermes spawns.
  75. Shared rather than per-session so a cached asset survives session churn.
  76. Lightpanda holds it in sqlite (WAL) with a best-effort write path, and
  77. ``--http-cache-entry-limit`` (upstream default 1000, not passed here)
  78. bounds it without Hermes managing eviction.
  79. """
  80. path = _state_dir() / "http-cache"
  81. path.mkdir(parents=True, exist_ok=True)
  82. return path
  83. _HTTP_CACHE_FLAG = "--http-cache-dir"
  84. @functools.lru_cache(maxsize=1)
  85. def _binary_supports_http_cache(binary: str) -> bool:
  86. """True if ``lightpanda serve`` accepts ``--http-cache-dir``.
  87. The flag landed upstream in 0.3.x; older binaries fatally reject it
  88. ("unknown argument"), which would break every launch. Probing ``help``
  89. output keeps working across future flag additions without parsing
  90. versions, and the lru_cache keeps it once per binary per process.
  91. """
  92. try:
  93. proc = subprocess.run(
  94. [binary, "help"],
  95. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=3.0,
  96. stdin=subprocess.DEVNULL,
  97. check=False,
  98. )
  99. return _HTTP_CACHE_FLAG in ((proc.stdout or "") + (proc.stderr or ""))
  100. except Exception as e:
  101. logger.debug("lightpanda http-cache probe failed (%s); assuming no", e)
  102. return False
  103. def _record_path(session_name: str) -> Path:
  104. return _state_dir() / f"{session_name}.json"
  105. def _browser_env() -> dict:
  106. from tools.browser_session import build_browser_env
  107. return build_browser_env()
  108. def _cdp_ready(url: str, timeout: float = 0.2) -> bool:
  109. try:
  110. from hermes_cli.browser_connect import is_browser_debug_ready
  111. return is_browser_debug_ready(url, timeout=timeout)
  112. except Exception as e:
  113. logger.debug("CDP readiness probe failed for %s: %s", url, e)
  114. return False
  115. def _read_log_tail(path: str) -> str:
  116. """Last non-empty line of the child's stderr log ('' when unreadable)."""
  117. try:
  118. text = Path(path).read_bytes()[-_STDERR_TAIL_LIMIT:].decode("utf-8", errors="replace")
  119. except OSError:
  120. return ""
  121. return next((ln for ln in reversed(text.strip().splitlines()) if ln.strip()), "")
  122. def _terminate(proc: subprocess.Popen, what: str = "lightpanda") -> None:
  123. """poll -> terminate -> wait(5) -> kill (fallback when the tree-kill failed)."""
  124. try:
  125. if proc.poll() is None:
  126. proc.terminate()
  127. try:
  128. proc.wait(timeout=5)
  129. except Exception:
  130. proc.kill()
  131. except Exception as e:
  132. logger.debug("%s terminate failed: %s", what, e)
  133. def _safe_start_time(pid: int) -> Optional[int]:
  134. try:
  135. from tools.process_registry import ProcessRegistry
  136. return ProcessRegistry._safe_host_start_time(pid)
  137. except Exception:
  138. return None
  139. def _tree_kill(pid: int, expected_start) -> None:
  140. """Tree-kill ``pid`` via ProcessRegistry, verifying its start time first."""
  141. from tools.process_registry import ProcessRegistry
  142. ProcessRegistry._terminate_host_pid(pid, expected_start=expected_start)
  143. def _write_record(server: LightpandaServer) -> None:
  144. record = {"pid": server.proc.pid, "port": server.port, "owner_pid": os.getpid(),
  145. "start_time": server.start_time, "started_at": time.time()}
  146. try:
  147. # Temp + rename: an orphan sweep (another Hermes, or a cleanup thread) unlinks a record it
  148. # cannot parse, so a torn in-place write would drop a live server's record.
  149. atomic_json_write(_state_dir() / f"{server.session_name}.json", record, indent=None)
  150. except OSError as e:
  151. logger.debug("could not write lightpanda record for %s: %s", server.session_name, e)
  152. def launch_lightpanda(session_name: str, *, block_private_networks: bool = False) -> tuple[Optional[LightpandaServer], Optional[str]]:
  153. """Start ``lightpanda serve`` on a free loopback port; ``(server, None)`` once ``/json/version`` answers,
  154. else ``(None, error)``. stderr goes to ``<state_dir>/<session>.log`` so a chatty child never blocks on a pipe."""
  155. binary = find_lightpanda_binary()
  156. if not binary:
  157. if os.name == "nt":
  158. return None, ("browser.engine is 'lightpanda' but Lightpanda has no Windows "
  159. "build. Set browser.engine to auto (or run Hermes under WSL2).")
  160. return None, ("browser.engine is 'lightpanda' but no lightpanda binary was found on PATH, ~/.lightpanda "
  161. f"or ~/.local/bin. {LIGHTPANDA_INSTALL_HINT}, or set browser.engine to auto.")
  162. port = _pick_free_loopback_port()
  163. argv = [binary, "serve", "--host", "127.0.0.1", "--port", str(port)]
  164. if _binary_supports_http_cache(binary):
  165. argv += [_HTTP_CACHE_FLAG, str(_http_cache_dir())]
  166. if block_private_networks:
  167. argv.append("--block-private-networks")
  168. log_path = str(_state_dir() / f"{session_name}.log")
  169. try:
  170. with open(log_path, "wb") as log_file:
  171. proc = subprocess.Popen(argv, stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL,
  172. # detached-spawn: ok — a crashed owner's server is reaped from its state record by the next Hermes (reap_orphaned_lightpanda); owning it is a follow-up with the launcher's Chromium
  173. stderr=log_file, env=_browser_env(), start_new_session=True)
  174. except (OSError, subprocess.SubprocessError) as e:
  175. return None, f"Failed to launch lightpanda serve ({binary}): {e}"
  176. url = f"http://127.0.0.1:{port}"
  177. deadline = time.monotonic() + _READY_TIMEOUT_S
  178. while True:
  179. rc = proc.poll()
  180. if rc is not None:
  181. tail = _read_log_tail(log_path)
  182. return None, f"lightpanda serve exited with code {rc} before {url}/json/version answered{f': {tail}' if tail else ''}"
  183. if _cdp_ready(url):
  184. break
  185. if time.monotonic() >= deadline:
  186. _terminate(proc)
  187. tail = _read_log_tail(log_path)
  188. detail = f" (last stderr line: {tail})" if tail else ""
  189. return None, f"lightpanda serve did not expose {url}/json/version within {int(_READY_TIMEOUT_S)}s{detail}"
  190. time.sleep(_POLL_INTERVAL_S)
  191. server = LightpandaServer(session_name, port, proc, log_path, _safe_start_time(proc.pid))
  192. _write_record(server)
  193. with _servers_lock:
  194. _servers[session_name] = server
  195. logger.info("Started lightpanda serve (pid %s, port %s) for session %s", proc.pid, port, session_name)
  196. return server, None
  197. def get_server(session_name: str) -> Optional[LightpandaServer]:
  198. with _servers_lock:
  199. return _servers.get(session_name)
  200. def stop_lightpanda(session_name: str) -> None:
  201. """Stop the server for ``session_name`` (tree-kill, plain terminate on failure) and drop its record."""
  202. with _servers_lock:
  203. server = _servers.pop(session_name, None)
  204. if server is not None and server.is_alive():
  205. try:
  206. _tree_kill(server.proc.pid, server.start_time)
  207. except Exception as e:
  208. logger.debug("lightpanda tree-kill failed for %s: %s", server.session_name, e)
  209. _terminate(server.proc)
  210. try:
  211. server.proc.wait(timeout=5)
  212. except Exception:
  213. _terminate(server.proc)
  214. try:
  215. (_state_dir() / f"{session_name}.json").unlink(missing_ok=True)
  216. except OSError as e:
  217. logger.debug("could not remove lightpanda record for %s: %s", session_name, e)
  218. if server is not None:
  219. logger.debug("Stopped lightpanda serve for session %s", session_name)
  220. def stop_all_lightpanda() -> None:
  221. """Stop every server this process started. Idempotent; safe from atexit."""
  222. with _servers_lock:
  223. names = list(_servers)
  224. for name in names:
  225. try:
  226. stop_lightpanda(name)
  227. except Exception as e:
  228. logger.debug("lightpanda stop failed for %s: %s", name, e)
  229. def _is_lightpanda_process(pid: int, port, start_time) -> bool:
  230. """True only when ``pid`` is verifiably the ``lightpanda serve`` we recorded."""
  231. try:
  232. import psutil
  233. proc = psutil.Process(pid)
  234. if "lightpanda" not in proc.name().lower():
  235. return False
  236. cmdline = proc.cmdline()
  237. if "serve" not in cmdline or str(port) not in cmdline:
  238. return False
  239. if start_time:
  240. from gateway.status import get_process_start_time
  241. return get_process_start_time(pid) == start_time
  242. except Exception:
  243. return False
  244. return True
  245. def reap_orphaned_lightpanda() -> int:
  246. """Kill ``lightpanda serve`` processes whose owning Hermes is gone; return the count. A live owner is
  247. never touched; a PID is only signalled after psutil confirms it is still ``lightpanda serve`` on the recorded port."""
  248. try:
  249. state_dir = _state_dir()
  250. except Exception as e:
  251. logger.debug("lightpanda state dir unavailable: %s", e)
  252. return 0
  253. try:
  254. from gateway.status import _pid_exists
  255. except Exception: # pragma: no cover - defensive
  256. return 0
  257. reaped = 0
  258. for record_path in sorted(state_dir.glob("*.json")):
  259. session_name = record_path.stem
  260. try:
  261. record = json.loads(record_path.read_text(encoding="utf-8-sig"))
  262. if not isinstance(record, dict):
  263. raise ValueError(f"expected a JSON object, got {type(record).__name__}")
  264. except (OSError, ValueError):
  265. record_path.unlink(missing_ok=True)
  266. continue
  267. owner_pid = record.get("owner_pid")
  268. if owner_pid == os.getpid():
  269. with _servers_lock:
  270. if session_name in _servers:
  271. continue
  272. elif owner_pid and _pid_exists(int(owner_pid)):
  273. continue
  274. pid = record.get("pid")
  275. if pid and _is_lightpanda_process(int(pid), record.get("port"), record.get("start_time")):
  276. try:
  277. _tree_kill(int(pid), record.get("start_time"))
  278. reaped += 1
  279. logger.info("Reaped orphaned lightpanda serve pid %s (session %s)", pid, session_name)
  280. except Exception as e:
  281. logger.debug("orphan lightpanda kill failed for pid %s: %s", pid, e)
  282. record_path.unlink(missing_ok=True)
  283. return reaped
  284. def using_lightpanda_engine() -> bool:
  285. """True when ``browser.engine`` is ``lightpanda``."""
  286. from tools.browser_session_cloud import get_browser_engine
  287. return get_browser_engine() == "lightpanda"
  288. def lightpanda_engine_status() -> tuple[bool, str]:
  289. """Whether ``browser.engine: lightpanda`` is actually in effect, and why.
  290. ``(False, "")`` when the engine isn't lightpanda; else the reason names the setting shadowing it, in
  291. session-creation precedence, with config-only gates (no network I/O) for ``/browser status`` and
  292. ``hermes doctor``."""
  293. if not using_lightpanda_engine():
  294. return False, ""
  295. from tools.browser_session_cdp import get_cdp_override_raw
  296. from tools.browser_session_cloud import get_cloud_provider, use_real_profile
  297. if get_cdp_override_raw():
  298. return False, "a CDP override is active (/browser connect or browser.cdp_url)"
  299. # Real profile before the cloud provider: browser_exec resolves it first, so it claims the session.
  300. if use_real_profile():
  301. return False, "browser.use_real_profile is on (Lightpanda cannot load a Chromium profile)"
  302. try:
  303. provider = get_cloud_provider()
  304. except ValueError: # an unknown explicit provider name: browser_exec reports it on use
  305. provider = None
  306. if provider is not None:
  307. name = getattr(provider, "display_name", "") or type(provider).__name__
  308. return False, f"cloud provider {name} is selected (browser.cloud_provider, or auto-detected from credentials)"
  309. from tools.browser_use_cli import _read_browser_cfg, is_legacy_browser_use_cloud_config
  310. if is_legacy_browser_use_cloud_config(_read_browser_cfg()):
  311. return False, "Browser Use cloud (BROWSER_USE_API_KEY) is selected"
  312. return True, "Hermes spawns `lightpanda serve` per browser session"