browser_use_cli.py 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043
  1. """``browser_exec``: drive a real browser through the Browser Use CLI engine (browser-harness,
  2. https://browser-use.com), Hermes' only core browser tool.
  3. The harness speaks CDP to whatever ``tools.browser_session`` resolved for the call: an operator
  4. ``BU_CDP_*`` endpoint, a ``/browser connect`` override, a cloud provider's session, or a local browser
  5. (real profile, Lightpanda, or the packaged Chromium started by ``tools.browser_launcher``).
  6. """
  7. import contextlib
  8. import contextvars
  9. import functools
  10. import importlib
  11. import importlib.util
  12. import json
  13. import logging
  14. import os
  15. import re
  16. import signal
  17. import subprocess
  18. import sys
  19. import threading
  20. import time
  21. from pathlib import Path
  22. from typing import Any, Callable, Dict, List, NamedTuple, Optional
  23. from hermes_cli.process_identity import (
  24. own_process_groups, register_child, release_process_group_once_empty, spawn_env, spawned_by_this_process)
  25. from hermes_constants import get_hermes_home, hermes_home_key
  26. from tools.browser_harness_dirs import (daemon_env, daemon_identity, fit_default_socket_dir, harness_pid_file,
  27. namespace_harness_dirs)
  28. from utils import is_truthy_value
  29. logger = logging.getLogger(__name__)
  30. _BROWSER_USE_PROVIDER = "browser-use"
  31. # The one credential the harness itself reads (BU_AUTOSPAWN of a Browser Use cloud browser).
  32. _HARNESS_PASSTHROUGH_KEYS: tuple[str, ...] = ("BROWSER_USE_API_KEY",)
  33. # Cloud daemon names become the BU_NAME env var
  34. _SESSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
  35. # Set on the env dict by the CDP resolvers when the resolved browser is EXCLUSIVE to this named session
  36. # (per-name provider / named BU cloud / Lightpanda). Popped before the subprocess launches — never exported.
  37. _PRIVATE_BROWSER_SENTINEL = "_HERMES_BU_PRIVATE_BROWSER"
  38. # Internal route provenance: this exec resolved to a browser on the Bot Desktop display and must use
  39. # the same human-control lease fence as the built-in browser tools. Popped before launching the CLI.
  40. _BOT_DESKTOP_BROWSER_SENTINEL = "_HERMES_BU_BOT_DESKTOP_BROWSER"
  41. # The note when the consented real profile could not load (the route fell back). Popped before launch.
  42. _REAL_PROFILE_FALLBACK_SENTINEL = "_HERMES_BU_REAL_PROFILE_FALLBACK"
  43. # local=True whose real profile could not load: run on the local sidecar browser. Popped before launch.
  44. _LOCAL_FALLBACK_SENTINEL = "_HERMES_BU_LOCAL_FALLBACK"
  45. # Where new/hidden tabs may be brought to the front: a browser_use_shot.ACTIVATE_* mode. Popped before launch.
  46. _CLOUD_BROWSER_SENTINEL = "_HERMES_BU_CLOUD_BROWSER"
  47. # Prepended to the model's code for named sessions on SHARED browsers (a /browser connect CDP override): the
  48. # harness daemon attaches to the first existing page at startup, so two fresh named daemons can land on the
  49. # SAME tab. Steering each onto a tab it created prevents clobbering. Runs once per daemon (marker keyed by
  50. # BU_NAME + daemon pid).
  51. _OWN_TAB_PREAMBLE = """\
  52. # hermes: pin this named session to its own tab (once per daemon process)
  53. def _hermes_ensure_own_tab():
  54. import os as _os, tempfile as _tf
  55. _name = _os.environ.get("BU_NAME", "default")
  56. try:
  57. # Key the marker by the daemon's pid so a daemon restart (which
  58. # re-attaches to the first shared page) re-pins automatically,
  59. # while agent-driven tab switches mid-session are left alone.
  60. from browser_harness import _ipc as _bipc
  61. _dpid = _bipc.pid_path(_name).read_text().strip() or "0"
  62. except Exception:
  63. _dpid = "0"
  64. _uid = _os.getuid() if hasattr(_os, "getuid") else 0
  65. _marker = _os.path.join(
  66. _tf.gettempdir(), "hermes-bu-owntab-%s-%s-%s" % (_uid, _name, _dpid)
  67. )
  68. if _os.path.exists(_marker):
  69. return
  70. try:
  71. # Force a fresh target: new_tab() would REUSE a blank current tab,
  72. # which is exactly the tab a sibling daemon may also hold.
  73. _tid = cdp("Target.createTarget", url="about:blank").get("targetId")
  74. if _tid:
  75. switch_tab(_tid)
  76. except Exception:
  77. pass # best-effort: worst case is pre-fix behavior
  78. try:
  79. open(_marker, "w").close()
  80. except OSError:
  81. pass
  82. _hermes_ensure_own_tab()
  83. del _hermes_ensure_own_tab
  84. """
  85. _DEFAULT_TIMEOUT_S = 300
  86. _MIN_TIMEOUT_S = 5
  87. _MAX_TIMEOUT_S = 1800
  88. _STDERR_CAP_CHARS = 4000
  89. _TASK_ID_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+") # filesystem-safe task ids
  90. # Screenshot paths printed by capture_screenshot(): POSIX or Windows drive-letter absolute.
  91. _IMAGE_PATH_RE = re.compile(r"((?:[A-Za-z]:[\\/]|/)[^\s\"']+?\.(?:png|jpe?g|webp))", re.IGNORECASE)
  92. # http(s) URL literals in exec code, checked against the browser URL policy
  93. _URL_RE = re.compile(r"https?://[^\s'\"\\)]+", re.IGNORECASE)
  94. _FHS_BIN_DIRS = ("/opt/homebrew/bin", "/opt/homebrew/sbin", "/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
  95. "/usr/bin", "/sbin", "/bin")
  96. def _quiet(fn: Callable[[], Any], default: Any, log_prefix: str = "") -> Any:
  97. """``fn()``, or ``default`` on any exception (debug-logged when ``log_prefix`` is set)."""
  98. try:
  99. return fn()
  100. except Exception as e:
  101. if log_prefix:
  102. logger.debug("%s: %s", log_prefix, e)
  103. return default
  104. def _lazy_call(module: str, name: str, default: Any, log_prefix: str) -> Any:
  105. """Call ``module.name()`` resolved at call time (tests stub the module / patch the
  106. attribute); on any failure log ``log_prefix`` and return ``default``."""
  107. return _quiet(lambda: getattr(importlib.import_module(module), name)(), default, log_prefix)
  108. def _real_profile_consented() -> bool:
  109. return _lazy_call("tools.browser_session_cloud", "use_real_profile", False, "real-profile consent lookup failed")
  110. def _set_cdp_env(env: dict, cdp: str) -> None:
  111. """Export a CDP endpoint under the BU_CDP_* contract (http(s) → URL, else WS)."""
  112. env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp
  113. def _has_cdp_env(env: dict) -> bool:
  114. return bool(env.get("BU_CDP_WS") or env.get("BU_CDP_URL"))
  115. def _export_session_cdp(env: dict, get_session_info: Callable[[str], Any], cache_key: str,
  116. fail_msg: Callable[[Exception], str], no_cdp_msg: str) -> Optional[str]:
  117. """Export the CDP endpoint from ``get_session_info(cache_key)``; error string on failure / no CDP."""
  118. try:
  119. cdp = str((get_session_info(cache_key) or {}).get("cdp_url") or "")
  120. except Exception as e:
  121. return fail_msg(e)
  122. if not cdp:
  123. return no_cdp_msg
  124. _set_cdp_env(env, cdp)
  125. return None
  126. class _CodeUrlVerdict(NamedTuple):
  127. """What one pass over a code blob's URL literals decided (see ``_code_url_verdict``)."""
  128. blocked: Optional[str]
  129. local_sidecar: bool
  130. def _code_url_verdict(code: str) -> _CodeUrlVerdict:
  131. """One pass over the code's URL literals: the blocking error, if any, and whether the call must run on
  132. the hybrid LOCAL sidecar. Each URL's routing predicate is evaluated ONCE and feeds both answers: it
  133. resolves DNS unmemoised, so a rebinding host evaluated twice could relax the private-address floor and
  134. still route to the cloud provider."""
  135. from tools.browser_url_policy import evaluate_url_safety, hybrid_routes_locally
  136. local_sidecar = False
  137. for url in _URL_RE.findall(code or ""):
  138. routes_local = hybrid_routes_locally(url)
  139. local_sidecar = local_sidecar or routes_local
  140. err = evaluate_url_safety(url, auto_local=routes_local)
  141. if err:
  142. return _CodeUrlVerdict(err.get("error", "Blocked: unsafe URL"), local_sidecar)
  143. return _CodeUrlVerdict(None, local_sidecar)
  144. def _harness_credentials() -> dict:
  145. """The harness's own credential from the active profile's secret scope, never ``os.environ``: under
  146. multiplex that holds the LAUNCH profile's key, and a served profile's browser runs on its own (or none)."""
  147. from agent.secret_scope import current_secret_scope, get_secret, serves_routed_profile
  148. routed = serves_routed_profile()
  149. scope = (current_secret_scope() or {}) if routed else None
  150. values = {key: (scope.get(key) if routed else get_secret(key)) for key in _HARNESS_PASSTHROUGH_KEYS}
  151. return {key: value for key, value in values.items() if value is not None}
  152. def _base_subprocess_env() -> dict:
  153. from tools.browser_session import build_browser_env
  154. env = build_browser_env()
  155. env.update(_harness_credentials())
  156. # The harness runs on Hermes's own interpreter, but a bundled Desktop install boots that
  157. # interpreter with its site dir on PYTHONPATH (no venv to activate), and the harness's daemon
  158. # re-runs sys.executable. Point PYTHONPATH at the harness's site dir, replacing whatever the
  159. # agent process inherited, so both the CLI and its daemon import the same packages.
  160. env.pop("PYTHONHOME", None)
  161. site_dir = _harness_site_dir()
  162. if site_dir:
  163. env["PYTHONPATH"] = site_dir
  164. else:
  165. env.pop("PYTHONPATH", None)
  166. env["PATH"] = _floor_subprocess_path(env.get("PATH", ""))
  167. env.setdefault("ANONYMIZED_TELEMETRY", "false")
  168. namespace_harness_dirs(env, _served_profile_tag()) # a served profile's own harness runtime/tmp dirs
  169. # A daemon the CLI starts inherits this tag; _own_harness_daemon owns only one that names us.
  170. env.update(spawn_env("browser-harness-daemon"))
  171. return env
  172. def _floor_subprocess_path(path: str) -> str:
  173. """Keep system commands reachable from profile workers with a minimal PATH.
  174. Browser subprocesses may launch shell helpers; version-manager-only paths
  175. omit /usr/bin. Reuse the shared browser PATH floor on POSIX.
  176. """
  177. if os.name == "nt":
  178. return path
  179. parts = [p for p in (path or "").split(os.pathsep) if p]
  180. return os.pathsep.join(parts + [d for d in _FHS_BIN_DIRS if d not in set(parts) and os.path.isdir(d)])
  181. def _read_browser_cfg() -> dict:
  182. """Return the ``browser:`` config section, or {} on any failure."""
  183. try:
  184. from tools.browser_session_cloud import browser_section
  185. return browser_section()
  186. except Exception as e:
  187. logger.debug("Could not read browser config section: %s", e)
  188. return {}
  189. def _use_gateway(browser_cfg: dict) -> bool:
  190. """True when the browser section selects the Nous Tool Gateway — by the current ``hermes tools``
  191. picker row (``cloud_provider: nous``) or the pre-picker ``use_gateway: true`` flag. Reading only
  192. the legacy flag missed every picker-configured gateway, and the direct-API branch it fell into
  193. holds no credentials in managed mode (#108310)."""
  194. if is_truthy_value(browser_cfg.get("use_gateway"), default=False):
  195. return True
  196. try:
  197. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER
  198. except Exception: # pragma: no cover — helper ships with the package
  199. return False
  200. return str(browser_cfg.get("cloud_provider") or "").strip().lower() == NOUS_MANAGED_PROVIDER
  201. def is_legacy_browser_use_cloud_config(browser_cfg: dict) -> bool:
  202. """True for pre-CLI direct-API Browser Use cloud configs: no cloud_provider (or ``browser-use``), not the
  203. Nous gateway, and BROWSER_USE_API_KEY set; the CLI then reaches Browser Use cloud natively."""
  204. if not isinstance(browser_cfg, dict):
  205. return False
  206. provider = str(browser_cfg.get("cloud_provider") or "").strip().lower()
  207. if provider not in {"browser-use", ""} or _use_gateway(browser_cfg):
  208. return False
  209. return _browser_use_api_key_set()
  210. def _browser_use_api_key_set() -> bool:
  211. """BROWSER_USE_API_KEY for the served profile: the only credential Hermes autospawns BU cloud with.
  212. Profile-scoped so a multiplexed secondary never inherits the default's cloud mode."""
  213. from agent.secret_scope import get_secret
  214. return bool(get_secret("BROWSER_USE_API_KEY", ""))
  215. def _harness_site_dir() -> Optional[str]:
  216. """The site dir Hermes's interpreter imports ``browser_harness`` from, or None."""
  217. spec = importlib.util.find_spec("browser_harness")
  218. if spec is None or not spec.origin:
  219. return None
  220. return str(Path(spec.origin).resolve().parent.parent)
  221. def _find_cli() -> Optional[list[str]]:
  222. """The Browser Use CLI's engine (browser-harness) is a core dependency of Hermes's own venv,
  223. so every install, the Desktop bundle included, runs it on the current interpreter."""
  224. if _harness_site_dir() is None:
  225. return None
  226. return [sys.executable, "-m", "browser_harness.run"]
  227. def browser_exec_available() -> bool:
  228. """check_fn: browser-harness is importable (a core dependency). Never launches or opens sockets."""
  229. return _find_cli() is not None
  230. def _workspace_dir(task_id: Optional[str]) -> Optional[str]:
  231. """Stable per-task scratch dir that persists across browser_exec calls"""
  232. if os.environ.get("BH_AGENT_WORKSPACE"):
  233. return os.environ["BH_AGENT_WORKSPACE"]
  234. try:
  235. safe = _TASK_ID_SAFE_RE.sub("_", str(task_id or "default"))[:80] or "default"
  236. path = Path(get_hermes_home()) / "cache" / "browser-use" / "workspace" / safe
  237. path.mkdir(parents=True, exist_ok=True)
  238. return str(path)
  239. except Exception as e:
  240. logger.debug("browser_exec workspace unavailable: %s", e)
  241. return None
  242. def _find_screenshot(stdout: str, since: float) -> Optional[str]:
  243. """Last screenshot path printed during this exec that exists and was written after
  244. the exec started, or None."""
  245. for path in reversed(_IMAGE_PATH_RE.findall(stdout or "")):
  246. try:
  247. if os.path.isfile(path) and os.path.getmtime(path) >= since - 1:
  248. return path
  249. except OSError:
  250. continue
  251. return None
  252. def _native_screenshot_result(result: dict[str, Any], path: str) -> Optional[dict[str, Any]]:
  253. """Build a multimodal tool result attaching path for vision models"""
  254. try:
  255. from tools.vision_tools import (_EMBED_MAX_DIMENSION,
  256. _resize_image_for_vision, _should_use_native_vision_fast_path)
  257. from tools.vision_tools_history_budget import resolve_embed_target_bytes
  258. if not _should_use_native_vision_fast_path():
  259. return None
  260. # History-reuse cap: this data URL bakes into the tool result and is re-sent every later turn —
  261. # same policy as the vision_analyze native embed.
  262. data_url = _resize_image_for_vision(Path(path), mime_type="image/png",
  263. max_base64_bytes=resolve_embed_target_bytes(),
  264. max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
  265. text = json.dumps(result, ensure_ascii=False)
  266. attached = text + "\n\nThe screenshot from this call is attached — inspect it with your native vision."
  267. return {"_multimodal": True, "text_summary": text, "meta": {"screenshot_path": path, "native_vision": True},
  268. "content": [{"type": "text", "text": attached}, {"type": "image_url", "image_url": {"url": data_url}}]}
  269. except Exception as e:
  270. logger.debug("Native screenshot attach failed (falling back to text): %s", e)
  271. return None
  272. def _served_profile_tag() -> str:
  273. """``""`` outside a served-profile scope (every legacy key stays byte-identical); under a
  274. multiplexed turn, the routed profile's home key — one profile's browser must never be handed
  275. to another that happens to use the same session name or task id (#110032)."""
  276. from hermes_constants import get_hermes_home_override, hermes_home_key
  277. return "" if get_hermes_home_override() is None else hermes_home_key()
  278. def _backend_cache_key(task_id: Optional[str], session_name: str = "", sidecar: bool = False) -> str:
  279. """Session-cache key for a backend browser: named sessions get their own; served profiles get their own.
  280. ``sidecar`` appends the hybrid ``::local`` suffix, which makes ``_get_session_info`` build local Chromium
  281. even when a cloud provider is configured."""
  282. key = f"bu-named-{session_name}" if session_name else (task_id or "browser-exec-default")
  283. tag = _served_profile_tag()
  284. key = f"{key}@{tag}" if tag else key
  285. return f"{key}::local" if sidecar else key
  286. def _sidecar_daemon_name(name: str) -> str:
  287. """BU_NAME of the harness daemon serving the hybrid local sidecar. The daemon reads BU_CDP_* once at
  288. spawn and outlives the call, so sharing the cloud route's daemon would relay a private URL to the
  289. cloud browser it is already attached to. User session names cannot start with ``_``."""
  290. return f"_local_{name or 'default'}"[:64]
  291. def _export_real_profile_note(env: dict, key: str) -> None:
  292. """The session layer decides real profile vs fresh browser once per key; carry its note (if it fell
  293. back) so the result reports the browser this call actually drives."""
  294. from tools.browser_session import real_profile_note
  295. note = real_profile_note(key)
  296. if note:
  297. env.setdefault(_REAL_PROFILE_FALLBACK_SENTINEL, note)
  298. def _mark_local_browser(env: dict, session: dict, key: str) -> None:
  299. """Flags for a LOCAL session browser: it runs on the Bot Desktop display (lease fence), and unless a
  300. persistent profile / the real-profile browser is shared by every key, it is exclusive to this key (no
  301. own-tab preamble: nothing to share a tab with)."""
  302. features = session.get("features") or {}
  303. env[_BOT_DESKTOP_BROWSER_SENTINEL] = "lightpanda" if features.get("lightpanda") else "local"
  304. if not session.get("shared_profile") and not features.get("real_profile"):
  305. env[_PRIVATE_BROWSER_SENTINEL] = "1"
  306. _export_real_profile_note(env, key)
  307. def _resolve_local_cdp(env: dict, task_id: Optional[str], session_name: str = "",
  308. sidecar: bool = False) -> Optional[str]:
  309. """Local browser for this cache key through the session layer: the consented real profile, Lightpanda
  310. (``browser.engine: lightpanda``), or the packaged Chromium. Never the harness's own discovery of the
  311. user's installed Chrome (chrome://inspect toggle + Allow popup per run, blocked on Chrome >=136)."""
  312. from tools.browser_lightpanda import using_lightpanda_engine
  313. from tools.browser_session import get_session_info
  314. key = _backend_cache_key(task_id, session_name, sidecar)
  315. session: dict = {}
  316. def _session_info(cache_key: str) -> dict:
  317. session.update(get_session_info(cache_key) or {})
  318. return session
  319. if _quiet(using_lightpanda_engine, False):
  320. fail = lambda e: (f"Lightpanda could not be started: {e} Set browser.engine to auto "
  321. "to use local Chrome, or switch backends via `hermes tools` → Browser Automation.")
  322. else:
  323. fail = lambda e: (f"The local browser could not be started: {e} "
  324. "Run `hermes tools` → Browser Automation to (re)install Chromium, or switch backends.")
  325. err = _export_session_cdp(env, _session_info, key, fail, "The local browser session returned no CDP endpoint.")
  326. if err is None:
  327. _mark_local_browser(env, session, key)
  328. return err
  329. def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str = "",
  330. force_local_sidecar: bool = False) -> Optional[str]:
  331. """Point the harness at the configured backend's CDP endpoint; error string on failure.
  332. Precedence: (1) ``BU_CDP_WS``/``BU_CDP_URL`` already in env (operator override); (2) ``BROWSER_CDP_URL``
  333. env / ``browser.cdp_url`` (``/browser connect``); (3) a cloud provider via ``browser_session.get_session_info``
  334. (per-key cache, expiry, reaper, atexit; a provider failure falls back to a local browser);
  335. (4) the local engine — ``browser.engine: lightpanda`` or Hermes' packaged Chromium via the launcher
  336. (never the harness's own discovery of the user's installed Chrome); (5) BU direct-API configs WITH
  337. ``BROWSER_USE_API_KEY`` → None (checked before (3)): the CLI reaches BU cloud natively (BU_AUTOSPAWN); a
  338. keyless ``browser-use`` provider (managed-gateway autodetect, or a selection missing its key) takes (3).
  339. ``session_name`` (BU_NAME) keys the session cache so each name gets its OWN browser — what makes named
  340. sessions concurrent-safe.
  341. ``force_local_sidecar`` skips step (3): the call runs on the local engine under the ``::local`` cache
  342. key (hybrid routing; see ``browser_url_policy.hybrid_routes_locally``). It sits BELOW the two operator overrides, and the
  343. sidecar verdict never fires while a CDP override is set."""
  344. if _has_cdp_env(env):
  345. return None
  346. from tools.browser_session import get_session_info
  347. from tools.browser_session_cdp import get_cdp_override
  348. from tools.browser_session_cloud import get_cloud_provider
  349. override = _quiet(get_cdp_override, "")
  350. if override:
  351. _set_cdp_env(env, override)
  352. return None
  353. if force_local_sidecar:
  354. return _resolve_local_cdp(env, task_id, session_name, sidecar=True)
  355. provider = _quiet(get_cloud_provider, None, "Cloud provider lookup failed")
  356. if provider is None:
  357. return _resolve_local_cdp(env, task_id, session_name)
  358. # Browser Use direct-API configs: the CLI talks to BU cloud natively (BU_AUTOSPAWN) — the
  359. # legacy provider would create a second, redundant session. Nous-gateway configs (cloud_provider: nous
  360. # from the picker, or the pre-picker use_gateway: true) DO resolve through the provider: the gateway
  361. # provisions the browser server-side and returns its CDP URL. Without BROWSER_USE_API_KEY the CLI cannot
  362. # autospawn, and the provider is either the managed gateway (autodetect on a never-configured install) or
  363. # a selection missing its key; both go through the session path (gateway CDP, or the session layer's
  364. # local-Chromium fallback) instead of leaving the harness to hunt for an installed Chrome.
  365. provider_key = str(getattr(provider, "name", "") or "").strip().lower()
  366. if provider_key == _BROWSER_USE_PROVIDER and not _use_gateway(_read_browser_cfg()) and _browser_use_api_key_set():
  367. env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon
  368. # Only intent: the harness attaches to a local Chrome already listening instead of autospawning, so
  369. # the live daemon's own report decides.
  370. from tools.browser_use_shot import ACTIVATE_IF_CLOUD
  371. env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_IF_CLOUD
  372. return None
  373. provider_name = type(provider).__name__
  374. session: dict = {}
  375. def _session_info(key: str) -> dict:
  376. session.update(get_session_info(key) or {})
  377. return session
  378. key = _backend_cache_key(task_id, session_name)
  379. err = _export_session_cdp(
  380. env, _session_info, key,
  381. lambda e: (f"Cloud browser provider {provider_name} failed to provide a session: {e}. "
  382. "Fix the provider configuration or switch backends via `hermes tools` → Browser Automation."),
  383. f"Cloud browser provider {provider_name} returned no CDP endpoint, so browser_exec "
  384. "cannot drive it. Pick a CDP-capable provider via `hermes tools` → Browser Automation.",
  385. )
  386. if session.get("fallback_from_cloud"):
  387. # The provider failed (rate limit, quota, outage) and the session layer fell back to a LOCAL
  388. # browser for this key; it already carries its CDP URL.
  389. logger.warning("browser_exec: %s failed (%s); using local Chromium for this session",
  390. provider_name, session.get("fallback_reason"))
  391. if err is None:
  392. _mark_local_browser(env, session, key)
  393. return err
  394. if err is None:
  395. from tools.browser_use_shot import ACTIVATE_ALWAYS
  396. env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_ALWAYS # the provider's own session: the harness connects only there
  397. # A provider browser keyed bu-named-<name> is exclusive to this session — the
  398. # own-tab preamble would just leak a blank tab into it.
  399. if err is None and session_name:
  400. env[_PRIVATE_BROWSER_SENTINEL] = "1"
  401. return err
  402. def _resolve_real_profile_cdp(env: dict, force_local: bool) -> Optional[str]:
  403. """Point the harness at the user's real-profile copy-browser (a SNAPSHOT of their default Chromium
  404. profile, hermes_cli.browser_connect) when consented. Two ways in: the effective backend is already local
  405. (no provider, CDP override, or legacy BU cloud config) → silent upgrade; or ``force_local`` (consent-gated
  406. ``local`` arg) → the user's browser even under a cloud backend. Operator overrides (BU_CDP_* env,
  407. /browser connect, ``browser.cdp_url``) own the session either way. A launch error is returned;
  408. ``_route_backend`` then falls back to the normal backend and reports it, never silently."""
  409. if not _real_profile_consented() or _has_cdp_env(env):
  410. return None
  411. from tools.browser_session_cdp import get_cdp_override_raw
  412. from tools.browser_session_cloud import get_cloud_provider
  413. from tools.browser_session_real_profile import real_profile_cdp
  414. if _quiet(get_cdp_override_raw, ""):
  415. return None
  416. # Only auto-upgrade genuinely-local attaches; any cloud path (provider, provider lookup failure, or
  417. # legacy BU cloud config) stays on its backend unless the model passes local=true.
  418. if not force_local and (_quiet(get_cloud_provider, object()) is not None
  419. or is_legacy_browser_use_cloud_config(_read_browser_cfg())):
  420. return None
  421. cdp, err = real_profile_cdp()
  422. if cdp and not err:
  423. _set_cdp_env(env, cdp)
  424. env[_BOT_DESKTOP_BROWSER_SENTINEL] = "local" # the user's real Chromium, whatever browser.engine says
  425. return err or None
  426. def _attach_vault_supervisor(env: dict, task_id: Optional[str]) -> None:
  427. """Attach the per-task CDP supervisor to the browser this exec drives so ``browser_vault`` fill has
  428. a secret-capable WebSocket (never argv) into the SAME browser. Only CDP-routed backends expose an
  429. endpoint; BU direct-cloud (BU_AUTOSPAWN) does not, and the vault tools report ``supervisor_required``."""
  430. cdp = env.get("BU_CDP_WS") or env.get("BU_CDP_URL")
  431. if not cdp:
  432. return
  433. try:
  434. from tools.browser_session_cdp import get_dialog_policy_config, resolve_cdp_override
  435. from tools.browser_supervisor import SUPERVISOR_REGISTRY
  436. policy, timeout_s = get_dialog_policy_config()
  437. SUPERVISOR_REGISTRY.get_or_start(task_id=task_id or "default", cdp_url=resolve_cdp_override(cdp),
  438. dialog_policy=policy, dialog_timeout_s=timeout_s)
  439. except Exception as exc:
  440. logger.debug("browser_exec: CDP supervisor attach failed (non-fatal): %s", exc)
  441. def _route_backend(env: dict, session: str, task_id: Optional[str], local: bool,
  442. force_local_sidecar: bool = False, real_profile: bool = True) -> Optional[str]:
  443. """Resolve where the harness connects; returns an error string or None. Real-profile consent runs
  444. BEFORE provider resolution so a hit short-circuits the cloud path via the BU_CDP_* env contract. Named
  445. sessions compose with the backend: BU_NAME namespaces the harness daemon (IPC socket, log, pid) and on
  446. provider backends additionally keys its own cloud browser.
  447. ``force_local_sidecar`` (a call carrying a private URL) declines the cloud provider for the hybrid local
  448. sidecar; unlike ``local`` (the consent-gated real-profile browser) it never touches the user's profile."""
  449. # ``real_profile=False``: this route already fell back, so it stays on its fresh browser instead
  450. # of swapping browsers mid-task once the profile frees up.
  451. rp_err = _resolve_real_profile_cdp(env, force_local=local) if real_profile else None
  452. # local=True is only served by the real-profile route; consent off must not pretend.
  453. if local and not _has_cdp_env(env) and not _real_profile_consented():
  454. return ("local=true was requested but browser.use_real_profile is off. Enable it in config.yaml "
  455. "(browser.use_real_profile: true) or the desktop Settings → Browser section, then retry.")
  456. # local=True promised the user's own machine: a profile that can't load runs on a fresh LOCAL browser
  457. # (the sidecar, which never loads the profile), never the cloud one. Without ``local`` the session
  458. # layer makes the one real-profile-or-fresh decision and reports it (_resolve_local_cdp).
  459. if local and not _has_cdp_env(env) and (rp_err or not real_profile):
  460. if rp_err:
  461. from tools.browser_session_real_profile import real_profile_fallback_note
  462. env[_REAL_PROFILE_FALLBACK_SENTINEL] = real_profile_fallback_note(rp_err)
  463. env[_LOCAL_FALLBACK_SENTINEL] = "1"
  464. force_local_sidecar = True
  465. return _resolve_backend_cdp(env, task_id, session_name=session,
  466. force_local_sidecar=force_local_sidecar)
  467. def _group_popen_kwargs() -> dict:
  468. """Popen kwargs starting the CLI in its own process group (a new session on POSIX) so a
  469. timeout can take down every process that inherited the capture pipes, not just the CLI
  470. child. Windows also hides the console window."""
  471. def _flags() -> dict:
  472. from hermes_cli._subprocess_compat import windows_hide_flags
  473. si = subprocess.STARTUPINFO()
  474. si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
  475. return {"creationflags": windows_hide_flags() | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0),
  476. "startupinfo": si}
  477. # detached-spawn: ok — awaited CLI; a timeout kills its whole group (_run_cli_killing_process_group)
  478. return _quiet(_flags, {}, "Windows hide-flags unavailable") if os.name == "nt" else {"start_new_session": True}
  479. def _clamp_timeout(timeout_s: Any) -> int:
  480. try:
  481. return max(_MIN_TIMEOUT_S, min(int(timeout_s), _MAX_TIMEOUT_S))
  482. except (TypeError, ValueError):
  483. return _DEFAULT_TIMEOUT_S
  484. # After a whole-group SIGKILL, every pipe holder is dead, so the drain below is normally
  485. # instant; the deadline only guards against a process outside the group still holding a pipe.
  486. _POST_KILL_DRAIN_S = 10.0
  487. def _kill_cli_process_group(proc) -> None:
  488. """SIGKILL the CLI's whole process group (POSIX; ``start_new_session`` made pgid == pid) or,
  489. on Windows, its process tree via ``taskkill /T /F`` — the only group-wide kill it offers."""
  490. if os.name == "nt":
  491. from hermes_cli._subprocess_compat import windows_hide_flags
  492. with contextlib.suppress(OSError, subprocess.SubprocessError):
  493. subprocess.run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], stdin=subprocess.DEVNULL,
  494. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
  495. check=False, creationflags=windows_hide_flags())
  496. return
  497. with contextlib.suppress(ProcessLookupError, PermissionError):
  498. os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — POSIX only, the nt branch returned above
  499. def _run_cli_killing_process_group(cmd, code, env, timeout):
  500. """Run the CLI in its own process group and kill the whole group on timeout.
  501. ``subprocess.run`` only kills the direct child on ``TimeoutExpired``; a grandchild that
  502. inherited the stdout/stderr pipes (browser_harness daemon / Chrome helper) is orphaned
  503. still holding them, and on Windows ``run()``'s unbounded post-kill ``communicate()`` then
  504. blocks on pipe EOF forever — so the tool call, plus its activity heartbeat, wedges (#106244).
  505. """
  506. proc = subprocess.Popen(
  507. cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
  508. text=True, encoding="utf-8", errors="replace", env=env, **_group_popen_kwargs(),
  509. )
  510. try:
  511. stdout, stderr = proc.communicate(input=code, timeout=timeout)
  512. except subprocess.TimeoutExpired:
  513. _kill_cli_process_group(proc)
  514. with contextlib.suppress(subprocess.TimeoutExpired):
  515. proc.communicate(timeout=_POST_KILL_DRAIN_S)
  516. raise
  517. return subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
  518. # How often a running browser_exec re-checks for the daemon its first call starts.
  519. _OWN_DURING_CALL_S = 0.5
  520. # Harness daemons this process drove: each latches BU_CDP_* at start and outlives every call, so a backend
  521. # swap (/browser connect|disconnect) leaves later calls in the old browser until these are stopped.
  522. _driven_daemons: dict[tuple[str, ...], str] = {} # daemon identity -> home
  523. _driven_daemons_lock = threading.Lock()
  524. _live_generation = functools.partial(_lazy_call, "tools.browser_session_cdp", "live_cdp_generation", 0,
  525. "live CDP generation lookup failed") # this profile's publish count
  526. # Route cache key -> whether its last browser_exec ran on the hybrid local sidecar.
  527. _exec_route_is_sidecar: dict[str, bool] = {}
  528. # Route cache key -> the note from its real-profile fallback; that route stays on the fallback browser.
  529. _real_profile_fallback: dict[str, str] = {}
  530. _real_profile_note_delivered: set = set()
  531. def stop_harness_daemons(home_key: Optional[str] = None) -> None:
  532. """Stop every harness daemon this process drove (only ``home_key``'s when given), through the
  533. harness's own identity-checked ``--reload``; the next browser_exec respawns one on the endpoint
  534. it resolves then."""
  535. with _driven_daemons_lock:
  536. daemons = sorted(key for key, home in _driven_daemons.items() if home_key in (None, home))
  537. for key in daemons:
  538. _driven_daemons.pop(key, None)
  539. _reload_daemons(daemons)
  540. def _published_daemon_pid(env: dict) -> Optional[int]:
  541. """The pid the harness published for the daemon ``env`` drives (``harness_pid_file``: JSON
  542. ``{"pid", "started"}`` or a bare pid), or None when there is none."""
  543. try:
  544. raw = harness_pid_file(env).read_text(encoding="utf-8-sig").strip()
  545. return int(json.loads(raw)["pid"]) if raw.startswith("{") else int(raw.split()[0])
  546. except (OSError, ValueError, KeyError, TypeError, IndexError):
  547. return None
  548. def _own_harness_daemon_until_owned(env: dict) -> Optional[bool]:
  549. """Periodic check while browser_exec runs: own the daemon as soon as it publishes its pid, then
  550. stop (False). It also stops once the daemon can never be ours (Windows, a managed worker, a daemon
  551. another Hermes started), instead of re-checking every tick for the whole call."""
  552. return None if _own_harness_daemon(env) is None else False
  553. def _reload_daemons(daemons) -> None: # each ``daemon_identity`` daemon, with its own dirs
  554. cmd = _find_cli() if daemons else None
  555. if not cmd:
  556. return
  557. env = _base_subprocess_env()
  558. for daemon in daemons:
  559. with contextlib.suppress(OSError, subprocess.SubprocessError):
  560. _run_cli_killing_process_group([*cmd, "--reload"], "", daemon_env(env, daemon), 15)
  561. with _owned_daemon_groups_lock:
  562. pgid = _owned_daemon_groups.pop(daemon, None)
  563. if pgid is not None:
  564. release_process_group_once_empty(pgid) # a daemon --reload failed to stop stays covered
  565. # Daemon identity (``daemon_identity``: its dirs + BU_NAME) -> process group of the harness daemon this
  566. # process owns. The harness starts its daemon with start_new_session so it outlives each CLI call (that
  567. # is how a session persists across browser_exec calls), which also means it outlives a SIGKILLed or
  568. # OOM-killed Hermes forever (#121095). Its group goes to this process's owner-death supervisor, so it
  569. # dies with us however we die.
  570. _owned_daemon_groups: dict[tuple[str, ...], int] = {}
  571. _owned_daemon_groups_lock = threading.Lock()
  572. def _own_harness_daemon(env: dict) -> Optional[bool]:
  573. """Hand the daemon behind ``env``'s BU_NAME to the owner-death supervisor.
  574. The harness publishes its daemon pid in its runtime dir (``harness_pid_file``: JSON
  575. ``{"pid", "started"}`` or a bare pid). The daemon is accepted only while it is this user's live
  576. ``browser_harness.daemon`` leading its own session and its spawn tag names this process, so a
  577. stale or planted pid file, or a daemon another Hermes started, never reaches the supervisor.
  578. Runs after every browser_exec (and while a long one runs, until the daemon is owned): a pid
  579. already owned for this daemon costs one file read.
  580. Returns True once the daemon is owned, False when it can never be (Windows, a managed worker, a
  581. daemon another Hermes started), None when there is nothing to own yet."""
  582. if os.name != "posix":
  583. return False # Windows: the harness daemon is outside this process's job; nothing to register.
  584. from agent.runtime_session_store import is_worker_process
  585. if is_worker_process():
  586. # A managed per-turn worker never owns it, so a killed worker does not take the browser the
  587. # next turn's worker drives along (its clean exit still runs the atexit browser cleanup).
  588. return False
  589. import psutil
  590. daemon = daemon_identity(env)
  591. pid = _published_daemon_pid(env)
  592. if pid is None:
  593. logger.debug("browser_exec: no harness daemon pid published for %s", daemon[-1])
  594. return None
  595. try:
  596. with _owned_daemon_groups_lock:
  597. if _owned_daemon_groups.get(daemon) == pid:
  598. return True
  599. proc = psutil.Process(pid)
  600. argv = proc.cmdline()
  601. if (os.getsid(pid) != pid or proc.uids().real != os.getuid() # windows-footgun: ok — posix-only, returned above
  602. or argv[argv.index("-m") + 1] != "browser_harness.daemon"):
  603. return None
  604. if not spawned_by_this_process(pid):
  605. return False # another Hermes on the same BU_NAME started it: that Hermes owns it
  606. except (OSError, ValueError, IndexError, psutil.Error) as exc:
  607. logger.debug("browser_exec: no harness daemon to supervise for %s: %s", daemon[-1], exc)
  608. return None
  609. with _owned_daemon_groups_lock:
  610. replaced = _owned_daemon_groups.get(daemon)
  611. _owned_daemon_groups[daemon] = pid
  612. own_process_groups([pid])
  613. register_child(pid, "browser-harness-daemon")
  614. if replaced is not None:
  615. release_process_group_once_empty(replaced, timeout=0) # the daemon it replaced; a survivor stays covered
  616. return True
  617. def _route_exec(env: dict, session: str, task_id: Optional[str], local: bool, local_sidecar: bool,
  618. route_key: str) -> tuple[Optional[str], bool]:
  619. """``_route_backend`` for one browser_exec call; returns ``(error, runs on the local sidecar)``.
  620. A real-profile fallback is remembered per route: the route keeps its fresh browser and the note
  621. waits until a result carries it (``_take_real_profile_note``)."""
  622. route_err = _route_backend(env, session, task_id, local, force_local_sidecar=local_sidecar,
  623. real_profile=route_key not in _real_profile_fallback)
  624. note = env.pop(_REAL_PROFILE_FALLBACK_SENTINEL, None)
  625. if note:
  626. _real_profile_fallback.setdefault(route_key, note)
  627. return route_err, local_sidecar or bool(env.pop(_LOCAL_FALLBACK_SENTINEL, None))
  628. def _route_exec_current(env: dict, session: str, task_id: Optional[str], local: bool,
  629. local_sidecar: bool, route_key: str) -> tuple[dict, Optional[str], bool, int]:
  630. """``_route_exec`` on a copy of ``env`` -> ``(routed env, error, sidecar, generation)``, re-routed (at
  631. most 3 tries) when /browser connect published meanwhile; ``_reload_if_republished`` covers dispatch."""
  632. for _attempt in range(3):
  633. routed = dict(env)
  634. generation = _live_generation()
  635. route_err, sidecar = _route_exec(routed, session, task_id, local, local_sidecar, route_key)
  636. if route_err or _live_generation() == generation:
  637. break
  638. return routed, route_err, sidecar, generation
  639. def _reload_if_republished(env: dict, generation: int) -> None:
  640. """Stop this call's daemon if the profile published a new endpoint while it spawned on the old one."""
  641. if _live_generation() == generation:
  642. return
  643. daemon = daemon_identity(env)
  644. with _driven_daemons_lock:
  645. _driven_daemons.pop(daemon, None)
  646. _reload_daemons([daemon])
  647. def _take_real_profile_note(route_key: str) -> dict:
  648. """The fallback note for the first result this route returns after falling back, then nothing."""
  649. if route_key not in _real_profile_fallback or route_key in _real_profile_note_delivered:
  650. return {}
  651. _real_profile_note_delivered.add(route_key)
  652. return {"real_profile_unavailable": _real_profile_fallback[route_key]}
  653. def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT_S,
  654. task_id: Optional[str] = None, local: bool = False):
  655. """Run Python code through the browser-use CLI, and return its output"""
  656. from agent.redact import redact_sensitive_text
  657. from tools.registry import tool_error, tool_result
  658. if not code or not code.strip():
  659. return tool_error("No code provided. Pass Python that uses the pre-imported helpers, e.g. new_tab(\"https://example.com\") then print(page_info()).")
  660. # The verdict is carried into the route, never re-derived: the predicate resolves DNS, so a second
  661. # evaluation could relax the floor and still pick the cloud for a rebinding host.
  662. verdict = _code_url_verdict(code)
  663. if verdict.blocked:
  664. return tool_error(verdict.blocked)
  665. # One endpoint per call: any private URL sends the whole call to the local sidecar. A call with no URL
  666. # literal stays on the route the previous call took.
  667. route_key = _backend_cache_key(task_id, session)
  668. local_sidecar = verdict.local_sidecar if _URL_RE.search(code) else _exec_route_is_sidecar.get(route_key, False)
  669. cmd = _find_cli()
  670. if not cmd:
  671. return tool_error("browser-harness is missing from Hermes's Python environment. "
  672. "Run `hermes update` to re-sync it.")
  673. env = _base_subprocess_env()
  674. if session:
  675. if not _SESSION_RE.match(session):
  676. return tool_error(f"Invalid session name {session!r}: use 1-64 letters, digits, "
  677. "dashes, or underscores (e.g. 'r7k2').")
  678. env["BU_NAME"] = session
  679. env, route_err, local_sidecar, generation = _route_exec_current(
  680. env, session, task_id, bool(local), local_sidecar, route_key)
  681. if route_err:
  682. return tool_error(route_err)
  683. if local_sidecar:
  684. env["BU_NAME"] = _sidecar_daemon_name(session)
  685. fit_default_socket_dir(env) # BU_NAME is final: a deep HOME's socket would overflow sun_path
  686. _exec_route_is_sidecar[route_key] = local_sidecar
  687. local_engine = env.pop(_BOT_DESKTOP_BROWSER_SENTINEL, None)
  688. bot_desktop_browser = bool(local_engine)
  689. _exec_local_engine.set(local_engine) # only local browsers carry the sentinel
  690. # SHARED browser (/browser connect CDP override): pin each named session to its own tab (see
  691. # _OWN_TAB_PREAMBLE). Private per-name browsers skip this — nothing to collide with.
  692. private_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLI
  693. if session and not private_browser:
  694. code = _OWN_TAB_PREAMBLE + code
  695. # Every backend: capture_screenshot() falls back to jpeg instead of hanging on a png a hidden tab never
  696. # renders; only on a cloud browser does it (and new_tab) bring the tab to the front (tools/browser_use_shot.py).
  697. from tools.browser_use_shot import ACTIVATE_NEVER, harness_command
  698. cmd = harness_command(cmd, activate=env.pop(_CLOUD_BROWSER_SENTINEL, None) or ACTIVATE_NEVER)
  699. workspace = _workspace_dir(task_id)
  700. if workspace:
  701. env["BH_AGENT_WORKSPACE"] = workspace
  702. # BU_AUTOSPAWN makes the CLI start a Browser Use cloud browser when no local
  703. # Chrome/CDP endpoint is reachable (their API key authenticates it)
  704. if "BU_AUTOSPAWN" not in env and is_legacy_browser_use_cloud_config(_read_browser_cfg()):
  705. env["BU_AUTOSPAWN"] = "1"
  706. timeout = _clamp_timeout(timeout_s)
  707. started = time.time()
  708. def dispatch() -> dict[str, Any]:
  709. _attach_vault_supervisor(env, task_id)
  710. with _driven_daemons_lock:
  711. daemon = daemon_identity(env)
  712. _driven_daemons[daemon] = hermes_home_key()
  713. # The first call starts the daemon and may run long (navigation, user code): own it as soon as
  714. # it publishes its pid, so a Hermes killed mid-call takes it along (the call's end owns it too).
  715. from agent.periodic_scheduler import schedule
  716. owning = schedule(lambda: _own_harness_daemon_until_owned(env), _OWN_DURING_CALL_S)
  717. try:
  718. return {"proc": _run_cli_killing_process_group(cmd, code, env, timeout)}
  719. except subprocess.TimeoutExpired:
  720. return {"error_result": tool_error(
  721. f"browser-use exec timed out after {timeout}s. The daemon may still be working; retry "
  722. f"with a larger timeout_s (max {_MAX_TIMEOUT_S}), or split the work into several calls that "
  723. "append to workspace files — anything already written to the workspace is preserved."
  724. )}
  725. except OSError as e:
  726. return {"error_result": tool_error(f"Failed to launch browser-use CLI: {e}")}
  727. finally:
  728. owning.cancel(wait=5) # an in-flight check finishes before the post-call one runs
  729. if bot_desktop_browser:
  730. from tools.browser_session import run_fenced
  731. dispatched = run_fenced({"features": {"local": True}}, dispatch)
  732. else:
  733. dispatched = dispatch()
  734. _reload_if_republished(env, generation)
  735. _own_harness_daemon(env)
  736. if "proc" not in dispatched:
  737. if "error_result" in dispatched:
  738. return dispatched["error_result"]
  739. return tool_result(dispatched)
  740. proc = dispatched["proc"]
  741. # browser_vault fill registers injected values with this forced model-egress
  742. # boundary. Preserve raw stdout only for screenshot-path detection below.
  743. result = {
  744. "success": proc.returncode == 0,
  745. "exit_code": proc.returncode,
  746. "output": redact_sensitive_text(proc.stdout, force=True),
  747. }
  748. if workspace:
  749. result["workspace"] = workspace
  750. if session:
  751. result["session"] = session
  752. result.update(_take_real_profile_note(route_key))
  753. stderr = redact_sensitive_text((proc.stderr or "").strip(), force=True)
  754. if len(stderr) > _STDERR_CAP_CHARS:
  755. stderr = stderr[:_STDERR_CAP_CHARS] + "\n… (stderr truncated)"
  756. if stderr:
  757. result["stderr"] = stderr
  758. screenshot = _find_screenshot(proc.stdout, started)
  759. if screenshot:
  760. result["screenshot_path"] = screenshot
  761. native = _native_screenshot_result(result, screenshot)
  762. if native is not None:
  763. return native
  764. return tool_result(result)
  765. _HEADER_BASE = (
  766. "Drive a real browser for JS pages, logins, forms and clicks (Browser Use CLI). `code` runs as full "
  767. "Python (stdlib available) with pre-imported browser helpers; stdout comes back in the result. Start `code` with a one-line "
  768. "comment describing the step for the user in plain language, max 60 chars "
  769. "(e.g. `# Searching Amazon for paper towels`) — the UI shows it as the step label.\n\n"
  770. "STATE: the browser session and workspace persist across calls; Python variables do NOT (fresh "
  771. "interpreter each call). The workspace dir is $BH_AGENT_WORKSPACE (also `workspace` in every result); "
  772. "functions defined in agent_helpers.py there are auto-imported into every call. For multi-item tasks "
  773. "('all N products / every entry'), append each batch to a JSON/CSV file in the workspace, then read it "
  774. "back and aggregate in code — dedupe/count/sort with Python, not in your head — and verify the "
  775. "collected count against what was asked before answering.\n\n"
  776. "Batch each sub-procedure (navigate, wait, extract, act) into one call — do not spend a call per "
  777. "action — but for long extractions prefer several medium calls that append to workspace files over "
  778. "one giant call, so progress survives timeouts."
  779. )
  780. _HEADER_VISION = (
  781. " Screenshots are attached to your context automatically: when the exec output contains a "
  782. "capture_screenshot() path, the image arrives with this tool's result and you inspect it directly "
  783. "with your own vision — never send browser screenshots to a separate vision tool."
  784. )
  785. _TEXT_FIRST = (
  786. "page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and "
  787. "js(\"document.querySelector('…').click()\") for clicks."
  788. )
  789. # The non-native header states a transport fact (screenshots do not ride back inside THIS
  790. # tool's results: routing is not ``native`` — e.g. an explicit ``auxiliary.vision`` pin — or
  791. # the route rejects tool-result media), never a verdict on the model's own vision: models
  792. # quote "your model cannot view images" against themselves and skip user images (#131568).
  793. # model_tools.get_tool_definitions() appends the vision_analyze pointer after
  794. # SCREENSHOTS_DETACHED when the session actually has that tool.
  795. SCREENSHOTS_DETACHED = " Screenshots are not attached to this tool's results."
  796. _HEADER_TEXT_ONLY = SCREENSHOTS_DETACHED + " Work text-first: " + _TEXT_FIRST
  797. # Lightpanda: no graphical renderer, and one CDP connection holds one page — a second
  798. # Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() sentence once
  799. # lightpanda-io/browser#1962 lands).
  800. _HEADER_LIGHTPANDA = (
  801. " The local engine is Lightpanda (no graphical renderer, one page per session): capture_screenshot() "
  802. "is unavailable, so work text-first: " + _TEXT_FIRST + " Navigate with new_tab(url) exactly once, then "
  803. "goto_url(url) for every later navigation — a second new_tab() fails with TargetAlreadyLoaded."
  804. )
  805. # Pinned quick-reference for the CLI's pre-imported helpers, replacing the live
  806. # ``browser-use skill`` fetch (uncontrolled third-party text in every schema: version
  807. # drift, supply-chain exposure, byte-unstable prompt). A/B benchmarked ~equal.
  808. _HELPERS_DIGEST = (
  809. "\n\nHELPERS (pre-imported): new_tab(url) opens/navigates (use for the FIRST navigation), goto_url(url) "
  810. "navigates the current tab, wait_for_load() after navigation, page_info() summarizes the current page "
  811. "state, js(expr) evaluates a JS expression and returns its value (js('document.title'); wrap function "
  812. "bodies as js('(() => {...})()') — a bare '() => {...}' returns the function itself, uncalled), "
  813. "fill_input(selector, text) types into inputs, click_at_xy(x, y) clicks viewport coordinates, "
  814. "capture_screenshot() saves and prints a screenshot path, cdp('Domain.method', **kwargs) is raw CDP — "
  815. "cdp('Accessibility.getFullAXTree')['nodes'] lists every element's role/name/backendDOMNodeId (filter "
  816. "in Python before printing; it is thousands of nodes), then cdp('DOM.getBoxModel', backendNodeId=n) "
  817. "gives click coordinates. ensure_real_tab() recovers from a stale/internal tab. Login walls: never guess "
  818. "credentials; see the vault note below if present, otherwise stop and ask the user."
  819. )
  820. def _description_header() -> str:
  821. """Header tailored to whether screenshots ride back natively in this tool's results."""
  822. if _lazy_call("tools.browser_lightpanda", "lightpanda_engine_status", (False, ""),
  823. "lightpanda engine status unavailable")[0]: # no screenshots, whatever the model sees
  824. return _HEADER_BASE + _HEADER_LIGHTPANDA
  825. if _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, ""):
  826. return _HEADER_BASE + _HEADER_VISION
  827. return _HEADER_BASE + _HEADER_TEXT_ONLY
  828. def _dynamic_schema_overrides() -> dict:
  829. overrides: dict = {"description": _description_header() + _HELPERS_DIGEST}
  830. # ``local`` exists ONLY when the user consented to real-profile browsing — everyone
  831. # else's schema carries zero extra surface. The caller memoizes on config.yaml mtime,
  832. # so toggling consent applies next session, not mid-chat.
  833. if _real_profile_consented():
  834. props = dict(BROWSER_EXEC_SCHEMA["parameters"]["properties"])
  835. props["local"] = {
  836. "type": "boolean", "default": False,
  837. "description": ("Drive the user's own local browser (a Hermes-managed copy of their real "
  838. "default-Chromium profile, logins/cookies included) instead of the configured "
  839. "cloud browser backend. Use when the user asks to act as themselves — their "
  840. "accounts, their sessions. No-op when the backend is already local. Default false."),
  841. }
  842. overrides["parameters"] = {**BROWSER_EXEC_SCHEMA["parameters"], "properties": props}
  843. return overrides
  844. BROWSER_EXEC_SCHEMA = {
  845. "name": "browser_exec",
  846. # Static fallback description, used only when the managed CLI is unavailable
  847. "description": (_HEADER_BASE + _HELPERS_DIGEST
  848. + "\n\n(The browser-use CLI is not installed yet. Install it with `hermes tools` (Browser Automation → Browser Use).)"),
  849. "parameters": {
  850. "type": "object",
  851. "properties": {
  852. "code": {"type": "string", "description": "Python code to execute using the pre-imported browser helpers. Use print(...) for any data you need back."},
  853. "session": {"type": "string", "description": "Named isolated browser session — its own daemon and (on cloud backends) own browser, so concurrent tasks don't share tabs. Reuse the same name on every related call; omit for the shared default session."},
  854. "timeout_s": {"type": "integer", "default": _DEFAULT_TIMEOUT_S,
  855. "description": f"Max seconds to wait for the code to finish (default {_DEFAULT_TIMEOUT_S}, max {_MAX_TIMEOUT_S})."},
  856. },
  857. "required": ["code"],
  858. },
  859. }
  860. # browser_exec is additionally gated at tool-definition time — sessions whose toolsets
  861. # lack ``terminal`` never see it (model_tools._compute_tool_definitions). check_fn only
  862. # answers "is browser-harness importable"; surface policy lives with the session.
  863. from tools.registry import registry
  864. # The LOCAL browser this context's browser_exec call ran on ("local" Chromium or "lightpanda", from its session),
  865. # else None: a configured cloud provider did not serve a hybrid ::local sidecar, cloud-fallback or real-profile
  866. # call, and browser.engine read afterwards may name another engine than the session's, so neither labels it.
  867. _exec_local_engine: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("browser_exec_local_engine",
  868. default=None)
  869. def _handle_browser_exec(args: dict, **kw):
  870. """Registry handler; each call counts against the backend that served it (shared metrics, opt-in)."""
  871. from hermes_cli.observability.shared_metrics_loop import record_browser_call
  872. from tools.browser_session_cloud import browser_backend_name
  873. token = _exec_local_engine.set(None)
  874. try:
  875. return record_browser_call(lambda: browser_exec(
  876. code=args.get("code", ""), session=args.get("session", "") or "",
  877. timeout_s=args.get("timeout_s", _DEFAULT_TIMEOUT_S), task_id=kw.get("task_id"),
  878. local=bool(args.get("local", False)),
  879. ), lambda: browser_backend_name(local_engine=_exec_local_engine.get()))
  880. finally:
  881. _exec_local_engine.reset(token)
  882. registry.register(
  883. name="browser_exec",
  884. toolset="browser-use",
  885. schema=BROWSER_EXEC_SCHEMA,
  886. handler=_handle_browser_exec,
  887. check_fn=browser_exec_available,
  888. dynamic_schema_overrides=_dynamic_schema_overrides,
  889. emoji="🌐",
  890. )