| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043 |
- """``browser_exec``: drive a real browser through the Browser Use CLI engine (browser-harness,
- https://browser-use.com), Hermes' only core browser tool.
- The harness speaks CDP to whatever ``tools.browser_session`` resolved for the call: an operator
- ``BU_CDP_*`` endpoint, a ``/browser connect`` override, a cloud provider's session, or a local browser
- (real profile, Lightpanda, or the packaged Chromium started by ``tools.browser_launcher``).
- """
- import contextlib
- import contextvars
- import functools
- import importlib
- import importlib.util
- import json
- import logging
- import os
- import re
- import signal
- import subprocess
- import sys
- import threading
- import time
- from pathlib import Path
- from typing import Any, Callable, Dict, List, NamedTuple, Optional
- from hermes_cli.process_identity import (
- own_process_groups, register_child, release_process_group_once_empty, spawn_env, spawned_by_this_process)
- from hermes_constants import get_hermes_home, hermes_home_key
- from tools.browser_harness_dirs import (daemon_env, daemon_identity, fit_default_socket_dir, harness_pid_file,
- namespace_harness_dirs)
- from utils import is_truthy_value
- logger = logging.getLogger(__name__)
- _BROWSER_USE_PROVIDER = "browser-use"
- # The one credential the harness itself reads (BU_AUTOSPAWN of a Browser Use cloud browser).
- _HARNESS_PASSTHROUGH_KEYS: tuple[str, ...] = ("BROWSER_USE_API_KEY",)
- # Cloud daemon names become the BU_NAME env var
- _SESSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
- # Set on the env dict by the CDP resolvers when the resolved browser is EXCLUSIVE to this named session
- # (per-name provider / named BU cloud / Lightpanda). Popped before the subprocess launches — never exported.
- _PRIVATE_BROWSER_SENTINEL = "_HERMES_BU_PRIVATE_BROWSER"
- # Internal route provenance: this exec resolved to a browser on the Bot Desktop display and must use
- # the same human-control lease fence as the built-in browser tools. Popped before launching the CLI.
- _BOT_DESKTOP_BROWSER_SENTINEL = "_HERMES_BU_BOT_DESKTOP_BROWSER"
- # The note when the consented real profile could not load (the route fell back). Popped before launch.
- _REAL_PROFILE_FALLBACK_SENTINEL = "_HERMES_BU_REAL_PROFILE_FALLBACK"
- # local=True whose real profile could not load: run on the local sidecar browser. Popped before launch.
- _LOCAL_FALLBACK_SENTINEL = "_HERMES_BU_LOCAL_FALLBACK"
- # Where new/hidden tabs may be brought to the front: a browser_use_shot.ACTIVATE_* mode. Popped before launch.
- _CLOUD_BROWSER_SENTINEL = "_HERMES_BU_CLOUD_BROWSER"
- # Prepended to the model's code for named sessions on SHARED browsers (a /browser connect CDP override): the
- # harness daemon attaches to the first existing page at startup, so two fresh named daemons can land on the
- # SAME tab. Steering each onto a tab it created prevents clobbering. Runs once per daemon (marker keyed by
- # BU_NAME + daemon pid).
- _OWN_TAB_PREAMBLE = """\
- # hermes: pin this named session to its own tab (once per daemon process)
- def _hermes_ensure_own_tab():
- import os as _os, tempfile as _tf
- _name = _os.environ.get("BU_NAME", "default")
- try:
- # Key the marker by the daemon's pid so a daemon restart (which
- # re-attaches to the first shared page) re-pins automatically,
- # while agent-driven tab switches mid-session are left alone.
- from browser_harness import _ipc as _bipc
- _dpid = _bipc.pid_path(_name).read_text().strip() or "0"
- except Exception:
- _dpid = "0"
- _uid = _os.getuid() if hasattr(_os, "getuid") else 0
- _marker = _os.path.join(
- _tf.gettempdir(), "hermes-bu-owntab-%s-%s-%s" % (_uid, _name, _dpid)
- )
- if _os.path.exists(_marker):
- return
- try:
- # Force a fresh target: new_tab() would REUSE a blank current tab,
- # which is exactly the tab a sibling daemon may also hold.
- _tid = cdp("Target.createTarget", url="about:blank").get("targetId")
- if _tid:
- switch_tab(_tid)
- except Exception:
- pass # best-effort: worst case is pre-fix behavior
- try:
- open(_marker, "w").close()
- except OSError:
- pass
- _hermes_ensure_own_tab()
- del _hermes_ensure_own_tab
- """
- _DEFAULT_TIMEOUT_S = 300
- _MIN_TIMEOUT_S = 5
- _MAX_TIMEOUT_S = 1800
- _STDERR_CAP_CHARS = 4000
- _TASK_ID_SAFE_RE = re.compile(r"[^A-Za-z0-9._-]+") # filesystem-safe task ids
- # Screenshot paths printed by capture_screenshot(): POSIX or Windows drive-letter absolute.
- _IMAGE_PATH_RE = re.compile(r"((?:[A-Za-z]:[\\/]|/)[^\s\"']+?\.(?:png|jpe?g|webp))", re.IGNORECASE)
- # http(s) URL literals in exec code, checked against the browser URL policy
- _URL_RE = re.compile(r"https?://[^\s'\"\\)]+", re.IGNORECASE)
- _FHS_BIN_DIRS = ("/opt/homebrew/bin", "/opt/homebrew/sbin", "/usr/local/sbin", "/usr/local/bin", "/usr/sbin",
- "/usr/bin", "/sbin", "/bin")
- def _quiet(fn: Callable[[], Any], default: Any, log_prefix: str = "") -> Any:
- """``fn()``, or ``default`` on any exception (debug-logged when ``log_prefix`` is set)."""
- try:
- return fn()
- except Exception as e:
- if log_prefix:
- logger.debug("%s: %s", log_prefix, e)
- return default
- def _lazy_call(module: str, name: str, default: Any, log_prefix: str) -> Any:
- """Call ``module.name()`` resolved at call time (tests stub the module / patch the
- attribute); on any failure log ``log_prefix`` and return ``default``."""
- return _quiet(lambda: getattr(importlib.import_module(module), name)(), default, log_prefix)
- def _real_profile_consented() -> bool:
- return _lazy_call("tools.browser_session_cloud", "use_real_profile", False, "real-profile consent lookup failed")
- def _set_cdp_env(env: dict, cdp: str) -> None:
- """Export a CDP endpoint under the BU_CDP_* contract (http(s) → URL, else WS)."""
- env["BU_CDP_URL" if cdp.startswith(("http://", "https://")) else "BU_CDP_WS"] = cdp
- def _has_cdp_env(env: dict) -> bool:
- return bool(env.get("BU_CDP_WS") or env.get("BU_CDP_URL"))
- def _export_session_cdp(env: dict, get_session_info: Callable[[str], Any], cache_key: str,
- fail_msg: Callable[[Exception], str], no_cdp_msg: str) -> Optional[str]:
- """Export the CDP endpoint from ``get_session_info(cache_key)``; error string on failure / no CDP."""
- try:
- cdp = str((get_session_info(cache_key) or {}).get("cdp_url") or "")
- except Exception as e:
- return fail_msg(e)
- if not cdp:
- return no_cdp_msg
- _set_cdp_env(env, cdp)
- return None
- class _CodeUrlVerdict(NamedTuple):
- """What one pass over a code blob's URL literals decided (see ``_code_url_verdict``)."""
- blocked: Optional[str]
- local_sidecar: bool
- def _code_url_verdict(code: str) -> _CodeUrlVerdict:
- """One pass over the code's URL literals: the blocking error, if any, and whether the call must run on
- the hybrid LOCAL sidecar. Each URL's routing predicate is evaluated ONCE and feeds both answers: it
- resolves DNS unmemoised, so a rebinding host evaluated twice could relax the private-address floor and
- still route to the cloud provider."""
- from tools.browser_url_policy import evaluate_url_safety, hybrid_routes_locally
- local_sidecar = False
- for url in _URL_RE.findall(code or ""):
- routes_local = hybrid_routes_locally(url)
- local_sidecar = local_sidecar or routes_local
- err = evaluate_url_safety(url, auto_local=routes_local)
- if err:
- return _CodeUrlVerdict(err.get("error", "Blocked: unsafe URL"), local_sidecar)
- return _CodeUrlVerdict(None, local_sidecar)
- def _harness_credentials() -> dict:
- """The harness's own credential from the active profile's secret scope, never ``os.environ``: under
- multiplex that holds the LAUNCH profile's key, and a served profile's browser runs on its own (or none)."""
- from agent.secret_scope import current_secret_scope, get_secret, serves_routed_profile
- routed = serves_routed_profile()
- scope = (current_secret_scope() or {}) if routed else None
- values = {key: (scope.get(key) if routed else get_secret(key)) for key in _HARNESS_PASSTHROUGH_KEYS}
- return {key: value for key, value in values.items() if value is not None}
- def _base_subprocess_env() -> dict:
- from tools.browser_session import build_browser_env
- env = build_browser_env()
- env.update(_harness_credentials())
- # The harness runs on Hermes's own interpreter, but a bundled Desktop install boots that
- # interpreter with its site dir on PYTHONPATH (no venv to activate), and the harness's daemon
- # re-runs sys.executable. Point PYTHONPATH at the harness's site dir, replacing whatever the
- # agent process inherited, so both the CLI and its daemon import the same packages.
- env.pop("PYTHONHOME", None)
- site_dir = _harness_site_dir()
- if site_dir:
- env["PYTHONPATH"] = site_dir
- else:
- env.pop("PYTHONPATH", None)
- env["PATH"] = _floor_subprocess_path(env.get("PATH", ""))
- env.setdefault("ANONYMIZED_TELEMETRY", "false")
- namespace_harness_dirs(env, _served_profile_tag()) # a served profile's own harness runtime/tmp dirs
- # A daemon the CLI starts inherits this tag; _own_harness_daemon owns only one that names us.
- env.update(spawn_env("browser-harness-daemon"))
- return env
- def _floor_subprocess_path(path: str) -> str:
- """Keep system commands reachable from profile workers with a minimal PATH.
- Browser subprocesses may launch shell helpers; version-manager-only paths
- omit /usr/bin. Reuse the shared browser PATH floor on POSIX.
- """
- if os.name == "nt":
- return path
- parts = [p for p in (path or "").split(os.pathsep) if p]
- return os.pathsep.join(parts + [d for d in _FHS_BIN_DIRS if d not in set(parts) and os.path.isdir(d)])
- def _read_browser_cfg() -> dict:
- """Return the ``browser:`` config section, or {} on any failure."""
- try:
- from tools.browser_session_cloud import browser_section
- return browser_section()
- except Exception as e:
- logger.debug("Could not read browser config section: %s", e)
- return {}
- def _use_gateway(browser_cfg: dict) -> bool:
- """True when the browser section selects the Nous Tool Gateway — by the current ``hermes tools``
- picker row (``cloud_provider: nous``) or the pre-picker ``use_gateway: true`` flag. Reading only
- the legacy flag missed every picker-configured gateway, and the direct-API branch it fell into
- holds no credentials in managed mode (#108310)."""
- if is_truthy_value(browser_cfg.get("use_gateway"), default=False):
- return True
- try:
- from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER
- except Exception: # pragma: no cover — helper ships with the package
- return False
- return str(browser_cfg.get("cloud_provider") or "").strip().lower() == NOUS_MANAGED_PROVIDER
- def is_legacy_browser_use_cloud_config(browser_cfg: dict) -> bool:
- """True for pre-CLI direct-API Browser Use cloud configs: no cloud_provider (or ``browser-use``), not the
- Nous gateway, and BROWSER_USE_API_KEY set; the CLI then reaches Browser Use cloud natively."""
- if not isinstance(browser_cfg, dict):
- return False
- provider = str(browser_cfg.get("cloud_provider") or "").strip().lower()
- if provider not in {"browser-use", ""} or _use_gateway(browser_cfg):
- return False
- return _browser_use_api_key_set()
- def _browser_use_api_key_set() -> bool:
- """BROWSER_USE_API_KEY for the served profile: the only credential Hermes autospawns BU cloud with.
- Profile-scoped so a multiplexed secondary never inherits the default's cloud mode."""
- from agent.secret_scope import get_secret
- return bool(get_secret("BROWSER_USE_API_KEY", ""))
- def _harness_site_dir() -> Optional[str]:
- """The site dir Hermes's interpreter imports ``browser_harness`` from, or None."""
- spec = importlib.util.find_spec("browser_harness")
- if spec is None or not spec.origin:
- return None
- return str(Path(spec.origin).resolve().parent.parent)
- def _find_cli() -> Optional[list[str]]:
- """The Browser Use CLI's engine (browser-harness) is a core dependency of Hermes's own venv,
- so every install, the Desktop bundle included, runs it on the current interpreter."""
- if _harness_site_dir() is None:
- return None
- return [sys.executable, "-m", "browser_harness.run"]
- def browser_exec_available() -> bool:
- """check_fn: browser-harness is importable (a core dependency). Never launches or opens sockets."""
- return _find_cli() is not None
- def _workspace_dir(task_id: Optional[str]) -> Optional[str]:
- """Stable per-task scratch dir that persists across browser_exec calls"""
- if os.environ.get("BH_AGENT_WORKSPACE"):
- return os.environ["BH_AGENT_WORKSPACE"]
- try:
- safe = _TASK_ID_SAFE_RE.sub("_", str(task_id or "default"))[:80] or "default"
- path = Path(get_hermes_home()) / "cache" / "browser-use" / "workspace" / safe
- path.mkdir(parents=True, exist_ok=True)
- return str(path)
- except Exception as e:
- logger.debug("browser_exec workspace unavailable: %s", e)
- return None
- def _find_screenshot(stdout: str, since: float) -> Optional[str]:
- """Last screenshot path printed during this exec that exists and was written after
- the exec started, or None."""
- for path in reversed(_IMAGE_PATH_RE.findall(stdout or "")):
- try:
- if os.path.isfile(path) and os.path.getmtime(path) >= since - 1:
- return path
- except OSError:
- continue
- return None
- def _native_screenshot_result(result: dict[str, Any], path: str) -> Optional[dict[str, Any]]:
- """Build a multimodal tool result attaching path for vision models"""
- try:
- from tools.vision_tools import (_EMBED_MAX_DIMENSION,
- _resize_image_for_vision, _should_use_native_vision_fast_path)
- from tools.vision_tools_history_budget import resolve_embed_target_bytes
- if not _should_use_native_vision_fast_path():
- return None
- # History-reuse cap: this data URL bakes into the tool result and is re-sent every later turn —
- # same policy as the vision_analyze native embed.
- data_url = _resize_image_for_vision(Path(path), mime_type="image/png",
- max_base64_bytes=resolve_embed_target_bytes(),
- max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
- text = json.dumps(result, ensure_ascii=False)
- attached = text + "\n\nThe screenshot from this call is attached — inspect it with your native vision."
- return {"_multimodal": True, "text_summary": text, "meta": {"screenshot_path": path, "native_vision": True},
- "content": [{"type": "text", "text": attached}, {"type": "image_url", "image_url": {"url": data_url}}]}
- except Exception as e:
- logger.debug("Native screenshot attach failed (falling back to text): %s", e)
- return None
- def _served_profile_tag() -> str:
- """``""`` outside a served-profile scope (every legacy key stays byte-identical); under a
- multiplexed turn, the routed profile's home key — one profile's browser must never be handed
- to another that happens to use the same session name or task id (#110032)."""
- from hermes_constants import get_hermes_home_override, hermes_home_key
- return "" if get_hermes_home_override() is None else hermes_home_key()
- def _backend_cache_key(task_id: Optional[str], session_name: str = "", sidecar: bool = False) -> str:
- """Session-cache key for a backend browser: named sessions get their own; served profiles get their own.
- ``sidecar`` appends the hybrid ``::local`` suffix, which makes ``_get_session_info`` build local Chromium
- even when a cloud provider is configured."""
- key = f"bu-named-{session_name}" if session_name else (task_id or "browser-exec-default")
- tag = _served_profile_tag()
- key = f"{key}@{tag}" if tag else key
- return f"{key}::local" if sidecar else key
- def _sidecar_daemon_name(name: str) -> str:
- """BU_NAME of the harness daemon serving the hybrid local sidecar. The daemon reads BU_CDP_* once at
- spawn and outlives the call, so sharing the cloud route's daemon would relay a private URL to the
- cloud browser it is already attached to. User session names cannot start with ``_``."""
- return f"_local_{name or 'default'}"[:64]
- def _export_real_profile_note(env: dict, key: str) -> None:
- """The session layer decides real profile vs fresh browser once per key; carry its note (if it fell
- back) so the result reports the browser this call actually drives."""
- from tools.browser_session import real_profile_note
- note = real_profile_note(key)
- if note:
- env.setdefault(_REAL_PROFILE_FALLBACK_SENTINEL, note)
- def _mark_local_browser(env: dict, session: dict, key: str) -> None:
- """Flags for a LOCAL session browser: it runs on the Bot Desktop display (lease fence), and unless a
- persistent profile / the real-profile browser is shared by every key, it is exclusive to this key (no
- own-tab preamble: nothing to share a tab with)."""
- features = session.get("features") or {}
- env[_BOT_DESKTOP_BROWSER_SENTINEL] = "lightpanda" if features.get("lightpanda") else "local"
- if not session.get("shared_profile") and not features.get("real_profile"):
- env[_PRIVATE_BROWSER_SENTINEL] = "1"
- _export_real_profile_note(env, key)
- def _resolve_local_cdp(env: dict, task_id: Optional[str], session_name: str = "",
- sidecar: bool = False) -> Optional[str]:
- """Local browser for this cache key through the session layer: the consented real profile, Lightpanda
- (``browser.engine: lightpanda``), or the packaged Chromium. Never the harness's own discovery of the
- user's installed Chrome (chrome://inspect toggle + Allow popup per run, blocked on Chrome >=136)."""
- from tools.browser_lightpanda import using_lightpanda_engine
- from tools.browser_session import get_session_info
- key = _backend_cache_key(task_id, session_name, sidecar)
- session: dict = {}
- def _session_info(cache_key: str) -> dict:
- session.update(get_session_info(cache_key) or {})
- return session
- if _quiet(using_lightpanda_engine, False):
- fail = lambda e: (f"Lightpanda could not be started: {e} Set browser.engine to auto "
- "to use local Chrome, or switch backends via `hermes tools` → Browser Automation.")
- else:
- fail = lambda e: (f"The local browser could not be started: {e} "
- "Run `hermes tools` → Browser Automation to (re)install Chromium, or switch backends.")
- err = _export_session_cdp(env, _session_info, key, fail, "The local browser session returned no CDP endpoint.")
- if err is None:
- _mark_local_browser(env, session, key)
- return err
- def _resolve_backend_cdp(env: dict, task_id: Optional[str], session_name: str = "",
- force_local_sidecar: bool = False) -> Optional[str]:
- """Point the harness at the configured backend's CDP endpoint; error string on failure.
- Precedence: (1) ``BU_CDP_WS``/``BU_CDP_URL`` already in env (operator override); (2) ``BROWSER_CDP_URL``
- env / ``browser.cdp_url`` (``/browser connect``); (3) a cloud provider via ``browser_session.get_session_info``
- (per-key cache, expiry, reaper, atexit; a provider failure falls back to a local browser);
- (4) the local engine — ``browser.engine: lightpanda`` or Hermes' packaged Chromium via the launcher
- (never the harness's own discovery of the user's installed Chrome); (5) BU direct-API configs WITH
- ``BROWSER_USE_API_KEY`` → None (checked before (3)): the CLI reaches BU cloud natively (BU_AUTOSPAWN); a
- keyless ``browser-use`` provider (managed-gateway autodetect, or a selection missing its key) takes (3).
- ``session_name`` (BU_NAME) keys the session cache so each name gets its OWN browser — what makes named
- sessions concurrent-safe.
- ``force_local_sidecar`` skips step (3): the call runs on the local engine under the ``::local`` cache
- key (hybrid routing; see ``browser_url_policy.hybrid_routes_locally``). It sits BELOW the two operator overrides, and the
- sidecar verdict never fires while a CDP override is set."""
- if _has_cdp_env(env):
- return None
- from tools.browser_session import get_session_info
- from tools.browser_session_cdp import get_cdp_override
- from tools.browser_session_cloud import get_cloud_provider
- override = _quiet(get_cdp_override, "")
- if override:
- _set_cdp_env(env, override)
- return None
- if force_local_sidecar:
- return _resolve_local_cdp(env, task_id, session_name, sidecar=True)
- provider = _quiet(get_cloud_provider, None, "Cloud provider lookup failed")
- if provider is None:
- return _resolve_local_cdp(env, task_id, session_name)
- # Browser Use direct-API configs: the CLI talks to BU cloud natively (BU_AUTOSPAWN) — the
- # legacy provider would create a second, redundant session. Nous-gateway configs (cloud_provider: nous
- # from the picker, or the pre-picker use_gateway: true) DO resolve through the provider: the gateway
- # provisions the browser server-side and returns its CDP URL. Without BROWSER_USE_API_KEY the CLI cannot
- # autospawn, and the provider is either the managed gateway (autodetect on a never-configured install) or
- # a selection missing its key; both go through the session path (gateway CDP, or the session layer's
- # local-Chromium fallback) instead of leaving the harness to hunt for an installed Chrome.
- provider_key = str(getattr(provider, "name", "") or "").strip().lower()
- if provider_key == _BROWSER_USE_PROVIDER and not _use_gateway(_read_browser_cfg()) and _browser_use_api_key_set():
- env[_PRIVATE_BROWSER_SENTINEL] = "1" # named BU cloud browsers are exclusive to their daemon
- # Only intent: the harness attaches to a local Chrome already listening instead of autospawning, so
- # the live daemon's own report decides.
- from tools.browser_use_shot import ACTIVATE_IF_CLOUD
- env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_IF_CLOUD
- return None
- provider_name = type(provider).__name__
- session: dict = {}
- def _session_info(key: str) -> dict:
- session.update(get_session_info(key) or {})
- return session
- key = _backend_cache_key(task_id, session_name)
- err = _export_session_cdp(
- env, _session_info, key,
- lambda e: (f"Cloud browser provider {provider_name} failed to provide a session: {e}. "
- "Fix the provider configuration or switch backends via `hermes tools` → Browser Automation."),
- f"Cloud browser provider {provider_name} returned no CDP endpoint, so browser_exec "
- "cannot drive it. Pick a CDP-capable provider via `hermes tools` → Browser Automation.",
- )
- if session.get("fallback_from_cloud"):
- # The provider failed (rate limit, quota, outage) and the session layer fell back to a LOCAL
- # browser for this key; it already carries its CDP URL.
- logger.warning("browser_exec: %s failed (%s); using local Chromium for this session",
- provider_name, session.get("fallback_reason"))
- if err is None:
- _mark_local_browser(env, session, key)
- return err
- if err is None:
- from tools.browser_use_shot import ACTIVATE_ALWAYS
- env[_CLOUD_BROWSER_SENTINEL] = ACTIVATE_ALWAYS # the provider's own session: the harness connects only there
- # A provider browser keyed bu-named-<name> is exclusive to this session — the
- # own-tab preamble would just leak a blank tab into it.
- if err is None and session_name:
- env[_PRIVATE_BROWSER_SENTINEL] = "1"
- return err
- def _resolve_real_profile_cdp(env: dict, force_local: bool) -> Optional[str]:
- """Point the harness at the user's real-profile copy-browser (a SNAPSHOT of their default Chromium
- profile, hermes_cli.browser_connect) when consented. Two ways in: the effective backend is already local
- (no provider, CDP override, or legacy BU cloud config) → silent upgrade; or ``force_local`` (consent-gated
- ``local`` arg) → the user's browser even under a cloud backend. Operator overrides (BU_CDP_* env,
- /browser connect, ``browser.cdp_url``) own the session either way. A launch error is returned;
- ``_route_backend`` then falls back to the normal backend and reports it, never silently."""
- if not _real_profile_consented() or _has_cdp_env(env):
- return None
- from tools.browser_session_cdp import get_cdp_override_raw
- from tools.browser_session_cloud import get_cloud_provider
- from tools.browser_session_real_profile import real_profile_cdp
- if _quiet(get_cdp_override_raw, ""):
- return None
- # Only auto-upgrade genuinely-local attaches; any cloud path (provider, provider lookup failure, or
- # legacy BU cloud config) stays on its backend unless the model passes local=true.
- if not force_local and (_quiet(get_cloud_provider, object()) is not None
- or is_legacy_browser_use_cloud_config(_read_browser_cfg())):
- return None
- cdp, err = real_profile_cdp()
- if cdp and not err:
- _set_cdp_env(env, cdp)
- env[_BOT_DESKTOP_BROWSER_SENTINEL] = "local" # the user's real Chromium, whatever browser.engine says
- return err or None
- def _attach_vault_supervisor(env: dict, task_id: Optional[str]) -> None:
- """Attach the per-task CDP supervisor to the browser this exec drives so ``browser_vault`` fill has
- a secret-capable WebSocket (never argv) into the SAME browser. Only CDP-routed backends expose an
- endpoint; BU direct-cloud (BU_AUTOSPAWN) does not, and the vault tools report ``supervisor_required``."""
- cdp = env.get("BU_CDP_WS") or env.get("BU_CDP_URL")
- if not cdp:
- return
- try:
- from tools.browser_session_cdp import get_dialog_policy_config, resolve_cdp_override
- from tools.browser_supervisor import SUPERVISOR_REGISTRY
- policy, timeout_s = get_dialog_policy_config()
- SUPERVISOR_REGISTRY.get_or_start(task_id=task_id or "default", cdp_url=resolve_cdp_override(cdp),
- dialog_policy=policy, dialog_timeout_s=timeout_s)
- except Exception as exc:
- logger.debug("browser_exec: CDP supervisor attach failed (non-fatal): %s", exc)
- def _route_backend(env: dict, session: str, task_id: Optional[str], local: bool,
- force_local_sidecar: bool = False, real_profile: bool = True) -> Optional[str]:
- """Resolve where the harness connects; returns an error string or None. Real-profile consent runs
- BEFORE provider resolution so a hit short-circuits the cloud path via the BU_CDP_* env contract. Named
- sessions compose with the backend: BU_NAME namespaces the harness daemon (IPC socket, log, pid) and on
- provider backends additionally keys its own cloud browser.
- ``force_local_sidecar`` (a call carrying a private URL) declines the cloud provider for the hybrid local
- sidecar; unlike ``local`` (the consent-gated real-profile browser) it never touches the user's profile."""
- # ``real_profile=False``: this route already fell back, so it stays on its fresh browser instead
- # of swapping browsers mid-task once the profile frees up.
- rp_err = _resolve_real_profile_cdp(env, force_local=local) if real_profile else None
- # local=True is only served by the real-profile route; consent off must not pretend.
- if local and not _has_cdp_env(env) and not _real_profile_consented():
- return ("local=true was requested but browser.use_real_profile is off. Enable it in config.yaml "
- "(browser.use_real_profile: true) or the desktop Settings → Browser section, then retry.")
- # local=True promised the user's own machine: a profile that can't load runs on a fresh LOCAL browser
- # (the sidecar, which never loads the profile), never the cloud one. Without ``local`` the session
- # layer makes the one real-profile-or-fresh decision and reports it (_resolve_local_cdp).
- if local and not _has_cdp_env(env) and (rp_err or not real_profile):
- if rp_err:
- from tools.browser_session_real_profile import real_profile_fallback_note
- env[_REAL_PROFILE_FALLBACK_SENTINEL] = real_profile_fallback_note(rp_err)
- env[_LOCAL_FALLBACK_SENTINEL] = "1"
- force_local_sidecar = True
- return _resolve_backend_cdp(env, task_id, session_name=session,
- force_local_sidecar=force_local_sidecar)
- def _group_popen_kwargs() -> dict:
- """Popen kwargs starting the CLI in its own process group (a new session on POSIX) so a
- timeout can take down every process that inherited the capture pipes, not just the CLI
- child. Windows also hides the console window."""
- def _flags() -> dict:
- from hermes_cli._subprocess_compat import windows_hide_flags
- si = subprocess.STARTUPINFO()
- si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
- return {"creationflags": windows_hide_flags() | getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0),
- "startupinfo": si}
- # detached-spawn: ok — awaited CLI; a timeout kills its whole group (_run_cli_killing_process_group)
- return _quiet(_flags, {}, "Windows hide-flags unavailable") if os.name == "nt" else {"start_new_session": True}
- def _clamp_timeout(timeout_s: Any) -> int:
- try:
- return max(_MIN_TIMEOUT_S, min(int(timeout_s), _MAX_TIMEOUT_S))
- except (TypeError, ValueError):
- return _DEFAULT_TIMEOUT_S
- # After a whole-group SIGKILL, every pipe holder is dead, so the drain below is normally
- # instant; the deadline only guards against a process outside the group still holding a pipe.
- _POST_KILL_DRAIN_S = 10.0
- def _kill_cli_process_group(proc) -> None:
- """SIGKILL the CLI's whole process group (POSIX; ``start_new_session`` made pgid == pid) or,
- on Windows, its process tree via ``taskkill /T /F`` — the only group-wide kill it offers."""
- if os.name == "nt":
- from hermes_cli._subprocess_compat import windows_hide_flags
- with contextlib.suppress(OSError, subprocess.SubprocessError):
- subprocess.run(["taskkill", "/T", "/F", "/PID", str(proc.pid)], stdin=subprocess.DEVNULL,
- capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10,
- check=False, creationflags=windows_hide_flags())
- return
- with contextlib.suppress(ProcessLookupError, PermissionError):
- os.killpg(proc.pid, signal.SIGKILL) # windows-footgun: ok — POSIX only, the nt branch returned above
- def _run_cli_killing_process_group(cmd, code, env, timeout):
- """Run the CLI in its own process group and kill the whole group on timeout.
- ``subprocess.run`` only kills the direct child on ``TimeoutExpired``; a grandchild that
- inherited the stdout/stderr pipes (browser_harness daemon / Chrome helper) is orphaned
- still holding them, and on Windows ``run()``'s unbounded post-kill ``communicate()`` then
- blocks on pipe EOF forever — so the tool call, plus its activity heartbeat, wedges (#106244).
- """
- proc = subprocess.Popen(
- cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
- text=True, encoding="utf-8", errors="replace", env=env, **_group_popen_kwargs(),
- )
- try:
- stdout, stderr = proc.communicate(input=code, timeout=timeout)
- except subprocess.TimeoutExpired:
- _kill_cli_process_group(proc)
- with contextlib.suppress(subprocess.TimeoutExpired):
- proc.communicate(timeout=_POST_KILL_DRAIN_S)
- raise
- return subprocess.CompletedProcess(cmd, proc.returncode, stdout, stderr)
- # How often a running browser_exec re-checks for the daemon its first call starts.
- _OWN_DURING_CALL_S = 0.5
- # Harness daemons this process drove: each latches BU_CDP_* at start and outlives every call, so a backend
- # swap (/browser connect|disconnect) leaves later calls in the old browser until these are stopped.
- _driven_daemons: dict[tuple[str, ...], str] = {} # daemon identity -> home
- _driven_daemons_lock = threading.Lock()
- _live_generation = functools.partial(_lazy_call, "tools.browser_session_cdp", "live_cdp_generation", 0,
- "live CDP generation lookup failed") # this profile's publish count
- # Route cache key -> whether its last browser_exec ran on the hybrid local sidecar.
- _exec_route_is_sidecar: dict[str, bool] = {}
- # Route cache key -> the note from its real-profile fallback; that route stays on the fallback browser.
- _real_profile_fallback: dict[str, str] = {}
- _real_profile_note_delivered: set = set()
- def stop_harness_daemons(home_key: Optional[str] = None) -> None:
- """Stop every harness daemon this process drove (only ``home_key``'s when given), through the
- harness's own identity-checked ``--reload``; the next browser_exec respawns one on the endpoint
- it resolves then."""
- with _driven_daemons_lock:
- daemons = sorted(key for key, home in _driven_daemons.items() if home_key in (None, home))
- for key in daemons:
- _driven_daemons.pop(key, None)
- _reload_daemons(daemons)
- def _published_daemon_pid(env: dict) -> Optional[int]:
- """The pid the harness published for the daemon ``env`` drives (``harness_pid_file``: JSON
- ``{"pid", "started"}`` or a bare pid), or None when there is none."""
- try:
- raw = harness_pid_file(env).read_text(encoding="utf-8-sig").strip()
- return int(json.loads(raw)["pid"]) if raw.startswith("{") else int(raw.split()[0])
- except (OSError, ValueError, KeyError, TypeError, IndexError):
- return None
- def _own_harness_daemon_until_owned(env: dict) -> Optional[bool]:
- """Periodic check while browser_exec runs: own the daemon as soon as it publishes its pid, then
- stop (False). It also stops once the daemon can never be ours (Windows, a managed worker, a daemon
- another Hermes started), instead of re-checking every tick for the whole call."""
- return None if _own_harness_daemon(env) is None else False
- def _reload_daemons(daemons) -> None: # each ``daemon_identity`` daemon, with its own dirs
- cmd = _find_cli() if daemons else None
- if not cmd:
- return
- env = _base_subprocess_env()
- for daemon in daemons:
- with contextlib.suppress(OSError, subprocess.SubprocessError):
- _run_cli_killing_process_group([*cmd, "--reload"], "", daemon_env(env, daemon), 15)
- with _owned_daemon_groups_lock:
- pgid = _owned_daemon_groups.pop(daemon, None)
- if pgid is not None:
- release_process_group_once_empty(pgid) # a daemon --reload failed to stop stays covered
- # Daemon identity (``daemon_identity``: its dirs + BU_NAME) -> process group of the harness daemon this
- # process owns. The harness starts its daemon with start_new_session so it outlives each CLI call (that
- # is how a session persists across browser_exec calls), which also means it outlives a SIGKILLed or
- # OOM-killed Hermes forever (#121095). Its group goes to this process's owner-death supervisor, so it
- # dies with us however we die.
- _owned_daemon_groups: dict[tuple[str, ...], int] = {}
- _owned_daemon_groups_lock = threading.Lock()
- def _own_harness_daemon(env: dict) -> Optional[bool]:
- """Hand the daemon behind ``env``'s BU_NAME to the owner-death supervisor.
- The harness publishes its daemon pid in its runtime dir (``harness_pid_file``: JSON
- ``{"pid", "started"}`` or a bare pid). The daemon is accepted only while it is this user's live
- ``browser_harness.daemon`` leading its own session and its spawn tag names this process, so a
- stale or planted pid file, or a daemon another Hermes started, never reaches the supervisor.
- Runs after every browser_exec (and while a long one runs, until the daemon is owned): a pid
- already owned for this daemon costs one file read.
- Returns True once the daemon is owned, False when it can never be (Windows, a managed worker, a
- daemon another Hermes started), None when there is nothing to own yet."""
- if os.name != "posix":
- return False # Windows: the harness daemon is outside this process's job; nothing to register.
- from agent.runtime_session_store import is_worker_process
- if is_worker_process():
- # A managed per-turn worker never owns it, so a killed worker does not take the browser the
- # next turn's worker drives along (its clean exit still runs the atexit browser cleanup).
- return False
- import psutil
- daemon = daemon_identity(env)
- pid = _published_daemon_pid(env)
- if pid is None:
- logger.debug("browser_exec: no harness daemon pid published for %s", daemon[-1])
- return None
- try:
- with _owned_daemon_groups_lock:
- if _owned_daemon_groups.get(daemon) == pid:
- return True
- proc = psutil.Process(pid)
- argv = proc.cmdline()
- if (os.getsid(pid) != pid or proc.uids().real != os.getuid() # windows-footgun: ok — posix-only, returned above
- or argv[argv.index("-m") + 1] != "browser_harness.daemon"):
- return None
- if not spawned_by_this_process(pid):
- return False # another Hermes on the same BU_NAME started it: that Hermes owns it
- except (OSError, ValueError, IndexError, psutil.Error) as exc:
- logger.debug("browser_exec: no harness daemon to supervise for %s: %s", daemon[-1], exc)
- return None
- with _owned_daemon_groups_lock:
- replaced = _owned_daemon_groups.get(daemon)
- _owned_daemon_groups[daemon] = pid
- own_process_groups([pid])
- register_child(pid, "browser-harness-daemon")
- if replaced is not None:
- release_process_group_once_empty(replaced, timeout=0) # the daemon it replaced; a survivor stays covered
- return True
- def _route_exec(env: dict, session: str, task_id: Optional[str], local: bool, local_sidecar: bool,
- route_key: str) -> tuple[Optional[str], bool]:
- """``_route_backend`` for one browser_exec call; returns ``(error, runs on the local sidecar)``.
- A real-profile fallback is remembered per route: the route keeps its fresh browser and the note
- waits until a result carries it (``_take_real_profile_note``)."""
- route_err = _route_backend(env, session, task_id, local, force_local_sidecar=local_sidecar,
- real_profile=route_key not in _real_profile_fallback)
- note = env.pop(_REAL_PROFILE_FALLBACK_SENTINEL, None)
- if note:
- _real_profile_fallback.setdefault(route_key, note)
- return route_err, local_sidecar or bool(env.pop(_LOCAL_FALLBACK_SENTINEL, None))
- def _route_exec_current(env: dict, session: str, task_id: Optional[str], local: bool,
- local_sidecar: bool, route_key: str) -> tuple[dict, Optional[str], bool, int]:
- """``_route_exec`` on a copy of ``env`` -> ``(routed env, error, sidecar, generation)``, re-routed (at
- most 3 tries) when /browser connect published meanwhile; ``_reload_if_republished`` covers dispatch."""
- for _attempt in range(3):
- routed = dict(env)
- generation = _live_generation()
- route_err, sidecar = _route_exec(routed, session, task_id, local, local_sidecar, route_key)
- if route_err or _live_generation() == generation:
- break
- return routed, route_err, sidecar, generation
- def _reload_if_republished(env: dict, generation: int) -> None:
- """Stop this call's daemon if the profile published a new endpoint while it spawned on the old one."""
- if _live_generation() == generation:
- return
- daemon = daemon_identity(env)
- with _driven_daemons_lock:
- _driven_daemons.pop(daemon, None)
- _reload_daemons([daemon])
- def _take_real_profile_note(route_key: str) -> dict:
- """The fallback note for the first result this route returns after falling back, then nothing."""
- if route_key not in _real_profile_fallback or route_key in _real_profile_note_delivered:
- return {}
- _real_profile_note_delivered.add(route_key)
- return {"real_profile_unavailable": _real_profile_fallback[route_key]}
- def browser_exec(code: str, session: str = "", timeout_s: int = _DEFAULT_TIMEOUT_S,
- task_id: Optional[str] = None, local: bool = False):
- """Run Python code through the browser-use CLI, and return its output"""
- from agent.redact import redact_sensitive_text
- from tools.registry import tool_error, tool_result
- if not code or not code.strip():
- return tool_error("No code provided. Pass Python that uses the pre-imported helpers, e.g. new_tab(\"https://example.com\") then print(page_info()).")
- # The verdict is carried into the route, never re-derived: the predicate resolves DNS, so a second
- # evaluation could relax the floor and still pick the cloud for a rebinding host.
- verdict = _code_url_verdict(code)
- if verdict.blocked:
- return tool_error(verdict.blocked)
- # One endpoint per call: any private URL sends the whole call to the local sidecar. A call with no URL
- # literal stays on the route the previous call took.
- route_key = _backend_cache_key(task_id, session)
- local_sidecar = verdict.local_sidecar if _URL_RE.search(code) else _exec_route_is_sidecar.get(route_key, False)
- cmd = _find_cli()
- if not cmd:
- return tool_error("browser-harness is missing from Hermes's Python environment. "
- "Run `hermes update` to re-sync it.")
- env = _base_subprocess_env()
- if session:
- if not _SESSION_RE.match(session):
- return tool_error(f"Invalid session name {session!r}: use 1-64 letters, digits, "
- "dashes, or underscores (e.g. 'r7k2').")
- env["BU_NAME"] = session
- env, route_err, local_sidecar, generation = _route_exec_current(
- env, session, task_id, bool(local), local_sidecar, route_key)
- if route_err:
- return tool_error(route_err)
- if local_sidecar:
- env["BU_NAME"] = _sidecar_daemon_name(session)
- fit_default_socket_dir(env) # BU_NAME is final: a deep HOME's socket would overflow sun_path
- _exec_route_is_sidecar[route_key] = local_sidecar
- local_engine = env.pop(_BOT_DESKTOP_BROWSER_SENTINEL, None)
- bot_desktop_browser = bool(local_engine)
- _exec_local_engine.set(local_engine) # only local browsers carry the sentinel
- # SHARED browser (/browser connect CDP override): pin each named session to its own tab (see
- # _OWN_TAB_PREAMBLE). Private per-name browsers skip this — nothing to collide with.
- private_browser = env.pop(_PRIVATE_BROWSER_SENTINEL, None) # always pop: never exported to the CLI
- if session and not private_browser:
- code = _OWN_TAB_PREAMBLE + code
- # Every backend: capture_screenshot() falls back to jpeg instead of hanging on a png a hidden tab never
- # renders; only on a cloud browser does it (and new_tab) bring the tab to the front (tools/browser_use_shot.py).
- from tools.browser_use_shot import ACTIVATE_NEVER, harness_command
- cmd = harness_command(cmd, activate=env.pop(_CLOUD_BROWSER_SENTINEL, None) or ACTIVATE_NEVER)
- workspace = _workspace_dir(task_id)
- if workspace:
- env["BH_AGENT_WORKSPACE"] = workspace
- # BU_AUTOSPAWN makes the CLI start a Browser Use cloud browser when no local
- # Chrome/CDP endpoint is reachable (their API key authenticates it)
- if "BU_AUTOSPAWN" not in env and is_legacy_browser_use_cloud_config(_read_browser_cfg()):
- env["BU_AUTOSPAWN"] = "1"
- timeout = _clamp_timeout(timeout_s)
- started = time.time()
- def dispatch() -> dict[str, Any]:
- _attach_vault_supervisor(env, task_id)
- with _driven_daemons_lock:
- daemon = daemon_identity(env)
- _driven_daemons[daemon] = hermes_home_key()
- # The first call starts the daemon and may run long (navigation, user code): own it as soon as
- # it publishes its pid, so a Hermes killed mid-call takes it along (the call's end owns it too).
- from agent.periodic_scheduler import schedule
- owning = schedule(lambda: _own_harness_daemon_until_owned(env), _OWN_DURING_CALL_S)
- try:
- return {"proc": _run_cli_killing_process_group(cmd, code, env, timeout)}
- except subprocess.TimeoutExpired:
- return {"error_result": tool_error(
- f"browser-use exec timed out after {timeout}s. The daemon may still be working; retry "
- f"with a larger timeout_s (max {_MAX_TIMEOUT_S}), or split the work into several calls that "
- "append to workspace files — anything already written to the workspace is preserved."
- )}
- except OSError as e:
- return {"error_result": tool_error(f"Failed to launch browser-use CLI: {e}")}
- finally:
- owning.cancel(wait=5) # an in-flight check finishes before the post-call one runs
- if bot_desktop_browser:
- from tools.browser_session import run_fenced
- dispatched = run_fenced({"features": {"local": True}}, dispatch)
- else:
- dispatched = dispatch()
- _reload_if_republished(env, generation)
- _own_harness_daemon(env)
- if "proc" not in dispatched:
- if "error_result" in dispatched:
- return dispatched["error_result"]
- return tool_result(dispatched)
- proc = dispatched["proc"]
- # browser_vault fill registers injected values with this forced model-egress
- # boundary. Preserve raw stdout only for screenshot-path detection below.
- result = {
- "success": proc.returncode == 0,
- "exit_code": proc.returncode,
- "output": redact_sensitive_text(proc.stdout, force=True),
- }
- if workspace:
- result["workspace"] = workspace
- if session:
- result["session"] = session
- result.update(_take_real_profile_note(route_key))
- stderr = redact_sensitive_text((proc.stderr or "").strip(), force=True)
- if len(stderr) > _STDERR_CAP_CHARS:
- stderr = stderr[:_STDERR_CAP_CHARS] + "\n… (stderr truncated)"
- if stderr:
- result["stderr"] = stderr
- screenshot = _find_screenshot(proc.stdout, started)
- if screenshot:
- result["screenshot_path"] = screenshot
- native = _native_screenshot_result(result, screenshot)
- if native is not None:
- return native
- return tool_result(result)
- _HEADER_BASE = (
- "Drive a real browser for JS pages, logins, forms and clicks (Browser Use CLI). `code` runs as full "
- "Python (stdlib available) with pre-imported browser helpers; stdout comes back in the result. Start `code` with a one-line "
- "comment describing the step for the user in plain language, max 60 chars "
- "(e.g. `# Searching Amazon for paper towels`) — the UI shows it as the step label.\n\n"
- "STATE: the browser session and workspace persist across calls; Python variables do NOT (fresh "
- "interpreter each call). The workspace dir is $BH_AGENT_WORKSPACE (also `workspace` in every result); "
- "functions defined in agent_helpers.py there are auto-imported into every call. For multi-item tasks "
- "('all N products / every entry'), append each batch to a JSON/CSV file in the workspace, then read it "
- "back and aggregate in code — dedupe/count/sort with Python, not in your head — and verify the "
- "collected count against what was asked before answering.\n\n"
- "Batch each sub-procedure (navigate, wait, extract, act) into one call — do not spend a call per "
- "action — but for long extractions prefer several medium calls that append to workspace files over "
- "one giant call, so progress survives timeouts."
- )
- _HEADER_VISION = (
- " Screenshots are attached to your context automatically: when the exec output contains a "
- "capture_screenshot() path, the image arrives with this tool's result and you inspect it directly "
- "with your own vision — never send browser screenshots to a separate vision tool."
- )
- _TEXT_FIRST = (
- "page_info() for state, js() for reading/extracting DOM text, fill_input(selector, text) for inputs, and "
- "js(\"document.querySelector('…').click()\") for clicks."
- )
- # The non-native header states a transport fact (screenshots do not ride back inside THIS
- # tool's results: routing is not ``native`` — e.g. an explicit ``auxiliary.vision`` pin — or
- # the route rejects tool-result media), never a verdict on the model's own vision: models
- # quote "your model cannot view images" against themselves and skip user images (#131568).
- # model_tools.get_tool_definitions() appends the vision_analyze pointer after
- # SCREENSHOTS_DETACHED when the session actually has that tool.
- SCREENSHOTS_DETACHED = " Screenshots are not attached to this tool's results."
- _HEADER_TEXT_ONLY = SCREENSHOTS_DETACHED + " Work text-first: " + _TEXT_FIRST
- # Lightpanda: no graphical renderer, and one CDP connection holds one page — a second
- # Target.createTarget fails with TargetAlreadyLoaded (drop the new_tab() sentence once
- # lightpanda-io/browser#1962 lands).
- _HEADER_LIGHTPANDA = (
- " The local engine is Lightpanda (no graphical renderer, one page per session): capture_screenshot() "
- "is unavailable, so work text-first: " + _TEXT_FIRST + " Navigate with new_tab(url) exactly once, then "
- "goto_url(url) for every later navigation — a second new_tab() fails with TargetAlreadyLoaded."
- )
- # Pinned quick-reference for the CLI's pre-imported helpers, replacing the live
- # ``browser-use skill`` fetch (uncontrolled third-party text in every schema: version
- # drift, supply-chain exposure, byte-unstable prompt). A/B benchmarked ~equal.
- _HELPERS_DIGEST = (
- "\n\nHELPERS (pre-imported): new_tab(url) opens/navigates (use for the FIRST navigation), goto_url(url) "
- "navigates the current tab, wait_for_load() after navigation, page_info() summarizes the current page "
- "state, js(expr) evaluates a JS expression and returns its value (js('document.title'); wrap function "
- "bodies as js('(() => {...})()') — a bare '() => {...}' returns the function itself, uncalled), "
- "fill_input(selector, text) types into inputs, click_at_xy(x, y) clicks viewport coordinates, "
- "capture_screenshot() saves and prints a screenshot path, cdp('Domain.method', **kwargs) is raw CDP — "
- "cdp('Accessibility.getFullAXTree')['nodes'] lists every element's role/name/backendDOMNodeId (filter "
- "in Python before printing; it is thousands of nodes), then cdp('DOM.getBoxModel', backendNodeId=n) "
- "gives click coordinates. ensure_real_tab() recovers from a stale/internal tab. Login walls: never guess "
- "credentials; see the vault note below if present, otherwise stop and ask the user."
- )
- def _description_header() -> str:
- """Header tailored to whether screenshots ride back natively in this tool's results."""
- if _lazy_call("tools.browser_lightpanda", "lightpanda_engine_status", (False, ""),
- "lightpanda engine status unavailable")[0]: # no screenshots, whatever the model sees
- return _HEADER_BASE + _HEADER_LIGHTPANDA
- if _lazy_call("tools.vision_tools", "_should_use_native_vision_fast_path", False, ""):
- return _HEADER_BASE + _HEADER_VISION
- return _HEADER_BASE + _HEADER_TEXT_ONLY
- def _dynamic_schema_overrides() -> dict:
- overrides: dict = {"description": _description_header() + _HELPERS_DIGEST}
- # ``local`` exists ONLY when the user consented to real-profile browsing — everyone
- # else's schema carries zero extra surface. The caller memoizes on config.yaml mtime,
- # so toggling consent applies next session, not mid-chat.
- if _real_profile_consented():
- props = dict(BROWSER_EXEC_SCHEMA["parameters"]["properties"])
- props["local"] = {
- "type": "boolean", "default": False,
- "description": ("Drive the user's own local browser (a Hermes-managed copy of their real "
- "default-Chromium profile, logins/cookies included) instead of the configured "
- "cloud browser backend. Use when the user asks to act as themselves — their "
- "accounts, their sessions. No-op when the backend is already local. Default false."),
- }
- overrides["parameters"] = {**BROWSER_EXEC_SCHEMA["parameters"], "properties": props}
- return overrides
- BROWSER_EXEC_SCHEMA = {
- "name": "browser_exec",
- # Static fallback description, used only when the managed CLI is unavailable
- "description": (_HEADER_BASE + _HELPERS_DIGEST
- + "\n\n(The browser-use CLI is not installed yet. Install it with `hermes tools` (Browser Automation → Browser Use).)"),
- "parameters": {
- "type": "object",
- "properties": {
- "code": {"type": "string", "description": "Python code to execute using the pre-imported browser helpers. Use print(...) for any data you need back."},
- "session": {"type": "string", "description": "Named isolated browser session — its own daemon and (on cloud backends) own browser, so concurrent tasks don't share tabs. Reuse the same name on every related call; omit for the shared default session."},
- "timeout_s": {"type": "integer", "default": _DEFAULT_TIMEOUT_S,
- "description": f"Max seconds to wait for the code to finish (default {_DEFAULT_TIMEOUT_S}, max {_MAX_TIMEOUT_S})."},
- },
- "required": ["code"],
- },
- }
- # browser_exec is additionally gated at tool-definition time — sessions whose toolsets
- # lack ``terminal`` never see it (model_tools._compute_tool_definitions). check_fn only
- # answers "is browser-harness importable"; surface policy lives with the session.
- from tools.registry import registry
- # The LOCAL browser this context's browser_exec call ran on ("local" Chromium or "lightpanda", from its session),
- # else None: a configured cloud provider did not serve a hybrid ::local sidecar, cloud-fallback or real-profile
- # call, and browser.engine read afterwards may name another engine than the session's, so neither labels it.
- _exec_local_engine: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar("browser_exec_local_engine",
- default=None)
- def _handle_browser_exec(args: dict, **kw):
- """Registry handler; each call counts against the backend that served it (shared metrics, opt-in)."""
- from hermes_cli.observability.shared_metrics_loop import record_browser_call
- from tools.browser_session_cloud import browser_backend_name
- token = _exec_local_engine.set(None)
- try:
- return record_browser_call(lambda: browser_exec(
- code=args.get("code", ""), session=args.get("session", "") or "",
- timeout_s=args.get("timeout_s", _DEFAULT_TIMEOUT_S), task_id=kw.get("task_id"),
- local=bool(args.get("local", False)),
- ), lambda: browser_backend_name(local_engine=_exec_local_engine.get()))
- finally:
- _exec_local_engine.reset(token)
- registry.register(
- name="browser_exec",
- toolset="browser-use",
- schema=BROWSER_EXEC_SCHEMA,
- handler=_handle_browser_exec,
- check_fn=browser_exec_available,
- dynamic_schema_overrides=_dynamic_schema_overrides,
- emoji="🌐",
- )
|