browser_use_shot.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. """Screenshot and tab helpers that browser_exec installs over browser-harness's own.
  2. Chrome draws no compositor frame for a hidden tab, and some remotes (Browser Use cloud) never answer
  3. ``Page.captureScreenshot`` with ``format: "png"`` for one, so the harness's ``capture_screenshot()`` hangs
  4. for its full 60 s IPC timeout. Its ``new_tab()`` and its daemon both create tabs with ``background=True``,
  5. so the tab the agent drives is often hidden. browser_harness is a pinned third-party dependency; this
  6. module is loaded by file path inside the harness process (``harness_command()``), so it imports nothing
  7. from Hermes and touches the harness only through the helpers module it is given.
  8. """
  9. from __future__ import annotations
  10. import base64
  11. import io
  12. import logging
  13. import os
  14. import tempfile
  15. from typing import Any, Callable
  16. # A png that has not arrived by then will not: a visible tab answers in well under a second, a hidden
  17. # one on Browser Use cloud never does. jpeg on the same hidden tab answers in a few seconds.
  18. PNG_DEADLINE_S = 10.0
  19. JPEG_DEADLINE_S = 60.0
  20. _HARNESS_MODULE = "browser_harness.helpers"
  21. _log = logging.getLogger("hermes_browser_use_shot")
  22. def _default_shot_path() -> str:
  23. try:
  24. from browser_harness import _ipc # the harness's own per-user temp dir
  25. return str(_ipc._TMP / "shot.png")
  26. except (ImportError, AttributeError):
  27. return os.path.join(tempfile.gettempdir(), "shot.png")
  28. def _bring_to_front_if_hidden(cdp: Callable[..., Any], current_tab: Callable[[], dict]) -> None:
  29. """Activate the attached tab when the page reports it is hidden. A visible tab is left alone, so a
  30. capture never takes window focus it does not need."""
  31. try:
  32. state = cdp("Runtime.evaluate", expression="document.visibilityState", returnByValue=True)
  33. if (state.get("result") or {}).get("value") != "hidden":
  34. return
  35. cdp("Target.activateTarget", targetId=current_tab()["targetId"])
  36. # A CDP error (Lightpanda has no Target.activateTarget) or a slow IPC round trip: capture anyway.
  37. except (RuntimeError, OSError, KeyError):
  38. return
  39. def _write_shot(data: bytes, path: str, fmt: str) -> None:
  40. """Write ``data`` (encoded as ``fmt``) to ``path``, re-encoding a jpeg as png when the path says png:
  41. callers and the screenshot embed trust the extension."""
  42. if fmt == "jpeg" and path.lower().endswith(".png"):
  43. from PIL import Image
  44. Image.open(io.BytesIO(data)).save(path, format="PNG")
  45. return
  46. with open(path, "wb") as f:
  47. f.write(data)
  48. def make_capture_screenshot(cdp: Callable[..., Any], current_tab: Callable[[], dict],
  49. activate_hidden: Callable[[], bool] = lambda: False) -> Callable[..., str]:
  50. """``activate_hidden()`` is true only on a browser no human watches (cloud): on the user's own or
  51. connected Chrome, activating the agent's tab would switch the tab they are looking at. Every backend
  52. keeps the bounded png attempt and the jpeg fallback."""
  53. def capture_screenshot(path=None, full=False, max_dim=None):
  54. """Save a PNG of the current viewport. Set max_dim=1800 on a 2x display to keep the file under
  55. the 2000px-per-side limit some image-aware LLMs enforce."""
  56. path = os.fspath(path or _default_shot_path()) # a pathlib.Path works too
  57. if activate_hidden():
  58. _bring_to_front_if_hidden(cdp, current_tab)
  59. fmt = "png"
  60. try:
  61. r = cdp("Page.captureScreenshot", _response_timeout=PNG_DEADLINE_S,
  62. format="png", captureBeyondViewport=full)
  63. except TimeoutError:
  64. fmt = "jpeg"
  65. try:
  66. r = cdp("Page.captureScreenshot", _response_timeout=JPEG_DEADLINE_S,
  67. format="jpeg", quality=90, captureBeyondViewport=full)
  68. except TimeoutError as e:
  69. raise RuntimeError(
  70. f"Page.captureScreenshot timed out (png after {PNG_DEADLINE_S:g}s, "
  71. f"jpeg after {JPEG_DEADLINE_S:g}s)") from e
  72. _write_shot(base64.b64decode(r["data"]), path, fmt)
  73. if max_dim:
  74. from PIL import Image
  75. img = Image.open(path)
  76. if max(img.size) > max_dim:
  77. img.thumbnail((max_dim, max_dim))
  78. img.save(path)
  79. return path
  80. return capture_screenshot
  81. def make_new_tab(new_tab: Callable[..., Any], cdp: Callable[..., Any],
  82. activate: Callable[[], bool] = lambda: True) -> Callable[..., Any]:
  83. def new_tab_in_front(url="about:blank"):
  84. """Open (or reuse a blank) tab, navigate to url, and bring it to the front on a cloud browser."""
  85. tid = new_tab(url)
  86. if tid and activate():
  87. try:
  88. cdp("Target.activateTarget", targetId=tid)
  89. # The tab exists either way; a provider that rejects activation leaves it in the background.
  90. except (RuntimeError, OSError) as e:
  91. _log.warning("new_tab: Target.activateTarget failed, tab left in background: %s", e)
  92. return tid
  93. return new_tab_in_front
  94. def _is_harness_helper(fn: Any) -> bool:
  95. """True for the harness's own helper; a same-named function from the agent's workspace
  96. ``agent_helpers.py`` (loaded into the helpers module at import) is the agent's choice."""
  97. return callable(fn) and getattr(fn, "__module__", "") == _HARNESS_MODULE
  98. # Which browser the tabs may be activated on, decided by browser_exec from the route it resolved.
  99. ACTIVATE_NEVER = "never" # local, connected or real-profile Chrome: a human may be looking at it
  100. ACTIVATE_ALWAYS = "always" # a cloud provider's session whose CDP endpoint Hermes itself resolved
  101. ACTIVATE_IF_CLOUD = "if-cloud" # BU_AUTOSPAWN: the harness picks the browser (a listening local Chrome wins)
  102. def _harness_daemon_is_cloud() -> bool:
  103. """True only when the live harness daemon says it drives a Browser Use cloud browser (it was started
  104. with that browser's id). Asked per call, after the harness has chosen or replaced its daemon; an
  105. unreachable or older daemon counts as not cloud."""
  106. from browser_harness import admin
  107. return admin.daemon_browser_kind() == "cloud"
  108. def _activation_check(mode: str) -> Callable[[], bool]:
  109. if mode == ACTIVATE_IF_CLOUD:
  110. return _harness_daemon_is_cloud
  111. return (lambda: True) if mode == ACTIVATE_ALWAYS else (lambda: False)
  112. def install(helpers: Any, activate: str = ACTIVATE_NEVER) -> None:
  113. """Rebind the helpers on the ``browser_harness.helpers`` module before ``browser_harness.run`` star-imports
  114. them, so the agent's code, the harness's step tracer and recorder all see these. Activation is for
  115. browsers no human is watching (cloud): there a background tab only costs throttling and blank
  116. screenshots, so new tabs and hidden tabs being captured are brought to the front; on a user's own
  117. Chrome the harness deliberately leaves the visible tab alone. ``activate`` is an ``ACTIVATE_*`` mode."""
  118. check = _activation_check(activate)
  119. if _is_harness_helper(helpers.capture_screenshot):
  120. helpers.capture_screenshot = make_capture_screenshot(helpers.cdp, helpers.current_tab,
  121. activate_hidden=check)
  122. if activate != ACTIVATE_NEVER and _is_harness_helper(helpers.new_tab):
  123. helpers.new_tab = make_new_tab(helpers.new_tab, helpers.cdp, activate=check)
  124. _HARNESS_MODULE_ARGS = ["-m", "browser_harness.run"]
  125. _LAUNCHER = """\
  126. import importlib.util, runpy
  127. spec = importlib.util.spec_from_file_location('hermes_browser_use_shot', {path!r})
  128. shot = importlib.util.module_from_spec(spec); spec.loader.exec_module(shot)
  129. import browser_harness.helpers
  130. shot.install(browser_harness.helpers, activate={activate!r})
  131. runpy.run_module('browser_harness.run', run_name='__main__', alter_sys=True)
  132. """
  133. def harness_command(cmd: list[str], activate: str) -> list[str]:
  134. """``python -m browser_harness.run`` as a launcher that installs these helpers first. The model's code
  135. still arrives untouched on stdin (no preamble shifting its traceback line numbers). This file is loaded
  136. by path: the harness interpreter has no Hermes checkout on sys.path."""
  137. if cmd[1:] != _HARNESS_MODULE_ARGS:
  138. return cmd
  139. return [cmd[0], "-c", _LAUNCHER.format(path=os.path.abspath(__file__), activate=activate)]