browser_url_policy.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202
  1. """URL safety for ``browser_exec`` (pre-scan of URL literals, hybrid local routing) and the SSRF helpers
  2. ``browser_cdp`` uses on raw CDP calls.
  3. Ordered floors: cloud metadata / IMDS refused unconditionally (a local Chromium on a cloud VM still
  4. reaches the host IMDS); private addresses refused unless the backend is local, the URL is served by the
  5. hybrid local sidecar, or ``browser.allow_private_urls``; then the website allow/deny policy.
  6. """
  7. from __future__ import annotations
  8. import ipaddress
  9. import logging
  10. import re
  11. import socket
  12. import urllib.parse
  13. from typing import Callable, Optional
  14. from tools.url_safety import (
  15. _BLOCKED_HOSTNAMES,
  16. _authority_ambiguity,
  17. _is_always_blocked_ip,
  18. _is_declared_fake_ip,
  19. _parse_ip,
  20. _url_hostname,
  21. is_always_blocked_url,
  22. is_safe_url,
  23. normalize_url_for_request,
  24. )
  25. from tools.website_policy import check_website_access
  26. logger = logging.getLogger(__name__)
  27. _PRIVATE_HOST_SUFFIXES = (".localhost", ".local", ".lan", ".internal")
  28. _CGNAT = ipaddress.ip_network("100.64.0.0/10")
  29. # URL-shaped literals in a JS expression (http/https only). fetch/XHR to a private host never updates
  30. # ``location.href``, so the literals are pre-screened instead. A literal ends at a quote, whitespace,
  31. # ``)``, ``]`` or ``<>``, except that one ``[...]`` segment (a bracketed IPv6 host such as
  32. # ``http://[::1]:8080/``) is read whole.
  33. _URL_BODY = r"""(?=[^\s'"`)\]<>])(?:[^\s'"`)\[\]<>]*\[[^\s'"`)\[\]<>]*\])?[^\s'"`)\]<>]*"""
  34. _JS_URL_LITERAL_RE = re.compile(r"https?://" + _URL_BODY, re.IGNORECASE)
  35. # Plus scheme-relative ``//host``, ``///host`` or ``\\host`` (browsers fetch each with the page's scheme and
  36. # read ``\`` as ``/``); not ``a//b`` or ``// note``. One scan, so a ``//`` inside an http(s) literal's path or
  37. # query (``https://x/?next=//y``) is part of that literal, not a second target.
  38. _URL_OR_SCHEME_RELATIVE_RE = re.compile(r"""(?:https?://|(?<![\w:/\\])[/\\]{2,}(?=[^/\\\s]))""" + _URL_BODY,
  39. re.IGNORECASE)
  40. _URL_BODY_RE = re.compile(_URL_BODY)
  41. # ``)``, ``]``, ``<`` and ``>`` are legal in userinfo, so a literal cut at one inside its authority may be
  42. # ``http://x)@host``: the rest of the authority (up to ``/ ? # \``) through its last ``@``.
  43. _USERINFO_TAIL_RE = re.compile(r"""[^\s'"`/?#\\]*@""")
  44. _LITERAL_HEAD_RE = re.compile(r"(?:https?:)?[/\\]+", re.IGNORECASE)
  45. _NUMERIC_HOST_RE = re.compile(r"[0-9a-fx.]+") # _url_hostname lowercases
  46. def _err(error: str, **extra) -> dict:
  47. return {"success": False, "error": error, **extra}
  48. def _private_ip(host: str) -> Optional[bool]:
  49. """None when ``host`` is not an IP literal."""
  50. try:
  51. ip = ipaddress.ip_address(host)
  52. except ValueError:
  53. return None
  54. if _is_declared_fake_ip(ip):
  55. return False
  56. return ip.is_private or ip.is_loopback or ip.is_link_local or ip in _CGNAT
  57. def url_is_private(url: str) -> bool:
  58. """True when the URL's host is (or resolves to) a private/LAN/loopback/CGNAT address.
  59. Routing oracle only: DNS failures are NOT private (the configured backend surfaces the error). A local
  60. proxy's declared fake-ip sentinel (``security.fake_ip_ranges``) is not private: the name is public and
  61. the cloud browser resolves it itself."""
  62. try:
  63. hostname = (urllib.parse.urlparse(url).hostname or "").strip().lower().rstrip(".")
  64. if not hostname:
  65. return False
  66. literal = _private_ip(hostname)
  67. if literal is not None:
  68. return literal
  69. if hostname == "localhost" or hostname.endswith(_PRIVATE_HOST_SUFFIXES):
  70. return True
  71. try:
  72. addr_info = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
  73. except socket.gaierror:
  74. return False
  75. return any(_private_ip(sockaddr[0]) for *_, sockaddr in addr_info)
  76. except (ValueError, OSError, UnicodeError) as exc:
  77. logger.debug("URL-privacy check failed for %s: %s", url, exc)
  78. return False
  79. def hybrid_routes_locally(url: str) -> bool:
  80. """True when ``url`` belongs on the hybrid LOCAL sidecar instead of the cloud provider: a provider
  81. configured, ``browser.auto_local_for_private_urls`` on, a private URL, and no CDP override (it owns
  82. the whole session)."""
  83. from tools.browser_session_cdp import get_cdp_override_raw
  84. from tools.browser_session_cloud import auto_local_for_private_urls, get_cloud_provider
  85. return (
  86. not get_cdp_override_raw()
  87. and get_cloud_provider() is not None
  88. and auto_local_for_private_urls()
  89. and url_is_private(url)
  90. )
  91. def _secret_url_error(url: str) -> Optional[dict]:
  92. """Refuse URLs embedding an API key/token (raw and URL-decoded, catching ``%2D`` tricks): a prompt
  93. injection could otherwise exfiltrate secrets through the URL."""
  94. from agent.redact import _PREFIX_RE
  95. if _PREFIX_RE.search(url) or _PREFIX_RE.search(urllib.parse.unquote(url)):
  96. return _err("Blocked: URL contains what appears to be an API key or token. Secrets must not be sent in URLs.")
  97. return None
  98. def _url_policy_error(url: str, *, auto_local: bool = False) -> Optional[dict]:
  99. """Backend-aware checks on a normalized URL; None if allowed.
  100. Credential-NAMED query params (``?token=``, ``?signature=``) are deliberately not a floor: magic links,
  101. OAuth callbacks and signed CDN assets are how the agent signs in, and a cloud browser already sees every
  102. cookie of the session. Hermes' own secrets in a URL are caught by ``_secret_url_error``."""
  103. from tools.browser_session_cloud import allow_private_urls, is_local_backend
  104. if is_always_blocked_url(url): # unconditional: IMDS via any browser exfiltrates host credentials (#16234)
  105. return _err("Blocked: URL targets a cloud metadata endpoint")
  106. if not is_local_backend() and not auto_local and not allow_private_urls() and not is_safe_url(url):
  107. return _err("Blocked: URL targets a private or internal address")
  108. blocked = check_website_access(url)
  109. if blocked:
  110. return _err(blocked["message"],
  111. blocked_by_policy={"host": blocked["host"], "rule": blocked["rule"], "source": blocked["source"]})
  112. return None
  113. def evaluate_url_safety(url: str, *, auto_local: bool = False) -> Optional[dict]:
  114. """None if ``url`` is safe to open, else an error dict. ``auto_local`` (served by the hybrid local
  115. sidecar) relaxes only the private-address floor, never the cloud-metadata one."""
  116. err = _secret_url_error(url)
  117. if err is None:
  118. url = normalize_url_for_request(url)
  119. err = _secret_url_error(url)
  120. return err or _url_policy_error(url, auto_local=auto_local)
  121. def eval_ssrf_guard_active() -> bool:
  122. """True when page-driven private-network access must be guarded: a non-local backend (cloud browser,
  123. off-host CDP, or a containerized terminal whose host browser reaches networks it can't) without
  124. ``browser.allow_private_urls``."""
  125. from tools.browser_session_cloud import allow_private_urls, is_local_backend
  126. return not is_local_backend() and not allow_private_urls()
  127. def url_blocked(url: str) -> bool:
  128. """True when ``url`` hits the always-blocked cloud-metadata floor or fails the SSRF guard."""
  129. return is_always_blocked_url(url) or not is_safe_url(url)
  130. def url_literals(text: str, *, protocol_relative: bool = False) -> list[str]:
  131. """Every ``http(s)://`` literal in free text (a JS expression, an HTML snippet, a CDP param); with
  132. ``protocol_relative``, each scheme-relative ``//host/...`` literal too, kept scheme-less. A literal cut
  133. inside its authority with an ``@`` ahead is also returned as the host a browser reads after that ``@``
  134. (``http://x)@host/`` -> ``http://host/``), next to the literal as cut."""
  135. if not isinstance(text, str):
  136. return []
  137. found = []
  138. for m in (_URL_OR_SCHEME_RELATIVE_RE if protocol_relative else _JS_URL_LITERAL_RE).finditer(text):
  139. found.append(m.group())
  140. head = _LITERAL_HEAD_RE.match(m.group()).group()
  141. if any(c in "/?#\\" for c in m.group()[len(head):]):
  142. continue # the authority ended inside the literal: its host is fixed
  143. tail = _USERINFO_TAIL_RE.match(text, m.end())
  144. host = tail and _URL_BODY_RE.match(text, tail.end())
  145. if host:
  146. found.append(head + host.group())
  147. return [m.rstrip(".,;") for m in found]
  148. def literal_targets_metadata(literal: str) -> bool:
  149. """True when a URL literal hits the cloud-metadata floor. A scheme-relative one (``//host``,
  150. ``\\\\host``) is judged by its literal host only (a metadata hostname or IP, decimal/hex/octal IPv4
  151. spellings included), never resolved, so a ``//TODO`` comment costs no DNS lookup."""
  152. if not literal.startswith(("/", "\\")):
  153. return is_always_blocked_url(literal)
  154. url = "http://" + literal.lstrip("/\\").replace("\\", "/")
  155. try:
  156. if _authority_ambiguity(url, floor=True): # a control character can move the host, as for http(s)
  157. return True
  158. host = _url_hostname(url, percent_decode=True)
  159. except ValueError:
  160. return False
  161. ip = _parse_ip(host)
  162. if ip is None and _NUMERIC_HOST_RE.fullmatch(host):
  163. try: # decimal, hex, octal or short IPv4 (``//0xa9fea9fe``), read as browsers do; still no DNS
  164. ip = ipaddress.IPv4Address(socket.inet_aton(host))
  165. except (OSError, ValueError):
  166. pass
  167. return host in _BLOCKED_HOSTNAMES or (ip is not None and _is_always_blocked_ip(ip))
  168. def expression_targets_private_url(expression: str, *, blocked: Callable[[str], bool] = url_blocked) -> Optional[str]:
  169. """The first ``http(s)://`` literal in a JS expression that ``blocked`` refuses (best-effort), else None."""
  170. return next((c for c in url_literals(expression) if blocked(c)), None)