url_safety.py 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650
  1. """URL safety checks — blocks requests to private/internal network addresses (SSRF).
  2. ``security.allow_private_urls: true`` disables private-IP blocking (DNS that resolves public
  3. names to private ranges); cloud metadata hostnames/IPs are **always** blocked. A local TUN proxy
  4. that answers DNS with a fake-ip block (Mihomo/Clash fake-ip, Surge enhanced) declares that block
  5. in ``security.fake_ip_ranges`` so its sentinel answers are dialable instead of looking private;
  6. the list is empty by default, so the sentinel stays blocked for everyone else. DNS rebinding
  7. (TOCTOU) is closed for Hermes-owned httpx paths by ``create_ssrf_safe_[async_]client()``, which
  8. re-apply the policy at TCP connect and dial the validated IP while preserving Host/SNI. Redirect
  9. bypass is mitigated by response hooks re-validating each target (``redirect_target_from_response``).
  10. """
  11. import ipaddress
  12. import logging
  13. import os
  14. import socket
  15. import asyncio
  16. import re
  17. from contextlib import contextmanager
  18. from typing import Any, Optional
  19. from urllib.parse import parse_qsl, quote, unquote, urljoin, urlparse, urlsplit, urlunsplit
  20. from hermes_constants import get_hermes_home_override
  21. from utils import is_truthy_value
  22. logger = logging.getLogger(__name__)
  23. # Proxy env vars: when set, the runtime should delegate DNS to the proxy.
  24. _PROXY_ENV_VARS = ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy")
  25. _HTTP_SCHEMES = frozenset({"http", "https"})
  26. _IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
  27. def _proxy_is_configured() -> bool:
  28. return any(os.environ.get(v) for v in _PROXY_ENV_VARS)
  29. def normalize_url_for_request(url: str) -> str:
  30. """ASCII-safe HTTP URL for Hermes-owned URL tools (IRI -> URI, e.g. ``https://wttr.in/Köln``).
  31. Preserves URL syntax and existing percent escapes while IDNA-encoding the host and
  32. percent-encoding non-ASCII path/query/fragment text. URL tool inputs only — never shell commands."""
  33. if not isinstance(url, str):
  34. return url
  35. raw = url.strip()
  36. if not raw:
  37. return raw
  38. # Repair model-emitted whitespace between scheme separator and authority
  39. # (``https:// docs.example``); that position is never meaningful in HTTP URLs.
  40. raw = re.sub(r"^([A-Za-z][A-Za-z0-9+.-]*://)\s+", r"\1", raw)
  41. try:
  42. parsed = urlsplit(raw)
  43. except ValueError:
  44. return raw
  45. if parsed.scheme.lower() not in _HTTP_SCHEMES:
  46. return raw
  47. netloc, hostname = parsed.netloc, parsed.hostname
  48. if hostname:
  49. try:
  50. ascii_host = hostname.encode("idna").decode("ascii")
  51. except UnicodeError:
  52. ascii_host = hostname
  53. if ascii_host != hostname:
  54. netloc = netloc.replace(hostname, ascii_host, 1)
  55. safe = "/%:@!$&'()*+,;="
  56. return urlunsplit((parsed.scheme, netloc, quote(parsed.path, safe=safe),
  57. quote(parsed.query, safe=safe + "?"), quote(parsed.fragment, safe=safe + "?")))
  58. # Unambiguously credential-bearing query param names. Deliberately narrow: bare
  59. # English words that double as page facets (``code``, ``key``, ``auth``,
  60. # ``session``, ``sig``) are EXCLUDED so ordinary browsing is not blocked.
  61. _SENSITIVE_QUERY_PARAM_NAMES = frozenset({
  62. "access_token", "api_key", "apikey", "auth_token", "authorization", "awsaccesskeyid",
  63. "client_secret", "credential", "credentials", "jwt", "password", "passwd", "secret",
  64. "session_id", "signature", "token", "x_amz_security_token", "x_amz_signature",
  65. "x-amz-security-token", "x-amz-signature"})
  66. def sensitive_query_param_name(url: str) -> Optional[str]:
  67. """First credential-named query parameter in ``url`` (with a value), if any. Checked before
  68. handing URLs to third-party fetch/browser backends: catches opaque magic links, OAuth codes,
  69. signed-URL signatures and custom ``?token=...`` values that prefix-based redaction misses."""
  70. if not isinstance(url, str) or "?" not in url:
  71. return None
  72. try:
  73. parsed = urlsplit(url.strip())
  74. except ValueError:
  75. return None
  76. if parsed.scheme.lower() not in _HTTP_SCHEMES or not parsed.query:
  77. return None
  78. return next((key for key, value in parse_qsl(parsed.query, keep_blank_values=True)
  79. if value and unquote(key).lower() in _SENSITIVE_QUERY_PARAM_NAMES), None)
  80. # Cloud metadata hostnames — always blocked regardless of DNS or config toggle.
  81. _BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal", "metadata.goog"})
  82. # Cloud metadata / credential endpoints (the #1 SSRF target) — always blocked, also in
  83. # IPv4-mapped IPv6 form (resolvers may return ``::ffff:x.x.x.x``; ipaddress treats those as distinct).
  84. _METADATA_V4 = (
  85. "169.254.169.254", # AWS/GCP/Azure/DO/Oracle metadata
  86. "169.254.170.2", # AWS ECS task metadata (task IAM creds)
  87. "169.254.169.253", # Azure IMDS wire server
  88. "100.100.100.200", # Alibaba Cloud metadata
  89. )
  90. _ALWAYS_BLOCKED_IPS = frozenset(
  91. {ipaddress.ip_address(ip) for ip in _METADATA_V4}
  92. | {ipaddress.ip_address("::ffff:" + ip) for ip in _METADATA_V4}
  93. | {ipaddress.ip_address("fd00:ec2::254")} # AWS metadata (IPv6)
  94. )
  95. # Entire link-local range (no legit agent target), plus its IPv4-mapped form.
  96. _ALWAYS_BLOCKED_NETWORKS = tuple(ipaddress.ip_network(n) for n in ("169.254.0.0/16", "::ffff:169.254.0.0/112"))
  97. # Exact HTTPS hostnames allowed to resolve to private/benchmark-space IPs
  98. # (QQ media legitimately resolves to 198.18.0.0/15 behind local proxy infra).
  99. _TRUSTED_PRIVATE_IP_HOSTS = frozenset({"multimedia.nt.qq.com.cn"})
  100. _MAX_SSRF_CONNECT_IPS = 8
  101. # 100.64.0.0/10 (CGNAT, RFC 6598) is neither is_private nor is_global in
  102. # ipaddress — must be blocked explicitly (Tailscale/WireGuard, cloud internal nets).
  103. _CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10")
  104. # RFC 2765 IPv4-translated wrapper ``::ffff:0:a.b.c.d`` — the second form resolvers may use to
  105. # answer an IPv4 name (on the reporting macOS host, fake-IP TUN DNS returns it alongside the
  106. # plain address). ``ip.ipv4_mapped`` does not read it; ``_embedded_ipv4`` handles it explicitly.
  107. _IPV4_TRANSLATED_NETWORK = ipaddress.ip_network("::ffff:0:0:0/96")
  108. # Address classes a ``security.fake_ip_ranges`` declaration can never excuse: a local proxy owns
  109. # none of them, and a declaration is trusted like ``allow_private_urls`` for whatever it names,
  110. # so an entry overlapping one of these (including 0.0.0.0/0 and ::/0) would make real internal
  111. # hosts dialable. Such entries are dropped with a warning instead.
  112. _FAKE_IP_UNDECLARABLE_NETWORKS = tuple(ipaddress.ip_network(n) for n in (
  113. "0.0.0.0/32", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", # unspecified/loopback/RFC 1918
  114. "169.254.0.0/16", "100.64.0.0/10", # link-local, CGNAT
  115. "::/128", "::1/128", "fc00::/7", "fe80::/10", # unspecified, loopback, ULA, link-local
  116. ))
  117. # Global toggle cache (process lifetime; see _global_allow_private_urls).
  118. _allow_private_resolved, _cached_allow_private = False, False
  119. _fake_ip_resolved, _cached_fake_ip_ranges = False, ()
  120. def _global_allow_private_urls() -> bool:
  121. """True when the user has opted out of private-IP blocking. Priority: ``HERMES_ALLOW_PRIVATE_URLS``
  122. env, ``security.allow_private_urls``, legacy ``browser.allow_private_urls``. Profile-scoped turns
  123. (``get_hermes_home_override()`` set) bypass the process-global cache — a multiplex gateway serves
  124. several profiles in one process; the first profile's opt-out must not disable blocking for later ones."""
  125. global _allow_private_resolved, _cached_allow_private
  126. if get_hermes_home_override() is not None:
  127. return _resolve_allow_private_urls()
  128. if not _allow_private_resolved:
  129. _allow_private_resolved, _cached_allow_private = True, _resolve_allow_private_urls()
  130. return _cached_allow_private
  131. def _resolve_allow_private_urls() -> bool:
  132. """Resolve the effective private-URL toggle from the active config scope."""
  133. env_val = os.getenv("HERMES_ALLOW_PRIVATE_URLS", "").strip().lower()
  134. if env_val in {"true", "1", "yes"}:
  135. return True
  136. if env_val in {"false", "0", "no"}:
  137. return False # explicit false does not fall through to config
  138. try:
  139. from hermes_cli.config import read_raw_config
  140. cfg = read_raw_config()
  141. for section in ("security", "browser"): # preferred, then legacy
  142. block = cfg.get(section, {})
  143. if isinstance(block, dict) and is_truthy_value(block.get("allow_private_urls"), default=False):
  144. return True
  145. except Exception:
  146. pass # config unavailable (tests, early import) — keep default
  147. return False
  148. def _reset_allow_private_cache() -> None:
  149. """Reset the cached toggle and the cached fake-ip ranges — only for tests."""
  150. global _allow_private_resolved, _cached_allow_private, _fake_ip_resolved, _cached_fake_ip_ranges
  151. _allow_private_resolved = _cached_allow_private = _fake_ip_resolved = False
  152. _cached_fake_ip_ranges = ()
  153. def _resolve_fake_ip_ranges() -> tuple:
  154. """CIDR blocks this host's local proxy answers DNS with (``security.fake_ip_ranges``).
  155. A TUN proxy in fake-ip mode answers every non-filtered name with an address from its own
  156. block; that answer is the proxy's sentinel, not an internal host's, so treating it as a
  157. private target blocks every outbound fetch on such a host (web_extract, platform attachment
  158. downloads, the browser relay). Empty by default: no host gets the exemption unless it
  159. declares one, and the sentinel range keeps the ordinary private-address verdict otherwise.
  160. """
  161. try:
  162. from hermes_cli.config import read_raw_config
  163. block = read_raw_config().get("security", {})
  164. raw = block.get("fake_ip_ranges") if isinstance(block, dict) else None
  165. except Exception:
  166. return () # config unavailable (tests, early import) — keep the secure default
  167. entries = [raw] if isinstance(raw, str) else raw if isinstance(raw, (list, tuple)) else ()
  168. networks = []
  169. for entry in entries:
  170. try:
  171. net = ipaddress.ip_network(str(entry).strip(), strict=False)
  172. except ValueError:
  173. logger.warning("Ignoring unparseable security.fake_ip_ranges entry: %r", entry)
  174. continue
  175. clash = next((r for r in _FAKE_IP_UNDECLARABLE_NETWORKS if r.version == net.version and net.overlaps(r)), None)
  176. if clash is not None:
  177. logger.warning("Ignoring security.fake_ip_ranges entry %r: it overlaps %s, which stays blocked", entry, clash)
  178. continue
  179. networks.append(net)
  180. return tuple(networks)
  181. def _global_fake_ip_ranges() -> tuple:
  182. """Process-lifetime cache with the same profile-scope bypass as ``_global_allow_private_urls``:
  183. a multiplex gateway must not apply the first profile's declaration to later ones."""
  184. global _fake_ip_resolved, _cached_fake_ip_ranges
  185. if get_hermes_home_override() is not None:
  186. return _resolve_fake_ip_ranges()
  187. if not _fake_ip_resolved:
  188. _fake_ip_resolved, _cached_fake_ip_ranges = True, _resolve_fake_ip_ranges()
  189. return _cached_fake_ip_ranges
  190. def _normalize_hostname(host: Optional[str]) -> str:
  191. return (host or "").strip().lower().rstrip(".")
  192. # urllib.parse and WHATWG parsers (browsers, Node-based fetch backends, also urllib3) can take a
  193. # different host from one string: WHATWG ends the authority at ``\``, percent-decodes the host and
  194. # accepts ``http:\\host`` or ``http:///host``; urlsplit does none of these. So
  195. # ``http://127.0.0.1:8080\@1.1.1.1`` is 1.1.1.1 here and 127.0.0.1:8080 in a browser. Such
  196. # URLs are refused rather than guessed at. WHATWG strips every C0 control from the ends, but urlsplit
  197. # keeps a trailing one in the host, so only ordinary whitespace is stripped before looking for controls.
  198. _URL_EDGE_WHITESPACE = " \t\n\r\x0b\x0c"
  199. # Dots IDNA maps to "." (ideographic, fullwidth, halfwidth ideographic); used when IDNA encoding fails.
  200. _IDNA_DOTS = str.maketrans(dict.fromkeys("。.。", "."))
  201. def _url_hostname(url: str, *, percent_decode: bool = False) -> str:
  202. """The host of *url* in the form a WHATWG client dials it: IDNA-encoded (so ``169.254.169.254。``
  203. is ``169.254.169.254``), lowercased, trailing dot dropped, optionally percent-decoded first."""
  204. host = urlsplit(url.strip(_URL_EDGE_WHITESPACE)).hostname or ""
  205. if percent_decode:
  206. host = unquote(host)
  207. try:
  208. host = host.encode("idna").decode("ascii")
  209. except UnicodeError:
  210. host = host.translate(_IDNA_DOTS)
  211. return _normalize_hostname(host)
  212. def _authority_ambiguity(url: str, *, floor: bool = False) -> Optional[str]:
  213. """Why another client could read a different host or port from an http(s) *url* than urlsplit
  214. does, else None. ``\\`` and ``@`` after the authority (path, query, fragment) are accepted: every
  215. parser has fixed the host by then, and WHATWG's ``\\`` -> ``/`` rewrite only changes the path.
  216. ``floor`` keeps only what can move the host itself (controls, no ``//``, ``\\`` in the authority):
  217. userinfo and an unparseable port (templated ``{port}``/``%d`` literals) leave the host as is, and
  218. the floor checks a percent-encoded host after decoding it."""
  219. url = url.strip(_URL_EDGE_WHITESPACE)
  220. parts = urlsplit(url)
  221. if parts.scheme.lower() not in _HTTP_SCHEMES:
  222. return None
  223. if any(ord(c) < 0x20 or ord(c) == 0x7F for c in url):
  224. return "control character in URL" # urlsplit silently drops tab/CR/LF anywhere
  225. if not parts.netloc:
  226. return "authority not introduced by //"
  227. if "\\" in parts.netloc:
  228. return "backslash in authority"
  229. if floor:
  230. return None
  231. _, at, host = parts.netloc.rpartition("@")
  232. if at:
  233. return "userinfo in authority"
  234. if "%" in host:
  235. return "percent-encoding in host"
  236. if any(c.isspace() for c in host):
  237. return "whitespace in host"
  238. try:
  239. parts.port
  240. except ValueError:
  241. return "invalid port"
  242. return None
  243. def _parse_ip(hostname: str) -> Optional[_IPAddress]:
  244. """IP object for a literal-IP hostname, else None."""
  245. try:
  246. return ipaddress.ip_address(hostname)
  247. except ValueError:
  248. return None
  249. def _iter_resolved_ips(addr_info: Any):
  250. """Yield ``(raw, ip_str, ip)`` per getaddrinfo answer. ``ip_str`` has any IPv6 scope ID
  251. (``%eth0``) stripped; ``ip`` is None when still unparseable — each caller decides skip/fail/raise."""
  252. for _family, _, _, _, sockaddr in addr_info:
  253. raw = sockaddr[0]
  254. ip_str = raw.split("%")[0]
  255. yield raw, ip_str, _parse_ip(ip_str)
  256. def _getaddrinfo(hostname: str, port: Optional[int] = None):
  257. return socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
  258. def _embedded_ipv4(ip: _IPAddress) -> _IPAddress:
  259. """The IPv4 address an IPv6 wrapper stands for — IPv4-mapped (``::ffff:x.x.x.x``) or
  260. IPv4-translated (``::ffff:0:x.x.x.x``); *ip* unchanged otherwise. ``ipaddress`` reads both
  261. wrappers as distinct IPv6 addresses, so every classification must see through them, or a
  262. resolver's sentinel / a cloud-metadata answer arrives as unrelated IPv6 space."""
  263. if isinstance(ip, ipaddress.IPv6Address):
  264. if ip.ipv4_mapped is not None:
  265. return ip.ipv4_mapped
  266. if ip in _IPV4_TRANSLATED_NETWORK:
  267. return ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF)
  268. return ip
  269. def _is_always_blocked_ip(ip: _IPAddress) -> bool:
  270. ip = _embedded_ipv4(ip)
  271. return ip in _ALWAYS_BLOCKED_IPS or any(ip in net for net in _ALWAYS_BLOCKED_NETWORKS)
  272. def _is_declared_fake_ip(ip: _IPAddress) -> bool:
  273. """True when *ip* falls in a block declared in ``security.fake_ip_ranges``. The dial still goes
  274. to the local proxy, which resolves and connects to the real target, so a declared block grants
  275. no reach an attacker lacks through the proxy's own DNS; undeclared ranges keep the private verdict."""
  276. ip = _embedded_ipv4(ip)
  277. return any(ip in net for net in _global_fake_ip_ranges())
  278. def _is_blocked_ip(ip: _IPAddress) -> bool:
  279. """Return True if the IP should be blocked for SSRF protection."""
  280. # IPv4-wrapped IPv6 (mapped ``::ffff:x.x.x.x`` or translated ``::ffff:0:x.x.x.x``) is
  281. # classified by its embedded IPv4.
  282. ip = _embedded_ipv4(ip)
  283. return (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
  284. or ip.is_multicast or ip.is_unspecified or ip in _CGNAT_NETWORK)
  285. def is_always_blocked_url(url: str) -> bool:
  286. """True when the URL targets the always-blocked floor (cloud metadata) — only the sentinel
  287. hostnames/IPs, regardless of backend, routing, or ``allow_private_urls``. For callers that
  288. deliberately bypass the full check (e.g. hybrid cloud browser routing private URLs to a local
  289. sidecar) but must still enforce the floor. False for ordinary private/loopback URLs, DNS
  290. failures and parse errors (the caller's ordinary fail-closed path handles those); True for an
  291. http(s) URL whose host another client could read differently, which may hide a metadata target.
  292. A percent-encoded host is judged decoded, as WHATWG clients read it."""
  293. try:
  294. reason = _authority_ambiguity(url, floor=True)
  295. if reason is not None:
  296. logger.warning("Blocked request — ambiguous URL authority (always-blocked floor): %s", reason)
  297. return True
  298. hostname = _url_hostname(url, percent_decode=True)
  299. if not hostname:
  300. return False
  301. if hostname in _BLOCKED_HOSTNAMES:
  302. logger.warning("Blocked request to internal hostname (always-blocked floor): %s", hostname)
  303. return True
  304. ip = _parse_ip(hostname)
  305. if ip is not None:
  306. if _is_always_blocked_ip(ip):
  307. logger.warning("Blocked request to cloud metadata address (always-blocked floor): %s", hostname)
  308. return True
  309. return False
  310. try:
  311. addr_info = _getaddrinfo(hostname)
  312. except socket.gaierror:
  313. return False # DNS failure is not part of the floor; caller's path handles it
  314. for raw, ip_str, resolved in _iter_resolved_ips(addr_info):
  315. if resolved is None:
  316. logger.warning("Unparseable IP address %r for hostname %s — skipping address", raw, hostname)
  317. continue
  318. if _is_always_blocked_ip(resolved):
  319. logger.warning(
  320. "Blocked request to cloud metadata address (always-blocked floor): %s -> %s", hostname, ip_str
  321. )
  322. return True
  323. return False
  324. except Exception as exc:
  325. # Parse/unexpected errors are not "always blocked"; caller decides fail-open/closed.
  326. logger.debug("is_always_blocked_url error for %s: %s", url, exc)
  327. return False
  328. def _allows_private_ip_resolution(hostname: str, scheme: str) -> bool:
  329. """Return True when a trusted HTTPS hostname may bypass IP-class blocking."""
  330. return scheme == "https" and hostname in _TRUSTED_PRIVATE_IP_HOSTS
  331. def _resolved_ip_block_reason(ip: _IPAddress, allow_private: bool) -> Optional[str]:
  332. """Why a resolved answer must be rejected, or None if it may be dialed. The metadata floor
  333. ignores ``allow_private``; ordinary private/internal classes are blocked only when it is False."""
  334. if _is_always_blocked_ip(ip):
  335. return "cloud metadata address"
  336. if not allow_private and _is_blocked_ip(ip) and not _is_declared_fake_ip(ip):
  337. return "private/internal address"
  338. return None
  339. def is_safe_url(url: str) -> bool:
  340. """True if the URL target is not a private/internal address. Resolves the hostname and checks
  341. every answer; fails closed on DNS errors, unexpected exceptions and URLs whose authority other
  342. clients could parse differently (userinfo included). ``allow_private_urls`` skips private-IP
  343. blocking, but cloud metadata endpoints remain blocked regardless."""
  344. try:
  345. scheme = (urlparse(url).scheme or "").strip().lower()
  346. if scheme not in _HTTP_SCHEMES:
  347. logger.warning("Blocked request — unsupported URL scheme: %s", scheme or "<empty>")
  348. return False
  349. reason = _authority_ambiguity(url)
  350. if reason is not None:
  351. # The reason only: the URL may carry userinfo credentials.
  352. logger.warning("Blocked request — ambiguous URL authority: %s", reason)
  353. return False
  354. hostname = _url_hostname(url)
  355. if not hostname:
  356. return False
  357. # Metadata hostnames are blocked BEFORE consulting the toggle.
  358. if hostname in _BLOCKED_HOSTNAMES:
  359. logger.warning("Blocked request to internal hostname: %s", hostname)
  360. return False
  361. allow_all_private = _global_allow_private_urls()
  362. allow_private_ip = _allows_private_ip_resolution(hostname, scheme)
  363. allow_private = allow_all_private or allow_private_ip
  364. try:
  365. addr_info = _getaddrinfo(hostname)
  366. except socket.gaierror:
  367. # Sandbox/proxy environments may block direct DNS; when a proxy is
  368. # configured, delegate resolution to it (metadata hostnames were already
  369. # rejected above). Literal IPs need no DNS, so a failure on one is not a
  370. # proxy symptom — keep them fail-closed.
  371. if _parse_ip(hostname) is None and _proxy_is_configured():
  372. logger.debug(
  373. "DNS resolution failed for %s — proxy configured, allowing through for proxy-side resolution",
  374. hostname)
  375. return True
  376. logger.warning("Blocked request — DNS resolution failed for: %s", hostname)
  377. return False
  378. for raw, ip_str, ip in _iter_resolved_ips(addr_info):
  379. if ip is None:
  380. logger.warning("Blocked request — unparseable IP address %r for hostname %s", raw, hostname)
  381. return False
  382. reason = _resolved_ip_block_reason(ip, allow_private)
  383. if reason is not None:
  384. logger.warning("Blocked request to %s: %s -> %s", reason, hostname, ip_str)
  385. return False
  386. if allow_all_private:
  387. logger.debug("Allowing private/internal resolution (security.allow_private_urls=true): %s", hostname)
  388. elif allow_private_ip:
  389. logger.debug("Allowing trusted hostname despite private/internal resolution: %s", hostname)
  390. return True
  391. except Exception as exc:
  392. # Fail closed: parsing edge cases must not become SSRF bypass vectors.
  393. logger.warning("Blocked request — URL safety check error for %s: %s", url, exc)
  394. return False
  395. async def async_is_safe_url(url: str) -> bool:
  396. """:func:`is_safe_url` with the blocking DNS work off the event loop."""
  397. return await asyncio.to_thread(is_safe_url, url)
  398. class SSRFConnectionBlocked(ValueError):
  399. """Raised when connect-time DNS resolution violates the URL safety policy."""
  400. def _resolved_http_connect_ips(host: str, port: int, scheme: str) -> list[str]:
  401. """Resolve and validate *host* at TCP-connect time; return dialable IP strings. Closes the
  402. DNS-rebinding gap between pre-flight validation and connect for direct httpx clients. The
  403. result is capped at ``_MAX_SSRF_CONNECT_IPS``, but EVERY answer is validated."""
  404. hostname = _normalize_hostname(host)
  405. if not hostname:
  406. raise SSRFConnectionBlocked("Blocked request with empty hostname")
  407. if hostname in _BLOCKED_HOSTNAMES:
  408. raise SSRFConnectionBlocked(f"Blocked request to internal hostname: {hostname}")
  409. allow_private = _global_allow_private_urls() or _allows_private_ip_resolution(hostname, scheme)
  410. try:
  411. addr_info = _getaddrinfo(hostname, port)
  412. except socket.gaierror as exc:
  413. raise SSRFConnectionBlocked(f"Blocked request - DNS resolution failed for: {hostname}") from exc
  414. safe_ips: list[str] = []
  415. for raw, ip_str, ip in _iter_resolved_ips(addr_info):
  416. if ip is None:
  417. raise SSRFConnectionBlocked(
  418. f"Blocked request - unparseable IP address {raw!r} for hostname {hostname}"
  419. ) from ValueError(f"{ip_str!r} does not appear to be an IPv4 or IPv6 address")
  420. reason = _resolved_ip_block_reason(ip, allow_private)
  421. if reason is not None:
  422. raise SSRFConnectionBlocked(f"Blocked request to {reason} during connect: {hostname} -> {ip_str}")
  423. if ip_str not in safe_ips and len(safe_ips) < _MAX_SSRF_CONNECT_IPS:
  424. safe_ips.append(ip_str)
  425. if not safe_ips:
  426. raise SSRFConnectionBlocked(f"Blocked request - DNS returned no results for: {hostname}")
  427. return safe_ips
  428. class _SSRFGuardedBackendBase:
  429. """httpcore backend that re-resolves + validates at connect time and dials a vetted IP.
  430. Host/SNI stay on the original hostname; Unix sockets are refused outright. Candidate IPs are
  431. tried in order and the last connect error is re-raised so callers see the real failure."""
  432. def __init__(self, backend: Any, schemes_by_origin_var: Any):
  433. self._backend = backend
  434. self._schemes_by_origin_var = schemes_by_origin_var
  435. def _connect_ips(self, host: str, port: int) -> list[str]:
  436. scheme = self._schemes_by_origin_var.get({}).get((host, port)) or ("https" if port == 443 else "http")
  437. return _resolved_http_connect_ips(host, port, scheme)
  438. @staticmethod
  439. def _connect_errors() -> tuple:
  440. import httpcore
  441. return (httpcore.ConnectError, httpcore.ConnectTimeout)
  442. @staticmethod
  443. def _no_usable_ips(host: str, last_exc: Exception | None) -> Exception:
  444. if last_exc is not None:
  445. return last_exc
  446. return SSRFConnectionBlocked(f"Blocked request - DNS returned no usable IPs for: {host}")
  447. def connect_unix_socket(self, path: str, timeout: float | None = None, socket_options: Any = None) -> Any:
  448. raise SSRFConnectionBlocked("Blocked Unix socket connection in SSRF-safe transport")
  449. class _SSRFGuardedAsyncNetworkBackend(_SSRFGuardedBackendBase):
  450. def __init__(self, schemes_by_origin_var: Any):
  451. from httpcore._backends.auto import AutoBackend
  452. super().__init__(AutoBackend(), schemes_by_origin_var)
  453. async def connect_tcp(self, host: str, port: int, timeout: float | None = None,
  454. local_address: str | None = None, socket_options: Any = None) -> Any:
  455. last_exc: Exception | None = None
  456. for ip in await asyncio.to_thread(self._connect_ips, host, port):
  457. try:
  458. return await self._backend.connect_tcp(
  459. ip, port, timeout=timeout, local_address=local_address, socket_options=socket_options)
  460. except self._connect_errors() as exc:
  461. last_exc = exc
  462. raise self._no_usable_ips(host, last_exc)
  463. async def connect_unix_socket(self, path: str, timeout: float | None = None, socket_options: Any = None) -> Any:
  464. raise SSRFConnectionBlocked("Blocked Unix socket connection in SSRF-safe transport")
  465. async def sleep(self, seconds: float) -> None:
  466. await self._backend.sleep(seconds)
  467. class _SSRFGuardedNetworkBackend(_SSRFGuardedBackendBase):
  468. def __init__(self, schemes_by_origin_var: Any):
  469. from httpcore._backends.sync import SyncBackend
  470. super().__init__(SyncBackend(), schemes_by_origin_var)
  471. def connect_tcp(self, host: str, port: int, timeout: float | None = None,
  472. local_address: str | None = None, socket_options: Any = None) -> Any:
  473. last_exc: Exception | None = None
  474. for ip in self._connect_ips(host, port):
  475. try:
  476. return self._backend.connect_tcp(
  477. ip, port, timeout=timeout, local_address=local_address, socket_options=socket_options)
  478. except self._connect_errors() as exc:
  479. last_exc = exc
  480. raise self._no_usable_ips(host, last_exc)
  481. def sleep(self, seconds: float) -> None:
  482. self._backend.sleep(seconds)
  483. def _origin_scheme_context(request: Any) -> dict[tuple[str, int], str]:
  484. host, port, scheme = request.url.host, request.url.port, request.url.scheme
  485. return {(host, port): scheme} if host and port is not None and scheme in _HTTP_SCHEMES else {}
  486. @contextmanager
  487. def _origin_scope(schemes_by_origin_var: Any, request: Any):
  488. """Expose the request's origin scheme to the connect-time backend for this request only."""
  489. token = schemes_by_origin_var.set(_origin_scheme_context(request))
  490. try:
  491. yield
  492. finally:
  493. schemes_by_origin_var.reset(token)
  494. def _install_ssrf_guard_on_transport(transport: Any, schemes_by_origin_var: Any, *, is_async: bool = False) -> None:
  495. """Swap the transport's pool network backend for the SSRF-guarded one (idempotent). Only the
  496. direct transport is guarded; proxy mounts delegate final-target resolution to the trusted proxy."""
  497. state = getattr(transport, "__dict__", {}) if transport is not None else {}
  498. if transport is None or state.get("_hermes_ssrf_guarded", False):
  499. return
  500. label = "async httpx transport" if is_async else "httpx transport"
  501. pool = state.get("_pool")
  502. if pool is None or not hasattr(pool, "_network_backend"):
  503. raise SSRFConnectionBlocked(f"Unsupported {label} cannot be made SSRF-safe")
  504. backend_cls = _SSRFGuardedAsyncNetworkBackend if is_async else _SSRFGuardedNetworkBackend
  505. pool._network_backend = backend_cls(schemes_by_origin_var)
  506. method_name = "handle_async_request" if is_async else "handle_request"
  507. handle = getattr(transport, method_name, None)
  508. if handle is None:
  509. raise SSRFConnectionBlocked(f"Unsupported {label} cannot be made SSRF-safe")
  510. async def guarded_async(request: Any) -> Any:
  511. with _origin_scope(schemes_by_origin_var, request):
  512. return await handle(request)
  513. def guarded_sync(request: Any) -> Any:
  514. with _origin_scope(schemes_by_origin_var, request):
  515. return handle(request)
  516. setattr(transport, method_name, guarded_async if is_async else guarded_sync)
  517. transport._hermes_ssrf_guarded = True
  518. def _install_ssrf_guard_on_client(client: Any, *, is_async: bool = False) -> None:
  519. """Guard ``client._transport`` only; ``_mounts`` (env/explicit proxies) stay untouched."""
  520. import contextvars
  521. var_name = "hermes_ssrf_async_origin_schemes" if is_async else "hermes_ssrf_origin_schemes"
  522. _install_ssrf_guard_on_transport(
  523. getattr(client, "__dict__", {}).get("_transport"), contextvars.ContextVar(var_name), is_async=is_async)
  524. def create_ssrf_safe_async_client(**kwargs: Any) -> Any:
  525. """``httpx.AsyncClient`` with connect-time SSRF validation: direct HTTP(S) connections are
  526. resolved, validated, and dialed by IP while the hostname is preserved for Host, SNI, and
  527. certificate verification. Proxied requests delegate resolution to the proxy."""
  528. import httpx
  529. client = httpx.AsyncClient(**kwargs)
  530. _install_ssrf_guard_on_client(client, is_async=True)
  531. return client
  532. def create_ssrf_safe_client(**kwargs: Any) -> Any:
  533. """Create an ``httpx.Client`` with connect-time SSRF validation."""
  534. import httpx
  535. client = httpx.Client(**kwargs)
  536. _install_ssrf_guard_on_client(client)
  537. return client
  538. def redirect_target_from_response(response: Any) -> Optional[str]:
  539. """Redirect target visible from inside an httpx response hook. ``response.next_request`` is
  540. frequently ``None`` inside hooks (populated later by the follower), which would make an SSRF
  541. redirect guard silently never fire — so resolve from ``Location`` first, then ``next_request``."""
  542. if not getattr(response, "is_redirect", False):
  543. return None
  544. location = (getattr(response, "headers", {}) or {}).get("location")
  545. if location:
  546. return urljoin(str(getattr(response, "url", "")), str(location))
  547. next_request = getattr(response, "next_request", None)
  548. return str(next_request.url) if next_request else None