| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650 |
- """URL safety checks — blocks requests to private/internal network addresses (SSRF).
- ``security.allow_private_urls: true`` disables private-IP blocking (DNS that resolves public
- names to private ranges); cloud metadata hostnames/IPs are **always** blocked. A local TUN proxy
- that answers DNS with a fake-ip block (Mihomo/Clash fake-ip, Surge enhanced) declares that block
- in ``security.fake_ip_ranges`` so its sentinel answers are dialable instead of looking private;
- the list is empty by default, so the sentinel stays blocked for everyone else. DNS rebinding
- (TOCTOU) is closed for Hermes-owned httpx paths by ``create_ssrf_safe_[async_]client()``, which
- re-apply the policy at TCP connect and dial the validated IP while preserving Host/SNI. Redirect
- bypass is mitigated by response hooks re-validating each target (``redirect_target_from_response``).
- """
- import ipaddress
- import logging
- import os
- import socket
- import asyncio
- import re
- from contextlib import contextmanager
- from typing import Any, Optional
- from urllib.parse import parse_qsl, quote, unquote, urljoin, urlparse, urlsplit, urlunsplit
- from hermes_constants import get_hermes_home_override
- from utils import is_truthy_value
- logger = logging.getLogger(__name__)
- # Proxy env vars: when set, the runtime should delegate DNS to the proxy.
- _PROXY_ENV_VARS = ("HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy")
- _HTTP_SCHEMES = frozenset({"http", "https"})
- _IPAddress = ipaddress.IPv4Address | ipaddress.IPv6Address
- def _proxy_is_configured() -> bool:
- return any(os.environ.get(v) for v in _PROXY_ENV_VARS)
- def normalize_url_for_request(url: str) -> str:
- """ASCII-safe HTTP URL for Hermes-owned URL tools (IRI -> URI, e.g. ``https://wttr.in/Köln``).
- Preserves URL syntax and existing percent escapes while IDNA-encoding the host and
- percent-encoding non-ASCII path/query/fragment text. URL tool inputs only — never shell commands."""
- if not isinstance(url, str):
- return url
- raw = url.strip()
- if not raw:
- return raw
- # Repair model-emitted whitespace between scheme separator and authority
- # (``https:// docs.example``); that position is never meaningful in HTTP URLs.
- raw = re.sub(r"^([A-Za-z][A-Za-z0-9+.-]*://)\s+", r"\1", raw)
- try:
- parsed = urlsplit(raw)
- except ValueError:
- return raw
- if parsed.scheme.lower() not in _HTTP_SCHEMES:
- return raw
- netloc, hostname = parsed.netloc, parsed.hostname
- if hostname:
- try:
- ascii_host = hostname.encode("idna").decode("ascii")
- except UnicodeError:
- ascii_host = hostname
- if ascii_host != hostname:
- netloc = netloc.replace(hostname, ascii_host, 1)
- safe = "/%:@!$&'()*+,;="
- return urlunsplit((parsed.scheme, netloc, quote(parsed.path, safe=safe),
- quote(parsed.query, safe=safe + "?"), quote(parsed.fragment, safe=safe + "?")))
- # Unambiguously credential-bearing query param names. Deliberately narrow: bare
- # English words that double as page facets (``code``, ``key``, ``auth``,
- # ``session``, ``sig``) are EXCLUDED so ordinary browsing is not blocked.
- _SENSITIVE_QUERY_PARAM_NAMES = frozenset({
- "access_token", "api_key", "apikey", "auth_token", "authorization", "awsaccesskeyid",
- "client_secret", "credential", "credentials", "jwt", "password", "passwd", "secret",
- "session_id", "signature", "token", "x_amz_security_token", "x_amz_signature",
- "x-amz-security-token", "x-amz-signature"})
- def sensitive_query_param_name(url: str) -> Optional[str]:
- """First credential-named query parameter in ``url`` (with a value), if any. Checked before
- handing URLs to third-party fetch/browser backends: catches opaque magic links, OAuth codes,
- signed-URL signatures and custom ``?token=...`` values that prefix-based redaction misses."""
- if not isinstance(url, str) or "?" not in url:
- return None
- try:
- parsed = urlsplit(url.strip())
- except ValueError:
- return None
- if parsed.scheme.lower() not in _HTTP_SCHEMES or not parsed.query:
- return None
- return next((key for key, value in parse_qsl(parsed.query, keep_blank_values=True)
- if value and unquote(key).lower() in _SENSITIVE_QUERY_PARAM_NAMES), None)
- # Cloud metadata hostnames — always blocked regardless of DNS or config toggle.
- _BLOCKED_HOSTNAMES = frozenset({"metadata.google.internal", "metadata.goog"})
- # Cloud metadata / credential endpoints (the #1 SSRF target) — always blocked, also in
- # IPv4-mapped IPv6 form (resolvers may return ``::ffff:x.x.x.x``; ipaddress treats those as distinct).
- _METADATA_V4 = (
- "169.254.169.254", # AWS/GCP/Azure/DO/Oracle metadata
- "169.254.170.2", # AWS ECS task metadata (task IAM creds)
- "169.254.169.253", # Azure IMDS wire server
- "100.100.100.200", # Alibaba Cloud metadata
- )
- _ALWAYS_BLOCKED_IPS = frozenset(
- {ipaddress.ip_address(ip) for ip in _METADATA_V4}
- | {ipaddress.ip_address("::ffff:" + ip) for ip in _METADATA_V4}
- | {ipaddress.ip_address("fd00:ec2::254")} # AWS metadata (IPv6)
- )
- # Entire link-local range (no legit agent target), plus its IPv4-mapped form.
- _ALWAYS_BLOCKED_NETWORKS = tuple(ipaddress.ip_network(n) for n in ("169.254.0.0/16", "::ffff:169.254.0.0/112"))
- # Exact HTTPS hostnames allowed to resolve to private/benchmark-space IPs
- # (QQ media legitimately resolves to 198.18.0.0/15 behind local proxy infra).
- _TRUSTED_PRIVATE_IP_HOSTS = frozenset({"multimedia.nt.qq.com.cn"})
- _MAX_SSRF_CONNECT_IPS = 8
- # 100.64.0.0/10 (CGNAT, RFC 6598) is neither is_private nor is_global in
- # ipaddress — must be blocked explicitly (Tailscale/WireGuard, cloud internal nets).
- _CGNAT_NETWORK = ipaddress.ip_network("100.64.0.0/10")
- # RFC 2765 IPv4-translated wrapper ``::ffff:0:a.b.c.d`` — the second form resolvers may use to
- # answer an IPv4 name (on the reporting macOS host, fake-IP TUN DNS returns it alongside the
- # plain address). ``ip.ipv4_mapped`` does not read it; ``_embedded_ipv4`` handles it explicitly.
- _IPV4_TRANSLATED_NETWORK = ipaddress.ip_network("::ffff:0:0:0/96")
- # Address classes a ``security.fake_ip_ranges`` declaration can never excuse: a local proxy owns
- # none of them, and a declaration is trusted like ``allow_private_urls`` for whatever it names,
- # so an entry overlapping one of these (including 0.0.0.0/0 and ::/0) would make real internal
- # hosts dialable. Such entries are dropped with a warning instead.
- _FAKE_IP_UNDECLARABLE_NETWORKS = tuple(ipaddress.ip_network(n) for n in (
- "0.0.0.0/32", "127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", # unspecified/loopback/RFC 1918
- "169.254.0.0/16", "100.64.0.0/10", # link-local, CGNAT
- "::/128", "::1/128", "fc00::/7", "fe80::/10", # unspecified, loopback, ULA, link-local
- ))
- # Global toggle cache (process lifetime; see _global_allow_private_urls).
- _allow_private_resolved, _cached_allow_private = False, False
- _fake_ip_resolved, _cached_fake_ip_ranges = False, ()
- def _global_allow_private_urls() -> bool:
- """True when the user has opted out of private-IP blocking. Priority: ``HERMES_ALLOW_PRIVATE_URLS``
- env, ``security.allow_private_urls``, legacy ``browser.allow_private_urls``. Profile-scoped turns
- (``get_hermes_home_override()`` set) bypass the process-global cache — a multiplex gateway serves
- several profiles in one process; the first profile's opt-out must not disable blocking for later ones."""
- global _allow_private_resolved, _cached_allow_private
- if get_hermes_home_override() is not None:
- return _resolve_allow_private_urls()
- if not _allow_private_resolved:
- _allow_private_resolved, _cached_allow_private = True, _resolve_allow_private_urls()
- return _cached_allow_private
- def _resolve_allow_private_urls() -> bool:
- """Resolve the effective private-URL toggle from the active config scope."""
- env_val = os.getenv("HERMES_ALLOW_PRIVATE_URLS", "").strip().lower()
- if env_val in {"true", "1", "yes"}:
- return True
- if env_val in {"false", "0", "no"}:
- return False # explicit false does not fall through to config
- try:
- from hermes_cli.config import read_raw_config
- cfg = read_raw_config()
- for section in ("security", "browser"): # preferred, then legacy
- block = cfg.get(section, {})
- if isinstance(block, dict) and is_truthy_value(block.get("allow_private_urls"), default=False):
- return True
- except Exception:
- pass # config unavailable (tests, early import) — keep default
- return False
- def _reset_allow_private_cache() -> None:
- """Reset the cached toggle and the cached fake-ip ranges — only for tests."""
- global _allow_private_resolved, _cached_allow_private, _fake_ip_resolved, _cached_fake_ip_ranges
- _allow_private_resolved = _cached_allow_private = _fake_ip_resolved = False
- _cached_fake_ip_ranges = ()
- def _resolve_fake_ip_ranges() -> tuple:
- """CIDR blocks this host's local proxy answers DNS with (``security.fake_ip_ranges``).
- A TUN proxy in fake-ip mode answers every non-filtered name with an address from its own
- block; that answer is the proxy's sentinel, not an internal host's, so treating it as a
- private target blocks every outbound fetch on such a host (web_extract, platform attachment
- downloads, the browser relay). Empty by default: no host gets the exemption unless it
- declares one, and the sentinel range keeps the ordinary private-address verdict otherwise.
- """
- try:
- from hermes_cli.config import read_raw_config
- block = read_raw_config().get("security", {})
- raw = block.get("fake_ip_ranges") if isinstance(block, dict) else None
- except Exception:
- return () # config unavailable (tests, early import) — keep the secure default
- entries = [raw] if isinstance(raw, str) else raw if isinstance(raw, (list, tuple)) else ()
- networks = []
- for entry in entries:
- try:
- net = ipaddress.ip_network(str(entry).strip(), strict=False)
- except ValueError:
- logger.warning("Ignoring unparseable security.fake_ip_ranges entry: %r", entry)
- continue
- clash = next((r for r in _FAKE_IP_UNDECLARABLE_NETWORKS if r.version == net.version and net.overlaps(r)), None)
- if clash is not None:
- logger.warning("Ignoring security.fake_ip_ranges entry %r: it overlaps %s, which stays blocked", entry, clash)
- continue
- networks.append(net)
- return tuple(networks)
- def _global_fake_ip_ranges() -> tuple:
- """Process-lifetime cache with the same profile-scope bypass as ``_global_allow_private_urls``:
- a multiplex gateway must not apply the first profile's declaration to later ones."""
- global _fake_ip_resolved, _cached_fake_ip_ranges
- if get_hermes_home_override() is not None:
- return _resolve_fake_ip_ranges()
- if not _fake_ip_resolved:
- _fake_ip_resolved, _cached_fake_ip_ranges = True, _resolve_fake_ip_ranges()
- return _cached_fake_ip_ranges
- def _normalize_hostname(host: Optional[str]) -> str:
- return (host or "").strip().lower().rstrip(".")
- # urllib.parse and WHATWG parsers (browsers, Node-based fetch backends, also urllib3) can take a
- # different host from one string: WHATWG ends the authority at ``\``, percent-decodes the host and
- # accepts ``http:\\host`` or ``http:///host``; urlsplit does none of these. So
- # ``http://127.0.0.1:8080\@1.1.1.1`` is 1.1.1.1 here and 127.0.0.1:8080 in a browser. Such
- # URLs are refused rather than guessed at. WHATWG strips every C0 control from the ends, but urlsplit
- # keeps a trailing one in the host, so only ordinary whitespace is stripped before looking for controls.
- _URL_EDGE_WHITESPACE = " \t\n\r\x0b\x0c"
- # Dots IDNA maps to "." (ideographic, fullwidth, halfwidth ideographic); used when IDNA encoding fails.
- _IDNA_DOTS = str.maketrans(dict.fromkeys("。.。", "."))
- def _url_hostname(url: str, *, percent_decode: bool = False) -> str:
- """The host of *url* in the form a WHATWG client dials it: IDNA-encoded (so ``169.254.169.254。``
- is ``169.254.169.254``), lowercased, trailing dot dropped, optionally percent-decoded first."""
- host = urlsplit(url.strip(_URL_EDGE_WHITESPACE)).hostname or ""
- if percent_decode:
- host = unquote(host)
- try:
- host = host.encode("idna").decode("ascii")
- except UnicodeError:
- host = host.translate(_IDNA_DOTS)
- return _normalize_hostname(host)
- def _authority_ambiguity(url: str, *, floor: bool = False) -> Optional[str]:
- """Why another client could read a different host or port from an http(s) *url* than urlsplit
- does, else None. ``\\`` and ``@`` after the authority (path, query, fragment) are accepted: every
- parser has fixed the host by then, and WHATWG's ``\\`` -> ``/`` rewrite only changes the path.
- ``floor`` keeps only what can move the host itself (controls, no ``//``, ``\\`` in the authority):
- userinfo and an unparseable port (templated ``{port}``/``%d`` literals) leave the host as is, and
- the floor checks a percent-encoded host after decoding it."""
- url = url.strip(_URL_EDGE_WHITESPACE)
- parts = urlsplit(url)
- if parts.scheme.lower() not in _HTTP_SCHEMES:
- return None
- if any(ord(c) < 0x20 or ord(c) == 0x7F for c in url):
- return "control character in URL" # urlsplit silently drops tab/CR/LF anywhere
- if not parts.netloc:
- return "authority not introduced by //"
- if "\\" in parts.netloc:
- return "backslash in authority"
- if floor:
- return None
- _, at, host = parts.netloc.rpartition("@")
- if at:
- return "userinfo in authority"
- if "%" in host:
- return "percent-encoding in host"
- if any(c.isspace() for c in host):
- return "whitespace in host"
- try:
- parts.port
- except ValueError:
- return "invalid port"
- return None
- def _parse_ip(hostname: str) -> Optional[_IPAddress]:
- """IP object for a literal-IP hostname, else None."""
- try:
- return ipaddress.ip_address(hostname)
- except ValueError:
- return None
- def _iter_resolved_ips(addr_info: Any):
- """Yield ``(raw, ip_str, ip)`` per getaddrinfo answer. ``ip_str`` has any IPv6 scope ID
- (``%eth0``) stripped; ``ip`` is None when still unparseable — each caller decides skip/fail/raise."""
- for _family, _, _, _, sockaddr in addr_info:
- raw = sockaddr[0]
- ip_str = raw.split("%")[0]
- yield raw, ip_str, _parse_ip(ip_str)
- def _getaddrinfo(hostname: str, port: Optional[int] = None):
- return socket.getaddrinfo(hostname, port, socket.AF_UNSPEC, socket.SOCK_STREAM)
- def _embedded_ipv4(ip: _IPAddress) -> _IPAddress:
- """The IPv4 address an IPv6 wrapper stands for — IPv4-mapped (``::ffff:x.x.x.x``) or
- IPv4-translated (``::ffff:0:x.x.x.x``); *ip* unchanged otherwise. ``ipaddress`` reads both
- wrappers as distinct IPv6 addresses, so every classification must see through them, or a
- resolver's sentinel / a cloud-metadata answer arrives as unrelated IPv6 space."""
- if isinstance(ip, ipaddress.IPv6Address):
- if ip.ipv4_mapped is not None:
- return ip.ipv4_mapped
- if ip in _IPV4_TRANSLATED_NETWORK:
- return ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF)
- return ip
- def _is_always_blocked_ip(ip: _IPAddress) -> bool:
- ip = _embedded_ipv4(ip)
- return ip in _ALWAYS_BLOCKED_IPS or any(ip in net for net in _ALWAYS_BLOCKED_NETWORKS)
- def _is_declared_fake_ip(ip: _IPAddress) -> bool:
- """True when *ip* falls in a block declared in ``security.fake_ip_ranges``. The dial still goes
- to the local proxy, which resolves and connects to the real target, so a declared block grants
- no reach an attacker lacks through the proxy's own DNS; undeclared ranges keep the private verdict."""
- ip = _embedded_ipv4(ip)
- return any(ip in net for net in _global_fake_ip_ranges())
- def _is_blocked_ip(ip: _IPAddress) -> bool:
- """Return True if the IP should be blocked for SSRF protection."""
- # IPv4-wrapped IPv6 (mapped ``::ffff:x.x.x.x`` or translated ``::ffff:0:x.x.x.x``) is
- # classified by its embedded IPv4.
- ip = _embedded_ipv4(ip)
- return (ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved
- or ip.is_multicast or ip.is_unspecified or ip in _CGNAT_NETWORK)
- def is_always_blocked_url(url: str) -> bool:
- """True when the URL targets the always-blocked floor (cloud metadata) — only the sentinel
- hostnames/IPs, regardless of backend, routing, or ``allow_private_urls``. For callers that
- deliberately bypass the full check (e.g. hybrid cloud browser routing private URLs to a local
- sidecar) but must still enforce the floor. False for ordinary private/loopback URLs, DNS
- failures and parse errors (the caller's ordinary fail-closed path handles those); True for an
- http(s) URL whose host another client could read differently, which may hide a metadata target.
- A percent-encoded host is judged decoded, as WHATWG clients read it."""
- try:
- reason = _authority_ambiguity(url, floor=True)
- if reason is not None:
- logger.warning("Blocked request — ambiguous URL authority (always-blocked floor): %s", reason)
- return True
- hostname = _url_hostname(url, percent_decode=True)
- if not hostname:
- return False
- if hostname in _BLOCKED_HOSTNAMES:
- logger.warning("Blocked request to internal hostname (always-blocked floor): %s", hostname)
- return True
- ip = _parse_ip(hostname)
- if ip is not None:
- if _is_always_blocked_ip(ip):
- logger.warning("Blocked request to cloud metadata address (always-blocked floor): %s", hostname)
- return True
- return False
- try:
- addr_info = _getaddrinfo(hostname)
- except socket.gaierror:
- return False # DNS failure is not part of the floor; caller's path handles it
- for raw, ip_str, resolved in _iter_resolved_ips(addr_info):
- if resolved is None:
- logger.warning("Unparseable IP address %r for hostname %s — skipping address", raw, hostname)
- continue
- if _is_always_blocked_ip(resolved):
- logger.warning(
- "Blocked request to cloud metadata address (always-blocked floor): %s -> %s", hostname, ip_str
- )
- return True
- return False
- except Exception as exc:
- # Parse/unexpected errors are not "always blocked"; caller decides fail-open/closed.
- logger.debug("is_always_blocked_url error for %s: %s", url, exc)
- return False
- def _allows_private_ip_resolution(hostname: str, scheme: str) -> bool:
- """Return True when a trusted HTTPS hostname may bypass IP-class blocking."""
- return scheme == "https" and hostname in _TRUSTED_PRIVATE_IP_HOSTS
- def _resolved_ip_block_reason(ip: _IPAddress, allow_private: bool) -> Optional[str]:
- """Why a resolved answer must be rejected, or None if it may be dialed. The metadata floor
- ignores ``allow_private``; ordinary private/internal classes are blocked only when it is False."""
- if _is_always_blocked_ip(ip):
- return "cloud metadata address"
- if not allow_private and _is_blocked_ip(ip) and not _is_declared_fake_ip(ip):
- return "private/internal address"
- return None
- def is_safe_url(url: str) -> bool:
- """True if the URL target is not a private/internal address. Resolves the hostname and checks
- every answer; fails closed on DNS errors, unexpected exceptions and URLs whose authority other
- clients could parse differently (userinfo included). ``allow_private_urls`` skips private-IP
- blocking, but cloud metadata endpoints remain blocked regardless."""
- try:
- scheme = (urlparse(url).scheme or "").strip().lower()
- if scheme not in _HTTP_SCHEMES:
- logger.warning("Blocked request — unsupported URL scheme: %s", scheme or "<empty>")
- return False
- reason = _authority_ambiguity(url)
- if reason is not None:
- # The reason only: the URL may carry userinfo credentials.
- logger.warning("Blocked request — ambiguous URL authority: %s", reason)
- return False
- hostname = _url_hostname(url)
- if not hostname:
- return False
- # Metadata hostnames are blocked BEFORE consulting the toggle.
- if hostname in _BLOCKED_HOSTNAMES:
- logger.warning("Blocked request to internal hostname: %s", hostname)
- return False
- allow_all_private = _global_allow_private_urls()
- allow_private_ip = _allows_private_ip_resolution(hostname, scheme)
- allow_private = allow_all_private or allow_private_ip
- try:
- addr_info = _getaddrinfo(hostname)
- except socket.gaierror:
- # Sandbox/proxy environments may block direct DNS; when a proxy is
- # configured, delegate resolution to it (metadata hostnames were already
- # rejected above). Literal IPs need no DNS, so a failure on one is not a
- # proxy symptom — keep them fail-closed.
- if _parse_ip(hostname) is None and _proxy_is_configured():
- logger.debug(
- "DNS resolution failed for %s — proxy configured, allowing through for proxy-side resolution",
- hostname)
- return True
- logger.warning("Blocked request — DNS resolution failed for: %s", hostname)
- return False
- for raw, ip_str, ip in _iter_resolved_ips(addr_info):
- if ip is None:
- logger.warning("Blocked request — unparseable IP address %r for hostname %s", raw, hostname)
- return False
- reason = _resolved_ip_block_reason(ip, allow_private)
- if reason is not None:
- logger.warning("Blocked request to %s: %s -> %s", reason, hostname, ip_str)
- return False
- if allow_all_private:
- logger.debug("Allowing private/internal resolution (security.allow_private_urls=true): %s", hostname)
- elif allow_private_ip:
- logger.debug("Allowing trusted hostname despite private/internal resolution: %s", hostname)
- return True
- except Exception as exc:
- # Fail closed: parsing edge cases must not become SSRF bypass vectors.
- logger.warning("Blocked request — URL safety check error for %s: %s", url, exc)
- return False
- async def async_is_safe_url(url: str) -> bool:
- """:func:`is_safe_url` with the blocking DNS work off the event loop."""
- return await asyncio.to_thread(is_safe_url, url)
- class SSRFConnectionBlocked(ValueError):
- """Raised when connect-time DNS resolution violates the URL safety policy."""
- def _resolved_http_connect_ips(host: str, port: int, scheme: str) -> list[str]:
- """Resolve and validate *host* at TCP-connect time; return dialable IP strings. Closes the
- DNS-rebinding gap between pre-flight validation and connect for direct httpx clients. The
- result is capped at ``_MAX_SSRF_CONNECT_IPS``, but EVERY answer is validated."""
- hostname = _normalize_hostname(host)
- if not hostname:
- raise SSRFConnectionBlocked("Blocked request with empty hostname")
- if hostname in _BLOCKED_HOSTNAMES:
- raise SSRFConnectionBlocked(f"Blocked request to internal hostname: {hostname}")
- allow_private = _global_allow_private_urls() or _allows_private_ip_resolution(hostname, scheme)
- try:
- addr_info = _getaddrinfo(hostname, port)
- except socket.gaierror as exc:
- raise SSRFConnectionBlocked(f"Blocked request - DNS resolution failed for: {hostname}") from exc
- safe_ips: list[str] = []
- for raw, ip_str, ip in _iter_resolved_ips(addr_info):
- if ip is None:
- raise SSRFConnectionBlocked(
- f"Blocked request - unparseable IP address {raw!r} for hostname {hostname}"
- ) from ValueError(f"{ip_str!r} does not appear to be an IPv4 or IPv6 address")
- reason = _resolved_ip_block_reason(ip, allow_private)
- if reason is not None:
- raise SSRFConnectionBlocked(f"Blocked request to {reason} during connect: {hostname} -> {ip_str}")
- if ip_str not in safe_ips and len(safe_ips) < _MAX_SSRF_CONNECT_IPS:
- safe_ips.append(ip_str)
- if not safe_ips:
- raise SSRFConnectionBlocked(f"Blocked request - DNS returned no results for: {hostname}")
- return safe_ips
- class _SSRFGuardedBackendBase:
- """httpcore backend that re-resolves + validates at connect time and dials a vetted IP.
- Host/SNI stay on the original hostname; Unix sockets are refused outright. Candidate IPs are
- tried in order and the last connect error is re-raised so callers see the real failure."""
- def __init__(self, backend: Any, schemes_by_origin_var: Any):
- self._backend = backend
- self._schemes_by_origin_var = schemes_by_origin_var
- def _connect_ips(self, host: str, port: int) -> list[str]:
- scheme = self._schemes_by_origin_var.get({}).get((host, port)) or ("https" if port == 443 else "http")
- return _resolved_http_connect_ips(host, port, scheme)
- @staticmethod
- def _connect_errors() -> tuple:
- import httpcore
- return (httpcore.ConnectError, httpcore.ConnectTimeout)
- @staticmethod
- def _no_usable_ips(host: str, last_exc: Exception | None) -> Exception:
- if last_exc is not None:
- return last_exc
- return SSRFConnectionBlocked(f"Blocked request - DNS returned no usable IPs for: {host}")
- def connect_unix_socket(self, path: str, timeout: float | None = None, socket_options: Any = None) -> Any:
- raise SSRFConnectionBlocked("Blocked Unix socket connection in SSRF-safe transport")
- class _SSRFGuardedAsyncNetworkBackend(_SSRFGuardedBackendBase):
- def __init__(self, schemes_by_origin_var: Any):
- from httpcore._backends.auto import AutoBackend
- super().__init__(AutoBackend(), schemes_by_origin_var)
- async def connect_tcp(self, host: str, port: int, timeout: float | None = None,
- local_address: str | None = None, socket_options: Any = None) -> Any:
- last_exc: Exception | None = None
- for ip in await asyncio.to_thread(self._connect_ips, host, port):
- try:
- return await self._backend.connect_tcp(
- ip, port, timeout=timeout, local_address=local_address, socket_options=socket_options)
- except self._connect_errors() as exc:
- last_exc = exc
- raise self._no_usable_ips(host, last_exc)
- async def connect_unix_socket(self, path: str, timeout: float | None = None, socket_options: Any = None) -> Any:
- raise SSRFConnectionBlocked("Blocked Unix socket connection in SSRF-safe transport")
- async def sleep(self, seconds: float) -> None:
- await self._backend.sleep(seconds)
- class _SSRFGuardedNetworkBackend(_SSRFGuardedBackendBase):
- def __init__(self, schemes_by_origin_var: Any):
- from httpcore._backends.sync import SyncBackend
- super().__init__(SyncBackend(), schemes_by_origin_var)
- def connect_tcp(self, host: str, port: int, timeout: float | None = None,
- local_address: str | None = None, socket_options: Any = None) -> Any:
- last_exc: Exception | None = None
- for ip in self._connect_ips(host, port):
- try:
- return self._backend.connect_tcp(
- ip, port, timeout=timeout, local_address=local_address, socket_options=socket_options)
- except self._connect_errors() as exc:
- last_exc = exc
- raise self._no_usable_ips(host, last_exc)
- def sleep(self, seconds: float) -> None:
- self._backend.sleep(seconds)
- def _origin_scheme_context(request: Any) -> dict[tuple[str, int], str]:
- host, port, scheme = request.url.host, request.url.port, request.url.scheme
- return {(host, port): scheme} if host and port is not None and scheme in _HTTP_SCHEMES else {}
- @contextmanager
- def _origin_scope(schemes_by_origin_var: Any, request: Any):
- """Expose the request's origin scheme to the connect-time backend for this request only."""
- token = schemes_by_origin_var.set(_origin_scheme_context(request))
- try:
- yield
- finally:
- schemes_by_origin_var.reset(token)
- def _install_ssrf_guard_on_transport(transport: Any, schemes_by_origin_var: Any, *, is_async: bool = False) -> None:
- """Swap the transport's pool network backend for the SSRF-guarded one (idempotent). Only the
- direct transport is guarded; proxy mounts delegate final-target resolution to the trusted proxy."""
- state = getattr(transport, "__dict__", {}) if transport is not None else {}
- if transport is None or state.get("_hermes_ssrf_guarded", False):
- return
- label = "async httpx transport" if is_async else "httpx transport"
- pool = state.get("_pool")
- if pool is None or not hasattr(pool, "_network_backend"):
- raise SSRFConnectionBlocked(f"Unsupported {label} cannot be made SSRF-safe")
- backend_cls = _SSRFGuardedAsyncNetworkBackend if is_async else _SSRFGuardedNetworkBackend
- pool._network_backend = backend_cls(schemes_by_origin_var)
- method_name = "handle_async_request" if is_async else "handle_request"
- handle = getattr(transport, method_name, None)
- if handle is None:
- raise SSRFConnectionBlocked(f"Unsupported {label} cannot be made SSRF-safe")
- async def guarded_async(request: Any) -> Any:
- with _origin_scope(schemes_by_origin_var, request):
- return await handle(request)
- def guarded_sync(request: Any) -> Any:
- with _origin_scope(schemes_by_origin_var, request):
- return handle(request)
- setattr(transport, method_name, guarded_async if is_async else guarded_sync)
- transport._hermes_ssrf_guarded = True
- def _install_ssrf_guard_on_client(client: Any, *, is_async: bool = False) -> None:
- """Guard ``client._transport`` only; ``_mounts`` (env/explicit proxies) stay untouched."""
- import contextvars
- var_name = "hermes_ssrf_async_origin_schemes" if is_async else "hermes_ssrf_origin_schemes"
- _install_ssrf_guard_on_transport(
- getattr(client, "__dict__", {}).get("_transport"), contextvars.ContextVar(var_name), is_async=is_async)
- def create_ssrf_safe_async_client(**kwargs: Any) -> Any:
- """``httpx.AsyncClient`` with connect-time SSRF validation: direct HTTP(S) connections are
- resolved, validated, and dialed by IP while the hostname is preserved for Host, SNI, and
- certificate verification. Proxied requests delegate resolution to the proxy."""
- import httpx
- client = httpx.AsyncClient(**kwargs)
- _install_ssrf_guard_on_client(client, is_async=True)
- return client
- def create_ssrf_safe_client(**kwargs: Any) -> Any:
- """Create an ``httpx.Client`` with connect-time SSRF validation."""
- import httpx
- client = httpx.Client(**kwargs)
- _install_ssrf_guard_on_client(client)
- return client
- def redirect_target_from_response(response: Any) -> Optional[str]:
- """Redirect target visible from inside an httpx response hook. ``response.next_request`` is
- frequently ``None`` inside hooks (populated later by the follower), which would make an SSRF
- redirect guard silently never fire — so resolve from ``Location`` first, then ``next_request``."""
- if not getattr(response, "is_redirect", False):
- return None
- location = (getattr(response, "headers", {}) or {}).get("location")
- if location:
- return urljoin(str(getattr(response, "url", "")), str(location))
- next_request = getattr(response, "next_request", None)
- return str(next_request.url) if next_request else None
|