| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158 |
- """Security policy for credential-bearing stdlib urllib requests."""
- from __future__ import annotations
- import copy
- import logging
- import ssl
- import urllib.parse
- import urllib.request
- from collections.abc import Callable, Iterable
- from typing import Any
- logger = logging.getLogger(__name__)
- # Headers safe to forward to a different origin. Everything else is dropped:
- # custom provider headers routinely carry credentials under arbitrary names.
- _CROSS_ORIGIN_SAFE_HEADERS = frozenset({"accept", "user-agent"})
- _DEFAULT_PORTS = {"http": 80, "https": 443}
- def url_origin(url: str) -> tuple[str, str, int | None]:
- """Return a normalized (scheme, hostname, effective port) origin."""
- parsed = urllib.parse.urlparse(url)
- scheme = (parsed.scheme or "").lower()
- # ``parsed.port`` raises ValueError on malformed ports — let that fail the
- # request closed instead of collapsing it to a default.
- port = parsed.port
- return scheme, (parsed.hostname or "").lower().rstrip("."), port if port is not None else _DEFAULT_PORTS.get(scheme)
- def _strip_headers(request, keep: frozenset[str]) -> None:
- """Drop every header on *request* whose lowercased name is not in *keep*."""
- for name, _value in list(request.header_items()):
- if name.lower() not in keep:
- request.remove_header(name)
- class SafeCredentialRedirectHandler(urllib.request.HTTPRedirectHandler):
- """Preserve request headers only while redirects stay on one origin."""
- def __init__(
- self, original_url: str, *, cross_origin_safe_headers: Iterable[str] = _CROSS_ORIGIN_SAFE_HEADERS
- ) -> None:
- self._original_origin = url_origin(original_url)
- self._cross_origin_safe_headers = frozenset(str(name).lower() for name in cross_origin_safe_headers)
- def redirect_request(self, req, fp, code, msg, headers, newurl):
- # Let urllib enforce status/method semantics first (notably 307/308).
- redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
- if redirected is None:
- return None
- # Allowlist rather than guessing credential header names: normalize_extra_headers
- # permits arbitrary secret-bearing names.
- if url_origin(urllib.parse.urljoin(req.full_url, newurl)) != self._original_origin:
- _strip_headers(redirected, self._cross_origin_safe_headers)
- return redirected
- class _CrossOriginRequestSanitizer(urllib.request.BaseHandler):
- """Strip headers after installed request processors have run."""
- # Request processors run in ascending order; infinity keeps this last so an
- # installed cookie/auth/instrumentation processor cannot re-add a secret after
- # the redirect handler sanitized the new Request (stable sort keeps this
- # appended handler after another infinity-ordered one).
- handler_order = float("inf") # type: ignore[assignment]
- def __init__(self, original_url: str) -> None:
- self._original_origin = url_origin(original_url)
- def _sanitize(self, request: urllib.request.Request):
- if url_origin(request.full_url) != self._original_origin:
- _strip_headers(request, _CROSS_ORIGIN_SAFE_HEADERS)
- return request
- http_request = _sanitize
- https_request = _sanitize
- def _resolved_https_context() -> ssl.SSLContext | None:
- """TLS context for Hermes-owned urllib openers.
- None means "use urllib's default", which — with the OS trust store
- installed process-wide — already verifies against the platform's
- certificates. There is nothing to resolve here any more.
- """
- from agent.ssl_verify import install_truststore
- install_truststore()
- return None
- def _secure_opener_from_installed_policy(original_url: str, *, ssl_context=None):
- """Clone the installed opener's handlers, replacing redirect policy only.
- When ``ssl_context`` is provided, the cloned HTTPS handler is replaced with
- one bound to that context so per-provider TLS settings (``ssl_ca_cert`` /
- ``ssl_verify``) apply to this request. When it is None, Hermes-owned
- openers verify against the OS trust store; an application-installed
- opener's TLS policy is preserved unchanged.
- """
- installed = getattr(urllib.request, "_opener", None)
- if installed is None:
- context = _resolved_https_context()
- installed = urllib.request.build_opener(*([] if context is None else [urllib.request.HTTPSHandler(context=context)]))
- _https_handler_cls = getattr(urllib.request, "HTTPSHandler", None)
- replace_https = ssl_context is not None and _https_handler_cls is not None
- handlers = [
- copy.copy(handler)
- for handler in getattr(installed, "handlers", ())
- if not isinstance(handler, urllib.request.HTTPRedirectHandler)
- and not (replace_https and isinstance(handler, _https_handler_cls))
- ]
- if replace_https:
- handlers.append(_https_handler_cls(context=ssl_context))
- handlers.append(SafeCredentialRedirectHandler(original_url))
- handlers.append(_CrossOriginRequestSanitizer(original_url))
- secured = urllib.request.build_opener(*handlers)
- # OpenerDirector injects addheaders after request processors (bypassing the
- # sanitizer on redirects), so carry them on the initial request instead.
- secured._hermes_initial_addheaders = list(getattr(installed, "addheaders", ()))
- secured.addheaders = []
- return secured
- def open_credentialed_url(
- request: urllib.request.Request,
- *,
- timeout: float,
- opener_factory: Callable[..., Any] | None = None,
- ssl_context=None,
- ):
- """Open a request without forwarding credentials across origins.
- The default preserves an application-installed opener's proxy, TLS,
- cookies, custom protocol handlers, and instrumentation while replacing its
- redirect handler. ``opener_factory`` is an explicit test seam; security is
- never disabled based on global ``urlopen`` identity.
- ``ssl_context`` (an ``ssl.SSLContext``) overrides the HTTPS handler's TLS
- policy for this request only. It is used to honor a custom provider's
- ``ssl_ca_cert`` / ``ssl_verify`` on the ``/models`` discovery path, which
- otherwise falls back to the process-wide platform trust store
- (``agent.ssl_verify.install_truststore``).
- """
- if opener_factory is None:
- opener = _secure_opener_from_installed_policy(request.full_url, ssl_context=ssl_context)
- for name, value in getattr(opener, "_hermes_initial_addheaders", ()):
- if not request.has_header(name):
- request.add_header(name, value)
- else:
- opener = opener_factory(SafeCredentialRedirectHandler(request.full_url))
- return opener.open(request, timeout=timeout)
- __all__ = ["SafeCredentialRedirectHandler", "open_credentialed_url", "url_origin"]
|