urllib_security.py 6.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158
  1. """Security policy for credential-bearing stdlib urllib requests."""
  2. from __future__ import annotations
  3. import copy
  4. import logging
  5. import ssl
  6. import urllib.parse
  7. import urllib.request
  8. from collections.abc import Callable, Iterable
  9. from typing import Any
  10. logger = logging.getLogger(__name__)
  11. # Headers safe to forward to a different origin. Everything else is dropped:
  12. # custom provider headers routinely carry credentials under arbitrary names.
  13. _CROSS_ORIGIN_SAFE_HEADERS = frozenset({"accept", "user-agent"})
  14. _DEFAULT_PORTS = {"http": 80, "https": 443}
  15. def url_origin(url: str) -> tuple[str, str, int | None]:
  16. """Return a normalized (scheme, hostname, effective port) origin."""
  17. parsed = urllib.parse.urlparse(url)
  18. scheme = (parsed.scheme or "").lower()
  19. # ``parsed.port`` raises ValueError on malformed ports — let that fail the
  20. # request closed instead of collapsing it to a default.
  21. port = parsed.port
  22. return scheme, (parsed.hostname or "").lower().rstrip("."), port if port is not None else _DEFAULT_PORTS.get(scheme)
  23. def _strip_headers(request, keep: frozenset[str]) -> None:
  24. """Drop every header on *request* whose lowercased name is not in *keep*."""
  25. for name, _value in list(request.header_items()):
  26. if name.lower() not in keep:
  27. request.remove_header(name)
  28. class SafeCredentialRedirectHandler(urllib.request.HTTPRedirectHandler):
  29. """Preserve request headers only while redirects stay on one origin."""
  30. def __init__(
  31. self, original_url: str, *, cross_origin_safe_headers: Iterable[str] = _CROSS_ORIGIN_SAFE_HEADERS
  32. ) -> None:
  33. self._original_origin = url_origin(original_url)
  34. self._cross_origin_safe_headers = frozenset(str(name).lower() for name in cross_origin_safe_headers)
  35. def redirect_request(self, req, fp, code, msg, headers, newurl):
  36. # Let urllib enforce status/method semantics first (notably 307/308).
  37. redirected = super().redirect_request(req, fp, code, msg, headers, newurl)
  38. if redirected is None:
  39. return None
  40. # Allowlist rather than guessing credential header names: normalize_extra_headers
  41. # permits arbitrary secret-bearing names.
  42. if url_origin(urllib.parse.urljoin(req.full_url, newurl)) != self._original_origin:
  43. _strip_headers(redirected, self._cross_origin_safe_headers)
  44. return redirected
  45. class _CrossOriginRequestSanitizer(urllib.request.BaseHandler):
  46. """Strip headers after installed request processors have run."""
  47. # Request processors run in ascending order; infinity keeps this last so an
  48. # installed cookie/auth/instrumentation processor cannot re-add a secret after
  49. # the redirect handler sanitized the new Request (stable sort keeps this
  50. # appended handler after another infinity-ordered one).
  51. handler_order = float("inf") # type: ignore[assignment]
  52. def __init__(self, original_url: str) -> None:
  53. self._original_origin = url_origin(original_url)
  54. def _sanitize(self, request: urllib.request.Request):
  55. if url_origin(request.full_url) != self._original_origin:
  56. _strip_headers(request, _CROSS_ORIGIN_SAFE_HEADERS)
  57. return request
  58. http_request = _sanitize
  59. https_request = _sanitize
  60. def _resolved_https_context() -> ssl.SSLContext | None:
  61. """TLS context for Hermes-owned urllib openers.
  62. None means "use urllib's default", which — with the OS trust store
  63. installed process-wide — already verifies against the platform's
  64. certificates. There is nothing to resolve here any more.
  65. """
  66. from agent.ssl_verify import install_truststore
  67. install_truststore()
  68. return None
  69. def _secure_opener_from_installed_policy(original_url: str, *, ssl_context=None):
  70. """Clone the installed opener's handlers, replacing redirect policy only.
  71. When ``ssl_context`` is provided, the cloned HTTPS handler is replaced with
  72. one bound to that context so per-provider TLS settings (``ssl_ca_cert`` /
  73. ``ssl_verify``) apply to this request. When it is None, Hermes-owned
  74. openers verify against the OS trust store; an application-installed
  75. opener's TLS policy is preserved unchanged.
  76. """
  77. installed = getattr(urllib.request, "_opener", None)
  78. if installed is None:
  79. context = _resolved_https_context()
  80. installed = urllib.request.build_opener(*([] if context is None else [urllib.request.HTTPSHandler(context=context)]))
  81. _https_handler_cls = getattr(urllib.request, "HTTPSHandler", None)
  82. replace_https = ssl_context is not None and _https_handler_cls is not None
  83. handlers = [
  84. copy.copy(handler)
  85. for handler in getattr(installed, "handlers", ())
  86. if not isinstance(handler, urllib.request.HTTPRedirectHandler)
  87. and not (replace_https and isinstance(handler, _https_handler_cls))
  88. ]
  89. if replace_https:
  90. handlers.append(_https_handler_cls(context=ssl_context))
  91. handlers.append(SafeCredentialRedirectHandler(original_url))
  92. handlers.append(_CrossOriginRequestSanitizer(original_url))
  93. secured = urllib.request.build_opener(*handlers)
  94. # OpenerDirector injects addheaders after request processors (bypassing the
  95. # sanitizer on redirects), so carry them on the initial request instead.
  96. secured._hermes_initial_addheaders = list(getattr(installed, "addheaders", ()))
  97. secured.addheaders = []
  98. return secured
  99. def open_credentialed_url(
  100. request: urllib.request.Request,
  101. *,
  102. timeout: float,
  103. opener_factory: Callable[..., Any] | None = None,
  104. ssl_context=None,
  105. ):
  106. """Open a request without forwarding credentials across origins.
  107. The default preserves an application-installed opener's proxy, TLS,
  108. cookies, custom protocol handlers, and instrumentation while replacing its
  109. redirect handler. ``opener_factory`` is an explicit test seam; security is
  110. never disabled based on global ``urlopen`` identity.
  111. ``ssl_context`` (an ``ssl.SSLContext``) overrides the HTTPS handler's TLS
  112. policy for this request only. It is used to honor a custom provider's
  113. ``ssl_ca_cert`` / ``ssl_verify`` on the ``/models`` discovery path, which
  114. otherwise falls back to the process-wide platform trust store
  115. (``agent.ssl_verify.install_truststore``).
  116. """
  117. if opener_factory is None:
  118. opener = _secure_opener_from_installed_policy(request.full_url, ssl_context=ssl_context)
  119. for name, value in getattr(opener, "_hermes_initial_addheaders", ()):
  120. if not request.has_header(name):
  121. request.add_header(name, value)
  122. else:
  123. opener = opener_factory(SafeCredentialRedirectHandler(request.full_url))
  124. return opener.open(request, timeout=timeout)
  125. __all__ = ["SafeCredentialRedirectHandler", "open_credentialed_url", "url_origin"]