ssl_verify.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. """TLS trust for Hermes. One authority, one trust source.
  2. Trust comes from the platform verifier via ``truststore``: CryptoAPI on
  3. Windows, Security.framework on macOS, and OpenSSL's own store (with a
  4. distro candidate sweep) on Linux. That is the machine's real answer to
  5. "is this chain valid", so a corporate MITM root installed by MDM, an
  6. internal CA, and a locked-down NixOS box all work with no configuration.
  7. ``install_truststore()`` runs once at process start and patches
  8. ``ssl.SSLContext`` process-wide, so every stack that builds a default
  9. context inherits it — httpx, requests/urllib3, aiohttp, AND the stdlib
  10. ``urllib.request`` call sites (the llama.cpp engine download among them)
  11. that a certifi-only or requests-only approach never reached.
  12. An exported ``SSL_CERT_FILE`` bundle is trusted ON TOP of the platform
  13. store; only EXPLICIT PER-PROVIDER CONFIG replaces that trust:
  14. ``ssl_ca_cert`` (a self-signed or internal endpoint's bundle) and
  15. ``ssl_verify: false`` (local development). Those are deliberate
  16. statements about one endpoint, not an ambient guess about the machine.
  17. """
  18. from __future__ import annotations
  19. import logging
  20. import os
  21. import ssl
  22. import threading
  23. from pathlib import Path
  24. from typing import Any, Optional
  25. logger = logging.getLogger(__name__)
  26. _installed: bool | None = None
  27. def install_truststore() -> bool:
  28. """Point every default SSLContext at the OS trust store. Idempotent.
  29. Returns True when the platform verifier is in force. A False return
  30. means truststore could not load (an unsupported platform, or a
  31. stripped payload); TLS still works off OpenSSL's compiled-in paths,
  32. it just won't see certificates the OS trusts.
  33. Call before the first HTTPS client is constructed.
  34. """
  35. global _installed
  36. if _installed is not None:
  37. return _installed
  38. try:
  39. import truststore
  40. truststore.inject_into_ssl()
  41. _installed = True
  42. logger.debug("TLS trust: platform store (truststore)")
  43. except Exception as exc:
  44. _installed = False
  45. logger.warning(
  46. "truststore unavailable (%s); falling back to OpenSSL's default "
  47. "trust paths. Certificates trusted only by the OS store — a "
  48. "corporate root, for instance — will not verify.",
  49. exc,
  50. )
  51. return _installed
  52. def _coerce_insecure(ssl_verify: Any) -> bool:
  53. if ssl_verify is False:
  54. return True
  55. if isinstance(ssl_verify, str) and ssl_verify.strip().lower() in {"false", "0", "no", "off"}:
  56. return True
  57. return False
  58. _CA_CONTEXTS: dict[tuple[str | None, bool], ssl.SSLContext] = {}
  59. _CA_CONTEXTS_LOCK = threading.Lock()
  60. def _stdlib_ssl_context_class() -> type[ssl.SSLContext]:
  61. """The un-injected stdlib ``ssl.SSLContext``.
  62. An explicit bundle must REPLACE OS trust, and truststore's context falls
  63. back to the OS verifier whenever the loaded bundle rejects a chain — so
  64. the bundle context has to be built from the stdlib class. Once injected
  65. (here, or by pm.launch/pm.worker before this module loads) the only
  66. handle on it is truststore's own saved reference; when nothing is
  67. injected — truststore absent or not installed — ``ssl.SSLContext`` is
  68. already the stdlib class and truststore must not be imported at all.
  69. """
  70. if ssl.SSLContext.__module__ == "ssl":
  71. return ssl.SSLContext
  72. from truststore._ssl_constants import _original_SSLContext
  73. return _original_SSLContext
  74. def _shared_context(ca_path: str | None, union: bool = False) -> ssl.SSLContext:
  75. """A stable context identity lets clients reuse the existing transport pool.
  76. ``union`` adds ``ca_path`` to the platform store (the ``SSL_CERT_FILE``
  77. contract) instead of replacing it (a provider ``ssl_ca_cert``).
  78. """
  79. with _CA_CONTEXTS_LOCK:
  80. ctx = _CA_CONTEXTS.get((ca_path, union))
  81. if ctx is None:
  82. if ca_path is not None and not union:
  83. # PROTOCOL_TLS_CLIENT sets hostname checking and CERT_REQUIRED;
  84. # assigning the original class's properties after injection recurses.
  85. ctx = _stdlib_ssl_context_class()(ssl.PROTOCOL_TLS_CLIENT)
  86. else:
  87. ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
  88. if not _installed:
  89. ctx.load_default_certs()
  90. if ca_path is not None:
  91. ctx.load_verify_locations(cafile=ca_path)
  92. _CA_CONTEXTS[(ca_path, union)] = ctx
  93. return ctx
  94. def platform_ssl_context() -> ssl.SSLContext:
  95. """The process-wide client context (cheap after the first call).
  96. An existing ``SSL_CERT_FILE`` bundle is trusted ON TOP of the platform store,
  97. as it was for httpx ``verify=True``; a missing or stale path falls back to
  98. the platform store alone.
  99. """
  100. install_truststore()
  101. ca = os.environ.get("SSL_CERT_FILE", "").strip()
  102. if ca and Path(ca).is_file():
  103. return _shared_context(str(Path(ca).resolve()), union=True)
  104. return _shared_context(None)
  105. _STALE_BUNDLE_OUTCOMES: dict[str, Optional[str]] = {}
  106. _STALE_BUNDLE_LOCK = threading.Lock()
  107. def certifi_bundle_path() -> Optional[str]:
  108. """certifi's CA bundle path, re-derived when a rotation invalidated it.
  109. ``certifi.where()`` resolves its path once, against the ``certifi`` package on the
  110. ``sys.path`` of the moment it was first called, and caches it in ``certifi.core``:
  111. a process that outlives an install-environment rotation — the package manager
  112. creates a new environment directory and deletes the old one — keeps pointing into
  113. the deleted directory for the rest of its life, and every TLS client it builds
  114. afterwards fails. Re-importing the package against the live ``sys.path`` repairs it
  115. process-wide; when even that finds nothing, the platform store is the answer and
  116. never an exception.
  117. The repair, and its warning, happen ONCE per dead path: a gateway builds clients
  118. per turn, per cron run and per model-metadata fetch, so an un-cached reload here
  119. would re-execute third-party module code and re-log on every client it builds.
  120. Returns a bundle path that exists on disk, else ``None``.
  121. """
  122. try:
  123. import certifi
  124. except ImportError:
  125. return None
  126. stale = certifi.where()
  127. if Path(stale).is_file():
  128. return stale
  129. return _recover_bundle_once(stale)
  130. def _recover_bundle_once(stale: str) -> Optional[str]:
  131. """Re-derive ``stale`` at most once, and report it at most once.
  132. A cached outcome is reused while it still holds (a re-derived path that exists, or
  133. an unrecoverable one); a later rotation dead-path is a new key and is repaired on
  134. its own.
  135. """
  136. with _STALE_BUNDLE_LOCK:
  137. if stale in _STALE_BUNDLE_OUTCOMES:
  138. cached = _STALE_BUNDLE_OUTCOMES[stale]
  139. if cached is None or Path(cached).is_file():
  140. return cached
  141. recovered = _reload_certifi_bundle()
  142. _STALE_BUNDLE_OUTCOMES[stale] = recovered
  143. if recovered is not None:
  144. logger.warning(
  145. "TLS trust: certifi's CA bundle %s no longer exists — the install "
  146. "environment it belongs to was replaced under this process; re-resolved "
  147. "to %s. TLS clients built before this point used the dead path.",
  148. stale,
  149. recovered,
  150. )
  151. return recovered
  152. logger.warning(
  153. "TLS trust: certifi's CA bundle %s no longer exists and certifi could not be "
  154. "re-resolved; using the platform trust store for the rest of this process.",
  155. stale,
  156. )
  157. return None
  158. def _reload_certifi_bundle() -> Optional[str]:
  159. """Re-import certifi from the live ``sys.path`` so its ``where()`` follows it.
  160. ``importlib.reload`` on its own does NOT re-derive the path with the real package:
  161. ``certifi.where`` is ``certifi.core.where`` re-exported by ``__init__``, and ``core``
  162. caches the resolved path in a module global, so re-executing ``__init__`` re-binds
  163. ``where`` from the still-loaded ``core`` — the replaced directory, cached a second
  164. time. Only dropping the package and its submodules from ``sys.modules`` makes both
  165. files re-execute against the live ``sys.path``. They are put back when the import
  166. fails, so the process is never left without the ``certifi`` it already had.
  167. ``None`` when certifi was never imported or its package is gone from every
  168. ``sys.path`` entry — the caller then falls back to the platform store.
  169. """
  170. import importlib
  171. import sys
  172. if "certifi" not in sys.modules:
  173. return None
  174. # The replaced directory is gone; a finder holding a cached listing of it would
  175. # still resolve the old package.
  176. importlib.invalidate_caches()
  177. dropped = {
  178. name: sys.modules.pop(name)
  179. for name in [n for n in sys.modules if n == "certifi" or n.startswith("certifi.")]
  180. }
  181. try:
  182. path = importlib.import_module("certifi").where()
  183. except Exception as exc: # the import re-executes third-party module code
  184. sys.modules.update(dropped)
  185. logger.debug("certifi could not be re-resolved: %s", exc, exc_info=True)
  186. return None
  187. return path if path and Path(path).is_file() else None
  188. def requests_verify() -> bool | str:
  189. """``verify`` for a ``requests`` call in this process.
  190. ``requests`` resolves ``verify=True`` through ``DEFAULT_CA_BUNDLE_PATH``, a string
  191. frozen when ``requests`` was imported — the one certifi consumer a module reload
  192. cannot reach, so a rotated environment leaves every ``requests`` call raising
  193. ``OSError: Could not find a suitable TLS CA certificate bundle``. The live bundle
  194. (certifi's, re-derived; else OpenSSL's own default) keeps those calls working;
  195. ``True`` only when nothing resolves, which is requests' own default. A healthy
  196. install keeps ``True`` too: requests reads ``REQUESTS_CA_BUNDLE``/``CURL_CA_BUNDLE``
  197. only for ``verify=True``, and an explicit path would silently bypass a user's
  198. proxy CA.
  199. """
  200. from requests.utils import DEFAULT_CA_BUNDLE_PATH
  201. if os.environ.get("REQUESTS_CA_BUNDLE") or os.environ.get("CURL_CA_BUNDLE"):
  202. return True
  203. if Path(DEFAULT_CA_BUNDLE_PATH).is_file():
  204. return True
  205. bundle = certifi_bundle_path()
  206. if bundle:
  207. return bundle
  208. defaults = ssl.get_default_verify_paths()
  209. for candidate in (defaults.cafile, defaults.capath):
  210. if candidate and Path(candidate).exists():
  211. return candidate
  212. # Windows keeps its roots in the system store with no OpenSSL file, and ``True``
  213. # would send requests back to the dead frozen path: hand it the store as a bundle.
  214. return _platform_store_bundle() or True
  215. _PLATFORM_STORE_BUNDLE: Optional[str] = None
  216. def _platform_store_bundle() -> Optional[str]:
  217. """The OS store's CA certificates written once to a PEM file under HERMES_HOME."""
  218. global _PLATFORM_STORE_BUNDLE
  219. if _PLATFORM_STORE_BUNDLE and Path(_PLATFORM_STORE_BUNDLE).is_file():
  220. return _PLATFORM_STORE_BUNDLE
  221. ctx = _stdlib_ssl_context_class()(ssl.PROTOCOL_TLS_CLIENT)
  222. ctx.load_default_certs()
  223. certs = ctx.get_ca_certs(binary_form=True)
  224. if not certs:
  225. return None
  226. from hermes_constants import get_hermes_home
  227. from utils import atomic_write_text
  228. path = get_hermes_home() / "cache" / "platform-ca-bundle.pem"
  229. try:
  230. path.parent.mkdir(parents=True, exist_ok=True)
  231. atomic_write_text(path, "".join(ssl.DER_cert_to_PEM_cert(der) for der in certs))
  232. except OSError as exc: # a full or read-only home must not replace the TLS fallback with a disk error
  233. logger.debug("platform CA bundle not written: %s", exc, exc_info=True)
  234. return None
  235. _PLATFORM_STORE_BUNDLE = str(path)
  236. return _PLATFORM_STORE_BUNDLE
  237. def _default_verify() -> bool | ssl.SSLContext:
  238. """``verify`` for the platform default path.
  239. httpx resolves ``verify=True`` through ``certifi.where()`` at client construction,
  240. which raises ``FileNotFoundError`` on a bundle path a rotation deleted. Hand the
  241. platform context over instead when no bundle can be found; a re-derivable bundle
  242. keeps ``True`` — the module reload already repaired certifi process-wide.
  243. """
  244. return True if certifi_bundle_path() else _shared_context(None)
  245. def resolve_httpx_verify(
  246. *,
  247. ca_bundle: Optional[str] = None,
  248. ssl_verify: Any = None,
  249. base_url: str = "",
  250. ) -> bool | ssl.SSLContext:
  251. """Resolve ``verify`` for an HTTP client.
  252. 1. ``ssl_verify: false`` — verification off (local development only)
  253. 2. explicit ``ca_bundle`` (the provider's ``ssl_ca_cert``) — that
  254. bundle INSTEAD of the platform store, for an endpoint whose chain
  255. the machine has no reason to trust
  256. 3. ``True`` — the platform store, via the process-wide install, or the
  257. platform context when the certifi bundle this process resolved at
  258. import no longer exists (a replaced install environment)
  259. ``base_url`` only labels the insecure-mode warning.
  260. """
  261. install_truststore()
  262. if _coerce_insecure(ssl_verify):
  263. logger.warning(
  264. "TLS certificate verification DISABLED (ssl_verify: false) for %s — "
  265. "this is intended for local development only and is unsafe on any "
  266. "network you do not fully control.",
  267. base_url or "a custom provider endpoint",
  268. )
  269. return False
  270. effective_ca = (ca_bundle or "").strip()
  271. if effective_ca:
  272. path = Path(effective_ca).expanduser()
  273. if path.is_file():
  274. return _shared_context(str(path.resolve()))
  275. logger.warning(
  276. "ssl_ca_cert path does not exist: %s — using the OS trust store instead",
  277. effective_ca,
  278. )
  279. # HTTPX reads CA env vars before the injected verifier gets control. Pass
  280. # the platform context directly so stale paths cannot break construction;
  281. # proxy environment handling remains enabled.
  282. if os.environ.get("SSL_CERT_FILE") or os.environ.get("SSL_CERT_DIR"):
  283. return _shared_context(None)
  284. return _default_verify()