registry.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. """Secret-source registry + apply orchestrator.
  2. Owns everything that must be uniform across backends: registration
  3. (name/scheme uniqueness, API-version gating), the wall-clock timeout around
  4. ``fetch()``, precedence (mapped beats bulk; within a shape ``secrets.sources``
  5. order, else registration order; first claim wins), ``override_existing``
  6. semantics (may beat .env/shell, never another source, never a protected var),
  7. cross-source conflict warnings, and provenance. Startup entry point:
  8. :func:`apply_all` via ``hermes_cli.env_loader``; plugins register through
  9. ``PluginContext.register_secret_source()`` → :func:`register_source`.
  10. """
  11. from __future__ import annotations
  12. import concurrent.futures
  13. import logging
  14. import os
  15. import threading
  16. from dataclasses import dataclass, field
  17. from pathlib import Path
  18. from typing import Dict, List, MutableMapping, Optional
  19. from agent.secret_sources.base import (
  20. SECRET_SOURCE_API_VERSION, ErrorKind, FetchResult, SecretSource, is_valid_env_name,
  21. reset_source_environment, set_source_environment,
  22. )
  23. from hermes_constants import hermes_home_key, normalize_scope
  24. logger = logging.getLogger(__name__)
  25. # Ordered registry: name → source. Dict insertion order doubles as the default
  26. # apply order. Origin is recorded so consumers never infer ownership from names.
  27. _SOURCES: dict[str, SecretSource] = {}
  28. _SOURCE_ORIGINS: dict[str, str] = {}
  29. _SCOPED_SOURCES: dict[str, dict[str, SecretSource]] = {}
  30. _BUILTINS_LOADED = False
  31. _REGISTRY_LOCK = threading.RLock()
  32. # (module, class, label) for the bundled sources, in registration order.
  33. _BUILTIN_SOURCES = (
  34. ("agent.secret_sources.bitwarden", "BitwardenSource", "Bitwarden"),
  35. ("agent.secret_sources.onepassword", "OnePasswordSource", "1Password"),
  36. ("agent.secret_sources.command", "CommandSource", "command"),
  37. )
  38. @dataclass
  39. class AppliedVar:
  40. """Provenance record for one env var the orchestrator set."""
  41. name: str
  42. source: str # SecretSource.name
  43. shape: str # "mapped" | "bulk"
  44. overrode_env: bool # replaced a pre-existing .env/shell value
  45. # The source may beat .env/shell for this var (``override_existing`` and not ``preserve_existing``), so a
  46. # dotenv reload may re-assert it; a gap-fill or preserved name must keep following .env edits (#74265).
  47. authoritative: bool = False
  48. @dataclass
  49. class SourceReport:
  50. """One source's outcome within an :class:`ApplyReport`."""
  51. name: str
  52. label: str
  53. result: FetchResult
  54. applied: list[str] = field(default_factory=list)
  55. skipped_existing: list[str] = field(default_factory=list) # .env/shell won
  56. skipped_claimed: list[str] = field(default_factory=list) # earlier source won
  57. skipped_protected: list[str] = field(default_factory=list) # bootstrap-auth guard
  58. skipped_invalid: list[str] = field(default_factory=list) # bad env-var name
  59. @dataclass
  60. class ApplyReport:
  61. """Merged outcome of one orchestrated apply pass."""
  62. sources: list[SourceReport] = field(default_factory=list)
  63. provenance: dict[str, AppliedVar] = field(default_factory=dict)
  64. conflicts: list[str] = field(default_factory=list) # human-readable warnings
  65. @property
  66. def applied_any(self) -> bool:
  67. return bool(self.provenance)
  68. # --- Registration -----------------------------------------------------------
  69. def _validate_source(source: SecretSource) -> Optional[str]:
  70. """Reason a source is unregistrable (already formatted for the log), or None."""
  71. if not isinstance(source, SecretSource):
  72. return f"Ignoring secret source {source!r}: does not inherit from SecretSource"
  73. name = source.name or ""
  74. if not name or not name.replace("_", "").isalnum() or name != name.lower():
  75. return f"Ignoring secret source with invalid name {name!r}"
  76. if source.api_version != SECRET_SOURCE_API_VERSION:
  77. return (f"Ignoring secret source '{name}': built against secret-source API "
  78. f"v{source.api_version}, this Hermes speaks v{SECRET_SOURCE_API_VERSION}")
  79. if source.shape not in ("mapped", "bulk"):
  80. return f"Ignoring secret source '{name}': shape must be 'mapped' or 'bulk', got {source.shape!r}"
  81. return None
  82. def register_source(source: SecretSource, *, replace: bool = False, builtin: bool = False,
  83. scope: Optional[str] = None) -> bool:
  84. """Register a secret source; True on success. Rejections are logged, never
  85. raised. ``replace`` allows same-name override (last-writer-wins); scheme
  86. collisions across *different* names are always rejected."""
  87. problem = _validate_source(source)
  88. if problem:
  89. logger.warning(problem)
  90. return False
  91. scope = normalize_scope(scope)
  92. name = source.name
  93. with _REGISTRY_LOCK:
  94. effective = dict(_SOURCES)
  95. if scope is not None:
  96. effective.update(_SCOPED_SOURCES.get(scope, {}))
  97. if name in effective and not replace:
  98. logger.warning("Secret source '%s' already registered; ignoring duplicate", name)
  99. return False
  100. owner = next((n for n, o in effective.items()
  101. if n != name and source.scheme and o.scheme == source.scheme), None)
  102. if owner:
  103. logger.warning("Ignoring secret source '%s': scheme '%s://' is already owned by source '%s'",
  104. name, source.scheme, owner)
  105. return False
  106. target = _SOURCES if scope is None else _SCOPED_SOURCES.setdefault(scope, {})
  107. target[name] = source
  108. if scope is None:
  109. _SOURCE_ORIGINS[name] = "builtin" if builtin else "plugin"
  110. return True
  111. def _merged(scope: Optional[str]) -> dict[str, SecretSource]:
  112. """Global sources overlaid with the scope's (default: current home) registrations."""
  113. merged = dict(_SOURCES)
  114. merged.update(_SCOPED_SOURCES.get(hermes_home_key(scope), {}))
  115. return merged
  116. def get_source(name: str, *, scope: Optional[str] = None) -> Optional[SecretSource]:
  117. _ensure_builtin_sources()
  118. with _REGISTRY_LOCK:
  119. return _merged(scope).get(name)
  120. def snapshot_registration(name: str, *, scope: Optional[str] = None) -> Optional[SecretSource]:
  121. """Return the registration owned by exactly one registry layer."""
  122. _ensure_builtin_sources()
  123. scope = normalize_scope(scope)
  124. with _REGISTRY_LOCK:
  125. return (_SOURCES if scope is None else _SCOPED_SOURCES.get(scope, {})).get(name)
  126. def restore_registration(name: str, current: SecretSource, previous: Optional[SecretSource], *,
  127. scope: Optional[str] = None) -> bool:
  128. """Restore a host-owned source registration if it is still current."""
  129. _ensure_builtin_sources()
  130. scope = normalize_scope(scope)
  131. with _REGISTRY_LOCK:
  132. target = _SOURCES if scope is None else _SCOPED_SOURCES.setdefault(scope, {})
  133. if target.get(name) is not current:
  134. return False
  135. if previous is None:
  136. target.pop(name, None)
  137. else:
  138. target[name] = previous
  139. if scope is not None and not target:
  140. _SCOPED_SOURCES.pop(scope, None)
  141. return True
  142. def list_sources(*, scope: Optional[str] = None) -> list[SecretSource]:
  143. _ensure_builtin_sources()
  144. with _REGISTRY_LOCK:
  145. return list(_merged(scope).values())
  146. def list_plugin_sources() -> list[SecretSource]:
  147. """Sources registered outside the bundled set: global ``"plugin"`` origins
  148. plus every scoped registration (bundled sources register with scope=None).
  149. Includes both legacy global plugin registrations (``_SOURCE_ORIGINS == "plugin"``) and the current
  150. scope's profile-keyed registrations — every scoped entry is plugin-registered by definition, since
  151. bundled sources register with ``scope=None`` (#64229 profile isolation).
  152. """
  153. _ensure_builtin_sources()
  154. with _REGISTRY_LOCK:
  155. merged = {n: s for n, s in _SOURCES.items() if _SOURCE_ORIGINS.get(n) == "plugin"}
  156. merged.update(_SCOPED_SOURCES.get(hermes_home_key(), {}))
  157. return list(merged.values())
  158. def _ensure_builtin_sources() -> None:
  159. """Idempotently register the bundled sources (lazy so import stays cheap;
  160. per-source guarded so one broken source can't block the others)."""
  161. global _BUILTINS_LOADED
  162. with _REGISTRY_LOCK:
  163. if _BUILTINS_LOADED:
  164. return
  165. _BUILTINS_LOADED = True
  166. for module_name, class_name, label in _BUILTIN_SOURCES:
  167. try:
  168. module = __import__(module_name, fromlist=[class_name])
  169. register_source(getattr(module, class_name)(), builtin=True)
  170. except Exception:
  171. logger.warning("Failed to register bundled %s secret source", label, exc_info=True)
  172. def _reset_registry_for_tests() -> None:
  173. global _BUILTINS_LOADED
  174. with _REGISTRY_LOCK:
  175. _SOURCES.clear()
  176. _SOURCE_ORIGINS.clear()
  177. _SCOPED_SOURCES.clear()
  178. _BUILTINS_LOADED = False
  179. # --- Orchestrated apply -----------------------------------------------------
  180. def _fetch_with_timeout(source: SecretSource, cfg: dict, home_path: Path,
  181. environ: MutableMapping[str, str]) -> FetchResult:
  182. """Run source.fetch() under a wall-clock budget; never raises.
  183. A worker thread enforces the budget: a source that blows it is reported as
  184. TIMEOUT and its eventual result discarded (the thread may linger until
  185. process exit — acceptable for a startup-only path).
  186. """
  187. timeout = source.fetch_timeout_seconds(cfg)
  188. executor = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix=f"secret-src-{source.name}")
  189. try:
  190. def _fetch() -> FetchResult:
  191. token = set_source_environment(environ)
  192. try:
  193. return source.fetch(cfg, home_path)
  194. finally:
  195. reset_source_environment(token)
  196. future = executor.submit(_fetch)
  197. try:
  198. result = future.result(timeout=timeout)
  199. except concurrent.futures.TimeoutError:
  200. future.cancel()
  201. return FetchResult().fail(f"fetch exceeded {timeout:.0f}s budget — startup continued "
  202. "without this source (raise secrets."
  203. f"{source.name}.timeout_seconds if the backend is just slow)",
  204. ErrorKind.TIMEOUT)
  205. except Exception as exc:
  206. return FetchResult().fail(f"fetch raised {type(exc).__name__}: {exc}", ErrorKind.INTERNAL)
  207. finally:
  208. executor.shutdown(wait=False)
  209. if not isinstance(result, FetchResult):
  210. return FetchResult().fail(f"fetch returned {type(result).__name__} instead of FetchResult",
  211. ErrorKind.INTERNAL)
  212. return result
  213. def _section(secrets_cfg: dict, name: str) -> dict:
  214. cfg = secrets_cfg.get(name)
  215. return cfg if isinstance(cfg, dict) else {}
  216. def _ordered_enabled_sources(secrets_cfg: dict, *, scope: Optional[str] = None) -> list[SecretSource]:
  217. """Enabled sources: ``secrets.sources`` order first, then registration order
  218. (mapped-vs-bulk precedence is applied on top by :func:`apply_all`)."""
  219. sources = {source.name: source for source in list_sources(scope=scope)}
  220. explicit = secrets_cfg.get("sources")
  221. names = [e for e in explicit if isinstance(e, str)] if isinstance(explicit, list) else []
  222. unknown = [n for n in names if n not in sources]
  223. if unknown:
  224. logger.warning("secrets.sources names unknown source(s): %s (known: %s)",
  225. ", ".join(unknown), ", ".join(sources) or "none")
  226. order = dict.fromkeys([n for n in names if n in sources] + list(sources)) # insertion-ordered set
  227. enabled: list[SecretSource] = []
  228. for name in order:
  229. try:
  230. if sources[name].is_enabled(_section(secrets_cfg, name)):
  231. enabled.append(sources[name])
  232. except Exception:
  233. logger.warning("Secret source '%s' is_enabled() raised; skipping", name, exc_info=True)
  234. return enabled
  235. def enabled_source_names(secrets_cfg: dict, home_path: Path) -> frozenset:
  236. """Names of the sources :func:`apply_all` would fetch for *home_path* right now (registered
  237. and enabled). A source missing here was removed or disabled, so a value it injected earlier
  238. is no longer backed by anything and must be revoked, not kept as process residue."""
  239. secrets_cfg = secrets_cfg if isinstance(secrets_cfg, dict) else {}
  240. return frozenset(s.name for s in _ordered_enabled_sources(secrets_cfg, scope=hermes_home_key(home_path)))
  241. def _active_profile_name(home_path: Optional[Path]) -> str:
  242. """Active profile name (``~/.hermes/profiles/<name>``); "" for the default profile."""
  243. if home_path is not None:
  244. resolved = Path(home_path)
  245. if resolved.parent.name == "profiles" and resolved.name:
  246. return resolved.name
  247. for env_name in ("HERMES_PROFILE_NAME", "HERMES_PROFILE"):
  248. value = os.environ.get(env_name, "").strip()
  249. if value and value != "default":
  250. return value
  251. return ""
  252. # Only credential-shaped names get auto-aliased — a random profile-suffixed
  253. # var should not silently hydrate an unsuffixed name.
  254. _ALIAS_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET", "_KEY", "_PASSWORD")
  255. def _profile_alias_target(var: str, profile: str) -> Optional[str]:
  256. """Map ``FOO_<PROFILE>`` to ``FOO`` for the active profile when safe."""
  257. suffix = "_" + profile.replace("-", "_").upper()
  258. if not profile or not var.endswith(suffix):
  259. return None
  260. alias = var[: -len(suffix)]
  261. return alias if alias and is_valid_env_name(alias) and alias.endswith(_ALIAS_SUFFIXES) else None
  262. class _Applier:
  263. """Apply phase state for one orchestrated pass: sequential, first-wins, attributed."""
  264. def __init__(self, env: MutableMapping[str, str], report: ApplyReport,
  265. protected: dict[str, str], preserve: frozenset) -> None:
  266. self.env, self.report, self.protected, self.preserve = env, report, protected, preserve
  267. self.claimed: dict[str, str] = {} # var → source name that won it
  268. def apply_source(self, source: SecretSource, cfg: dict, result: FetchResult,
  269. profile: str, supplied_directly: set) -> None:
  270. sr = SourceReport(name=source.name, label=source.label or source.name, result=result)
  271. self.report.sources.append(sr)
  272. if not result.ok:
  273. return
  274. try:
  275. override = source.override_existing(cfg)
  276. except Exception:
  277. override = False
  278. for var, value in result.secrets.items():
  279. if not isinstance(var, str) or not isinstance(value, str):
  280. continue
  281. if not self._try_apply(sr, source, override, var, value) or not profile:
  282. continue
  283. alias = _profile_alias_target(var, profile)
  284. if (alias and alias not in supplied_directly and alias not in self.claimed
  285. and self._try_apply(sr, source, override, alias, value)):
  286. result.warnings.append(f"applied profile-scoped {var} as {alias} (active profile {profile!r})")
  287. def _try_apply(self, sr: SourceReport, source: SecretSource, override: bool,
  288. var: str, value: str) -> bool:
  289. """Apply one var through the shared guard chain. True = applied."""
  290. if not is_valid_env_name(var):
  291. sr.skipped_invalid.append(var)
  292. return False
  293. if var in self.protected:
  294. sr.skipped_protected.append(var)
  295. return False
  296. if var in self.claimed:
  297. sr.skipped_claimed.append(var)
  298. self.report.conflicts.append(f"{var}: kept value from {self.claimed[var]}; "
  299. f"{source.name} also supplies it (first source wins — "
  300. "remove one binding or reorder secrets.sources)")
  301. return False
  302. existed = bool(self.env.get(var))
  303. if existed and (var in self.preserve or not override):
  304. sr.skipped_existing.append(var)
  305. return False
  306. self.env[var] = value
  307. self.claimed[var] = source.name
  308. sr.applied.append(var)
  309. self.report.provenance[var] = AppliedVar(var, source.name, source.shape, overrode_env=existed,
  310. authoritative=override and var not in self.preserve)
  311. return True
  312. def apply_all(secrets_cfg: dict, home_path: Path,
  313. environ: Optional[MutableMapping[str, str]] = None) -> ApplyReport:
  314. """Fetch from every enabled source and apply the merged result to ``environ``
  315. (default ``os.environ``).
  316. Precedence per env var, most-specific intent first: (1) ``secrets.preserve_existing``
  317. names always keep a pre-existing value, even against ``override_existing: true``;
  318. (2) pre-existing .env/shell value, unless the winning source has
  319. ``override_existing: true``; (3) mapped sources in configured order; (4) bulk
  320. sources in configured order. First claim wins: a later source carrying the same
  321. var gets ``skipped_claimed`` plus a conflict warning — never a silent clobber,
  322. and ``override_existing`` never applies across sources.
  323. Profile aliasing: under a named profile an applied ``FOO_<PROFILE>``
  324. (credential-shaped suffixes only) also hydrates canonical ``FOO``, under the
  325. same guards; disabled with ``secrets.profile_alias: false``.
  326. 1. 2. 3. 4. See #58073.
  327. See #51447.
  328. """
  329. env = environ if environ is not None else os.environ
  330. report = ApplyReport()
  331. secrets_cfg = secrets_cfg if isinstance(secrets_cfg, dict) else {}
  332. enabled = _ordered_enabled_sources(secrets_cfg, scope=hermes_home_key(home_path))
  333. if not enabled:
  334. return report
  335. preserve_raw = secrets_cfg.get("preserve_existing")
  336. preserve = frozenset(n.strip() for n in preserve_raw if isinstance(n, str) and n.strip()
  337. ) if isinstance(preserve_raw, list) else frozenset()
  338. profile = _active_profile_name(home_path) if secrets_cfg.get("profile_alias", True) else ""
  339. # Mapped outranks bulk regardless of list order.
  340. ordered = [s for s in enabled if s.shape == "mapped"] + [s for s in enabled if s.shape == "bulk"]
  341. fetches: list[tuple[SecretSource, dict, FetchResult]] = []
  342. protected: dict[str, str] = {} # var → source that protects it
  343. for source in ordered:
  344. cfg = _section(secrets_cfg, source.name)
  345. result = _fetch_with_timeout(source, cfg, home_path, env)
  346. fetches.append((source, cfg, result))
  347. try:
  348. for var in source.protected_env_vars(cfg):
  349. protected.setdefault(var, source.name)
  350. except Exception:
  351. pass
  352. # An alias never shadows a var some source supplies by its real name.
  353. supplied_directly = {v for _, _, r in fetches if r.ok for v in r.secrets if isinstance(v, str)}
  354. applier = _Applier(env, report, protected, preserve)
  355. for source, cfg, result in fetches:
  356. applier.apply_source(source, cfg, result, profile, supplied_directly)
  357. return report