env_loader.py 59 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121
  1. """Helpers for loading Hermes .env files consistently across entrypoints."""
  2. from __future__ import annotations
  3. import codecs
  4. import io
  5. import itertools
  6. import logging
  7. import os
  8. import sys
  9. import threading
  10. from pathlib import Path
  11. # Kept at module level on purpose: importing this module must fail when the dotenv install is
  12. # wiped (#57828) so early recovery provably runs before third-party imports (test_early_recovery).
  13. # The parser internals are imported lazily below because gateway tests stub ``sys.modules["dotenv"]``.
  14. import dotenv
  15. from hermes_cli.left_core_migration import profile_env_names
  16. from utils import atomic_replace, mkstemp_beside
  17. logger = logging.getLogger(__name__)
  18. # The ONLY env vars sanitized on load: credentials must be pure ASCII (they become HTTP header values);
  19. # arbitrary user env vars are never silently altered.
  20. _CREDENTIAL_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET", "_KEY")
  21. # Once-per-process guards: load_hermes_dotenv() runs repeatedly (user + project env, gateway hot-reload,
  22. # lazy imports mid-turn, tests) so warnings/logs fire once per key/path/home.
  23. _WARNED_KEYS: set[str] = set() # credential names already given the non-ASCII warning
  24. _WARNED_UTF32_PATHS: set[str] = set() # .env paths already given the UTF-32 refuse-to-mangle warning
  25. _SCOPED_SKIP_LOGGED: set[str] = set() # routed profile homes whose multiplex dotenv skip was logged
  26. _WARNED_UNREADABLE_PATHS: set[str] = set() # unreadable profile .env paths already given the warning
  27. # env-var name → source label ("bitwarden", …) for externally injected credentials; setup / `hermes
  28. # model` tell users WHERE a key came from when .env lacks it.
  29. _SECRET_SOURCES: dict[str, str] = {}
  30. # Every env-var name an external source SUPPLIED for some home, whether it was applied or lost to a
  31. # pre-existing process value (``skipped_existing``). ``_SECRET_SOURCES`` is provenance metadata and only
  32. # names applied values; the scrub that keeps a launch profile's source-supplied names out of a routed
  33. # child must see the skipped ones too, or a name already in the process env leaks with the launch value.
  34. _SOURCE_SUPPLIED_NAMES: set[str] = set()
  35. # Every KEY name a dotenv file loaded into ``os.environ`` during this process's lifetime. A key removed
  36. # or renamed in the launch ``.env`` after boot stays in ``os.environ`` (dotenv never unsets), but a
  37. # re-parse of the current file no longer names it — so the launch-residue strip for a routed child must
  38. # work from what was LOADED, not from what the file says now. Additive for the process lifetime.
  39. _LOADED_DOTENV_KEYS: set[str] = set()
  40. # KEY names loaded from the administrator-managed ``.env`` (``_apply_managed_env``). Kept OUT of the launch
  41. # residue: those values are policy that beats the user's own ``.env`` for every profile, so a routed child
  42. # must keep them — and keep them LAST, over the routed profile's scope (review on f5f88d5058).
  43. _MANAGED_DOTENV_KEYS: set[str] = set()
  44. # Immutable per-home snapshots: os.environ is shared across profiles and a later home's apply may overwrite it.
  45. _SECRET_SOURCE_VALUES_BY_HOME: dict[str, dict[str, str]] = {}
  46. # Per home: the subset of the snapshot a dotenv reload may re-assert — see ``AppliedVar.authoritative`` (#74265).
  47. _SECRET_SOURCE_RESTORE_BY_HOME: dict[str, dict[str, str]] = {}
  48. # Per home: what the process-global path WROTE into ``os.environ`` for an external source — name →
  49. # (source name, value written, value the name held before the first write or None). The per-home
  50. # snapshot above is republished on every pass, but ``os.environ`` is not: without this record a value a
  51. # since-removed or disabled source injected stayed in the process env, and single-profile
  52. # ``get_secret()`` kept serving it through its ``os.environ`` fallback until restart. Deliberately NOT
  53. # cleared by a per-home ``reset_secret_source_cache``: it describes ``os.environ``, which a reset does not
  54. # touch, and the next pass needs it to revoke.
  55. _SECRET_SOURCE_WRITES_BY_HOME: dict[str, dict[str, tuple[str, str, str | None]]] = {}
  56. # HERMES_HOME paths already pulled external secrets for: load_hermes_dotenv() runs at import time from
  57. # several hot modules, so without this the Bitwarden status line prints 3-5x per startup and the config
  58. # re-parse + ASCII sweep re-run each time (Bitwarden's own cache only saves the network call).
  59. _APPLIED_HOMES: set[str] = set()
  60. _SECRET_SOURCE_CACHE_LOCK = threading.RLock()
  61. # What THIS process has published from dotenv files, per variable: (value before our first publish, or
  62. # None if absent; last value we published; load pass that published it). Reloads run per gateway turn and
  63. # per cron fire, and a line like ``PATH=/x:${PATH}`` interpolated against an environ that already holds the
  64. # previous reload's output grows by ``/x:`` every time until child spawns fail with E2BIG (#109902). A new
  65. # pass resolves against the baseline instead — but only where the environ still holds exactly what we
  66. # published, so a value the shell, config bridge, or an external secret source changed since is not frozen.
  67. # One process-wide record (not per home/project scope): the environ is process-wide, so alternating
  68. # callers (gateway with a project .env, cron without; home A then B) must peel each other's output too.
  69. _DOTENV_PUBLISHED: dict[str, tuple[str | None, str, int]] = {}
  70. _DOTENV_PASSES = itertools.count()
  71. _DOTENV_LOCK = threading.RLock()
  72. # Per-process credentials a parent mints and injects into the child's environment (the Desktop shell /
  73. # a link-style launcher spawns `hermes dashboard` with a fresh HERMES_DASHBOARD_SESSION_TOKEN and keeps
  74. # the same token for its own /api probes). They are never .env configuration, so a persisted value in
  75. # ~/.hermes/.env must not replace an injected one — the parent would then 401 against its own child
  76. # (#115955). A value an earlier dotenv pass published is still reloaded normally.
  77. _SPAWN_CREDENTIAL_KEYS: frozenset[str] = frozenset({"HERMES_DASHBOARD_SESSION_TOKEN"})
  78. # Behavioral routing keys a parent Hermes process injects into child env that silently redirect a profile
  79. # onto the wrong provider path; these — and ONLY these — are scrubbed at startup when absent from the
  80. # profile's .env. Credentials are excluded: shell exports are a documented way to supply them, and
  81. # read-time secret-scope checks (agent/secret_scope.py) own cross-profile credential isolation.
  82. # Providers that left core keep theirs listed on their ``LEFT_CORE`` row (``profile_env``), read statically:
  83. # this runs before provider discovery, and with the plugin absent nothing else names them.
  84. _PROFILE_MANAGED_ENV_KEYS: frozenset[str] = frozenset({
  85. "HERMES_ACP_AUTH_METHOD", "HERMES_ACP_AUTO_APPROVE",
  86. }) | profile_env_names()
  87. def _env_keys_defined_in_dotenv(path: Path) -> set[str]:
  88. """KEY names assigned in a dotenv file (including empty ``KEY=``), via the same tokenizer that installs
  89. profile scopes — a key the installer sees is a key the dashboard scrub sees (BOM'd first line included)."""
  90. from agent.secret_scope import load_env_file
  91. return set(load_env_file(path))
  92. def _clear_known_keys_missing_from_dotenv(path: Path) -> None:
  93. """After ``.env`` loaded with override, delete inherited ``_PROFILE_MANAGED_ENV_KEYS`` it does not
  94. define. Deliberately NARROW: only keys that change *which provider path* is used.
  95. Does **not** run when the ``.env`` file does not exist (bare-profile case, which follows ``#66930`` /
  96. ``#67027`` semantics).
  97. """
  98. if not path.exists():
  99. return
  100. defined = _env_keys_defined_in_dotenv(path)
  101. for key in _PROFILE_MANAGED_ENV_KEYS:
  102. if key not in defined and key in os.environ:
  103. del os.environ[key]
  104. def get_secret_source(env_var: str) -> str | None:
  105. """Source label that supplied ``env_var`` (``"bitwarden"`` …), None for .env/shell keys. Metadata only —
  106. never authorization to persist the raw value."""
  107. return _SECRET_SOURCES.get(env_var)
  108. def _record_supplied_names(report) -> set[str]:
  109. """Every name *report*'s sources supplied — applied, or skipped because a value already existed —
  110. recorded into ``_SOURCE_SUPPLIED_NAMES`` so the routed-child scrub knows the source owns it."""
  111. supplied = set(report.provenance)
  112. for src in report.sources:
  113. supplied.update(src.skipped_existing)
  114. _SOURCE_SUPPLIED_NAMES.update(supplied)
  115. return supplied
  116. def secret_source_names() -> tuple[str, ...]:
  117. """Every env-var name some profile's external secret source APPLIED (names only — the map is
  118. process-wide, so a value must be resolved through the active profile's secret scope). Consumers that
  119. forward source values into a child (MCP stdio env) want exactly these; see ``source_supplied_names``
  120. for the wider set the routed-child scrub needs."""
  121. return tuple(_SECRET_SOURCES)
  122. def source_supplied_names() -> tuple[str, ...]:
  123. """Every env-var name an external source supplied for any home — applied, or lost to a pre-existing
  124. process value (``skipped_existing``). The launch value in ``os.environ`` is still not a routed
  125. profile's to inherit, so the strip must see the skipped names too."""
  126. return tuple(sorted(set(_SECRET_SOURCES) | _SOURCE_SUPPLIED_NAMES))
  127. def launch_dotenv_keys() -> frozenset[str]:
  128. """KEY names any NON-managed dotenv file loaded into this process's ``os.environ`` so far (see
  129. ``_LOADED_DOTENV_KEYS``); the launch profile's residue set for routed children."""
  130. return frozenset(_LOADED_DOTENV_KEYS)
  131. def dotenv_published_values(name: str) -> frozenset[str]:
  132. """The values dotenv loads in this process last wrote to ``name``, its name matched case-insensitively:
  133. on Windows ``https_proxy`` in a .env and an inherited ``HTTPS_PROXY`` are one environment variable."""
  134. folded = name.upper()
  135. with _DOTENV_LOCK:
  136. return frozenset(value for key, (_base, value, _pass) in _DOTENV_PUBLISHED.items() if key.upper() == folded)
  137. def managed_dotenv_keys() -> frozenset[str]:
  138. """KEY names the administrator-managed ``.env`` loaded (see ``_MANAGED_DOTENV_KEYS``). Policy for
  139. every profile: never stripped from a routed child, and re-applied over the routed scope."""
  140. return frozenset(_MANAGED_DOTENV_KEYS)
  141. def get_secret_source_values(hermes_home: str | os.PathLike) -> dict[str, str]:
  142. """Return the external-secret value snapshot for ``hermes_home``."""
  143. return dict(_SECRET_SOURCE_VALUES_BY_HOME.get(str(Path(hermes_home).resolve()), {}))
  144. def hydrate_profile_secret_sources(hermes_home: str | os.PathLike) -> dict[str, str]:
  145. """Resolve one profile's configured sources without mutating ``os.environ``: multiplex gateways route
  146. turns to profiles that never ran the process-global dotenv path, so resolve against a private mapping
  147. seeded from that ``.env`` and record the per-home snapshot for ``build_profile_secret_scope()``.
  148. Fail-open / once-per-home like ``_apply_external_secret_sources``; never returns plaintext .env entries.
  149. The one fail-closed case is D32's: under the remote config backend, a source that supplies a
  150. protected name raises :class:`ConfigBackendUnavailable` and records nothing for the home."""
  151. with _SECRET_SOURCE_CACHE_LOCK:
  152. return _hydrate_profile_secret_sources(Path(hermes_home))
  153. def _hydrate_profile_secret_sources(home: Path) -> dict[str, str]:
  154. """Locked implementation for :func:`hydrate_profile_secret_sources`."""
  155. from agent.safe_worker_policy import worker_config_snapshot
  156. if worker_config_snapshot() is not None:
  157. return {}
  158. home_key = str(home.resolve())
  159. if home_key in _APPLIED_HOMES:
  160. return get_secret_source_values(home)
  161. # A retry must not keep serving a partial result after the source is removed, disabled, or can no
  162. # longer be evaluated. Publish only the snapshot established by this attempt.
  163. _SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
  164. _SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
  165. try:
  166. cfg = _load_secrets_config(home)
  167. except Exception:
  168. return {}
  169. if not cfg:
  170. return {}
  171. try:
  172. from agent.secret_scope import _is_global_env, load_env_file
  173. from agent.secret_sources.registry import apply_all
  174. local_env = {name: value for name, value in os.environ.items() if _is_global_env(name)}
  175. local_env.update(load_env_file(home / ".env"))
  176. # Mirror load_hermes_dotenv()'s .op.env bootstrap (1Password token lives in gitignored .op.env)
  177. # or cold profiles fail 1Password hydration. .env wins.
  178. # Without seeding it here a cold profile configured for the supported .op.env flow fails 1Password
  179. # hydration (sweeper review on #74549). .env values win — never override an existing key.
  180. op_env = home / ".op.env"
  181. if op_env.exists():
  182. for _name, _value in load_env_file(op_env).items():
  183. local_env.setdefault(_name, _value)
  184. local_env["HERMES_HOME"] = str(home)
  185. env_before = dict(local_env)
  186. report = apply_all(cfg, home, environ=local_env)
  187. except Exception:
  188. return {}
  189. if not report.sources:
  190. return {}
  191. # D32, as on the process-global path: a source must never supply the config backend's own
  192. # deployment or plane credential, for a routed profile either (this snapshot feeds that
  193. # profile's secret scope, which the Portal/plane resolvers read). Checked before anything of
  194. # this attempt is recorded: the remote backend refuses the profile (raises, nothing cached, so
  195. # every retry refuses again until the mapping is removed); the file backend drops the selector.
  196. from hermes_cli.config_backend import get_config_backend
  197. _refuse_protected_env_from_sources(report, get_config_backend(), env_before, env=local_env, routed_home=home)
  198. # Routed profiles have no runtime reset path. Keep a failed source retryable so correcting its
  199. # profile-local bootstrap credentials takes effect on the next turn; successful sources from a
  200. # mixed report are still snapshotted below and can be used while the failed source recovers.
  201. if all(src.result.ok for src in report.sources):
  202. _APPLIED_HOMES.add(home_key)
  203. # Same ownership bookkeeping as the process-global path: a name this profile's source supplied — applied,
  204. # or skipped because the private mapping already had it — is a source-owned name the routed-child scrub
  205. # must know about, or a sibling still inherits the launch value for it (review on f5f88d5058).
  206. _record_supplied_names(report)
  207. values: dict[str, str] = {}
  208. for name, applied in report.provenance.items():
  209. value = local_env.get(name)
  210. if value is None:
  211. continue
  212. _SECRET_SOURCES[name] = applied.source
  213. values[name] = value
  214. _SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
  215. return dict(values)
  216. def reset_secret_source_cache(hermes_home: str | os.PathLike | None = None) -> None:
  217. """Forget applied homes so the next load re-pulls (tests, long-running processes after config edits).
  218. ``hermes_home`` limits the reset to ONE home: a multiplex gateway keeps every profile's snapshot in
  219. this process, and a per-fire cron re-pull or a plugin-discovery refresh for one home must not wipe
  220. a sibling's hydrated snapshot — the sibling's next scope build would run empty until it re-hydrated
  221. (#102041)."""
  222. if hermes_home is None:
  223. _APPLIED_HOMES.clear()
  224. _SECRET_SOURCES.clear()
  225. _SOURCE_SUPPLIED_NAMES.clear()
  226. _SECRET_SOURCE_VALUES_BY_HOME.clear()
  227. _SECRET_SOURCE_RESTORE_BY_HOME.clear()
  228. _SECRET_SOURCE_WRITES_BY_HOME.clear()
  229. return
  230. home_key = str(Path(hermes_home).resolve())
  231. _APPLIED_HOMES.discard(home_key)
  232. _SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
  233. _SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
  234. def format_secret_source_suffix(env_var: str) -> str:
  235. """``" (from Bitwarden)"``-style suffix; ``""`` for .env/shell keys (only external sources are named)."""
  236. source = get_secret_source(env_var)
  237. if not source:
  238. return ""
  239. if source == "bitwarden":
  240. return " (from Bitwarden)"
  241. # Registry label (e.g. "1Password"); raw name for unknown sources (uninstalled plugin, tests).
  242. try:
  243. from agent.secret_sources.registry import get_source
  244. registered = get_source(source)
  245. if registered is not None and registered.label:
  246. return f" (from {registered.label})"
  247. except Exception:
  248. pass
  249. return f" (from {source})"
  250. def _format_offending_chars(value: str, limit: int = 3) -> str:
  251. """Compact ``U+XXXX ('c'), ...`` summary of non-ASCII codepoints."""
  252. seen: list[str] = []
  253. for ch in value:
  254. if ord(ch) > 127:
  255. label = f"U+{ord(ch):04X}"
  256. if ch.isprintable():
  257. label += f" ({ch!r})"
  258. if label not in seen:
  259. seen.append(label)
  260. if len(seen) >= limit:
  261. break
  262. return ", ".join(seen)
  263. def _sanitize_loaded_credentials() -> None:
  264. """Strip non-ASCII from credential env vars (``_CREDENTIAL_SUFFIXES``) so the codebase never sees them.
  265. Emits a one-line warning to stderr when characters are stripped. Silent stripping would mask copy-paste
  266. corruption (Unicode lookalike glyphs from PDFs / rich-text editors, ZWSP from web pages) as opaque
  267. provider-side "invalid API key" errors (see #6843).
  268. """
  269. for key, value in list(os.environ.items()):
  270. if not any(key.endswith(suffix) for suffix in _CREDENTIAL_SUFFIXES):
  271. continue
  272. if value.isascii():
  273. continue
  274. cleaned = value.encode("ascii", errors="ignore").decode("ascii")
  275. os.environ[key] = cleaned
  276. if key in _WARNED_KEYS:
  277. continue
  278. _WARNED_KEYS.add(key)
  279. stripped = len(value) - len(cleaned)
  280. detail = _format_offending_chars(value) or "non-printable"
  281. print(f" Warning: {key} contained {stripped} non-ASCII character"
  282. f"{'s' if stripped != 1 else ''} ({detail}) — stripped so the "
  283. f"key can be sent as an HTTP header.", file=sys.stderr)
  284. print(
  285. " This usually means the key was copy-pasted from a PDF, "
  286. "rich-text editor, or web page that substituted lookalike\n"
  287. " Unicode glyphs for ASCII letters. If authentication fails "
  288. "(e.g. \"API key not valid\"), re-copy the key from the\n"
  289. " provider's dashboard and run `hermes setup` (or edit the "
  290. ".env file in a plain-text editor).",
  291. file=sys.stderr,
  292. )
  293. def _parse_dotenv_file(path: Path) -> list[tuple[str, str | None]]:
  294. """``(name, raw value)`` assignments of one dotenv file, uninterpolated, in file order."""
  295. raw = path.read_bytes()
  296. try:
  297. # utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
  298. # first key name and silently drop it from os.environ under its canonical name.
  299. text = raw.decode("utf-8-sig")
  300. except UnicodeDecodeError:
  301. raw = raw.removeprefix(codecs.BOM_UTF8)
  302. text = raw.decode("latin-1")
  303. # Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
  304. # exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
  305. from dotenv.main import DotEnv
  306. return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
  307. def _dotenv_lookup_env(load_pass: int | None) -> dict[str, str]:
  308. """``os.environ`` with every value an earlier pass (any pass, for ``None``) published peeled back to
  309. its baseline — what ``${VAR}`` resolves against (see ``_DOTENV_PUBLISHED``). Caller holds the lock."""
  310. lookup_env = dict(os.environ)
  311. for name, (baseline, published, published_pass) in _DOTENV_PUBLISHED.items():
  312. if published_pass != load_pass and lookup_env.get(name) == published:
  313. if baseline is None:
  314. del lookup_env[name] # absent, so ``${VAR:-default}`` takes the default again
  315. else:
  316. lookup_env[name] = baseline
  317. return lookup_env
  318. def _resolve_dotenv_assignments(
  319. assignments: list[tuple[str, str | None]], lookup_env: dict[str, str], *, override: bool,
  320. ) -> dict[str, str | None]:
  321. """Expand ``${VAR}`` / ``${VAR:-default}`` like ``dotenv.main.resolve_variables``, minus the live
  322. ``os.environ``: a line sees *lookup_env* and the file's earlier lines (earlier lines win under
  323. *override*, the environment otherwise)."""
  324. from dotenv.variables import parse_variables
  325. resolved: dict[str, str | None] = {}
  326. for name, value in assignments:
  327. if value is not None:
  328. lookup = {**lookup_env, **resolved} if override else {**resolved, **lookup_env}
  329. value = "".join(atom.resolve(lookup) for atom in parse_variables(value))
  330. resolved[name] = value
  331. return resolved
  332. def resolve_home_dotenv(home: Path) -> dict[str, str]:
  333. """What ``load_hermes_dotenv`` would publish from *home*'s dotenv layers, read-only: the home's
  334. ``.env`` then the administrator-managed ``.env``, both with override and ``${VAR}`` expanded by the
  335. same rules against the process's own environment (this process's dotenv output peeled back), each
  336. layer seeing the one before. External secret sources and the project ``.env`` are not included."""
  337. from hermes_cli import managed_scope
  338. managed_dir = managed_scope.get_managed_dir()
  339. layers = [home / ".env", *([managed_dir / ".env"] if managed_dir is not None else [])]
  340. with _DOTENV_LOCK:
  341. lookup_env = _dotenv_lookup_env(None)
  342. out: dict[str, str] = {}
  343. for path in layers:
  344. try:
  345. assignments = _parse_dotenv_file(path)
  346. except OSError:
  347. continue
  348. for name, value in _resolve_dotenv_assignments(assignments, lookup_env, override=True).items():
  349. if value is not None:
  350. lookup_env[name] = out[name] = value
  351. return out
  352. def _load_dotenv_with_fallback(
  353. path: Path, *, override: bool, load_pass: int | None = None, managed: bool = False,
  354. ) -> None:
  355. """Load one dotenv file into ``os.environ`` like ``dotenv.load_dotenv`` — same parser, same
  356. ``${VAR}`` / ``${VAR:-default}`` / precedence rules — except that ``${VAR}`` resolves against the value
  357. VAR had before this process's earlier passes published it (see ``_DOTENV_PUBLISHED``).
  358. ``load_pass`` groups the layered files of one ``load_hermes_dotenv`` call: within a pass a later layer
  359. (project, managed) still sees the earlier layer's output, as it always did; only OTHER passes' output
  360. is peeled. A bare call (``hermes send``'s direct reload) is its own pass."""
  361. assignments = _parse_dotenv_file(path)
  362. with _DOTENV_LOCK:
  363. if load_pass is None:
  364. load_pass = next(_DOTENV_PASSES)
  365. resolved = _resolve_dotenv_assignments(assignments, _dotenv_lookup_env(load_pass), override=override)
  366. for name, value in resolved.items():
  367. if value is None or (not override and name in os.environ):
  368. continue
  369. current = os.environ.get(name)
  370. record = _DOTENV_PUBLISHED.get(name)
  371. ours = record is not None and current == record[1]
  372. if name in _SPAWN_CREDENTIAL_KEYS and current and not ours:
  373. continue # parent-minted per-process credential: .env must not split it from the parent
  374. # Ours and untouched since → keep the original baseline; anything else is a newer outside value.
  375. baseline = record[0] if ours else current
  376. os.environ[name] = value
  377. _DOTENV_PUBLISHED[name] = (baseline, value, load_pass)
  378. # Every key this file defines, for the launch-residue strip: dotenv never unsets, so a key later
  379. # removed from the launch .env stays in os.environ and a re-parse of the file no longer names it.
  380. # Managed keys are recorded separately: they are administrator policy, not launch-profile residue.
  381. (_MANAGED_DOTENV_KEYS if managed else _LOADED_DOTENV_KEYS).update(name for name, _value in assignments)
  382. _sanitize_loaded_credentials() # httpx encodes headers as ASCII
  383. def _dotenv_assignments(path: Path) -> list:
  384. """``(name, raw value)`` pairs of one dotenv file as the loader will see it once
  385. ``_sanitize_env_file_if_needed`` has run: :func:`_parse_dotenv_file` plus the sanitizer's UTF-16
  386. rewrite and NUL strip, for the pre-boot bootstrap reads that run before the sanitizer."""
  387. raw = path.read_bytes()
  388. if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
  389. # Notepad "Unicode": ``_sanitize_env_file_if_needed`` rewrites it as UTF-8 before the loader
  390. # reads it, so parse it as that rewrite will -- bootstrap / stage2 run before the sanitizer
  391. # and must select the same backend the runtime then boots.
  392. try:
  393. with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
  394. text = f.read()
  395. except UnicodeDecodeError:
  396. text = raw.decode("latin-1")
  397. else:
  398. try:
  399. # utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
  400. # first key name and silently drop it from os.environ under its canonical name.
  401. text = raw.decode("utf-8-sig")
  402. except UnicodeDecodeError:
  403. raw = raw.removeprefix(codecs.BOM_UTF8)
  404. text = raw.decode("latin-1")
  405. text = text.replace("\x00", "") # the sanitizer strips NULs too (BOM-less UTF-16 / NUL-padded ASCII)
  406. # Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
  407. # exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
  408. from dotenv.main import DotEnv
  409. return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
  410. def _bootstrap_env(home: Path | None = None, project_env: Path | None = None) -> dict:
  411. """The process env as ``load_hermes_dotenv`` leaves it for the names its dotenv layers define:
  412. the home's ``.env`` (override), ``.op.env`` (fills gaps, only without a process
  413. ``OP_SERVICE_ACCOUNT_TOKEN``), the caller's *project_env* (fills gaps when the home has a
  414. ``.env``, else overrides) and the managed ``.env`` (override, last); ``${VAR}`` resolved, the
  415. loader's parser. Read-only: nothing is published and no file is rewritten (sanitizing imports
  416. hermes_cli.config, whose import-time config read must not run before the backend is decided)."""
  417. home = Path(home) if home is not None else _process_hermes_home()
  418. env = dict(os.environ)
  419. user_env = home / ".env"
  420. layers: list[tuple[Path, bool]] = [(user_env, True)]
  421. if not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
  422. layers.append((home / ".op.env", False))
  423. if project_env is not None:
  424. layers.append((Path(project_env), not user_env.exists()))
  425. try:
  426. from hermes_cli import managed_scope
  427. managed_dir = managed_scope.get_managed_dir()
  428. except Exception: # health: allow BLE001 -- managed scope never blocks startup, as in _apply_managed_env
  429. managed_dir = None
  430. if managed_dir is not None:
  431. layers.append((managed_dir / ".env", True))
  432. for path, override in layers:
  433. try:
  434. assignments = _dotenv_assignments(path)
  435. except OSError:
  436. continue
  437. resolved = _resolve_dotenv_assignments(assignments, env, override=override)
  438. for name, value in resolved.items():
  439. if value is not None and (override or name not in env):
  440. env[name] = value
  441. return env
  442. def selected_config_backend_name(home: Path | None = None) -> str:
  443. """The config backend ``load_hermes_dotenv`` selects for *home*, decided without loading or
  444. booting anything. For a caller that must decide before Hermes starts (Docker stage2 skips the
  445. config-file migration in remote mode, D12)."""
  446. from hermes_cli.config_backend import BACKEND_ENV
  447. return (_bootstrap_env(home).get(BACKEND_ENV) or "").strip().lower() or "file"
  448. def apply_config_bootstrap_env(backend_names, home: Path | None = None, project_env: Path | None = None) -> None:
  449. """Publish ``backend_names(kind)`` (the selected backend's deployment and credential, D26/D32) from the
  450. launch home's ``.env`` and the managed ``.env``, with the values ``load_hermes_dotenv`` gives
  451. them. Called before the first config read of a process, which can come before
  452. ``load_hermes_dotenv`` (``hermes_cli.config`` reads config at import time): the backend is
  453. selected and authenticated from the same deployment either way. A file deployment gets only its
  454. selector (the file backend needs nothing else before boot); values the process env holds and
  455. no file defines are left alone; ``load_hermes_dotenv`` later publishes the same values."""
  456. from agent.safe_worker_policy import worker_config_snapshot
  457. from hermes_cli.config_backend import BACKEND_ENV
  458. # A frozen-policy worker reads no profile dotenv (``load_hermes_dotenv`` returns early there):
  459. # its bootstrap supplied the process env explicitly, so nothing is published from the files.
  460. if worker_config_snapshot() is not None:
  461. return
  462. env = _bootstrap_env(home, project_env)
  463. # The selector itself always, so a file deployment is not read as another kind meanwhile.
  464. names = {BACKEND_ENV} | set(backend_names((env.get(BACKEND_ENV) or "").strip().lower() or "file"))
  465. for name in names:
  466. value = env.get(name)
  467. if value is not None and os.environ.get(name) != value:
  468. os.environ[name] = value
  469. def _sanitize_env_file_if_needed(path: Path) -> None:
  470. """Pre-sanitize a .env file before python-dotenv reads it. Sniffs a leading BOM *before* any text
  471. decode: UTF-16 (Notepad "Unicode") is rewritten as clean UTF-8; UTF-32 is refused (left untouched) so
  472. we never fall through to the errors=replace corruption path."""
  473. if not path.exists():
  474. return
  475. try:
  476. from hermes_cli.config import _sanitize_env_lines
  477. except ImportError:
  478. return # early bootstrap — config module not available yet
  479. try:
  480. raw = path.read_bytes()
  481. except Exception:
  482. return
  483. # ORDER MATTERS: BOM_UTF32_LE (FF FE 00 00) startswith BOM_UTF16_LE (FF FE); UTF-16 first would mangle it.
  484. force_utf8_rewrite = False
  485. if raw.startswith(codecs.BOM_UTF32_LE) or raw.startswith(codecs.BOM_UTF32_BE):
  486. # Lazy import keeps the module import block identical to #65124's codecs/io additions so the two PRs
  487. # auto-merge either order.
  488. path_key = str(path.resolve())
  489. if path_key not in _WARNED_UTF32_PATHS:
  490. _WARNED_UTF32_PATHS.add(path_key)
  491. logger.warning("Skipping .env sanitize for %s: UTF-32 BOM detected; "
  492. "leaving file untouched to avoid corruption", path)
  493. return
  494. if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
  495. # "utf-16" uses the BOM for endianness and strips it; newline=None matches open()'s universal
  496. # newlines (not splitlines()'s extra boundaries like U+2028) so sanitize sees the same lines.
  497. try:
  498. with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
  499. original = f.readlines()
  500. except UnicodeDecodeError:
  501. return
  502. force_utf8_rewrite = True # always rewrite UTF-16 as UTF-8 so the dotenv load sees a canonical file
  503. else:
  504. # utf-8-sig strips a UTF-8 BOM; errors=replace so embedded NULs can be stripped below.
  505. try:
  506. with open(path, encoding="utf-8-sig", errors="replace") as f:
  507. original = f.readlines()
  508. except Exception:
  509. return
  510. # errors=replace turns undecodable leading bytes into U+FFFD; persisting would glue them onto
  511. # the first key name permanently — leave the file untouched instead.
  512. if original and original[0].startswith("\ufffd"):
  513. return
  514. try:
  515. # Strip NULs (os.environ raises ValueError on them); also repairs BOM-less UTF-16 (NUL-padded ASCII).
  516. stripped = [line.replace("\x00", "") for line in original]
  517. sanitized = _sanitize_env_lines(stripped)
  518. if sanitized != original or force_utf8_rewrite:
  519. fd, tmp = mkstemp_beside(path, suffix=".tmp", prefix=".env_")
  520. try:
  521. with os.fdopen(fd, "w", encoding="utf-8") as f:
  522. f.writelines(sanitized)
  523. f.flush()
  524. os.fsync(f.fileno())
  525. atomic_replace(tmp, path)
  526. except BaseException:
  527. try:
  528. os.unlink(tmp)
  529. except OSError:
  530. pass
  531. raise
  532. except Exception:
  533. pass # best-effort — don't block gateway startup
  534. def dotenv_unreadable(path: Path) -> bool:
  535. """True when *path* exists but this process may not read it."""
  536. try:
  537. with open(path, "rb"):
  538. return False
  539. except PermissionError:
  540. return True
  541. except OSError: # missing, or a failure that is not a refusal
  542. return False
  543. def _load_user_dotenv(path: Path, *, load_pass: int) -> bool:
  544. """Load the profile ``.env``; False when it exists but cannot be read and managed scope pins
  545. ``terminal.backend``. A sandbox launcher withholds the file, passes the keys it allows in as
  546. environment and pins the backend. Without that pin the ``PermissionError`` propagates: the file may
  547. be the only place the backend is set, and skipping it would run terminal commands on the host."""
  548. try:
  549. _load_dotenv_with_fallback(path, override=True, load_pass=load_pass)
  550. except PermissionError:
  551. from hermes_cli import managed_scope
  552. if managed_scope.pinned_terminal_backend() is None:
  553. raise
  554. if str(path) not in _WARNED_UNREADABLE_PATHS:
  555. _WARNED_UNREADABLE_PATHS.add(str(path))
  556. logger.warning("Cannot read %s; managed scope pins terminal.backend, so it loads as absent", path)
  557. return False
  558. return True
  559. def load_hermes_dotenv(
  560. *,
  561. hermes_home: str | os.PathLike | None = None,
  562. project_env: str | os.PathLike | None = None,
  563. load_external_secrets: bool = True,
  564. ) -> list[Path]:
  565. """Load Hermes env files: ``~/.hermes/.env`` overrides stale shell exports; project ``.env`` is a dev
  566. fallback that only fills gaps when the user env exists (and overrides shell vars when it does not)."""
  567. from agent.safe_worker_policy import worker_config_snapshot
  568. # The private bootstrap supplies credentials explicitly; do not rehydrate
  569. # profile behavior, external secret plugins, or managed env in this worker.
  570. if worker_config_snapshot() is not None:
  571. return []
  572. # Process home on purpose (never the per-turn override): a startup .env load must not follow a routed
  573. # profile — see the multiplex guard below.
  574. from hermes_constants import get_process_hermes_home
  575. home_path = Path(hermes_home) if hermes_home else get_process_hermes_home()
  576. # Multiplex gateway: while a routed profile-home override is active, copying that profile's .env
  577. # into os.environ would expose its credentials to sibling turns and every spawned child. The launch
  578. # home's own .env is process configuration and still loads: the launch profile's scoped bodies bind
  579. # an override naming the launch home too, and skipping it hid launch-only credentials such as a
  580. # fallback_providers key from the process env (#125530). Both the load's target AND the active
  581. # home must be the launch home: a launch-targeted load inside a FOREIGN turn re-bridges terminal.*
  582. # from the config the override resolves to, i.e. the routed profile's cwd into the shared env.
  583. # External sources still refresh against the profile mapping.
  584. # (``is_multiplex_active()`` is also true, context-locally, for a routed cron fire in the desktop
  585. # backend — see ``cron.scheduler_provider.routed_profile_fire``.)
  586. from agent.secret_scope import is_multiplex_active
  587. from hermes_constants import get_hermes_home, get_hermes_home_override
  588. launch_home = _process_hermes_home().resolve()
  589. if (is_multiplex_active() and get_hermes_home_override() is not None
  590. and (home_path.resolve() != launch_home or get_hermes_home().resolve() != launch_home)):
  591. home_key = str(home_path.resolve())
  592. if home_key not in _SCOPED_SKIP_LOGGED:
  593. _SCOPED_SKIP_LOGGED.add(home_key)
  594. logger.debug("multiplex: skipping process-global dotenv load for routed "
  595. "profile home %s (credentials resolve via the profile scope)", home_path)
  596. if load_external_secrets:
  597. from hermes_cli import _early_recovery
  598. if not _early_recovery._should_skip_external_secret_sources():
  599. hydrate_profile_secret_sources(home_path)
  600. return []
  601. loaded: list[Path] = []
  602. user_env = home_path / ".env"
  603. project_env_path = Path(project_env) if project_env else None
  604. load_pass = next(_DOTENV_PASSES) # one pass: later layers below see the earlier layers' output
  605. # Before anything below imports hermes_cli.config (the .env sanitizer does), whose import-time
  606. # config read selects and boots the backend: publish the deployment and plane credential from
  607. # THIS load's layers, the project .env included, with the precedence the loads below give them.
  608. from hermes_cli.config_backend import bootstrap_deployment
  609. bootstrap_deployment(home_path, project_env_path)
  610. if user_env.exists(): # normalize formatting / strip NULs before parsing
  611. _sanitize_env_file_if_needed(user_env)
  612. if project_env_path and project_env_path.exists():
  613. _sanitize_env_file_if_needed(project_env_path)
  614. if user_env.exists() and _load_user_dotenv(user_env, load_pass=load_pass):
  615. loaded.append(user_env)
  616. _clear_known_keys_missing_from_dotenv(user_env) # mirrors reload_env(): inherited keys must not leak
  617. # .op.env AFTER .env so .env wins, but the bootstrap OP_SERVICE_ACCOUNT_TOKEN reaches
  618. # apply_onepassword_secrets() even in cron with no shell state; gitignored so the token never enters
  619. # the committed .env. override=False lets a systemd `EnvironmentFile=-…/.op.env` token win.
  620. op_env = home_path / ".op.env"
  621. if op_env.exists() and not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
  622. _load_dotenv_with_fallback(op_env, override=False, load_pass=load_pass)
  623. if project_env_path and project_env_path.exists():
  624. _load_dotenv_with_fallback(project_env_path, override=not user_env.exists(), load_pass=load_pass)
  625. loaded.append(project_env_path)
  626. # The override=True loads above wrote the raw .env line (``__BITWARDEN_MANAGED__`` placeholder, stale
  627. # token) back over a value an external source resolved on an earlier call, and the source pass below is a
  628. # once-per-home no-op — so the clobber stuck for the life of the process (#74265). Re-assert only what the
  629. # source is authoritative for; managed scope, applied last with override=True, still wins on purpose.
  630. if _SECRET_SOURCE_RESTORE_BY_HOME:
  631. for name, value in _SECRET_SOURCE_RESTORE_BY_HOME.get(str(home_path.resolve()), {}).items():
  632. if os.environ.get(name) != value:
  633. os.environ[name] = value
  634. # The config backend is selected here — after the .env files, before the first config.yaml read
  635. # (``_apply_external_secret_sources`` reads ``secrets:``) — so an unavailable backend stops the
  636. # process before any reader could fall back to defaults (config-config design §4.4, D11/D32). A
  637. # remote backend fetches this home's config here, failing closed; the file backend does nothing.
  638. from hermes_cli.config_backend import get_config_backend
  639. # The administrator-managed .env is part of the deployment the backend boots from (it may be
  640. # where the selector, plane URL or instance id live). The override=True loads above may have
  641. # replaced those names with the home's values, so re-publish the deployment names (only those)
  642. # with the bootstrap precedence -- managed last -- before the boot. The full managed overlay
  643. # still applies once, last below, where it keeps its precedence over secret-source results.
  644. bootstrap_deployment(home_path, project_env_path)
  645. get_config_backend().boot(home_path)
  646. # External sources are skipped for the updater (dotenv + managed env still load): ``update`` must not
  647. # import optional secret-manager libs (Bitwarden → cryptography → _rust.pyd) into the process replacing
  648. # that env on Windows, and a fresh retry after a deferred dependency install would otherwise make the
  649. # self-lock preflight exit 2 again.
  650. from hermes_cli import _early_recovery
  651. # External secret sources are skipped in two updater situations: 1. ``load_external_secrets=False`` —
  652. # the caller is an ``update`` invocation that must not import optional secret-manager libraries
  653. # (Bitwarden → cryptography → ``_rust.pyd``) into the process that replaces that same environment on
  654. # Windows (#73381, #86735). 2. A fresh ``hermes update`` retry just completed a deferred dependency
  655. # install before importing this module. Do not remap native secret-source dependencies in that same
  656. # updater process or the self-lock preflight will recreate the marker and exit 2 again. Dotenv and
  657. # managed env still load in both cases; only external source resolution is unnecessary for the updater.
  658. if load_external_secrets and not _early_recovery._should_skip_external_secret_sources():
  659. _apply_external_secret_sources(home_path)
  660. _apply_managed_env(load_pass=load_pass)
  661. # config.yaml owns terminal.*, but the override=True loads above let a stale TERMINAL_ENV=docker in
  662. # ~/.hermes/.env win on every reload and flip the backend mid-session in long-lived processes.
  663. # Re-apply the explicit terminal keys LAST, after the managed overlay, so the merged config lands.
  664. # config.yaml is the documented source of truth for terminal.* settings, but the dotenv loads above run
  665. # with override=True — so a stale TERMINAL_ENV=docker left in ~/.hermes/.env (e.g. written by an older
  666. # `hermes setup` before the user switched terminal.backend in config.yaml) silently wins again on every
  667. # reload. Startup launchers bridge config→env once, but long-lived processes (gateway per-turn reload,
  668. # cron standalone runs) call load_hermes_dotenv() repeatedly and used to flip the effective backend back
  669. # to the stale .env value mid-session (#29186, #67323).
  670. _reapply_terminal_config_bridge(home_path)
  671. return loaded
  672. def _reapply_terminal_config_bridge(home_path: Path) -> None:
  673. """Re-assert config.yaml's explicit ``terminal.*`` keys over reloaded .env via the single shared bridge
  674. ``apply_terminal_config_to_env`` (also used by terminal_tool and the TUI/dashboard launchers) so the
  675. semantics can't drift between sites."""
  676. try:
  677. if Path(home_path).resolve() != _process_hermes_home().resolve():
  678. return
  679. from hermes_cli.config import apply_terminal_config_to_env
  680. apply_terminal_config_to_env(env=None)
  681. except Exception:
  682. pass
  683. def _apply_managed_env(*, load_pass: int | None = None) -> None:
  684. """Apply the managed-scope .env last, with override, so it beats user/shell. Does NOT stop the agent
  685. from later mutating os.environ (v1 relies on filesystem permissions). Fail-open: never blocks startup."""
  686. try:
  687. from hermes_cli import managed_scope
  688. managed_dir = managed_scope.get_managed_dir()
  689. except Exception:
  690. return
  691. if managed_dir is None:
  692. return
  693. managed_env = managed_dir / ".env"
  694. if not managed_env.exists():
  695. return
  696. _sanitize_env_file_if_needed(managed_env)
  697. _load_dotenv_with_fallback(managed_env, override=True, load_pass=load_pass, managed=True)
  698. def _revoke_secret_source_writes(home_path: Path, *, keep) -> None:
  699. """Take back what a source wrote into ``os.environ`` for *home_path* once nothing backs it.
  700. ``keep(name, source)`` says which recorded writes are still owned. Every other write is revoked,
  701. but only while ``os.environ`` still holds exactly the value the source wrote: a value another owner
  702. replaced since (the profile's ``.env``, the administrator-managed ``.env``, a shell export, a later
  703. source) is theirs and stays. A revoked name gets back the value it held before the source's first
  704. write, or is removed when it had none."""
  705. home_key = str(Path(home_path).resolve())
  706. writes = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key)
  707. if not writes:
  708. return
  709. try:
  710. from agent.secret_scope import load_env_file
  711. owned_by_dotenv = set(load_env_file(Path(home_path) / ".env")) | _MANAGED_DOTENV_KEYS
  712. except Exception:
  713. owned_by_dotenv = set(_MANAGED_DOTENV_KEYS)
  714. kept: dict[str, tuple[str, str, str | None]] = {}
  715. for name, (source, value, prior) in writes.items():
  716. if keep(name, source):
  717. kept[name] = (source, value, prior)
  718. continue
  719. if name not in owned_by_dotenv and os.environ.get(name) == value:
  720. if prior is None:
  721. os.environ.pop(name, None)
  722. else:
  723. os.environ[name] = prior
  724. still_attributed = any(
  725. other.get(name, ("",))[0] == source
  726. for key, other in _SECRET_SOURCE_WRITES_BY_HOME.items() if key != home_key)
  727. if _SECRET_SOURCES.get(name) == source and not still_attributed:
  728. _SECRET_SOURCES.pop(name, None)
  729. if kept:
  730. _SECRET_SOURCE_WRITES_BY_HOME[home_key] = kept
  731. else:
  732. _SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
  733. def _refuse_protected_env_from_sources(report, backend=None, environ_before=None, env=None, *, routed_home=None) -> None:
  734. """D32: the config backend's own credential (which plane, which agent, which token) must never
  735. come from a secret source — a source that could change it could point config at another plane.
  736. Exits (the backend is unusable), even when the pre-existing value won.
  737. *backend* is the backend selected BEFORE the sources ran: judging by the backend selected after
  738. them would let a source that writes ``HERMES_CONFIG_BACKEND=file`` switch remote mode off and
  739. disarm this very check. The selector itself is never a source's to change, under any backend:
  740. a source's write to it is reverted in *env* (the mapping the sources wrote into; default
  741. ``os.environ``) to its *environ_before* value before anything else.
  742. Startup runs the sources against a private staging copy of the environment and calls this
  743. BEFORE publishing it (:func:`_publish_staged_env`), so a refused value is never visible to a
  744. concurrent request — the poller reads the plane URL and credential live from ``os.environ``."""
  745. if env is None:
  746. env = os.environ
  747. from hermes_cli.config_backend import BACKEND_ENV, ConfigBackendUnavailable, get_config_backend
  748. if backend is None:
  749. backend = get_config_backend()
  750. supplied = set(report.provenance)
  751. for src in report.sources:
  752. supplied.update(getattr(src, "skipped_existing", ()) or ())
  753. protected = backend.protected_env_names() | {BACKEND_ENV}
  754. clash = sorted(supplied & protected)
  755. if not clash:
  756. return
  757. if environ_before is not None: # undo the source's writes to protected names before deciding
  758. for name in clash:
  759. if name in environ_before:
  760. env[name] = environ_before[name]
  761. else:
  762. env.pop(name, None)
  763. if not backend.protected_env_names():
  764. # The file backend has no plane credential: only the selector clashed. Reverted, and dropped
  765. # from the report so no provenance/snapshot re-applies it later; keep going.
  766. for name in clash:
  767. report.provenance.pop(name, None)
  768. for src in report.sources:
  769. for bucket in (getattr(src, "applied", None), getattr(src, "skipped_existing", None)):
  770. if isinstance(bucket, list) and name in bucket:
  771. bucket.remove(name)
  772. print(f" Secret sources: ignored {BACKEND_ENV} from a secrets: source (the config backend is "
  773. "chosen by .env or the process environment only).", file=sys.stderr)
  774. return
  775. outcome = (f"Hermes does not serve profile home {routed_home} with it." if routed_home is not None
  776. else "Hermes does not start with it.")
  777. raise ConfigBackendUnavailable(
  778. f"A secrets: source supplies {', '.join(clash)}, which the {backend.name!r} config "
  779. "backend uses to reach its config plane. That credential must come from auth.json or .env, never "
  780. f"from a secret source; remove the mapping. {outcome}")
  781. def _publish_staged_env(staged: dict, environ_before: dict) -> None:
  782. """Copy into ``os.environ`` exactly the names the sources set in *staged* (value differs from
  783. *environ_before*). Sources only ever set names, so nothing is removed; a name nobody's source
  784. touched is left alone even if another thread changed it meanwhile."""
  785. for name, value in staged.items():
  786. if environ_before.get(name) != value:
  787. os.environ[name] = value
  788. def _apply_external_secret_sources(home_path: Path) -> None:
  789. """Pull secrets from every enabled external source into env — AFTER dotenv (sources need .env bootstrap
  790. tokens), BEFORE Hermes reads credentials; failures never block startup. Precedence/conflicts/provenance
  791. live in ``registry.apply_all``; this wrapper owns the once-per-home guard, the post-apply ASCII sweep,
  792. the ``_SECRET_SOURCES`` map and status lines."""
  793. home_key = str(Path(home_path).resolve())
  794. if home_key in _APPLIED_HOMES:
  795. return
  796. # Neither early return marks the home applied: a malformed config.yaml would otherwise permanently
  797. # disable secret loading for this process, and an unmarked home picks up a config change on the next
  798. # load (the signature-cached read is cheap).
  799. try:
  800. cfg = _load_secrets_config(home_path)
  801. except Exception:
  802. # See #40597.
  803. return
  804. # No source configured / enabled any more: whatever one wrote earlier is no longer backed (#126982
  805. # review). A config READ failure above keeps them — fail-open, same as the fetch-failure path below.
  806. if not cfg:
  807. _revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
  808. return
  809. # Defer the registry import until a source is enabled — bitwarden eagerly loads cryptography._rust.pyd,
  810. # which makes the Windows updater self-lock before its preflight. Detect by *shape* (dict with enabled
  811. # flag), not names, so plugin/test sources pass and a plain dict entry never forces the crypto load.
  812. any_enabled = any(isinstance(v, dict) and v.get("enabled") is True for v in cfg.values())
  813. if not any_enabled:
  814. _revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
  815. return
  816. try:
  817. from agent.secret_sources.registry import apply_all, enabled_source_names
  818. except ImportError:
  819. return
  820. # Revoke a removed/disabled source's writes BEFORE the pass: left in place, its stale value would make
  821. # a still-enabled source that supplies the same name skip it as pre-existing (``skipped_existing``).
  822. try:
  823. active = enabled_source_names(cfg, home_path)
  824. except Exception:
  825. active = None
  826. if active is not None:
  827. _revoke_secret_source_writes(home_path, keep=lambda _name, source: source in active)
  828. environ_before = dict(os.environ)
  829. # Latched before the sources run: a source must not be able to re-select the backend (D32).
  830. from hermes_cli.config_backend import get_config_backend
  831. backend_before = get_config_backend()
  832. # The sources write into a private staging copy, published only after the D32 check: written
  833. # straight into os.environ, a forbidden plane URL or credential would be live for any request
  834. # (the config poller, another thread's write) until the check reverted it.
  835. staged = dict(environ_before)
  836. try:
  837. report = apply_all(cfg, home_path, environ=staged)
  838. except Exception:
  839. return
  840. if not report.sources: # no source enabled: keep retrying cheaply so flipping one on takes effect
  841. return
  842. _refuse_protected_env_from_sources(report, backend_before, environ_before, env=staged)
  843. _publish_staged_env(staged, environ_before)
  844. # A real fetch attempt happened (success OR error): mark the home so the 3-5 import-time calls per
  845. # startup don't re-fetch / re-print (error retries are opt-in via reset_secret_source_cache()).
  846. # Marking AFTER the attempt keeps the earlier failure paths retryable.
  847. _APPLIED_HOMES.add(home_key)
  848. if report.applied_any:
  849. _sanitize_loaded_credentials() # vault values carry the same copy-paste corruption risk as .env
  850. for name, applied in report.provenance.items():
  851. _SECRET_SOURCES[name] = applied.source
  852. # Snapshot EVERY name a source supplied, not just the newly applied ones. A name the source supplied
  853. # but the pre-existing process value won (``skipped_existing``) is still this home's effective value
  854. # for it — and on the unscoped path os.environ IS this home's environment. Skipping those names
  855. # latched an EMPTY snapshot whenever the key was already in the env: after the first cron/plugin
  856. # re-pull (the previous apply's own write-back shadows every key) or from boot under systemd
  857. # ``EnvironmentFile=``. Under multiplex the scope is the only credential source, so an empty
  858. # snapshot failed every default-profile turn for the process lifetime (#102041).
  859. values: dict[str, str] = {}
  860. supplied = _record_supplied_names(report)
  861. for name in supplied:
  862. if name in os.environ:
  863. values[name] = os.environ[name]
  864. _record_secret_source_writes(home_key, report, supplied, environ_before)
  865. if values:
  866. _SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
  867. _SECRET_SOURCE_RESTORE_BY_HOME[home_key] = {
  868. n: values[n] for n, a in report.provenance.items() if a.authoritative and n in values}
  869. for src in report.sources:
  870. if src.applied:
  871. print(f" {src.label}: applied {len(src.applied)} "
  872. f"secret{'s' if len(src.applied) != 1 else ''}", file=sys.stderr)
  873. if src.result.error:
  874. print(f" {src.label}: {src.result.error}", file=sys.stderr)
  875. hint = _remediation_hint(src.name, src.result.error_kind, cfg, scope=home_key)
  876. if hint:
  877. print(f" {src.label}: → {hint}", file=sys.stderr)
  878. for warn in src.result.warnings:
  879. print(f" {src.label}: {warn}", file=sys.stderr)
  880. for conflict in report.conflicts:
  881. print(f" Secret sources: {conflict}", file=sys.stderr)
  882. def _record_secret_source_writes(home_key: str, report, supplied: set[str], environ_before: dict) -> None:
  883. """Refresh ``_SECRET_SOURCE_WRITES_BY_HOME`` after one process-global pass, revoking a write its
  884. still-enabled source has stopped supplying (rotated out, mapping removed). A source whose fetch
  885. FAILED this pass keeps its earlier writes: a transient vault outage is not a revocation."""
  886. failed = {src.name for src in report.sources if not src.result.ok}
  887. previous = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {})
  888. _revoke_secret_source_writes(
  889. Path(home_key),
  890. keep=lambda name, source: name in report.provenance or name in supplied or source in failed)
  891. writes = {name: rec for name, rec in _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {}).items()
  892. if name not in report.provenance and os.environ.get(name) == rec[1]}
  893. for name, applied in report.provenance.items():
  894. if name not in os.environ:
  895. continue
  896. before = environ_before.get(name)
  897. earlier = previous.get(name)
  898. # Re-applied over our own earlier write: the pre-source value is the one recorded then.
  899. prior = earlier[2] if earlier is not None and earlier[1] == before else before
  900. writes[name] = (applied.source, os.environ[name], prior)
  901. if writes:
  902. _SECRET_SOURCE_WRITES_BY_HOME[home_key] = writes
  903. else:
  904. _SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
  905. def _remediation_hint(source_name: str, error_kind, secrets_cfg: dict, *, scope: str | None = None) -> str:
  906. """The failed source's one-line fix-it hint; a plugin remediation() could raise and startup must not."""
  907. try:
  908. from agent.secret_sources.registry import get_source
  909. source = get_source(source_name, scope=scope)
  910. if source is None:
  911. return ""
  912. src_cfg = secrets_cfg.get(source_name)
  913. src_cfg = src_cfg if isinstance(src_cfg, dict) else {}
  914. return str(source.remediation(error_kind, src_cfg) or "").strip()
  915. except Exception:
  916. return ""
  917. def _load_secrets_config(home_path: Path) -> dict:
  918. """Read just the ``secrets:`` section of config.yaml, isolated so a malformed config can't break dotenv."""
  919. from hermes_cli.config_backend import config_exists, read_config_doc_readonly
  920. config_path = home_path / "config.yaml"
  921. if not config_exists(config_path):
  922. return {}
  923. # Prefer the shared raw-config cache: this is the first config.yaml read of a normal startup, so
  924. # populating it lets main.py's early bridge and hermes_logging reuse one parse instead of 3-4.
  925. if home_path == _process_hermes_home():
  926. try:
  927. from hermes_cli.config import read_raw_config
  928. data = read_raw_config() or {}
  929. return data.get("secrets") or {}
  930. except Exception:
  931. pass
  932. # Routed profiles re-enter their scope on every poll/turn; only re-parse after the file changed.
  933. try:
  934. data = read_config_doc_readonly(config_path) or {}
  935. except Exception:
  936. return {}
  937. return data.get("secrets") or {}
  938. def _process_hermes_home() -> Path:
  939. """The HERMES_HOME the running process was launched under.
  940. Must be the *true* process home, ignoring any context-local
  941. ``set_hermes_home_override`` a per-request task has installed. Both
  942. callers depend on that:
  943. * ``_reapply_terminal_config_bridge`` guards "only re-bridge config into
  944. the shared ``os.environ`` when THIS load is for the process's own
  945. profile". If this followed the task override, a per-turn handler scoped
  946. to a *secondary* profile (the multiplex dashboard serving every profile
  947. from one process) would satisfy the guard and bridge that profile's
  948. ``terminal.backend`` into the shared env — e.g. a ``local`` sibling
  949. profile clobbering the launch profile's ``TERMINAL_ENV=ssh`` while
  950. leaving its ``TERMINAL_SSH_*`` untouched, so the session silently runs
  951. commands locally (cross-profile terminal-backend leak).
  952. * ``_load_secrets_config`` uses it to decide whether the shared
  953. (mtime,size)-keyed config cache is safe to reuse; under an override it
  954. must fall through to an isolated parse of the scoped profile.
  955. ``hermes_constants.get_routing_process_hermes_home()`` is the override-immune
  956. resolver built for exactly this; delegate to it. It is also immune to a host
  957. that mirrors the served profile into the live ``HERMES_HOME`` env var
  958. (``pin_process_hermes_home``): without that, the mirrored profile satisfied
  959. the guard above and bridged ITS ``terminal.*`` into the shared env.
  960. """
  961. try:
  962. from hermes_constants import get_routing_process_hermes_home
  963. return get_routing_process_hermes_home()
  964. except Exception:
  965. return Path.home() / ".hermes"