| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121 |
- """Helpers for loading Hermes .env files consistently across entrypoints."""
- from __future__ import annotations
- import codecs
- import io
- import itertools
- import logging
- import os
- import sys
- import threading
- from pathlib import Path
- # Kept at module level on purpose: importing this module must fail when the dotenv install is
- # wiped (#57828) so early recovery provably runs before third-party imports (test_early_recovery).
- # The parser internals are imported lazily below because gateway tests stub ``sys.modules["dotenv"]``.
- import dotenv
- from hermes_cli.left_core_migration import profile_env_names
- from utils import atomic_replace, mkstemp_beside
- logger = logging.getLogger(__name__)
- # The ONLY env vars sanitized on load: credentials must be pure ASCII (they become HTTP header values);
- # arbitrary user env vars are never silently altered.
- _CREDENTIAL_SUFFIXES = ("_API_KEY", "_TOKEN", "_SECRET", "_KEY")
- # Once-per-process guards: load_hermes_dotenv() runs repeatedly (user + project env, gateway hot-reload,
- # lazy imports mid-turn, tests) so warnings/logs fire once per key/path/home.
- _WARNED_KEYS: set[str] = set() # credential names already given the non-ASCII warning
- _WARNED_UTF32_PATHS: set[str] = set() # .env paths already given the UTF-32 refuse-to-mangle warning
- _SCOPED_SKIP_LOGGED: set[str] = set() # routed profile homes whose multiplex dotenv skip was logged
- _WARNED_UNREADABLE_PATHS: set[str] = set() # unreadable profile .env paths already given the warning
- # env-var name → source label ("bitwarden", …) for externally injected credentials; setup / `hermes
- # model` tell users WHERE a key came from when .env lacks it.
- _SECRET_SOURCES: dict[str, str] = {}
- # Every env-var name an external source SUPPLIED for some home, whether it was applied or lost to a
- # pre-existing process value (``skipped_existing``). ``_SECRET_SOURCES`` is provenance metadata and only
- # names applied values; the scrub that keeps a launch profile's source-supplied names out of a routed
- # child must see the skipped ones too, or a name already in the process env leaks with the launch value.
- _SOURCE_SUPPLIED_NAMES: set[str] = set()
- # Every KEY name a dotenv file loaded into ``os.environ`` during this process's lifetime. A key removed
- # or renamed in the launch ``.env`` after boot stays in ``os.environ`` (dotenv never unsets), but a
- # re-parse of the current file no longer names it — so the launch-residue strip for a routed child must
- # work from what was LOADED, not from what the file says now. Additive for the process lifetime.
- _LOADED_DOTENV_KEYS: set[str] = set()
- # KEY names loaded from the administrator-managed ``.env`` (``_apply_managed_env``). Kept OUT of the launch
- # residue: those values are policy that beats the user's own ``.env`` for every profile, so a routed child
- # must keep them — and keep them LAST, over the routed profile's scope (review on f5f88d5058).
- _MANAGED_DOTENV_KEYS: set[str] = set()
- # Immutable per-home snapshots: os.environ is shared across profiles and a later home's apply may overwrite it.
- _SECRET_SOURCE_VALUES_BY_HOME: dict[str, dict[str, str]] = {}
- # Per home: the subset of the snapshot a dotenv reload may re-assert — see ``AppliedVar.authoritative`` (#74265).
- _SECRET_SOURCE_RESTORE_BY_HOME: dict[str, dict[str, str]] = {}
- # Per home: what the process-global path WROTE into ``os.environ`` for an external source — name →
- # (source name, value written, value the name held before the first write or None). The per-home
- # snapshot above is republished on every pass, but ``os.environ`` is not: without this record a value a
- # since-removed or disabled source injected stayed in the process env, and single-profile
- # ``get_secret()`` kept serving it through its ``os.environ`` fallback until restart. Deliberately NOT
- # cleared by a per-home ``reset_secret_source_cache``: it describes ``os.environ``, which a reset does not
- # touch, and the next pass needs it to revoke.
- _SECRET_SOURCE_WRITES_BY_HOME: dict[str, dict[str, tuple[str, str, str | None]]] = {}
- # HERMES_HOME paths already pulled external secrets for: load_hermes_dotenv() runs at import time from
- # several hot modules, so without this the Bitwarden status line prints 3-5x per startup and the config
- # re-parse + ASCII sweep re-run each time (Bitwarden's own cache only saves the network call).
- _APPLIED_HOMES: set[str] = set()
- _SECRET_SOURCE_CACHE_LOCK = threading.RLock()
- # What THIS process has published from dotenv files, per variable: (value before our first publish, or
- # None if absent; last value we published; load pass that published it). Reloads run per gateway turn and
- # per cron fire, and a line like ``PATH=/x:${PATH}`` interpolated against an environ that already holds the
- # previous reload's output grows by ``/x:`` every time until child spawns fail with E2BIG (#109902). A new
- # pass resolves against the baseline instead — but only where the environ still holds exactly what we
- # published, so a value the shell, config bridge, or an external secret source changed since is not frozen.
- # One process-wide record (not per home/project scope): the environ is process-wide, so alternating
- # callers (gateway with a project .env, cron without; home A then B) must peel each other's output too.
- _DOTENV_PUBLISHED: dict[str, tuple[str | None, str, int]] = {}
- _DOTENV_PASSES = itertools.count()
- _DOTENV_LOCK = threading.RLock()
- # Per-process credentials a parent mints and injects into the child's environment (the Desktop shell /
- # a link-style launcher spawns `hermes dashboard` with a fresh HERMES_DASHBOARD_SESSION_TOKEN and keeps
- # the same token for its own /api probes). They are never .env configuration, so a persisted value in
- # ~/.hermes/.env must not replace an injected one — the parent would then 401 against its own child
- # (#115955). A value an earlier dotenv pass published is still reloaded normally.
- _SPAWN_CREDENTIAL_KEYS: frozenset[str] = frozenset({"HERMES_DASHBOARD_SESSION_TOKEN"})
- # Behavioral routing keys a parent Hermes process injects into child env that silently redirect a profile
- # onto the wrong provider path; these — and ONLY these — are scrubbed at startup when absent from the
- # profile's .env. Credentials are excluded: shell exports are a documented way to supply them, and
- # read-time secret-scope checks (agent/secret_scope.py) own cross-profile credential isolation.
- # Providers that left core keep theirs listed on their ``LEFT_CORE`` row (``profile_env``), read statically:
- # this runs before provider discovery, and with the plugin absent nothing else names them.
- _PROFILE_MANAGED_ENV_KEYS: frozenset[str] = frozenset({
- "HERMES_ACP_AUTH_METHOD", "HERMES_ACP_AUTO_APPROVE",
- }) | profile_env_names()
- def _env_keys_defined_in_dotenv(path: Path) -> set[str]:
- """KEY names assigned in a dotenv file (including empty ``KEY=``), via the same tokenizer that installs
- profile scopes — a key the installer sees is a key the dashboard scrub sees (BOM'd first line included)."""
- from agent.secret_scope import load_env_file
- return set(load_env_file(path))
- def _clear_known_keys_missing_from_dotenv(path: Path) -> None:
- """After ``.env`` loaded with override, delete inherited ``_PROFILE_MANAGED_ENV_KEYS`` it does not
- define. Deliberately NARROW: only keys that change *which provider path* is used.
- Does **not** run when the ``.env`` file does not exist (bare-profile case, which follows ``#66930`` /
- ``#67027`` semantics).
- """
- if not path.exists():
- return
- defined = _env_keys_defined_in_dotenv(path)
- for key in _PROFILE_MANAGED_ENV_KEYS:
- if key not in defined and key in os.environ:
- del os.environ[key]
- def get_secret_source(env_var: str) -> str | None:
- """Source label that supplied ``env_var`` (``"bitwarden"`` …), None for .env/shell keys. Metadata only —
- never authorization to persist the raw value."""
- return _SECRET_SOURCES.get(env_var)
- def _record_supplied_names(report) -> set[str]:
- """Every name *report*'s sources supplied — applied, or skipped because a value already existed —
- recorded into ``_SOURCE_SUPPLIED_NAMES`` so the routed-child scrub knows the source owns it."""
- supplied = set(report.provenance)
- for src in report.sources:
- supplied.update(src.skipped_existing)
- _SOURCE_SUPPLIED_NAMES.update(supplied)
- return supplied
- def secret_source_names() -> tuple[str, ...]:
- """Every env-var name some profile's external secret source APPLIED (names only — the map is
- process-wide, so a value must be resolved through the active profile's secret scope). Consumers that
- forward source values into a child (MCP stdio env) want exactly these; see ``source_supplied_names``
- for the wider set the routed-child scrub needs."""
- return tuple(_SECRET_SOURCES)
- def source_supplied_names() -> tuple[str, ...]:
- """Every env-var name an external source supplied for any home — applied, or lost to a pre-existing
- process value (``skipped_existing``). The launch value in ``os.environ`` is still not a routed
- profile's to inherit, so the strip must see the skipped names too."""
- return tuple(sorted(set(_SECRET_SOURCES) | _SOURCE_SUPPLIED_NAMES))
- def launch_dotenv_keys() -> frozenset[str]:
- """KEY names any NON-managed dotenv file loaded into this process's ``os.environ`` so far (see
- ``_LOADED_DOTENV_KEYS``); the launch profile's residue set for routed children."""
- return frozenset(_LOADED_DOTENV_KEYS)
- def dotenv_published_values(name: str) -> frozenset[str]:
- """The values dotenv loads in this process last wrote to ``name``, its name matched case-insensitively:
- on Windows ``https_proxy`` in a .env and an inherited ``HTTPS_PROXY`` are one environment variable."""
- folded = name.upper()
- with _DOTENV_LOCK:
- return frozenset(value for key, (_base, value, _pass) in _DOTENV_PUBLISHED.items() if key.upper() == folded)
- def managed_dotenv_keys() -> frozenset[str]:
- """KEY names the administrator-managed ``.env`` loaded (see ``_MANAGED_DOTENV_KEYS``). Policy for
- every profile: never stripped from a routed child, and re-applied over the routed scope."""
- return frozenset(_MANAGED_DOTENV_KEYS)
- def get_secret_source_values(hermes_home: str | os.PathLike) -> dict[str, str]:
- """Return the external-secret value snapshot for ``hermes_home``."""
- return dict(_SECRET_SOURCE_VALUES_BY_HOME.get(str(Path(hermes_home).resolve()), {}))
- def hydrate_profile_secret_sources(hermes_home: str | os.PathLike) -> dict[str, str]:
- """Resolve one profile's configured sources without mutating ``os.environ``: multiplex gateways route
- turns to profiles that never ran the process-global dotenv path, so resolve against a private mapping
- seeded from that ``.env`` and record the per-home snapshot for ``build_profile_secret_scope()``.
- Fail-open / once-per-home like ``_apply_external_secret_sources``; never returns plaintext .env entries.
- The one fail-closed case is D32's: under the remote config backend, a source that supplies a
- protected name raises :class:`ConfigBackendUnavailable` and records nothing for the home."""
- with _SECRET_SOURCE_CACHE_LOCK:
- return _hydrate_profile_secret_sources(Path(hermes_home))
- def _hydrate_profile_secret_sources(home: Path) -> dict[str, str]:
- """Locked implementation for :func:`hydrate_profile_secret_sources`."""
- from agent.safe_worker_policy import worker_config_snapshot
- if worker_config_snapshot() is not None:
- return {}
- home_key = str(home.resolve())
- if home_key in _APPLIED_HOMES:
- return get_secret_source_values(home)
- # A retry must not keep serving a partial result after the source is removed, disabled, or can no
- # longer be evaluated. Publish only the snapshot established by this attempt.
- _SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
- _SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
- try:
- cfg = _load_secrets_config(home)
- except Exception:
- return {}
- if not cfg:
- return {}
- try:
- from agent.secret_scope import _is_global_env, load_env_file
- from agent.secret_sources.registry import apply_all
- local_env = {name: value for name, value in os.environ.items() if _is_global_env(name)}
- local_env.update(load_env_file(home / ".env"))
- # Mirror load_hermes_dotenv()'s .op.env bootstrap (1Password token lives in gitignored .op.env)
- # or cold profiles fail 1Password hydration. .env wins.
- # Without seeding it here a cold profile configured for the supported .op.env flow fails 1Password
- # hydration (sweeper review on #74549). .env values win — never override an existing key.
- op_env = home / ".op.env"
- if op_env.exists():
- for _name, _value in load_env_file(op_env).items():
- local_env.setdefault(_name, _value)
- local_env["HERMES_HOME"] = str(home)
- env_before = dict(local_env)
- report = apply_all(cfg, home, environ=local_env)
- except Exception:
- return {}
- if not report.sources:
- return {}
- # D32, as on the process-global path: a source must never supply the config backend's own
- # deployment or plane credential, for a routed profile either (this snapshot feeds that
- # profile's secret scope, which the Portal/plane resolvers read). Checked before anything of
- # this attempt is recorded: the remote backend refuses the profile (raises, nothing cached, so
- # every retry refuses again until the mapping is removed); the file backend drops the selector.
- from hermes_cli.config_backend import get_config_backend
- _refuse_protected_env_from_sources(report, get_config_backend(), env_before, env=local_env, routed_home=home)
- # Routed profiles have no runtime reset path. Keep a failed source retryable so correcting its
- # profile-local bootstrap credentials takes effect on the next turn; successful sources from a
- # mixed report are still snapshotted below and can be used while the failed source recovers.
- if all(src.result.ok for src in report.sources):
- _APPLIED_HOMES.add(home_key)
- # Same ownership bookkeeping as the process-global path: a name this profile's source supplied — applied,
- # or skipped because the private mapping already had it — is a source-owned name the routed-child scrub
- # must know about, or a sibling still inherits the launch value for it (review on f5f88d5058).
- _record_supplied_names(report)
- values: dict[str, str] = {}
- for name, applied in report.provenance.items():
- value = local_env.get(name)
- if value is None:
- continue
- _SECRET_SOURCES[name] = applied.source
- values[name] = value
- _SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
- return dict(values)
- def reset_secret_source_cache(hermes_home: str | os.PathLike | None = None) -> None:
- """Forget applied homes so the next load re-pulls (tests, long-running processes after config edits).
- ``hermes_home`` limits the reset to ONE home: a multiplex gateway keeps every profile's snapshot in
- this process, and a per-fire cron re-pull or a plugin-discovery refresh for one home must not wipe
- a sibling's hydrated snapshot — the sibling's next scope build would run empty until it re-hydrated
- (#102041)."""
- if hermes_home is None:
- _APPLIED_HOMES.clear()
- _SECRET_SOURCES.clear()
- _SOURCE_SUPPLIED_NAMES.clear()
- _SECRET_SOURCE_VALUES_BY_HOME.clear()
- _SECRET_SOURCE_RESTORE_BY_HOME.clear()
- _SECRET_SOURCE_WRITES_BY_HOME.clear()
- return
- home_key = str(Path(hermes_home).resolve())
- _APPLIED_HOMES.discard(home_key)
- _SECRET_SOURCE_VALUES_BY_HOME.pop(home_key, None)
- _SECRET_SOURCE_RESTORE_BY_HOME.pop(home_key, None)
- def format_secret_source_suffix(env_var: str) -> str:
- """``" (from Bitwarden)"``-style suffix; ``""`` for .env/shell keys (only external sources are named)."""
- source = get_secret_source(env_var)
- if not source:
- return ""
- if source == "bitwarden":
- return " (from Bitwarden)"
- # Registry label (e.g. "1Password"); raw name for unknown sources (uninstalled plugin, tests).
- try:
- from agent.secret_sources.registry import get_source
- registered = get_source(source)
- if registered is not None and registered.label:
- return f" (from {registered.label})"
- except Exception:
- pass
- return f" (from {source})"
- def _format_offending_chars(value: str, limit: int = 3) -> str:
- """Compact ``U+XXXX ('c'), ...`` summary of non-ASCII codepoints."""
- seen: list[str] = []
- for ch in value:
- if ord(ch) > 127:
- label = f"U+{ord(ch):04X}"
- if ch.isprintable():
- label += f" ({ch!r})"
- if label not in seen:
- seen.append(label)
- if len(seen) >= limit:
- break
- return ", ".join(seen)
- def _sanitize_loaded_credentials() -> None:
- """Strip non-ASCII from credential env vars (``_CREDENTIAL_SUFFIXES``) so the codebase never sees them.
- Emits a one-line warning to stderr when characters are stripped. Silent stripping would mask copy-paste
- corruption (Unicode lookalike glyphs from PDFs / rich-text editors, ZWSP from web pages) as opaque
- provider-side "invalid API key" errors (see #6843).
- """
- for key, value in list(os.environ.items()):
- if not any(key.endswith(suffix) for suffix in _CREDENTIAL_SUFFIXES):
- continue
- if value.isascii():
- continue
- cleaned = value.encode("ascii", errors="ignore").decode("ascii")
- os.environ[key] = cleaned
- if key in _WARNED_KEYS:
- continue
- _WARNED_KEYS.add(key)
- stripped = len(value) - len(cleaned)
- detail = _format_offending_chars(value) or "non-printable"
- print(f" Warning: {key} contained {stripped} non-ASCII character"
- f"{'s' if stripped != 1 else ''} ({detail}) — stripped so the "
- f"key can be sent as an HTTP header.", file=sys.stderr)
- print(
- " This usually means the key was copy-pasted from a PDF, "
- "rich-text editor, or web page that substituted lookalike\n"
- " Unicode glyphs for ASCII letters. If authentication fails "
- "(e.g. \"API key not valid\"), re-copy the key from the\n"
- " provider's dashboard and run `hermes setup` (or edit the "
- ".env file in a plain-text editor).",
- file=sys.stderr,
- )
- def _parse_dotenv_file(path: Path) -> list[tuple[str, str | None]]:
- """``(name, raw value)`` assignments of one dotenv file, uninterpolated, in file order."""
- raw = path.read_bytes()
- try:
- # utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
- # first key name and silently drop it from os.environ under its canonical name.
- text = raw.decode("utf-8-sig")
- except UnicodeDecodeError:
- raw = raw.removeprefix(codecs.BOM_UTF8)
- text = raw.decode("latin-1")
- # Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
- # exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
- from dotenv.main import DotEnv
- return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
- def _dotenv_lookup_env(load_pass: int | None) -> dict[str, str]:
- """``os.environ`` with every value an earlier pass (any pass, for ``None``) published peeled back to
- its baseline — what ``${VAR}`` resolves against (see ``_DOTENV_PUBLISHED``). Caller holds the lock."""
- lookup_env = dict(os.environ)
- for name, (baseline, published, published_pass) in _DOTENV_PUBLISHED.items():
- if published_pass != load_pass and lookup_env.get(name) == published:
- if baseline is None:
- del lookup_env[name] # absent, so ``${VAR:-default}`` takes the default again
- else:
- lookup_env[name] = baseline
- return lookup_env
- def _resolve_dotenv_assignments(
- assignments: list[tuple[str, str | None]], lookup_env: dict[str, str], *, override: bool,
- ) -> dict[str, str | None]:
- """Expand ``${VAR}`` / ``${VAR:-default}`` like ``dotenv.main.resolve_variables``, minus the live
- ``os.environ``: a line sees *lookup_env* and the file's earlier lines (earlier lines win under
- *override*, the environment otherwise)."""
- from dotenv.variables import parse_variables
- resolved: dict[str, str | None] = {}
- for name, value in assignments:
- if value is not None:
- lookup = {**lookup_env, **resolved} if override else {**resolved, **lookup_env}
- value = "".join(atom.resolve(lookup) for atom in parse_variables(value))
- resolved[name] = value
- return resolved
- def resolve_home_dotenv(home: Path) -> dict[str, str]:
- """What ``load_hermes_dotenv`` would publish from *home*'s dotenv layers, read-only: the home's
- ``.env`` then the administrator-managed ``.env``, both with override and ``${VAR}`` expanded by the
- same rules against the process's own environment (this process's dotenv output peeled back), each
- layer seeing the one before. External secret sources and the project ``.env`` are not included."""
- from hermes_cli import managed_scope
- managed_dir = managed_scope.get_managed_dir()
- layers = [home / ".env", *([managed_dir / ".env"] if managed_dir is not None else [])]
- with _DOTENV_LOCK:
- lookup_env = _dotenv_lookup_env(None)
- out: dict[str, str] = {}
- for path in layers:
- try:
- assignments = _parse_dotenv_file(path)
- except OSError:
- continue
- for name, value in _resolve_dotenv_assignments(assignments, lookup_env, override=True).items():
- if value is not None:
- lookup_env[name] = out[name] = value
- return out
- def _load_dotenv_with_fallback(
- path: Path, *, override: bool, load_pass: int | None = None, managed: bool = False,
- ) -> None:
- """Load one dotenv file into ``os.environ`` like ``dotenv.load_dotenv`` — same parser, same
- ``${VAR}`` / ``${VAR:-default}`` / precedence rules — except that ``${VAR}`` resolves against the value
- VAR had before this process's earlier passes published it (see ``_DOTENV_PUBLISHED``).
- ``load_pass`` groups the layered files of one ``load_hermes_dotenv`` call: within a pass a later layer
- (project, managed) still sees the earlier layer's output, as it always did; only OTHER passes' output
- is peeled. A bare call (``hermes send``'s direct reload) is its own pass."""
- assignments = _parse_dotenv_file(path)
- with _DOTENV_LOCK:
- if load_pass is None:
- load_pass = next(_DOTENV_PASSES)
- resolved = _resolve_dotenv_assignments(assignments, _dotenv_lookup_env(load_pass), override=override)
- for name, value in resolved.items():
- if value is None or (not override and name in os.environ):
- continue
- current = os.environ.get(name)
- record = _DOTENV_PUBLISHED.get(name)
- ours = record is not None and current == record[1]
- if name in _SPAWN_CREDENTIAL_KEYS and current and not ours:
- continue # parent-minted per-process credential: .env must not split it from the parent
- # Ours and untouched since → keep the original baseline; anything else is a newer outside value.
- baseline = record[0] if ours else current
- os.environ[name] = value
- _DOTENV_PUBLISHED[name] = (baseline, value, load_pass)
- # Every key this file defines, for the launch-residue strip: dotenv never unsets, so a key later
- # removed from the launch .env stays in os.environ and a re-parse of the file no longer names it.
- # Managed keys are recorded separately: they are administrator policy, not launch-profile residue.
- (_MANAGED_DOTENV_KEYS if managed else _LOADED_DOTENV_KEYS).update(name for name, _value in assignments)
- _sanitize_loaded_credentials() # httpx encodes headers as ASCII
- def _dotenv_assignments(path: Path) -> list:
- """``(name, raw value)`` pairs of one dotenv file as the loader will see it once
- ``_sanitize_env_file_if_needed`` has run: :func:`_parse_dotenv_file` plus the sanitizer's UTF-16
- rewrite and NUL strip, for the pre-boot bootstrap reads that run before the sanitizer."""
- raw = path.read_bytes()
- if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
- # Notepad "Unicode": ``_sanitize_env_file_if_needed`` rewrites it as UTF-8 before the loader
- # reads it, so parse it as that rewrite will -- bootstrap / stage2 run before the sanitizer
- # and must select the same backend the runtime then boots.
- try:
- with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
- text = f.read()
- except UnicodeDecodeError:
- text = raw.decode("latin-1")
- else:
- try:
- # utf-8-sig strips a leading BOM (PowerShell 5.1 / Notepad); plain utf-8 would keep U+FEFF on the
- # first key name and silently drop it from os.environ under its canonical name.
- text = raw.decode("utf-8-sig")
- except UnicodeDecodeError:
- raw = raw.removeprefix(codecs.BOM_UTF8)
- text = raw.decode("latin-1")
- text = text.replace("\x00", "") # the sanitizer strips NULs too (BOM-less UTF-16 / NUL-padded ASCII)
- # Imported here, not at module level: gateway tests stub ``sys.modules["dotenv"]`` with a bare module
- # exposing only ``load_dotenv``, and ``gateway.run`` imports this module at import time.
- from dotenv.main import DotEnv
- return list(DotEnv(dotenv_path=None, stream=io.StringIO(text), interpolate=False).parse())
- def _bootstrap_env(home: Path | None = None, project_env: Path | None = None) -> dict:
- """The process env as ``load_hermes_dotenv`` leaves it for the names its dotenv layers define:
- the home's ``.env`` (override), ``.op.env`` (fills gaps, only without a process
- ``OP_SERVICE_ACCOUNT_TOKEN``), the caller's *project_env* (fills gaps when the home has a
- ``.env``, else overrides) and the managed ``.env`` (override, last); ``${VAR}`` resolved, the
- loader's parser. Read-only: nothing is published and no file is rewritten (sanitizing imports
- hermes_cli.config, whose import-time config read must not run before the backend is decided)."""
- home = Path(home) if home is not None else _process_hermes_home()
- env = dict(os.environ)
- user_env = home / ".env"
- layers: list[tuple[Path, bool]] = [(user_env, True)]
- if not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
- layers.append((home / ".op.env", False))
- if project_env is not None:
- layers.append((Path(project_env), not user_env.exists()))
- try:
- from hermes_cli import managed_scope
- managed_dir = managed_scope.get_managed_dir()
- except Exception: # health: allow BLE001 -- managed scope never blocks startup, as in _apply_managed_env
- managed_dir = None
- if managed_dir is not None:
- layers.append((managed_dir / ".env", True))
- for path, override in layers:
- try:
- assignments = _dotenv_assignments(path)
- except OSError:
- continue
- resolved = _resolve_dotenv_assignments(assignments, env, override=override)
- for name, value in resolved.items():
- if value is not None and (override or name not in env):
- env[name] = value
- return env
- def selected_config_backend_name(home: Path | None = None) -> str:
- """The config backend ``load_hermes_dotenv`` selects for *home*, decided without loading or
- booting anything. For a caller that must decide before Hermes starts (Docker stage2 skips the
- config-file migration in remote mode, D12)."""
- from hermes_cli.config_backend import BACKEND_ENV
- return (_bootstrap_env(home).get(BACKEND_ENV) or "").strip().lower() or "file"
- def apply_config_bootstrap_env(backend_names, home: Path | None = None, project_env: Path | None = None) -> None:
- """Publish ``backend_names(kind)`` (the selected backend's deployment and credential, D26/D32) from the
- launch home's ``.env`` and the managed ``.env``, with the values ``load_hermes_dotenv`` gives
- them. Called before the first config read of a process, which can come before
- ``load_hermes_dotenv`` (``hermes_cli.config`` reads config at import time): the backend is
- selected and authenticated from the same deployment either way. A file deployment gets only its
- selector (the file backend needs nothing else before boot); values the process env holds and
- no file defines are left alone; ``load_hermes_dotenv`` later publishes the same values."""
- from agent.safe_worker_policy import worker_config_snapshot
- from hermes_cli.config_backend import BACKEND_ENV
- # A frozen-policy worker reads no profile dotenv (``load_hermes_dotenv`` returns early there):
- # its bootstrap supplied the process env explicitly, so nothing is published from the files.
- if worker_config_snapshot() is not None:
- return
- env = _bootstrap_env(home, project_env)
- # The selector itself always, so a file deployment is not read as another kind meanwhile.
- names = {BACKEND_ENV} | set(backend_names((env.get(BACKEND_ENV) or "").strip().lower() or "file"))
- for name in names:
- value = env.get(name)
- if value is not None and os.environ.get(name) != value:
- os.environ[name] = value
- def _sanitize_env_file_if_needed(path: Path) -> None:
- """Pre-sanitize a .env file before python-dotenv reads it. Sniffs a leading BOM *before* any text
- decode: UTF-16 (Notepad "Unicode") is rewritten as clean UTF-8; UTF-32 is refused (left untouched) so
- we never fall through to the errors=replace corruption path."""
- if not path.exists():
- return
- try:
- from hermes_cli.config import _sanitize_env_lines
- except ImportError:
- return # early bootstrap — config module not available yet
- try:
- raw = path.read_bytes()
- except Exception:
- return
- # ORDER MATTERS: BOM_UTF32_LE (FF FE 00 00) startswith BOM_UTF16_LE (FF FE); UTF-16 first would mangle it.
- force_utf8_rewrite = False
- if raw.startswith(codecs.BOM_UTF32_LE) or raw.startswith(codecs.BOM_UTF32_BE):
- # Lazy import keeps the module import block identical to #65124's codecs/io additions so the two PRs
- # auto-merge either order.
- path_key = str(path.resolve())
- if path_key not in _WARNED_UTF32_PATHS:
- _WARNED_UTF32_PATHS.add(path_key)
- logger.warning("Skipping .env sanitize for %s: UTF-32 BOM detected; "
- "leaving file untouched to avoid corruption", path)
- return
- if raw.startswith(codecs.BOM_UTF16_LE) or raw.startswith(codecs.BOM_UTF16_BE):
- # "utf-16" uses the BOM for endianness and strips it; newline=None matches open()'s universal
- # newlines (not splitlines()'s extra boundaries like U+2028) so sanitize sees the same lines.
- try:
- with io.TextIOWrapper(io.BytesIO(raw), encoding="utf-16", newline=None) as f:
- original = f.readlines()
- except UnicodeDecodeError:
- return
- force_utf8_rewrite = True # always rewrite UTF-16 as UTF-8 so the dotenv load sees a canonical file
- else:
- # utf-8-sig strips a UTF-8 BOM; errors=replace so embedded NULs can be stripped below.
- try:
- with open(path, encoding="utf-8-sig", errors="replace") as f:
- original = f.readlines()
- except Exception:
- return
- # errors=replace turns undecodable leading bytes into U+FFFD; persisting would glue them onto
- # the first key name permanently — leave the file untouched instead.
- if original and original[0].startswith("\ufffd"):
- return
- try:
- # Strip NULs (os.environ raises ValueError on them); also repairs BOM-less UTF-16 (NUL-padded ASCII).
- stripped = [line.replace("\x00", "") for line in original]
- sanitized = _sanitize_env_lines(stripped)
- if sanitized != original or force_utf8_rewrite:
- fd, tmp = mkstemp_beside(path, suffix=".tmp", prefix=".env_")
- try:
- with os.fdopen(fd, "w", encoding="utf-8") as f:
- f.writelines(sanitized)
- f.flush()
- os.fsync(f.fileno())
- atomic_replace(tmp, path)
- except BaseException:
- try:
- os.unlink(tmp)
- except OSError:
- pass
- raise
- except Exception:
- pass # best-effort — don't block gateway startup
- def dotenv_unreadable(path: Path) -> bool:
- """True when *path* exists but this process may not read it."""
- try:
- with open(path, "rb"):
- return False
- except PermissionError:
- return True
- except OSError: # missing, or a failure that is not a refusal
- return False
- def _load_user_dotenv(path: Path, *, load_pass: int) -> bool:
- """Load the profile ``.env``; False when it exists but cannot be read and managed scope pins
- ``terminal.backend``. A sandbox launcher withholds the file, passes the keys it allows in as
- environment and pins the backend. Without that pin the ``PermissionError`` propagates: the file may
- be the only place the backend is set, and skipping it would run terminal commands on the host."""
- try:
- _load_dotenv_with_fallback(path, override=True, load_pass=load_pass)
- except PermissionError:
- from hermes_cli import managed_scope
- if managed_scope.pinned_terminal_backend() is None:
- raise
- if str(path) not in _WARNED_UNREADABLE_PATHS:
- _WARNED_UNREADABLE_PATHS.add(str(path))
- logger.warning("Cannot read %s; managed scope pins terminal.backend, so it loads as absent", path)
- return False
- return True
- def load_hermes_dotenv(
- *,
- hermes_home: str | os.PathLike | None = None,
- project_env: str | os.PathLike | None = None,
- load_external_secrets: bool = True,
- ) -> list[Path]:
- """Load Hermes env files: ``~/.hermes/.env`` overrides stale shell exports; project ``.env`` is a dev
- fallback that only fills gaps when the user env exists (and overrides shell vars when it does not)."""
- from agent.safe_worker_policy import worker_config_snapshot
- # The private bootstrap supplies credentials explicitly; do not rehydrate
- # profile behavior, external secret plugins, or managed env in this worker.
- if worker_config_snapshot() is not None:
- return []
- # Process home on purpose (never the per-turn override): a startup .env load must not follow a routed
- # profile — see the multiplex guard below.
- from hermes_constants import get_process_hermes_home
- home_path = Path(hermes_home) if hermes_home else get_process_hermes_home()
- # Multiplex gateway: while a routed profile-home override is active, copying that profile's .env
- # into os.environ would expose its credentials to sibling turns and every spawned child. The launch
- # home's own .env is process configuration and still loads: the launch profile's scoped bodies bind
- # an override naming the launch home too, and skipping it hid launch-only credentials such as a
- # fallback_providers key from the process env (#125530). Both the load's target AND the active
- # home must be the launch home: a launch-targeted load inside a FOREIGN turn re-bridges terminal.*
- # from the config the override resolves to, i.e. the routed profile's cwd into the shared env.
- # External sources still refresh against the profile mapping.
- # (``is_multiplex_active()`` is also true, context-locally, for a routed cron fire in the desktop
- # backend — see ``cron.scheduler_provider.routed_profile_fire``.)
- from agent.secret_scope import is_multiplex_active
- from hermes_constants import get_hermes_home, get_hermes_home_override
- launch_home = _process_hermes_home().resolve()
- if (is_multiplex_active() and get_hermes_home_override() is not None
- and (home_path.resolve() != launch_home or get_hermes_home().resolve() != launch_home)):
- home_key = str(home_path.resolve())
- if home_key not in _SCOPED_SKIP_LOGGED:
- _SCOPED_SKIP_LOGGED.add(home_key)
- logger.debug("multiplex: skipping process-global dotenv load for routed "
- "profile home %s (credentials resolve via the profile scope)", home_path)
- if load_external_secrets:
- from hermes_cli import _early_recovery
- if not _early_recovery._should_skip_external_secret_sources():
- hydrate_profile_secret_sources(home_path)
- return []
- loaded: list[Path] = []
- user_env = home_path / ".env"
- project_env_path = Path(project_env) if project_env else None
- load_pass = next(_DOTENV_PASSES) # one pass: later layers below see the earlier layers' output
- # Before anything below imports hermes_cli.config (the .env sanitizer does), whose import-time
- # config read selects and boots the backend: publish the deployment and plane credential from
- # THIS load's layers, the project .env included, with the precedence the loads below give them.
- from hermes_cli.config_backend import bootstrap_deployment
- bootstrap_deployment(home_path, project_env_path)
- if user_env.exists(): # normalize formatting / strip NULs before parsing
- _sanitize_env_file_if_needed(user_env)
- if project_env_path and project_env_path.exists():
- _sanitize_env_file_if_needed(project_env_path)
- if user_env.exists() and _load_user_dotenv(user_env, load_pass=load_pass):
- loaded.append(user_env)
- _clear_known_keys_missing_from_dotenv(user_env) # mirrors reload_env(): inherited keys must not leak
- # .op.env AFTER .env so .env wins, but the bootstrap OP_SERVICE_ACCOUNT_TOKEN reaches
- # apply_onepassword_secrets() even in cron with no shell state; gitignored so the token never enters
- # the committed .env. override=False lets a systemd `EnvironmentFile=-…/.op.env` token win.
- op_env = home_path / ".op.env"
- if op_env.exists() and not os.environ.get("OP_SERVICE_ACCOUNT_TOKEN"):
- _load_dotenv_with_fallback(op_env, override=False, load_pass=load_pass)
- if project_env_path and project_env_path.exists():
- _load_dotenv_with_fallback(project_env_path, override=not user_env.exists(), load_pass=load_pass)
- loaded.append(project_env_path)
- # The override=True loads above wrote the raw .env line (``__BITWARDEN_MANAGED__`` placeholder, stale
- # token) back over a value an external source resolved on an earlier call, and the source pass below is a
- # once-per-home no-op — so the clobber stuck for the life of the process (#74265). Re-assert only what the
- # source is authoritative for; managed scope, applied last with override=True, still wins on purpose.
- if _SECRET_SOURCE_RESTORE_BY_HOME:
- for name, value in _SECRET_SOURCE_RESTORE_BY_HOME.get(str(home_path.resolve()), {}).items():
- if os.environ.get(name) != value:
- os.environ[name] = value
- # The config backend is selected here — after the .env files, before the first config.yaml read
- # (``_apply_external_secret_sources`` reads ``secrets:``) — so an unavailable backend stops the
- # process before any reader could fall back to defaults (config-config design §4.4, D11/D32). A
- # remote backend fetches this home's config here, failing closed; the file backend does nothing.
- from hermes_cli.config_backend import get_config_backend
- # The administrator-managed .env is part of the deployment the backend boots from (it may be
- # where the selector, plane URL or instance id live). The override=True loads above may have
- # replaced those names with the home's values, so re-publish the deployment names (only those)
- # with the bootstrap precedence -- managed last -- before the boot. The full managed overlay
- # still applies once, last below, where it keeps its precedence over secret-source results.
- bootstrap_deployment(home_path, project_env_path)
- get_config_backend().boot(home_path)
- # External sources are skipped for the updater (dotenv + managed env still load): ``update`` must not
- # import optional secret-manager libs (Bitwarden → cryptography → _rust.pyd) into the process replacing
- # that env on Windows, and a fresh retry after a deferred dependency install would otherwise make the
- # self-lock preflight exit 2 again.
- from hermes_cli import _early_recovery
- # External secret sources are skipped in two updater situations: 1. ``load_external_secrets=False`` —
- # the caller is an ``update`` invocation that must not import optional secret-manager libraries
- # (Bitwarden → cryptography → ``_rust.pyd``) into the process that replaces that same environment on
- # Windows (#73381, #86735). 2. A fresh ``hermes update`` retry just completed a deferred dependency
- # install before importing this module. Do not remap native secret-source dependencies in that same
- # updater process or the self-lock preflight will recreate the marker and exit 2 again. Dotenv and
- # managed env still load in both cases; only external source resolution is unnecessary for the updater.
- if load_external_secrets and not _early_recovery._should_skip_external_secret_sources():
- _apply_external_secret_sources(home_path)
- _apply_managed_env(load_pass=load_pass)
- # config.yaml owns terminal.*, but the override=True loads above let a stale TERMINAL_ENV=docker in
- # ~/.hermes/.env win on every reload and flip the backend mid-session in long-lived processes.
- # Re-apply the explicit terminal keys LAST, after the managed overlay, so the merged config lands.
- # config.yaml is the documented source of truth for terminal.* settings, but the dotenv loads above run
- # with override=True — so a stale TERMINAL_ENV=docker left in ~/.hermes/.env (e.g. written by an older
- # `hermes setup` before the user switched terminal.backend in config.yaml) silently wins again on every
- # reload. Startup launchers bridge config→env once, but long-lived processes (gateway per-turn reload,
- # cron standalone runs) call load_hermes_dotenv() repeatedly and used to flip the effective backend back
- # to the stale .env value mid-session (#29186, #67323).
- _reapply_terminal_config_bridge(home_path)
- return loaded
- def _reapply_terminal_config_bridge(home_path: Path) -> None:
- """Re-assert config.yaml's explicit ``terminal.*`` keys over reloaded .env via the single shared bridge
- ``apply_terminal_config_to_env`` (also used by terminal_tool and the TUI/dashboard launchers) so the
- semantics can't drift between sites."""
- try:
- if Path(home_path).resolve() != _process_hermes_home().resolve():
- return
- from hermes_cli.config import apply_terminal_config_to_env
- apply_terminal_config_to_env(env=None)
- except Exception:
- pass
- def _apply_managed_env(*, load_pass: int | None = None) -> None:
- """Apply the managed-scope .env last, with override, so it beats user/shell. Does NOT stop the agent
- from later mutating os.environ (v1 relies on filesystem permissions). Fail-open: never blocks startup."""
- try:
- from hermes_cli import managed_scope
- managed_dir = managed_scope.get_managed_dir()
- except Exception:
- return
- if managed_dir is None:
- return
- managed_env = managed_dir / ".env"
- if not managed_env.exists():
- return
- _sanitize_env_file_if_needed(managed_env)
- _load_dotenv_with_fallback(managed_env, override=True, load_pass=load_pass, managed=True)
- def _revoke_secret_source_writes(home_path: Path, *, keep) -> None:
- """Take back what a source wrote into ``os.environ`` for *home_path* once nothing backs it.
- ``keep(name, source)`` says which recorded writes are still owned. Every other write is revoked,
- but only while ``os.environ`` still holds exactly the value the source wrote: a value another owner
- replaced since (the profile's ``.env``, the administrator-managed ``.env``, a shell export, a later
- source) is theirs and stays. A revoked name gets back the value it held before the source's first
- write, or is removed when it had none."""
- home_key = str(Path(home_path).resolve())
- writes = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key)
- if not writes:
- return
- try:
- from agent.secret_scope import load_env_file
- owned_by_dotenv = set(load_env_file(Path(home_path) / ".env")) | _MANAGED_DOTENV_KEYS
- except Exception:
- owned_by_dotenv = set(_MANAGED_DOTENV_KEYS)
- kept: dict[str, tuple[str, str, str | None]] = {}
- for name, (source, value, prior) in writes.items():
- if keep(name, source):
- kept[name] = (source, value, prior)
- continue
- if name not in owned_by_dotenv and os.environ.get(name) == value:
- if prior is None:
- os.environ.pop(name, None)
- else:
- os.environ[name] = prior
- still_attributed = any(
- other.get(name, ("",))[0] == source
- for key, other in _SECRET_SOURCE_WRITES_BY_HOME.items() if key != home_key)
- if _SECRET_SOURCES.get(name) == source and not still_attributed:
- _SECRET_SOURCES.pop(name, None)
- if kept:
- _SECRET_SOURCE_WRITES_BY_HOME[home_key] = kept
- else:
- _SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
- def _refuse_protected_env_from_sources(report, backend=None, environ_before=None, env=None, *, routed_home=None) -> None:
- """D32: the config backend's own credential (which plane, which agent, which token) must never
- come from a secret source — a source that could change it could point config at another plane.
- Exits (the backend is unusable), even when the pre-existing value won.
- *backend* is the backend selected BEFORE the sources ran: judging by the backend selected after
- them would let a source that writes ``HERMES_CONFIG_BACKEND=file`` switch remote mode off and
- disarm this very check. The selector itself is never a source's to change, under any backend:
- a source's write to it is reverted in *env* (the mapping the sources wrote into; default
- ``os.environ``) to its *environ_before* value before anything else.
- Startup runs the sources against a private staging copy of the environment and calls this
- BEFORE publishing it (:func:`_publish_staged_env`), so a refused value is never visible to a
- concurrent request — the poller reads the plane URL and credential live from ``os.environ``."""
- if env is None:
- env = os.environ
- from hermes_cli.config_backend import BACKEND_ENV, ConfigBackendUnavailable, get_config_backend
- if backend is None:
- backend = get_config_backend()
- supplied = set(report.provenance)
- for src in report.sources:
- supplied.update(getattr(src, "skipped_existing", ()) or ())
- protected = backend.protected_env_names() | {BACKEND_ENV}
- clash = sorted(supplied & protected)
- if not clash:
- return
- if environ_before is not None: # undo the source's writes to protected names before deciding
- for name in clash:
- if name in environ_before:
- env[name] = environ_before[name]
- else:
- env.pop(name, None)
- if not backend.protected_env_names():
- # The file backend has no plane credential: only the selector clashed. Reverted, and dropped
- # from the report so no provenance/snapshot re-applies it later; keep going.
- for name in clash:
- report.provenance.pop(name, None)
- for src in report.sources:
- for bucket in (getattr(src, "applied", None), getattr(src, "skipped_existing", None)):
- if isinstance(bucket, list) and name in bucket:
- bucket.remove(name)
- print(f" Secret sources: ignored {BACKEND_ENV} from a secrets: source (the config backend is "
- "chosen by .env or the process environment only).", file=sys.stderr)
- return
- outcome = (f"Hermes does not serve profile home {routed_home} with it." if routed_home is not None
- else "Hermes does not start with it.")
- raise ConfigBackendUnavailable(
- f"A secrets: source supplies {', '.join(clash)}, which the {backend.name!r} config "
- "backend uses to reach its config plane. That credential must come from auth.json or .env, never "
- f"from a secret source; remove the mapping. {outcome}")
- def _publish_staged_env(staged: dict, environ_before: dict) -> None:
- """Copy into ``os.environ`` exactly the names the sources set in *staged* (value differs from
- *environ_before*). Sources only ever set names, so nothing is removed; a name nobody's source
- touched is left alone even if another thread changed it meanwhile."""
- for name, value in staged.items():
- if environ_before.get(name) != value:
- os.environ[name] = value
- def _apply_external_secret_sources(home_path: Path) -> None:
- """Pull secrets from every enabled external source into env — AFTER dotenv (sources need .env bootstrap
- tokens), BEFORE Hermes reads credentials; failures never block startup. Precedence/conflicts/provenance
- live in ``registry.apply_all``; this wrapper owns the once-per-home guard, the post-apply ASCII sweep,
- the ``_SECRET_SOURCES`` map and status lines."""
- home_key = str(Path(home_path).resolve())
- if home_key in _APPLIED_HOMES:
- return
- # Neither early return marks the home applied: a malformed config.yaml would otherwise permanently
- # disable secret loading for this process, and an unmarked home picks up a config change on the next
- # load (the signature-cached read is cheap).
- try:
- cfg = _load_secrets_config(home_path)
- except Exception:
- # See #40597.
- return
- # No source configured / enabled any more: whatever one wrote earlier is no longer backed (#126982
- # review). A config READ failure above keeps them — fail-open, same as the fetch-failure path below.
- if not cfg:
- _revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
- return
- # Defer the registry import until a source is enabled — bitwarden eagerly loads cryptography._rust.pyd,
- # which makes the Windows updater self-lock before its preflight. Detect by *shape* (dict with enabled
- # flag), not names, so plugin/test sources pass and a plain dict entry never forces the crypto load.
- any_enabled = any(isinstance(v, dict) and v.get("enabled") is True for v in cfg.values())
- if not any_enabled:
- _revoke_secret_source_writes(home_path, keep=lambda _name, _source: False)
- return
- try:
- from agent.secret_sources.registry import apply_all, enabled_source_names
- except ImportError:
- return
- # Revoke a removed/disabled source's writes BEFORE the pass: left in place, its stale value would make
- # a still-enabled source that supplies the same name skip it as pre-existing (``skipped_existing``).
- try:
- active = enabled_source_names(cfg, home_path)
- except Exception:
- active = None
- if active is not None:
- _revoke_secret_source_writes(home_path, keep=lambda _name, source: source in active)
- environ_before = dict(os.environ)
- # Latched before the sources run: a source must not be able to re-select the backend (D32).
- from hermes_cli.config_backend import get_config_backend
- backend_before = get_config_backend()
- # The sources write into a private staging copy, published only after the D32 check: written
- # straight into os.environ, a forbidden plane URL or credential would be live for any request
- # (the config poller, another thread's write) until the check reverted it.
- staged = dict(environ_before)
- try:
- report = apply_all(cfg, home_path, environ=staged)
- except Exception:
- return
- if not report.sources: # no source enabled: keep retrying cheaply so flipping one on takes effect
- return
- _refuse_protected_env_from_sources(report, backend_before, environ_before, env=staged)
- _publish_staged_env(staged, environ_before)
- # A real fetch attempt happened (success OR error): mark the home so the 3-5 import-time calls per
- # startup don't re-fetch / re-print (error retries are opt-in via reset_secret_source_cache()).
- # Marking AFTER the attempt keeps the earlier failure paths retryable.
- _APPLIED_HOMES.add(home_key)
- if report.applied_any:
- _sanitize_loaded_credentials() # vault values carry the same copy-paste corruption risk as .env
- for name, applied in report.provenance.items():
- _SECRET_SOURCES[name] = applied.source
- # Snapshot EVERY name a source supplied, not just the newly applied ones. A name the source supplied
- # but the pre-existing process value won (``skipped_existing``) is still this home's effective value
- # for it — and on the unscoped path os.environ IS this home's environment. Skipping those names
- # latched an EMPTY snapshot whenever the key was already in the env: after the first cron/plugin
- # re-pull (the previous apply's own write-back shadows every key) or from boot under systemd
- # ``EnvironmentFile=``. Under multiplex the scope is the only credential source, so an empty
- # snapshot failed every default-profile turn for the process lifetime (#102041).
- values: dict[str, str] = {}
- supplied = _record_supplied_names(report)
- for name in supplied:
- if name in os.environ:
- values[name] = os.environ[name]
- _record_secret_source_writes(home_key, report, supplied, environ_before)
- if values:
- _SECRET_SOURCE_VALUES_BY_HOME[home_key] = values
- _SECRET_SOURCE_RESTORE_BY_HOME[home_key] = {
- n: values[n] for n, a in report.provenance.items() if a.authoritative and n in values}
- for src in report.sources:
- if src.applied:
- print(f" {src.label}: applied {len(src.applied)} "
- f"secret{'s' if len(src.applied) != 1 else ''}", file=sys.stderr)
- if src.result.error:
- print(f" {src.label}: {src.result.error}", file=sys.stderr)
- hint = _remediation_hint(src.name, src.result.error_kind, cfg, scope=home_key)
- if hint:
- print(f" {src.label}: → {hint}", file=sys.stderr)
- for warn in src.result.warnings:
- print(f" {src.label}: {warn}", file=sys.stderr)
- for conflict in report.conflicts:
- print(f" Secret sources: {conflict}", file=sys.stderr)
- def _record_secret_source_writes(home_key: str, report, supplied: set[str], environ_before: dict) -> None:
- """Refresh ``_SECRET_SOURCE_WRITES_BY_HOME`` after one process-global pass, revoking a write its
- still-enabled source has stopped supplying (rotated out, mapping removed). A source whose fetch
- FAILED this pass keeps its earlier writes: a transient vault outage is not a revocation."""
- failed = {src.name for src in report.sources if not src.result.ok}
- previous = _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {})
- _revoke_secret_source_writes(
- Path(home_key),
- keep=lambda name, source: name in report.provenance or name in supplied or source in failed)
- writes = {name: rec for name, rec in _SECRET_SOURCE_WRITES_BY_HOME.get(home_key, {}).items()
- if name not in report.provenance and os.environ.get(name) == rec[1]}
- for name, applied in report.provenance.items():
- if name not in os.environ:
- continue
- before = environ_before.get(name)
- earlier = previous.get(name)
- # Re-applied over our own earlier write: the pre-source value is the one recorded then.
- prior = earlier[2] if earlier is not None and earlier[1] == before else before
- writes[name] = (applied.source, os.environ[name], prior)
- if writes:
- _SECRET_SOURCE_WRITES_BY_HOME[home_key] = writes
- else:
- _SECRET_SOURCE_WRITES_BY_HOME.pop(home_key, None)
- def _remediation_hint(source_name: str, error_kind, secrets_cfg: dict, *, scope: str | None = None) -> str:
- """The failed source's one-line fix-it hint; a plugin remediation() could raise and startup must not."""
- try:
- from agent.secret_sources.registry import get_source
- source = get_source(source_name, scope=scope)
- if source is None:
- return ""
- src_cfg = secrets_cfg.get(source_name)
- src_cfg = src_cfg if isinstance(src_cfg, dict) else {}
- return str(source.remediation(error_kind, src_cfg) or "").strip()
- except Exception:
- return ""
- def _load_secrets_config(home_path: Path) -> dict:
- """Read just the ``secrets:`` section of config.yaml, isolated so a malformed config can't break dotenv."""
- from hermes_cli.config_backend import config_exists, read_config_doc_readonly
- config_path = home_path / "config.yaml"
- if not config_exists(config_path):
- return {}
- # Prefer the shared raw-config cache: this is the first config.yaml read of a normal startup, so
- # populating it lets main.py's early bridge and hermes_logging reuse one parse instead of 3-4.
- if home_path == _process_hermes_home():
- try:
- from hermes_cli.config import read_raw_config
- data = read_raw_config() or {}
- return data.get("secrets") or {}
- except Exception:
- pass
- # Routed profiles re-enter their scope on every poll/turn; only re-parse after the file changed.
- try:
- data = read_config_doc_readonly(config_path) or {}
- except Exception:
- return {}
- return data.get("secrets") or {}
- def _process_hermes_home() -> Path:
- """The HERMES_HOME the running process was launched under.
- Must be the *true* process home, ignoring any context-local
- ``set_hermes_home_override`` a per-request task has installed. Both
- callers depend on that:
- * ``_reapply_terminal_config_bridge`` guards "only re-bridge config into
- the shared ``os.environ`` when THIS load is for the process's own
- profile". If this followed the task override, a per-turn handler scoped
- to a *secondary* profile (the multiplex dashboard serving every profile
- from one process) would satisfy the guard and bridge that profile's
- ``terminal.backend`` into the shared env — e.g. a ``local`` sibling
- profile clobbering the launch profile's ``TERMINAL_ENV=ssh`` while
- leaving its ``TERMINAL_SSH_*`` untouched, so the session silently runs
- commands locally (cross-profile terminal-backend leak).
- * ``_load_secrets_config`` uses it to decide whether the shared
- (mtime,size)-keyed config cache is safe to reuse; under an override it
- must fall through to an isolated parse of the scoped profile.
- ``hermes_constants.get_routing_process_hermes_home()`` is the override-immune
- resolver built for exactly this; delegate to it. It is also immune to a host
- that mirrors the served profile into the live ``HERMES_HOME`` env var
- (``pin_process_hermes_home``): without that, the mirrored profile satisfied
- the guard above and bridged ITS ``terminal.*`` into the shared env.
- """
- try:
- from hermes_constants import get_routing_process_hermes_home
- return get_routing_process_hermes_home()
- except Exception:
- return Path.home() / ".hermes"
|