| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576 |
- """Where the user config layer lives: the ``ConfigBackend`` seam (config-config design §4.1, D10/D11).
- Every read and write of a profile's user ``config.yaml`` goes through the backend returned by
- :func:`get_config_backend` (``scripts/check_config_yaml_readers.py`` and
- ``scripts/check_config_yaml_writers.py`` reject direct file access elsewhere). The key per call is
- the hermes home (profile): one multiplexed process serves N profiles, and every config cache
- already keys on the config path, so a backend keeps per-home state.
- :class:`FileBackend` (the default) is today's behaviour byte for byte:
- the version is the ``file_signature`` stat tuple, reads are ``open`` + ``fast_safe_load`` and raise
- exactly what those raise (``FileNotFoundError`` for an absent file, ``OSError``, YAML errors), and
- writes are the ruamel round-trip writers in ``utils``. Callers keep their own error handling, so a
- backend whose layer always exists simply never takes the "no file" branches. Every other kind
- (``HERMES_CONFIG_BACKEND=<kind>``, e.g. ``remote``) is the package ``plugins/config_backends/<kind>/``,
- whose ``get_backend()`` returns the instance; this module names no backend beyond that.
- The managed scope (``/etc/hermes``) is NOT a backend concern: it stays an overlay applied on top of
- whatever user layer the backend returns.
- """
- from __future__ import annotations
- import contextlib
- import contextvars
- import importlib
- import os
- import re
- import sys
- import threading
- from dataclasses import dataclass, field
- from pathlib import Path
- from typing import Any, Mapping, Optional, Protocol, Union, runtime_checkable
- CONFIG_FILENAME = "config.yaml"
- BACKEND_ENV = "HERMES_CONFIG_BACKEND"
- PathLike = Union[str, "os.PathLike[str]"]
- @dataclass(frozen=True)
- class UserLayer:
- """One read of a profile's user layer."""
- doc: Any # the parsed root as written (None for an empty document)
- version: tuple[Any, ...] # the cache signature this doc was read at (etag for a remote backend)
- locks: Mapping[str, str] = field(default_factory=dict) # dotted prefix -> level that locks it
- provenance: str = "" # where the doc came from, for diagnostics
- @dataclass(frozen=True)
- class Changes:
- """A write to one profile's user layer.
- ``document`` replaces the whole user-layer document (``atomic_config_write``: the file backend
- merges it onto the on-disk document, comment-preserving); ``set`` / ``unset`` are single dotted
- keys (``atomic_roundtrip_yaml_update`` semantics). A backend applies ``document`` first.
- """
- document: Optional[dict] = None
- set: Mapping[str, Any] = field(default_factory=dict)
- unset: tuple[str, ...] = ()
- extra_content_on_create: Optional[str] = None
- @runtime_checkable
- class ConfigBackend(Protocol):
- name: str
- def read_user_doc(self, home: Path) -> Any:
- """The parsed user layer as written (None for an empty document); version checks are separate."""
- ...
- def read_user_doc_readonly(self, home: Path) -> Any:
- """``read_user_doc(home)`` re-read only when ``version`` changes; never mutate it."""
- ...
- def version(self, home: Path) -> tuple[Any, ...]:
- """Cheap change signature; changes whenever the user layer does."""
- ...
- def cache_version(self, home: Path, version: tuple[Any, ...]) -> tuple[Any, ...]:
- """``version`` (just taken) plus whatever a cache needs to tell apart writes it cannot see."""
- ...
- def exists(self, home: Path) -> bool: ...
- def probe_readable(self, home: Path) -> None:
- """Raise ``OSError`` while the user layer cannot be read; no parse."""
- ...
- def write_changes(self, home: Path, changes: Changes) -> None: ...
- def locked(self, home: Path, dotted: str) -> Optional[str]:
- """The level that locks ``dotted`` (prefix-aware), or None."""
- ...
- def supports_file_tooling(self) -> bool:
- """Whether tools that copy/edit/back up the config FILE (profile clone, backup/restore,
- ``config edit``, last-known-good backups, ``migrate_config`` write-back) may run."""
- ...
- def honors_managed_config(self) -> bool:
- """Whether the managed ``/etc/hermes/config.yaml`` overlay applies (D18: not in remote mode)."""
- ...
- def boot(self, home: Path) -> None:
- """Called once per ``load_hermes_dotenv`` after the ``.env`` loads and before the first
- config read (design §4.4): a backend that fetches config does it here, failing closed."""
- ...
- def protected_env_names(self) -> frozenset:
- """Env names no external secret source may supply (the plane credential, D32)."""
- ...
- def deployment_env_names(self) -> frozenset:
- """Non-secret env names that select and address this backend (see :func:`deployment_env_names`)."""
- ...
- def bootstrap_env_names(self) -> frozenset:
- """Env names published from the launch home's and the managed ``.env`` before the first
- config read (design §4.4 steps 1-2): what this backend needs to select, address and
- authenticate itself before any config exists."""
- ...
- def preserve_read_base(self) -> contextlib.AbstractContextManager:
- """A context in which a read does not become the base a later document save is diffed
- against (see :func:`keeping_read_base`)."""
- ...
- def doctor_warnings(self, home: Path) -> tuple:
- """``(warning, hint, issue)`` triples ``hermes doctor`` reports after a successful read."""
- ...
- def apply_in_memory(self, home: Path, doc: dict) -> None:
- """Replace the user layer in memory only (in-memory migration, D12). Only called for a
- backend without file tooling; nothing is sent or written."""
- ...
- def describe(self, home: Path) -> str:
- """One line naming where the user layer comes from and its state (status line, doctor)."""
- ...
- def destination(self, home: Path) -> str:
- """Where a write lands, for a user-facing "saved in ..." line (no state, no network)."""
- ...
- class ConfigBackendUnavailable(SystemExit):
- """The selected backend cannot serve config. A ``SystemExit`` on purpose: many config readers
- fail open with ``except Exception`` → defaults, and serving defaults instead of the selected
- backend's config is exactly the silent failure this must not become."""
- class ConfigWriteError(RuntimeError):
- """A config write the backend refused or could not complete; nothing was changed. ``code`` is
- a machine code (the plane's ``error`` for a remote refusal)."""
- def __init__(self, message: str, *, code: str = "config_write_failed") -> None:
- super().__init__(message)
- self.code = code
- class ConfigLockedError(ConfigWriteError):
- """The write touches a path locked by an upper level (D7, D10)."""
- def __init__(self, path: str, locked_by: str, message: Optional[str] = None) -> None:
- super().__init__(message or f"{path} is locked by the {locked_by} level", code="config_key_locked")
- self.path = path
- self.locked_by = locked_by
- class ConfigValueError(ConfigWriteError):
- """A value the backend cannot store (a YAML-only type, a secret literal); nothing was sent."""
- def __init__(self, message: str, *, code: str = "config_value_invalid") -> None:
- super().__init__(message, code=code)
- class FileBackend:
- """The user layer is ``<home>/config.yaml`` on local disk."""
- name = "file"
- # Path-level primitives: the file backend's whole behaviour, also used for explicit files
- # that are not a home's config.yaml (see ``_route``).
- @staticmethod
- def version_path(path: Path) -> tuple[int, int, int, int]:
- st = path.stat() # before the import: a missing file must not need the utils stack (PM runtime)
- from utils import file_signature
- return file_signature(st)
- @staticmethod
- def read_path(path: Path) -> Any:
- from utils import fast_safe_load
- with open(path, encoding="utf-8-sig") as f:
- return fast_safe_load(f)
- @staticmethod
- def read_path_readonly(path: Path) -> Any:
- from utils import load_yaml_file_readonly
- return load_yaml_file_readonly(path)
- @staticmethod
- def write_path(path: Path, changes: Changes) -> None:
- from utils import atomic_roundtrip_yaml_save, atomic_roundtrip_yaml_update
- if changes.document is not None:
- atomic_roundtrip_yaml_save(path, changes.document, extra_content_on_create=changes.extra_content_on_create)
- for key, value in changes.set.items():
- atomic_roundtrip_yaml_update(path, key, value)
- for key in changes.unset:
- atomic_roundtrip_yaml_update(path, key, None)
- def config_path(self, home: Path) -> Path:
- return Path(home) / CONFIG_FILENAME
- def read_user_doc(self, home: Path) -> Any:
- return self.read_path(self.config_path(home))
- def read_user_doc_readonly(self, home: Path) -> Any:
- return self.read_path_readonly(self.config_path(home))
- def version(self, home: Path) -> tuple[int, int, int, int]:
- return self.version_path(self.config_path(home))
- def cache_version(self, home: Path, version: tuple[Any, ...]) -> tuple[Any, ...]:
- # utils.file_cache_version's rule. ``version`` is config_version's opaque token, so the
- # racy test takes the file's own signature.
- from utils import racy_content_digest
- return (*version, racy_content_digest(self.config_path(home), self.version(home)))
- def exists(self, home: Path) -> bool:
- return self.config_path(home).exists()
- def probe_readable(self, home: Path) -> None:
- path = self.config_path(home)
- with open(path, "rb") as f:
- f.read(1)
- def write_changes(self, home: Path, changes: Changes) -> None:
- self.write_path(self.config_path(home), changes)
- def locked(self, home: Path, dotted: str) -> Optional[str]:
- return None # a local file has no levels; the managed scope is a separate overlay
- def supports_file_tooling(self) -> bool:
- return True
- def honors_managed_config(self) -> bool:
- return True
- def boot(self, home: Path) -> None:
- return None
- def protected_env_names(self) -> frozenset:
- return frozenset()
- def deployment_env_names(self) -> frozenset:
- return frozenset()
- def bootstrap_env_names(self) -> frozenset:
- return frozenset()
- def preserve_read_base(self) -> contextlib.AbstractContextManager:
- return contextlib.nullcontext()
- def doctor_warnings(self, home: Path) -> tuple:
- return ()
- def apply_in_memory(self, home: Path, doc: dict) -> None:
- raise NotImplementedError("the file backend persists migrations to config.yaml")
- def describe(self, home: Path) -> str:
- return str(self.config_path(home))
- def destination(self, home: Path) -> str:
- return str(self.config_path(home))
- _FILE_BACKEND = FileBackend()
- # Reentrant: publishing the deployment can import modules whose import-time config read selects
- # the backend again on the SAME thread (``_BOOTSTRAP_OWNER``); every other thread waits for it.
- _BOOTSTRAP_LOCK = threading.RLock()
- _BOOTSTRAPPED = False
- _BOOTSTRAP_OWNER: Optional[int] = None
- def _bootstrap_names(kind: str) -> frozenset:
- """The env names backend *kind* (as the ``.env`` layers select it) needs before the first read;
- none for a kind this build lacks (selection refuses it afterwards)."""
- try:
- return _backend_for(kind).bootstrap_env_names()
- except ConfigBackendUnavailable:
- return frozenset()
- def _ensure_bootstrap_env() -> None:
- """Once per process, before the first selection: the deployment that selects, addresses and
- authenticates the backend comes from the process env, the launch home's ``.env`` and the
- managed ``.env`` (design §4.4 steps 1-2, D26/D32) — also when the first config read comes before
- ``load_hermes_dotenv`` (``hermes_cli.config`` reads config at import time)."""
- if not _BOOTSTRAPPED:
- _bootstrap(None, None, once=True)
- def bootstrap_deployment(home: Optional[PathLike] = None, project_env: Optional[PathLike] = None) -> None:
- """Publish the deployment from an explicit dotenv load's layers (``load_hermes_dotenv``: its
- home and project ``.env``) before that load's first config read. Runs even when the
- once-per-process bootstrap already ran: an earlier one did not know this load's project file."""
- _bootstrap(Path(home) if home is not None else None,
- Path(project_env) if project_env is not None else None, once=False)
- def _bootstrap(home: Optional[Path], project_env: Optional[Path], *, once: bool) -> None:
- global _BOOTSTRAPPED, _BOOTSTRAP_OWNER
- with _BOOTSTRAP_LOCK: # another thread's bootstrap in progress: wait for it to publish
- if (once and _BOOTSTRAPPED) or _BOOTSTRAP_OWNER == threading.get_ident():
- return # done, or this thread's own bootstrap re-entering (import-time config read)
- _BOOTSTRAP_OWNER = threading.get_ident()
- try:
- from hermes_cli.env_loader import apply_config_bootstrap_env
- # names resolved only for the kind the .env layers select
- apply_config_bootstrap_env(_bootstrap_names, home, project_env)
- except ImportError: # no usable python-dotenv (a bare installer interpreter): no .env loads either
- pass
- finally:
- # Complete only once the deployment is published: a concurrent first reader that saw
- # the flag earlier would select (and read) the file backend of a remote deployment.
- _BOOTSTRAP_OWNER = None
- _BOOTSTRAPPED = True
- def get_config_backend() -> ConfigBackend:
- """The backend selected by ``HERMES_CONFIG_BACKEND`` (default ``file``).
- Read from the environment on every call, before any config read, so no config value can
- select it (D11). Raises :class:`ConfigBackendUnavailable` for a backend this build lacks.
- """
- _ensure_bootstrap_env()
- return _backend_for(os.environ.get(BACKEND_ENV, "").strip().lower() or "file")
- # A kind is one package name under plugins/config_backends/: never a dotted or relative path.
- _KIND_RE = re.compile(r"[a-z][a-z0-9_]{0,63}")
- _BACKENDS_PACKAGE = "plugins.config_backends"
- def _backend_for(kind: str) -> ConfigBackend:
- """``file``, or the instance ``plugins.config_backends.<kind>.get_backend()`` returns. The
- package is imported directly, never through the general plugin loader (which reads config, D11)."""
- if kind == "file":
- return _FILE_BACKEND
- unknown = f"{BACKEND_ENV}={kind!r} is not a config backend this build provides (expected 'file' or 'remote')."
- if not _KIND_RE.fullmatch(kind):
- raise ConfigBackendUnavailable(unknown)
- name = f"{_BACKENDS_PACKAGE}.{kind}"
- module = sys.modules.get(name)
- if module is None:
- try:
- module = importlib.import_module(name)
- except ImportError as exc:
- if exc.name not in (name, _BACKENDS_PACKAGE, "plugins"):
- raise # the backend exists but is broken: show that, not "unknown"
- raise ConfigBackendUnavailable(unknown) from exc
- get_backend = getattr(module, "get_backend", None)
- if not callable(get_backend):
- raise ConfigBackendUnavailable(unknown)
- return get_backend()
- class _ExplicitFile(FileBackend):
- """A config file that is not a home's user layer (a test fixture, an import source): the file
- backend's behaviour on that exact path, whichever backend is selected."""
- def __init__(self, path: Path):
- self._path = path
- def config_path(self, home: Path) -> Path:
- return self._path
- def _route(config_path: PathLike) -> tuple[ConfigBackend, Path]:
- """``(backend, home)`` serving a config path through the one ``ConfigBackend`` surface.
- ``<home>/config.yaml`` is that home's user layer and goes to the selected backend. Any other
- file name is an explicit file, served by the file backend on that path.
- """
- path = Path(config_path)
- if path.name == CONFIG_FILENAME:
- return get_config_backend(), path.parent
- return _ExplicitFile(path), path.parent
- def local_file_layer(config_path: PathLike) -> tuple[ConfigBackend, Path]:
- """``(backend, home)`` for the LOCAL file at *config_path*, whichever backend is selected.
- Only for PM's dependency runtime: it publishes plugin selections to the local file and
- compare-and-swaps those bytes, and it carries neither the remote backend's client nor the
- plane credential. Everything else goes through :func:`_route`."""
- path = Path(config_path)
- return _ExplicitFile(path), path.parent
- def read_config_doc(config_path: PathLike) -> Any:
- """Parsed root of a config file (None when empty); raises like ``open`` + ``fast_safe_load``."""
- backend, home = _route(config_path)
- return backend.read_user_doc(home)
- def read_config_doc_readonly(config_path: PathLike) -> Any:
- """Signature-cached :func:`read_config_doc`; the result is shared — never mutate it."""
- backend, home = _route(config_path)
- return backend.read_user_doc_readonly(home)
- def config_version(config_path: PathLike) -> tuple[Any, ...]:
- """Cache signature of a config file; raises ``FileNotFoundError`` / ``OSError`` like ``stat``."""
- backend, home = _route(config_path)
- return backend.version(home)
- def config_cache_version(config_path: PathLike) -> tuple[Any, ...]:
- """Signature for a cache of a config file's parse: :func:`config_version` can stay equal across a
- same-tick same-size rewrite, which would pin the older parse for the process lifetime."""
- version = config_version(config_path) # the one stat; its errors (and test seams) stay config_version's
- backend, home = _route(config_path)
- return backend.cache_version(home, version)
- def config_cache_version_or_none(config_path: PathLike) -> Optional[tuple[Any, ...]]:
- """:func:`config_cache_version`, or None when the config is absent or cannot be read."""
- try:
- return config_cache_version(config_path)
- except OSError:
- return None
- def config_version_or_none(config_path: PathLike) -> Optional[tuple[Any, ...]]:
- """:func:`config_version`, or None when the config is absent or cannot be stat'ed."""
- try:
- return config_version(config_path)
- except OSError:
- return None
- def version_token(version: Optional[tuple[Any, ...]]) -> float:
- """A number that changes with a backend ``version`` (0 = no config), for clients that only
- compare successive values: the file backend's version leads with ``st_mtime_ns``, kept as seconds.
- Any other int-led version is scaled the same way, which is fine: only changes are compared."""
- if not version:
- return 0
- if isinstance(version[0], int):
- return version[0] / 1_000_000_000
- import zlib
- return float(zlib.crc32(repr(version).encode("utf-8")) or 1)
- def probe_config_readable(config_path: PathLike) -> None:
- """Raise ``OSError`` while a config file cannot be read; no parse."""
- backend, home = _route(config_path)
- backend.probe_readable(home)
- def config_exists(config_path: PathLike) -> bool:
- backend, home = _route(config_path)
- return backend.exists(home)
- def write_config_document(config_path: PathLike, document: dict, *, extra_content_on_create: Optional[str] = None) -> None:
- """Replace a config file's document (comment-preserving merge for the file backend)."""
- backend, home = _route(config_path)
- backend.write_changes(home, Changes(document=document, extra_content_on_create=extra_content_on_create))
- def config_destination(config_path: PathLike) -> str:
- """Where a write to *config_path* lands, for "✓ Set ... in <here>" lines: the file path for
- the file backend and explicit files, the backend's own name for the user layer otherwise."""
- backend, home = _route(config_path)
- return backend.destination(home)
- def write_config_key(config_path: PathLike, key_path: str, value: Any) -> None:
- """Set one dotted key (``None`` removes it) — ``atomic_roundtrip_yaml_update`` semantics."""
- write_config_keys(config_path, {key_path: value})
- def write_config_keys(config_path: PathLike, updates: Mapping[str, Any]) -> None:
- """Set several related dotted keys (``None`` removes one) as ONE write.
- A remote backend sends them as one CAS PATCH, so a refusal of any of them (a lock, a value
- check) changes none of them: a grouped edit such as a model switch (default + provider + ...)
- never lands half-applied. The file backend applies them key by key, as before."""
- backend, home = _route(config_path)
- backend.write_changes(home, Changes(set={k: v for k, v in updates.items() if v is not None},
- unset=tuple(k for k, v in updates.items() if v is None)))
- def keeping_read_base(read):
- """Run ``read()`` (a pre-write guard read) without making it the read a later document save
- is diffed against: the caller's document came from ITS earlier read (Remote Config). A
- context that never read keeps the guard read. No-op for the file backend."""
- with get_config_backend().preserve_read_base():
- return read()
- # Dotted prefixes a document save in this context must write whole: a lock on a changed key under
- # one refuses the save (ConfigLockedError) instead of being silently omitted from it.
- _REFUSE_LOCKED_OMISSIONS: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar(
- "config_refuse_locked_omissions", default=())
- @contextlib.contextmanager
- def refusing_locked_omissions(*prefixes: str):
- """Within the block, a document save that changes a key under one of *prefixes* (dotted, e.g.
- ``"model"``) and finds it locked raises :class:`ConfigLockedError` and sends nothing, instead
- of dropping the locked key and landing the rest (Remote Config bulk-save rule). Checked against
- the locks the save itself sees, so a lock published after an earlier preflight still refuses;
- a CAS retry re-checks every key it sends. No-op for the file backend (no locks)."""
- token = _REFUSE_LOCKED_OMISSIONS.set(_REFUSE_LOCKED_OMISSIONS.get() + tuple(prefixes))
- try:
- yield
- finally:
- _REFUSE_LOCKED_OMISSIONS.reset(token)
- def refused_locked_omission_prefixes() -> tuple[str, ...]:
- return _REFUSE_LOCKED_OMISSIONS.get()
- # Dotted keys a document save in this context stores at the user layer even when the value equals
- # what an upper level already supplies (an explicit `config set`): the file backend always writes
- # them, and a backend with levels must not drop them as "no change".
- _PINNED_KEYS: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar("config_pinned_keys", default=())
- @contextlib.contextmanager
- def pinning_keys(*keys: str):
- """Within the block, a document save stores *keys* (dotted) at the user layer even when
- unchanged from the inherited value. No-op for the file backend (it writes the whole file)."""
- token = _PINNED_KEYS.set(_PINNED_KEYS.get() + tuple(keys))
- try:
- yield
- finally:
- _PINNED_KEYS.reset(token)
- def pinned_keys() -> tuple[str, ...]:
- return _PINNED_KEYS.get()
- def write_changes(config_path: PathLike, changes: Changes) -> None:
- """One :class:`Changes` write. Unlike :func:`write_config_keys`, a ``None`` in ``set`` stores
- an explicit null (Remote Config: "no value", masking an inherited scalar); the file backend
- writes it as YAML null."""
- backend, home = _route(config_path)
- backend.write_changes(home, changes)
- def deployment_env_names() -> frozenset:
- """The selected backend's deployment env names (which backend, which plane, which instance).
- The deployment is a property of the instance, not of a profile, but it usually arrives in the
- launch profile's ``.env``, which a child acting for ANOTHER profile drops as launch residue
- (``strip_launch_profile_env``). Such a child re-reads only its own profile's ``.env`` and would
- silently select the file backend, so these names are never residue. Credentials are not part
- of it: the child resolves its own, and fails closed without them."""
- return get_config_backend().deployment_env_names()
- def supports_file_tooling() -> bool:
- """Gate for tools that copy, edit or back up the config FILE itself (§4.3, §4.7)."""
- return get_config_backend().supports_file_tooling()
- def require_file_tooling(what: str) -> None:
- """Refuse *what* (a file-copy tool) with a clear message when the backend has no config file."""
- if not supports_file_tooling():
- raise ConfigBackendUnavailable(
- f"{what} copies the local config.yaml, and the {get_config_backend().name!r} config "
- "backend has none. It is not available with this backend.")
|