"""The remote config backend (config-config design §4, contract.md §11, §15). The user layer of each profile home is the effective document the config plane resolves for (this instance, that profile). It lives in memory only: fetched at boot (fail closed, D2), kept fresh by a per-process ETag poller (D9), and written back as key-level changes on the profile level with CAS (D10). Nothing is read from or written to a local ``config.yaml``. """ from __future__ import annotations import contextlib import contextvars import copy import itertools import logging import os import random import sys import threading import time from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Tuple from hermes_cli.config_backend import ( Changes, ConfigBackendUnavailable, ConfigLockedError, ConfigValueError, ConfigWriteError, UserLayer, pinned_keys, refused_locked_omission_prefixes) from . import client from .credentials import PLANE_CREDENTIAL_ENV_NAMES, PlaneCredentialError from .diff import (_get_path, _pop_path, _set_path, apply_intent, diff, encode_changes, intent_diff, json_equal, strip_locked, write_check) from .paths import Path as KeyPath from .paths import PathError, covering_lock, covers, decode, encode, from_dotted from .values import secret_literal_path, to_wire logger = logging.getLogger(__name__) POLL_SECONDS = 300.0 # contract §11.3; a plane-served interval is a later change MIN_POLL_SECONDS = 30.0 # contract §11.3: lower values are clamped BOOT_RETRY_DELAYS = (10.0, 20.0) # contract §11.4: attempts at t = 0, ~10 s, ~30 s MAX_RETRY_WAIT = 20.0 def poll_interval() -> float: return max(POLL_SECONDS, MIN_POLL_SECONDS) def _latest_config_version() -> int: from hermes_cli.config_defaults import DEFAULT_CONFIG version = DEFAULT_CONFIG.get("_config_version") return version if isinstance(version, int) else 1 def _log_notes(notes: list[str]) -> None: """Log messages collected while a backend lock was held (never log under one: a log handler can read config, and a thread holding the handler's lock may be waiting for that lock).""" for note in notes: if note.startswith("debug:"): logger.debug("%s", note[len("debug:"):]) else: logger.warning("%s", note) class _FetchFailed(Exception): def __init__(self, detail: str, *, retryable: bool, retry_after: Optional[float] = None, status: Optional[int] = None) -> None: super().__init__(detail) self.retryable = retryable self.retry_after = retry_after self.status = status @dataclass class _ProfileState: home: Path profile: str server_config: dict[str, Any] = field(default_factory=dict) # EffectiveResponse.config, as sent doc: dict[str, Any] = field(default_factory=dict) # what readers get (+ _config_version, migrated) etag: str = "" profile_version: int = 0 # The profile level's stored writerConfigVersion: None = stored null, _UNKNOWN = not reported. profile_writer: Any = None other_writers: list[int] = field(default_factory=list) # the other levels' int stamps (R6) locks: list[tuple[KeyPath, str]] = field(default_factory=list) provenance: dict[str, str] = field(default_factory=dict) changed_ns: int = 0 gen: int = 0 installed: int = 0 # bumped by every _install; a GET started before a newer install is stale base: Optional[dict[str, Any]] = None # the migrated doc writes diff against (None = server_config) postprocessed: bool = False # st.doc is the completed migration output of generation `installed` fetched_at: float = 0.0 last_error: Optional[str] = None next_poll: float = 0.0 lock: threading.RLock = field(default_factory=threading.RLock) _UNKNOWN = object() @dataclass class _Private: """An in-memory migration's working copy of one profile's document (D12). The migration steps read and persist through ``hermes_cli.config`` like any reader and writer; while one runs, THAT thread sees this copy instead of the published one (``_PRIVATE``), and everyone else keeps seeing the published doc with its matching diff base. The migrated doc and its base are published together, in one step, when the migration ends: a concurrent writer never diffs a half-migrated document against the unmigrated base, or the reverse.""" st: _ProfileState doc: dict[str, Any] serial: int gen: int = 0 _PRIVATE: contextvars.ContextVar[Optional[_Private]] = contextvars.ContextVar("remote_config_private", default=None) _PRIVATE_SERIAL = itertools.count(1) # The diff base each profile had when THIS context last read it: {id(state): base}. A document # save means "turn what I read into this", so its intent is diffed against that read, never # against whatever a poll (or another writer) installed between the read and the save. Replaced, # never mutated: copied contexts (threads, tasks) must not share one dict. _READ_BASE: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar("remote_config_read_base", default=None) def _remember_read(st: _ProfileState) -> None: """Record (by reference: ``st.base`` / ``st.server_config`` are replaced, never mutated) the base this context just read under st.lock.""" seen = dict(_READ_BASE.get() or {}) seen[id(st)] = (st, st.base if st.base is not None else st.server_config) _READ_BASE.set(seen) def _read_base_for(st: _ProfileState) -> Optional[dict[str, Any]]: held = (_READ_BASE.get() or {}).get(id(st)) if held is None or held[0] is not st or held[1] is None: return None # never read in this context, or a different (reset) profile state base = copy.deepcopy(held[1]) base.pop("_config_version", None) return base def _private_for(st: _ProfileState) -> Optional[_Private]: m = _PRIVATE.get() return m if m is not None and m.st is st else None # Plane refusals caused by the submitted value (contract §2.5, §11.1): surfaced to the caller as # ConfigValueError with the plane's code. config_request_invalid is deliberately absent: a request # the plane cannot parse is this client's bug, not the user's value, so it stays opaque. _VALUE_REFUSALS = frozenset({ (400, "config_secret_literal"), (400, "config_value_invalid"), (400, "config_path_invalid"), (400, "config_path_reserved"), (413, "config_level_too_large"), (413, "config_body_too_large"), }) def _profile_writer(body: dict[str, Any]) -> Any: """The profile level's stored ``writerConfigVersion`` (int or None), or ``_UNKNOWN``.""" for lv in body.get("levels") or []: if isinstance(lv, dict) and lv.get("kind") == "profile": w = lv.get("writerConfigVersion") return w if w is None or isinstance(w, int) else _UNKNOWN return _UNKNOWN def _effective_ok(body: dict[str, Any]) -> bool: return (isinstance(body.get("config"), dict) and isinstance(body.get("profileVersion"), int) and isinstance(body.get("locks"), list)) class RemoteBackend: name = "remote" def __init__(self) -> None: self._states: dict[str, _ProfileState] = {} self._fetch_locks: dict[str, threading.Lock] = {} self._lock = threading.Lock() self._local = threading.local() # .booting: the home keys this thread is loading self._poller: Optional[threading.Thread] = None self._poller_pid: Optional[int] = None self._stop = threading.Event() self._managed_warned = False self._unknown_warned: set = set() # --- homes and state -------------------------------------------------------------------- @staticmethod def _key(home: Path) -> str: from hermes_constants import hermes_home_key return hermes_home_key(home) @staticmethod def _profile_for(home: Path) -> str: from hermes_constants import PROFILE_ID_RE, profile_name_for_home name = profile_name_for_home(home) if not name or not PROFILE_ID_RE.match(name): raise ConfigBackendUnavailable( f"Remote Config: {home} is not a Hermes profile home, so it has no remote config " "(HERMES_CONFIG_BACKEND=remote serves profile homes only).") return name def _state(self, home: Path) -> _ProfileState: key = self._key(home) st = self._states.get(key) if st is not None: poller = self._poller # A forked child has no poller thread, and a poller that died must not leave every # cached profile without updates until the next new profile happens to re-arm it. if poller is None or self._poller_pid != os.getpid() or not poller.is_alive(): self._ensure_poller() return st if key in self._booting(): # This thread is loading this profile (resolving its credential or fetching): the read # can only be served by the load it is part of. Waiting would wait on itself forever. raise ConfigBackendUnavailable( f"Remote Config: a config read for {home} re-entered the load of that profile's config " "on the same thread (code on the boot path read config before it was loaded). This is " "a Hermes bug; refusing the read instead of deadlocking.") with self._lock: fetch_lock = self._fetch_locks.setdefault(key, threading.Lock()) booting = self._booting() booting.add(key) try: st = self._boot_fetch(Path(home), key, fetch_lock) finally: booting.discard(key) self._ensure_poller() return st def _booting(self) -> set[str]: """Home keys this thread is loading now (a thread-local set).""" keys = getattr(self._local, "booting", None) if keys is None: keys = self._local.booting = set() return keys def _roster(self) -> list[_ProfileState]: with self._lock: return list(self._states.values()) def _boot_fetch(self, home: Path, key: str, fetch_lock: threading.Lock) -> _ProfileState: """Boot fetch with bounded retry (contract §11.4); fails closed (D2). Each attempt resolves the credential first, holding no backend lock, then takes the fetch lock only for the GET: other threads wait there for the fetch in flight and share its result. Nothing is logged and nobody is waited for while that lock is held.""" profile = self._profile_for(home) if not client.instance_id(): raise ConfigBackendUnavailable( f"Remote Config: {client.INSTANCE_ENV} is not set, so Hermes cannot identify this agent " "to the config plane. Hermes does not start without its remote config.") delays = list(BOOT_RETRY_DELAYS) while True: notes: list[str] = [] try: token = self._bearer(home) with fetch_lock: st = self._states.get(key) if st is None: st = _ProfileState(home=home, profile=profile) self._fetch_into(st, conditional=False, token=token, notes=notes) with self._lock: # the poller snapshots the roster under the same lock self._states[key] = st _log_notes(notes) return st except _FetchFailed as exc: _log_notes(notes) if exc.retryable and delays: wait = min(max(exc.retry_after or 0.0, delays.pop(0)), MAX_RETRY_WAIT) logger.warning("Remote Config: fetch for profile %r failed (%s); retrying in %.0fs", profile, exc, wait) time.sleep(wait) continue raise ConfigBackendUnavailable( f"Remote Config: cannot load the config for profile {profile!r} from " f"{client.shown_url()}: {exc}. Hermes does not start without its remote config " "(HERMES_CONFIG_BACKEND=remote has no local fallback).") from exc @staticmethod def _bearer(home: Path) -> str: try: return client.bearer(home) except PlaneCredentialError as exc: raise _FetchFailed(str(exc), retryable=exc.retryable) from exc def _fetch_into(self, st: _ProfileState, *, conditional: bool, token: Optional[str] = None, notes: Optional[list[str]] = None) -> bool: """GET ``/self`` into ``st``; True when the document changed. Raises :class:`_FetchFailed`. The GET runs outside ``st.lock`` (a slow plane must not stall writers), so a write can install a newer document while it is in flight. Its response is then stale, even when its ``profileVersion`` matches (an upper level may have changed in between): it is dropped, and the next poll fetches again. Messages go to ``notes`` (logged by the caller once it holds no backend lock), or are logged here when there is no list.""" if notes is None: notes = [] try: return self._fetch_into(st, conditional=conditional, token=token, notes=notes) finally: _log_notes(notes) started = st.installed try: resp = client.request("GET", st.home, st.profile, etag=st.etag if conditional else None, token=token) except PlaneCredentialError as exc: raise _FetchFailed(str(exc), retryable=exc.retryable) from exc except client.TransportError as exc: raise _FetchFailed(str(exc), retryable=True) from exc if resp.status == 304 and conditional: st.fetched_at, st.last_error = time.time(), None return False if resp.status == 200 and _effective_ok(resp.body): with st.lock: if st.installed != started: notes.append(f"debug:Remote Config: dropped a fetch for profile {st.profile!r} that a write overtook") return False notes.extend(self._install(st, resp.body, resp.etag)) return True if resp.status == 200: raise _FetchFailed("the plane returned a malformed response", retryable=True) retryable = resp.status >= 500 or resp.status == 429 raise _FetchFailed(f"HTTP {resp.status} {resp.error}: {resp.message}", retryable=retryable, retry_after=resp.retry_after, status=resp.status) def _install(self, st: _ProfileState, body: dict[str, Any], etag: Optional[str]) -> list[str]: """Install a fetched document (the caller holds st.lock); returns warnings to log later.""" notes: list[str] = [] config = body["config"] doc = copy.deepcopy(config) doc.pop("_config_version", None) # reserved on the wire (§3.6); never expected here # R6: the in-memory schema version is the oldest writer's; migrations run in memory (D12). writer_versions: list[int] = [ lv["writerConfigVersion"] for lv in body.get("levels") or [] if isinstance(lv, dict) and isinstance(lv.get("writerConfigVersion"), int)] doc["_config_version"] = min(writer_versions) if writer_versions else _latest_config_version() locks: list[tuple[KeyPath, str]] = [] for entry in body.get("locks") or []: try: locks.append((decode(entry["path"]), str(entry["level"]))) except (KeyError, TypeError, PathError): notes.append(f"Remote Config: ignoring malformed lock entry {entry!r}") st.server_config = config st.doc = doc st.etag = etag or str(body.get("etag") or "") st.profile_version = int(body["profileVersion"]) st.profile_writer = _profile_writer(body) st.other_writers = [ lv["writerConfigVersion"] for lv in body.get("levels") or [] if isinstance(lv, dict) and lv.get("kind") != "profile" and isinstance(lv.get("writerConfigVersion"), int)] st.locks = locks st.provenance = dict(body.get("provenance") or {}) st.changed_ns = time.time_ns() st.gen += 1 st.installed += 1 st.base = None st.postprocessed = False st.fetched_at, st.last_error = time.time(), None st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1) return notes _MAX_POSTPROCESS = 8 # passes when polls keep installing newer docs mid-migration def _completed(self, st: _ProfileState) -> bool: """Bring the published doc to its completed postprocess output; False when that is not possible yet (``hermes_cli.config`` still importing during the boot fetch).""" if _private_for(st) is not None: return True # a migration's own reads use its private copy (never another pass) for _ in range(self._MAX_POSTPROCESS): ready = self._postprocess(st) if not ready: return False with st.lock: if st.postprocessed: return True raise ConfigWriteError("Remote Config: the profile's config kept changing while it was migrated in memory", code="config_version_conflict") def _postprocess(self, st: _ProfileState) -> bool: """First read after a change: warn on unknown keys and migrate in memory (D12). Runs on a read, not at fetch, because both need ``hermes_cli.config``, which may still be importing when the boot fetch runs (then False: nothing could be done). No single flight: a reader or writer that finds the published doc unmigrated runs its own private pass, even while another thread's pass is in flight, and the first pass to finish publishes. Waiting for the other pass instead could deadlock (a migration reads through hermes_cli.config's _CONFIG_LOCK, which the waiting caller may hold), and returning the unmigrated doc meanwhile would hand that caller legacy settings the migration removes.""" if _private_for(st) is not None: return True # a migration's own reads: never start another one with st.lock: if st.postprocessed: return True # The doc, its schema version and its generation are captured together: a poll may # install a newer doc at any point after this, and migrating that doc from THIS version # would run steps its data never needed (and, D12, drop settings it holds on purpose). seen = st.installed current = int(st.doc.get("_config_version") or 0) doc = copy.deepcopy(st.doc) try: from hermes_cli.config import _known_top_level_keys from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION except ImportError: return False # still importing; the next read retries for key in sorted(set(doc) - _known_top_level_keys() - {"_config_version"}): if key not in self._unknown_warned: self._unknown_warned.add(key) logger.warning("Remote Config: unknown config key %r is ignored by this Hermes version", key) latest = _latest_config_version() if SUPPORT_FLOOR_VERSION <= current < latest: self._migrate_in_memory(st, seen, current, doc) return True if current < SUPPORT_FLOOR_VERSION: logger.warning("Remote Config: profile %r was written by config version %d, below the " "migration floor %d; not migrated", st.profile, current, SUPPORT_FLOOR_VERSION) with st.lock: if st.installed == seen: # a doc installed meanwhile needs its own pass st.postprocessed = True return True def _run_private(self, st: _ProfileState, doc: dict[str, Any], current: int) -> dict[str, Any]: """Run the migrations from schema *current* over a private copy of *doc* and return it. Nothing is published: only this thread sees the copy while the steps run (:class:`_Private`).""" from hermes_cli.config_migrations import run_migrations from hermes_constants import reset_hermes_home_override, set_hermes_home_override m = _Private(st=st, doc=copy.deepcopy(doc), serial=next(_PRIVATE_SERIAL)) m.doc["_config_version"] = current private_token = _PRIVATE.set(m) home_token = set_hermes_home_override(st.home) try: # config_only: a read (or a refused write's check) must change no local file (D12). run_migrations(current, {"env_added": [], "config_added": [], "warnings": []}, True, config_only=True) finally: reset_hermes_home_override(home_token) _PRIVATE.reset(private_token) return m.doc def _migrate_in_memory(self, st: _ProfileState, started: int, current: int, doc: dict[str, Any]) -> None: """Migrate doc generation *started* (*doc*, schema version *current*) in memory (D12). The steps work on a private copy; the result is published with its diff base in one step, and only while generation *started* is still the installed one and no concurrent pass over it has published first: once a poll or write has installed another doc, nothing of this migration lands on it (that doc gets its own pass). The migrated doc becomes the base later writes diff against: a reader was handed the migrated doc, so a write that leaves the migration's changes in place must not send them (D12: never write the migration back).""" with st.lock: if st.installed != started or st.postprocessed: return # replaced, or another pass already published this generation migrated = self._run_private(st, doc, current) migrated["_config_version"] = _latest_config_version() base = copy.deepcopy(migrated) base.pop("_config_version", None) with st.lock: if st.installed != started or st.postprocessed: return # a poll or write installed a newer doc, or a concurrent pass published self._publish_locked(st, migrated, base) st.postprocessed = True logger.info("Remote Config: migrated profile %r in memory from config version %d", st.profile, current) @staticmethod def _publish_locked(st: _ProfileState, doc: dict[str, Any], base: dict[str, Any]) -> None: st.doc = doc st.base = base st.changed_ns = time.time_ns() st.gen += 1 # --- ConfigBackend: reads --------------------------------------------------------------- def _snapshot(self, st: _ProfileState, take): """``take()`` under st.lock once the published doc is completed postprocess output (or this thread's own migration copy). A poll that installs a newer doc between completing and taking sends the caller round again, so no reader ever sees an unmigrated doc.""" for _ in range(self._MAX_POSTPROCESS): ready = self._completed(st) with st.lock: if not ready or st.postprocessed or _private_for(st) is not None: return take() raise ConfigWriteError("Remote Config: the profile's config kept changing while it was read", code="config_version_conflict") def read_user_layer(self, home: Path) -> UserLayer: st = self._state(home) def take(): m = _private_for(st) if m is None: _remember_read(st) return (copy.deepcopy(m.doc if m is not None else st.doc), self._version_of(st), {encode(p): level for p, level in st.locks}) doc, version, locks = self._snapshot(st, take) return UserLayer(doc=doc, version=version, locks=locks, provenance=f"remote:{client.shown_url()} profile={st.profile}") def read_user_doc(self, home: Path) -> Any: return self.read_user_layer(home).doc def probe_readable(self, home: Path) -> None: # Readable once this home's layer is fetched; an unreachable plane raises # ConfigBackendUnavailable (never an OSError: there is no local file to retry). self._state(home) def read_user_doc_readonly(self, home: Path) -> Any: st = self._state(home) def take(): m = _private_for(st) if m is None: _remember_read(st) return m.doc if m is not None else st.doc return self._snapshot(st, take) @staticmethod def _version_of(st: _ProfileState) -> tuple[Any, ...]: # Leads with a nanosecond timestamp like the file backend's st_mtime_ns (the TUI's config # watcher reads element 0 as a time); the etag and local generation make it exact. A # migration's own reads see its private copy under a version of their own, so no config # cache ever pairs the published version with a private doc, or the reverse. It must not # START with the published version either: config caches match a signature as a prefix. m = _private_for(st) if m is not None: return (-m.serial, m.gen, st.changed_ns, st.etag, st.gen) return (st.changed_ns, st.etag, st.gen) def version(self, home: Path) -> tuple[Any, ...]: # A config cache validates on this signature and then hands out the doc it cached for it: # that doc is what the caller now holds, so this is the read a later save diffs against. st = self._state(home) with st.lock: if _private_for(st) is None and st.postprocessed: _remember_read(st) return self._version_of(st) def cache_version(self, home: Path, version: tuple[Any, ...]) -> tuple[Any, ...]: # The version is exact (etag + local generation): a remote write the cache cannot see # always changes it, so nothing is added (the file backend adds a racy content digest). return version def exists(self, home: Path) -> bool: return True # the plane always resolves a document (an absent level is empty, contract R4) def locked(self, home: Path, dotted: str) -> Optional[str]: path = from_dotted(dotted) for lock, level in self._state(home).locks: if len(lock) <= len(path) and tuple(path[:len(lock)]) == lock: return level return None # --- ConfigBackend: writes -------------------------------------------------------------- _MAX_PREPARE = 3 # re-preparations when a poll lands between preparing and sending def write_changes(self, home: Path, changes: Changes) -> None: """Send ``changes`` as one PATCH with CAS (contract §15.2), re-read and retry once on a conflict (§15.3). The edit is fixed once, against the doc the caller read (its *intent*: the key-level sets and unsets that turn that doc into ``changes``). A retry re-diffs that intent against the re-read doc, with the re-read locks; it never replays the caller's whole document, which would turn every key another writer changed meanwhile into an edit of ours.""" st = self._state(home) try: # before st.lock: resolving the credential may read files and the network token = client.bearer(st.home) except PlaneCredentialError as exc: raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc notes: list[str] = [] try: self._send_changes(st, changes, token, notes) finally: _log_notes(notes) # never under st.lock def _send_changes(self, st: _ProfileState, changes: Changes, token: str, notes: list[str]) -> None: intent: Optional[tuple[dict[KeyPath, Any], list[KeyPath], frozenset]] = None for attempt in (1, 2): for _ in range(self._MAX_PREPARE): # Diff against the doc readers get, i.e. migrated in memory (D12) — also after the # CAS re-read below or a poll landing mid-prepare. Outside st.lock: a migration # reads through hermes_cli.config. ready = self._completed(st) with st.lock: if ready and not st.postprocessed: continue # a poll installed a newer doc since: complete that one first seen = st.installed if intent is None: intent = self._intent(st, changes) built = self._patch_body(st, *intent) if built is None: return body, sets, unsets = built check = self._migration_check_input(st, body, sets, unsets) if check is not None: # outside st.lock, like any migration run self._check_migration_keeps(st, sets, *check) with st.lock: if st.installed != seen: continue # a poll landed meanwhile: prepare again against it try: resp = client.request("PATCH", st.home, st.profile, body=body, token=token) except (PlaneCredentialError, client.TransportError) as exc: raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc if resp.status == 200 and _effective_ok(resp.body): notes.extend(self._install(st, resp.body, resp.etag)) _remember_read(st) # the caller now holds what it wrote notes.extend(self._inherited_notes(st, unsets)) return if resp.status == 409 and attempt == 1: try: # contract §15.3: re-read, re-diff, retry once self._fetch_into(st, conditional=False, token=token, notes=notes) except _FetchFailed as exc: raise ConfigWriteError(f"Remote Config write conflict and re-read failed: {exc}", code="config_version_conflict") from exc break raise self._write_error(resp) else: raise ConfigWriteError("Remote Config write not sent: the profile's config kept changing " "while the write was prepared", code="config_version_conflict") def _diff_base(self, st: _ProfileState) -> dict[str, Any]: base = copy.deepcopy(st.base if st.base is not None else st.server_config) base.pop("_config_version", None) return base def _intent(self, st: _ProfileState, changes: Changes ) -> tuple[dict[KeyPath, Any], list[KeyPath], frozenset]: """The key-level edit ``changes`` makes to the last read doc (§15.2 steps 1-5). A document is diffed against the base THIS context read it from (``_READ_BASE``): a poll landing between the caller's read and its save must not turn every key the poll changed into an edit of ours, sent with the already-current expectedVersion (no 409 to catch it). A keyed set/unset names its own keys, so it applies to the current base. The third item is the leaf paths a keyed ``set`` (or, for a document save, a :func:`~hermes_cli.config_backend.pinning_keys` block) names: they are stored at the profile level even when the value equals the inherited one, as the file backend writes them into ``config.yaml``. Otherwise ``config set k v`` with ``v`` equal to the upper level's value would report success, store nothing, and follow the next upper-level change.""" base = self._diff_base(st) if changes.document is not None: read = _read_base_for(st) if _private_for(st) is None else None if read is not None: base = read new = to_wire(copy.deepcopy(changes.document)) if not isinstance(new, dict): raise ConfigValueError("the config document must be a mapping") else: new = copy.deepcopy(base) for key, value in changes.set.items(): path = from_dotted(key) _set_path(new, path, to_wire(value, path)) for key in changes.unset: _pop_path(new, from_dotted(key)) new.pop("_config_version", None) if changes.document is not None: base, new, dropped = strip_locked(base, new, st.locks) for prefix in refused_locked_omission_prefixes(): whole = from_dotted(prefix) for path in dropped: if covers(whole, path) or covers(path, whole): level = next((lv for lk, lv in st.locks if lk == path), "upper") raise ConfigLockedError(encode(path), level, f"{encode(path)} is locked by the {level} level of Remote Config") if dropped: print(f"Note: {len(dropped)} setting(s) locked by Remote Config were not saved: " f"{', '.join(sorted(encode(p) for p in dropped))}", file=sys.stderr) # Leaf-level: a key added under a section the doc lacked stays a key-level edit, so a # retry after 409 keeps the siblings another writer put in that section meanwhile. sets, unsets = intent_diff(base, new) pinned: set[KeyPath] = set() named = list(changes.set) + (list(pinned_keys()) if changes.document is not None else []) for key in named: path = from_dotted(key) found, wire = _get_path(new, path) if not found: # a locked key a document save dropped continue leaves = intent_diff({}, wire, path)[0] if isinstance(wire, dict) and wire else {path: wire} for leaf, leaf_value in leaves.items(): sets.setdefault(leaf, leaf_value) pinned.add(leaf) return sets, unsets, frozenset(pinned) def _patch_body(self, st: _ProfileState, intent_sets: dict[KeyPath, Any], unsets: list[KeyPath], pinned: frozenset = frozenset() ) -> Optional[tuple[dict[str, Any], dict[KeyPath, Any], list[KeyPath]]]: """The PATCH body applying the intent to the current doc, or None when that is no change. A *pinned* key equal to the inherited value is still sent, unless a lock already freezes it (nothing to store, and refusing a no-op pick of the locked value helps nobody).""" base = self._diff_base(st) new = copy.deepcopy(base) apply_intent(new, intent_sets, unsets) sets, unsets = diff(base, new) for path in pinned: value = intent_sets[path] if isinstance(value, dict) and not value: continue # "make this a mapping" (apply_intent): sent above when absent, kept when present if path not in sets and covering_lock(st.locks, path) is None \ and not any(covers(p, path) for p in sets): sets[path] = copy.deepcopy(value) refused = write_check(sets, unsets, st.locks) if refused is not None: raise ConfigLockedError(encode(refused[0]), refused[1], f"{encode(refused[0])} is locked by the {refused[1]} level of Remote Config") for p, v in sets.items(): bad = secret_literal_path(p, v) if bad is not None: raise ConfigValueError( f"{encode(bad)} is a secret-shaped key: Remote Config stores only a ${{VAR}} reference " "there, never the secret itself. Put the value in .env and set a ${VAR} reference.", code="config_secret_literal") if not sets and not unsets: return None set_body, unset_body = encode_changes(sets, unsets) body: dict[str, Any] = {"expectedVersion": st.profile_version} # The stamp describes the profile level's STORED data (contract §5.3: absent = unchanged). # Stamp only when that data is already at this agent's version, or unstamped (null: readers # already treat it as current, R6). An older stamp means the level still holds pre-migration # keys that D12 forbids writing back migrated; re-stamping would make later readers skip # those migration steps. A newer stamp must not be lowered either. latest = _latest_config_version() if st.profile_writer is None or st.profile_writer == latest: body["writerConfigVersion"] = latest if set_body: body["set"] = set_body if unset_body: body["unset"] = unset_body return body, sets, unsets def _migration_check_input(self, st: _ProfileState, body: dict[str, Any], sets: dict[KeyPath, Any], unsets: list[KeyPath]) -> Optional[tuple[dict[str, Any], int]]: """``(stored doc after this write, its schema version)`` when readers will migrate it in memory after the write (R6: the oldest level's stamp), else None. Call under st.lock.""" if not sets: return None try: from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION except ImportError: return None latest = _latest_config_version() profile = latest if "writerConfigVersion" in body else st.profile_writer stamps = [*st.other_writers, *([profile] if isinstance(profile, int) else [])] after = min(stamps) if stamps else latest if not SUPPORT_FLOOR_VERSION <= after < latest: return None stored = copy.deepcopy(st.server_config) stored.pop("_config_version", None) for p, v in sets.items(): _set_path(stored, p, copy.deepcopy(v)) for p in unsets: _pop_path(stored, p) return stored, after def _check_migration_keeps(self, st: _ProfileState, sets: dict[KeyPath, Any], stored: dict[str, Any], after: int) -> None: """Refuse a write whose values every later read would migrate away. The plane stores the write as sent, but under a level stamp older than this agent's schema (the stamp keeps describing the level's untouched legacy data, contract §5.3), so every reader — this process and any later one — re-runs the migrations from that stamp over the stored doc. A value one of those steps rewrites (an old default the user set on purpose) would be acknowledged and then never read back. Refused instead, before anything is sent.""" migrated = self._run_private(st, stored, after) latest = _latest_config_version() for p, v in sets.items(): found, got = _get_path(migrated, p) changed = _first_change(p, v, found, got) if changed is not None: raise ConfigValueError( f"Remote Config: {encode(changed)} cannot be saved with that value. This profile's remote " f"settings are stored at config schema v{after}, and every read migrates them to " f"v{latest} in memory, which rewrites that value (migrations are never written back, " "D12), so it would never be read back. Nothing was sent; choose another value, or " "have the profile level re-saved at the current schema on the config plane.", code="config_migration_conflict") @staticmethod def _write_error(resp: client.Response) -> ConfigWriteError: if resp.status == 403 and resp.error == "config_key_locked": return ConfigLockedError(str(resp.body.get("path") or ""), str(resp.body.get("lockedBy") or "upper"), f"Remote Config refused the write: {resp.message}") if (resp.status, resp.error) in _VALUE_REFUSALS: # The submitted value is the problem (contract §2.5 table): the caller's to fix, so it # keeps its code and the plane's refusal text, which names a path and a reason, never a # value (config_secret_literal never echoes it). detail = resp.message or resp.error path, reason = resp.body.get("path"), resp.body.get("reason") extras = [f"{k} {v}" for k, v in (("path", path), ("reason", reason)) if isinstance(v, str) and v and v not in detail] if extras: detail = f"{detail} ({', '.join(extras)})" return ConfigValueError(f"Remote Config refused the write: {detail}", code=resp.error) return ConfigWriteError(f"Remote Config refused the write ({resp.status} {resp.error}): {resp.message}", code=resp.error) @staticmethod def _inherited_notes(st: _ProfileState, unsets: list[KeyPath]) -> list[str]: """R5: removing a key at the profile level cannot remove a value an upper level supplies.""" notes: list[str] = [] for p in unsets: node: Any = st.server_config for seg in p: if not isinstance(node, dict) or seg not in node: break node = node[seg] else: notes.append(f"Remote Config: {encode(p)} is still set by an upper level; removing it from this " "profile cannot remove an inherited value (set it to null for 'no value')") return notes def apply_in_memory(self, home: Path, doc: dict) -> None: st = self._state(home) m = _private_for(st) if m is not None: # a migration step: lands in that migration's private copy only m.doc = copy.deepcopy(doc) m.gen += 1 return with st.lock: # outside a migration: the doc and the base writes diff against move together doc = copy.deepcopy(doc) base = copy.deepcopy(doc) base.pop("_config_version", None) self._publish_locked(st, doc, base) # --- ConfigBackend: capabilities and boot ----------------------------------------------- def supports_file_tooling(self) -> bool: return False def honors_managed_config(self) -> bool: return False def protected_env_names(self) -> frozenset: return PLANE_CREDENTIAL_ENV_NAMES def deployment_env_names(self) -> frozenset: from hermes_cli.config_backend import BACKEND_ENV return frozenset({BACKEND_ENV, client.URL_ENV, client.INSTANCE_ENV}) def bootstrap_env_names(self) -> frozenset: return PLANE_CREDENTIAL_ENV_NAMES # the deployment (selector, plane, instance) and the credential @contextlib.contextmanager def preserve_read_base(self): held = _READ_BASE.get() try: yield finally: if held is not None: _READ_BASE.set(held) def doctor_warnings(self, home: Path) -> tuple: managed = managed_config_file() if managed is None: return () return ((f"{managed} exists but is ignored in remote config mode", "(remove it)", f"Remove {managed}: remote config mode takes config and locks only from Remote Config"),) def boot(self, home: Path) -> None: """Design §4.4 steps 2-4: fetch this home's layer (fail closed) and flag a managed file.""" self._state(home) if not self._managed_warned: self._managed_warned = True path = managed_config_file() if path is not None: msg = (f"WARNING: {path} exists but is IGNORED: HERMES_CONFIG_BACKEND=remote takes config " "and locks only from Remote Config (D18). Remove the file.") logger.warning(msg) print(msg, file=sys.stderr) def destination(self, home: Path) -> str: return f"Remote Config (profile {self._profile_for(home)!r} level)" def describe(self, home: Path) -> str: st = self._states.get(self._key(home)) head = f"Remote Config {client.shown_url()} (instance {client.instance_id() or ''})" if st is None: return f"{head}: profile not fetched yet" age = int(time.time() - st.fetched_at) if st.fetched_at else -1 tail = f", last poll failed: {st.last_error}" if st.last_error else "" return (f"{head}: profile {st.profile!r}, profile level v{st.profile_version}, " f"{len(st.locks)} lock(s), fetched {age}s ago{tail}") # --- poller (D9) ------------------------------------------------------------------------ def _ensure_poller(self) -> None: with self._lock: if self._poller is not None and self._poller.is_alive() and self._poller_pid == os.getpid(): return self._poller_pid = os.getpid() # a forked child re-arms its own thread self._poller = threading.Thread(target=self._poll_loop, name="remote-config-poll", daemon=True) self._poller.start() def _poll_loop(self) -> None: while not self._stop.is_set(): try: now = time.monotonic() roster = self._roster() # a snapshot: a lazily added profile must not break iteration for st in roster: if st.next_poll <= now: self.poll_one(st) wake = min((st.next_poll for st in self._roster()), default=now + poll_interval()) except Exception: logger.warning("Remote Config: poll loop error; retrying", exc_info=True) wake = time.monotonic() + MIN_POLL_SECONDS self._stop.wait(max(1.0, wake - time.monotonic())) def poll_one(self, st: _ProfileState) -> bool: """One conditional GET (contract §11.3). Never raises and never exits: an error keeps the in-memory document and is retried next tick.""" try: changed = self._fetch_into(st, conditional=True) except _FetchFailed as exc: st.last_error = str(exc) # contract §11.3: a refused credential needs an operator; anything else is a blip. level = logging.ERROR if exc.status in (401, 403) else logging.WARNING logger.log(level, "Remote Config: poll for profile %r failed (%s); keeping the loaded config", st.profile, exc) changed = False except Exception as exc: st.last_error = str(exc) logger.warning("Remote Config: poll for profile %r failed", st.profile, exc_info=True) changed = False st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1) if changed: logger.info("Remote Config: profile %r changed (profile level v%d)", st.profile, st.profile_version) return changed def poll_all(self) -> None: for st in self._roster(): self.poll_one(st) def managed_config_file() -> Optional[Path]: from hermes_cli import managed_scope managed_dir = managed_scope.get_managed_dir() path = managed_dir / "config.yaml" if managed_dir else None if path is None: return None return path if path.exists() else None # config-reader: ok — existence only, to warn it is ignored (D18) def _first_change(path: KeyPath, sent: Any, found: bool, got: Any) -> Optional[KeyPath]: """The first path at or under *path* where *got* (as read back) differs from *sent*.""" if not found: return path if isinstance(sent, dict) and isinstance(got, dict): for key, value in sent.items(): changed = _first_change(path + (key,), value, key in got, got.get(key)) if changed is not None: return changed return None try: return None if json_equal(to_wire(copy.deepcopy(got), path), sent) else path except ConfigValueError: return path