| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951 |
- """The remote config backend (config-config design §4, contract.md §11, §15).
- The user layer of each profile home is the effective document the config plane resolves for
- (this instance, that profile). It lives in memory only: fetched at boot (fail closed, D2), kept
- fresh by a per-process ETag poller (D9), and written back as key-level changes on the profile
- level with CAS (D10). Nothing is read from or written to a local ``config.yaml``.
- """
- from __future__ import annotations
- import contextlib
- import contextvars
- import copy
- import itertools
- import logging
- import os
- import random
- import sys
- import threading
- import time
- from dataclasses import dataclass, field
- from pathlib import Path
- from typing import Any, Dict, List, Optional, Tuple
- from hermes_cli.config_backend import (
- Changes, ConfigBackendUnavailable, ConfigLockedError, ConfigValueError, ConfigWriteError, UserLayer,
- pinned_keys, refused_locked_omission_prefixes)
- from . import client
- from .credentials import PLANE_CREDENTIAL_ENV_NAMES, PlaneCredentialError
- from .diff import (_get_path, _pop_path, _set_path, apply_intent, diff, encode_changes, intent_diff, json_equal,
- strip_locked, write_check)
- from .paths import Path as KeyPath
- from .paths import PathError, covering_lock, covers, decode, encode, from_dotted
- from .values import secret_literal_path, to_wire
- logger = logging.getLogger(__name__)
- POLL_SECONDS = 300.0 # contract §11.3; a plane-served interval is a later change
- MIN_POLL_SECONDS = 30.0 # contract §11.3: lower values are clamped
- BOOT_RETRY_DELAYS = (10.0, 20.0) # contract §11.4: attempts at t = 0, ~10 s, ~30 s
- MAX_RETRY_WAIT = 20.0
- def poll_interval() -> float:
- return max(POLL_SECONDS, MIN_POLL_SECONDS)
- def _latest_config_version() -> int:
- from hermes_cli.config_defaults import DEFAULT_CONFIG
- version = DEFAULT_CONFIG.get("_config_version")
- return version if isinstance(version, int) else 1
- def _log_notes(notes: list[str]) -> None:
- """Log messages collected while a backend lock was held (never log under one: a log handler
- can read config, and a thread holding the handler's lock may be waiting for that lock)."""
- for note in notes:
- if note.startswith("debug:"):
- logger.debug("%s", note[len("debug:"):])
- else:
- logger.warning("%s", note)
- class _FetchFailed(Exception):
- def __init__(self, detail: str, *, retryable: bool, retry_after: Optional[float] = None,
- status: Optional[int] = None) -> None:
- super().__init__(detail)
- self.retryable = retryable
- self.retry_after = retry_after
- self.status = status
- @dataclass
- class _ProfileState:
- home: Path
- profile: str
- server_config: dict[str, Any] = field(default_factory=dict) # EffectiveResponse.config, as sent
- doc: dict[str, Any] = field(default_factory=dict) # what readers get (+ _config_version, migrated)
- etag: str = ""
- profile_version: int = 0
- # The profile level's stored writerConfigVersion: None = stored null, _UNKNOWN = not reported.
- profile_writer: Any = None
- other_writers: list[int] = field(default_factory=list) # the other levels' int stamps (R6)
- locks: list[tuple[KeyPath, str]] = field(default_factory=list)
- provenance: dict[str, str] = field(default_factory=dict)
- changed_ns: int = 0
- gen: int = 0
- installed: int = 0 # bumped by every _install; a GET started before a newer install is stale
- base: Optional[dict[str, Any]] = None # the migrated doc writes diff against (None = server_config)
- postprocessed: bool = False # st.doc is the completed migration output of generation `installed`
- fetched_at: float = 0.0
- last_error: Optional[str] = None
- next_poll: float = 0.0
- lock: threading.RLock = field(default_factory=threading.RLock)
- _UNKNOWN = object()
- @dataclass
- class _Private:
- """An in-memory migration's working copy of one profile's document (D12).
- The migration steps read and persist through ``hermes_cli.config`` like any reader and writer;
- while one runs, THAT thread sees this copy instead of the published one (``_PRIVATE``), and
- everyone else keeps seeing the published doc with its matching diff base. The migrated doc and
- its base are published together, in one step, when the migration ends: a concurrent writer
- never diffs a half-migrated document against the unmigrated base, or the reverse."""
- st: _ProfileState
- doc: dict[str, Any]
- serial: int
- gen: int = 0
- _PRIVATE: contextvars.ContextVar[Optional[_Private]] = contextvars.ContextVar("remote_config_private", default=None)
- _PRIVATE_SERIAL = itertools.count(1)
- # The diff base each profile had when THIS context last read it: {id(state): base}. A document
- # save means "turn what I read into this", so its intent is diffed against that read, never
- # against whatever a poll (or another writer) installed between the read and the save. Replaced,
- # never mutated: copied contexts (threads, tasks) must not share one dict.
- _READ_BASE: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar("remote_config_read_base", default=None)
- def _remember_read(st: _ProfileState) -> None:
- """Record (by reference: ``st.base`` / ``st.server_config`` are replaced, never mutated) the
- base this context just read under st.lock."""
- seen = dict(_READ_BASE.get() or {})
- seen[id(st)] = (st, st.base if st.base is not None else st.server_config)
- _READ_BASE.set(seen)
- def _read_base_for(st: _ProfileState) -> Optional[dict[str, Any]]:
- held = (_READ_BASE.get() or {}).get(id(st))
- if held is None or held[0] is not st or held[1] is None:
- return None # never read in this context, or a different (reset) profile state
- base = copy.deepcopy(held[1])
- base.pop("_config_version", None)
- return base
- def _private_for(st: _ProfileState) -> Optional[_Private]:
- m = _PRIVATE.get()
- return m if m is not None and m.st is st else None
- # Plane refusals caused by the submitted value (contract §2.5, §11.1): surfaced to the caller as
- # ConfigValueError with the plane's code. config_request_invalid is deliberately absent: a request
- # the plane cannot parse is this client's bug, not the user's value, so it stays opaque.
- _VALUE_REFUSALS = frozenset({
- (400, "config_secret_literal"), (400, "config_value_invalid"), (400, "config_path_invalid"),
- (400, "config_path_reserved"), (413, "config_level_too_large"), (413, "config_body_too_large"),
- })
- def _profile_writer(body: dict[str, Any]) -> Any:
- """The profile level's stored ``writerConfigVersion`` (int or None), or ``_UNKNOWN``."""
- for lv in body.get("levels") or []:
- if isinstance(lv, dict) and lv.get("kind") == "profile":
- w = lv.get("writerConfigVersion")
- return w if w is None or isinstance(w, int) else _UNKNOWN
- return _UNKNOWN
- def _effective_ok(body: dict[str, Any]) -> bool:
- return (isinstance(body.get("config"), dict) and isinstance(body.get("profileVersion"), int)
- and isinstance(body.get("locks"), list))
- class RemoteBackend:
- name = "remote"
- def __init__(self) -> None:
- self._states: dict[str, _ProfileState] = {}
- self._fetch_locks: dict[str, threading.Lock] = {}
- self._lock = threading.Lock()
- self._local = threading.local() # .booting: the home keys this thread is loading
- self._poller: Optional[threading.Thread] = None
- self._poller_pid: Optional[int] = None
- self._stop = threading.Event()
- self._managed_warned = False
- self._unknown_warned: set = set()
- # --- homes and state --------------------------------------------------------------------
- @staticmethod
- def _key(home: Path) -> str:
- from hermes_constants import hermes_home_key
- return hermes_home_key(home)
- @staticmethod
- def _profile_for(home: Path) -> str:
- from hermes_constants import PROFILE_ID_RE, profile_name_for_home
- name = profile_name_for_home(home)
- if not name or not PROFILE_ID_RE.match(name):
- raise ConfigBackendUnavailable(
- f"Remote Config: {home} is not a Hermes profile home, so it has no remote config "
- "(HERMES_CONFIG_BACKEND=remote serves profile homes only).")
- return name
- def _state(self, home: Path) -> _ProfileState:
- key = self._key(home)
- st = self._states.get(key)
- if st is not None:
- poller = self._poller
- # A forked child has no poller thread, and a poller that died must not leave every
- # cached profile without updates until the next new profile happens to re-arm it.
- if poller is None or self._poller_pid != os.getpid() or not poller.is_alive():
- self._ensure_poller()
- return st
- if key in self._booting():
- # This thread is loading this profile (resolving its credential or fetching): the read
- # can only be served by the load it is part of. Waiting would wait on itself forever.
- raise ConfigBackendUnavailable(
- f"Remote Config: a config read for {home} re-entered the load of that profile's config "
- "on the same thread (code on the boot path read config before it was loaded). This is "
- "a Hermes bug; refusing the read instead of deadlocking.")
- with self._lock:
- fetch_lock = self._fetch_locks.setdefault(key, threading.Lock())
- booting = self._booting()
- booting.add(key)
- try:
- st = self._boot_fetch(Path(home), key, fetch_lock)
- finally:
- booting.discard(key)
- self._ensure_poller()
- return st
- def _booting(self) -> set[str]:
- """Home keys this thread is loading now (a thread-local set)."""
- keys = getattr(self._local, "booting", None)
- if keys is None:
- keys = self._local.booting = set()
- return keys
- def _roster(self) -> list[_ProfileState]:
- with self._lock:
- return list(self._states.values())
- def _boot_fetch(self, home: Path, key: str, fetch_lock: threading.Lock) -> _ProfileState:
- """Boot fetch with bounded retry (contract §11.4); fails closed (D2).
- Each attempt resolves the credential first, holding no backend lock, then takes the fetch
- lock only for the GET: other threads wait there for the fetch in flight and share its
- result. Nothing is logged and nobody is waited for while that lock is held."""
- profile = self._profile_for(home)
- if not client.instance_id():
- raise ConfigBackendUnavailable(
- f"Remote Config: {client.INSTANCE_ENV} is not set, so Hermes cannot identify this agent "
- "to the config plane. Hermes does not start without its remote config.")
- delays = list(BOOT_RETRY_DELAYS)
- while True:
- notes: list[str] = []
- try:
- token = self._bearer(home)
- with fetch_lock:
- st = self._states.get(key)
- if st is None:
- st = _ProfileState(home=home, profile=profile)
- self._fetch_into(st, conditional=False, token=token, notes=notes)
- with self._lock: # the poller snapshots the roster under the same lock
- self._states[key] = st
- _log_notes(notes)
- return st
- except _FetchFailed as exc:
- _log_notes(notes)
- if exc.retryable and delays:
- wait = min(max(exc.retry_after or 0.0, delays.pop(0)), MAX_RETRY_WAIT)
- logger.warning("Remote Config: fetch for profile %r failed (%s); retrying in %.0fs",
- profile, exc, wait)
- time.sleep(wait)
- continue
- raise ConfigBackendUnavailable(
- f"Remote Config: cannot load the config for profile {profile!r} from "
- f"{client.shown_url()}: {exc}. Hermes does not start without its remote config "
- "(HERMES_CONFIG_BACKEND=remote has no local fallback).") from exc
- @staticmethod
- def _bearer(home: Path) -> str:
- try:
- return client.bearer(home)
- except PlaneCredentialError as exc:
- raise _FetchFailed(str(exc), retryable=exc.retryable) from exc
- def _fetch_into(self, st: _ProfileState, *, conditional: bool, token: Optional[str] = None,
- notes: Optional[list[str]] = None) -> bool:
- """GET ``/self`` into ``st``; True when the document changed. Raises :class:`_FetchFailed`.
- The GET runs outside ``st.lock`` (a slow plane must not stall writers), so a write can
- install a newer document while it is in flight. Its response is then stale, even when its
- ``profileVersion`` matches (an upper level may have changed in between): it is dropped, and
- the next poll fetches again. Messages go to ``notes`` (logged by the caller once it holds no
- backend lock), or are logged here when there is no list."""
- if notes is None:
- notes = []
- try:
- return self._fetch_into(st, conditional=conditional, token=token, notes=notes)
- finally:
- _log_notes(notes)
- started = st.installed
- try:
- resp = client.request("GET", st.home, st.profile, etag=st.etag if conditional else None, token=token)
- except PlaneCredentialError as exc:
- raise _FetchFailed(str(exc), retryable=exc.retryable) from exc
- except client.TransportError as exc:
- raise _FetchFailed(str(exc), retryable=True) from exc
- if resp.status == 304 and conditional:
- st.fetched_at, st.last_error = time.time(), None
- return False
- if resp.status == 200 and _effective_ok(resp.body):
- with st.lock:
- if st.installed != started:
- notes.append(f"debug:Remote Config: dropped a fetch for profile {st.profile!r} that a write overtook")
- return False
- notes.extend(self._install(st, resp.body, resp.etag))
- return True
- if resp.status == 200:
- raise _FetchFailed("the plane returned a malformed response", retryable=True)
- retryable = resp.status >= 500 or resp.status == 429
- raise _FetchFailed(f"HTTP {resp.status} {resp.error}: {resp.message}", retryable=retryable,
- retry_after=resp.retry_after, status=resp.status)
- def _install(self, st: _ProfileState, body: dict[str, Any], etag: Optional[str]) -> list[str]:
- """Install a fetched document (the caller holds st.lock); returns warnings to log later."""
- notes: list[str] = []
- config = body["config"]
- doc = copy.deepcopy(config)
- doc.pop("_config_version", None) # reserved on the wire (§3.6); never expected here
- # R6: the in-memory schema version is the oldest writer's; migrations run in memory (D12).
- writer_versions: list[int] = [
- lv["writerConfigVersion"] for lv in body.get("levels") or []
- if isinstance(lv, dict) and isinstance(lv.get("writerConfigVersion"), int)]
- doc["_config_version"] = min(writer_versions) if writer_versions else _latest_config_version()
- locks: list[tuple[KeyPath, str]] = []
- for entry in body.get("locks") or []:
- try:
- locks.append((decode(entry["path"]), str(entry["level"])))
- except (KeyError, TypeError, PathError):
- notes.append(f"Remote Config: ignoring malformed lock entry {entry!r}")
- st.server_config = config
- st.doc = doc
- st.etag = etag or str(body.get("etag") or "")
- st.profile_version = int(body["profileVersion"])
- st.profile_writer = _profile_writer(body)
- st.other_writers = [
- lv["writerConfigVersion"] for lv in body.get("levels") or []
- if isinstance(lv, dict) and lv.get("kind") != "profile" and isinstance(lv.get("writerConfigVersion"), int)]
- st.locks = locks
- st.provenance = dict(body.get("provenance") or {})
- st.changed_ns = time.time_ns()
- st.gen += 1
- st.installed += 1
- st.base = None
- st.postprocessed = False
- st.fetched_at, st.last_error = time.time(), None
- st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1)
- return notes
- _MAX_POSTPROCESS = 8 # passes when polls keep installing newer docs mid-migration
- def _completed(self, st: _ProfileState) -> bool:
- """Bring the published doc to its completed postprocess output; False when that is not
- possible yet (``hermes_cli.config`` still importing during the boot fetch)."""
- if _private_for(st) is not None:
- return True # a migration's own reads use its private copy (never another pass)
- for _ in range(self._MAX_POSTPROCESS):
- ready = self._postprocess(st)
- if not ready:
- return False
- with st.lock:
- if st.postprocessed:
- return True
- raise ConfigWriteError("Remote Config: the profile's config kept changing while it was migrated in memory",
- code="config_version_conflict")
- def _postprocess(self, st: _ProfileState) -> bool:
- """First read after a change: warn on unknown keys and migrate in memory (D12). Runs on a
- read, not at fetch, because both need ``hermes_cli.config``, which may still be importing
- when the boot fetch runs (then False: nothing could be done).
- No single flight: a reader or writer that finds the published doc unmigrated runs its own
- private pass, even while another thread's pass is in flight, and the first pass to finish
- publishes. Waiting for the other pass instead could deadlock (a migration reads through
- hermes_cli.config's _CONFIG_LOCK, which the waiting caller may hold), and returning the
- unmigrated doc meanwhile would hand that caller legacy settings the migration removes."""
- if _private_for(st) is not None:
- return True # a migration's own reads: never start another one
- with st.lock:
- if st.postprocessed:
- return True
- # The doc, its schema version and its generation are captured together: a poll may
- # install a newer doc at any point after this, and migrating that doc from THIS version
- # would run steps its data never needed (and, D12, drop settings it holds on purpose).
- seen = st.installed
- current = int(st.doc.get("_config_version") or 0)
- doc = copy.deepcopy(st.doc)
- try:
- from hermes_cli.config import _known_top_level_keys
- from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION
- except ImportError:
- return False # still importing; the next read retries
- for key in sorted(set(doc) - _known_top_level_keys() - {"_config_version"}):
- if key not in self._unknown_warned:
- self._unknown_warned.add(key)
- logger.warning("Remote Config: unknown config key %r is ignored by this Hermes version", key)
- latest = _latest_config_version()
- if SUPPORT_FLOOR_VERSION <= current < latest:
- self._migrate_in_memory(st, seen, current, doc)
- return True
- if current < SUPPORT_FLOOR_VERSION:
- logger.warning("Remote Config: profile %r was written by config version %d, below the "
- "migration floor %d; not migrated", st.profile, current, SUPPORT_FLOOR_VERSION)
- with st.lock:
- if st.installed == seen: # a doc installed meanwhile needs its own pass
- st.postprocessed = True
- return True
- def _run_private(self, st: _ProfileState, doc: dict[str, Any], current: int) -> dict[str, Any]:
- """Run the migrations from schema *current* over a private copy of *doc* and return it.
- Nothing is published: only this thread sees the copy while the steps run (:class:`_Private`)."""
- from hermes_cli.config_migrations import run_migrations
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- m = _Private(st=st, doc=copy.deepcopy(doc), serial=next(_PRIVATE_SERIAL))
- m.doc["_config_version"] = current
- private_token = _PRIVATE.set(m)
- home_token = set_hermes_home_override(st.home)
- try:
- # config_only: a read (or a refused write's check) must change no local file (D12).
- run_migrations(current, {"env_added": [], "config_added": [], "warnings": []}, True, config_only=True)
- finally:
- reset_hermes_home_override(home_token)
- _PRIVATE.reset(private_token)
- return m.doc
- def _migrate_in_memory(self, st: _ProfileState, started: int, current: int, doc: dict[str, Any]) -> None:
- """Migrate doc generation *started* (*doc*, schema version *current*) in memory (D12).
- The steps work on a private copy; the result is published with its diff base in one step,
- and only while generation *started* is still the installed one and no concurrent pass over
- it has published first: once a poll or write has installed another doc, nothing of this
- migration lands on it (that doc gets its own pass).
- The migrated doc becomes the base later writes diff against: a reader was handed the
- migrated doc, so a write that leaves the migration's changes in place must not send them
- (D12: never write the migration back)."""
- with st.lock:
- if st.installed != started or st.postprocessed:
- return # replaced, or another pass already published this generation
- migrated = self._run_private(st, doc, current)
- migrated["_config_version"] = _latest_config_version()
- base = copy.deepcopy(migrated)
- base.pop("_config_version", None)
- with st.lock:
- if st.installed != started or st.postprocessed:
- return # a poll or write installed a newer doc, or a concurrent pass published
- self._publish_locked(st, migrated, base)
- st.postprocessed = True
- logger.info("Remote Config: migrated profile %r in memory from config version %d", st.profile, current)
- @staticmethod
- def _publish_locked(st: _ProfileState, doc: dict[str, Any], base: dict[str, Any]) -> None:
- st.doc = doc
- st.base = base
- st.changed_ns = time.time_ns()
- st.gen += 1
- # --- ConfigBackend: reads ---------------------------------------------------------------
- def _snapshot(self, st: _ProfileState, take):
- """``take()`` under st.lock once the published doc is completed postprocess output (or
- this thread's own migration copy). A poll that installs a newer doc between completing
- and taking sends the caller round again, so no reader ever sees an unmigrated doc."""
- for _ in range(self._MAX_POSTPROCESS):
- ready = self._completed(st)
- with st.lock:
- if not ready or st.postprocessed or _private_for(st) is not None:
- return take()
- raise ConfigWriteError("Remote Config: the profile's config kept changing while it was read",
- code="config_version_conflict")
- def read_user_layer(self, home: Path) -> UserLayer:
- st = self._state(home)
- def take():
- m = _private_for(st)
- if m is None:
- _remember_read(st)
- return (copy.deepcopy(m.doc if m is not None else st.doc), self._version_of(st),
- {encode(p): level for p, level in st.locks})
- doc, version, locks = self._snapshot(st, take)
- return UserLayer(doc=doc, version=version, locks=locks,
- provenance=f"remote:{client.shown_url()} profile={st.profile}")
- def read_user_doc(self, home: Path) -> Any:
- return self.read_user_layer(home).doc
- def probe_readable(self, home: Path) -> None:
- # Readable once this home's layer is fetched; an unreachable plane raises
- # ConfigBackendUnavailable (never an OSError: there is no local file to retry).
- self._state(home)
- def read_user_doc_readonly(self, home: Path) -> Any:
- st = self._state(home)
- def take():
- m = _private_for(st)
- if m is None:
- _remember_read(st)
- return m.doc if m is not None else st.doc
- return self._snapshot(st, take)
- @staticmethod
- def _version_of(st: _ProfileState) -> tuple[Any, ...]:
- # Leads with a nanosecond timestamp like the file backend's st_mtime_ns (the TUI's config
- # watcher reads element 0 as a time); the etag and local generation make it exact. A
- # migration's own reads see its private copy under a version of their own, so no config
- # cache ever pairs the published version with a private doc, or the reverse. It must not
- # START with the published version either: config caches match a signature as a prefix.
- m = _private_for(st)
- if m is not None:
- return (-m.serial, m.gen, st.changed_ns, st.etag, st.gen)
- return (st.changed_ns, st.etag, st.gen)
- def version(self, home: Path) -> tuple[Any, ...]:
- # A config cache validates on this signature and then hands out the doc it cached for it:
- # that doc is what the caller now holds, so this is the read a later save diffs against.
- st = self._state(home)
- with st.lock:
- if _private_for(st) is None and st.postprocessed:
- _remember_read(st)
- return self._version_of(st)
- def cache_version(self, home: Path, version: tuple[Any, ...]) -> tuple[Any, ...]:
- # The version is exact (etag + local generation): a remote write the cache cannot see
- # always changes it, so nothing is added (the file backend adds a racy content digest).
- return version
- def exists(self, home: Path) -> bool:
- return True # the plane always resolves a document (an absent level is empty, contract R4)
- def locked(self, home: Path, dotted: str) -> Optional[str]:
- path = from_dotted(dotted)
- for lock, level in self._state(home).locks:
- if len(lock) <= len(path) and tuple(path[:len(lock)]) == lock:
- return level
- return None
- # --- ConfigBackend: writes --------------------------------------------------------------
- _MAX_PREPARE = 3 # re-preparations when a poll lands between preparing and sending
- def write_changes(self, home: Path, changes: Changes) -> None:
- """Send ``changes`` as one PATCH with CAS (contract §15.2), re-read and retry once on a
- conflict (§15.3).
- The edit is fixed once, against the doc the caller read (its *intent*: the key-level sets
- and unsets that turn that doc into ``changes``). A retry re-diffs that intent against the
- re-read doc, with the re-read locks; it never replays the caller's whole document, which
- would turn every key another writer changed meanwhile into an edit of ours."""
- st = self._state(home)
- try: # before st.lock: resolving the credential may read files and the network
- token = client.bearer(st.home)
- except PlaneCredentialError as exc:
- raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc
- notes: list[str] = []
- try:
- self._send_changes(st, changes, token, notes)
- finally:
- _log_notes(notes) # never under st.lock
- def _send_changes(self, st: _ProfileState, changes: Changes, token: str, notes: list[str]) -> None:
- intent: Optional[tuple[dict[KeyPath, Any], list[KeyPath], frozenset]] = None
- for attempt in (1, 2):
- for _ in range(self._MAX_PREPARE):
- # Diff against the doc readers get, i.e. migrated in memory (D12) — also after the
- # CAS re-read below or a poll landing mid-prepare. Outside st.lock: a migration
- # reads through hermes_cli.config.
- ready = self._completed(st)
- with st.lock:
- if ready and not st.postprocessed:
- continue # a poll installed a newer doc since: complete that one first
- seen = st.installed
- if intent is None:
- intent = self._intent(st, changes)
- built = self._patch_body(st, *intent)
- if built is None:
- return
- body, sets, unsets = built
- check = self._migration_check_input(st, body, sets, unsets)
- if check is not None: # outside st.lock, like any migration run
- self._check_migration_keeps(st, sets, *check)
- with st.lock:
- if st.installed != seen:
- continue # a poll landed meanwhile: prepare again against it
- try:
- resp = client.request("PATCH", st.home, st.profile, body=body, token=token)
- except (PlaneCredentialError, client.TransportError) as exc:
- raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc
- if resp.status == 200 and _effective_ok(resp.body):
- notes.extend(self._install(st, resp.body, resp.etag))
- _remember_read(st) # the caller now holds what it wrote
- notes.extend(self._inherited_notes(st, unsets))
- return
- if resp.status == 409 and attempt == 1:
- try: # contract §15.3: re-read, re-diff, retry once
- self._fetch_into(st, conditional=False, token=token, notes=notes)
- except _FetchFailed as exc:
- raise ConfigWriteError(f"Remote Config write conflict and re-read failed: {exc}",
- code="config_version_conflict") from exc
- break
- raise self._write_error(resp)
- else:
- raise ConfigWriteError("Remote Config write not sent: the profile's config kept changing "
- "while the write was prepared", code="config_version_conflict")
- def _diff_base(self, st: _ProfileState) -> dict[str, Any]:
- base = copy.deepcopy(st.base if st.base is not None else st.server_config)
- base.pop("_config_version", None)
- return base
- def _intent(self, st: _ProfileState, changes: Changes
- ) -> tuple[dict[KeyPath, Any], list[KeyPath], frozenset]:
- """The key-level edit ``changes`` makes to the last read doc (§15.2 steps 1-5).
- A document is diffed against the base THIS context read it from (``_READ_BASE``): a poll
- landing between the caller's read and its save must not turn every key the poll changed
- into an edit of ours, sent with the already-current expectedVersion (no 409 to catch it).
- A keyed set/unset names its own keys, so it applies to the current base.
- The third item is the leaf paths a keyed ``set`` (or, for a document save, a
- :func:`~hermes_cli.config_backend.pinning_keys` block) names: they are stored at the profile
- level even when the value equals the inherited one, as the file backend writes them into
- ``config.yaml``. Otherwise ``config set k v`` with ``v`` equal to the upper level's value
- would report success, store nothing, and follow the next upper-level change."""
- base = self._diff_base(st)
- if changes.document is not None:
- read = _read_base_for(st) if _private_for(st) is None else None
- if read is not None:
- base = read
- new = to_wire(copy.deepcopy(changes.document))
- if not isinstance(new, dict):
- raise ConfigValueError("the config document must be a mapping")
- else:
- new = copy.deepcopy(base)
- for key, value in changes.set.items():
- path = from_dotted(key)
- _set_path(new, path, to_wire(value, path))
- for key in changes.unset:
- _pop_path(new, from_dotted(key))
- new.pop("_config_version", None)
- if changes.document is not None:
- base, new, dropped = strip_locked(base, new, st.locks)
- for prefix in refused_locked_omission_prefixes():
- whole = from_dotted(prefix)
- for path in dropped:
- if covers(whole, path) or covers(path, whole):
- level = next((lv for lk, lv in st.locks if lk == path), "upper")
- raise ConfigLockedError(encode(path), level,
- f"{encode(path)} is locked by the {level} level of Remote Config")
- if dropped:
- print(f"Note: {len(dropped)} setting(s) locked by Remote Config were not saved: "
- f"{', '.join(sorted(encode(p) for p in dropped))}", file=sys.stderr)
- # Leaf-level: a key added under a section the doc lacked stays a key-level edit, so a
- # retry after 409 keeps the siblings another writer put in that section meanwhile.
- sets, unsets = intent_diff(base, new)
- pinned: set[KeyPath] = set()
- named = list(changes.set) + (list(pinned_keys()) if changes.document is not None else [])
- for key in named:
- path = from_dotted(key)
- found, wire = _get_path(new, path)
- if not found: # a locked key a document save dropped
- continue
- leaves = intent_diff({}, wire, path)[0] if isinstance(wire, dict) and wire else {path: wire}
- for leaf, leaf_value in leaves.items():
- sets.setdefault(leaf, leaf_value)
- pinned.add(leaf)
- return sets, unsets, frozenset(pinned)
- def _patch_body(self, st: _ProfileState, intent_sets: dict[KeyPath, Any], unsets: list[KeyPath],
- pinned: frozenset = frozenset()
- ) -> Optional[tuple[dict[str, Any], dict[KeyPath, Any], list[KeyPath]]]:
- """The PATCH body applying the intent to the current doc, or None when that is no change.
- A *pinned* key equal to the inherited value is still sent, unless a lock already freezes
- it (nothing to store, and refusing a no-op pick of the locked value helps nobody)."""
- base = self._diff_base(st)
- new = copy.deepcopy(base)
- apply_intent(new, intent_sets, unsets)
- sets, unsets = diff(base, new)
- for path in pinned:
- value = intent_sets[path]
- if isinstance(value, dict) and not value:
- continue # "make this a mapping" (apply_intent): sent above when absent, kept when present
- if path not in sets and covering_lock(st.locks, path) is None \
- and not any(covers(p, path) for p in sets):
- sets[path] = copy.deepcopy(value)
- refused = write_check(sets, unsets, st.locks)
- if refused is not None:
- raise ConfigLockedError(encode(refused[0]), refused[1],
- f"{encode(refused[0])} is locked by the {refused[1]} level of Remote Config")
- for p, v in sets.items():
- bad = secret_literal_path(p, v)
- if bad is not None:
- raise ConfigValueError(
- f"{encode(bad)} is a secret-shaped key: Remote Config stores only a ${{VAR}} reference "
- "there, never the secret itself. Put the value in .env and set a ${VAR} reference.",
- code="config_secret_literal")
- if not sets and not unsets:
- return None
- set_body, unset_body = encode_changes(sets, unsets)
- body: dict[str, Any] = {"expectedVersion": st.profile_version}
- # The stamp describes the profile level's STORED data (contract §5.3: absent = unchanged).
- # Stamp only when that data is already at this agent's version, or unstamped (null: readers
- # already treat it as current, R6). An older stamp means the level still holds pre-migration
- # keys that D12 forbids writing back migrated; re-stamping would make later readers skip
- # those migration steps. A newer stamp must not be lowered either.
- latest = _latest_config_version()
- if st.profile_writer is None or st.profile_writer == latest:
- body["writerConfigVersion"] = latest
- if set_body:
- body["set"] = set_body
- if unset_body:
- body["unset"] = unset_body
- return body, sets, unsets
- def _migration_check_input(self, st: _ProfileState, body: dict[str, Any], sets: dict[KeyPath, Any],
- unsets: list[KeyPath]) -> Optional[tuple[dict[str, Any], int]]:
- """``(stored doc after this write, its schema version)`` when readers will migrate it in
- memory after the write (R6: the oldest level's stamp), else None. Call under st.lock."""
- if not sets:
- return None
- try:
- from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION
- except ImportError:
- return None
- latest = _latest_config_version()
- profile = latest if "writerConfigVersion" in body else st.profile_writer
- stamps = [*st.other_writers, *([profile] if isinstance(profile, int) else [])]
- after = min(stamps) if stamps else latest
- if not SUPPORT_FLOOR_VERSION <= after < latest:
- return None
- stored = copy.deepcopy(st.server_config)
- stored.pop("_config_version", None)
- for p, v in sets.items():
- _set_path(stored, p, copy.deepcopy(v))
- for p in unsets:
- _pop_path(stored, p)
- return stored, after
- def _check_migration_keeps(self, st: _ProfileState, sets: dict[KeyPath, Any],
- stored: dict[str, Any], after: int) -> None:
- """Refuse a write whose values every later read would migrate away.
- The plane stores the write as sent, but under a level stamp older than this agent's schema
- (the stamp keeps describing the level's untouched legacy data, contract §5.3), so every
- reader — this process and any later one — re-runs the migrations from that stamp over the
- stored doc. A value one of those steps rewrites (an old default the user set on purpose)
- would be acknowledged and then never read back. Refused instead, before anything is sent."""
- migrated = self._run_private(st, stored, after)
- latest = _latest_config_version()
- for p, v in sets.items():
- found, got = _get_path(migrated, p)
- changed = _first_change(p, v, found, got)
- if changed is not None:
- raise ConfigValueError(
- f"Remote Config: {encode(changed)} cannot be saved with that value. This profile's remote "
- f"settings are stored at config schema v{after}, and every read migrates them to "
- f"v{latest} in memory, which rewrites that value (migrations are never written back, "
- "D12), so it would never be read back. Nothing was sent; choose another value, or "
- "have the profile level re-saved at the current schema on the config plane.",
- code="config_migration_conflict")
- @staticmethod
- def _write_error(resp: client.Response) -> ConfigWriteError:
- if resp.status == 403 and resp.error == "config_key_locked":
- return ConfigLockedError(str(resp.body.get("path") or ""), str(resp.body.get("lockedBy") or "upper"),
- f"Remote Config refused the write: {resp.message}")
- if (resp.status, resp.error) in _VALUE_REFUSALS:
- # The submitted value is the problem (contract §2.5 table): the caller's to fix, so it
- # keeps its code and the plane's refusal text, which names a path and a reason, never a
- # value (config_secret_literal never echoes it).
- detail = resp.message or resp.error
- path, reason = resp.body.get("path"), resp.body.get("reason")
- extras = [f"{k} {v}" for k, v in (("path", path), ("reason", reason))
- if isinstance(v, str) and v and v not in detail]
- if extras:
- detail = f"{detail} ({', '.join(extras)})"
- return ConfigValueError(f"Remote Config refused the write: {detail}", code=resp.error)
- return ConfigWriteError(f"Remote Config refused the write ({resp.status} {resp.error}): {resp.message}",
- code=resp.error)
- @staticmethod
- def _inherited_notes(st: _ProfileState, unsets: list[KeyPath]) -> list[str]:
- """R5: removing a key at the profile level cannot remove a value an upper level supplies."""
- notes: list[str] = []
- for p in unsets:
- node: Any = st.server_config
- for seg in p:
- if not isinstance(node, dict) or seg not in node:
- break
- node = node[seg]
- else:
- notes.append(f"Remote Config: {encode(p)} is still set by an upper level; removing it from this "
- "profile cannot remove an inherited value (set it to null for 'no value')")
- return notes
- def apply_in_memory(self, home: Path, doc: dict) -> None:
- st = self._state(home)
- m = _private_for(st)
- if m is not None: # a migration step: lands in that migration's private copy only
- m.doc = copy.deepcopy(doc)
- m.gen += 1
- return
- with st.lock: # outside a migration: the doc and the base writes diff against move together
- doc = copy.deepcopy(doc)
- base = copy.deepcopy(doc)
- base.pop("_config_version", None)
- self._publish_locked(st, doc, base)
- # --- ConfigBackend: capabilities and boot -----------------------------------------------
- def supports_file_tooling(self) -> bool:
- return False
- def honors_managed_config(self) -> bool:
- return False
- def protected_env_names(self) -> frozenset:
- return PLANE_CREDENTIAL_ENV_NAMES
- def deployment_env_names(self) -> frozenset:
- from hermes_cli.config_backend import BACKEND_ENV
- return frozenset({BACKEND_ENV, client.URL_ENV, client.INSTANCE_ENV})
- def bootstrap_env_names(self) -> frozenset:
- return PLANE_CREDENTIAL_ENV_NAMES # the deployment (selector, plane, instance) and the credential
- @contextlib.contextmanager
- def preserve_read_base(self):
- held = _READ_BASE.get()
- try:
- yield
- finally:
- if held is not None:
- _READ_BASE.set(held)
- def doctor_warnings(self, home: Path) -> tuple:
- managed = managed_config_file()
- if managed is None:
- return ()
- return ((f"{managed} exists but is ignored in remote config mode", "(remove it)",
- f"Remove {managed}: remote config mode takes config and locks only from Remote Config"),)
- def boot(self, home: Path) -> None:
- """Design §4.4 steps 2-4: fetch this home's layer (fail closed) and flag a managed file."""
- self._state(home)
- if not self._managed_warned:
- self._managed_warned = True
- path = managed_config_file()
- if path is not None:
- msg = (f"WARNING: {path} exists but is IGNORED: HERMES_CONFIG_BACKEND=remote takes config "
- "and locks only from Remote Config (D18). Remove the file.")
- logger.warning(msg)
- print(msg, file=sys.stderr)
- def destination(self, home: Path) -> str:
- return f"Remote Config (profile {self._profile_for(home)!r} level)"
- def describe(self, home: Path) -> str:
- st = self._states.get(self._key(home))
- head = f"Remote Config {client.shown_url()} (instance {client.instance_id() or '<unset>'})"
- if st is None:
- return f"{head}: profile not fetched yet"
- age = int(time.time() - st.fetched_at) if st.fetched_at else -1
- tail = f", last poll failed: {st.last_error}" if st.last_error else ""
- return (f"{head}: profile {st.profile!r}, profile level v{st.profile_version}, "
- f"{len(st.locks)} lock(s), fetched {age}s ago{tail}")
- # --- poller (D9) ------------------------------------------------------------------------
- def _ensure_poller(self) -> None:
- with self._lock:
- if self._poller is not None and self._poller.is_alive() and self._poller_pid == os.getpid():
- return
- self._poller_pid = os.getpid() # a forked child re-arms its own thread
- self._poller = threading.Thread(target=self._poll_loop, name="remote-config-poll", daemon=True)
- self._poller.start()
- def _poll_loop(self) -> None:
- while not self._stop.is_set():
- try:
- now = time.monotonic()
- roster = self._roster() # a snapshot: a lazily added profile must not break iteration
- for st in roster:
- if st.next_poll <= now:
- self.poll_one(st)
- wake = min((st.next_poll for st in self._roster()), default=now + poll_interval())
- except Exception:
- logger.warning("Remote Config: poll loop error; retrying", exc_info=True)
- wake = time.monotonic() + MIN_POLL_SECONDS
- self._stop.wait(max(1.0, wake - time.monotonic()))
- def poll_one(self, st: _ProfileState) -> bool:
- """One conditional GET (contract §11.3). Never raises and never exits: an error keeps the
- in-memory document and is retried next tick."""
- try:
- changed = self._fetch_into(st, conditional=True)
- except _FetchFailed as exc:
- st.last_error = str(exc)
- # contract §11.3: a refused credential needs an operator; anything else is a blip.
- level = logging.ERROR if exc.status in (401, 403) else logging.WARNING
- logger.log(level, "Remote Config: poll for profile %r failed (%s); keeping the loaded config",
- st.profile, exc)
- changed = False
- except Exception as exc:
- st.last_error = str(exc)
- logger.warning("Remote Config: poll for profile %r failed", st.profile, exc_info=True)
- changed = False
- st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1)
- if changed:
- logger.info("Remote Config: profile %r changed (profile level v%d)", st.profile, st.profile_version)
- return changed
- def poll_all(self) -> None:
- for st in self._roster():
- self.poll_one(st)
- def managed_config_file() -> Optional[Path]:
- from hermes_cli import managed_scope
- managed_dir = managed_scope.get_managed_dir()
- path = managed_dir / "config.yaml" if managed_dir else None
- if path is None:
- return None
- return path if path.exists() else None # config-reader: ok — existence only, to warn it is ignored (D18)
- def _first_change(path: KeyPath, sent: Any, found: bool, got: Any) -> Optional[KeyPath]:
- """The first path at or under *path* where *got* (as read back) differs from *sent*."""
- if not found:
- return path
- if isinstance(sent, dict) and isinstance(got, dict):
- for key, value in sent.items():
- changed = _first_change(path + (key,), value, key in got, got.get(key))
- if changed is not None:
- return changed
- return None
- try:
- return None if json_equal(to_wire(copy.deepcopy(got), path), sent) else path
- except ConfigValueError:
- return path
|