backend.py 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951
  1. """The remote config backend (config-config design §4, contract.md §11, §15).
  2. The user layer of each profile home is the effective document the config plane resolves for
  3. (this instance, that profile). It lives in memory only: fetched at boot (fail closed, D2), kept
  4. fresh by a per-process ETag poller (D9), and written back as key-level changes on the profile
  5. level with CAS (D10). Nothing is read from or written to a local ``config.yaml``.
  6. """
  7. from __future__ import annotations
  8. import contextlib
  9. import contextvars
  10. import copy
  11. import itertools
  12. import logging
  13. import os
  14. import random
  15. import sys
  16. import threading
  17. import time
  18. from dataclasses import dataclass, field
  19. from pathlib import Path
  20. from typing import Any, Dict, List, Optional, Tuple
  21. from hermes_cli.config_backend import (
  22. Changes, ConfigBackendUnavailable, ConfigLockedError, ConfigValueError, ConfigWriteError, UserLayer,
  23. pinned_keys, refused_locked_omission_prefixes)
  24. from . import client
  25. from .credentials import PLANE_CREDENTIAL_ENV_NAMES, PlaneCredentialError
  26. from .diff import (_get_path, _pop_path, _set_path, apply_intent, diff, encode_changes, intent_diff, json_equal,
  27. strip_locked, write_check)
  28. from .paths import Path as KeyPath
  29. from .paths import PathError, covering_lock, covers, decode, encode, from_dotted
  30. from .values import secret_literal_path, to_wire
  31. logger = logging.getLogger(__name__)
  32. POLL_SECONDS = 300.0 # contract §11.3; a plane-served interval is a later change
  33. MIN_POLL_SECONDS = 30.0 # contract §11.3: lower values are clamped
  34. BOOT_RETRY_DELAYS = (10.0, 20.0) # contract §11.4: attempts at t = 0, ~10 s, ~30 s
  35. MAX_RETRY_WAIT = 20.0
  36. def poll_interval() -> float:
  37. return max(POLL_SECONDS, MIN_POLL_SECONDS)
  38. def _latest_config_version() -> int:
  39. from hermes_cli.config_defaults import DEFAULT_CONFIG
  40. version = DEFAULT_CONFIG.get("_config_version")
  41. return version if isinstance(version, int) else 1
  42. def _log_notes(notes: list[str]) -> None:
  43. """Log messages collected while a backend lock was held (never log under one: a log handler
  44. can read config, and a thread holding the handler's lock may be waiting for that lock)."""
  45. for note in notes:
  46. if note.startswith("debug:"):
  47. logger.debug("%s", note[len("debug:"):])
  48. else:
  49. logger.warning("%s", note)
  50. class _FetchFailed(Exception):
  51. def __init__(self, detail: str, *, retryable: bool, retry_after: Optional[float] = None,
  52. status: Optional[int] = None) -> None:
  53. super().__init__(detail)
  54. self.retryable = retryable
  55. self.retry_after = retry_after
  56. self.status = status
  57. @dataclass
  58. class _ProfileState:
  59. home: Path
  60. profile: str
  61. server_config: dict[str, Any] = field(default_factory=dict) # EffectiveResponse.config, as sent
  62. doc: dict[str, Any] = field(default_factory=dict) # what readers get (+ _config_version, migrated)
  63. etag: str = ""
  64. profile_version: int = 0
  65. # The profile level's stored writerConfigVersion: None = stored null, _UNKNOWN = not reported.
  66. profile_writer: Any = None
  67. other_writers: list[int] = field(default_factory=list) # the other levels' int stamps (R6)
  68. locks: list[tuple[KeyPath, str]] = field(default_factory=list)
  69. provenance: dict[str, str] = field(default_factory=dict)
  70. changed_ns: int = 0
  71. gen: int = 0
  72. installed: int = 0 # bumped by every _install; a GET started before a newer install is stale
  73. base: Optional[dict[str, Any]] = None # the migrated doc writes diff against (None = server_config)
  74. postprocessed: bool = False # st.doc is the completed migration output of generation `installed`
  75. fetched_at: float = 0.0
  76. last_error: Optional[str] = None
  77. next_poll: float = 0.0
  78. lock: threading.RLock = field(default_factory=threading.RLock)
  79. _UNKNOWN = object()
  80. @dataclass
  81. class _Private:
  82. """An in-memory migration's working copy of one profile's document (D12).
  83. The migration steps read and persist through ``hermes_cli.config`` like any reader and writer;
  84. while one runs, THAT thread sees this copy instead of the published one (``_PRIVATE``), and
  85. everyone else keeps seeing the published doc with its matching diff base. The migrated doc and
  86. its base are published together, in one step, when the migration ends: a concurrent writer
  87. never diffs a half-migrated document against the unmigrated base, or the reverse."""
  88. st: _ProfileState
  89. doc: dict[str, Any]
  90. serial: int
  91. gen: int = 0
  92. _PRIVATE: contextvars.ContextVar[Optional[_Private]] = contextvars.ContextVar("remote_config_private", default=None)
  93. _PRIVATE_SERIAL = itertools.count(1)
  94. # The diff base each profile had when THIS context last read it: {id(state): base}. A document
  95. # save means "turn what I read into this", so its intent is diffed against that read, never
  96. # against whatever a poll (or another writer) installed between the read and the save. Replaced,
  97. # never mutated: copied contexts (threads, tasks) must not share one dict.
  98. _READ_BASE: contextvars.ContextVar[Optional[dict]] = contextvars.ContextVar("remote_config_read_base", default=None)
  99. def _remember_read(st: _ProfileState) -> None:
  100. """Record (by reference: ``st.base`` / ``st.server_config`` are replaced, never mutated) the
  101. base this context just read under st.lock."""
  102. seen = dict(_READ_BASE.get() or {})
  103. seen[id(st)] = (st, st.base if st.base is not None else st.server_config)
  104. _READ_BASE.set(seen)
  105. def _read_base_for(st: _ProfileState) -> Optional[dict[str, Any]]:
  106. held = (_READ_BASE.get() or {}).get(id(st))
  107. if held is None or held[0] is not st or held[1] is None:
  108. return None # never read in this context, or a different (reset) profile state
  109. base = copy.deepcopy(held[1])
  110. base.pop("_config_version", None)
  111. return base
  112. def _private_for(st: _ProfileState) -> Optional[_Private]:
  113. m = _PRIVATE.get()
  114. return m if m is not None and m.st is st else None
  115. # Plane refusals caused by the submitted value (contract §2.5, §11.1): surfaced to the caller as
  116. # ConfigValueError with the plane's code. config_request_invalid is deliberately absent: a request
  117. # the plane cannot parse is this client's bug, not the user's value, so it stays opaque.
  118. _VALUE_REFUSALS = frozenset({
  119. (400, "config_secret_literal"), (400, "config_value_invalid"), (400, "config_path_invalid"),
  120. (400, "config_path_reserved"), (413, "config_level_too_large"), (413, "config_body_too_large"),
  121. })
  122. def _profile_writer(body: dict[str, Any]) -> Any:
  123. """The profile level's stored ``writerConfigVersion`` (int or None), or ``_UNKNOWN``."""
  124. for lv in body.get("levels") or []:
  125. if isinstance(lv, dict) and lv.get("kind") == "profile":
  126. w = lv.get("writerConfigVersion")
  127. return w if w is None or isinstance(w, int) else _UNKNOWN
  128. return _UNKNOWN
  129. def _effective_ok(body: dict[str, Any]) -> bool:
  130. return (isinstance(body.get("config"), dict) and isinstance(body.get("profileVersion"), int)
  131. and isinstance(body.get("locks"), list))
  132. class RemoteBackend:
  133. name = "remote"
  134. def __init__(self) -> None:
  135. self._states: dict[str, _ProfileState] = {}
  136. self._fetch_locks: dict[str, threading.Lock] = {}
  137. self._lock = threading.Lock()
  138. self._local = threading.local() # .booting: the home keys this thread is loading
  139. self._poller: Optional[threading.Thread] = None
  140. self._poller_pid: Optional[int] = None
  141. self._stop = threading.Event()
  142. self._managed_warned = False
  143. self._unknown_warned: set = set()
  144. # --- homes and state --------------------------------------------------------------------
  145. @staticmethod
  146. def _key(home: Path) -> str:
  147. from hermes_constants import hermes_home_key
  148. return hermes_home_key(home)
  149. @staticmethod
  150. def _profile_for(home: Path) -> str:
  151. from hermes_constants import PROFILE_ID_RE, profile_name_for_home
  152. name = profile_name_for_home(home)
  153. if not name or not PROFILE_ID_RE.match(name):
  154. raise ConfigBackendUnavailable(
  155. f"Remote Config: {home} is not a Hermes profile home, so it has no remote config "
  156. "(HERMES_CONFIG_BACKEND=remote serves profile homes only).")
  157. return name
  158. def _state(self, home: Path) -> _ProfileState:
  159. key = self._key(home)
  160. st = self._states.get(key)
  161. if st is not None:
  162. poller = self._poller
  163. # A forked child has no poller thread, and a poller that died must not leave every
  164. # cached profile without updates until the next new profile happens to re-arm it.
  165. if poller is None or self._poller_pid != os.getpid() or not poller.is_alive():
  166. self._ensure_poller()
  167. return st
  168. if key in self._booting():
  169. # This thread is loading this profile (resolving its credential or fetching): the read
  170. # can only be served by the load it is part of. Waiting would wait on itself forever.
  171. raise ConfigBackendUnavailable(
  172. f"Remote Config: a config read for {home} re-entered the load of that profile's config "
  173. "on the same thread (code on the boot path read config before it was loaded). This is "
  174. "a Hermes bug; refusing the read instead of deadlocking.")
  175. with self._lock:
  176. fetch_lock = self._fetch_locks.setdefault(key, threading.Lock())
  177. booting = self._booting()
  178. booting.add(key)
  179. try:
  180. st = self._boot_fetch(Path(home), key, fetch_lock)
  181. finally:
  182. booting.discard(key)
  183. self._ensure_poller()
  184. return st
  185. def _booting(self) -> set[str]:
  186. """Home keys this thread is loading now (a thread-local set)."""
  187. keys = getattr(self._local, "booting", None)
  188. if keys is None:
  189. keys = self._local.booting = set()
  190. return keys
  191. def _roster(self) -> list[_ProfileState]:
  192. with self._lock:
  193. return list(self._states.values())
  194. def _boot_fetch(self, home: Path, key: str, fetch_lock: threading.Lock) -> _ProfileState:
  195. """Boot fetch with bounded retry (contract §11.4); fails closed (D2).
  196. Each attempt resolves the credential first, holding no backend lock, then takes the fetch
  197. lock only for the GET: other threads wait there for the fetch in flight and share its
  198. result. Nothing is logged and nobody is waited for while that lock is held."""
  199. profile = self._profile_for(home)
  200. if not client.instance_id():
  201. raise ConfigBackendUnavailable(
  202. f"Remote Config: {client.INSTANCE_ENV} is not set, so Hermes cannot identify this agent "
  203. "to the config plane. Hermes does not start without its remote config.")
  204. delays = list(BOOT_RETRY_DELAYS)
  205. while True:
  206. notes: list[str] = []
  207. try:
  208. token = self._bearer(home)
  209. with fetch_lock:
  210. st = self._states.get(key)
  211. if st is None:
  212. st = _ProfileState(home=home, profile=profile)
  213. self._fetch_into(st, conditional=False, token=token, notes=notes)
  214. with self._lock: # the poller snapshots the roster under the same lock
  215. self._states[key] = st
  216. _log_notes(notes)
  217. return st
  218. except _FetchFailed as exc:
  219. _log_notes(notes)
  220. if exc.retryable and delays:
  221. wait = min(max(exc.retry_after or 0.0, delays.pop(0)), MAX_RETRY_WAIT)
  222. logger.warning("Remote Config: fetch for profile %r failed (%s); retrying in %.0fs",
  223. profile, exc, wait)
  224. time.sleep(wait)
  225. continue
  226. raise ConfigBackendUnavailable(
  227. f"Remote Config: cannot load the config for profile {profile!r} from "
  228. f"{client.shown_url()}: {exc}. Hermes does not start without its remote config "
  229. "(HERMES_CONFIG_BACKEND=remote has no local fallback).") from exc
  230. @staticmethod
  231. def _bearer(home: Path) -> str:
  232. try:
  233. return client.bearer(home)
  234. except PlaneCredentialError as exc:
  235. raise _FetchFailed(str(exc), retryable=exc.retryable) from exc
  236. def _fetch_into(self, st: _ProfileState, *, conditional: bool, token: Optional[str] = None,
  237. notes: Optional[list[str]] = None) -> bool:
  238. """GET ``/self`` into ``st``; True when the document changed. Raises :class:`_FetchFailed`.
  239. The GET runs outside ``st.lock`` (a slow plane must not stall writers), so a write can
  240. install a newer document while it is in flight. Its response is then stale, even when its
  241. ``profileVersion`` matches (an upper level may have changed in between): it is dropped, and
  242. the next poll fetches again. Messages go to ``notes`` (logged by the caller once it holds no
  243. backend lock), or are logged here when there is no list."""
  244. if notes is None:
  245. notes = []
  246. try:
  247. return self._fetch_into(st, conditional=conditional, token=token, notes=notes)
  248. finally:
  249. _log_notes(notes)
  250. started = st.installed
  251. try:
  252. resp = client.request("GET", st.home, st.profile, etag=st.etag if conditional else None, token=token)
  253. except PlaneCredentialError as exc:
  254. raise _FetchFailed(str(exc), retryable=exc.retryable) from exc
  255. except client.TransportError as exc:
  256. raise _FetchFailed(str(exc), retryable=True) from exc
  257. if resp.status == 304 and conditional:
  258. st.fetched_at, st.last_error = time.time(), None
  259. return False
  260. if resp.status == 200 and _effective_ok(resp.body):
  261. with st.lock:
  262. if st.installed != started:
  263. notes.append(f"debug:Remote Config: dropped a fetch for profile {st.profile!r} that a write overtook")
  264. return False
  265. notes.extend(self._install(st, resp.body, resp.etag))
  266. return True
  267. if resp.status == 200:
  268. raise _FetchFailed("the plane returned a malformed response", retryable=True)
  269. retryable = resp.status >= 500 or resp.status == 429
  270. raise _FetchFailed(f"HTTP {resp.status} {resp.error}: {resp.message}", retryable=retryable,
  271. retry_after=resp.retry_after, status=resp.status)
  272. def _install(self, st: _ProfileState, body: dict[str, Any], etag: Optional[str]) -> list[str]:
  273. """Install a fetched document (the caller holds st.lock); returns warnings to log later."""
  274. notes: list[str] = []
  275. config = body["config"]
  276. doc = copy.deepcopy(config)
  277. doc.pop("_config_version", None) # reserved on the wire (§3.6); never expected here
  278. # R6: the in-memory schema version is the oldest writer's; migrations run in memory (D12).
  279. writer_versions: list[int] = [
  280. lv["writerConfigVersion"] for lv in body.get("levels") or []
  281. if isinstance(lv, dict) and isinstance(lv.get("writerConfigVersion"), int)]
  282. doc["_config_version"] = min(writer_versions) if writer_versions else _latest_config_version()
  283. locks: list[tuple[KeyPath, str]] = []
  284. for entry in body.get("locks") or []:
  285. try:
  286. locks.append((decode(entry["path"]), str(entry["level"])))
  287. except (KeyError, TypeError, PathError):
  288. notes.append(f"Remote Config: ignoring malformed lock entry {entry!r}")
  289. st.server_config = config
  290. st.doc = doc
  291. st.etag = etag or str(body.get("etag") or "")
  292. st.profile_version = int(body["profileVersion"])
  293. st.profile_writer = _profile_writer(body)
  294. st.other_writers = [
  295. lv["writerConfigVersion"] for lv in body.get("levels") or []
  296. if isinstance(lv, dict) and lv.get("kind") != "profile" and isinstance(lv.get("writerConfigVersion"), int)]
  297. st.locks = locks
  298. st.provenance = dict(body.get("provenance") or {})
  299. st.changed_ns = time.time_ns()
  300. st.gen += 1
  301. st.installed += 1
  302. st.base = None
  303. st.postprocessed = False
  304. st.fetched_at, st.last_error = time.time(), None
  305. st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1)
  306. return notes
  307. _MAX_POSTPROCESS = 8 # passes when polls keep installing newer docs mid-migration
  308. def _completed(self, st: _ProfileState) -> bool:
  309. """Bring the published doc to its completed postprocess output; False when that is not
  310. possible yet (``hermes_cli.config`` still importing during the boot fetch)."""
  311. if _private_for(st) is not None:
  312. return True # a migration's own reads use its private copy (never another pass)
  313. for _ in range(self._MAX_POSTPROCESS):
  314. ready = self._postprocess(st)
  315. if not ready:
  316. return False
  317. with st.lock:
  318. if st.postprocessed:
  319. return True
  320. raise ConfigWriteError("Remote Config: the profile's config kept changing while it was migrated in memory",
  321. code="config_version_conflict")
  322. def _postprocess(self, st: _ProfileState) -> bool:
  323. """First read after a change: warn on unknown keys and migrate in memory (D12). Runs on a
  324. read, not at fetch, because both need ``hermes_cli.config``, which may still be importing
  325. when the boot fetch runs (then False: nothing could be done).
  326. No single flight: a reader or writer that finds the published doc unmigrated runs its own
  327. private pass, even while another thread's pass is in flight, and the first pass to finish
  328. publishes. Waiting for the other pass instead could deadlock (a migration reads through
  329. hermes_cli.config's _CONFIG_LOCK, which the waiting caller may hold), and returning the
  330. unmigrated doc meanwhile would hand that caller legacy settings the migration removes."""
  331. if _private_for(st) is not None:
  332. return True # a migration's own reads: never start another one
  333. with st.lock:
  334. if st.postprocessed:
  335. return True
  336. # The doc, its schema version and its generation are captured together: a poll may
  337. # install a newer doc at any point after this, and migrating that doc from THIS version
  338. # would run steps its data never needed (and, D12, drop settings it holds on purpose).
  339. seen = st.installed
  340. current = int(st.doc.get("_config_version") or 0)
  341. doc = copy.deepcopy(st.doc)
  342. try:
  343. from hermes_cli.config import _known_top_level_keys
  344. from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION
  345. except ImportError:
  346. return False # still importing; the next read retries
  347. for key in sorted(set(doc) - _known_top_level_keys() - {"_config_version"}):
  348. if key not in self._unknown_warned:
  349. self._unknown_warned.add(key)
  350. logger.warning("Remote Config: unknown config key %r is ignored by this Hermes version", key)
  351. latest = _latest_config_version()
  352. if SUPPORT_FLOOR_VERSION <= current < latest:
  353. self._migrate_in_memory(st, seen, current, doc)
  354. return True
  355. if current < SUPPORT_FLOOR_VERSION:
  356. logger.warning("Remote Config: profile %r was written by config version %d, below the "
  357. "migration floor %d; not migrated", st.profile, current, SUPPORT_FLOOR_VERSION)
  358. with st.lock:
  359. if st.installed == seen: # a doc installed meanwhile needs its own pass
  360. st.postprocessed = True
  361. return True
  362. def _run_private(self, st: _ProfileState, doc: dict[str, Any], current: int) -> dict[str, Any]:
  363. """Run the migrations from schema *current* over a private copy of *doc* and return it.
  364. Nothing is published: only this thread sees the copy while the steps run (:class:`_Private`)."""
  365. from hermes_cli.config_migrations import run_migrations
  366. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  367. m = _Private(st=st, doc=copy.deepcopy(doc), serial=next(_PRIVATE_SERIAL))
  368. m.doc["_config_version"] = current
  369. private_token = _PRIVATE.set(m)
  370. home_token = set_hermes_home_override(st.home)
  371. try:
  372. # config_only: a read (or a refused write's check) must change no local file (D12).
  373. run_migrations(current, {"env_added": [], "config_added": [], "warnings": []}, True, config_only=True)
  374. finally:
  375. reset_hermes_home_override(home_token)
  376. _PRIVATE.reset(private_token)
  377. return m.doc
  378. def _migrate_in_memory(self, st: _ProfileState, started: int, current: int, doc: dict[str, Any]) -> None:
  379. """Migrate doc generation *started* (*doc*, schema version *current*) in memory (D12).
  380. The steps work on a private copy; the result is published with its diff base in one step,
  381. and only while generation *started* is still the installed one and no concurrent pass over
  382. it has published first: once a poll or write has installed another doc, nothing of this
  383. migration lands on it (that doc gets its own pass).
  384. The migrated doc becomes the base later writes diff against: a reader was handed the
  385. migrated doc, so a write that leaves the migration's changes in place must not send them
  386. (D12: never write the migration back)."""
  387. with st.lock:
  388. if st.installed != started or st.postprocessed:
  389. return # replaced, or another pass already published this generation
  390. migrated = self._run_private(st, doc, current)
  391. migrated["_config_version"] = _latest_config_version()
  392. base = copy.deepcopy(migrated)
  393. base.pop("_config_version", None)
  394. with st.lock:
  395. if st.installed != started or st.postprocessed:
  396. return # a poll or write installed a newer doc, or a concurrent pass published
  397. self._publish_locked(st, migrated, base)
  398. st.postprocessed = True
  399. logger.info("Remote Config: migrated profile %r in memory from config version %d", st.profile, current)
  400. @staticmethod
  401. def _publish_locked(st: _ProfileState, doc: dict[str, Any], base: dict[str, Any]) -> None:
  402. st.doc = doc
  403. st.base = base
  404. st.changed_ns = time.time_ns()
  405. st.gen += 1
  406. # --- ConfigBackend: reads ---------------------------------------------------------------
  407. def _snapshot(self, st: _ProfileState, take):
  408. """``take()`` under st.lock once the published doc is completed postprocess output (or
  409. this thread's own migration copy). A poll that installs a newer doc between completing
  410. and taking sends the caller round again, so no reader ever sees an unmigrated doc."""
  411. for _ in range(self._MAX_POSTPROCESS):
  412. ready = self._completed(st)
  413. with st.lock:
  414. if not ready or st.postprocessed or _private_for(st) is not None:
  415. return take()
  416. raise ConfigWriteError("Remote Config: the profile's config kept changing while it was read",
  417. code="config_version_conflict")
  418. def read_user_layer(self, home: Path) -> UserLayer:
  419. st = self._state(home)
  420. def take():
  421. m = _private_for(st)
  422. if m is None:
  423. _remember_read(st)
  424. return (copy.deepcopy(m.doc if m is not None else st.doc), self._version_of(st),
  425. {encode(p): level for p, level in st.locks})
  426. doc, version, locks = self._snapshot(st, take)
  427. return UserLayer(doc=doc, version=version, locks=locks,
  428. provenance=f"remote:{client.shown_url()} profile={st.profile}")
  429. def read_user_doc(self, home: Path) -> Any:
  430. return self.read_user_layer(home).doc
  431. def probe_readable(self, home: Path) -> None:
  432. # Readable once this home's layer is fetched; an unreachable plane raises
  433. # ConfigBackendUnavailable (never an OSError: there is no local file to retry).
  434. self._state(home)
  435. def read_user_doc_readonly(self, home: Path) -> Any:
  436. st = self._state(home)
  437. def take():
  438. m = _private_for(st)
  439. if m is None:
  440. _remember_read(st)
  441. return m.doc if m is not None else st.doc
  442. return self._snapshot(st, take)
  443. @staticmethod
  444. def _version_of(st: _ProfileState) -> tuple[Any, ...]:
  445. # Leads with a nanosecond timestamp like the file backend's st_mtime_ns (the TUI's config
  446. # watcher reads element 0 as a time); the etag and local generation make it exact. A
  447. # migration's own reads see its private copy under a version of their own, so no config
  448. # cache ever pairs the published version with a private doc, or the reverse. It must not
  449. # START with the published version either: config caches match a signature as a prefix.
  450. m = _private_for(st)
  451. if m is not None:
  452. return (-m.serial, m.gen, st.changed_ns, st.etag, st.gen)
  453. return (st.changed_ns, st.etag, st.gen)
  454. def version(self, home: Path) -> tuple[Any, ...]:
  455. # A config cache validates on this signature and then hands out the doc it cached for it:
  456. # that doc is what the caller now holds, so this is the read a later save diffs against.
  457. st = self._state(home)
  458. with st.lock:
  459. if _private_for(st) is None and st.postprocessed:
  460. _remember_read(st)
  461. return self._version_of(st)
  462. def cache_version(self, home: Path, version: tuple[Any, ...]) -> tuple[Any, ...]:
  463. # The version is exact (etag + local generation): a remote write the cache cannot see
  464. # always changes it, so nothing is added (the file backend adds a racy content digest).
  465. return version
  466. def exists(self, home: Path) -> bool:
  467. return True # the plane always resolves a document (an absent level is empty, contract R4)
  468. def locked(self, home: Path, dotted: str) -> Optional[str]:
  469. path = from_dotted(dotted)
  470. for lock, level in self._state(home).locks:
  471. if len(lock) <= len(path) and tuple(path[:len(lock)]) == lock:
  472. return level
  473. return None
  474. # --- ConfigBackend: writes --------------------------------------------------------------
  475. _MAX_PREPARE = 3 # re-preparations when a poll lands between preparing and sending
  476. def write_changes(self, home: Path, changes: Changes) -> None:
  477. """Send ``changes`` as one PATCH with CAS (contract §15.2), re-read and retry once on a
  478. conflict (§15.3).
  479. The edit is fixed once, against the doc the caller read (its *intent*: the key-level sets
  480. and unsets that turn that doc into ``changes``). A retry re-diffs that intent against the
  481. re-read doc, with the re-read locks; it never replays the caller's whole document, which
  482. would turn every key another writer changed meanwhile into an edit of ours."""
  483. st = self._state(home)
  484. try: # before st.lock: resolving the credential may read files and the network
  485. token = client.bearer(st.home)
  486. except PlaneCredentialError as exc:
  487. raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc
  488. notes: list[str] = []
  489. try:
  490. self._send_changes(st, changes, token, notes)
  491. finally:
  492. _log_notes(notes) # never under st.lock
  493. def _send_changes(self, st: _ProfileState, changes: Changes, token: str, notes: list[str]) -> None:
  494. intent: Optional[tuple[dict[KeyPath, Any], list[KeyPath], frozenset]] = None
  495. for attempt in (1, 2):
  496. for _ in range(self._MAX_PREPARE):
  497. # Diff against the doc readers get, i.e. migrated in memory (D12) — also after the
  498. # CAS re-read below or a poll landing mid-prepare. Outside st.lock: a migration
  499. # reads through hermes_cli.config.
  500. ready = self._completed(st)
  501. with st.lock:
  502. if ready and not st.postprocessed:
  503. continue # a poll installed a newer doc since: complete that one first
  504. seen = st.installed
  505. if intent is None:
  506. intent = self._intent(st, changes)
  507. built = self._patch_body(st, *intent)
  508. if built is None:
  509. return
  510. body, sets, unsets = built
  511. check = self._migration_check_input(st, body, sets, unsets)
  512. if check is not None: # outside st.lock, like any migration run
  513. self._check_migration_keeps(st, sets, *check)
  514. with st.lock:
  515. if st.installed != seen:
  516. continue # a poll landed meanwhile: prepare again against it
  517. try:
  518. resp = client.request("PATCH", st.home, st.profile, body=body, token=token)
  519. except (PlaneCredentialError, client.TransportError) as exc:
  520. raise ConfigWriteError(f"Remote Config write failed: {exc}", code="config_plane_unreachable") from exc
  521. if resp.status == 200 and _effective_ok(resp.body):
  522. notes.extend(self._install(st, resp.body, resp.etag))
  523. _remember_read(st) # the caller now holds what it wrote
  524. notes.extend(self._inherited_notes(st, unsets))
  525. return
  526. if resp.status == 409 and attempt == 1:
  527. try: # contract §15.3: re-read, re-diff, retry once
  528. self._fetch_into(st, conditional=False, token=token, notes=notes)
  529. except _FetchFailed as exc:
  530. raise ConfigWriteError(f"Remote Config write conflict and re-read failed: {exc}",
  531. code="config_version_conflict") from exc
  532. break
  533. raise self._write_error(resp)
  534. else:
  535. raise ConfigWriteError("Remote Config write not sent: the profile's config kept changing "
  536. "while the write was prepared", code="config_version_conflict")
  537. def _diff_base(self, st: _ProfileState) -> dict[str, Any]:
  538. base = copy.deepcopy(st.base if st.base is not None else st.server_config)
  539. base.pop("_config_version", None)
  540. return base
  541. def _intent(self, st: _ProfileState, changes: Changes
  542. ) -> tuple[dict[KeyPath, Any], list[KeyPath], frozenset]:
  543. """The key-level edit ``changes`` makes to the last read doc (§15.2 steps 1-5).
  544. A document is diffed against the base THIS context read it from (``_READ_BASE``): a poll
  545. landing between the caller's read and its save must not turn every key the poll changed
  546. into an edit of ours, sent with the already-current expectedVersion (no 409 to catch it).
  547. A keyed set/unset names its own keys, so it applies to the current base.
  548. The third item is the leaf paths a keyed ``set`` (or, for a document save, a
  549. :func:`~hermes_cli.config_backend.pinning_keys` block) names: they are stored at the profile
  550. level even when the value equals the inherited one, as the file backend writes them into
  551. ``config.yaml``. Otherwise ``config set k v`` with ``v`` equal to the upper level's value
  552. would report success, store nothing, and follow the next upper-level change."""
  553. base = self._diff_base(st)
  554. if changes.document is not None:
  555. read = _read_base_for(st) if _private_for(st) is None else None
  556. if read is not None:
  557. base = read
  558. new = to_wire(copy.deepcopy(changes.document))
  559. if not isinstance(new, dict):
  560. raise ConfigValueError("the config document must be a mapping")
  561. else:
  562. new = copy.deepcopy(base)
  563. for key, value in changes.set.items():
  564. path = from_dotted(key)
  565. _set_path(new, path, to_wire(value, path))
  566. for key in changes.unset:
  567. _pop_path(new, from_dotted(key))
  568. new.pop("_config_version", None)
  569. if changes.document is not None:
  570. base, new, dropped = strip_locked(base, new, st.locks)
  571. for prefix in refused_locked_omission_prefixes():
  572. whole = from_dotted(prefix)
  573. for path in dropped:
  574. if covers(whole, path) or covers(path, whole):
  575. level = next((lv for lk, lv in st.locks if lk == path), "upper")
  576. raise ConfigLockedError(encode(path), level,
  577. f"{encode(path)} is locked by the {level} level of Remote Config")
  578. if dropped:
  579. print(f"Note: {len(dropped)} setting(s) locked by Remote Config were not saved: "
  580. f"{', '.join(sorted(encode(p) for p in dropped))}", file=sys.stderr)
  581. # Leaf-level: a key added under a section the doc lacked stays a key-level edit, so a
  582. # retry after 409 keeps the siblings another writer put in that section meanwhile.
  583. sets, unsets = intent_diff(base, new)
  584. pinned: set[KeyPath] = set()
  585. named = list(changes.set) + (list(pinned_keys()) if changes.document is not None else [])
  586. for key in named:
  587. path = from_dotted(key)
  588. found, wire = _get_path(new, path)
  589. if not found: # a locked key a document save dropped
  590. continue
  591. leaves = intent_diff({}, wire, path)[0] if isinstance(wire, dict) and wire else {path: wire}
  592. for leaf, leaf_value in leaves.items():
  593. sets.setdefault(leaf, leaf_value)
  594. pinned.add(leaf)
  595. return sets, unsets, frozenset(pinned)
  596. def _patch_body(self, st: _ProfileState, intent_sets: dict[KeyPath, Any], unsets: list[KeyPath],
  597. pinned: frozenset = frozenset()
  598. ) -> Optional[tuple[dict[str, Any], dict[KeyPath, Any], list[KeyPath]]]:
  599. """The PATCH body applying the intent to the current doc, or None when that is no change.
  600. A *pinned* key equal to the inherited value is still sent, unless a lock already freezes
  601. it (nothing to store, and refusing a no-op pick of the locked value helps nobody)."""
  602. base = self._diff_base(st)
  603. new = copy.deepcopy(base)
  604. apply_intent(new, intent_sets, unsets)
  605. sets, unsets = diff(base, new)
  606. for path in pinned:
  607. value = intent_sets[path]
  608. if isinstance(value, dict) and not value:
  609. continue # "make this a mapping" (apply_intent): sent above when absent, kept when present
  610. if path not in sets and covering_lock(st.locks, path) is None \
  611. and not any(covers(p, path) for p in sets):
  612. sets[path] = copy.deepcopy(value)
  613. refused = write_check(sets, unsets, st.locks)
  614. if refused is not None:
  615. raise ConfigLockedError(encode(refused[0]), refused[1],
  616. f"{encode(refused[0])} is locked by the {refused[1]} level of Remote Config")
  617. for p, v in sets.items():
  618. bad = secret_literal_path(p, v)
  619. if bad is not None:
  620. raise ConfigValueError(
  621. f"{encode(bad)} is a secret-shaped key: Remote Config stores only a ${{VAR}} reference "
  622. "there, never the secret itself. Put the value in .env and set a ${VAR} reference.",
  623. code="config_secret_literal")
  624. if not sets and not unsets:
  625. return None
  626. set_body, unset_body = encode_changes(sets, unsets)
  627. body: dict[str, Any] = {"expectedVersion": st.profile_version}
  628. # The stamp describes the profile level's STORED data (contract §5.3: absent = unchanged).
  629. # Stamp only when that data is already at this agent's version, or unstamped (null: readers
  630. # already treat it as current, R6). An older stamp means the level still holds pre-migration
  631. # keys that D12 forbids writing back migrated; re-stamping would make later readers skip
  632. # those migration steps. A newer stamp must not be lowered either.
  633. latest = _latest_config_version()
  634. if st.profile_writer is None or st.profile_writer == latest:
  635. body["writerConfigVersion"] = latest
  636. if set_body:
  637. body["set"] = set_body
  638. if unset_body:
  639. body["unset"] = unset_body
  640. return body, sets, unsets
  641. def _migration_check_input(self, st: _ProfileState, body: dict[str, Any], sets: dict[KeyPath, Any],
  642. unsets: list[KeyPath]) -> Optional[tuple[dict[str, Any], int]]:
  643. """``(stored doc after this write, its schema version)`` when readers will migrate it in
  644. memory after the write (R6: the oldest level's stamp), else None. Call under st.lock."""
  645. if not sets:
  646. return None
  647. try:
  648. from hermes_cli.config_migrations import SUPPORT_FLOOR_VERSION
  649. except ImportError:
  650. return None
  651. latest = _latest_config_version()
  652. profile = latest if "writerConfigVersion" in body else st.profile_writer
  653. stamps = [*st.other_writers, *([profile] if isinstance(profile, int) else [])]
  654. after = min(stamps) if stamps else latest
  655. if not SUPPORT_FLOOR_VERSION <= after < latest:
  656. return None
  657. stored = copy.deepcopy(st.server_config)
  658. stored.pop("_config_version", None)
  659. for p, v in sets.items():
  660. _set_path(stored, p, copy.deepcopy(v))
  661. for p in unsets:
  662. _pop_path(stored, p)
  663. return stored, after
  664. def _check_migration_keeps(self, st: _ProfileState, sets: dict[KeyPath, Any],
  665. stored: dict[str, Any], after: int) -> None:
  666. """Refuse a write whose values every later read would migrate away.
  667. The plane stores the write as sent, but under a level stamp older than this agent's schema
  668. (the stamp keeps describing the level's untouched legacy data, contract §5.3), so every
  669. reader — this process and any later one — re-runs the migrations from that stamp over the
  670. stored doc. A value one of those steps rewrites (an old default the user set on purpose)
  671. would be acknowledged and then never read back. Refused instead, before anything is sent."""
  672. migrated = self._run_private(st, stored, after)
  673. latest = _latest_config_version()
  674. for p, v in sets.items():
  675. found, got = _get_path(migrated, p)
  676. changed = _first_change(p, v, found, got)
  677. if changed is not None:
  678. raise ConfigValueError(
  679. f"Remote Config: {encode(changed)} cannot be saved with that value. This profile's remote "
  680. f"settings are stored at config schema v{after}, and every read migrates them to "
  681. f"v{latest} in memory, which rewrites that value (migrations are never written back, "
  682. "D12), so it would never be read back. Nothing was sent; choose another value, or "
  683. "have the profile level re-saved at the current schema on the config plane.",
  684. code="config_migration_conflict")
  685. @staticmethod
  686. def _write_error(resp: client.Response) -> ConfigWriteError:
  687. if resp.status == 403 and resp.error == "config_key_locked":
  688. return ConfigLockedError(str(resp.body.get("path") or ""), str(resp.body.get("lockedBy") or "upper"),
  689. f"Remote Config refused the write: {resp.message}")
  690. if (resp.status, resp.error) in _VALUE_REFUSALS:
  691. # The submitted value is the problem (contract §2.5 table): the caller's to fix, so it
  692. # keeps its code and the plane's refusal text, which names a path and a reason, never a
  693. # value (config_secret_literal never echoes it).
  694. detail = resp.message or resp.error
  695. path, reason = resp.body.get("path"), resp.body.get("reason")
  696. extras = [f"{k} {v}" for k, v in (("path", path), ("reason", reason))
  697. if isinstance(v, str) and v and v not in detail]
  698. if extras:
  699. detail = f"{detail} ({', '.join(extras)})"
  700. return ConfigValueError(f"Remote Config refused the write: {detail}", code=resp.error)
  701. return ConfigWriteError(f"Remote Config refused the write ({resp.status} {resp.error}): {resp.message}",
  702. code=resp.error)
  703. @staticmethod
  704. def _inherited_notes(st: _ProfileState, unsets: list[KeyPath]) -> list[str]:
  705. """R5: removing a key at the profile level cannot remove a value an upper level supplies."""
  706. notes: list[str] = []
  707. for p in unsets:
  708. node: Any = st.server_config
  709. for seg in p:
  710. if not isinstance(node, dict) or seg not in node:
  711. break
  712. node = node[seg]
  713. else:
  714. notes.append(f"Remote Config: {encode(p)} is still set by an upper level; removing it from this "
  715. "profile cannot remove an inherited value (set it to null for 'no value')")
  716. return notes
  717. def apply_in_memory(self, home: Path, doc: dict) -> None:
  718. st = self._state(home)
  719. m = _private_for(st)
  720. if m is not None: # a migration step: lands in that migration's private copy only
  721. m.doc = copy.deepcopy(doc)
  722. m.gen += 1
  723. return
  724. with st.lock: # outside a migration: the doc and the base writes diff against move together
  725. doc = copy.deepcopy(doc)
  726. base = copy.deepcopy(doc)
  727. base.pop("_config_version", None)
  728. self._publish_locked(st, doc, base)
  729. # --- ConfigBackend: capabilities and boot -----------------------------------------------
  730. def supports_file_tooling(self) -> bool:
  731. return False
  732. def honors_managed_config(self) -> bool:
  733. return False
  734. def protected_env_names(self) -> frozenset:
  735. return PLANE_CREDENTIAL_ENV_NAMES
  736. def deployment_env_names(self) -> frozenset:
  737. from hermes_cli.config_backend import BACKEND_ENV
  738. return frozenset({BACKEND_ENV, client.URL_ENV, client.INSTANCE_ENV})
  739. def bootstrap_env_names(self) -> frozenset:
  740. return PLANE_CREDENTIAL_ENV_NAMES # the deployment (selector, plane, instance) and the credential
  741. @contextlib.contextmanager
  742. def preserve_read_base(self):
  743. held = _READ_BASE.get()
  744. try:
  745. yield
  746. finally:
  747. if held is not None:
  748. _READ_BASE.set(held)
  749. def doctor_warnings(self, home: Path) -> tuple:
  750. managed = managed_config_file()
  751. if managed is None:
  752. return ()
  753. return ((f"{managed} exists but is ignored in remote config mode", "(remove it)",
  754. f"Remove {managed}: remote config mode takes config and locks only from Remote Config"),)
  755. def boot(self, home: Path) -> None:
  756. """Design §4.4 steps 2-4: fetch this home's layer (fail closed) and flag a managed file."""
  757. self._state(home)
  758. if not self._managed_warned:
  759. self._managed_warned = True
  760. path = managed_config_file()
  761. if path is not None:
  762. msg = (f"WARNING: {path} exists but is IGNORED: HERMES_CONFIG_BACKEND=remote takes config "
  763. "and locks only from Remote Config (D18). Remove the file.")
  764. logger.warning(msg)
  765. print(msg, file=sys.stderr)
  766. def destination(self, home: Path) -> str:
  767. return f"Remote Config (profile {self._profile_for(home)!r} level)"
  768. def describe(self, home: Path) -> str:
  769. st = self._states.get(self._key(home))
  770. head = f"Remote Config {client.shown_url()} (instance {client.instance_id() or '<unset>'})"
  771. if st is None:
  772. return f"{head}: profile not fetched yet"
  773. age = int(time.time() - st.fetched_at) if st.fetched_at else -1
  774. tail = f", last poll failed: {st.last_error}" if st.last_error else ""
  775. return (f"{head}: profile {st.profile!r}, profile level v{st.profile_version}, "
  776. f"{len(st.locks)} lock(s), fetched {age}s ago{tail}")
  777. # --- poller (D9) ------------------------------------------------------------------------
  778. def _ensure_poller(self) -> None:
  779. with self._lock:
  780. if self._poller is not None and self._poller.is_alive() and self._poller_pid == os.getpid():
  781. return
  782. self._poller_pid = os.getpid() # a forked child re-arms its own thread
  783. self._poller = threading.Thread(target=self._poll_loop, name="remote-config-poll", daemon=True)
  784. self._poller.start()
  785. def _poll_loop(self) -> None:
  786. while not self._stop.is_set():
  787. try:
  788. now = time.monotonic()
  789. roster = self._roster() # a snapshot: a lazily added profile must not break iteration
  790. for st in roster:
  791. if st.next_poll <= now:
  792. self.poll_one(st)
  793. wake = min((st.next_poll for st in self._roster()), default=now + poll_interval())
  794. except Exception:
  795. logger.warning("Remote Config: poll loop error; retrying", exc_info=True)
  796. wake = time.monotonic() + MIN_POLL_SECONDS
  797. self._stop.wait(max(1.0, wake - time.monotonic()))
  798. def poll_one(self, st: _ProfileState) -> bool:
  799. """One conditional GET (contract §11.3). Never raises and never exits: an error keeps the
  800. in-memory document and is retried next tick."""
  801. try:
  802. changed = self._fetch_into(st, conditional=True)
  803. except _FetchFailed as exc:
  804. st.last_error = str(exc)
  805. # contract §11.3: a refused credential needs an operator; anything else is a blip.
  806. level = logging.ERROR if exc.status in (401, 403) else logging.WARNING
  807. logger.log(level, "Remote Config: poll for profile %r failed (%s); keeping the loaded config",
  808. st.profile, exc)
  809. changed = False
  810. except Exception as exc:
  811. st.last_error = str(exc)
  812. logger.warning("Remote Config: poll for profile %r failed", st.profile, exc_info=True)
  813. changed = False
  814. st.next_poll = time.monotonic() + poll_interval() * random.uniform(0.9, 1.1)
  815. if changed:
  816. logger.info("Remote Config: profile %r changed (profile level v%d)", st.profile, st.profile_version)
  817. return changed
  818. def poll_all(self) -> None:
  819. for st in self._roster():
  820. self.poll_one(st)
  821. def managed_config_file() -> Optional[Path]:
  822. from hermes_cli import managed_scope
  823. managed_dir = managed_scope.get_managed_dir()
  824. path = managed_dir / "config.yaml" if managed_dir else None
  825. if path is None:
  826. return None
  827. return path if path.exists() else None # config-reader: ok — existence only, to warn it is ignored (D18)
  828. def _first_change(path: KeyPath, sent: Any, found: bool, got: Any) -> Optional[KeyPath]:
  829. """The first path at or under *path* where *got* (as read back) differs from *sent*."""
  830. if not found:
  831. return path
  832. if isinstance(sent, dict) and isinstance(got, dict):
  833. for key, value in sent.items():
  834. changed = _first_change(path + (key,), value, key in got, got.get(key))
  835. if changed is not None:
  836. return changed
  837. return None
  838. try:
  839. return None if json_equal(to_wire(copy.deepcopy(got), path), sent) else path
  840. except ConfigValueError:
  841. return path