update_host_obligation.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480
  1. """Host-scoped update→restart obligation for ``hermes update``.
  2. Multiplex-only (Teknium ruling): exactly ONE ``hermes gateway run`` per host serves every
  3. profile, so "this pull still owes the fleet a restart" is a property of the HOST, not of one
  4. profile's ``HERMES_HOME``. The legacy ``$HERMES_HOME/fleet_restart_pending`` marker was
  5. per-home: ``hermes -p coder update`` armed and cleared coder's copy while restarting the
  6. SHARED process, and every other profile's CLI could neither see nor discharge that obligation
  7. — it simply armed its own and re-killed the same host process.
  8. The record therefore lives beside the host rendezvous record, in
  9. :func:`gateway.host_rendezvous.host_state_dir` (``$HERMES_GATEWAY_LOCK_DIR`` else
  10. ``$XDG_STATE_HOME/hermes/gateway-locks``) — the one cross-profile, per-OS-user state root the
  11. tree already has. It is written once per host and installation (keyed by the checkout), read by
  12. every profile's CLI, and cleared once.
  13. The same "one host process, not one per profile" identity is what
  14. :func:`collapse_units_to_host_processes` applies to enumerated systemd units: leftover
  15. per-profile ``hermes-gateway-<p>.service`` units on a multiplexed host all point at the same
  16. live ``MainPID``, so restarting each one restarts the host process N times.
  17. """
  18. from __future__ import annotations
  19. import base64
  20. import contextlib
  21. import hashlib
  22. import json
  23. import logging
  24. import os
  25. import stat
  26. import sys
  27. import tempfile
  28. import time
  29. from pathlib import Path
  30. from typing import Any, Callable, Iterable, Optional
  31. from hermes_dir_acl import make_user_dir
  32. logger = logging.getLogger("hermes_cli.update_cmd")
  33. #: The record before it carried an install key, beside ``host-gateway.json`` / ``host-serve.json``.
  34. #: Still read, and cleared by a discharge that judged it, so a debt an older release armed survives
  35. #: the upgrade; it names no install, so it keeps its old meaning: owed by whichever install reads it.
  36. HOST_OBLIGATION_NAME = "host-update-restart.json"
  37. #: The per-``HERMES_HOME`` breadcrumb an unwritable host record falls back to; every reader honours it.
  38. PROFILE_MARKER_NAME = "fleet_restart_pending"
  39. _RECORD_VERSION = 1
  40. def host_obligation_path() -> Path:
  41. """This installation's host obligation record.
  42. Keyed by the checkout this module runs from: every profile of one install arms, reads and
  43. clears the same record, while a second install of the same OS user (another checkout, a ZIP
  44. install) owes its own, so its completed restart never erases this install's debt (review S3).
  45. """
  46. # Not ``gateway.host_rendezvous.host_state_dir``: ``gateway.status`` imports ``utils`` -> ruamel,
  47. # absent from the historical interpreter an old updater's takeover arms this record in; the
  48. # recovery module's stdlib copy of the rule is drift-tested against the gateway resolver.
  49. from hermes_cli.update_restart_recovery import _host_state_dir
  50. install = os.path.normcase(str(Path(__file__).resolve().parents[1]))
  51. key = hashlib.sha256(install.encode("utf-8")).hexdigest()[:16]
  52. return Path(_host_state_dir()) / f"{Path(HOST_OBLIGATION_NAME).stem}-{key}.json"
  53. def _owed_path() -> Path:
  54. """The record a reader honours: this install's, else an older release's unkeyed one."""
  55. path = host_obligation_path()
  56. legacy = path.with_name(HOST_OBLIGATION_NAME)
  57. return legacy if not os.path.exists(path) and os.path.exists(legacy) else path # never raises
  58. def _write_record(path: Path, record: dict) -> None:
  59. """``utils.atomic_json_write(path, record, mode=0o600)`` in stdlib only (see above)."""
  60. replace_bytes(path, json.dumps(record, indent=2, ensure_ascii=False).encode("utf-8"))
  61. def read_host_obligation() -> Optional[dict]:
  62. """The published obligation record, or ``None`` when absent/corrupt/foreign-versioned."""
  63. return _read_record(_owed_path())
  64. def _read_record(path: Path) -> Optional[dict]:
  65. try:
  66. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  67. except (OSError, UnicodeDecodeError, ValueError):
  68. return None
  69. if not isinstance(payload, dict) or payload.get("version") != _RECORD_VERSION:
  70. return None
  71. return payload
  72. def host_obligation_present() -> bool:
  73. """True when the record FILE exists, parseable or not.
  74. Fail-closed: a corrupt record is an obligation whose terms are unknown, never a discharged
  75. one — the restart is still owed and the reader falls back to "no recorded inventory".
  76. """
  77. try:
  78. return _owed_path().is_file()
  79. except OSError:
  80. return False
  81. def amend_host_obligation(**fields: Any) -> None:
  82. """Merge ``fields`` into the armed record (test/diagnostic surface). Never raises."""
  83. _update_record(lambda record: record.update(fields), "amend")
  84. def write_host_obligation(
  85. *, expected_sha: str = "", runtimes: Optional[list] = None, profile: str = "", owner: str = ""
  86. ) -> bool:
  87. """Arm the host obligation. True when it was written. Never raises.
  88. Re-arming from a second profile for the SAME pulled SHA keeps the existing record (and its
  89. ``restarted`` proof) instead of resetting it: the host owes one restart, not one per profile.
  90. ``owner`` (an update run's commit-point token) joins the record's ``owners``: a run that
  91. fails before its move hands back only its own stake (``release_host_obligation``), never
  92. another run's debt for the same SHA. The first owner also stores what it found there
  93. (``found``), which the last owner to leave puts back; an owner retargeting to another SHA
  94. keeps that baseline (``_baseline_for``).
  95. """
  96. path = host_obligation_path()
  97. try:
  98. with _record_mutex(path):
  99. return _write_locked(path, expected_sha=expected_sha, runtimes=runtimes, profile=profile, owner=owner)
  100. except OSError as exc: # the mutex busy or unopenable: the caller falls back to the per-home marker
  101. logger.debug("Could not lock the host update-restart obligation: %s", exc)
  102. return False
  103. def arm_host_obligation(
  104. marker: Path, *, expected_sha: str = "", runtimes: Optional[list] = None, profile: str = "", owner: str = ""
  105. ) -> bool:
  106. """Arm the host record, else ``marker`` (the arming profile's per-home breadcrumb). Never raises.
  107. The one fallback for every arm (``hermes update``'s commit point, a historical updater's
  108. takeover). An unwritable host state dir (``HERMES_GATEWAY_LOCK_DIR`` on a read-only mount, a
  109. container UID that does not own ``$HOME``) must never disarm the obligation (#117275), but the
  110. marker is visible to the arming profile only: it settles the debt only on an install PROVEN to
  111. have no other profile, and an unreadable profile roster proves nothing (review S2 residual).
  112. Otherwise the marker is still left for this profile, the gap is said out loud, and the result
  113. is False: the commit point refuses to move; a takeover, whose tree already moved, keeps the warning.
  114. """
  115. if write_host_obligation(expected_sha=expected_sha, runtimes=runtimes, profile=profile, owner=owner):
  116. return True
  117. lines = [f"started={time.time()}", f"pid={os.getpid()}"] + ([f"expected_sha={expected_sha}"] if expected_sha else [])
  118. if runtimes is not None:
  119. lines.append("inventory=" + json.dumps({"version": 1, "runtimes": runtimes}))
  120. hidden = ""
  121. try:
  122. replace_bytes(marker, ("\n".join(lines) + "\n").encode("utf-8"))
  123. except OSError as exc:
  124. hidden = f" or {marker} ({exc})"
  125. if not hidden:
  126. try:
  127. if _named_profiles_exist():
  128. hidden = f" ({marker} hides it from the other profiles)"
  129. except OSError as exc:
  130. hidden = f" ({marker} may hide it from other profiles: the profile list is unreadable: {exc})"
  131. if not hidden:
  132. logger.warning("Host update-restart obligation (%s) is unwritable; armed the per-home marker %s instead.",
  133. host_obligation_path(), marker)
  134. return True
  135. logger.error("Could not arm the update-restart obligation in %s%s; an interrupted update will not warn.",
  136. host_obligation_path(), hidden)
  137. # ASCII: a historical updater's takeover may print through a legacy console code page.
  138. print(" Warning: could not record the pending gateway-restart obligation where every profile sees it "
  139. "(state dir not writable) - restart gateways with `hermes gateway restart` if this update is "
  140. "interrupted.", file=sys.stderr)
  141. return False
  142. def _named_profiles_exist() -> bool:
  143. """True when the install has a live named profile beside ``default``, read from a COMPLETE roster.
  144. ``profiles.list_profile_names`` suppresses enumeration errors and reads an unreadable roster as
  145. "default only"; here any error listing the roster or reading an entry's identity raises instead.
  146. Same identity rule as ``hermes_constants.named_profile_is_live`` (stdlib only: see above).
  147. """
  148. from hermes_constants import _PROFILE_IDENTITY_MARKERS, PROFILE_ID_RE, get_default_hermes_root, profile_tombstone_path
  149. try:
  150. entries = list((get_default_hermes_root() / "profiles").iterdir())
  151. except (FileNotFoundError, NotADirectoryError):
  152. return False
  153. for home in entries:
  154. if home.name == "default" or not PROFILE_ID_RE.match(home.name) or not stat.S_ISDIR(_mode(home, os.stat)):
  155. continue
  156. identity = any((mode := _mode(home / name, os.lstat)) and not stat.S_ISDIR(mode)
  157. for name in _PROFILE_IDENTITY_MARKERS)
  158. if identity and not _mode(profile_tombstone_path(home), os.lstat):
  159. return True
  160. return False
  161. def _mode(path: Path, probe: Callable[[Path], os.stat_result]) -> int:
  162. """``probe(path).st_mode``; 0 when absent. Any other error raises: unreadable is unknown, not absent."""
  163. try:
  164. return probe(path).st_mode
  165. except (FileNotFoundError, NotADirectoryError):
  166. return 0
  167. def _write_locked(path: Path, *, expected_sha: str, runtimes: Optional[list], profile: str, owner: str) -> bool:
  168. existing = _read_record(path)
  169. try:
  170. found = _baseline_for(owner, existing, path) if owner else None
  171. except OSError as exc: # unreadable is not absent: a guessed ``found`` would delete it later
  172. logger.debug("Could not read the host update-restart obligation: %s", exc)
  173. return False
  174. if existing is not None and expected_sha and existing.get("expected_sha") == expected_sha:
  175. # Same pull, second profile: the host owes ONE restart, so keep the standing record (and
  176. # any proof that the restart already happened) rather than resetting it. A later arm that
  177. # carries the owed inventory still upgrades it — an inventory-less record owes no set.
  178. fields: dict[str, Any] = {}
  179. owners = _owners(existing)
  180. if owner and owner not in owners:
  181. fields["owners"] = [*owners, owner]
  182. if not owners:
  183. fields["found"] = found
  184. inventory = {"version": 1, "runtimes": runtimes}
  185. if runtimes is not None and existing.get("inventory") != inventory:
  186. fields["inventory"] = inventory
  187. if not fields:
  188. return True
  189. try:
  190. _write_record(path, {**existing, **fields})
  191. except Exception as exc: # health: allow BLE001 -- never raises: the caller falls back to the per-home marker
  192. logger.debug("Could not amend host update-restart obligation: %s", exc)
  193. return False
  194. return True
  195. payload: dict[str, Any] = {
  196. "version": _RECORD_VERSION,
  197. "started": time.time(),
  198. "pid": os.getpid(),
  199. "armed_by_profile": profile or "",
  200. "expected_sha": expected_sha or "",
  201. }
  202. if owner:
  203. payload["owners"] = [owner]
  204. payload["found"] = found
  205. if runtimes is not None:
  206. payload["inventory"] = {"version": 1, "runtimes": runtimes}
  207. try:
  208. make_user_dir(path.parent, parents=True, exist_ok=True)
  209. _write_record(path, payload)
  210. except Exception as exc: # health: allow BLE001 -- never raises: the caller falls back to the per-home marker
  211. logger.debug("Could not write host update-restart obligation: %s", exc)
  212. return False
  213. return True
  214. @contextlib.contextmanager
  215. def _record_mutex(path: Path):
  216. """Serialize every read-judge-write of the record across processes (``update_lock.marker_mutex``).
  217. The sidecar sits in a subdirectory: the state dir is the gateway lock dir, whose ``--replace``
  218. cleanup unlinks every top-level ``*.lock``, and a deleted sidecar splits the lock in two.
  219. """
  220. # One guard for every install's record (and the unkeyed one an older release still locks).
  221. guard = path.parent / f".{Path(HOST_OBLIGATION_NAME).stem}.mutex"
  222. make_user_dir(guard, parents=True, exist_ok=True)
  223. with _marker_mutex()(guard / "record"):
  224. yield
  225. def _marker_mutex():
  226. """``update_lock.marker_mutex``. A historical updater imported ``update_lock`` before its pull and
  227. then lazily imports this pulled module, so its in-memory copy predates the mutex: load the pulled
  228. ``update_lock`` (stdlib-only) beside it, which locks the same sidecar every current process does."""
  229. import importlib.util
  230. import sys
  231. from hermes_cli import update_lock
  232. if hasattr(update_lock, "marker_mutex"):
  233. return update_lock.marker_mutex
  234. name = "hermes_cli._update_lock_pulled"
  235. pulled = sys.modules.get(name)
  236. if pulled is None:
  237. spec = importlib.util.spec_from_file_location(name, update_lock.__file__)
  238. pulled = importlib.util.module_from_spec(spec)
  239. sys.modules[name] = pulled # before exec: dataclasses resolve the module by name
  240. spec.loader.exec_module(pulled)
  241. return pulled.marker_mutex
  242. def _owners(record: dict) -> list[str]:
  243. owners = record.get("owners")
  244. return [str(o) for o in owners] if isinstance(owners, list) else []
  245. def _baseline_for(owner: str, existing: Optional[dict], path: Path) -> Optional[str]:
  246. """What ``owner``'s release must put back (base64; ``None`` = absent): the record as it stood
  247. before ``owner``'s first stake in it (review R1).
  248. A run re-arms with one owner as it retargets (CP0 branch, origin pull, upstream fork ff), so
  249. "the record already has owners" never means "nothing was here": a new owner keeps the bytes
  250. there now, other owners' stakes included; an owner already in the record carries the baseline
  251. it saved then, or, when it shared the record with others, the shared record minus its own stake.
  252. """
  253. owners = _owners(existing or {})
  254. if owner not in owners:
  255. return _found_field(path)
  256. others = [o for o in owners if o != owner]
  257. if not others:
  258. return (existing or {}).get("found")
  259. shared = json.dumps({**(existing or {}), "owners": others}, indent=2, ensure_ascii=False)
  260. return base64.b64encode(shared.encode("utf-8")).decode("ascii")
  261. def _found_field(path: Path) -> Optional[str]:
  262. """The record's current bytes (base64), ``None`` when absent: what a first owner puts back.
  263. Any other read error raises: custody is never guessed."""
  264. try:
  265. return base64.b64encode(path.read_bytes()).decode("ascii")
  266. except FileNotFoundError:
  267. return None
  268. def release_host_obligation(owner: str) -> None:
  269. """Hand back ``owner``'s stake in the record. When no other run still owes through it, the
  270. record goes back to what its first owner found (``found``; absent = unlinked).
  271. A record without ``owner`` was rewritten since (another run's newer pull): it is theirs and
  272. stays. Raises OSError when the record cannot be rewritten (the caller keeps the debt armed).
  273. Judged and rewritten under ``_record_mutex``, which every arm takes too, so no arm can land
  274. between the look and the write and be undone by it (kshitijk4poor F22/N05).
  275. """
  276. path = host_obligation_path()
  277. if not owner or owner not in _owners(_read_record(path) or {}):
  278. return # nothing of ours to hand back: no lock taken, no state dir created
  279. with _record_mutex(path):
  280. _release_locked(path, owner)
  281. def _release_locked(path: Path, owner: str) -> None:
  282. record = _read_record(path)
  283. owners = _owners(record) if record is not None else []
  284. if owner not in owners:
  285. return # rewritten since the unlocked look (another run's arm): theirs now
  286. remaining = [o for o in owners if o != owner]
  287. if remaining:
  288. _write_record(path, {**record, "owners": remaining})
  289. return
  290. try:
  291. found = base64.b64decode(record["found"], validate=True) if record.get("found") is not None else None
  292. except (TypeError, ValueError):
  293. return # a damaged ``found`` cannot be put back: the debt stays armed, never guessed away
  294. if found is None:
  295. path.unlink(missing_ok=True)
  296. else:
  297. replace_bytes(path, found)
  298. def replace_bytes(path: Path, data: bytes) -> None:
  299. """``data`` at ``path`` as one durable record: a fresh 0600 ``mkstemp`` file (never written
  300. through a planted alias), fsynced, renamed over ``path``, then the rename itself made durable."""
  301. fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=f".{path.stem}_", suffix=".tmp")
  302. try:
  303. with os.fdopen(fd, "wb") as handle:
  304. handle.write(data)
  305. handle.flush()
  306. os.fsync(handle.fileno())
  307. os.replace(tmp, path)
  308. except BaseException:
  309. with contextlib.suppress(OSError):
  310. os.unlink(tmp)
  311. raise
  312. if os.name != "nt": # Windows opens no directory handle; NTFS journals the rename itself
  313. directory = os.open(path.parent, os.O_RDONLY)
  314. try:
  315. os.fsync(directory)
  316. finally:
  317. os.close(directory)
  318. def clear_host_obligation() -> None:
  319. """Discharge the record this install's readers judged (``_owed_path``). Never raises.
  320. That is this install's own record whenever it has one. An older release's unkeyed record names
  321. no install: while this install has its own record, the unkeyed one may be another
  322. installation's standing debt that nothing here judged, so it stays (review S3 residual). It goes
  323. only when it was the record judged (this install armed none): it keeps its old meaning, owed by
  324. whichever install reads it, and an upgrade's own interrupted debt must stay dischargeable.
  325. """
  326. try:
  327. _owed_path().unlink(missing_ok=True)
  328. except OSError as exc:
  329. logger.debug("Could not clear host update-restart obligation: %s", exc)
  330. def obligation_fields() -> Optional[dict[str, str]]:
  331. """The obligation in the legacy ``key=value`` field shape, or ``None`` when unarmed.
  332. Keeps one parser for both sources: the fields a reader needs (``expected_sha``, the
  333. serialized ``inventory``) are identical whether they came from the host record or from an
  334. in-flight legacy per-home marker.
  335. """
  336. record = read_host_obligation()
  337. if record is None:
  338. return None
  339. fields = {"expected_sha": str(record.get("expected_sha") or "")}
  340. inventory = record.get("inventory")
  341. if inventory is not None:
  342. fields["inventory"] = json.dumps(inventory)
  343. return fields
  344. def mark_host_restart_completed(sha: str) -> None:
  345. """Record that the host process was restarted onto ``sha``. Never raises."""
  346. _update_record(
  347. lambda record: record.update(restarted={"sha": sha or "", "pid": os.getpid(), "at": time.time()}),
  348. "stamp restart completion on")
  349. def _update_record(change: Callable[[dict], None], what: str) -> None:
  350. """Read, ``change`` and rewrite an armed record under ``_record_mutex`` (an arm's new owner
  351. stake is never overwritten by a stale copy). Never raises; no record, no write."""
  352. path = _owed_path()
  353. if _read_record(path) is None:
  354. return
  355. try:
  356. with _record_mutex(path):
  357. record = _read_record(path)
  358. if record is None:
  359. return
  360. change(record)
  361. _write_record(path, record)
  362. except Exception as exc: # health: allow BLE001 -- never raises: a lost stamp costs a repeat restart, never the debt
  363. logger.debug("Could not %s host update-restart obligation: %s", what, exc)
  364. def host_restart_already_completed(sha: Optional[str]) -> bool:
  365. """True when THIS host obligation was already restarted onto ``sha``.
  366. The guard that makes the catch-up restart idempotent per host: a second profile running
  367. ``hermes update`` must attach to the first restart's outcome, never kill the shared
  368. multiplexer again.
  369. """
  370. record = read_host_obligation()
  371. if record is None or not sha:
  372. return False
  373. restarted = record.get("restarted")
  374. return isinstance(restarted, dict) and str(restarted.get("sha") or "") == sha
  375. def collapse_units_to_host_processes(
  376. units: Iterable[str], main_pid: Callable[[str], int]
  377. ) -> tuple[list[str], dict[str, str]]:
  378. """Split enumerated units into ``(restart, {legacy_unit: covering_unit})``.
  379. Units resolving to the same live ``MainPID`` are ONE host process; restarting each of them
  380. restarts that process N times, which on a multiplexed host is an N-fold outage triggered by
  381. leftover per-profile units. A unit with no readable main PID (inactive, unprivileged scope)
  382. keeps its own restart: identity that cannot be proved is never collapsed away.
  383. """
  384. restart: list[str] = []
  385. covered: dict[str, str] = {}
  386. owner_by_pid: dict[int, str] = {}
  387. for unit in units:
  388. try:
  389. pid = int(main_pid(unit) or 0)
  390. except Exception:
  391. # Identity that cannot be proved keeps its own restart; a probe failure of any kind
  392. # must never abort the whole pass.
  393. pid = 0
  394. if pid <= 0:
  395. restart.append(unit)
  396. continue
  397. owner = owner_by_pid.get(pid)
  398. if owner is None:
  399. owner_by_pid[pid] = unit
  400. restart.append(unit)
  401. else:
  402. covered[unit] = owner
  403. return restart, covered