update_receipt.py 65 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312
  1. """Structured update receipts + post-update fleet version verification.
  2. Phase 1 of the fleet-update reliability plan (#91277): the updater must
  3. *prove* its outcome instead of assuming it.
  4. Two additive capabilities, both designed so a failure inside them can never
  5. break an update (every public entry point is exception-swallowing):
  6. 1. **Update receipt** — a machine-readable JSON record of what one
  7. ``hermes update`` run discovered, did, skipped (and why), written to
  8. the ROOT home's ``logs/update_receipts/`` (``get_default_hermes_root()``,
  9. never a sticky profile's ``HERMES_HOME``). Silent-failure classes this
  10. makes visible: #88848 (helper died after "success" printed), #74973
  11. (restart silently skipped), #85753 (restart phase never ran), #81193
  12. (desktop shows failure for a successful update).
  13. 2. **Fleet version verification** — after the restart phase, read every
  14. profile's ``gateway_state.json``, compare each live gateway's stamped
  15. ``code_sha`` (written by ``gateway/status.py`` on every runtime-status
  16. write) against the freshly-updated checkout's HEAD, and print a fleet
  17. version matrix. Mixed-version fleets (#88654, #69754, #77553, #56717)
  18. become a loud, actionable report instead of a latent state.
  19. Deployment-kind awareness (docker/image-managed installs) rides on
  20. ``hermes_cli.version_info.get_code_identity()``: a packaged build reports
  21. its install-stamp provenance (``source="docker"``/``"nix"``/…) and the
  22. receipt records that the install is not in-place updatable.
  23. """
  24. from __future__ import annotations
  25. import contextvars
  26. import copy
  27. import json
  28. import logging
  29. import os
  30. import re
  31. import subprocess
  32. import sys
  33. import time
  34. import uuid
  35. from contextlib import contextmanager, suppress
  36. from datetime import datetime, timezone, UTC
  37. from pathlib import Path
  38. from typing import Any, Optional
  39. logger = logging.getLogger(__name__)
  40. _RECEIPT_KEEP = 20 # keep the last N receipts in the root home's receipt directory
  41. #: Terminal records finalized by THIS process, by update id (see ``finalized_receipt``).
  42. _FINALIZED: dict[str, dict[str, Any]] = {}
  43. COMMAND_BOUNDARY_STOP_REASON = "completed at command boundary"
  44. # Receipt state is per-CONTEXT, not a module global: a nested
  45. # ``hermes update`` receipt (or one in another thread) must never clobber
  46. # the outer one, and the boundary finalize must see exactly its own
  47. # process's receipt. Same pattern as pm.receipt's ContextVars — no
  48. # manager object.
  49. _current: contextvars.ContextVar[Optional[UpdateReceipt]] = contextvars.ContextVar(
  50. "update_receipt_current", default=None
  51. )
  52. #: The terminal outcome of the run the enclosing command scope finalized or adopted (from a
  53. #: completion child): ``committed_success`` answers the command boundary's gateway status from it.
  54. _scope_terminal: contextvars.ContextVar[Optional[dict[str, Any]]] = contextvars.ContextVar(
  55. "update_receipt_scope_terminal", default=None
  56. )
  57. @contextmanager
  58. def update_receipt_scope():
  59. """Keep the command's finalization guard away from an enclosing update."""
  60. token = _current.set(None)
  61. terminal = _scope_terminal.set({})
  62. try:
  63. yield
  64. finally:
  65. _scope_terminal.reset(terminal)
  66. _current.reset(token)
  67. def adopt_terminal_receipt(data: Any) -> None:
  68. """A completion child finalized this command's run: its terminal outcome is this scope's too."""
  69. cell = _scope_terminal.get()
  70. if cell is not None and isinstance(data, dict) and data.get("finished_at"):
  71. cell["outcome"] = data.get("outcome")
  72. def committed_success() -> bool:
  73. """True when this command's run is closed and its receipt says ``success`` (C3).
  74. The gateway ``/update`` status written at the command boundary follows the receipt: an
  75. interrupt or error that escapes after the run closed as a success never reports it failed.
  76. """
  77. cell = _scope_terminal.get()
  78. return _current.get() is None and bool(cell) and cell.get("outcome") == "success"
  79. def current_correlation_id() -> Optional[str]:
  80. """The update correlation id in force in this context, or None.
  81. Derived from the OPEN update receipt itself — one source of truth, no
  82. duplicate id variable: a nested update's begin replaces the current
  83. receipt (so syncs begun under it capture the nested id), and its
  84. finalize RESTORES the outer receipt via the ContextVar token (so the
  85. outer id comes back for the rest of the outer update)."""
  86. current = _current.get()
  87. return current.correlation_id if current is not None else None
  88. def _utc_now_iso() -> str:
  89. return datetime.now(UTC).isoformat()
  90. def _code_identity(refresh: bool = False) -> dict[str, Any]:
  91. """Running-code identity, or ``{}`` when the probe fails."""
  92. with suppress(Exception):
  93. from hermes_cli.version_info import get_code_identity
  94. return get_code_identity(refresh=refresh) or {}
  95. return {}
  96. def _str_records(entries: Any, keys: tuple[str, ...], *, pid: bool = False) -> list[dict[str, Any]]:
  97. """Dict entries reduced to stringified ``keys`` (plus an int ``pid`` first when requested)."""
  98. records = []
  99. for entry in entries:
  100. if not isinstance(entry, dict):
  101. continue
  102. record: dict[str, Any] = {"pid": int(entry.get("pid", 0) or 0)} if pid else {}
  103. record.update({key: str(entry.get(key, "")) for key in keys})
  104. records.append(record)
  105. return records
  106. def _launcher_correlation_id() -> Optional[str]:
  107. """Correlation id handed in by an external launcher, or None.
  108. Desktop's managed SSH update exports ``HERMES_UPDATE_CORRELATION_ID`` and
  109. only accepts a receipt whose ``correlation_id`` equals it. This is separate
  110. from ``update_id``, which stays the CLI's own id for pm sync receipts."""
  111. return os.environ.get("HERMES_UPDATE_CORRELATION_ID", "").strip() or None
  112. class UpdateReceipt:
  113. """Collects the observable facts of one ``hermes update`` run."""
  114. def __init__(self) -> None:
  115. self.data: dict[str, Any] = {
  116. "schema": 1, "started_at": _utc_now_iso(), "finished_at": None,
  117. "argv": list(sys.argv), "pid": os.getpid(),
  118. "outcome": "running", # running | success | partial | failed
  119. "pre_update": _code_identity(), "post_update": {},
  120. "steps": [], "skips": [], "gateway_restart": {}, "fleet": [],
  121. }
  122. # The id binds this update to the pm sync receipts begun under it
  123. # (pm.receipt captures it via the _correlation ContextVar).
  124. self.correlation_id = uuid.uuid4().hex
  125. self.data["update_id"] = self.correlation_id
  126. self.data["correlation_id"] = _launcher_correlation_id()
  127. # The dashboard action that spawned this run: the receipt store is root-wide, so the
  128. # action-status route certifies an outcome only from a receipt naming ITS action.
  129. self.data["action_id"] = os.environ.get("HERMES_ACTION_ID", "").strip() or None
  130. def step(self, name: str, ok: bool, detail: str = "") -> None:
  131. self.data["steps"].append({"name": name, "ok": bool(ok), "detail": detail, "at": _utc_now_iso()})
  132. def skip(self, name: str, reason: str) -> None:
  133. self.data["skips"].append({"name": name, "reason": reason, "at": _utc_now_iso()})
  134. def stage(self, name: str, outcome: str, **facts: str) -> None:
  135. # Stage END marks: a stage's duration is the gap since the previous mark (or started_at).
  136. self.data.setdefault("stages", []).append({"name": name, "outcome": outcome, "at": _utc_now_iso(), **facts})
  137. def fact(self, key: str, value: Any) -> None:
  138. self.data[key] = value
  139. def followup(self, step: str, reason: str) -> None:
  140. self.data.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()})
  141. def gateway_restart_result(
  142. self, *, restarted_services: list | None = None, relaunched_profiles: list | None = None,
  143. externally_supervised_profiles: list | None = None, killed_pids: list | None = None,
  144. failed_units: list | None = None, incomplete: bool = False, phase_error: str = "",
  145. fresh_recovery: dict[str, Any] | None = None,
  146. ) -> None:
  147. result: dict[str, Any] = {
  148. "restarted_services": list(restarted_services or []),
  149. "relaunched_profiles": list(relaunched_profiles or []),
  150. "externally_supervised_profiles": list(externally_supervised_profiles or []),
  151. "killed_pids": [int(p) for p in (killed_pids or [])],
  152. "failed_units": [str(u) for u in (failed_units or [])],
  153. "incomplete": bool(incomplete),
  154. "phase_error": phase_error,
  155. }
  156. if fresh_recovery is not None:
  157. # Conservative outcome vocabulary: "verified" is the only bucket allowed to claim
  158. # supervisor coverage; "relaunch_attempted" means the relaunch exited 0 without
  159. # independent supervisor observation. "skipped" preserves runtimes (manual gateways,
  160. # serve/dashboard entries) the pass deliberately did not touch.
  161. persisted: dict[str, Any] = {
  162. key: [str(profile) for profile in fresh_recovery.get(key, [])]
  163. for key in ("requested", "verified", "relaunch_attempted", "failed")
  164. }
  165. persisted["skipped"] = _str_records(
  166. fresh_recovery.get("skipped", []), ("profile", "kind", "supervisor", "reason")
  167. )
  168. # ``hermes serve`` hosts tui_gateway and is not a gateway profile, so neither the
  169. # per-profile buckets above nor the fleet-version matrix can describe it. Persist its
  170. # unit outcomes and any process that survived on the pre-update generation, or the
  171. # receipt keeps claiming a clean recovery the operator's box contradicts.
  172. serve_units = fresh_recovery.get("serve_units") or {}
  173. persisted["serve_units"] = {
  174. key: [str(unit) for unit in (serve_units.get(key) or [])] for key in ("verified", "failed")
  175. }
  176. persisted["stale_runtimes"] = _str_records(
  177. fresh_recovery.get("stale_runtimes", []), ("kind", "profile", "supervisor"), pid=True
  178. )
  179. result["fresh_recovery"] = persisted
  180. self.data["gateway_restart"] = result
  181. def finalize(self, outcome: str) -> None:
  182. if outcome == "success" and self.data.get("user_action"):
  183. outcome = "partial" # committed, but the user still has to act (record_user_action)
  184. self.data["outcome"] = outcome
  185. self.data["finished_at"] = _utc_now_iso()
  186. self.data["post_update"] = _code_identity(refresh=True)
  187. def _receipt_dir() -> Path:
  188. # ``hermes_constants`` (stdlib-only), never ``hermes_cli.config``: the receipt must be
  189. # writable from the refused/failed paths where config loading itself may be what broke
  190. # (#112465, #112558). The ROOT home, never a sticky profile's: an update mutates the
  191. # checkout every profile shares, and the Desktop and the hand-off scripts read the root.
  192. from hermes_constants import get_default_hermes_root
  193. return get_default_hermes_root() / "logs" / "update_receipts"
  194. def _receipt_dirs() -> list[Path]:
  195. """The root store, then the profile's own when a named profile makes them differ. ``pm/``
  196. writes its sync receipts to ``get_hermes_home()`` and receipts from before the root move live
  197. there too, so readers take both and writers mirror ``latest.json`` into it (``hermes -p <name>
  198. pm status`` reads only that one)."""
  199. from hermes_constants import get_hermes_home
  200. root, profile = _receipt_dir(), get_hermes_home() / "logs" / "update_receipts"
  201. return [root] if profile.resolve() == root.resolve() else [root, profile]
  202. def _write_latest(payload: bytes) -> None:
  203. """Replace ``latest.json`` in every store a reader of this home consults."""
  204. from hermes_cli.runtime_state import _atomic_bytes
  205. for directory in _receipt_dirs():
  206. directory.mkdir(parents=True, exist_ok=True)
  207. _atomic_bytes(directory / "latest.json", payload)
  208. def _run_file(directory: Path, data: dict[str, Any]) -> Path:
  209. """One archive file per run, named at begin so the running and terminal records coincide."""
  210. stamp = re.sub(r"[^0-9]", "", str(data.get("started_at") or ""))[:14] or time.strftime("%Y%m%d%H%M%S")
  211. return directory / f"update_{stamp[:8]}_{stamp[8:]}_{data.get('pid') or os.getpid()}_{data.get('update_id')}.json"
  212. def read_run_record(update_id: str) -> Optional[tuple[Path, dict[str, Any]]]:
  213. """The run's own archive ``(path, record)`` in the root store, or None. Never raises.
  214. The ONE lookup of a run's archive (named at begin, so its running and terminal records share
  215. one file); never ``latest.json``, which another profile or run may have replaced since. A file
  216. that cannot be read, is not a JSON object, or names another run is not this run's record:
  217. callers treat all of those exactly like a missing archive.
  218. """
  219. with suppress(OSError):
  220. for path in _receipt_dir().glob(f"update_*_{update_id}.json"):
  221. with suppress(OSError, ValueError):
  222. record = json.loads(path.read_text(encoding="utf-8-sig"))
  223. if isinstance(record, dict) and record.get("update_id") == update_id:
  224. return path, record
  225. return None
  226. def _persist_running(data: dict[str, Any]) -> None:
  227. """Write the open run to disk: a killed update still leaves its own record. Never raises."""
  228. with suppress(Exception):
  229. from hermes_cli.runtime_state import _atomic_bytes
  230. directory = _receipt_dir()
  231. directory.mkdir(parents=True, exist_ok=True)
  232. path = _run_file(directory, data)
  233. # A completion child can finalize while its parent still holds the pre-child
  234. # context. That stale running snapshot must never undo the terminal receipt.
  235. stored = read_run_record(str(data.get("update_id")))
  236. if stored is not None and stored[1].get("finished_at"):
  237. return
  238. from hermes_cli.process_identity import _process_create_time
  239. payload = (json.dumps({**data, "writer_pid": os.getpid(),
  240. "writer_create_time": _process_create_time(os.getpid())},
  241. indent=2, default=str) + "\n").encode("utf-8")
  242. _atomic_bytes(path, payload)
  243. _write_latest(payload)
  244. def _owner_alive(record: dict[str, Any]) -> bool:
  245. from hermes_cli.process_identity import _pid_alive_matches
  246. identities = {}
  247. for key, time_key in (("pid", "pid_create_time"), ("writer_pid", "writer_create_time")):
  248. pid = record.get(key)
  249. if isinstance(pid, int) and pid > 0:
  250. # Older receipts recorded no writer incarnation. Do not let a duplicate
  251. # bare PID override the original owner's proven creation-time mismatch.
  252. create_time = record.get(time_key)
  253. if pid not in identities or identities[pid] is None:
  254. identities[pid] = create_time
  255. return any(_pid_alive_matches(pid, create_time) is not False
  256. for pid, create_time in identities.items())
  257. def reconcile_interrupted_runs() -> list[dict[str, Any]]:
  258. """Mark ``running`` records whose processes are gone as ``interrupted``; returns them.
  259. The start-of-run reclaimer for the durable running receipt: the next update says plainly
  260. that the previous one was killed and at which stage, instead of reporting its outcome as
  261. whatever ran before it. Never raises.
  262. """
  263. interrupted: list[dict[str, Any]] = []
  264. with suppress(Exception):
  265. from hermes_cli.runtime_state import _atomic_bytes
  266. directory = _receipt_dir()
  267. latest = read_latest_receipt() or {}
  268. for path in sorted(directory.glob("update_*.json"), key=lambda p: p.stat().st_mtime)[-_RECEIPT_KEEP:]:
  269. with suppress(Exception):
  270. record = json.loads(path.read_text(encoding="utf-8-sig"))
  271. if not isinstance(record, dict) or record.get("outcome") != "running" or _owner_alive(record):
  272. continue
  273. stages = record.get("stages") or []
  274. last = stages[-1].get("name") if stages and isinstance(stages[-1], dict) else None
  275. record.update(outcome="interrupted", interrupted_detected_at=_utc_now_iso(),
  276. stop_reason=f"process exited during the update (last stage: {last or 'start'})")
  277. payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8")
  278. _atomic_bytes(path, payload)
  279. if latest.get("update_id") == record.get("update_id"):
  280. _write_latest(payload)
  281. interrupted.append(record)
  282. for record in interrupted:
  283. print(f"⚠ The previous update ({record.get('started_at')}) was interrupted before it finished "
  284. f"({record.get('stop_reason')}); this update completes the work it still owed.")
  285. return interrupted
  286. def begin_update_receipt(*, previous: dict | None = None, correlation_id: str | None = None) -> None:
  287. """Start recording a new update receipt, durably ``running`` until it finalizes.
  288. Nested updates are safe: the previous receipt (if any) is preserved
  289. behind the ContextVar token and comes back when this one finalizes —
  290. a nested begin/finalize never drops the outer update's receipt or
  291. correlation. Never raises."""
  292. try:
  293. receipt = UpdateReceipt()
  294. if previous:
  295. receipt.data.update(copy.deepcopy(previous))
  296. receipt.correlation_id = correlation_id or receipt.correlation_id
  297. receipt.data.update(update_id=receipt.correlation_id, outcome="running", finished_at=None)
  298. # A handoff receipt from an older interpreter may predate the field.
  299. receipt.data["correlation_id"] = receipt.data.get("correlation_id") or _launcher_correlation_id()
  300. with suppress(Exception):
  301. from hermes_cli.process_identity import _process_create_time
  302. receipt.data.setdefault("pid_create_time", _process_create_time(receipt.data["pid"]))
  303. if not previous:
  304. reconcile_interrupted_runs()
  305. # The running record is about to replace latest.json: snapshot the previous record's
  306. # manual serve rows (raw, no side effects) for finalize's carry-forward. A handed-off
  307. # receipt from an older interpreter predates the field, so it needs the same snapshot.
  308. if not previous or "carried_manual_serves" not in previous:
  309. with suppress(Exception):
  310. prior = read_latest_receipt() or {}
  311. rows = list((prior.get("plan") or {}).get("runtimes") or [])
  312. rows += list(prior.get("pending_manual_serves") or [])
  313. rows += list(prior.get("carried_manual_serves") or []) # a run killed before finalize
  314. from hermes_cli.update_serve_obligations import is_reminder_row
  315. receipt.data["carried_manual_serves"] = [row for row in rows if is_reminder_row(row)]
  316. except Exception as exc: # pragma: no cover - defensive
  317. logger.debug("Could not start update receipt: %s", exc)
  318. return
  319. receipt.current_token = _current.set(receipt)
  320. _persist_running(receipt.data)
  321. def persist_running_receipt() -> None:
  322. """Re-assert the open run on disk (the completion child calls this after resuming it)."""
  323. current = _current.get()
  324. if current is not None and current.data.get("outcome") == "running":
  325. _persist_running(current.data)
  326. #: Follow-up steps whose failure leaves the source-update tail owed (``source-completion-pending``).
  327. TAIL_FOLLOWUPS = frozenset({"dependencies", "launchers", "build", "maintenance", "profile_sync",
  328. "config_migration", "completion"})
  329. #: Follow-ups that mean the BUILD stage did not succeed (C3: the receipt names what actually failed):
  330. #: the products or the launchers the build publishes failed, or the tail raised before proving them.
  331. #: Owed dependencies, config migration or maintenance keep the tail armed but say nothing about the
  332. #: build; they are reported as their own follow-ups.
  333. BUILD_FOLLOWUPS = frozenset({"launchers", "build", "completion"})
  334. def record_build_stage(followups) -> None:
  335. """Mark the build stage from the owed ``(step, reason)`` follow-ups: failed only for a build one."""
  336. record_stage("build", "failed" if any(step in BUILD_FOLLOWUPS for step, _ in followups) else "success")
  337. def record_followup(step: str, reason: str, *, retry: str = "the next launch or `hermes update` retries it") -> None:
  338. """A post-commit step failed: print ⚠, keep the run a success, and say what is still owed.
  339. The code already committed, so the update does not fail; the step's own obligation (the
  340. source tail, the fleet restart, the launch-time bytecode sweep) stays armed and the next
  341. launch or ``hermes update`` retries it. Never raises. The printed line is a protocol: the
  342. Desktop hand-offs (scripts/desktop-update/posix.sh, windows.ps1) parse ``Update follow-up
  343. '<step>' did not finish:`` to report every owed step, so keep it one whole line.
  344. """
  345. reason = " ".join(str(reason).split())[:500] or "failed"
  346. print(f" ⚠ Update follow-up '{step}' did not finish: {reason} ({retry})", flush=True)
  347. current = _current.get()
  348. if current is None:
  349. return
  350. _record("followup", f"update followup {step}", step, reason)
  351. current = _current.get()
  352. if current is not None:
  353. _persist_running(current.data)
  354. def record_user_action(step: str, reason: str) -> None:
  355. """The code committed, but something only the user can do is still owed. Never raises.
  356. Unlike a follow-up nothing retries it (a stash whose restore conflicted stays parked until the
  357. user re-applies it), so the run can never be a plain success: it finalizes ``partial`` and
  358. ``hermes update`` exits 1, as #122557 established for an unrestored autostash.
  359. """
  360. _record("fact", f"update user action {step}", "user_action", {"step": step, "reason": " ".join(str(reason).split())[:500]})
  361. persist_running_receipt()
  362. def amend_terminal_followup(update_id: str, step: str, reason: str) -> None:
  363. """A follow-up that failed after the run's receipt was finalized lands on that receipt. Never raises."""
  364. found = read_run_record(update_id)
  365. if found is None:
  366. return
  367. path, record = found
  368. record.setdefault("followups", []).append({"step": step, "reason": reason, "at": _utc_now_iso()})
  369. payload = (json.dumps(record, indent=2, default=str) + "\n").encode("utf-8")
  370. with suppress(Exception): # an unwritable store: the printed follow-up line still names it
  371. from hermes_cli.runtime_state import _atomic_bytes
  372. _atomic_bytes(path, payload)
  373. if (read_latest_receipt() or {}).get("update_id") == update_id:
  374. _write_latest(payload)
  375. def owe_followup(update_id: Optional[str], step: str, reason: str, **retry: str) -> None:
  376. """A post-commit step failed: record it on the run while its receipt is open, else amend the
  377. run's closed receipt. Amending an OPEN run would append a second copy beside the one
  378. ``record_followup`` just persisted. Never raises."""
  379. record_followup(step, reason, **retry)
  380. if _current.get() is None and update_id:
  381. amend_terminal_followup(update_id, step, reason)
  382. def _record(method: str, what: str, *args: Any, **kwargs: Any) -> None:
  383. """Invoke ``method`` on the active receipt; no-op when none, never raises.
  384. Copy-on-write, same rule as pm.receipt: a copied context (copy_context,
  385. asyncio.to_thread) inherits the SAME receipt object — mutate a clone
  386. and re-set it in THIS context only, so a child's records never leak
  387. into (or corrupt) the parent's receipt.
  388. """
  389. try:
  390. import copy
  391. current = _current.get()
  392. if current is None:
  393. return
  394. clone = copy.copy(current)
  395. clone.data = copy.deepcopy(current.data)
  396. getattr(clone, method)(*args, **kwargs)
  397. _current.set(clone)
  398. except Exception as exc: # pragma: no cover - defensive
  399. logger.debug("Could not record %s: %s", what, exc)
  400. def record_step(name: str, ok: bool, detail: str = "") -> None:
  401. """Record one update step outcome. No-op when no receipt is active."""
  402. _record("step", f"update step {name}", name, ok, detail)
  403. def record_skip(name: str, reason: str) -> None:
  404. """Record a skipped step WITH the reason it was skipped."""
  405. _record("skip", f"update skip {name}", name, reason)
  406. def record_stage(name: str, outcome: str, **facts: str) -> None:
  407. """Mark the END of a pipeline stage (``success``/``failed``/``skipped``) with a timestamp."""
  408. _record("stage", f"update stage {name}", name, outcome, **facts)
  409. # Each stage boundary refreshes the durable running record (a kill names its last stage).
  410. persist_running_receipt()
  411. def record_fact(key: str, value: Any) -> None:
  412. """Set one top-level receipt field (e.g. ``initiator``)."""
  413. _record("fact", f"update fact {key}", key, value)
  414. def record_gateway_restart(**kwargs: Any) -> None:
  415. """Record the gateway restart phase outcome (see UpdateReceipt)."""
  416. _record("gateway_restart_result", "gateway restart result", **kwargs)
  417. def record_stop_reason(reason: str) -> None:
  418. """Name the exit that is about to stop this run, as one closed token (``stop_class``).
  419. Called on the line before the ``sys.exit``/raise it names, so the last one recorded is the
  420. exit that fired. The token must be one of ``shared_metrics_contract.UPDATE_STOP_CLASSES``
  421. (anything else is ignored by the metric); the human ``stop_reason`` and printed text are
  422. untouched. No-op when no receipt is open; never raises.
  423. """
  424. _record("fact", f"update stop {reason}", "stop_class", reason)
  425. #: git could not write a file it needs (``index.lock``'s directory, a tracked file): an OS permission
  426. #: error, not a lock another git holds. SSH's ``Permission denied (publickey)`` is an auth failure.
  427. _GIT_FILE_PERMISSION = re.compile(r"Permission denied(?! *\()|\bEACCES\b")
  428. def git_error_stop_class(exc: BaseException) -> Optional[str]:
  429. """The closed stop token for a git ``CalledProcessError`` that is about to end the run, or None.
  430. Reads only the failed argv and fixed git/Hermes phrases in its output, never stores either.
  431. """
  432. if not isinstance(exc, subprocess.CalledProcessError):
  433. return None
  434. output = exc.stderr or exc.output or ""
  435. output = output.decode("utf-8", "replace") if isinstance(output, bytes) else str(output)
  436. argv = [str(part) for part in exc.cmd] if isinstance(exc.cmd, (list, tuple)) else str(exc.cmd or "").split()
  437. return git_output_stop_class(output, argv, exc.returncode)
  438. def git_output_stop_class(output: str, argv: list[str], returncode: Optional[int] = None) -> Optional[str]:
  439. """:func:`git_error_stop_class` for a git call that returned instead of raising (``_git_run``)."""
  440. if "index.lock" in output and "File exists" in output:
  441. return "git_index_locked" # another git (or a killed one, #132089) holds the index lock
  442. if _GIT_FILE_PERMISSION.search(output):
  443. return "permission_denied" # git may not write a file it needs (index.lock's dir, a tracked file)
  444. if returncode == 124 and "timed out after" in output:
  445. return "git_timeout" # update_cmd._git_run's own timeout text
  446. if "No space left on device" in output:
  447. return "disk_full"
  448. if "stash" in argv:
  449. return "local_changes_blocked" # update_cmd_stash._push_stash saved nothing
  450. if {"checkout", "merge", "reset"} & set(argv):
  451. return "checkout_move_failed"
  452. return None
  453. def _started_by_handoff_partner() -> str | None:
  454. """``delegate`` (the hand-off launched this process as its update child) or ``partner`` when
  455. the update marker names the Desktop hand-off this process runs under, else None.
  456. For an exit before the receipt opens, where this process may NOT hold the lock: at the
  457. update-lock refusal the marker belongs to whichever update holds it, so "a pid other than
  458. ours" (update_cmd._record_update_initiator, which runs under the lock) would call every
  459. refusal Desktop-started. The claim is ours only when it names us as its delegate (the posix and
  460. Windows hand-offs write that line before the update runs) or names our hand-off partner on
  461. line 1 or the delegate line (``HERMES_UPDATE_HANDOFF_PID``, the Tauri updater; our parent, a
  462. launcher between the hand-off and us). The marker is read raw: no liveness probe and no
  463. cleanup (those are lock policy, not a metrics side effect).
  464. """
  465. from hermes_cli.update_lock import _handoff_pid, _parse_marker, update_marker_path
  466. marker = _parse_marker(update_marker_path().read_bytes())
  467. if marker.started_at is None:
  468. return None
  469. pid = os.getpid()
  470. # Our parent too: on Windows the hand-off names the exact process it created (suspended), and
  471. # hermes_bootstrap's relaunch (or the legacy .cmd's cmd.exe) makes the updater its child.
  472. if marker.delegate_pid in (pid, os.getppid()):
  473. return "delegate"
  474. partners = {_handoff_pid(), os.getppid()} - {None, 0, 1, pid}
  475. return "partner" if partners & {marker.pid, marker.delegate_pid} else None
  476. def record_stop_without_receipt(reason: str, outcome: str) -> None:
  477. """Count an exit that fires before this run's receipt opens, in shared metrics only.
  478. No receipt is written, so the exit behaves exactly as before: at the update-lock refusal
  479. ``latest.json`` and the running record belong to the update holding the lock (opening one
  480. would replace its pointer and reconcile its archive), and the Git-operation refusal runs before
  481. the receipt, plan and snapshot by design. The row is derived from the same fields a final
  482. receipt carries (``outcome`` is ``refused`` or ``failed``), including the initiator, so a
  483. Desktop-started refusal reads ``kind=desktop`` like the same run's receipt would. Never raises.
  484. """
  485. with suppress(Exception):
  486. now = _utc_now_iso()
  487. data = {
  488. "schema": 1, "update_id": uuid.uuid4().hex, "started_at": now, "finished_at": now,
  489. "outcome": "refused" if outcome == "refused" else "failed",
  490. "stop_class": reason, "pre_update": {}, "stages": [], "steps": [], "fleet": [],
  491. }
  492. with suppress(Exception): # no marker, or an unreadable one: the CLI default
  493. claim = _started_by_handoff_partner()
  494. if claim:
  495. data["initiator"] = "desktop"
  496. if claim == "delegate":
  497. # The Desktop checked the marker, handed this run off and quit; its result dialog
  498. # reports the refusal as a failed update, so the row must not read a benign one.
  499. data["outcome"] = "failed"
  500. from hermes_cli.observability.shared_metrics_update import record_update_receipt
  501. record_update_receipt(data)
  502. def finalize_update_receipt(outcome: str, fleet: list | None = None, stop_reason: str = "") -> Optional[Path]:
  503. """Finalize + persist the receipt (``success``/``partial``/``failed``/``refused``); path or None.
  504. Exactly-once by construction: the context's receipt is popped first, so a second call (e.g. the
  505. command-boundary safety net after an inner path already finalized) is a no-op returning None.
  506. The receipt is popped via its OWN begin token, so a nested update's finalize RESTORES the outer
  507. update's open receipt and correlation instead of discarding them.
  508. """
  509. current = _current.get()
  510. if current is None:
  511. return None
  512. receipt = copy.copy(current)
  513. receipt.data = copy.deepcopy(current.data)
  514. token = getattr(receipt, "current_token", None)
  515. try:
  516. if token is not None:
  517. _current.reset(token)
  518. else: # pragma: no cover - receipts begun before token binding
  519. _current.set(None)
  520. except ValueError:
  521. # Token from another context (finalize ran in a copied context) —
  522. # pop THIS context only, so exactly-once still holds.
  523. _current.set(None)
  524. try:
  525. receipt.finalize(outcome)
  526. if stop_reason:
  527. receipt.data["stop_reason"] = stop_reason
  528. # The terminal facts exist before the store is touched; a store that refuses the write
  529. # (after the commit point) must not erase them for this process's own caller.
  530. _FINALIZED[str(receipt.data.get("update_id"))] = receipt.data
  531. adopt_terminal_receipt(receipt.data)
  532. if fleet is not None:
  533. receipt.data["fleet"] = fleet
  534. # Manual serve restart obligations outlive one receipt rotation: carry the previous
  535. # receipt's still-pending rows forward so the startup warning survives (see
  536. # update_serve_obligations).
  537. from hermes_cli.update_serve_obligations import retain_receipt_manual_serves
  538. if "carried_manual_serves" in receipt.data:
  539. prior = {"pending_manual_serves": receipt.data.pop("carried_manual_serves") or []}
  540. else: # a receipt begun before the running record existed
  541. prior = read_latest_receipt() or {}
  542. pending = retain_receipt_manual_serves(prior)
  543. if pending:
  544. receipt.data["pending_manual_serves"] = pending
  545. # EMBED the pm sync sections (the settled receipts contract): the
  546. # update's rebuild/bisect ran through pm's own sync receipt, which
  547. # finalizes before this one. Fold in only the completion filed
  548. # under THIS update's correlation id (pm.receipt.last_for_update,
  549. # per-context, never latest.json) — a sync from before this update
  550. # began, a standalone sync, or one in a concurrent thread cannot
  551. # be misattributed, and a nested update's sync cannot displace
  552. # this update's own. ONE file still carries the whole story
  553. # (desktop reads a single latest.json).
  554. try:
  555. from pm import receipt as pm_receipt
  556. sync = pm_receipt.last_for_update(receipt.correlation_id, consume=True)
  557. if isinstance(sync, dict):
  558. for key in ("venv_rebuild", "plugin_bisect", "feature_list", "steps", "exit_code"):
  559. if sync.get(key) is not None:
  560. receipt.data[f"pm_{key}"] = sync[key]
  561. if sync.get("outcome") is not None:
  562. receipt.data["pm_sync_outcome"] = sync.get("outcome")
  563. if sync.get("warnings"):
  564. receipt.data["pm_warnings"] = sync["warnings"]
  565. if sync.get("refusal") is not None:
  566. receipt.data["pm_refusal"] = sync["refusal"]
  567. except Exception as exc: # pragma: no cover — embedding is additive
  568. logger.debug("pm sync-section embed skipped: %s", exc)
  569. directory = _receipt_dir()
  570. directory.mkdir(parents=True, exist_ok=True)
  571. # Unique name: stamp+pid collides for nested/concurrent receipts in
  572. # the same process+second — the correlation id makes the name unique
  573. # per update run. Atomic write for BOTH the stamped receipt and the
  574. # latest.json pointer (no torn readers).
  575. from hermes_cli.runtime_state import _atomic_bytes
  576. # The run's own file (named at begin): the durable running record becomes terminal.
  577. path = _run_file(directory, receipt.data)
  578. payload = (json.dumps(receipt.data, indent=2, default=str) + "\n").encode("utf-8")
  579. _atomic_bytes(path, payload)
  580. with suppress(Exception): # stable pointer for the dashboard/desktop
  581. _write_latest(payload)
  582. _prune_old_receipts(directory)
  583. _publish_shared_metrics(receipt.data)
  584. return path
  585. except Exception as exc:
  586. # Visible, not debug: a run that pulled code and left no receipt is exactly the run
  587. # operators need to post-mortem, and INFO-level logs discard debug (#112465, #112558).
  588. logger.warning("Could not write update receipt (%s): %s", outcome, exc)
  589. print(f" ⚠ Update receipt not written: {exc}")
  590. return None
  591. def _collection_enabled_now() -> Optional[bool]:
  592. """Shared-metrics consent via the ALREADY-LOADED config module (never an import: this interpreter
  593. predates the checkout swap). None when it cannot tell (not loaded, unreadable config)."""
  594. reader = getattr(sys.modules.get("hermes_cli.config"), "read_raw_config_readonly", None)
  595. if reader is None:
  596. return None
  597. try:
  598. config: Any = reader()
  599. except Exception:
  600. return None
  601. if type(config) is not dict: # FailedConfigRead: a fallback, not what the user chose
  602. return None
  603. for key in ("telemetry", "shared_metrics"):
  604. config = config.get(key) if isinstance(config, dict) else None
  605. return isinstance(config, dict) and config.get("enabled") is True
  606. #: The leading label of a stop reason the parked copy may keep, exactly what the classifier reads:
  607. #: an exception type name (plus its errno token) or one of the fixed phrases Hermes writes
  608. #: (shared_metrics_update._STOP_REASON_PREFIX_CLASSES); never the message after it.
  609. _PARKED_REASON_LABEL = re.compile(
  610. r"(?:[A-Z][A-Za-z0-9_]{0,63}:(?: \[(?:Errno|WinError) \d+\])?"
  611. r"|historical takeover preparation failed|Windows gateway recovery failed)")
  612. _STOP_CLASS_TOKEN = re.compile(r"[a-z][a-z0-9_]{0,39}")
  613. def _metric_receipt(data: dict[str, Any]) -> dict[str, Any]:
  614. """Only what shared_metrics_update.update_receipt_fields reads; never argv or step text.
  615. Every field the classifier reads is kept in closed form so a parked run classifies exactly as
  616. the same run finalized in-process: the exit's ``stop_class`` token, the exit code, the stop
  617. reason's leading label (``-`` for any other text), and flags for the restart/user-action facts.
  618. """
  619. raw_pre, raw_restart, raw_reason = (data.get(key) for key in ("pre_update", "gateway_restart", "stop_reason"))
  620. pre: dict[str, Any] = raw_pre if isinstance(raw_pre, dict) else {}
  621. restart: dict[str, Any] = raw_restart if isinstance(raw_restart, dict) else {}
  622. reason: str = raw_reason if isinstance(raw_reason, str) else ""
  623. label = _PARKED_REASON_LABEL.match(reason)
  624. stop_class = data.get("stop_class")
  625. exit_code = data.get("exit_code")
  626. return {
  627. "schema": data.get("schema"),
  628. "update_id": data.get("update_id"), "started_at": data.get("started_at"),
  629. "finished_at": data.get("finished_at"), "outcome": data.get("outcome"),
  630. "initiator": "desktop" if data.get("initiator") == "desktop" else None,
  631. "pre_update": {"commit_date": pre.get("commit_date")},
  632. "stages": [
  633. {key: mark[key] for key in ("name", "outcome", "at", "mode") if key in mark}
  634. for mark in data.get("stages") or () if isinstance(mark, dict)
  635. ],
  636. "steps": [
  637. {"name": "admission", "ok": bool(step.get("ok"))}
  638. for step in data.get("steps") or () if isinstance(step, dict) and step.get("name") == "admission"
  639. ],
  640. "fleet": [{"state": row.get("state")} for row in data.get("fleet") or () if isinstance(row, dict)],
  641. "stop_class": stop_class if isinstance(stop_class, str) and _STOP_CLASS_TOKEN.fullmatch(stop_class) else None,
  642. "exit_code": exit_code if isinstance(exit_code, int) and not isinstance(exit_code, bool) else None,
  643. "stop_reason": label.group(0) if label else ("-" if reason else ""),
  644. "user_action": bool(data.get("user_action")),
  645. "gateway_restart": {key: bool(restart.get(key)) for key in ("incomplete", "phase_error", "failed_units")},
  646. "runtime_outcomes": [
  647. {"outcome": "failed"} for row in data.get("runtime_outcomes") or ()
  648. if isinstance(row, dict) and row.get("outcome") == "failed"
  649. ],
  650. }
  651. def _consent_reader_importable() -> bool:
  652. """Whether this interpreter can load the config reader the recorder's consent gate uses.
  653. False in the bare ``-I -S`` bootstrap interpreter that finalizes an update whose dependency
  654. preparation failed (update_completion._settle_after_commit): it has the new tree but no
  655. third-party packages, so ``hermes_cli.config`` dies on ``ruamel``. Only asked of an interpreter
  656. that runs the new tree (never the pre-pull one, which must import nothing); the recorder's
  657. pre-gate imports the same module next, so a working interpreter loads nothing extra."""
  658. try:
  659. import hermes_cli.config
  660. except ImportError: # ruamel and every other third-party package are missing here
  661. return False
  662. return True
  663. def _bare_collection_enabled() -> Optional[bool]:
  664. """Consent as far as an interpreter without the config reader can tell: the loaded module's
  665. answer if one is loaded, False when the profile has no config.yaml (the shipped default is off),
  666. else None (unknowable here; the next normal start decides)."""
  667. enabled = _collection_enabled_now()
  668. if enabled is not None:
  669. return enabled
  670. from hermes_constants import get_hermes_home
  671. try:
  672. (get_hermes_home() / "config.yaml").stat() # config-reader: ok — bare interpreter has no config stack; presence probe only
  673. except FileNotFoundError:
  674. return False
  675. except OSError:
  676. return None
  677. return None
  678. def _park_metric_receipt(data: dict[str, Any], enabled: Optional[bool]) -> None:
  679. """Keep the bounded fields for the next Hermes start (stdlib only): ``report_pending_updates``
  680. records them when collection is on, ``begin_process`` purges them when it is off. Collection
  681. known off: park nothing and purge what an earlier run parked."""
  682. from hermes_constants import get_hermes_home
  683. from hermes_cli.runtime_state import _atomic_bytes
  684. pending = get_hermes_home() / "telemetry" / "shared_metrics" / "pending_updates" # = PENDING_DIRNAME
  685. if enabled is False:
  686. import shutil
  687. shutil.rmtree(pending, ignore_errors=True) # opted out: nothing parked may be counted later
  688. return
  689. pending.mkdir(parents=True, exist_ok=True)
  690. _atomic_bytes(pending / f"{data.get('update_id')}.json", json.dumps(_metric_receipt(data), default=str).encode())
  691. def _publish_shared_metrics(data: dict[str, Any]) -> None:
  692. """hermes.update.run/stage from this FINAL receipt; must never fail or slow the update.
  693. Rows are emitted only by an interpreter that can read consent with the real config reader.
  694. Any other interpreter parks the bounded fields instead, and the next normal start applies the
  695. collection gate (records them, or purges them when collection is off)."""
  696. with suppress(Exception):
  697. pre, post = data.get("pre_update") or {}, data.get("post_update") or {}
  698. if data.get("pid") == os.getpid() and not (pre.get("sha") and pre.get("sha") == post.get("sha")):
  699. # This interpreter began the run before the checkout swap: importing now would load
  700. # pulled code into it. Consent comes from the config module it already has loaded.
  701. _park_metric_receipt(data, _collection_enabled_now())
  702. return
  703. if not _consent_reader_importable():
  704. # The bare bootstrap interpreter (dependency preparation failed): no config reader, so
  705. # it can neither read consent nor emit. Park, unless consent is knowably off.
  706. _park_metric_receipt(data, _bare_collection_enabled())
  707. return
  708. from hermes_cli.observability.shared_metrics_update import record_update_receipt
  709. record_update_receipt(data)
  710. def finalized_receipt(update_id: str) -> Optional[dict[str, Any]]:
  711. """The terminal record this process finalized for ``update_id``, published or not.
  712. The completion child answers its parent from this when the receipt store refused the
  713. terminal write: the run is still correlated and terminal, only its archive is missing
  714. (already reported by ``⚠ Update receipt not written``).
  715. """
  716. data = _FINALIZED.get(str(update_id))
  717. return copy.deepcopy(data) if data is not None else None
  718. def finalize_pending_update_receipt(exit_code: Optional[int] = None, stop_reason: str = "") -> Optional[Path]:
  719. """Command-boundary safety net: persist a still-open receipt, if any. Never raises.
  720. ``hermes update`` has many early ``sys.exit`` paths (preflight refusals, venv-holder refusal,
  721. fetch failure) predating the inner finalize calls; finalizing here means refused/failed runs —
  722. where a receipt matters most — leave a record. Exit 0/None → ``success``, exit 2 → ``refused``
  723. (preflight convention), else → ``failed`` (``partial`` when the run committed and only owes a
  724. user action, see ``record_user_action``).
  725. No-op when no receipt is open (the inner paths already finalized — exactly-once via the popped
  726. per-context receipt) or when recording was never started. See #91283.
  727. """
  728. current = _current.get()
  729. if current is None:
  730. return None
  731. outcome = ("success" if exit_code in (0, None) else "refused" if exit_code == 2
  732. else "partial" if current.data.get("user_action") else "failed")
  733. if exit_code is not None:
  734. with suppress(Exception):
  735. clone = copy.copy(current)
  736. clone.data = copy.deepcopy(current.data)
  737. clone.data["exit_code"] = int(exit_code)
  738. _current.set(clone)
  739. return finalize_update_receipt(outcome, stop_reason=stop_reason)
  740. def resume_run_record() -> Optional[Path]:
  741. """Carry the open run forward from its own archive when another process took it further.
  742. The completion child resumes this process's snapshot and persists stages (or closes the run)
  743. the snapshot never saw; closing the run from that stale snapshot would erase them. Returns the
  744. archive when the run is already closed there (the open receipt is dropped: nothing is left to
  745. finalize), else None. Never raises.
  746. """
  747. current = _current.get()
  748. found = read_run_record(str(current.data.get("update_id"))) if current is not None else None
  749. if found is None:
  750. return None
  751. path, record = found
  752. if record.get("finished_at"):
  753. try:
  754. _current.reset(current.current_token)
  755. except ValueError: # token from another context: pop THIS context only
  756. _current.set(None)
  757. return path
  758. record.pop("writer_pid", None)
  759. clone = copy.copy(current)
  760. clone.data = record
  761. _current.set(clone)
  762. return None
  763. def finalize_interrupted_update_receipt(stop_reason: str, *, exit_code: int = 130) -> Optional[Path]:
  764. """Close a run the operator interrupted AFTER the commit point as ``interrupted``. Never raises.
  765. Not ``failed``: the code already moved, so "still on the previous version" would be false; the
  766. armed obligations finish the rest. The run's own on-disk record is preferred when it is further
  767. along (the completion child persisted stages this process never saw).
  768. """
  769. print("⚠ Interrupted after the code was updated: the new code is in place; its remaining steps are owed "
  770. "and the next launch or `hermes update` finishes them.", flush=True)
  771. if _current.get() is None:
  772. return None
  773. closed = resume_run_record()
  774. if closed is not None: # the completion child already closed the run
  775. return closed
  776. with suppress(Exception):
  777. clone = copy.copy(_current.get())
  778. clone.data = copy.deepcopy(clone.data)
  779. clone.data["exit_code"] = int(exit_code)
  780. _current.set(clone)
  781. return finalize_update_receipt("interrupted", stop_reason=stop_reason)
  782. def _prune_old_receipts(directory: Path) -> None:
  783. with suppress(Exception):
  784. receipts = (p for p in directory.glob("update_*.json") if p.is_file())
  785. for stale in sorted(receipts, key=lambda p: p.stat().st_mtime, reverse=True)[_RECEIPT_KEEP:]:
  786. with suppress(OSError):
  787. stale.unlink()
  788. def settle_latest_receipt_fleet(fleet: list[dict[str, Any]], *, discharges) -> bool:
  789. """Record on ``latest.json`` that the fleet it still reports as owed now serves the checkout.
  790. A failed receipt whose plan rows cannot be matched to a live gateway (unknown identity,
  791. pre-pull SHAs) keeps ``hermes update`` exiting 1 and every CLI start warning about mixed
  792. modules, long after the operator's ``hermes gateway restart`` fixed the fleet (#117051). The
  793. caller has just verified every live row is current at the checkout SHA; persisting that
  794. matrix as the receipt's post-restart ``fleet`` (and un-flagging ``gateway_restart``) is what
  795. lets the stale-runtime readers see the recovery. ``discharges(settled_receipt)`` decides on
  796. the in-memory copy; ``latest.json`` is rewritten only when it answers True, so a catch-up
  797. that still owes the restart leaves the receipt byte-identical. Only the ``latest.json`` pointer is
  798. rewritten; the archived per-run file keeps the original outcome. Never raises.
  799. """
  800. try:
  801. path = _receipt_dir() / "latest.json"
  802. receipt = json.loads(path.read_text(encoding="utf-8-sig"))
  803. if not isinstance(receipt, dict):
  804. return False
  805. receipt["fleet"] = list(fleet)
  806. gateway_restart = receipt.get("gateway_restart")
  807. if not isinstance(gateway_restart, dict):
  808. gateway_restart = {}
  809. gateway_restart.update({"incomplete": False, "phase_error": ""})
  810. gateway_restart["settled_from_live_fleet_at"] = _utc_now_iso()
  811. receipt["gateway_restart"] = gateway_restart
  812. if not discharges(receipt):
  813. return False
  814. _write_latest((json.dumps(receipt, indent=2, default=str) + "\n").encode("utf-8"))
  815. return True
  816. except Exception as exc:
  817. logger.debug("Could not settle latest update receipt from the live fleet: %s", exc)
  818. return False
  819. def read_latest_receipt() -> Optional[dict[str, Any]]:
  820. """Read the most recent update receipt, or None. Never raises."""
  821. with suppress(Exception):
  822. # The newest pointer wins (the root on a tie): the last writer of either store, exactly
  823. # as when updater and pm shared one folder.
  824. points = [d / "latest.json" for d in _receipt_dirs() if (d / "latest.json").is_file()]
  825. if not points:
  826. return None
  827. path = max(points, key=lambda p: p.stat().st_mtime)
  828. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  829. return payload if isinstance(payload, dict) else None
  830. return None
  831. def read_receipt_for_action(action_id: str) -> Optional[dict[str, Any]]:
  832. """Newest receipt written by the dashboard action ``action_id`` (``latest.json`` first, then
  833. the retained archive), or None. Never raises."""
  834. latest = read_latest_receipt()
  835. if latest and latest.get("action_id") == action_id:
  836. return latest
  837. with suppress(Exception):
  838. archives = (p for d in _receipt_dirs() for p in d.glob("update_*.json"))
  839. for path in sorted(archives, key=lambda p: p.stat().st_mtime, reverse=True):
  840. with suppress(Exception):
  841. payload = json.loads(path.read_text(encoding="utf-8-sig"))
  842. if isinstance(payload, dict) and payload.get("action_id") == action_id:
  843. return payload
  844. return None
  845. def _profile_homes() -> list[tuple[str, Path]]:
  846. """``(profile, home)`` for the default home plus every valid named profile dir, sorted."""
  847. from hermes_cli.profiles import _get_default_hermes_home, _get_profiles_root, _PROFILE_ID_RE
  848. homes: list[tuple[str, Path]] = []
  849. default_home = _get_default_hermes_home()
  850. if default_home.is_dir():
  851. homes.append(("default", default_home))
  852. root = _get_profiles_root()
  853. if root.is_dir():
  854. homes.extend(
  855. (entry.name, entry)
  856. for entry in sorted(root.iterdir())
  857. if entry.is_dir() and entry.name != "default" and _PROFILE_ID_RE.match(entry.name)
  858. )
  859. return homes
  860. def _socket_identity(home: Path) -> Optional[tuple[int, dict]]:
  861. """``(pid, identity)`` declared by the gateway owning ``home``'s control socket, else None.
  862. A live ``identify`` answer is authoritative — no PID-reuse or stale-file heuristics. Callers
  863. fall back to ``gateway_state.json`` for gateways that predate the socket or whose socket
  864. didn't bind.
  865. """
  866. try:
  867. # Prefer the gateway-owned control socket (#92091): identity declared by the process itself,
  868. # including its own supervisor provenance — no argv/PID inference. Scan fallback below.
  869. from gateway.control_socket import identify_gateway
  870. identity = identify_gateway(home)
  871. return (int(identity.get("pid")), identity) if identity else None
  872. except Exception: # probe failure, no gateway, or an unparseable pid
  873. return None
  874. _CODE_ROOT_MAX_DEPTH = 8
  875. def _code_root_for_path(raw: Any) -> Optional[Path]:
  876. """Return the Hermes checkout containing an absolute process path."""
  877. if not isinstance(raw, str) or not raw:
  878. return None
  879. with suppress(Exception):
  880. candidate = Path(raw)
  881. if not candidate.is_absolute():
  882. return None
  883. for parent in [candidate, *candidate.parents][:_CODE_ROOT_MAX_DEPTH]:
  884. if (parent / "hermes_cli" / "main.py").is_file():
  885. return parent.resolve()
  886. return None
  887. def _updater_code_root() -> Optional[Path]:
  888. return _code_root_for_path(str(Path(__file__).resolve()))
  889. def _gateway_code_root(pid: int, home: Path) -> Optional[Path]:
  890. """Resolve the checkout served by a verified live gateway when possible."""
  891. # Older gateways cannot publish a new identity field, but their pid-guarded
  892. # status record already carries sys.argv (whose first item is the resolved
  893. # module path for ``python -m`` launches).
  894. with suppress(Exception):
  895. from gateway.status import read_runtime_status
  896. record = read_runtime_status(home / "gateway_state.json") or {}
  897. if int(record.get("pid")) == pid:
  898. for probe in record.get("argv") or []:
  899. root = _code_root_for_path(probe)
  900. if root:
  901. return root
  902. with suppress(Exception):
  903. import psutil # type: ignore
  904. process = psutil.Process(pid)
  905. with suppress(Exception):
  906. root = _code_root_for_path((process.environ() or {}).get("VIRTUAL_ENV"))
  907. if root:
  908. return root
  909. probes: list[Any] = []
  910. with suppress(Exception):
  911. probes.append(process.exe())
  912. with suppress(Exception):
  913. probes.extend(process.cmdline() or [])
  914. for probe in probes:
  915. root = _code_root_for_path(probe)
  916. if root:
  917. return root
  918. return None
  919. EXTERNAL_STATE = "external"
  920. # A gateway this updater runs INSIDE that accepted a self-restart request: it is still on the
  921. # pre-update code by construction and restarts once the updater exits (#100179 / #119597).
  922. RESTART_PENDING_STATE = "restart_pending"
  923. def row_is_external(row: Any) -> bool:
  924. """Whether a fleet row serves a checkout this update did not touch."""
  925. return isinstance(row, dict) and row.get("state") == EXTERNAL_STATE
  926. def _fleet_row(
  927. profile: str, pid: int, code_sha: Any, code_version: Any, expected_sha: Any,
  928. state: str = "unknown", code_root: Optional[Path] = None,
  929. expected_root: Optional[Path] = None, served_profiles: Any = None,
  930. self_restart_pending: Optional[set] = None,
  931. ) -> dict[str, Any]:
  932. if state == "unknown" and code_root and expected_root and code_root != expected_root:
  933. state = EXTERNAL_STATE
  934. if state == "unknown" and code_sha and expected_sha:
  935. state = "current" if str(code_sha) == str(expected_sha) else "stale"
  936. if state == "stale" and self_restart_pending and pid in self_restart_pending:
  937. state = RESTART_PENDING_STATE
  938. row = {
  939. "profile": profile, "pid": pid, "code_sha": str(code_sha) if code_sha else None,
  940. "code_version": code_version, "state": state,
  941. "code_root": str(code_root) if code_root else None,
  942. }
  943. # A live, identity-verified multiplexer represents every profile in this
  944. # list. Keep the field only when its shape is usable: callers use it to
  945. # discharge per-profile restart obligations, so corrupt status must not
  946. # widen coverage.
  947. if isinstance(served_profiles, list) and served_profiles and all(
  948. isinstance(name, str) and name for name in served_profiles
  949. ):
  950. row["served_profiles"] = list(dict.fromkeys(served_profiles))
  951. return row
  952. # Runtime-status states that do not describe a gateway that should be running now — no down row.
  953. _NOT_EXPECTED_STATES = {"stopped", "startup_failed"}
  954. def collect_fleet_versions(
  955. *, pre_restart_pids: Optional[list[int]] = None, self_restart_pending: Optional[set] = None,
  956. ) -> list[dict[str, Any]]:
  957. """Snapshot every profile's gateway code identity vs. the current tree.
  958. ``self_restart_pending`` — pids of gateways that are ancestors of this updater and accepted a
  959. self-restart request (cron update inside the gateway tree, #100179). They can only restart after
  960. this process exits, so their pre-update ``code_sha`` is expected: such a row is
  961. ``restart_pending`` instead of ``stale`` and does not fail the matrix (#119597). Every other
  962. live gateway on the old sha keeps its ``stale`` verdict.
  963. Rollout safety: ``down`` requires membership in ``pre_restart_pids`` — a stale state file from a
  964. long-dead gateway (machine reboot, manual kill weeks ago) must NOT fail every future update.
  965. Without a pre-restart snapshot (``None``/empty) dead PIDs are skipped (historical behavior).
  966. ``stale`` — gateway stamped a code_sha that differs from the updated checkout's HEAD (it is still
  967. serving pre-update modules). ``unknown`` — gateway predates the code-identity stamp (started before this
  968. feature landed), identity could not be resolved, or the state file's live PID is not the
  969. verified gateway for that home (``live_gateway_pid_for_home``): ``write_runtime_status`` re-stamps
  970. ``pid``/``code_sha`` for whatever process writes it, so a foreign writer must never read as
  971. ``current`` (#110420, sibling of #109680). ``down`` — the gateway was ALIVE when this update
  972. started (``pre_restart_pids``), its runtime status still says running, but the PID is dead and no
  973. successor rewrote the record: the restart phase stopped it and nothing came back. Without this row a
  974. killed-and-never-replaced gateway produced NO entry at all and the matrix passed silently (Phase-1
  975. verification gap, #88848/#74973 class).
  976. """
  977. _pre_restart = {int(p) for p in (pre_restart_pids or []) if isinstance(p, int)}
  978. _pending = {int(p) for p in (self_restart_pending or ()) if isinstance(p, int)}
  979. results: list[dict[str, Any]] = []
  980. expected_sha = _code_identity(refresh=True).get("sha")
  981. expected_root = _updater_code_root()
  982. try:
  983. from gateway.status import (
  984. live_gateway_pid_for_home,
  985. read_runtime_status,
  986. runtime_status_pid_is_live,
  987. )
  988. for profile, home in _profile_homes():
  989. sock = _socket_identity(home)
  990. if sock is not None:
  991. pid, identity = sock
  992. row = _fleet_row(
  993. profile, pid, identity.get("code_sha"), identity.get("code_version"), expected_sha,
  994. served_profiles=identity.get("served_profiles"),
  995. code_root=_gateway_code_root(pid, home), expected_root=expected_root,
  996. self_restart_pending=_pending,
  997. )
  998. results.append({**row, "source": "socket"})
  999. continue
  1000. record = read_runtime_status(home / "gateway_state.json")
  1001. if not record:
  1002. continue
  1003. try:
  1004. pid = int(record.get("pid"))
  1005. except (TypeError, ValueError):
  1006. continue
  1007. # A state file is only a fallback claim. Its SHA is evidence about
  1008. # its own PID only when the profile's canonical identity resolver
  1009. # verifies that same live gateway.
  1010. if live_gateway_pid_for_home(home) == pid:
  1011. results.append(
  1012. _fleet_row(
  1013. profile, pid, record.get("code_sha"), record.get("code_version"), expected_sha,
  1014. served_profiles=record.get("served_profiles"),
  1015. code_root=_gateway_code_root(pid, home), expected_root=expected_root,
  1016. self_restart_pending=_pending,
  1017. )
  1018. )
  1019. continue
  1020. # A live non-gateway (or a gateway for another profile) can write a
  1021. # plausible state file. Keep the fail-open visibility row, but never
  1022. # let that file's self-reported SHA or version classify or label
  1023. # the process — both claims have the same trust problem.
  1024. if runtime_status_pid_is_live(record):
  1025. results.append(_fleet_row(profile, pid, None, None, None))
  1026. continue
  1027. # Dead PID (or a live PID recycled by an unrelated process during the update's own
  1028. # churn): a DOWN row only when this exact pid was alive at update start AND the record
  1029. # still claims a running state — "the restart phase stopped it and nothing came back."
  1030. # Everything else (clean stop, startup failure, long-dead stale record) keeps the no-row
  1031. # behavior so the rollout can't false-positive. ``_pre_restart`` is a bare PID set, not
  1032. # (pid, start_time) pairs, so a recycled PID from gateway A landing in B's stale record
  1033. # could still mislabel B as down — inherent to the snapshot's data model.
  1034. # See #93258.
  1035. gw_state = record.get("gateway_state")
  1036. if pid in _pre_restart and isinstance(gw_state, str) and gw_state and gw_state not in _NOT_EXPECTED_STATES:
  1037. results.append(_fleet_row(profile, pid, None, record.get("code_version"), None, state="down"))
  1038. except Exception as exc:
  1039. logger.debug("Fleet version probe failed: %s", exc)
  1040. return results
  1041. _FLEET_ROW_LINES = {
  1042. "current": " ✓ {profile} (pid {pid}) @ {short} — up to date",
  1043. "stale": " ✗ {profile} (pid {pid}) @ {short} — STALE (pre-update code)",
  1044. "down": " ✗ {profile} — DOWN (gateway was running before the update; pid {pid} is gone and nothing replaced it)",
  1045. "external": " ◆ {profile} (pid {pid}) @ {short} — separate checkout, not updated by this run",
  1046. RESTART_PENDING_STATE: (
  1047. " ↻ {profile} (pid {pid}) @ {short} — restart pending (deferred until this process exits;"
  1048. " the update runs inside this gateway)"
  1049. ),
  1050. }
  1051. _FLEET_ROW_UNKNOWN = " ? {profile} (pid {pid}) — version unknown (gateway predates version stamping; restart to enable)"
  1052. # A gateway pid the pre-update snapshot did not know that had not published its code identity when
  1053. # the settle window closed (#112634): most likely the successor this update relaunched, still
  1054. # booting, so "restart to enable" would be wrong — but the poll never observed the restart itself,
  1055. # so the copy does not claim one.
  1056. _FLEET_ROW_IDENTITY_PENDING = (
  1057. " ? {profile} (pid {pid}) — new pid since the update, code identity not published yet"
  1058. " — re-check with `hermes gateway status`"
  1059. )
  1060. def print_fleet_version_matrix(fleet: list[dict[str, Any]]) -> bool:
  1061. """Print the post-update fleet version matrix.
  1062. Returns True when at least one gateway is provably stale (still serving pre-update code) OR
  1063. provably down (killed by the restart phase, nothing came back), so the caller can escalate.
  1064. ``unknown`` entries are reported but do NOT fail the update: gateways started before the
  1065. code-identity stamp existed have no sha to compare, and failing them would be a false-positive
  1066. storm. ``restart_pending`` entries (the gateway this updater runs inside, self-restart
  1067. accepted) are on the old code by construction and do not fail it either (#119597).
  1068. """
  1069. if not fleet:
  1070. return False
  1071. print()
  1072. print("Fleet version check:")
  1073. states = set()
  1074. external_roots: list[str] = []
  1075. for entry in fleet:
  1076. sha = entry.get("code_sha")
  1077. states.add(entry.get("state"))
  1078. if row_is_external(entry) and entry.get("code_root"):
  1079. external_roots.append(f"{entry.get('profile')}: {entry.get('code_root')}")
  1080. fallback = _FLEET_ROW_IDENTITY_PENDING if entry.get("identity_pending") else _FLEET_ROW_UNKNOWN
  1081. print(_FLEET_ROW_LINES.get(entry.get("state"), fallback).format(
  1082. profile=entry.get("profile"), pid=entry.get("pid"), short=sha[:8] if isinstance(sha, str) and sha else "?",
  1083. ))
  1084. if external_roots:
  1085. print()
  1086. print(" ℹ These profiles run their own checkout and are updated separately:")
  1087. for line in external_roots:
  1088. print(f" {line}")
  1089. if RESTART_PENDING_STATE in states:
  1090. print()
  1091. print(" ℹ A restart-pending gateway has not yet been verified on the new code;")
  1092. print(" check after this update exits with `hermes gateway status`.")
  1093. stale_or_down = sum(1 for entry in fleet if entry.get("state") in ("stale", "down"))
  1094. if stale_or_down:
  1095. print()
  1096. if "stale" in states:
  1097. print(" ⚠ Stale gateways keep serving pre-update code until restarted.")
  1098. if "down" in states:
  1099. print(" ⚠ Down gateways stopped serving messaging entirely.")
  1100. # The code is committed (the update exits 0), but the fleet is not on it yet: say what
  1101. # is still owed after ``✓ Update complete!``; every CLI start repeats it until restarted.
  1102. print()
  1103. print(
  1104. f"⚠ Gateway restart still owed: {stale_or_down} gateway(s) still running the old code (or stopped).")
  1105. print(" Run `hermes gateway restart` (or `hermes -p <profile> gateway restart` for a named")
  1106. print(" profile), then `hermes gateway status` to confirm.")
  1107. return stale_or_down > 0