update_cmd_fleet_verify.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503
  1. """Post-restart fleet verification for ``hermes update`` (``update_cmd_fleet`` sibling).
  2. After the restart phase: compare every live gateway's stamped code against the checkout, reconcile
  3. the pre-update plan against what the restart did, keep the restart owed (a ``gateway_restart``
  4. follow-up, never a failed update — contract C3) or clear the obligation, and finalize the receipt.
  5. Restart-phase helpers stay in ``update_cmd_fleet`` and are read through it so their patches apply.
  6. """
  7. import json
  8. import logging
  9. import subprocess
  10. import time as _time
  11. from contextlib import suppress
  12. from hermes_cli.update_cmd_common import _best_effort
  13. # Log-record parity with the origin module.
  14. logger = logging.getLogger("hermes_cli.update_cmd")
  15. # A supervisor can report a restarted unit active before the gateway finishes its
  16. # bootstrap and publishes ``gateway_state.json``. Keep the readiness poll bounded,
  17. # but allow the default systemd startup budget plus status-publication slack.
  18. _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS = 120.0
  19. def _print_legacy_units_warning() -> None:
  20. """Legacy hermes.service fights hermes-gateway.service over the bot token; warn on
  21. every update until migrated."""
  22. from hermes_cli.gateway import (has_legacy_hermes_units, _find_legacy_hermes_units, supports_systemd_services)
  23. if not (supports_systemd_services() and has_legacy_hermes_units()):
  24. return
  25. print()
  26. print("⚠ Legacy Hermes gateway unit(s) detected:")
  27. for name, path, is_sys in _find_legacy_hermes_units():
  28. scope = "system" if is_sys else "user"
  29. print(f" {path} ({scope} scope)")
  30. print()
  31. print(" These pre-rename units (hermes.service) fight the current")
  32. print(" hermes-gateway.service for the bot token and cause SIGTERM")
  33. print(" flap loops. Remove them with:")
  34. print()
  35. print(" hermes gateway migrate-legacy")
  36. print()
  37. print(" (add `sudo` if any are in system scope)")
  38. def _collect_fleet_snapshot(restart, rows_expected: bool) -> list:
  39. """Fleet version rows, polled over a bounded settle window when runtimes are expected.
  40. Gateways need time to rewrite gateway_state.json; Windows resumes DETACHED (~10s boot),
  41. so a single 2s sleep reported "no rows" on healthy resumes. A "down" row may be a
  42. detached replacement still booting: poll until none remain or the deadline passes.
  43. Pre-restart PIDs make a gateway stopped WITHOUT verified replacement a DOWN row (owed restart)
  44. instead of no row at all. An ``unknown`` row whose pid is NOT a pre-restart pid is a successor
  45. that has not published its code identity yet (a relaunched gateway can sit ~10s between process
  46. start and its first runtime-status write, #112634) — keep polling; at the deadline it is flagged
  47. ``identity_pending`` so the matrix does not call it a pre-stamping gateway.
  48. """
  49. from hermes_cli.update_receipt import collect_fleet_versions
  50. pending = getattr(restart, "self_restart_pending_pids", None) or None
  51. if not rows_expected:
  52. return collect_fleet_versions(
  53. pre_restart_pids=restart.pre_restart_gateway_pids, self_restart_pending=pending)
  54. pre_pids = restart.pre_restart_gateway_pids
  55. _fleet_deadline = _time.monotonic() + _FLEET_PROBE_SETTLE_TIMEOUT_SECONDS
  56. while True:
  57. _time.sleep(2.0)
  58. snapshot = collect_fleet_versions(pre_restart_pids=pre_pids, self_restart_pending=pending)
  59. unstamped = [row for row in snapshot if _fleet_row_identity_pending(row, pre_pids)]
  60. if snapshot and not unstamped and not any(row.get("state") == "down" for row in snapshot):
  61. return snapshot
  62. if _time.monotonic() >= _fleet_deadline or _restarted_units_gone(
  63. getattr(restart, "restarted_scoped_units", ())):
  64. for row in unstamped:
  65. row["identity_pending"] = True
  66. return snapshot
  67. def _fleet_row_identity_pending(row: dict, pre_restart_pids) -> bool:
  68. """An ``unknown`` row with no sha from a pid that did not exist at update start: a relaunched
  69. gateway still booting, not a gateway that predates version stamping. A surviving pre-restart pid
  70. (or no pid snapshot at all) is settled as-is — waiting cannot change what it publishes."""
  71. if row.get("state") != "unknown" or row.get("code_sha"):
  72. return False
  73. if pre_restart_pids is None:
  74. return False
  75. return row.get("pid") not in {int(p) for p in pre_restart_pids if isinstance(p, int)}
  76. def _restarted_units_gone(scoped_units) -> bool:
  77. """True when every restarted systemd unit is LOADED in its scope and neither active nor
  78. activating: the successor died, nothing will publish a state stamp, so the settle poll should fail
  79. closed now instead of at the deadline. Anything inconclusive keeps waiting: no units, systemctl
  80. missing/slow, or ``LoadState=not-found`` — a unit name asked in a scope that does not own it
  81. answers ``inactive`` exactly like a dead unit (#112466), so only a loaded unit can prove death."""
  82. from hermes_cli import update_cmd_fleet as fleet
  83. if not scoped_units:
  84. return False
  85. scope_cmds = dict(fleet._SYSTEMD_SCOPES)
  86. for scoped in scoped_units:
  87. scope, _, name = scoped.partition("/")
  88. try:
  89. stdout = fleet._systemctl(scope_cmds[scope] + ["show", "-p", "LoadState,ActiveState", name], timeout=5).stdout
  90. except (KeyError, FileNotFoundError, subprocess.TimeoutExpired):
  91. return False
  92. props = dict(line.split("=", 1) for line in stdout.splitlines() if "=" in line)
  93. if props.get("LoadState") != "loaded":
  94. return False
  95. if props.get("ActiveState") in ("active", "activating", "reloading"):
  96. return False
  97. return True
  98. def _live_gateway_pids_from_fleet(fleet_rows: list) -> dict:
  99. """Profile -> gateway PIDs alive after the restart phase, from the fleet snapshot.
  100. Incarnation evidence for gateway reconciliation (``match_runtime_outcomes``'s
  101. ``live_gateway_pids``): the plan identifies a gateway by the profile it SERVES while the restart
  102. bookkeeping names the SERVICE, and a service can serve a profile its name does not encode — the
  103. profile-scoped name matcher then never credits the planned runtime. A ``down`` row reports the
  104. PRE-restart PID (nothing replaced it), so it is never a successor; rows without a usable
  105. profile/PID are skipped.
  106. """
  107. live: dict = {}
  108. for row in fleet_rows:
  109. pid, profile = row.get("pid"), row.get("profile")
  110. if profile and isinstance(pid, int) and row.get("state") != "down":
  111. live.setdefault(str(profile), set()).add(pid)
  112. return live
  113. def _gateway_unit_runs_one_of(unit: str, pids: set) -> bool:
  114. """Every scope that has ``hermes-gateway*`` *unit* loaded runs it as one of *pids*.
  115. The restart bookkeeping keeps bare unit names, so it cannot say which scope failed: a loaded twin
  116. in the other scope that is not proven keeps the failure. Anything unreadable proves nothing.
  117. """
  118. from hermes_cli import update_cmd_fleet as fleet
  119. if not pids or not (unit == "hermes-gateway" or unit.startswith("hermes-gateway-")):
  120. return False
  121. loaded = False
  122. for _scope, scope_cmd in fleet._SYSTEMD_SCOPES:
  123. try:
  124. stdout = fleet._systemctl(scope_cmd + ["show", "-p", "LoadState,MainPID", unit], timeout=5).stdout
  125. props = dict(line.split("=", 1) for line in (stdout or "").splitlines() if "=" in line)
  126. if props.get("LoadState") != "loaded":
  127. continue
  128. loaded = True
  129. if int(props.get("MainPID") or 0) not in pids:
  130. return False
  131. except (OSError, subprocess.TimeoutExpired, ValueError):
  132. return False
  133. return loaded
  134. def _settle_units_proven_current(restart, fleet_rows: list) -> bool:
  135. """Credit each failed gateway unit whose own MainPID the fleet probe shows serving this checkout.
  136. A restart call can time out while systemd still finishes the job (a unit whose stop takes
  137. minutes), so the restart phase's "failed" can be a gateway that came up on the new code moments
  138. later. Proof is the unit's own process: a ``current`` row whose pid is the unit's MainPID and was
  139. not running when the restart phase began. Another profile's current row never vouches for it.
  140. Returns True when a unit was credited; the phase's own incompleteness is then recomputed. The
  141. caller saves the restart result once verification is done: a later check can still owe a restart.
  142. """
  143. pre = restart.pre_restart_gateway_pids
  144. if pre is None or not restart.failed_or_stale_units:
  145. return False
  146. fresh = {row.get("pid") for row in fleet_rows if row.get("state") == "current"} - set(pre)
  147. proven = [unit for unit in dict.fromkeys(restart.failed_or_stale_units) if _gateway_unit_runs_one_of(unit, fresh)]
  148. if not proven:
  149. return False
  150. restart.failed_or_stale_units[:] = [u for u in restart.failed_or_stale_units if u not in proven]
  151. restart.unrestarted_units[:] = [u for u in restart.unrestarted_units if u not in proven]
  152. restart.restarted_services.extend(u for u in proven if u not in restart.restarted_services)
  153. for unit in proven:
  154. print(f" ✓ {unit} finished restarting onto the new code after the restart phase stopped waiting")
  155. restart.incomplete = bool(restart.failed_or_stale_units or restart.phase_errors)
  156. if not restart.incomplete:
  157. # The restart phase withheld this stamp; every other profile's CLI reads it.
  158. from hermes_cli import update_cmd_fleet as fleet
  159. from hermes_cli.update_host_obligation import mark_host_restart_completed
  160. mark_host_restart_completed(fleet._restart_identity_sha())
  161. return True
  162. def _record_owed_gateway_inventory(plan, stopped_unmapped_pids=()) -> None:
  163. """Name the pre-update gateways on the obligation once their restart is owed.
  164. The pull arms the obligation before the restart phase knows what it owes. An inventory-less
  165. record on a host whose gateway then died at boot is settled by the gateway-less discharge
  166. (nothing live, nothing named), which would silence the owed-restart warning that replaces a
  167. failed update under contract C3. A named inventory keeps it until those gateways run HEAD.
  168. A gateway stopped with no known profile is recorded as an unmapped row for the same reason: it
  169. has no ``gateway_state.json`` the gateway-less probe could see, so absence would settle it.
  170. A readable inventory (a same-SHA catch-up keeps the standing record) only GROWS: rows this run
  171. newly owes are appended, so a mapped successor cannot discharge a newly stopped unmapped one.
  172. """
  173. from hermes_cli import update_cmd_fleet as fleet
  174. armed = fleet._fleet_restart_obligation_armed()
  175. fields = fleet._obligation_fields() if armed else {}
  176. if fields is None:
  177. return # unreadable terms stay fail-closed
  178. recorded: list = []
  179. if fields.get("inventory") not in (None, "", "null"):
  180. try:
  181. inventory = json.loads(fields["inventory"])
  182. fleet._marker_owed_gateways(inventory) # validates; malformed/unsupported stays untouched
  183. except ValueError:
  184. return
  185. recorded = inventory["runtimes"]
  186. from dataclasses import asdict
  187. rows = [
  188. asdict(runtime) for runtime in getattr(plan, "runtimes", ()) or ()
  189. if getattr(runtime, "kind", None) == "gateway"
  190. and isinstance(getattr(runtime, "profile", None), str)
  191. and runtime.profile.strip() and runtime.profile != "unknown"
  192. ] + [{"kind": "gateway", "profile": None, "pid": pid, "stopped_unmapped": True}
  193. for pid in sorted(stopped_unmapped_pids)]
  194. rows = [row for row in rows if row not in recorded]
  195. if rows:
  196. # Not armed any more (a pre-restart probe settled an inventory-less record): re-arm, the
  197. # restart is owed. The SHA is the code the fleet must be proven current on.
  198. sha = fields.get("expected_sha", "") if armed else (fleet._current_checkout_sha() or "")
  199. fleet._write_fleet_restart_pending_marker(expected_sha=sha, runtimes=recorded + rows)
  200. def _named_gateways_still_owed() -> bool:
  201. """True when the armed obligation NAMES gateways (recorded by an earlier owed restart) and the
  202. live fleet still does not prove them current. An inventory-less record keeps its old rules."""
  203. from hermes_cli import update_cmd_fleet as fleet
  204. if not fleet._fleet_restart_obligation_armed():
  205. return False
  206. fields = fleet._obligation_fields()
  207. if fields is None or fields.get("inventory") in (None, "", "null"):
  208. return False
  209. return not fleet._marker_only_restart_obsolete()
  210. def _is_serve_unit(name: object) -> bool:
  211. """A serve/dashboard unit by the fleet loop's own name gate (``hermes-serve*``/``hermes-dashboard*``,
  212. bare or ``.service``). Anything else it records as failed (gateway units, launchd labels,
  213. listing-failure markers) is gateway evidence."""
  214. from hermes_cli.update_cmd_fleet import _is_serve_or_dashboard_unit
  215. return _is_serve_or_dashboard_unit(f"{str(name).removesuffix('.service')}.service")
  216. def _restart_phase_owes_gateway(restart) -> bool:
  217. """Did the restart phase itself leave a gateway owed? Its incompleteness counts against the
  218. gateways unless every failure it recorded is a serve/dashboard unit (#141021)."""
  219. if not restart.incomplete:
  220. return False
  221. failed = list(restart.failed_or_stale_units)
  222. return bool(restart.phase_errors) or not failed or not all(map(_is_serve_unit, failed))
  223. def _split_unaccounted(outcomes) -> tuple[bool, set]:
  224. """``(gateway missed, serve/dashboard PIDs missed)`` among the unaccounted runtime outcomes."""
  225. from hermes_cli.update_inventory import _SERVE_KINDS
  226. missed = [o for o in outcomes if o.get("outcome") == "unaccounted"]
  227. serve_pids = {o["pid"] for o in missed if o.get("kind") in _SERVE_KINDS and isinstance(o.get("pid"), int)}
  228. return any(o.get("kind") not in _SERVE_KINDS for o in missed), serve_pids
  229. # The Desktop hand-offs (scripts/desktop-update/posix.sh, windows.ps1) drop their "Run hermes gateway
  230. # restart" hint only when every gateway_restart reason starts with this prefix, so every serve-only
  231. # part below starts with it and the gateway part always comes first (#141071).
  232. SERVE_ONLY_REASON_PREFIX = "serve/dashboard "
  233. _GATEWAY_OWED_REASON = "gateways may still run pre-update code or were stopped without a successor"
  234. _GATEWAY_RECOVERY = "; recover with `hermes gateway restart`"
  235. def _owed_restart_reason(restart, stopped, *, gateway_owed: bool, stale_serves: set, down_serves: set) -> str:
  236. """The ``gateway_restart`` follow-up's reason, naming what is actually owed (#141021).
  237. The step id stays ``gateway_restart`` (Desktop hand-offs parse it); only the text changes. A run
  238. whose gateways all verified current but whose dashboard unit could not be restarted must not tell
  239. the user to run `hermes gateway restart`, which never touches that unit."""
  240. parts = []
  241. if gateway_owed:
  242. parts.append(
  243. _GATEWAY_OWED_REASON
  244. + (f" (stopped PIDs {', '.join(map(str, stopped))})" if stopped else "")
  245. + _GATEWAY_RECOVERY)
  246. units = sorted({str(u).removesuffix(".service") for u in restart.failed_or_stale_units if _is_serve_unit(u)})
  247. if units:
  248. parts.append(
  249. f"{SERVE_ONLY_REASON_PREFIX}unit(s) {', '.join(units)} were not restarted onto the new code;"
  250. " see the systemctl advice above")
  251. if down_serves:
  252. parts.append(
  253. f"{SERVE_ONLY_REASON_PREFIX}process(es) {', '.join(map(str, sorted(down_serves)))} were stopped during the"
  254. " update and not brought back; relaunch `hermes serve` / `hermes dashboard`")
  255. if stale_serves - down_serves:
  256. parts.append(
  257. f"{SERVE_ONLY_REASON_PREFIX}process(es) {', '.join(map(str, sorted(stale_serves - down_serves)))} still run"
  258. " pre-update code; restart them as advised above")
  259. # Nothing classified (an unknown incompleteness): keep the historical, conservative wording.
  260. return "; ".join(parts) or (_GATEWAY_OWED_REASON + _GATEWAY_RECOVERY)
  261. def _verify_fleet_after_update(restart, *, _pre_update_plan, _windows_gateway_resume, update_complete):
  262. """Post-restart verification: legacy-unit warning, dashboard cleanup, stale serve
  263. probe, fleet version matrix, plan-vs-execution reconciliation, receipt finalize.
  264. Never fails the committed update (contract C3): when any gateway may still be stale it
  265. records a ``gateway_restart`` follow-up and leaves ``fleet_restart_pending`` armed for the
  266. next catch-up and the startup warning; otherwise clears the marker.
  267. """
  268. from hermes_cli import update_cmd_fleet as fleet
  269. from hermes_cli.update_cmd import (
  270. _m, _surviving_pre_update_serve_runtimes, _warn_stale_serve_runtimes,
  271. )
  272. from hermes_cli.update_cmd_maint import _refresh_dashboard_after_update
  273. with _best_effort('Legacy unit check during update failed: %s'):
  274. _print_legacy_units_warning()
  275. # Restart a managed dashboard via systemd or stop stale manual ones (raw-killing
  276. # a systemd-owned PID reads as clean stop and leaves the Cloudflare origin dead).
  277. # Which family is owed decides what the follow-up tells the user to run (#141021). Read before the
  278. # dashboard pass can flip ``incomplete``.
  279. _gateway_owed = _restart_phase_owes_gateway(restart)
  280. # Units the fleet loop already attempted aren't redone — restarted ones, and failed ones too: the
  281. # loop already reported each failure (with its systemctl advice), and a second attempt only
  282. # duplicates it or, for a system unit without root, adds a polkit/tty error (#141021).
  283. # A dashboard it stopped and could not bring back is a promised restart that did not happen.
  284. _dashboards_down = _refresh_dashboard_after_update(
  285. already_restarted_units=set(restart.restarted_services) | set(restart.failed_or_stale_units))
  286. if _dashboards_down:
  287. restart.incomplete = True
  288. _stale_serves_owed: set = set()
  289. # Success-path twin of the abort-recovery probe: the restart phase only touches
  290. # units, so a unit-less `hermes serve` keeps stale sys.modules. Runs AFTER
  291. # dashboard cleanup so a respawned manual dashboard isn't a survivor. Rows feed
  292. # reconciliation (survivor → owed restart); ``None`` = probe failed, stays fail-closed.
  293. # Check if any pre-update serve/dashboard runtimes survived on pre-update code generations (#100479).
  294. # This is the SUCCESS-path twin of the abort-recovery probe above: the restart phase only restarts
  295. # units, so an sshd-spawned `serve --isolated` or a manual `hermes serve` (no unit) is left running its
  296. # pre-update sys.modules graph — and its cron ticker keeps firing agent jobs that ImportError on every
  297. # symbol added in the pulled range. The rows also feed the plan-vs-execution reconciliation below, so a
  298. # survivor is escalated (an owed gateway_restart follow-up) instead of merely printed.
  299. _stale_serve_rows: list | None = None
  300. with _best_effort('Failed to check for surviving serve runtimes: %s'):
  301. _stale_serve_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan)
  302. if _stale_serve_rows:
  303. _warn_stale_serve_runtimes(_stale_serve_rows)
  304. print()
  305. print("Tip: You can now select a provider and model:")
  306. print(" hermes model # Select provider and model")
  307. # Compare every live gateway's stamped code_sha against the fresh checkout
  308. # instead of assuming the restart phase worked.
  309. # Phase 1 (#91277): post-update fleet version verification.
  310. _fleet_snapshot: list = []
  311. _units_settled = False
  312. with _best_effort('Fleet version verification failed: %s'):
  313. from hermes_cli.update_receipt import print_fleet_version_matrix
  314. # Cross-platform "rows expected" signal: (restarted_services or killed_pids)
  315. # never fires on Windows (pause/resume populates neither), so a healthy
  316. # resumed gateway yielded zero rows and exit 0.
  317. # See #93406.
  318. # A gateway stopped WITHOUT a successor ("Restart manually") publishes no row by design,
  319. # so it must not count as an expected one — otherwise an update whose only live gateways
  320. # were unmapped reports "no rows" after correctly stopping them.
  321. _pre_restart, _killed = restart.fleet_probe_signals()
  322. _fleet_rows_expected = _m()._fleet_probe_expected_runtimes(
  323. _pre_update_plan, _pre_restart, _windows_gateway_resume, restart.restarted_services, _killed,
  324. )
  325. _fleet_snapshot = _collect_fleet_snapshot(restart, _fleet_rows_expected)
  326. _units_settled = _settle_units_proven_current(restart, _fleet_snapshot)
  327. if _units_settled:
  328. _gateway_owed = _restart_phase_owes_gateway(restart)
  329. restart.incomplete = restart.incomplete or bool(_dashboards_down)
  330. if print_fleet_version_matrix(_fleet_snapshot):
  331. restart.incomplete = _gateway_owed = True
  332. # A proven-stale survivor must not keep running (its ticker yields every tick and
  333. # nothing else restarts it, #117275): hand it to the drain-first restart path.
  334. from hermes_cli.update_cmd_stale_survivors import signal_stale_fleet_survivors
  335. signal_stale_fleet_survivors(_fleet_snapshot, restart, fleet._gateway_drain_budget())
  336. elif not _fleet_snapshot and _fleet_rows_expected:
  337. # collect_fleet_versions() swallows every failure, so zero rows with
  338. # expected runtimes is indistinguishable from health — keep the restart owed.
  339. print(
  340. # Fleet probe returned zero rows even though at least one gateway runtime was (or may have
  341. # been) live pre-update — POSIX restart bookkeeping, the pre-restart PID snapshot, the
  342. # pre-update plan inventory, or the Windows pause/resume token all count as that signal.
  343. # Every failure path inside collect_fleet_versions() is swallowed via logger.debug(), so an
  344. # empty list is indistinguishable from a healthy fleet in the current output. Treat it as
  345. # verification failure so the receipt carries a gateway_restart follow-up (#93406).
  346. "\n⚠ Fleet version check returned no rows even though"
  347. " gateway runtimes were expected — verification incomplete."
  348. )
  349. restart.incomplete = _gateway_owed = True
  350. # Printed after the probe, not by the restart phase: only units it did not prove current.
  351. fleet._warn_incomplete_gateway_fleet_restart(getattr(restart, "unrestarted_units", None) or [])
  352. # Every runtime the PLAN saw must appear in restart bookkeeping; an
  353. # unaccounted one is a silent miss and escalates like a STALE/DOWN row.
  354. with _best_effort('Runtime-outcome reconciliation failed: %s'):
  355. # An unaccounted runtime is the silent-miss class (a platform branch re-discovered its own targets
  356. # and skipped one the inventory knew about) — escalate it exactly like a STALE/DOWN fleet row. See
  357. # #91277.
  358. if _pre_update_plan is not None and _pre_update_plan.runtimes:
  359. from hermes_cli.update_inventory import (match_runtime_outcomes, report_unaccounted_runtimes)
  360. from hermes_cli.update_receipt import row_is_external
  361. # Gateway incarnation evidence, from the post-restart fleet snapshot collected above: a
  362. # service can serve a profile its own name does not encode (root-home launchd label +
  363. # sticky active profile, hashed custom HERMES_HOME), so the bookkeeping's service names
  364. # alone cannot credit the planned runtime. A profile the probe produced no row for simply
  365. # has no evidence — that runtime stays on the name-matching path (and logs it).
  366. _live_gateway_pids = _live_gateway_pids_from_fleet(_fleet_snapshot)
  367. _runtime_outcomes = match_runtime_outcomes(
  368. _pre_update_plan,
  369. restarted_services=restart.restarted_services,
  370. relaunched_profiles=restart.relaunched_profiles,
  371. externally_supervised_profiles=restart.externally_supervised_profiles,
  372. killed_pids=restart.killed_pids,
  373. failed_units=restart.failed_or_stale_units,
  374. # Serve/dashboard reconcile by incarnation liveness, not unit names.
  375. # See #100479.
  376. stale_serve_pids=(
  377. {row.get("pid") for row in _stale_serve_rows}
  378. if _stale_serve_rows is not None
  379. else None
  380. ),
  381. failed_respawn_pids=_dashboards_down,
  382. # A symlinked profile served by another install's checkout (#120240).
  383. external_gateway_pids={row.get("pid") for row in _fleet_snapshot if row_is_external(row)},
  384. live_gateway_pids=_live_gateway_pids,
  385. )
  386. from dataclasses import asdict
  387. from hermes_cli.update_serve_obligations import defer_manual_serve
  388. for runtime, outcome in zip(_pre_update_plan.runtimes, _runtime_outcomes):
  389. if outcome["outcome"] == "unaccounted" and defer_manual_serve(asdict(runtime), require_alive=True):
  390. outcome["outcome"] = "deferred"
  391. if report_unaccounted_runtimes(_runtime_outcomes):
  392. restart.incomplete = True
  393. _missed_gateway, _missed_serves = _split_unaccounted(_runtime_outcomes)
  394. _gateway_owed |= _missed_gateway
  395. _stale_serves_owed |= _missed_serves
  396. with suppress(Exception):
  397. import hermes_cli.update_receipt as _ur
  398. _active = _ur._current.get()
  399. if _active is not None:
  400. _active.data["runtime_outcomes"] = _runtime_outcomes
  401. if not restart.incomplete:
  402. with _best_effort('Owed-gateway check failed: %s'):
  403. # Only on an otherwise-complete run: the check can settle (clear) the host obligation,
  404. # which an incomplete run must not do mid-verification. An incomplete run keeps the
  405. # obligation armed anyway, and the startup warning names any older owed gateway.
  406. if _named_gateways_still_owed():
  407. # A gateway a previous run could not restart is still not serving HEAD (nothing
  408. # live to restart this time): keep owing it rather than clearing the obligation.
  409. print(" ⚠ A gateway owed a restart by a previous update is still not serving this checkout.")
  410. restart.incomplete = _gateway_owed = True
  411. if getattr(restart, "stopped_unmapped_pids", None):
  412. # A gateway stopped with no successor is owed a restart, not "accounted for": keep the
  413. # fleet obligation armed so the startup warning names it until someone restarts it.
  414. restart.incomplete = _gateway_owed = True
  415. if restart.incomplete:
  416. # Code is committed (contract C3): a gateway that may still run stale modules is a
  417. # follow-up, never a failed update. The fleet obligation stays armed, every CLI start
  418. # warns about it, and the next `hermes update` retries the restart.
  419. from hermes_cli.update_receipt import record_followup
  420. stopped = sorted(getattr(restart, "stopped_unmapped_pids", None) or ())
  421. record_followup("gateway_restart", _owed_restart_reason(
  422. restart, stopped, gateway_owed=_gateway_owed, stale_serves=_stale_serves_owed,
  423. down_serves=set(_dashboards_down or ())))
  424. with _best_effort('Fleet restart inventory not recorded: %s'):
  425. _record_owed_gateway_inventory(_pre_update_plan, stopped)
  426. if _units_settled:
  427. # Credited units changed the restart result; save it with the final verdict so it agrees with
  428. # the follow-up above.
  429. restart.record_receipt()
  430. with _best_effort('Update receipt finalize failed: %s'):
  431. from hermes_cli.update_receipt import finalize_update_receipt
  432. # A False ``update_complete`` (unsafe SQLite runtime, owed tail work) never fails the
  433. # run: maintenance already reported it as a follow-up. It only vetoes migration below.
  434. _receipt_path = finalize_update_receipt("success", fleet=_fleet_snapshot)
  435. if _receipt_path is not None:
  436. logger.info("Update receipt written: %s", _receipt_path)
  437. if restart.incomplete:
  438. return
  439. fleet._clear_fleet_restart_pending_marker()
  440. if not update_complete:
  441. # Fleet caught up, but the selected runtime's SQLite is unsafe or the tail is still owed
  442. # (both reported by the completion): leave the topology alone until they are repaired.
  443. return
  444. # Fleet is healthy on the new code: fold per-profile gateways into one multiplexer when nothing
  445. # blocks it (deterministic; never prompts), else print the blockers and the one-liner to run later.
  446. try:
  447. from hermes_cli.gateway_migrate import maybe_auto_migrate_after_update
  448. maybe_auto_migrate_after_update()
  449. except (Exception, SystemExit) as exc: # health: allow BLE001 -- a SystemExit here must not fail a committed update
  450. logger.warning('Multiplex auto-migration after update failed: %s', exc)
  451. print(f" ⚠ Gateway multiplex migration did not finish: {exc} (run `hermes gateway migrate` later)")