update_abort_recovery.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282
  1. """Fresh-process recovery after the update's in-process restart phase aborts. Separate owner
  2. from ``update_cmd``: its own vocabulary (``verified`` / ``relaunch_attempted`` / ``failed``, serve units, survivors) and
  3. its own fail-closed contract."""
  4. from __future__ import annotations
  5. import json
  6. import logging
  7. import os
  8. import shutil
  9. import subprocess
  10. import sys
  11. logger = logging.getLogger(__name__)
  12. def _serve_unit_recovery_available() -> bool:
  13. """Can a fresh process restart ``hermes-serve*``/``hermes-dashboard*`` units on this host?"""
  14. return sys.platform == "linux" and bool(shutil.which("systemctl"))
  15. def _surviving_pre_update_serve_runtimes(plan) -> list[dict]:
  16. """Pre-update serve/dashboard runtimes that are STILL the same process — i.e. live on the
  17. pre-update code generation. Identity is the incarnation ``(pid, create_time)``, never the PID
  18. alone: ``ledger_entries()`` prunes dead entries, but a correctly restarted serve can come back
  19. on the same number. Fail closed on missing evidence (unreadable ledger, no incarnation on either
  20. side): the runtime counts as surviving."""
  21. planned: dict[int, dict] = {}
  22. try:
  23. for runtime in getattr(plan, "runtimes", ()) or ():
  24. if getattr(runtime, "kind", None) not in ("serve", "dashboard"):
  25. continue
  26. pid = getattr(runtime, "pid", None)
  27. if not isinstance(pid, int) or pid <= 0:
  28. continue
  29. detail = getattr(runtime, "detail", None)
  30. detail = detail if isinstance(detail, dict) else {}
  31. planned[pid] = {
  32. "pid": pid, "kind": str(getattr(runtime, "kind", "")),
  33. "profile": str(getattr(runtime, "profile", "")),
  34. "supervisor": str(getattr(runtime, "supervisor", "")),
  35. "_create_time": _numeric(detail.get("create_time"))}
  36. if detail.get("systemd_unit"):
  37. # The owning unit and scope name the exact restart command (#141021).
  38. planned[pid]["systemd_unit"] = str(detail["systemd_unit"])
  39. planned[pid]["systemd_scope"] = detail.get("systemd_scope")
  40. except Exception as exc:
  41. logger.debug("Could not read planned serve runtimes: %s", exc)
  42. return []
  43. if not planned:
  44. return []
  45. try:
  46. from hermes_cli.process_identity import ledger_entries
  47. live: dict[int, float | None] = {
  48. entry["pid"]: _numeric(entry.get("create_time"))
  49. for entry in ledger_entries()
  50. if entry.get("purpose") in ("serve", "dashboard") and isinstance(entry.get("pid"), int)}
  51. except Exception as exc:
  52. logger.debug("Serve/dashboard survivor probe failed: %s", exc)
  53. live = None
  54. def _still_live(pid, row) -> bool:
  55. if live is None:
  56. return True
  57. if pid not in live:
  58. return False
  59. planned_created, live_created = row["_create_time"], live[pid]
  60. # Same number, different process: the pre-update runtime is gone and something new
  61. # registered under its PID. Not a survivor.
  62. return not (
  63. planned_created is not None and live_created is not None
  64. and abs(float(live_created) - float(planned_created)) >= 2.0)
  65. # The operator-facing row drops the incarnation (a matching key only).
  66. survivors = [
  67. {k: v for k, v in row.items() if k != "_create_time"}
  68. for pid, row in planned.items() if _still_live(pid, row)]
  69. return sorted(survivors, key=lambda row: row["pid"])
  70. def _numeric(value):
  71. return value if isinstance(value, (int, float)) else None
  72. def _qualified_serve_skips(skip_units) -> list[dict]:
  73. """Scope-qualify the units the aborted phase already settled: ``<scope>/<unit>`` because
  74. ``hermes-serve.service`` can exist in BOTH the user and system manager (two processes)."""
  75. rows: list[dict] = []
  76. for entry in sorted(skip_units or ()):
  77. scope, sep, unit = str(entry).partition("/")
  78. if sep and scope in ("user", "system") and unit:
  79. rows.append({"scope": scope, "unit": unit})
  80. elif entry:
  81. rows.append({"unit": str(entry)})
  82. return rows
  83. def _run_fresh_recovery_process(
  84. profiles, candidates, *, gateway_mode: bool, recover_serve: bool, skip_units
  85. ) -> subprocess.CompletedProcess | None:
  86. """Spawn ``hermes_cli.update_restart_recovery --stdin`` detached from this process; None when it
  87. could not run (no systemd-run in gateway mode, OSError, timeout) — the caller fails closed."""
  88. command = [sys.executable, "-m", "hermes_cli.update_restart_recovery", "--stdin"]
  89. env = os.environ.copy()
  90. env["HERMES_UPDATE_RESTART_RECOVERY"] = "1"
  91. for marker in ("_HERMES_GATEWAY", "HERMES_GATEWAY", "HERMES_GATEWAY_MODE"):
  92. env.pop(marker, None)
  93. # A gateway-triggered update may run inside the gateway's systemd cgroup: put the recovery in a
  94. # transient user scope or KillMode terminates it with the old service. No systemd-run -> fail
  95. # closed rather than pretend the in-cgroup child is independent.
  96. if gateway_mode and sys.platform == "linux":
  97. systemd_run = shutil.which("systemd-run")
  98. if not systemd_run:
  99. logger.warning("Cannot isolate fresh gateway recovery from the gateway cgroup")
  100. return None
  101. command = [systemd_run, "--user", "--scope", "--quiet", "--collect", "--", *command]
  102. kwargs = {
  103. "input": json.dumps({
  104. "profiles": profiles, "supervisors": candidates,
  105. "serve_units": {"recover": recover_serve, "skip": _qualified_serve_skips(skip_units)}}),
  106. "capture_output": True, "text": True, "encoding": "utf-8", "errors": "replace",
  107. "env": env,
  108. # Gateway profiles run sequentially at up to 90s each, plus the serve pass's own
  109. # restart + settle budget — don't kill a recovery that was working.
  110. "timeout": max(180, 30 + 90 * len(profiles) + (150 if recover_serve else 0))}
  111. if sys.platform == "win32":
  112. kwargs["creationflags"] = (
  113. getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) | getattr(subprocess, "DETACHED_PROCESS", 0))
  114. else:
  115. # detached-spawn: ok — awaited with a timeout; the gateway restart it runs must not share the updater's group
  116. kwargs["start_new_session"] = True
  117. try:
  118. return subprocess.run(command, **kwargs, check=False)
  119. except (OSError, subprocess.TimeoutExpired) as exc:
  120. logger.warning("Fresh gateway restart recovery failed: %s", exc)
  121. return None
  122. def _parse_serve_units(raw_serve, *, recover_serve: bool) -> dict[str, list]:
  123. """Validate the child's ``serve_units`` block; an unreadable block is not "nothing to do" when
  124. serve recovery was requested (those units may still serve the pre-update generation)."""
  125. if (
  126. isinstance(raw_serve, dict)
  127. and isinstance(raw_serve.get("verified"), list) and isinstance(raw_serve.get("failed"), list)
  128. and all(isinstance(unit, str) for unit in (*raw_serve["verified"], *raw_serve["failed"]))):
  129. return {"verified": sorted(raw_serve["verified"]), "failed": sorted(raw_serve["failed"])}
  130. if recover_serve:
  131. logger.warning("Fresh recovery returned an invalid serve-unit result")
  132. return {"verified": [], "failed": ["<unreadable>"]}
  133. return {"verified": [], "failed": []}
  134. def _recover_gateway_restart_after_abort(
  135. plan, *, gateway_mode: bool, skip_profiles: set[str] | None = None,
  136. skip_units: set[str] | None = None) -> dict[str, list]:
  137. """Retry supervised gateway restarts from a clean Python process. Only inventory-classified
  138. supervisor-owned profiles.
  139. ``skip_units`` names the units the aborted phase already settled, as ``<scope>/<unit>``. The scope is
  140. part of the identity, not decoration: ``hermes-serve.service`` can exist in both the user and the system
  141. manager as two different processes, and an unqualified token would let a settled one suppress recovery
  142. of a stale one (review on #96235).
  143. """
  144. from hermes_cli.update_cmd import _gateway_recovery_partition
  145. candidates, skipped = _gateway_recovery_partition(plan, skip_profiles=skip_profiles)
  146. profiles = sorted(candidates)
  147. recover_serve = _serve_unit_recovery_available()
  148. def _result(requested, verified, relaunch_attempted, failed, serve_units=None) -> dict[str, list]:
  149. return {
  150. "requested": requested, "verified": verified, "relaunch_attempted": relaunch_attempted,
  151. "failed": failed, "skipped": skipped,
  152. "serve_units": {"verified": [], "failed": []} if serve_units is None else serve_units}
  153. if not profiles and not recover_serve:
  154. return _result([], [], [], [])
  155. def _all_failed() -> dict[str, list]:
  156. return _result(profiles, [], [], profiles)
  157. result = _run_fresh_recovery_process(
  158. profiles, candidates, gateway_mode=gateway_mode, recover_serve=recover_serve, skip_units=skip_units)
  159. if result is None:
  160. return _all_failed()
  161. if result.returncode != 0:
  162. logger.warning("Fresh gateway restart recovery exited %s", result.returncode)
  163. return _all_failed()
  164. try:
  165. recovery_result = json.loads(result.stdout or "")
  166. verified = recovery_result.get("verified")
  167. relaunch_attempted = recovery_result.get("relaunch_attempted")
  168. failed = recovery_result.get("failed")
  169. raw_serve = recovery_result.get("serve_units") or {"verified": [], "failed": []}
  170. except (AttributeError, TypeError, ValueError):
  171. logger.warning("Fresh gateway restart recovery returned invalid JSON")
  172. return _all_failed()
  173. serve_units = _parse_serve_units(raw_serve, recover_serve=recover_serve)
  174. buckets = (verified, relaunch_attempted, failed)
  175. reported: list[str] = []
  176. if all(isinstance(bucket, list) for bucket in buckets):
  177. reported = [*verified, *relaunch_attempted, *failed]
  178. if (
  179. not all(isinstance(bucket, list) for bucket in buckets)
  180. or any(not isinstance(profile, str) for profile in reported)
  181. or set(reported) != set(profiles) or len(reported) != len(set(reported))):
  182. logger.warning("Fresh gateway restart recovery returned incomplete profiles")
  183. return _all_failed()
  184. verified, relaunch_attempted, failed = sorted(verified), sorted(relaunch_attempted), sorted(failed)
  185. covered_map = recovery_result.get("covered")
  186. for owner, others in (covered_map if isinstance(covered_map, dict) else {}).items():
  187. if isinstance(owner, str) and isinstance(others, list) and others:
  188. print(
  189. f" • One host gateway serves {owner} and {', '.join(str(o) for o in others)} — "
  190. f"restarted once through {owner}; that restart is their outcome."
  191. )
  192. for names, text in (
  193. (verified, " ✓ Restarted supervised gateway(s) in a fresh process (systemd-verified active): "),
  194. (relaunch_attempted, " ⚠ Relaunch attempted in a fresh process but not"
  195. " supervisor-verified (check these gateways manually): "),
  196. (serve_units["verified"], " ✓ Restarted serve unit(s) in a fresh process (new main PID observed): "),
  197. (serve_units["failed"], " ⚠ Could not verify a replacement for serve unit(s): ")):
  198. if names:
  199. print(text + ", ".join(names))
  200. return _result(profiles, verified, relaunch_attempted, failed, serve_units)
  201. def _warn_stale_serve_runtimes(rows) -> None:
  202. """Name the serve/dashboard processes still on pre-update code: ``hermes serve`` hosts
  203. ``tui_gateway.server``, and an un-restarted unit keeps the pre-pull ``sys.modules`` graph so
  204. every chat turn fails with an ``ImportError`` no gateway row explains."""
  205. if not rows:
  206. return
  207. print(
  208. " ⚠ These serve/dashboard processes still run pre-update code"
  209. " (they started before the checkout changed):")
  210. for row in rows:
  211. print(
  212. f" pid {row.get('pid')} — {row.get('kind')}"
  213. f" (profile {row.get('profile') or 'default'}, {row.get('supervisor') or 'unknown'})")
  214. unit_rows = [row for row in rows if row.get("supervisor") == "systemd"]
  215. if len(unit_rows) < len(rows):
  216. print(" Ask their owner to relaunch `hermes serve` / `hermes dashboard`, or reconnect Desktop for an SSH backend.")
  217. # A unit-managed backend restarts through its own unit and scope — never "ask the owner to
  218. # relaunch", and never a hardcoded user-scope hermes-serve (#141021).
  219. from hermes_cli.main_dashboard import _format_restart_commands, _systemd_restart_commands
  220. for row in unit_rows:
  221. commands = _systemd_restart_commands(row.get("systemd_unit") or "<unit>", row.get("systemd_scope"))
  222. print(f" pid {row.get('pid')} is managed by systemd: {_format_restart_commands(commands)}")
  223. if sys.platform == "darwin" and any(row.get("supervisor") == "launchd" for row in rows):
  224. print(" For launchd-managed backends: `launchctl kickstart -k gui/$UID/<label>`.")
  225. def _owed_stale_serve_rows(rows) -> list[dict]:
  226. """Survivors the updater itself owes a restart for. A Desktop-supervised serve is excluded: the
  227. recovery pass is forbidden to restart it (it hosts the live Desktop chats), so counting it would
  228. end every update with the Desktop open as incomplete/exit 1. It is still named by
  229. :func:`_warn_stale_serve_runtimes` and recorded in the receipt. See #111494. (The
  230. fleet-restart-pending marker draws the same boundary for its own inventory, so a supervisor-owned
  231. serve row no longer keeps that warning armed either.)"""
  232. from hermes_cli.update_inventory import CLIENT_OWNED_SERVE_SUPERVISORS
  233. return [row for row in (rows or []) if row.get("supervisor") not in CLIENT_OWNED_SERVE_SUPERVISORS]
  234. def _abort_recovery_is_complete(
  235. *, planned_gateway_profiles, covered_gateway_profiles, recovery_result, stale_runtime_rows
  236. ) -> bool:
  237. """May a fresh-process recovery clear the incomplete flag? Only when EVERY inventoried runtime
  238. family is accounted for. Empty ``planned_gateway_profiles`` is deliberately NOT completeness:
  239. with no gateway leg to prove, ``_restart_phase_failure_is_incomplete`` + the stale rows decide.
  240. """
  241. result = recovery_result or {}
  242. return bool(
  243. planned_gateway_profiles
  244. and set(planned_gateway_profiles) <= set(covered_gateway_profiles)
  245. and not (result.get("failed") or result.get("relaunch_attempted"))
  246. and not (result.get("serve_units") or {}).get("failed")
  247. and not _owed_stale_serve_rows(stale_runtime_rows))