update_restart_recovery.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469
  1. """Restart supervised gateway profiles from a clean Python generation.
  2. The normal update command keeps executing in the interpreter that started before
  3. ``git pull``. This module is deliberately small: it imports no gateway code
  4. itself and launches the regular per-profile gateway command in a new
  5. interpreter. It is used only after the in-process restart phase has raised, so
  6. that the recovery path cannot inherit the stale ``sys.modules`` graph that
  7. caused the failure.
  8. Outcome vocabulary (deliberately conservative):
  9. - ``verified`` — the relaunch command exited 0 AND the profile's
  10. systemd unit was independently observed ``active`` afterwards. This is the
  11. only outcome that may claim supervisor coverage.
  12. - ``relaunch_attempted`` — the relaunch command exited 0 but no independent
  13. supervisor observation was possible (non-systemd supervisor, ``systemctl``
  14. missing, or the unit probe was inconclusive). ``rc == 0`` from
  15. ``gateway restart`` is not proof that the new code generation is running,
  16. so this outcome must never be treated as verified coverage.
  17. - ``failed`` — the relaunch command errored, timed out, or exited
  18. non-zero.
  19. The pass covers two runtime families, because the in-process restart phase
  20. covers both and an abort can strand either one (#92145):
  21. - **gateway profiles**, relaunched through the existing per-profile
  22. ``hermes_cli.main -p <profile> gateway restart`` command; and
  23. - **``hermes-serve*`` systemd units**, restarted directly through
  24. ``systemctl``. ``hermes serve`` is not a gateway profile and has no
  25. per-profile relaunch command, but it is the runtime that hosts
  26. ``tui_gateway.server``: the process the original report saw answering every
  27. chat turn with an ``ImportError`` for a symbol that existed on disk. The
  28. unit family is enumerated from systemd itself rather than from the update
  29. inventory, so a manually launched or Desktop-owned ``hermes serve`` — which
  30. has no relaunch authority — can never enter this path.
  31. Serve-unit identity is always ``<scope>/<unit>`` (``user/hermes-serve``,
  32. ``system/hermes-serve``). The two managers can each own a unit of the same
  33. name and they are different processes: an identity that projects the scope
  34. away lets one settled unit suppress recovery of the other, and lets one
  35. scope's outcome speak for the other's. Scope is therefore never dropped and
  36. reconstructed — not in the skip payload, not in ``verified``/``failed``, not
  37. in the receipt.
  38. """
  39. from __future__ import annotations
  40. import argparse
  41. import json
  42. import os
  43. import re
  44. import shutil
  45. import subprocess
  46. import sys
  47. import time
  48. from collections.abc import Callable, Iterable, Mapping
  49. from typing import Any
  50. _RECOVERY_ENV = "HERMES_UPDATE_RESTART_RECOVERY"
  51. _GATEWAY_MARKERS = ("_HERMES_GATEWAY", "HERMES_GATEWAY", "HERMES_GATEWAY_MODE")
  52. _PROFILE_RESTART_TIMEOUT = 90
  53. _VERIFY_TIMEOUT = 15
  54. from hermes_constants import PROFILE_ID_RE as _PROFILE_ID_RE
  55. _SUPERVISOR_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,31}$")
  56. _UNIT_RE = re.compile(r"^hermes-(serve|dashboard)(-[a-z0-9][a-z0-9_-]{0,63})?\.service$")
  57. _SERVE_UNIT_PATTERNS = ("hermes-serve*", "hermes-dashboard*")
  58. _SCOPE_LABELS = ("user", "system")
  59. _UNIT_RESTART_TIMEOUT = 60
  60. _UNIT_SETTLE_ATTEMPTS = 10
  61. _UNIT_SETTLE_DELAY = 1.0
  62. def _run_quiet(run: Callable[..., Any], argv: list[str], *, timeout: int, **extra: Any) -> Any | None:
  63. """Run ``argv`` capturing text output; ``None`` when it errors or times out."""
  64. try:
  65. return run(
  66. argv, capture_output=True, text=True, encoding="utf-8", errors="replace",
  67. check=False, timeout=timeout, **extra,
  68. )
  69. except (OSError, subprocess.TimeoutExpired):
  70. return None
  71. def _stdout(result: Any) -> str:
  72. return (getattr(result, "stdout", "") or "").strip()
  73. def _succeeded(result: Any) -> bool:
  74. return result is not None and getattr(result, "returncode", 1) == 0
  75. def _launch_profile() -> str:
  76. """Profile this recovery process was launched as: ``<root>/profiles/<name>`` in ``HERMES_HOME`` names it,
  77. anything else is the default profile. Mirrors ``hermes_cli.profiles.profile_root_for_env_home`` without
  78. importing it — this module stays stdlib-only at import time."""
  79. home = os.environ.get("HERMES_HOME", "").strip()
  80. if home:
  81. path = os.path.normpath(home)
  82. if os.path.basename(os.path.dirname(path)) == "profiles":
  83. return os.path.basename(path)
  84. return "default"
  85. def _child_environment(profile: str | None = None) -> dict[str, str]:
  86. """Return an environment that cannot self-identify as the gateway owner.
  87. One updater environment relaunches EVERY profile, so a child for another profile would inherit
  88. this process's authorization gates (``DISCORD_ALLOWED_CHANNELS``, ``GATEWAY_ALLOW_ALL_USERS`` ...)
  89. and enforce them as its own — its ``.env`` only overwrites the keys it defines (#113270). Gates are
  90. dropped when *profile* is not the launch profile; a same-profile child keeps an operator export.
  91. Restarting the host multiplexer from a named profile does not copy this process's environ: the
  92. child is ``host_gateway_child_env`` (``served_profile_child_env`` for the default root) so the
  93. launching profile's platform credentials never become the primary adapter's.
  94. """
  95. launch = _launch_profile()
  96. target = profile if profile is not None else launch
  97. if target == "default" and launch != "default":
  98. from tools.environments.local import host_gateway_child_env
  99. env = host_gateway_child_env()
  100. else:
  101. env = os.environ.copy()
  102. if profile is not None and profile != launch:
  103. from tools.environments.local_env_policy import strip_profile_gate_env
  104. strip_profile_gate_env(env)
  105. for marker in _GATEWAY_MARKERS:
  106. env.pop(marker, None)
  107. env[_RECOVERY_ENV] = "1"
  108. return env
  109. def _run_profile_restart(profile: str, *, run: Callable[..., Any]) -> bool:
  110. """Run one profile restart without inheriting the updater's process state."""
  111. kwargs: dict[str, Any] = {"stdin": subprocess.DEVNULL, "env": _child_environment(profile)}
  112. if os.name == "nt":
  113. kwargs["creationflags"] = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0) | getattr(subprocess, "DETACHED_PROCESS", 0)
  114. else:
  115. # detached-spawn: ok — awaited with a timeout; the gateway restart it runs must not share the updater's group
  116. kwargs["start_new_session"] = True
  117. # `--interrupt`: the update owns this restart (drain-first, 31637312899); a live session must not
  118. # strand the profile on pre-update code the way a refusal would. `--detach`: this bounded call
  119. # must not itself become a service-less profile's gateway (killed at the timeout below).
  120. argv = [sys.executable, "-m", "hermes_cli.main", "-p", profile, "gateway", "restart", "--interrupt", "--detach"]
  121. return _succeeded(_run_quiet(run, argv, timeout=_PROFILE_RESTART_TIMEOUT, **kwargs))
  122. def _systemd_unit_candidates(profile: str) -> tuple[str, ...]:
  123. """Unit names the existing systemd gateway lifecycle produces per profile."""
  124. if profile == "default":
  125. return ("hermes-gateway.service", "gateway.service", "gateway-default.service")
  126. return (f"hermes-gateway-{profile}.service", f"gateway-{profile}.service")
  127. def _systemd_verified_active(profile: str, *, run: Callable[..., Any]) -> bool:
  128. """True only when systemd itself reports the profile's unit active — the observation separating
  129. ``verified`` from ``relaunch_attempted``. Any failure (no ``systemctl``, probe error, unit not
  130. ``active``) means we could NOT verify, never that the restart failed."""
  131. systemctl = shutil.which("systemctl")
  132. return bool(systemctl) and any(
  133. _unit_is_active([systemctl, "--user"], unit, run=run, require_rc0=True)
  134. for unit in _systemd_unit_candidates(profile)
  135. )
  136. def _host_state_dir() -> str:
  137. """The path ``gateway.host_rendezvous.host_state_dir()`` resolves, computed locally.
  138. This module imports no Hermes code at runtime — importing the freshly pulled tree is exactly
  139. what aborted the phase that calls us — so the rule is duplicated here rather than shared.
  140. """
  141. override = os.environ.get("HERMES_GATEWAY_LOCK_DIR")
  142. if override:
  143. return override
  144. state_home = os.environ.get("XDG_STATE_HOME") or ""
  145. if not os.path.isabs(state_home):
  146. state_home = os.path.join(os.path.expanduser("~"), ".local", "state")
  147. return os.path.join(state_home, "hermes", "gateway-locks")
  148. def _pid_is_live(pid: int) -> bool:
  149. """Liveness of ``pid``: ``psutil`` when importable, else the POSIX signal-0 probe.
  150. The signal probe is POSIX-only by construction — on Windows ``os.kill(pid, 0)`` sends a real
  151. control event and can kill the target — so an unimportable psutil there means "cannot prove".
  152. """
  153. try:
  154. import psutil
  155. return bool(psutil.pid_exists(pid))
  156. except Exception:
  157. pass
  158. if os.name == "nt":
  159. return False
  160. try:
  161. os.kill(pid, 0) # windows-footgun: ok — POSIX-only branch, guarded by os.name above
  162. except PermissionError:
  163. return True
  164. except OSError:
  165. return False
  166. return True
  167. def _host_served_profiles() -> set[str]:
  168. """Profiles the ONE live host gateway multiplexes, from its rendezvous record.
  169. Restarting any one of them restarts the same process, so they are a single restart target.
  170. Empty (no collapsing, today's per-profile behaviour) when the record is absent, unreadable,
  171. dead, or when liveness cannot be probed — a missed collapse costs an extra restart, a wrong
  172. one would skip a profile that really has its own process.
  173. """
  174. try:
  175. with open(os.path.join(_host_state_dir(), "host-gateway.json"), encoding="utf-8-sig") as handle:
  176. record = json.load(handle)
  177. except (OSError, UnicodeDecodeError, ValueError):
  178. return set()
  179. if not isinstance(record, dict) or record.get("role") != "gateway":
  180. return set()
  181. pid = record.get("pid")
  182. profiles = record.get("profiles")
  183. if not isinstance(pid, int) or pid <= 0 or not isinstance(profiles, list) or not _pid_is_live(pid):
  184. return set()
  185. return {name for name in profiles if isinstance(name, str) and name}
  186. def restart_profiles(
  187. profiles: Iterable[str], *, supervisors: Mapping[str, str] | None = None, run: Callable[..., Any] = subprocess.run
  188. ) -> dict[str, Any]:
  189. """Restart the supplied profiles (only ones whose inventory identified a service supervisor).
  190. Profiles served by the SAME host gateway process are one restart target: a host multiplexes
  191. every profile, so N payload profiles meant N sequential ``gateway restart`` calls, each
  192. killing the successor the previous pass had just verified. The group is restarted exactly
  193. once through one representative and the rest are reported under ``covered`` with that
  194. restart's outcome.
  195. A profile only lands in ``verified`` when its supervisor is systemd and ``systemctl --user is-
  196. active`` independently confirms the unit after the relaunch command succeeded.
  197. """
  198. supervisors = supervisors or {}
  199. result: dict[str, Any] = {"verified": [], "relaunch_attempted": [], "failed": []}
  200. requested = sorted({p for p in profiles if isinstance(p, str) and p})
  201. served = _host_served_profiles()
  202. group = [profile for profile in requested if profile in served]
  203. representative = group[0] if len(group) > 1 else None
  204. covered = group[1:] if representative else []
  205. for profile in requested:
  206. if profile in covered:
  207. continue
  208. if not _run_profile_restart(profile, run=run):
  209. bucket = "failed"
  210. elif supervisors.get(profile) == "systemd" and _systemd_verified_active(profile, run=run):
  211. bucket = "verified"
  212. else:
  213. bucket = "relaunch_attempted"
  214. result[bucket].append(profile)
  215. if profile == representative:
  216. # One process: the representative's observed outcome IS these profiles' outcome.
  217. result[bucket].extend(covered)
  218. result["covered"] = {representative: covered} if representative else {}
  219. return result
  220. def _systemctl_scopes() -> list[tuple[str, list[str]]]:
  221. """``(label, systemctl argv)`` for the user and system scopes (the pair the in-process phase walks), or nothing.
  222. ``systemctl`` comes from ``shutil.which`` so this module never imports a Hermes platform helper —
  223. importing the freshly pulled tree is exactly what aborted the phase that called us. Scopes carry
  224. their label because the same unit name in both managers is two different processes.
  225. """
  226. systemctl = shutil.which("systemctl")
  227. if not systemctl or sys.platform != "linux":
  228. return []
  229. return [("user", [systemctl, "--user"]), ("system", [systemctl])]
  230. def _listed_serve_units(scope: list[str], *, run: Callable[..., Any]) -> list[str]:
  231. """Serve/dashboard units systemd knows about in one scope, validated by name."""
  232. units: list[str] = []
  233. for pattern in _SERVE_UNIT_PATTERNS:
  234. result = _run_quiet(
  235. run,
  236. scope + ["list-units", pattern, "--plain", "--no-legend", "--no-pager"],
  237. timeout=_VERIFY_TIMEOUT,
  238. )
  239. if result is None:
  240. continue
  241. # The glob is a systemd pattern, not a name gate (`hermes-serve*` also matches
  242. # `hermes-server.service`): require the exact base unit or the profile family.
  243. for line in (getattr(result, "stdout", "") or "").splitlines():
  244. parts = line.split()
  245. if parts and _UNIT_RE.fullmatch(parts[0]) and parts[0] not in units:
  246. units.append(parts[0])
  247. return units
  248. def _unit_main_pid(scope: list[str], unit: str, *, run: Callable[..., Any]) -> int:
  249. """The unit's ``MainPID`` via ``systemctl show``; ``0`` when absent or unreadable."""
  250. result = _run_quiet(run, scope + ["show", unit, "--property=MainPID", "--value"], timeout=_VERIFY_TIMEOUT)
  251. try:
  252. return int(_stdout(result) or 0) if _succeeded(result) else 0
  253. except ValueError:
  254. return 0
  255. def _unit_is_active(scope: list[str], unit: str, *, run: Callable[..., Any], require_rc0: bool = False) -> bool:
  256. result = _run_quiet(run, scope + ["is-active", unit], timeout=_VERIFY_TIMEOUT)
  257. if result is None or (require_rc0 and not _succeeded(result)):
  258. return False
  259. return _stdout(result) == "active"
  260. def _serve_unit_replaced(
  261. scope: list[str], unit: str, previous_pid: int, *, run: Callable[..., Any], sleep: Callable[[float], Any]
  262. ) -> bool:
  263. """Did the unit come back on a NEW main process?
  264. ``restart`` returning 0 is not evidence: a live process can keep serving the pre-update
  265. generation while every status command reports success. A changed ``MainPID`` on an ``active``
  266. unit is the observation that the old interpreter and its stale ``sys.modules`` are gone.
  267. """
  268. for attempt in range(_UNIT_SETTLE_ATTEMPTS):
  269. if attempt:
  270. sleep(_UNIT_SETTLE_DELAY)
  271. if _unit_is_active(scope, unit, run=run):
  272. current = _unit_main_pid(scope, unit, run=run)
  273. if current > 0 and current != previous_pid:
  274. return True
  275. return False
  276. def _split_skip_entry(entry: Any) -> tuple[str, str]:
  277. """``(scope_label, base_unit)`` of a skip entry in either the mapping or ``scope/unit`` shape."""
  278. if isinstance(entry, Mapping):
  279. scope_label = str(entry.get("scope") or "")
  280. unit = str(entry.get("unit") or "")
  281. else:
  282. scope_label, sep, unit = str(entry).partition("/")
  283. if not sep:
  284. scope_label, unit = "", scope_label
  285. return scope_label, unit.removesuffix(".service")
  286. def _normalized_skips(skip_units: Iterable[Any]) -> tuple[set[tuple[str, str]], set[str]]:
  287. """Split already-settled units into scope-qualified and legacy entries (a bare ``"hermes-serve"``
  288. carries no scope and cannot say WHICH of two same-named processes was settled)."""
  289. qualified: set[tuple[str, str]] = set()
  290. legacy: set[str] = set()
  291. for entry in skip_units or ():
  292. scope_label, base = _split_skip_entry(entry)
  293. if base:
  294. (qualified.add((scope_label, base)) if scope_label in _SCOPE_LABELS else legacy.add(base))
  295. return qualified, legacy
  296. def restart_serve_units(
  297. *, skip_units: Iterable[Any] = (), run: Callable[..., Any] = subprocess.run, sleep: Callable[[float], Any] = time.sleep
  298. ) -> dict[str, list[str]]:
  299. """Restart every active ``hermes-serve*``/``hermes-dashboard*`` systemd unit from this process.
  300. Units are enumerated from systemd, never from the update inventory, so a manually launched or
  301. Desktop-owned ``hermes serve`` (no unit) structurally cannot be touched here.
  302. """
  303. skipped_qualified, skipped_legacy = _normalized_skips(skip_units)
  304. # (scope, base unit) -> replaced? The same unit name in the user and the system
  305. # scope is two processes; each is proven, reported and accounted for on its own.
  306. outcomes: dict[tuple[str, str], bool] = {}
  307. seen: set[tuple[str, str]] = set()
  308. for scope_label, scope in _systemctl_scopes():
  309. for unit in _listed_serve_units(scope, run=run):
  310. base = unit.removesuffix(".service")
  311. target = (scope_label, base)
  312. if target in seen or target in skipped_qualified or base in skipped_legacy:
  313. continue
  314. seen.add(target)
  315. if not _unit_is_active(scope, unit, run=run):
  316. continue # not running: nothing serves a stale generation from it
  317. previous_pid = _unit_main_pid(scope, unit, run=run)
  318. # No readable main PID: a replacement can't be observed, so it can't be claimed
  319. # (restarting blind and reporting success is the failure mode this module removes).
  320. # A failed restart includes the unprivileged system-scope case; no sudo probe —
  321. # an unverifiable unit must read as failed so the update stays explicitly incomplete.
  322. outcomes[target] = previous_pid > 0 and _succeeded(
  323. _run_quiet(run, scope + ["--no-ask-password", "restart", unit], timeout=_UNIT_RESTART_TIMEOUT)
  324. ) and _serve_unit_replaced(scope, unit, previous_pid, run=run, sleep=sleep)
  325. # ``<scope>/<unit>`` is the only identity this module reports for a serve unit.
  326. return {
  327. "verified": sorted(f"{scope}/{base}" for (scope, base), ok in outcomes.items() if ok),
  328. "failed": sorted(f"{scope}/{base}" for (scope, base), ok in outcomes.items() if not ok),
  329. }
  330. def _parse_payload(stream) -> tuple[list[str], dict[str, str], bool, list[str]]:
  331. payload = json.load(stream)
  332. if not isinstance(payload, dict):
  333. payload = {}
  334. profiles = payload.get("profiles")
  335. if not isinstance(profiles, list):
  336. raise ValueError("recovery payload must contain a profiles list")
  337. if any(not isinstance(profile, str) or not _PROFILE_ID_RE.fullmatch(profile) for profile in profiles):
  338. raise ValueError("recovery profiles contain an invalid profile id")
  339. raw_supervisors = payload.get("supervisors")
  340. supervisors: dict[str, str] = {}
  341. if raw_supervisors is not None:
  342. if not isinstance(raw_supervisors, dict) or any(
  343. not isinstance(profile, str)
  344. or not isinstance(supervisor, str)
  345. or not _PROFILE_ID_RE.fullmatch(profile)
  346. or not _SUPERVISOR_RE.fullmatch(supervisor)
  347. for profile, supervisor in raw_supervisors.items()
  348. ):
  349. raise ValueError("recovery supervisors map is invalid")
  350. supervisors = dict(raw_supervisors)
  351. raw_serve = payload.get("serve_units")
  352. recover_serve = False
  353. skip_units: list[str] = []
  354. if raw_serve is not None:
  355. if not isinstance(raw_serve, dict):
  356. raise ValueError("recovery serve_units block is invalid")
  357. recover_serve = bool(raw_serve.get("recover"))
  358. raw_skip = raw_serve.get("skip") or []
  359. if not isinstance(raw_skip, list) or any(not isinstance(entry, (str, dict)) for entry in raw_skip):
  360. raise ValueError("recovery serve_units skip list is invalid")
  361. for entry in raw_skip:
  362. # A skip entry names one already-settled process. The qualified shape carries the systemd
  363. # scope (`hermes-serve.service` can exist in BOTH managers; settling one says nothing about
  364. # the other). A bare string is the legacy shape a pre-update interpreter can still send,
  365. # read as scope-agnostic by `restart_serve_units`.
  366. if isinstance(entry, dict):
  367. if not isinstance(entry.get("unit"), str):
  368. raise ValueError("recovery serve_units skip list is invalid")
  369. if not isinstance(entry.get("scope"), str):
  370. entry = {"unit": entry["unit"]}
  371. scope_label, base = _split_skip_entry(entry)
  372. # Only shapes systemd can produce for this family (a skip is a name filter, never a command
  373. # argument). An unrecognized scope DROPS the entry rather than raising: a dropped skip costs at
  374. # most one extra restart-and-verify; honouring an unreadable one could suppress recovery.
  375. if (scope_label and scope_label not in _SCOPE_LABELS) or not _UNIT_RE.fullmatch(f"{base}.service"):
  376. continue
  377. skip_units.append(f"{scope_label}/{base}" if scope_label else base)
  378. return profiles, supervisors, recover_serve, skip_units
  379. def main(argv: list[str] | None = None) -> int:
  380. parser = argparse.ArgumentParser(description=__doc__)
  381. parser.add_argument("--stdin", action="store_true", help=argparse.SUPPRESS)
  382. args = parser.parse_args(argv)
  383. if not args.stdin:
  384. parser.error("this command is an internal update-recovery entry point")
  385. try:
  386. profiles, supervisors, recover_serve, skip_units = _parse_payload(sys.stdin)
  387. result = restart_profiles(profiles, supervisors=supervisors)
  388. result["serve_units"] = (
  389. restart_serve_units(skip_units=skip_units) if recover_serve else {"verified": [], "failed": []}
  390. )
  391. except (ValueError, json.JSONDecodeError) as exc:
  392. print(json.dumps({
  393. "error": str(exc), "verified": [], "relaunch_attempted": [], "failed": [],
  394. "serve_units": {"verified": [], "failed": []},
  395. }))
  396. return 2
  397. print(json.dumps(result, sort_keys=True))
  398. return 1 if result["failed"] or result["serve_units"]["failed"] else 0
  399. if __name__ == "__main__":
  400. raise SystemExit(main())