update_inventory.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632
  1. """Runtime inventory + update plan for the fleet-update pipeline.
  2. One read-only pass answering, BEFORE any mutation: which Hermes runtimes run on this machine, how
  3. each is deployed, which ones this update touches, and how each restarts. Every collector is a
  4. side-effect-free probe, so ``hermes update --plan`` is safe on a live fleet.
  5. """
  6. from __future__ import annotations
  7. import logging
  8. import shlex
  9. import sys
  10. from contextlib import contextmanager, suppress
  11. from dataclasses import dataclass, field, asdict, fields as dataclass_fields
  12. from typing import Any, Callable, Optional
  13. logger = logging.getLogger(__name__)
  14. @dataclass
  15. class RuntimeRecord:
  16. """One running (or expected) Hermes runtime on this machine."""
  17. kind: str # gateway | dashboard | serve
  18. profile: str
  19. pid: Optional[int] = None
  20. supervisor: str = "manual" # systemd | launchd | desktop | windows-service | service | manual | manual-serve
  21. code_sha: Optional[str] = None # stamped running-code sha
  22. # See #91283.
  23. code_version: Optional[str] = None
  24. restart_via: str = "" # mechanism id, see _RESTART_MECHANISMS
  25. detail: dict = field(default_factory=dict)
  26. @dataclass
  27. class UpdatePlan:
  28. """The full pre-update picture: install shape + runtimes + actions."""
  29. install_method: str = "unknown" # git | docker | nix | apt | ...
  30. updatable_in_place: bool = True
  31. update_mechanism: str = "hermes update"
  32. expected_sha: Optional[str] = None # current checkout HEAD (pre-pull)
  33. expected_version: Optional[str] = None
  34. profiles: list = field(default_factory=list)
  35. runtimes: list = field(default_factory=list) # list[RuntimeRecord]
  36. def to_dict(self) -> dict[str, Any]:
  37. return asdict(self) # recursive: RuntimeRecord entries become dicts
  38. @classmethod
  39. def from_dict(cls, data: dict[str, Any]) -> UpdatePlan:
  40. """Inverse of :meth:`to_dict` (the plan crosses the post-swap hand-off as JSON)."""
  41. fields_ = {f.name for f in dataclass_fields(cls)}
  42. plan = cls(**{k: v for k, v in data.items() if k in fields_ and k != "runtimes"})
  43. record_fields = {f.name for f in dataclass_fields(RuntimeRecord)}
  44. plan.runtimes = [
  45. RuntimeRecord(**{k: v for k, v in r.items() if k in record_fields})
  46. for r in data.get("runtimes") or [] if isinstance(r, dict)
  47. ]
  48. return plan
  49. def _detect_supervisor_for_pid(pid: int, service_pids: set, windows_service_pids: set | None = None) -> str:
  50. """Classify how a live gateway PID is supervised."""
  51. if windows_service_pids and pid in windows_service_pids:
  52. # SCM-supervised Windows gateway: the update pause machinery stops the SERVICE via sc.exe
  53. # instead of killing the child, so reconciliation must plan it under its own mechanism id.
  54. # See #91277.
  55. return "windows-service"
  56. if pid not in service_pids:
  57. return "manual"
  58. with suppress(Exception):
  59. from hermes_cli.gateway import is_macos, supports_systemd_services
  60. if supports_systemd_services():
  61. return "systemd"
  62. if is_macos():
  63. return "launchd"
  64. return "service"
  65. # THE restart policy table: restart execution consumes these ids via match_runtime_outcomes / the
  66. # update's restart phase, and the receipt records per-runtime outcomes against them. Display
  67. # strings are derived by describe_restart_mechanism — never the other way around.
  68. _RESTART_MECHANISMS = {
  69. "systemd": "systemd", "launchd": "launchd", "desktop": "desktop",
  70. "windows-service": "windows-service", "manual-serve": "respawn-argv",
  71. "desktop-ssh": "desktop-ssh",
  72. }
  73. _MECHANISM_DESCRIPTIONS = {
  74. "systemd": "systemctl restart (drain-first SIGUSR1 when supported)",
  75. "launchd": "launchctl kickstart -k (drain-first, per-label domain)",
  76. "desktop": "Desktop app respawns its serve backend",
  77. "windows-service": "sc.exe stop before venv mutation, sc.exe start after update",
  78. "respawn-argv": "stop before code swap, relaunch with recorded launch args",
  79. "desktop-ssh": "the remote Desktop that spawned it over SSH respawns it when it reconnects",
  80. }
  81. _SERVE_KINDS = ("serve", "dashboard")
  82. # Serve backends a Desktop client owns and recycles: this app's own pool child (``desktop``) or one
  83. # another machine's Desktop spawned here over SSH (``desktop-ssh``). The updater never restarts
  84. # either; stopping one out from under its client only makes the client respawn it.
  85. CLIENT_OWNED_SERVE_SUPERVISORS = frozenset({"desktop", "desktop-ssh"})
  86. def _restart_mechanism(supervisor: str, profile: str) -> str:
  87. """Machine-readable restart mechanism id for a runtime.
  88. THE policy table (#91277 Phase 2): restart execution consumes these ids via
  89. :func:`match_runtime_outcomes` / the update's restart phase, and the receipt records per-runtime
  90. outcomes against them. Display strings are derived by :func:`describe_restart_mechanism` — never the
  91. other way around.
  92. """
  93. return _RESTART_MECHANISMS.get(supervisor, "manual")
  94. def describe_restart_mechanism(mechanism: str, profile: str) -> str:
  95. """Human-readable description of a restart mechanism id."""
  96. return _MECHANISM_DESCRIPTIONS.get(mechanism) or (
  97. f"hermes -p {profile} gateway restart" if profile != "default" else "hermes gateway restart"
  98. )
  99. def _runtime(
  100. kind: str, profile: str, pid: Optional[int], supervisor: str,
  101. code_sha: Any = None, code_version: Any = None, **extra: Any,
  102. ) -> RuntimeRecord:
  103. """A :class:`RuntimeRecord` with ``restart_via`` derived from its supervisor."""
  104. return RuntimeRecord(
  105. kind=kind, profile=profile, pid=pid, supervisor=supervisor,
  106. code_sha=str(code_sha) if code_sha else None, code_version=code_version,
  107. restart_via=_restart_mechanism(supervisor, profile), **extra,
  108. )
  109. @contextmanager
  110. def _probe(label: str):
  111. """Run one inventory collector; a failure is logged at debug and yields fewer rows, never an exception."""
  112. try:
  113. yield
  114. except Exception as exc:
  115. logger.debug("%s failed: %s", label, exc)
  116. def _collect_install_shape(plan: UpdatePlan) -> None:
  117. with _probe("Install-method probe"):
  118. from hermes_cli.config import detect_install_method, get_managed_system, recommended_update_command_for_method
  119. method = detect_install_method()
  120. managed = get_managed_system()
  121. plan.install_method = managed or method
  122. plan.updatable_in_place = method in ("git", "unknown") and not managed
  123. # Baked image provenance is authoritative when present: a bind-mounted checkout inside a
  124. # container can look like `git` while the running filesystem is an immutable image.
  125. # Fail-closed: an invalid marker still flips the plan to not-updatable.
  126. with _probe("Image provenance probe"):
  127. # See #91277.
  128. from hermes_cli.image_provenance import read_image_provenance
  129. provenance = read_image_provenance()
  130. if provenance is not None:
  131. plan.updatable_in_place = False
  132. if provenance.valid and provenance.manager:
  133. plan.install_method = provenance.manager
  134. plan.update_mechanism = recommended_update_command_for_method(method)
  135. def _supervisor_classifier() -> Callable[[int], str]:
  136. """``pid -> supervisor`` over the service-PID sets; each probe degrades to an empty set."""
  137. service_pids: set = set()
  138. with _probe("Service-PID probe"):
  139. from hermes_cli.gateway import _get_service_pids
  140. service_pids = _get_service_pids(all_profiles=True) or set()
  141. # Windows SCM services (no-op off Windows): the update's pause phase stops these via `sc.exe
  142. # stop` / restarts via `sc.exe start`, so the plan must carry the matching mechanism id.
  143. # --- SCM-supervised gateway PIDs (Windows) ------------------------------
  144. # find_windows_gateway_services() maps validated gateway PIDs through process ancestry to running SCM
  145. # service PIDs (no-op off Windows). See #91277.
  146. windows_service_pids: set = set()
  147. with _probe("Windows SCM service-ownership probe"):
  148. from hermes_cli.gateway import find_windows_gateway_services
  149. windows_service_pids = {int(service.gateway_pid) for service in find_windows_gateway_services()}
  150. return lambda pid: _detect_supervisor_for_pid(pid, service_pids, windows_service_pids)
  151. def _collect_gateway_runtimes(plan: UpdatePlan, profile_homes: list, seen: set[int]) -> None:
  152. """Per-profile gateways: control-socket identity first (declared by the process itself, including
  153. supervisor provenance — no argv/PID inference), ``gateway_state.json`` fallback, then PID-file
  154. mapped gateways no status record covers."""
  155. supervisor = _supervisor_classifier()
  156. with _probe("Gateway-state inventory"):
  157. from gateway.status import live_gateway_pid_for_home, read_runtime_status
  158. from hermes_cli.update_receipt import _socket_identity
  159. for profile, home in profile_homes:
  160. sock = _socket_identity(home)
  161. if sock is not None:
  162. pid, record = sock
  163. if pid in seen:
  164. continue # one multiplex gateway answers identify for several homes — one record per process
  165. seen.add(pid)
  166. from gateway.control_socket import launch_supervisor
  167. # The protocol's "none" (manual or desktop) is no answer: classify it outside-in.
  168. declared = launch_supervisor(record)
  169. sup = declared if declared else supervisor(pid)
  170. else:
  171. # Verified identity, not bare PID existence: a ``stopped`` record whose PID was recycled
  172. # by an unrelated process fabricated a phantom gateway the restart phase could never
  173. # touch, so `hermes update` exited partial (#109680).
  174. pid = live_gateway_pid_for_home(home)
  175. if pid is None or pid in seen:
  176. continue
  177. record = read_runtime_status(home / "gateway_state.json") or {}
  178. seen.add(pid)
  179. sup = supervisor(pid)
  180. plan.runtimes.append(_runtime("gateway", profile, pid, sup, record.get("code_sha"), record.get("code_version")))
  181. with _probe("PID-file gateway inventory"):
  182. from hermes_cli.gateway import find_profile_gateway_processes
  183. for proc in find_profile_gateway_processes():
  184. if proc.pid not in seen:
  185. seen.add(proc.pid)
  186. plan.runtimes.append(_runtime("gateway", proc.profile, proc.pid, supervisor(proc.pid)))
  187. def _loaded_backend_launchd_jobs() -> list:
  188. """Loaded launchd dashboard/serve jobs for supervisor classification.
  189. The probe itself is darwin-gated (``[]`` on every other host); here any failure also degrades
  190. to ``[]`` — classification falls back to the spawner probe and never aborts the inventory.
  191. See #116503."""
  192. with suppress(Exception):
  193. from hermes_cli import main_dashboard as _dash
  194. return _dash._loaded_launchd_backend_jobs()
  195. return []
  196. def _launchd_owner_for_ledger_entry(entry: dict, pid: int, jobs: list) -> tuple[str, str, int | None] | None:
  197. """``(domain, label, live_pid)`` of the loaded launchd job owning this ledger row, if any.
  198. A KeepAlive LaunchAgent backend's recorded spawner (the bootstrap shell) is long dead, so the
  199. spawner probe alone misreads the row as ``manual-serve`` — and a respawn-argv restart then
  200. fights the job's own KeepAlive respawn. The loaded-job match (live PID, an ancestor, or the
  201. normalized ``ProgramArguments``) is the authoritative classification. See #116503."""
  202. with suppress(Exception):
  203. from hermes_cli import main_dashboard as _dash
  204. from hermes_cli.dashboard_procs import _process_ancestors
  205. try:
  206. cmdline = shlex.split(str(entry.get("argv") or "")) or None
  207. except ValueError:
  208. cmdline = None
  209. return _dash._launchd_job_owning_backend(pid, cmdline, jobs, ancestors=_process_ancestors(pid))
  210. return None
  211. def _systemd_owner_for_ledger_pid(pid: int) -> tuple[str, str | None] | None:
  212. """``(unit, scope)`` of the systemd unit whose live ``MainPID`` is *pid*, if any.
  213. A unit-launched backend sets neither ``HERMES_SPAWN`` nor ``HERMES_PARENT_PID``, so the spawner
  214. probe reads ``manual-serve`` and the update tells the user to "ask the owner to relaunch
  215. `hermes dashboard`" for a process systemd owns (#141021). Ownership is the unit's MainPID, not
  216. cgroup membership (a hand-started dashboard inside some unrelated unit's cgroup is still manual);
  217. ``None`` off Linux, when unreadable, or when no unit owns it."""
  218. with suppress(Exception):
  219. from hermes_cli import main_dashboard as _dash
  220. return _dash._systemd_owner_for_pid(pid)
  221. return None
  222. def _is_desktop_ssh_ledger_entry(entry: dict) -> bool:
  223. """Is this row the backend a (possibly remote) Desktop spawned over SSH? The canonical argv
  224. predicate also classifies rows written before the ledger carried ``isolated``, which is exactly
  225. the pre-update serve the first update after this change inventories."""
  226. from hermes_cli._startup_fast import is_desktop_ssh_backend_argv
  227. try:
  228. return is_desktop_ssh_backend_argv(shlex.split(str(entry.get("argv") or "")))
  229. except ValueError:
  230. return False
  231. def _collect_ledger_runtimes(plan: UpdatePlan, seen: set[int]) -> None:
  232. """Serve/dashboard backends from the spawn ledger — runtimes the gateway collectors can never see
  233. (a manual `hermes serve --host <ip>` for a remote Desktop, a long-lived `hermes dashboard`).
  234. ledger_entries() live-verifies (pid, create_time) so PID reuse never fabricates a row. Desktop-
  235. supervised backends (spawner still alive) restart via the Desktop's own respawn, not ours.
  236. A backend owned by a loaded launchd job is classified ``launchd`` (kickstart restart, never a
  237. detached argv respawn) — the spawner probe cannot see that (#116503); one that is a systemd
  238. unit's MainPID is ``systemd`` for the same reason (#141021)."""
  239. with _probe("Serve/dashboard ledger inventory"):
  240. from hermes_cli.process_identity import ledger_entries, spawner_is_dead
  241. launchd_jobs = _loaded_backend_launchd_jobs()
  242. for entry in ledger_entries():
  243. purpose, pid = entry.get("purpose"), entry.get("pid")
  244. if purpose not in _SERVE_KINDS or not isinstance(pid, int) or pid in seen:
  245. continue
  246. seen.add(pid)
  247. # detail.create_time: process incarnation, not just the numeric PID — a post-update
  248. # survivor probe comparing PIDs alone calls a NEW serve that reused the number a survivor.
  249. detail = {
  250. "argv": entry.get("argv") or "", "host": entry.get("host") or "",
  251. "port": entry.get("port"), "create_time": entry.get("create_time"),
  252. }
  253. job = _launchd_owner_for_ledger_entry(entry, pid, launchd_jobs) if launchd_jobs else None
  254. if job:
  255. supervisor, detail["launchd_domain"], detail["launchd_label"] = "launchd", job[0], job[1]
  256. elif _is_desktop_ssh_ledger_entry(entry):
  257. # No local spawner, so the probe below would read manual-serve and file a reminder
  258. # nobody here can discharge; its token file and owner nonce belong to the client.
  259. supervisor = "desktop-ssh"
  260. elif spawner_is_dead(entry) is False:
  261. supervisor = "desktop"
  262. elif unit := _systemd_owner_for_ledger_pid(pid):
  263. supervisor, detail["systemd_unit"], detail["systemd_scope"] = "systemd", unit[0], unit[1]
  264. else:
  265. supervisor = "manual-serve"
  266. plan.runtimes.append(_runtime(
  267. str(purpose), str(entry.get("profile") or "default"), pid, supervisor, detail=detail,
  268. ))
  269. def collect_runtime_inventory() -> UpdatePlan:
  270. """Build the pre-update plan. Read-only; never raises — every collector degrades independently.
  271. The result is embeddable in the update receipt and printable via :func:`print_update_plan`.
  272. """
  273. plan = UpdatePlan()
  274. _collect_install_shape(plan)
  275. with _probe("Code-identity probe"):
  276. from hermes_cli.version_info import get_code_identity
  277. identity = get_code_identity(refresh=True)
  278. plan.expected_sha = identity.get("sha")
  279. plan.expected_version = identity.get("version")
  280. profile_homes: list = []
  281. with _probe("Profile enumeration"):
  282. from hermes_cli.update_receipt import _profile_homes
  283. profile_homes = _profile_homes()
  284. plan.profiles = [name for name, _ in profile_homes]
  285. seen: set[int] = set()
  286. _collect_gateway_runtimes(plan, profile_homes, seen)
  287. _collect_ledger_runtimes(plan, seen)
  288. return plan
  289. def print_update_plan(plan: UpdatePlan) -> None:
  290. """Human-readable plan — what the update will touch and how."""
  291. print("Update plan:")
  292. install = f" Install: {plan.install_method}"
  293. if plan.expected_version:
  294. install += f" (v{plan.expected_version}" + (f" @ {plan.expected_sha[:8]}" if plan.expected_sha else "") + ")"
  295. print(install)
  296. if not plan.updatable_in_place:
  297. print(" ⚠ This install is NOT updatable in place.")
  298. print(f" Update via: {plan.update_mechanism}")
  299. print(f" Profiles: {', '.join(plan.profiles) if plan.profiles else '(none found)'}")
  300. if not plan.runtimes:
  301. print(" Running Hermes services: none detected — code swap only.")
  302. return
  303. print(f" Running services to restart ({len(plan.runtimes)}):")
  304. for runtime in plan.runtimes:
  305. sha = f" @ {runtime.code_sha[:8]}" if runtime.code_sha else ""
  306. print(f" • {runtime.kind} [{runtime.profile}] pid {runtime.pid} — {runtime.supervisor}{sha}")
  307. print(f" restart: {describe_restart_mechanism(runtime.restart_via, runtime.profile)}")
  308. def _serve_unit_matches_profile(profile: str, unit: object) -> bool:
  309. """Does *unit* name a ``hermes-serve*``/``hermes-dashboard*`` unit for *profile*? (OWN vocabulary;
  310. the gateway's ``hermes-gateway*`` names never cover serve/dashboard runtimes.)
  311. Exact names only — ``work`` must not claim ``hermes-serve-workbench`` — and a scope prefix
  312. (``user/hermes-serve``) is tolerated because the restart phase records scope-qualified identities in
  313. some lists. See #100479.
  314. """
  315. name = str(unit).removesuffix(".service").rsplit("/", 1)[-1]
  316. suffix = "" if profile == "default" else f"-{profile}"
  317. return name in {f"hermes-serve{suffix}", f"hermes-dashboard{suffix}"}
  318. def _gateway_service_matches_profile(profile: str, service: object) -> bool:
  319. """Match an exact gateway service/label (systemd/launchd/s6 shapes) to a profile.
  320. Never substring-match: ``foo`` must not claim ``hermes-gateway-foobar.service``.
  321. Launchd labels are ``ai.hermes.gateway`` / ``ai.hermes.gateway-<profile>`` — they do
  322. not contain the substring ``hermes-gateway``, so a successful macOS kickstart must
  323. still credit the planned default gateway. A scope prefix (``user/hermes-gateway``,
  324. ``gui/501/ai.hermes.gateway``) is stripped the same way serve units are.
  325. """
  326. name = str(service).removesuffix(".service").rsplit("/", 1)[-1]
  327. if profile == "default":
  328. return name in {"hermes-gateway", "ai.hermes.gateway", "gateway", "gateway-default"}
  329. return name in {f"hermes-gateway-{profile}", f"ai.hermes.gateway-{profile}", f"gateway-{profile}"}
  330. def _gateway_named_in(r: RuntimeRecord, names: set) -> bool:
  331. # Gateway-only vocabulary: a serve/dashboard that merely shares the profile is a
  332. # different process. Exact label match (systemd + launchd + s6), not substring.
  333. return any(_gateway_service_matches_profile(r.profile, name) for name in names)
  334. def match_runtime_outcomes(
  335. plan: UpdatePlan, *, restarted_services: list, relaunched_profiles: list,
  336. externally_supervised_profiles: list, killed_pids: set, failed_units: list,
  337. stale_serve_pids: set | None = None, failed_respawn_pids: set | None = None,
  338. external_gateway_pids: set | None = None,
  339. live_gateway_pids: dict[str, set[int]] | None = None,
  340. ) -> list[dict[str, Any]]:
  341. """Reconcile the plan's runtimes against what the restart phase DID.
  342. The platform restart branches each re-discover their own targets, so a runtime the plan saw can
  343. be missed with no signal. Returns one ``{kind, profile, pid, mechanism, outcome}`` row per
  344. planned runtime; outcome is ``restarted``, ``stopped``, ``failed``, ``deferred``, ``external``
  345. or ``unaccounted`` (no bookkeeping mentions it — the blind-spot tripwire). Never raises.
  346. Serve/dashboard runtimes are reconciled in their OWN vocabulary and never borrow the gateway's
  347. outcome: with ``stale_serve_pids`` a pre-update serve whose incarnation is gone counts as
  348. ``restarted``, one still alive is ``unaccounted``; without the probe an untouched serve stays
  349. ``unaccounted``. A Desktop-supervised serve is ``deferred`` only when the survivor probe RAN
  350. and still lists its pid: the restart phase is forbidden to restart it out from under the app (it
  351. hosts the live Desktop chats), so it is handed back to its supervisor and surfaced. Without a
  352. probe result it remains ``unaccounted``, rather than claiming the app owns an unknown
  353. incarnation. ``failed_respawn_pids`` (serves the dashboard cleanup stopped and could not bring
  354. back) read ``failed`` before the probe, whose "gone" is exactly what a failed respawn looks like
  355. (#109290). The probe itself fails closed (unreadable ledger -> every planned serve is listed as
  356. surviving), so ``deferred`` means "not shown to be gone", not "observed alive". See #111494.
  357. ``live_gateway_pids`` (profile -> gateway PIDs alive for that profile AFTER the restart phase:
  358. control-socket identity, or a runtime-status record whose PID is still live) carries the same
  359. incarnation evidence for gateways: the plan identifies a gateway by the
  360. profile it SERVES, while the restart bookkeeping names the SERVICE it runs under, and those two
  361. disagree whenever a service serves a profile its name does not encode — a root-home launchd
  362. label running the sticky active profile, a hash-suffixed systemd unit for a custom
  363. ``HERMES_HOME``. No name rule can bridge that, so a planned PID that is gone while a gateway
  364. answers for the same profile counts as ``restarted``. A missing successor, or the planned PID
  365. still answering, stays ``unaccounted`` — the tripwire keeps its teeth.
  366. That credit needs an unambiguous baseline: a profile with SEVERAL planned gateway runtimes cannot
  367. be attributed to one successor (the fleet probe publishes at most one row per profile), so one
  368. replacement cannot have replaced two planned processes. Such a profile keeps the name-path
  369. verdict instead of letting an untouched sibling vanish behind a successor that can only have
  370. replaced one of them. That resolves conservatively on purpose: pairing a successor to the planned
  371. process it replaced would need a start-time identity that neither the plan record nor the fleet
  372. row carries, and a tripwire firing on an anomalous plan is the intended direction.
  373. Credit is deliberately NOT gated on restart bookkeeping either: the failure this fallback exists
  374. for already had bookkeeping (``restarted_services`` named the service label — only the
  375. profile↔label name match failed), and a gateway launchd respawned by itself has no restart-phase
  376. record at all while the fleet row still proves its successor runs the new code.
  377. See #91277.
  378. They never borrow the gateway's outcome: ``relaunched_profiles`` and ``hermes-gateway*`` name a
  379. different process that shares the profile, nothing more. See #100479.
  380. ``external_gateway_pids``: gateway pids the post-restart fleet probe verified as serving a
  381. DIFFERENT checkout than this updater (a ``profiles/<name>`` symlinked to a separate install).
  382. An untouched gateway in that set is ``external`` — another install owns its restart — instead
  383. of ``unaccounted``. No evidence keeps the tripwire armed. See #120240.
  384. """
  385. outcomes: list[dict[str, Any]] = []
  386. try:
  387. failed_set = {str(u) for u in (failed_units or [])}
  388. restarted_set = {str(s) for s in (restarted_services or [])}
  389. relaunched = set(relaunched_profiles or []) | set(externally_supervised_profiles or [])
  390. killed = {int(p) for p in (killed_pids or set())}
  391. stale_serves = {int(p) for p in stale_serve_pids} if stale_serve_pids is not None else None
  392. failed_respawns = {int(p) for p in (failed_respawn_pids or set())}
  393. external = {p for p in (external_gateway_pids or ()) if isinstance(p, int)}
  394. successors = (
  395. {str(profile): {p for p in pids if isinstance(p, int)} for profile, pids in live_gateway_pids.items()}
  396. if live_gateway_pids is not None
  397. else None
  398. )
  399. def _bookkeeping_resolves(r: RuntimeRecord) -> bool:
  400. """Terminal verdict already available from the restart phase's bookkeeping."""
  401. if r.pid is not None and r.pid in killed:
  402. return True
  403. if r.profile in relaunched:
  404. return True
  405. return _gateway_named_in(r, failed_set) or _gateway_named_in(r, restarted_set)
  406. # Baseline identity: the fleet probe publishes at most one row per profile, so successor
  407. # evidence is only attributable when ONE planned gateway for that profile still needs a
  408. # verdict. Serve/dashboard rows are different processes, and rows bookkeeping already resolved
  409. # (killed / relaunched / name-matched) are not candidates either — a stopped orphan must not
  410. # make its surviving sibling's successor look ambiguous.
  411. pending_gateways: dict[str, int] = {}
  412. for _planned in plan.runtimes:
  413. if (isinstance(_planned, RuntimeRecord) and _planned.kind == "gateway"
  414. and not _bookkeeping_resolves(_planned)):
  415. pending_gateways[_planned.profile] = pending_gateways.get(_planned.profile, 0) + 1
  416. def _outcome(r: RuntimeRecord) -> str:
  417. killed_here = r.pid is not None and r.pid in killed
  418. if r.kind in _SERVE_KINDS:
  419. if killed_here:
  420. return "stopped"
  421. if r.pid in failed_respawns or any(_serve_unit_matches_profile(r.profile, u) for u in failed_set):
  422. return "failed"
  423. if stale_serves is not None and r.pid not in stale_serves:
  424. # Incarnation-verified: the pre-update process is gone (replaced by its unit / the
  425. # dashboard cleanup respawn / the Desktop app).
  426. return "restarted"
  427. if r.supervisor in CLIENT_OWNED_SERVE_SUPERVISORS:
  428. if stale_serves is not None:
  429. # Still alive on pre-update code, but a Desktop client owns it and the restart phase
  430. # must not kill it (_DESKTOP_SERVE_SKIP_REASON); only that client can pick up the new code.
  431. return "deferred"
  432. return "unaccounted"
  433. if stale_serves is not None:
  434. return "unaccounted"
  435. return "restarted" if any(_serve_unit_matches_profile(r.profile, s) for s in restarted_set) else "unaccounted"
  436. if r.profile in relaunched:
  437. return "restarted"
  438. if killed_here:
  439. return "stopped"
  440. if _gateway_named_in(r, failed_set):
  441. return "failed"
  442. if _gateway_named_in(r, restarted_set):
  443. return "restarted"
  444. if successors is not None and r.pid is not None:
  445. # Incarnation-verified last resort: the planned process is gone and a gateway answers
  446. # for the same profile now. Name-independent on purpose — the supervising service's
  447. # label/unit may encode the install root (or a hashed home) instead of the served
  448. # profile, which the profile-scoped name matcher above can never credit.
  449. live = successors.get(r.profile)
  450. ambiguous = pending_gateways.get(r.profile, 0) > 1
  451. if live and r.pid not in live and not ambiguous:
  452. return "restarted"
  453. if ambiguous:
  454. logger.debug(
  455. "%s planned gateway runtimes for profile %r still need a verdict — successor "
  456. "evidence cannot attribute the restart, leaving pid %s on the name path",
  457. pending_gateways[r.profile], r.profile, r.pid,
  458. )
  459. elif not live:
  460. # No row for this profile: the fallback has nothing to work with and reconciliation
  461. # stays on the service-name path. Logged because the tripwire below reads identically
  462. # whether the evidence was missing or the restart was actually missed.
  463. logger.debug(
  464. "No post-restart gateway evidence for profile %r (planned pid %s via %s) — "
  465. "reconciling on restart bookkeeping alone",
  466. r.profile, r.pid, r.restart_via,
  467. )
  468. return "external" if r.pid in external else "unaccounted"
  469. for r in plan.runtimes:
  470. if isinstance(r, RuntimeRecord):
  471. outcomes.append(
  472. {"kind": r.kind, "profile": r.profile, "pid": r.pid, "mechanism": r.restart_via, "outcome": _outcome(r),
  473. **{k: r.detail[k] for k in ("systemd_unit", "systemd_scope") if k in (r.detail or {})}}
  474. )
  475. except Exception as exc:
  476. logger.debug("Runtime-outcome reconciliation failed: %s", exc)
  477. return outcomes
  478. def report_unaccounted_runtimes(outcomes: list[dict[str, Any]]) -> bool:
  479. """Print a loud warning for runtimes the restart phase never touched.
  480. Returns True when at least one planned runtime is unaccounted; the caller escalates like a
  481. STALE/DOWN fleet row (an owed ``gateway_restart`` follow-up) — a promised restart silently missed is the class this phase
  482. exists to kill.
  483. """
  484. manual = [o for o in outcomes if o.get("outcome") == "deferred" and o.get("mechanism") == "respawn-argv"]
  485. if manual:
  486. print()
  487. print(" ⚠ Manual serve restarts deferred to their owner (reminders retained until the old processes exit):")
  488. for o in manual:
  489. print(f" • {o['kind']} [{o['profile']}] pid {o['pid']}: relaunch `hermes serve` / `hermes dashboard`, or reconnect Desktop for an SSH backend")
  490. deferred = [o for o in outcomes if o.get("outcome") == "deferred" and o.get("mechanism") != "respawn-argv"]
  491. if deferred:
  492. # Surfaced but not escalated: the updater has no authority over these, so holding
  493. # ``fleet_restart_pending`` for them would never be discharged. See #111494.
  494. print()
  495. print(" ℹ Left to the Desktop app (still on pre-update code until it is relaunched):")
  496. for o in deferred:
  497. action = ("owned by a Desktop connected over SSH; it picks up the update when that Desktop reconnects"
  498. if o.get("mechanism") == "desktop-ssh" else "relaunch the Desktop app to pick up the update")
  499. print(f" • {o['kind']} [{o['profile']}] pid {o['pid']} — {action}")
  500. external = [o for o in outcomes if o.get("outcome") == "external"]
  501. if external:
  502. # Surfaced but not escalated: another install's updater owns these restarts. See #120240.
  503. print()
  504. print(" ℹ Left to their own install (separate checkout, not restarted by this update):")
  505. for o in external:
  506. print(f" • {o['kind']} [{o['profile']}] pid {o['pid']}")
  507. missed = [o for o in outcomes if o.get("outcome") == "unaccounted"]
  508. if not missed:
  509. return False
  510. print()
  511. print(" ⚠ Planned runtimes the restart phase never touched:")
  512. for o in missed:
  513. print(f" ✗ {o['kind']} [{o['profile']}] pid {o['pid']} — planned mechanism: {o['mechanism']}")
  514. print(" Restart them manually, then verify:")
  515. if any(o.get("kind") not in _SERVE_KINDS for o in missed):
  516. print(" hermes gateway restart # active profile")
  517. print(" hermes -p <profile> gateway restart # named profile")
  518. if any(o.get("kind") in _SERVE_KINDS for o in missed):
  519. # A serve/dashboard is not reachable by any `gateway restart` command: name the process, not the wrong verb.
  520. # See #100479.
  521. from hermes_cli.main_dashboard import _systemd_restart_commands
  522. for o in missed:
  523. if o.get("kind") in _SERVE_KINDS and o.get("systemd_unit"):
  524. # The owning unit and scope are known: name its exact command (#141021).
  525. for command in _systemd_restart_commands(o["systemd_unit"], o.get("systemd_scope")):
  526. print(f" {command} # pid {o['pid']}")
  527. if any(o.get("kind") in _SERVE_KINDS and not o.get("systemd_unit") for o in missed):
  528. if sys.platform == "linux":
  529. print(" systemctl --user restart hermes-serve.service # unit-managed serve")
  530. print(" relaunch `hermes serve` / `hermes dashboard`")
  531. return True
  532. def record_plan_in_receipt(plan: UpdatePlan) -> None:
  533. """Attach the inventory to the active update receipt. Never raises."""
  534. try:
  535. import hermes_cli.update_receipt as ur
  536. current = ur._current.get()
  537. if current is not None:
  538. current.data["plan"] = plan.to_dict()
  539. except Exception as exc:
  540. logger.debug("Could not record plan in receipt: %s", exc)