update_cmd_fleet.py 93 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842
  1. """Gateway fleet restart + post-update verification for ``hermes update``.
  2. Split out of ``hermes_cli/update_cmd.py``; every name is re-imported there so
  3. ``hermes_cli.update_cmd.<name>`` keeps resolving/monkeypatching. Origin helpers are
  4. imported lazily inside each function (no import cycle; test patches stay effective).
  5. """
  6. import json
  7. import logging
  8. import re
  9. from contextlib import suppress
  10. import os
  11. import subprocess
  12. import sys
  13. import time as _time
  14. from dataclasses import dataclass, field
  15. from pathlib import Path
  16. from hermes_cli.update_cmd_common import _best_effort
  17. from hermes_cli.update_host_obligation import PROFILE_MARKER_NAME
  18. from hermes_cli.update_inventory import _gateway_service_matches_profile
  19. # Log-record parity with the origin module.
  20. logger = logging.getLogger("hermes_cli.update_cmd")
  21. # Under HERMES_HOME (not next to the venv): records the fleet-restart obligation
  22. # after a pull advanced HEAD; cleared only when the restart completes or nothing ran.
  23. # The existing ``.update-incomplete`` / ``.lazy-refresh-incomplete`` markers gate dependency/venv repair;
  24. # this one is the fleet-restart obligation after a git pull that advanced HEAD (#95294).
  25. _FLEET_RESTART_PENDING_NAME = PROFILE_MARKER_NAME
  26. _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"})
  27. _SYSTEMD_SCOPES = (("user", ["systemctl", "--user"]), ("system", ["systemctl"]))
  28. _LIST_GATEWAY_UNITS = [
  29. "list-units", "hermes-gateway*", "hermes-serve*", "hermes-dashboard*",
  30. "--plain", "--no-legend", "--no-pager",
  31. ]
  32. def _write_gateway_update_exit_code(ok: bool) -> None:
  33. from hermes_cli.update_cmd import get_hermes_home
  34. from hermes_cli.update_receipt import committed_success
  35. from pm.filesystem import durable_write_bytes
  36. # A run whose receipt already closed as a success is never reported failed (C3, review
  37. # regression 3): the command boundary's catch-alls call this with False.
  38. ok = ok or committed_success()
  39. path = get_hermes_home() / ".update_exit_code"
  40. with suppress(OSError):
  41. # Atomic: the gateway watcher polls this file and reads an empty one as exit 1.
  42. durable_write_bytes(path, b"0" if ok else b"1")
  43. def _fleet_restart_pending_marker_path() -> Path:
  44. """LEGACY per-``HERMES_HOME`` breadcrumb. Read-compat only — nothing writes it any more.
  45. One host runs one multiplexing gateway, so the pull→restart obligation is host-scoped
  46. (``hermes_cli/update_host_obligation.py``). An obligation armed by the old per-profile code
  47. is still read and cleared here so an in-flight update is discharged after the upgrade.
  48. """
  49. from hermes_cli.update_cmd import get_hermes_home
  50. return get_hermes_home() / _FLEET_RESTART_PENDING_NAME
  51. def _write_fleet_restart_pending_marker(*, expected_sha: str = "", runtimes: list[dict] | None = None, owner: str = "") -> bool:
  52. """Arm the HOST pull→restart obligation. Never raises; False when no store every profile reads took it.
  53. An unwritable host state dir falls back to this profile's per-home marker, which settles the
  54. debt only on an install proven to have no other profile; otherwise the arm returns False and
  55. the commit point refuses to move (``update_host_obligation.arm_host_obligation``, review S2).
  56. """
  57. if runtimes == []:
  58. # An explicit empty inventory owes no restart (e.g. Desktop-hosted `serve` with no
  59. # gateway services). Arming the marker here leaves a breadcrumb nothing can discharge:
  60. # a no-gateway host would then fail every later ``hermes update`` (#115311).
  61. return True
  62. from hermes_cli.update_cmd import _m
  63. from hermes_cli.update_host_obligation import arm_host_obligation
  64. if _m()._pytest_owns_live_checkout(_fleet_restart_pending_marker_path().parent):
  65. logger.debug("Skipping fleet-restart-pending obligation under pytest (live checkout)")
  66. return True
  67. return arm_host_obligation(_fleet_restart_pending_marker_path(), expected_sha=expected_sha, runtimes=runtimes,
  68. profile=_current_profile_name(), owner=owner)
  69. def _current_profile_name() -> str:
  70. """Profile whose CLI armed the obligation (diagnostics only — the record is host-scoped)."""
  71. try:
  72. from hermes_cli.profiles import get_active_profile_name
  73. return get_active_profile_name() or "default"
  74. except Exception:
  75. return ""
  76. def _clear_fleet_restart_pending_marker() -> None:
  77. """Discharge the obligation for the whole host (legacy per-home marker included). Never raises."""
  78. from hermes_cli.update_cmd import _m
  79. from hermes_cli.update_host_obligation import clear_host_obligation
  80. clear_host_obligation()
  81. _m()._clear_marker_file(_fleet_restart_pending_marker_path(), label="fleet-restart-pending")
  82. def _fleet_restart_obligation_armed() -> bool:
  83. """True when this HOST owes a fleet restart — from any profile's CLI."""
  84. from hermes_cli.update_host_obligation import host_obligation_present
  85. if host_obligation_present():
  86. return True
  87. with suppress(OSError):
  88. return _fleet_restart_pending_marker_path().is_file()
  89. return False
  90. def _obligation_fields() -> dict[str, str] | None:
  91. """Armed obligation as ``key=value`` fields: HOST record first, then the legacy marker.
  92. ``None`` means nothing armed OR a malformed record; both must leave the obligation standing.
  93. """
  94. from hermes_cli.update_host_obligation import host_obligation_present, obligation_fields
  95. fields = obligation_fields()
  96. if fields is not None:
  97. return fields
  98. if host_obligation_present():
  99. # The record exists but its terms are unknown (corrupt, or a NEWER CLI's version). An
  100. # unrelated legacy marker's inventory cannot discharge terms nobody can read: fail closed.
  101. return None
  102. try:
  103. text = _fleet_restart_pending_marker_path().read_text(encoding="utf-8-sig")
  104. except (OSError, UnicodeError):
  105. return None
  106. legacy: dict[str, str] = {}
  107. for line in text.splitlines():
  108. key, sep, value = line.partition("=")
  109. if not sep or key in legacy:
  110. return None
  111. legacy[key] = value
  112. return legacy
  113. def _current_checkout_sha() -> str | None:
  114. """Current on-disk checkout HEAD, or None if it cannot be resolved."""
  115. from hermes_cli.update_cmd import _capture_head_sha, _m
  116. try:
  117. from hermes_cli.version_info import read_checkout_sha
  118. return read_checkout_sha()
  119. except Exception:
  120. return _capture_head_sha(["git"], _m().PROJECT_ROOT)
  121. def _receipt_looks_unfinished(receipt: dict) -> bool:
  122. """True when *receipt* is from an update that did not finish cleanly.
  123. The command boundary stamps a ``stop_reason`` on every receipt, including clean
  124. ones (``completed at command boundary``, ``sys.exit(0)``); it must not make a
  125. successful receipt look unfinished, or the next ``hermes update`` retriggers
  126. ``fleet_restart_pending`` from pre-pull plan SHAs (#98022).
  127. """
  128. exit_code = receipt.get("exit_code")
  129. outcome = receipt.get("outcome")
  130. if exit_code not in (0, None) or outcome in ("failed", "partial", "running"):
  131. return True
  132. gateway_restart = receipt.get("gateway_restart")
  133. if isinstance(gateway_restart, dict) and gateway_restart.get("incomplete"):
  134. return True
  135. # A stop_reason alone (update_contract refusals: outcome="refused", no exit_code)
  136. # counts only when nothing else vouched for success.
  137. succeeded = exit_code == 0 or outcome == "success"
  138. return bool(receipt.get("stop_reason")) and not succeeded
  139. def _receipt_reports_stale_runtime(receipt: dict, expected_sha: str | None = None) -> bool:
  140. """True when ``update_receipts/latest.json`` records a runtime SHA skew.
  141. Prefer the post-restart ``fleet`` matrix. ``plan.runtimes[].code_sha`` is captured
  142. *before* the pull, so a finished update's plan always looks stale and must not
  143. retrigger a restart; consult it only for an unfinished receipt.
  144. See #95294.
  145. """
  146. from hermes_cli.update_cmd import _current_checkout_sha
  147. if not isinstance(receipt, dict):
  148. return False
  149. expected_sha = expected_sha or _current_checkout_sha()
  150. if not expected_sha:
  151. return False
  152. def _sha_mismatch(code_sha) -> bool:
  153. return bool(code_sha) and str(code_sha) != str(expected_sha)
  154. from hermes_cli.update_receipt import row_is_external
  155. fleet = receipt.get("fleet")
  156. if isinstance(fleet, list) and fleet:
  157. return any(
  158. isinstance(entry, dict)
  159. and not row_is_external(entry)
  160. and (entry.get("state") == "stale" or _sha_mismatch(entry.get("code_sha")))
  161. for entry in fleet
  162. )
  163. if not _receipt_looks_unfinished(receipt):
  164. return False
  165. plan = receipt.get("plan")
  166. if not isinstance(plan, dict):
  167. return False
  168. return any(
  169. isinstance(runtime, dict) and _sha_mismatch(runtime.get("code_sha"))
  170. for runtime in plan.get("runtimes") or []
  171. )
  172. _SUPERVISED_SERVE_BACKENDS = frozenset(
  173. {"manual-serve", "desktop", "desktop-ssh", "systemd", "launchd", "windows-service", "service"}
  174. )
  175. # Backends whose supervisor restarts the process without any updater bookkeeping. ``manual-serve``
  176. # is excluded: it owes a durable handoff (``defer_manual_serve``) before it stops counting.
  177. # The inventory writer classifies a serve/dashboard row as launchd, desktop, desktop-ssh, systemd (a unit's
  178. # MainPID, #141021) or manual-serve (``update_inventory._collect_ledger_runtimes``); ``windows-service`` and
  179. # ``service`` mirror ``_SUPERVISED_SERVE_BACKENDS`` for parity only — they are set for gateway rows alone.
  180. _SUPERVISOR_OWNED_SERVE_BACKENDS = _SUPERVISED_SERVE_BACKENDS - {"manual-serve"}
  181. def _receipt_owed_gateways(receipt: dict, pending_manual: list[dict]) -> set[tuple[str, str]] | None:
  182. """Pure coverage classification after manual retention of this receipt snapshot.
  183. Empty means this receipt owes no gateways, not that an independent marker owes none. Unknown identities, unclassified serve backends and failed manual transfers make coverage unverified.
  184. """
  185. plan = receipt.get("plan") or {}
  186. entries: list[tuple[object, str | None]] = [(entry, None) for entry in plan.get("runtimes") or []]
  187. entries.extend((entry, None) for entry in receipt.get("pending_manual_serves") or [])
  188. entries.extend((entry, None) for entry in receipt.get("carried_manual_serves") or [])
  189. entries.extend((entry, "gateway") for entry in receipt.get("fleet") or [])
  190. owed: set[tuple[str, str]] = set()
  191. unverified = False
  192. for entry, default_kind in entries:
  193. if not isinstance(entry, dict):
  194. unverified = True
  195. continue
  196. kind = entry.get("kind", default_kind)
  197. profile = entry.get("profile")
  198. # A serve/dashboard row is outside the gateway matrix's evidence, not evidence against the
  199. # gateways it does cover: a supervised backend (desktop, systemd, launchd) is its
  200. # supervisor's to restart, and a manual-serve row outside the retention list is the serve
  201. # obligation mechanism's — a host running a dashboard carries such a row in every receipt,
  202. # and a blanket veto made the gateway warning permanently undischargeable there (#115090).
  203. # Only an unclassified backend or a failed manual transfer still makes coverage unverified.
  204. if kind in ("serve", "dashboard") and entry.get("supervisor") in _SUPERVISED_SERVE_BACKENDS and entry not in pending_manual:
  205. continue
  206. if kind != "gateway" or not profile or profile == "unknown":
  207. unverified = True
  208. continue
  209. owed.add((kind, profile))
  210. return None if unverified else owed
  211. def _fleet_covered_gateways(fleet: list) -> set[tuple[str, str]] | None:
  212. """``(kind, profile)`` identities the live rows vouch for; ``None`` when any row is unidentified.
  213. A multiplexer's row carries the ``served_profiles`` its runtime status records (``_fleet_row``
  214. keeps the field only when well-formed), so one live process covers every profile it serves.
  215. """
  216. covered: set[tuple[str, str]] = set()
  217. for row in fleet:
  218. profile = row.get("profile") if isinstance(row, dict) else None
  219. if not profile or profile == "unknown":
  220. return None # unidentified runtime: the matrix cannot vouch for it
  221. covered.add(("gateway", profile))
  222. covered.update(("gateway", served) for served in row.get("served_profiles") or [])
  223. return covered
  224. def _live_fleet_covers_receipt(expected_sha: str | None, receipt: dict, owed: set[tuple[str, str]] | None, *, accept_states: tuple = ("current",)) -> bool:
  225. """Require a successor at the expected SHA for every owed gateway identity."""
  226. if not expected_sha:
  227. return False
  228. from hermes_cli.update_receipt import collect_fleet_versions, row_is_external
  229. try:
  230. if owed is None:
  231. return False
  232. if not owed:
  233. return bool((receipt.get("plan") or {}).get("runtimes"))
  234. fleet = collect_fleet_versions()
  235. # State labels are checkout-relative; completed restarts may accept stale rows at the pulled SHA.
  236. if not fleet or any(
  237. row.get("state") not in accept_states or row.get("code_sha") != expected_sha
  238. for row in fleet if not row_is_external(row)
  239. ):
  240. return False
  241. covered = _fleet_covered_gateways(fleet)
  242. return covered is not None and owed <= covered
  243. except Exception as exc:
  244. logger.debug("Could not reconcile pending fleet identities: %s", exc)
  245. return False
  246. # A gateway the restart phase stopped without knowing its profile: no live row can be matched to it,
  247. # so only a live, current gateway of this checkout settles it, never an empty host (""=no profile).
  248. _UNMAPPED_GATEWAY = ("gateway", "")
  249. def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None:
  250. """The ``("gateway", profile)`` set a marker's inventory owes; None when it recorded none.
  251. Raises ValueError for a malformed or unsupported inventory, which keeps the marker.
  252. """
  253. from hermes_cli.update_cmd_fleet_gatewayless import runtime_outside_gateway_evidence
  254. if inventory is None:
  255. return None
  256. if not isinstance(inventory, dict) or inventory.get("version") != 1:
  257. raise ValueError("unsupported fleet-restart inventory")
  258. runtimes = inventory.get("runtimes")
  259. if not isinstance(runtimes, list):
  260. raise ValueError("fleet-restart inventory has no runtime list")
  261. owed: set[tuple[str, str]] = set()
  262. for runtime in runtimes:
  263. if not isinstance(runtime, dict):
  264. raise ValueError("fleet-restart inventory row is not an object")
  265. if runtime_outside_gateway_evidence(runtime):
  266. continue
  267. if runtime.get("kind") == "gateway" and runtime.get("stopped_unmapped"):
  268. owed.add(_UNMAPPED_GATEWAY)
  269. continue
  270. profile = runtime.get("profile")
  271. if runtime.get("kind") != "gateway" or not isinstance(profile, str) or not profile.strip() or profile == "unknown":
  272. raise ValueError("fleet-restart inventory row is not an identified gateway")
  273. owed.add(("gateway", profile))
  274. return owed
  275. def _discharge_gatewayless_marker(checkout_sha: str, expected_sha: str) -> bool:
  276. """Settle an inventory-less marker on a host with no live gateway (#118742).
  277. Only when the host itself shows nothing the update could still owe a restart to, and HEAD
  278. still holds the code it pulled.
  279. """
  280. from hermes_cli.update_cmd_fleet_checkout import checkout_contains
  281. from hermes_cli.update_cmd_fleet_gatewayless import host_owes_no_gateway_restart
  282. try:
  283. gatewayless = (checkout_sha == expected_sha or checkout_contains(expected_sha)) and host_owes_no_gateway_restart()
  284. except Exception as exc:
  285. logger.debug("Gateway-less host probe failed; keeping fleet-restart-pending marker: %s", exc)
  286. return False
  287. if not gatewayless:
  288. return False
  289. _clear_fleet_restart_pending_marker()
  290. logger.debug("Fleet-restart-pending marker discharged: host runs no gateway at %s", checkout_sha[:10])
  291. return True
  292. def _marker_only_restart_obsolete() -> bool:
  293. """Settle only the inventory stored with this marker's target SHA.
  294. Historical receipts cannot narrow this obligation. Malformed or unsupported inventories stay
  295. fail-closed; empty discovery never proves a stopped gateway recovered. Two shapes record no
  296. obligation and settle without one: an explicit empty inventory (a pull that found no gateway,
  297. #115311) clears outright, and an inventory-less marker (the pre-inventory writer, or a tail
  298. that died before its inventory was recorded, #115638) clears once every live gateway is
  299. current on the checkout — there is no recorded owed set, so the fleet running the code on disk
  300. is the whole of the evidence the marker's warning can be about, even after HEAD moved past
  301. ``expected_sha`` by an out-of-band pull — and so does an inventory-less record armed with no
  302. SHA at all (a no-op update whose head capture failed, #125952): with no owed set and no SHA,
  303. the checkout is the only code it can be held to. With no live gateway at all, the inventory-less marker
  304. asks the host instead (``update_cmd_fleet_gatewayless``): it clears when no profile left a
  305. gateway that should be running and every live runtime is supervisor-owned or handed off, so a
  306. Desktop-only install stops failing every later update (#118742).
  307. A serve/dashboard row whose supervisor owns the restart (Desktop backend, systemd/launchd
  308. unit, Windows service) is outside the gateway matrix's evidence, not evidence against it —
  309. the same boundary ``_receipt_owed_gateways`` draws for receipts (#115090) and the restart
  310. phase draws for the Desktop backend (#111494). Counting it made the warning permanently
  311. undischargeable on every host that runs a dashboard. A manual-serve row still needs its
  312. durable handoff (``defer_manual_serve``), and an unclassified backend stays fail-closed.
  313. Discharging here strands nobody: the same row is still accounted at update time by
  314. ``update_inventory.report_unaccounted_runtimes``, which prints it and records an owed restart when the restart
  315. phase never touched it — this marker only stops re-warning about it on every later startup.
  316. """
  317. from hermes_cli.update_cmd_fleet_checkout import checkout_contains
  318. try:
  319. fields = _obligation_fields()
  320. if fields is None:
  321. return False
  322. expected_sha = fields.get("expected_sha", "").strip()
  323. owed = _marker_owed_gateways(json.loads(fields.get("inventory", "null")))
  324. except (OSError, UnicodeError, ValueError):
  325. return False
  326. if owed is not None and not owed:
  327. # A pull that recorded no gateway runtime owes no restart; clearing avoids the
  328. # stuck "Fleet restart incomplete" loop on Desktop-hosted (no-service) installs.
  329. _clear_fleet_restart_pending_marker()
  330. logger.debug("Fleet-restart-pending marker discharged: no gateway obligation recorded")
  331. return True
  332. if owed is not None and not expected_sha:
  333. return False # an inventoried obligation without its SHA can never be proven
  334. checkout_sha = _current_checkout_sha()
  335. if owed is not None and checkout_sha != expected_sha and not checkout_contains(expected_sha):
  336. return False # a newer pull moved HEAD; it owns a fresh obligation
  337. # HEAD may sit past ``expected_sha`` by a carried local commit (a cherry-picked hotfix) that no
  338. # pull made and no fresh obligation covers; the fleet is held to the code it actually runs, which
  339. # is what an equality gate on ``expected_sha`` could never discharge (#119367).
  340. target_sha = checkout_sha
  341. if not target_sha:
  342. return False
  343. try:
  344. from hermes_cli.update_receipt import collect_fleet_versions, row_is_external
  345. fleet = collect_fleet_versions()
  346. except Exception as exc:
  347. logger.debug("Fleet probe failed; keeping fleet-restart-pending marker: %s", exc)
  348. return False
  349. if not fleet or (not expected_sha and all(row_is_external(row) for row in fleet)):
  350. if owed is not None or not expected_sha:
  351. # Absence cannot prove recovery of the recorded inventory / unnamed code; a fleet
  352. # whose every row serves ANOTHER checkout root is absence too, not evidence.
  353. return False
  354. return _discharge_gatewayless_marker(checkout_sha, expected_sha)
  355. covered = _fleet_covered_gateways(fleet)
  356. if covered is None:
  357. return False # unidentified runtime: the matrix cannot vouch for it
  358. for row in fleet:
  359. if row_is_external(row):
  360. continue
  361. if row.get("state") != "current" or str(row.get("code_sha")) != target_sha:
  362. return False # stale / down / unknown-identity row still owes the restart
  363. if owed is not None and not owed - {_UNMAPPED_GATEWAY} <= covered:
  364. return False # A gateway this marker owns is absent (down) or unidentifiable.
  365. if owed and _UNMAPPED_GATEWAY in owed and not (_fleet_covered_gateways(
  366. [row for row in fleet if not row_is_external(row)]) or set()) - owed:
  367. # An unmapped stopped gateway has no identity to match, so only a local gateway that no
  368. # mapped owed one explains can be its successor; another checkout's gateway never is.
  369. return False
  370. _clear_fleet_restart_pending_marker()
  371. logger.debug(
  372. "Fleet-restart-pending marker discharged: %d gateway(s) already serve %s",
  373. len(fleet), target_sha[:10],
  374. )
  375. return True
  376. def _receipt_restart_phase_completed(receipt: dict) -> str | None:
  377. """Return the pulled SHA when the restart phase completed, even if a later step failed."""
  378. gateway_restart = receipt.get("gateway_restart")
  379. if not isinstance(gateway_restart, dict) or not gateway_restart:
  380. return None
  381. if gateway_restart.get("incomplete") or gateway_restart.get("phase_error"):
  382. return None
  383. post_sha = (receipt.get("post_update") or {}).get("sha")
  384. return str(post_sha) if post_sha else None
  385. def _pending_fleet_restart_needed(*, receipt: dict | None = None, pending_manual: list[dict] | None = None) -> bool:
  386. """Require identity-matched gateways at checkout HEAD for update catch-up."""
  387. from hermes_cli.update_cmd import _current_checkout_sha
  388. from hermes_cli.update_receipt import read_latest_receipt
  389. from hermes_cli.update_serve_obligations import retain_receipt_manual_serves
  390. if receipt is None:
  391. receipt = read_latest_receipt() or {}
  392. if pending_manual is None:
  393. pending_manual = retain_receipt_manual_serves(receipt)
  394. # The HOST obligation owns its inventory; latest.json can belong to an older update.
  395. if _fleet_restart_obligation_armed():
  396. return not _marker_only_restart_obsolete()
  397. owed = _receipt_owed_gateways(receipt, pending_manual)
  398. if not _receipt_reports_stale_runtime(receipt):
  399. return False
  400. return not _live_fleet_covers_receipt(_current_checkout_sha(), receipt, owed)
  401. def _update_owes_fleet_restart(*, receipt: dict | None = None, pending_manual: list[dict] | None = None) -> bool:
  402. """Hold a completed restart to the code it pulled, not a later checkout HEAD."""
  403. from hermes_cli.update_cmd import _current_checkout_sha
  404. from hermes_cli.update_receipt import read_latest_receipt
  405. from hermes_cli.update_serve_obligations import retain_receipt_manual_serves
  406. if receipt is None:
  407. receipt = read_latest_receipt() or {}
  408. if pending_manual is None:
  409. pending_manual = retain_receipt_manual_serves(receipt)
  410. # A completed older receipt cannot discharge an independent host obligation's inventory.
  411. if _fleet_restart_obligation_armed():
  412. return not _marker_only_restart_obsolete()
  413. owed = _receipt_owed_gateways(receipt, pending_manual)
  414. if not _receipt_reports_stale_runtime(receipt):
  415. return False
  416. restarted_to = _receipt_restart_phase_completed(receipt)
  417. # A fleet an operator has since restarted onto a moved checkout (``hermes gateway restart`` —
  418. # the remedy this warning names) has nothing of the update left to owe either.
  419. if restarted_to and _live_fleet_covers_receipt(restarted_to, receipt, owed, accept_states=("current", "stale")):
  420. return False
  421. return not _live_fleet_covers_receipt(_current_checkout_sha(), receipt, owed)
  422. def _warn_pending_fleet_restart(*, startup: bool = False) -> None:
  423. """Print the specific interrupted-update fleet-restart warning."""
  424. stream = sys.stderr if startup else sys.stdout
  425. print("⚠ A previous `hermes update` pulled new code but did not restart running gateways.", file=stream)
  426. print(" Gateways may still be serving pre-update modules (mixed sys.modules).", file=stream)
  427. if startup:
  428. print(" Run `hermes update` or `hermes gateway restart`.", file=stream)
  429. def _warn_pending_fleet_restart_on_startup() -> None:
  430. """Cheap CLI-startup hint. Never restarts; never raises."""
  431. from hermes_cli.update_receipt import read_latest_receipt
  432. from hermes_cli.update_serve_obligations import retain_receipt_manual_serves, warn_pending_manual_serves
  433. receipt = read_latest_receipt() or {}
  434. pending_manual = None
  435. with suppress(Exception):
  436. pending_manual = retain_receipt_manual_serves(receipt)
  437. with suppress(Exception):
  438. if _update_owes_fleet_restart(receipt=receipt, pending_manual=pending_manual):
  439. _warn_pending_fleet_restart(startup=True)
  440. with suppress(Exception):
  441. warn_pending_manual_serves(startup=True, pending_manual=pending_manual)
  442. def _systemd_gateway_unit_listings(on_list_timeout=None):
  443. """Yield ``(scope, scope_cmd, list-units CompletedProcess)`` per systemd scope that answered.
  444. A missing systemctl skips the scope silently; a listing timeout skips it after
  445. ``on_list_timeout(scope, exc)`` (when given) so the other scope is still processed.
  446. """
  447. for scope, scope_cmd in _SYSTEMD_SCOPES:
  448. try:
  449. result = _systemctl(scope_cmd + _LIST_GATEWAY_UNITS, timeout=10)
  450. except FileNotFoundError:
  451. continue
  452. except subprocess.TimeoutExpired as exc:
  453. if on_list_timeout is not None:
  454. on_list_timeout(scope, exc)
  455. continue
  456. yield scope, scope_cmd, result
  457. def _needs_sudo(scope: str) -> bool:
  458. return (
  459. scope == "system"
  460. and hasattr(os, "geteuid")
  461. and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only
  462. )
  463. def _unit_main_pid(scope_cmd: list, svc_name: str) -> int:
  464. """Live ``MainPID`` of a unit; ``0`` when inactive, unprivileged or unreadable.
  465. Property reads need no manage-units privileges, and an unreadable PID is never collapsed:
  466. identity that cannot be proved keeps its own restart.
  467. """
  468. try:
  469. result = _systemctl(list(scope_cmd) + ["show", svc_name, "--property=MainPID", "--value"], timeout=10)
  470. except (OSError, subprocess.TimeoutExpired):
  471. return 0
  472. if getattr(result, "returncode", 1) != 0:
  473. return 0
  474. try:
  475. return int((getattr(result, "stdout", "") or "").strip() or 0)
  476. except ValueError:
  477. return 0
  478. def _restart_systemd_gateway_units_best_effort(failed: list, listings) -> None:
  479. """Restart every hermes-gateway/serve unit ONCE PER LIVE HOST PROCESS.
  480. One host runs one multiplexing gateway, so leftover per-profile units
  481. (``hermes-gateway-<profile>.service``) all point at the SAME live ``MainPID``; restarting
  482. each in turn restarts the host gateway N times — a self-inflicted N-fold outage triggered
  483. by one update. Units that share a live main PID are collapsed to one representative and the
  484. others are named as LEGACY units to migrate, never silently dropped.
  485. """
  486. from hermes_cli.update_host_obligation import collapse_units_to_host_processes
  487. answered = set()
  488. targets: dict[str, tuple[str, list, str]] = {} # "<scope>/<unit>" -> (scope, scope_cmd, unit)
  489. for scope, scope_cmd, result in listings:
  490. answered.add(scope)
  491. if result.returncode != 0:
  492. failed.append(f"systemd-{scope} (listing failed)")
  493. continue
  494. _for_each_systemd_gateway_unit(
  495. result.stdout,
  496. process_unit=lambda svc_name, _scope=scope, _cmd=scope_cmd: targets.setdefault(
  497. f"{_scope}/{svc_name}", (_scope, _cmd, svc_name)),
  498. on_unit_timeout=lambda svc_name, exc: failed.append(svc_name),
  499. )
  500. keys = list(targets)
  501. covered: dict[str, str] = {}
  502. if len(keys) > 1:
  503. # Only worth a `systemctl show` round when several units could be one process.
  504. keys, covered = collapse_units_to_host_processes(
  505. keys, lambda key: _unit_main_pid(targets[key][1], targets[key][2]))
  506. for unit_key, owner_key in covered.items():
  507. print(
  508. f" • {unit_key} is a legacy per-profile unit sharing one host gateway process with "
  509. f"{owner_key}; restarting it again would restart that process twice. Fold the units "
  510. "together with: hermes gateway migrate"
  511. )
  512. for key in keys:
  513. scope, scope_cmd, svc_name = targets[key]
  514. if not _systemd_unit_owned_by_update(scope_cmd, svc_name):
  515. continue
  516. manage_cmd = list(scope_cmd) + ["--no-ask-password"]
  517. if _needs_sudo(scope):
  518. manage_cmd = ["sudo", "-n"] + manage_cmd
  519. try:
  520. result = _systemctl_reset_and_restart(manage_cmd, svc_name, scope_cmd=scope_cmd)
  521. if result.returncode != 0 or not _wait_for_service_active(scope_cmd, svc_name):
  522. failed.append(svc_name)
  523. except subprocess.TimeoutExpired:
  524. failed.append(svc_name)
  525. # A timeout or missing executable is not an empty scope.
  526. failed.extend(f"systemd-{scope} (listing unavailable)" for scope, _ in _SYSTEMD_SCOPES if scope not in answered)
  527. def _live_fleet_current_rows() -> list[dict] | None:
  528. """The fleet matrix when the probe finds at least one gateway and every row is ``current``
  529. at the checkout SHA (identity known); ``None`` on any unknown/stale/down row or a failed
  530. probe (restart)."""
  531. checkout_sha = _current_checkout_sha()
  532. if not checkout_sha:
  533. return None
  534. try:
  535. from hermes_cli.update_receipt import collect_fleet_versions
  536. fleet = collect_fleet_versions()
  537. except Exception as exc:
  538. logger.debug("Pending fleet restart: fleet probe failed: %s", exc)
  539. return None
  540. if not fleet or _fleet_covered_gateways(fleet) is None:
  541. return None
  542. if all(row.get("state") == "current" and str(row.get("code_sha")) == checkout_sha for row in fleet):
  543. return fleet
  544. return None
  545. def _restart_identity_sha() -> str:
  546. """The SHA a completed host restart is stamped with; ``""`` when nothing names the code.
  547. ``_current_checkout_sha()`` is ``None`` on every non-git install (zip, pip, Docker), and an
  548. empty stamp can never match, so the per-host restart-once guard would be inert exactly on the
  549. installs it exists for: each profile's ``hermes update`` would re-kill the one shared
  550. multiplexer. The obligation's own ``expected_sha`` — else the receipt's post-update identity —
  551. names the same pulled code.
  552. """
  553. sha = _current_checkout_sha()
  554. if sha:
  555. return str(sha)
  556. sha = ((_obligation_fields() or {}).get("expected_sha") or "").strip()
  557. if sha:
  558. return sha
  559. with suppress(Exception):
  560. from hermes_cli.update_receipt import read_latest_receipt
  561. post_update = (read_latest_receipt() or {}).get("post_update")
  562. if isinstance(post_update, dict):
  563. return str(post_update.get("sha") or "")
  564. return ""
  565. def _fleet_restart_skip_reason(plan) -> str | None:
  566. """Why the completion tail may leave the fleet alone, or ``None`` when a restart is owed.
  567. Every route (pulled, already-current, ZIP) now finishes through the same completion
  568. tail, so the guards the old catch-up path carried live here: one host runs ONE
  569. multiplexing gateway, so a second profile's ``hermes update`` attaches to the restart the
  570. first one already stamped (#95294), and a fleet already serving the checkout code (a
  571. no-op update, a manual ``hermes gateway restart`` seconds ago) is not re-killed (#117051).
  572. The second guard needs BOTH the pre-update plan and the live probe: the live matrix only
  573. lists gateways, so a planned ``serve`` still on pre-update code (or any runtime without a
  574. stamped identity) keeps the restart — the reconciliation there is what surfaces it.
  575. """
  576. from hermes_cli.update_host_obligation import host_restart_already_completed
  577. checkout_sha = _restart_identity_sha()
  578. if host_restart_already_completed(checkout_sha):
  579. return "this host's gateway was already restarted for this update"
  580. if (checkout_sha and plan is not None and plan.runtimes
  581. and all(str(runtime.code_sha) == checkout_sha for runtime in plan.runtimes)
  582. and _live_fleet_current_rows() is not None):
  583. return "every running gateway already serves the checkout code"
  584. return None
  585. def _run_pending_fleet_restart() -> bool:
  586. """Historical retry hook; new retries use the ordinary completion owner."""
  587. from hermes_cli._old_updater import stop_for_relaunch
  588. stop_for_relaunch(incomplete=True)
  589. def _systemctl(cmd: list, *, timeout: float):
  590. """Run a systemctl (or sudo systemctl) invocation, capturing utf-8 text with a timeout."""
  591. return subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False)
  592. def _systemctl_reset_and_restart(manage_cmd: list, svc_name: str, *, scope_cmd: list | None = None):
  593. """``reset-failed`` then ``restart``: a unit parked in failed state by systemd's own
  594. auto-restart can wedge a plain ``restart`` against RestartSec backoff and stay dead."""
  595. # Property reads need no manage-units privileges: narrow sudoers may permit
  596. # restart/reset-failed but deny show. Keep the same user/system manager scope.
  597. from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job, systemd_restart_timeout
  598. scope_cmd = scope_cmd if scope_cmd is not None else manage_cmd
  599. timeout = systemd_restart_timeout(scope_cmd, svc_name)
  600. _systemctl(manage_cmd + ["reset-failed", svc_name], timeout=10)
  601. return run_unit_job(manage_cmd, scope_cmd, "restart", svc_name, timeout=timeout)
  602. def _systemd_unit_owned_by_update(scope_cmd: list, svc_name: str) -> bool:
  603. """Gate a unit restart on the unit's home being one this update owns (#93349).
  604. ``hermes-gateway*`` is an account-wide namespace: a second install's ``hermes update`` used
  605. to drain and restart the account's real ``hermes-gateway.service`` because the unit was
  606. listed, not because it ran the updated code. Foreign or unreadable ownership prints a notice
  607. and leaves the unit alone; it is not a failed restart.
  608. """
  609. from hermes_cli.update_fleet_scope import announce_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope
  610. home = systemd_unit_hermes_home(scope_cmd, svc_name)
  611. if home is not None and home_in_update_scope(home):
  612. return True
  613. announce_skipped_runtime("systemd unit", svc_name, home)
  614. return False
  615. def _scoped_manual_gateway_pids(pids, *, keep=(), quiet: bool = False) -> list[int]:
  616. """*pids* whose live home this update owns (plus *keep*, PIDs already mapped to this
  617. install's profile PID files); every other gateway process is named and left running."""
  618. from hermes_cli.update_fleet_scope import announce_skipped_runtime, partition_gateway_pids_by_scope
  619. keep = set(keep)
  620. owned, foreign = partition_gateway_pids_by_scope([pid for pid in pids if pid not in keep])
  621. if not quiet:
  622. for pid, home in foreign:
  623. announce_skipped_runtime("gateway process", f"PID {pid}", home)
  624. return [pid for pid in pids if pid in keep or pid in owned]
  625. def _is_serve_or_dashboard_unit(unit: str) -> bool:
  626. """``hermes-serve*``/``hermes-dashboard*`` (``.service``) by the same exact-or-hyphenated shape."""
  627. return (
  628. unit == "hermes-serve.service"
  629. or unit.startswith("hermes-serve-")
  630. # #125297: ``hermes-dashboard*`` units are systemd-supervised dashboard backends — the
  631. # same fleet this pass restarts. Leaving them out meant a successful update reported
  632. # the dashboard ``deferred`` (still on pre-update code) while nothing ever restarted it.
  633. or unit == "hermes-dashboard.service"
  634. or unit.startswith("hermes-dashboard-")
  635. )
  636. def _is_hermes_gateway_unit(unit: str) -> bool:
  637. """Exact base unit or hyphenated profile family only: ``startswith("hermes-serve")``
  638. would accept ``hermes-server.service``."""
  639. return (
  640. # list-units is already pattern-filtered, but keep the name gate so a stray non-gateway/serve line
  641. # cannot enter the restart path. See #83595.
  642. unit == "hermes-gateway.service"
  643. or unit.startswith("hermes-gateway-")
  644. or _is_serve_or_dashboard_unit(unit)
  645. )
  646. def _for_each_systemd_gateway_unit(list_units_stdout: str, *, process_unit, on_unit_timeout) -> None:
  647. """Process each hermes-gateway*/hermes-serve* unit from ``systemctl list-units``.
  648. ``TimeoutExpired`` from ``process_unit`` is isolated per unit via ``on_unit_timeout``
  649. so one wedged systemctl call cannot abort the rest of the fleet.
  650. See #68523.
  651. """
  652. for line in (list_units_stdout or "").strip().splitlines():
  653. parts = line.split()
  654. if not parts:
  655. continue
  656. unit = parts[0]
  657. if not unit.endswith(".service") or not _is_hermes_gateway_unit(unit):
  658. continue
  659. svc_name = unit.removesuffix(".service")
  660. try:
  661. process_unit(svc_name)
  662. except subprocess.TimeoutExpired as exc:
  663. on_unit_timeout(svc_name, exc)
  664. def _service_unit_supports_graceful_sigusr1_restart(svc_name: str) -> bool:
  665. """Whether *svc_name* wires SIGUSR1 to a graceful drain-then-restart.
  666. Only ``hermes-gateway*`` runs ``gateway/run.py`` (the handler); SIGUSR1 would just
  667. kill ``hermes-serve*`` and burn the drain budget, so those go straight to the blunt
  668. restart. Same exact/hyphenated shape as ``_for_each_systemd_gateway_unit`` so a
  669. near-prefix unit like ``hermes-gatewayd`` is never signalled.
  670. See #83438.
  671. """
  672. return svc_name == "hermes-gateway" or svc_name.startswith("hermes-gateway-")
  673. def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None:
  674. """Print an explicit incomplete-update warning for unrestarted units."""
  675. from hermes_cli.gateway import is_macos
  676. if not failed_units:
  677. return
  678. ordered = list(dict.fromkeys(failed_units)) # de-dup, discovery order
  679. print()
  680. print("⚠ Update incomplete — some units were not restarted:")
  681. for name in ordered:
  682. print(f" - {name}")
  683. if is_macos():
  684. # A label lands here when launchd wasn't supervising a live process after
  685. # the restart — likely deregistered, which `launchctl kickstart` can't revive.
  686. # See #88848.
  687. print(" Listed services may be deregistered from launchd, or still")
  688. print(" running pre-update code (mixed sys.modules). Recover with:")
  689. print(" hermes gateway status")
  690. print(" launchctl list | grep <label>")
  691. print(" launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/<label>.plist")
  692. return
  693. print(" Skipped units may still be running pre-update code (mixed")
  694. print(" sys.modules). Restart them manually, then verify:")
  695. print(" hermes gateway status")
  696. if any(not name.startswith("ai.hermes.") for name in ordered):
  697. print(" systemctl --user restart <unit> # user-scope")
  698. print(" sudo systemctl restart <unit> # system-scope")
  699. if any(name.startswith("ai.hermes.") for name in ordered):
  700. print(" launchctl kickstart -k gui/$UID/<label> # macOS (or user/$UID)")
  701. def _restart_launchd_gateway_after_update(
  702. *, supervision_verify: bool = True, self_restart_pending: set | None = None,
  703. ) -> tuple[list, list]:
  704. """Restart the invoking profile's launchd gateway after an update.
  705. No ``launchctl list`` gating: a booted-out job (plist present, definition
  706. deregistered) fails it, and it can exit non-zero while the job is alive — gating on it
  707. silently skipped the restart yet printed "Update complete!". When the plist exists
  708. ``launchd_restart()`` always runs; every failure path is loud with a manual recovery
  709. command. Returns ``(restarted_labels, failed_labels)``; with ``supervision_verify``
  710. success also requires a fresh supervised PID ("the call returned" is not "supervised").
  711. 74973 (salvage #75021 by @jeff-mettel): the restart used to be gated on ``launchctl list <label>``
  712. exiting 0. A *booted-out* job — plist present, definition deregistered from launchd (crashed helper,
  713. manual bootout, failed prior update) — fails that check, so the whole branch silently skipped: no
  714. restart, no message, ``KeepAlive`` unable to revive a definition launchd no longer knows, and the update
  715. still printed "Update complete!".
  716. See #88848.
  717. """
  718. from hermes_cli.gateway import (
  719. get_launchd_label, get_launchd_plist_path, launchd_restart, wait_for_launchd_gateway_supervision,
  720. _is_pid_ancestor_of_current_process, _launchctl_supervised_pid,
  721. )
  722. current_label = get_launchd_label()
  723. old_pid = None
  724. try:
  725. if not get_launchd_plist_path().exists():
  726. return [], [] # not a launchd install — nothing to do or warn
  727. # Snapshot BEFORE the restart: "supervising some pid" was true before too, so only a pid that
  728. # actually changed distinguishes a restart from a no-op (the sibling loop's contract). Read-only
  729. # and verification-only — the restart itself is never gated on `launchctl list` (#74973).
  730. old_pid = _launchctl_supervised_pid(current_label) if supervision_verify else None
  731. try:
  732. launchd_restart()
  733. except subprocess.CalledProcessError as e:
  734. stderr = (getattr(e, "stderr", "") or "").strip()
  735. print(
  736. f" ⚠ Gateway restart failed: {stderr}\n"
  737. " The gateway may be DOWN on pre-update code. "
  738. "Recover manually: hermes gateway restart"
  739. )
  740. return [], [current_label]
  741. except (FileNotFoundError, subprocess.TimeoutExpired) as e:
  742. # A plist exists, so a gateway is SUPPOSED to be supervised; a broken/wedged
  743. # launchctl is not proof nothing needs restarting. Count it, tell the operator.
  744. print(
  745. # The old code `pass`ed here (#74973's second silent variant); count it and tell the operator.
  746. " ⚠ Could not restart the gateway "
  747. f"({e.__class__.__name__}: {e}).\n"
  748. " Recover manually: hermes gateway restart"
  749. )
  750. return [], [current_label]
  751. if not supervision_verify:
  752. return [current_label], []
  753. if old_pid is not None and _is_pid_ancestor_of_current_process(old_pid):
  754. # launchd_restart() handed the restart to the gateway this updater runs INSIDE (cron job in
  755. # the gateway tree, #100179): it exits only after this process does, so no fresh supervised
  756. # pid can appear while we wait. Record it as pending for the fleet matrix (#119597).
  757. if self_restart_pending is not None:
  758. self_restart_pending.add(old_pid)
  759. return [current_label], []
  760. # launchd_restart() returning only means "restart REQUESTED" (async). A helper dying
  761. # before first bootstrap, or a bootstrap exiting 0 without registering (macOS 26.6.1),
  762. # would otherwise reach "Update complete!" unsupervised. Verified domain-agnostically:
  763. # domain locate fails on macOS-26 per-user domains.
  764. # launchd_restart() returning is only "restart REQUESTED" — the self-restart branch hands work to the
  765. # running gateway, a plist reload to a detached helper; both asynchronous. See #88848.
  766. if wait_for_launchd_gateway_supervision(label=current_label, old_pid=old_pid):
  767. return [current_label], []
  768. print(
  769. f" ✗ {current_label} restarted but launchd is not supervising a new process for it.\n"
  770. " Check logs, then: hermes gateway restart"
  771. )
  772. return [], [current_label]
  773. def _restart_macos_launchd_gateways(
  774. restarted_services: list, failed_or_stale_units: list, drain_budget: float, *, require_supervision: bool = False,
  775. self_restart_pending: set | None = None,
  776. ) -> None:
  777. """Restart every launchd-managed gateway after an update (macOS).
  778. The pull is shared across profiles, so every ``ai.hermes.gateway*`` LaunchAgent
  779. must reload it or siblings stay on pre-update ``sys.modules`` (systemd parity).
  780. Invoking profile uses ``launchd_restart()``; siblings get the same drain-first
  781. sequence with their domain (``gui/<uid>`` vs ``user/<uid>``) resolved per label so
  782. none is kickstarted in the wrong domain. ``TimeoutExpired`` is isolated per label.
  783. See #41403.
  784. The invoking profile keeps the existing ``launchd_restart()`` treatment (self-restart request → graceful
  785. drain → kickstart). ``subprocess.TimeoutExpired`` is isolated per label so one wedged launchctl call
  786. cannot leave the rest of the fleet on old code (#68523).
  787. """
  788. from hermes_cli.gateway import (
  789. get_launchd_label, get_launchd_plist_path, launchd_gateway_labels_for_install, legacy_launchd_labels_for_install,
  790. _graceful_restart_via_sigusr1, _launchd_kickstart,
  791. _locate_launchd_gateway_service, _wait_for_launchd_service_pid,
  792. )
  793. if require_supervision:
  794. listing = subprocess.run(["launchctl", "list"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, check=False)
  795. if listing.returncode != 0:
  796. failed_or_stale_units.append("launchd (listing failed)")
  797. return
  798. from hermes_cli.update_cmd_posix_pause import already_restarted
  799. current_label = get_launchd_label()
  800. resumed = already_restarted()["labels"] # paused for the update, restarted on the new code
  801. _restarted, _failed = ([], []) if current_label in resumed else _restart_launchd_gateway_after_update(
  802. supervision_verify=True, self_restart_pending=self_restart_pending)
  803. restarted_services.extend(_restarted)
  804. failed_or_stale_units.extend(_failed)
  805. derived_labels = launchd_gateway_labels_for_install()
  806. # Units labelled before the profile-name suffix scheme (ai.hermes.gateway-<hash>) are invisible
  807. # to the derivation; legacy_launchd_labels_for_install() credits one only when its plist is
  808. # provably this install's, so the #41403 boundary (never touch another install's fleet) holds.
  809. # See #115254.
  810. legacy_labels = legacy_launchd_labels_for_install(exclude=set(derived_labels) | {current_label})
  811. if legacy_labels:
  812. print(f" ↻ legacy-labelled units of this install join the restart: {', '.join(legacy_labels)}")
  813. from hermes_cli.update_fleet_scope import describe_skipped_runtime, launchd_label_foreign_home
  814. for label in derived_labels + legacy_labels:
  815. if label == current_label or label in resumed:
  816. continue
  817. # Labels are account-global: root B's default profile derives the same bare label root A
  818. # installed. A plist pinning a foreign HERMES_HOME is another install's job (#93349).
  819. if (foreign_home := launchd_label_foreign_home(label)) is not None:
  820. print(describe_skipped_runtime("launchd job", label, foreign_home))
  821. continue
  822. try:
  823. # Locate = liveness + domain in one probe; kickstart and fresh-PID checks
  824. # reuse that domain so a sibling is never probed in one and restarted in another.
  825. domain, old_pid = _locate_launchd_gateway_service(label)
  826. if domain is None:
  827. if require_supervision and get_launchd_plist_path().with_name(f"{label}.plist").exists():
  828. failed_or_stale_units.append(label)
  829. continue # A profile without an installed job has no restart target.
  830. graceful_ok = False
  831. if old_pid is not None and old_pid > 0:
  832. print(f" → {label}: draining (up to {drain_budget:.0f}s)...")
  833. from hermes_cli.update_cmd_drain_report import drain_progress_reporter
  834. graceful_ok = _graceful_restart_via_sigusr1(
  835. old_pid, drain_timeout=drain_budget,
  836. on_progress=drain_progress_reporter(_gateway_home_for_pid(old_pid), budget_s=drain_budget))
  837. if graceful_ok and _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=10.0, domain=domain):
  838. # KeepAlive already respawned it on new code — a kickstart would kill it.
  839. restarted_services.append(label)
  840. continue
  841. try:
  842. _launchd_kickstart(label, domain)
  843. except subprocess.CalledProcessError as e:
  844. stderr = (getattr(e, "stderr", "") or "").strip()
  845. failed_or_stale_units.append(label)
  846. print(
  847. f" ⚠ Failed to restart {label}: {stderr}\n"
  848. f" Recover manually: launchctl kickstart -k {domain}/{label}"
  849. )
  850. continue
  851. if _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=15.0, domain=domain):
  852. restarted_services.append(label)
  853. else:
  854. failed_or_stale_units.append(label)
  855. print(
  856. f" ✗ {label} failed to come back after restart.\n"
  857. f" Check logs, then: launchctl kickstart -k {domain}/{label}"
  858. )
  859. except subprocess.TimeoutExpired:
  860. failed_or_stale_units.append(label)
  861. print(f" ⚠ launchctl timed out restarting {label}; continuing with remaining gateways")
  862. def _surviving_gateway_pids_after_failed_restart():
  863. """Best-effort PIDs of gateways still running after the restart phase died.
  864. ``None`` when undeterminable (notably ``hermes_cli.gateway`` no longer importing
  865. under the replaced checkout). Callers treat ``None`` and non-empty as "assume
  866. stale"; only a positive empty result proves nothing needs restarting.
  867. """
  868. try:
  869. from hermes_cli.gateway import find_gateway_pids
  870. return _scoped_manual_gateway_pids(find_gateway_pids(all_profiles=True), quiet=True)
  871. except Exception as exc: # pragma: no cover - defensive
  872. logger.debug("Could not probe for surviving gateways after update: %s", exc)
  873. return None
  874. _MANUAL_GATEWAY_SKIP_REASON = (
  875. "manual gateway has no supervisor relaunch authority; left running for explicit operator restart"
  876. )
  877. _DESKTOP_SERVE_SKIP_REASON = (
  878. "desktop app owns and respawns this serve backend;"
  879. " the recovery pass must not restart it out from under its supervisor"
  880. )
  881. # NOT a claim that no supervisor exists: a unit's MainPID row is classified "systemd" (#141021),
  882. # and "manual-serve" still covers a process inside some unit's cgroup it is not MainPID of, or
  883. # unreadable ownership. Unit-backed serves are recovered by the fresh child's systemd pass;
  884. # survivors reported by _surviving_pre_update_serve_runtimes.
  885. _SERVE_SKIP_REASON = (
  886. "no per-profile relaunch command reaches a serve/dashboard runtime; recovered by the fresh"
  887. " systemd unit pass when a hermes-serve*/hermes-dashboard* unit owns it, else left running for explicit"
  888. " operator restart"
  889. )
  890. # A launchd-owned backend (#116503): the fresh child has no per-label kickstart for serve/dashboard
  891. # jobs, but the post-update dashboard cleanup pass kickstarts the loaded job — never a detached
  892. # argv respawn, which would fight the job's own KeepAlive.
  893. _LAUNCHD_SERVE_SKIP_REASON = (
  894. "launchd job owns this backend; the post-update dashboard cleanup kickstarts the job through"
  895. " launchd, never a detached argv respawn that would fight its KeepAlive"
  896. )
  897. def _gateway_recovery_partition(plan, *, skip_profiles: set[str] | None = None) -> tuple[dict[str, str], list[dict]]:
  898. """Partition pre-update runtimes into fresh-restart candidates and skips.
  899. Uses only the pre-checkout inventory: re-importing ``hermes_cli.gateway`` in the
  900. failing interpreter is what raises the original ``ImportError``. Returns
  901. ``(candidates, skipped)``: profile → supervisor for supervised gateways the fresh
  902. process may restart; every other inventoried runtime with an explicit reason so
  903. nothing vanishes silently. Skipped serve/dashboard is NOT unrecoverable: the fresh
  904. child's ``hermes-serve*`` systemd pass enumerates units from systemd; leftovers are
  905. caught by :func:`_surviving_pre_update_serve_runtimes`.
  906. """
  907. skip_profiles = skip_profiles or set()
  908. candidates: dict[str, str] = {}
  909. skipped: list[dict] = []
  910. with _best_effort('Could not prepare fresh gateway restart profiles: %s'):
  911. for runtime in getattr(plan, "runtimes", ()) or ():
  912. kind = getattr(runtime, "kind", None)
  913. profile = getattr(runtime, "profile", None)
  914. supervisor = getattr(runtime, "supervisor", None)
  915. if not isinstance(profile, str) or not profile:
  916. continue
  917. if kind == "gateway":
  918. if profile in skip_profiles:
  919. continue
  920. if supervisor in _FRESH_RESTART_SUPERVISORS:
  921. candidates.setdefault(profile, str(supervisor))
  922. continue
  923. reason = _MANUAL_GATEWAY_SKIP_REASON
  924. elif kind in ("serve", "dashboard"):
  925. from hermes_cli.update_inventory import CLIENT_OWNED_SERVE_SUPERVISORS
  926. if supervisor in CLIENT_OWNED_SERVE_SUPERVISORS:
  927. reason = _DESKTOP_SERVE_SKIP_REASON
  928. elif supervisor == "launchd":
  929. reason = _LAUNCHD_SERVE_SKIP_REASON
  930. else:
  931. reason = _SERVE_SKIP_REASON
  932. else:
  933. continue
  934. skipped.append({"profile": profile, "kind": str(kind), "supervisor": str(supervisor), "reason": reason})
  935. return candidates, skipped
  936. def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None:
  937. """Print a recovery warning when the whole restart phase raised.
  938. Previously a blanket debug-logged ``except Exception`` erased every drain/restart
  939. line, so "Update complete!" exited 0 while the gateway kept serving pre-update
  940. modules and died on the next turn with an ImportError.
  941. Issue #78574: the gateway auto-restart phase was wrapped in a blanket ``except Exception`` that only
  942. logged at debug level, so an early failure (e.g. importing ``hermes_cli.gateway`` from the freshly
  943. pulled checkout) erased every drain/restart line from the update output.
  944. """
  945. print()
  946. print(f"⚠ Update incomplete — gateway auto-restart failed: {exc}")
  947. if pids:
  948. listed = ", ".join(str(pid) for pid in pids)
  949. print(f" Gateway process(es) still running pre-update code: {listed}")
  950. else:
  951. print(" Any gateway still running is serving pre-update code")
  952. print(" (mixed sys.modules) against the updated checkout.")
  953. print(" Restart it manually, then verify:")
  954. print(" hermes gateway restart")
  955. print(" hermes gateway status")
  956. def _drain_or_signal_gateway_for_update(
  957. pid: int, drain_budget: float, label: str, *, self_restart_pending: set | None = None,
  958. ) -> bool:
  959. """Three-way triage (shared by systemd and bare-process paths) for handing a
  960. running gateway over to new code. Returns True when signalled/stopped.
  961. 1. Gateway is an ancestor of this process (auto-update cron inside the gateway
  962. tree): waiting is circular (gateway waits on in-flight work → cron session
  963. waits on update → update waits on gateway) and the 1800s force-drain cap burns.
  964. So fire-and-forget: signal restart and return; it completes once THIS process exits.
  965. The pid lands in ``self_restart_pending`` so the fleet matrix can tell "restart
  966. deferred until the updater exits" from "restart never happened" (#119597): the
  967. ancestor is still serving the old code when the matrix runs, by construction.
  968. 2. Event loop provably wedged: SIGUSR1 can never drain it; bounded SIGTERM→SIGKILL.
  969. 3. Live out-of-tree gateway: graceful SIGUSR1 drain up to ``drain_budget``.
  970. The wedged-loop probe cannot break it: the cron session posts activity every ~180s (process-tool poll
  971. return), so it is "actively waiting forever" and never marked wedged — the gateway burns the full
  972. force-drain cap (1800s) before killing its own updater's session. See #86684.
  973. """
  974. from hermes_cli.gateway import (
  975. GATEWAY_LOOP_WEDGED, _escalate_wedged_gateway, _graceful_restart_via_sigusr1,
  976. _is_pid_ancestor_of_current_process, _request_gateway_self_restart, probe_gateway_loop_liveness,
  977. )
  978. if _is_pid_ancestor_of_current_process(pid):
  979. print(
  980. f" → {label}: update is running inside this gateway's "
  981. "process tree — signalling restart and letting the gateway "
  982. "drain itself (avoids the cron-update deadlock, #100179)"
  983. )
  984. accepted = _request_gateway_self_restart(pid)
  985. if accepted and self_restart_pending is not None:
  986. self_restart_pending.add(pid)
  987. return accepted
  988. if probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
  989. print(f" ⚠ {label}: gateway event loop is unresponsive — skipping drain, forcing a bounded stop...")
  990. _escalate_wedged_gateway(pid)
  991. return True
  992. print(f" → {label}: draining (up to {drain_budget:.0f}s)...")
  993. from hermes_cli.update_cmd_drain_report import drain_progress_reporter
  994. return _graceful_restart_via_sigusr1(
  995. pid, drain_timeout=drain_budget,
  996. on_progress=drain_progress_reporter(_gateway_home_for_pid(pid), budget_s=drain_budget))
  997. def _gateway_home_for_pid(pid: int):
  998. """HERMES_HOME of the gateway ``pid`` per the fleet inventory, else None (own profile's file)."""
  999. with suppress(Exception):
  1000. from hermes_cli.update_receipt import _profile_homes
  1001. from gateway.status import read_runtime_status
  1002. for _profile, home in _profile_homes():
  1003. record = read_runtime_status(home / "gateway_state.json") or {}
  1004. if record.get("pid") == pid:
  1005. return home
  1006. return None
  1007. def _sudo_noninteractive_ok(targeted_probe: list) -> bool:
  1008. """True when this user can elevate without a prompt.
  1009. ``sudo -n true`` first; a refusal is inconclusive because a NOPASSWD sudoers entry scoped
  1010. to one command (the hardened shape) rejects the blanket probe, so fall back to running
  1011. ``sudo -n <targeted_probe>`` — callers pass a non-destructive stand-in for the argv they
  1012. are about to elevate.
  1013. """
  1014. try:
  1015. if subprocess.run(["sudo", "-n", "true"], capture_output=True, timeout=5, check=False).returncode == 0:
  1016. return True
  1017. # Blanket sudo refused — a targeted NOPASSWD sudoers entry may still work.
  1018. return subprocess.run(["sudo", "-n", *targeted_probe], capture_output=True, timeout=5, check=False).returncode == 0
  1019. except (OSError, subprocess.TimeoutExpired):
  1020. return False
  1021. def _resolve_manage_cmd(cache: dict, scope_: str, scope_cmd_: list, svc_name_: str):
  1022. """Resolve the command prefix for manage-units verbs (None ⇒ no privilege path).
  1023. Manage-units verbs on a *system* service trigger a polkit prompt for non-root
  1024. users, which flashes and dies inside our captured 10-15s subprocess. Root → plain
  1025. systemctl; else ``sudo -n`` blanket probe, then a targeted ``reset-failed`` probe
  1026. so a least-privilege sudoers entry scoped to hermes-gateway* qualifies (idempotent
  1027. no-op we run before every privileged restart anyway). On None the caller must SKIP
  1028. the restart (without draining first!). ``--no-ask-password`` prevents polkit hangs.
  1029. """
  1030. if scope_ in cache:
  1031. return cache[scope_]
  1032. cmd = scope_cmd_ + ["--no-ask-password"]
  1033. if _needs_sudo(scope_):
  1034. sudo_cmd = ["sudo", "-n"] + cmd
  1035. cmd = sudo_cmd if _sudo_noninteractive_ok(cmd + ["reset-failed", svc_name_]) else None
  1036. cache[scope_] = cmd
  1037. return cmd
  1038. def _repair_unit_without_fatal_exit_park(svc_name: str, scope: str) -> None:
  1039. """A unit whose restart policy predates ``RestartPreventExitStatus=78`` crash-loops on the PERMANENT
  1040. exit: a ``Restart=on-failure`` system unit restarted ~180x on a host-attach refusal while the
  1041. regenerated user units parked (#118282). The gateway rewrites its USER unit at boot; a SYSTEM unit
  1042. lives in /etc, so rewrite it here when we are root, else name the repair."""
  1043. from hermes_cli.gateway import (
  1044. _SYSTEM_UNIT_DIR, GATEWAY_FATAL_CONFIG_EXIT_CODE, get_service_name,
  1045. refresh_systemd_unit_if_needed, user_systemd_unit_dir,
  1046. )
  1047. system = scope == "system"
  1048. unit_path = (_SYSTEM_UNIT_DIR if system else user_systemd_unit_dir()) / f"{svc_name}.service"
  1049. try:
  1050. parked = re.search(rf"^RestartPreventExitStatus=.*\b{GATEWAY_FATAL_CONFIG_EXIT_CODE}\b", unit_path.read_text(encoding="utf-8-sig"), re.MULTILINE)
  1051. except OSError:
  1052. return
  1053. if parked:
  1054. return
  1055. if system and not _needs_sudo(scope) and svc_name == get_service_name():
  1056. # The refresh adopts the unit's HERMES_HOME into os.environ (sudo strips it); the rest of the
  1057. # update keeps running for the invoking profile.
  1058. launch_home = os.environ.get("HERMES_HOME")
  1059. try:
  1060. refresh_systemd_unit_if_needed(system=True)
  1061. finally:
  1062. if launch_home is None:
  1063. os.environ.pop("HERMES_HOME", None)
  1064. else:
  1065. os.environ["HERMES_HOME"] = launch_home
  1066. return
  1067. print(
  1068. f" ⚠ {svc_name} lacks RestartPreventExitStatus={GATEWAY_FATAL_CONFIG_EXIT_CODE}: a permanent refusal "
  1069. f"(exit {GATEWAY_FATAL_CONFIG_EXIT_CODE}) would crash-loop it instead of parking.\n"
  1070. f" Repair: {'sudo ' if system else ''}hermes gateway install{' --system' if system else ''}"
  1071. )
  1072. def _restart_one_systemd_gateway_unit(
  1073. svc_name: str, *, scope: str, scope_cmd: list, drain_budget: float, _manage_cmd_cache: dict,
  1074. restarted_services: list, failed_or_stale_units: list, self_restart_pending: set | None = None,
  1075. ) -> None:
  1076. """Restart one active systemd gateway/serve unit: graceful SIGUSR1 drain, then forced restart.
  1077. Appends settled names to ``restarted_services`` and failures to ``failed_or_stale_units``.
  1078. """
  1079. check = _systemctl(scope_cmd + ["is-active", svc_name], timeout=5)
  1080. if check.stdout.strip() != "active":
  1081. return
  1082. if not _systemd_unit_owned_by_update(scope_cmd, svc_name):
  1083. return
  1084. _repair_unit_without_fatal_exit_park(svc_name, scope)
  1085. # None ⇒ no non-interactive privilege path; avoid manage-units verbs
  1086. # entirely or polkit prompts inside the captured subprocess.
  1087. _manage_cmd = _resolve_manage_cmd(_manage_cmd_cache, scope, scope_cmd, svc_name)
  1088. # Graceful SIGUSR1 first so in-flight runs drain: handler → request_restart(via_service=True)
  1089. # → drain → exit, Restart=always respawns. hermes-serve has no handler → blunt restart below.
  1090. _main_pid = 0
  1091. if _service_unit_supports_graceful_sigusr1_restart(svc_name):
  1092. try:
  1093. _show = _systemctl(scope_cmd + ["show", svc_name, "--property=MainPID", "--value"], timeout=5)
  1094. _main_pid = int((_show.stdout or "").strip() or 0)
  1095. except (ValueError, subprocess.TimeoutExpired, FileNotFoundError):
  1096. _main_pid = 0
  1097. # Three-way triage (ancestor / wedged / graceful drain).
  1098. _graceful_ok = _main_pid > 0 and _drain_or_signal_gateway_for_update(
  1099. _main_pid, drain_budget, svc_name, self_restart_pending=self_restart_pending)
  1100. if _graceful_ok:
  1101. from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job, service_restart_sec, systemd_restart_timeout
  1102. # ``Restart=always`` respawns only after RestartSec (60s in our unit; dead time for a
  1103. # voluntary restart). ``reset-failed`` + ``start`` skips it (~1-3s); if RestartSec already
  1104. # elapsed, ``start`` is a no-op and we fall through to the poll. Needs manage-units
  1105. # privileges; without them auto-restart still fires after RestartSec.
  1106. if _manage_cmd is not None:
  1107. _systemctl(_manage_cmd + ["reset-failed", svc_name], timeout=10)
  1108. run_unit_job(_manage_cmd, scope_cmd, "start", svc_name,
  1109. timeout=systemd_restart_timeout(scope_cmd, svc_name, start_only=True))
  1110. if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
  1111. restarted_services.append(svc_name)
  1112. return
  1113. # Passive poll: auto-restart fires after RestartSec regardless of
  1114. # privileges — primary when _manage_cmd is None, fallback otherwise.
  1115. _restart_sec = service_restart_sec(scope_cmd, svc_name, default=0.0)
  1116. if _manage_cmd is None and _restart_sec > 5.0:
  1117. print(
  1118. f" → {svc_name}: waiting for systemd "
  1119. f"auto-restart (~{int(_restart_sec)}s; "
  1120. "no root for an immediate restart)..."
  1121. )
  1122. if _wait_for_service_active(scope_cmd, svc_name, timeout=max(10.0, _restart_sec + 10.0)):
  1123. restarted_services.append(svc_name)
  1124. return
  1125. # Exited but not respawned (older unit without Restart=on-failure /
  1126. # RestartForceExitStatus=75); fall through to forced restart.
  1127. print(f" ⚠ {svc_name} drained but didn't relaunch — forcing restart")
  1128. # Forcing needs manage-units privileges; without a non-interactive path
  1129. # polkit would prompt inside the captured subprocess — skip, instruct.
  1130. if _manage_cmd is None:
  1131. failed_or_stale_units.append(svc_name)
  1132. print(
  1133. f" ⚠ {svc_name} is a system service and restarting it needs root.\n"
  1134. f" Restart it manually to load the new version:\n"
  1135. f" sudo systemctl restart {svc_name}\n"
  1136. f" To let `hermes update` restart it automatically, allow\n"
  1137. f" passwordless sudo for systemctl, or run updates with sudo."
  1138. )
  1139. return
  1140. # Blunt restart — only when the graceful path failed (no SIGUSR1 wiring, drain over
  1141. # budget, restart-policy mismatch). Mirrors `hermes gateway restart` (`systemd_restart()`).
  1142. restart = _systemctl_reset_and_restart(_manage_cmd, svc_name, scope_cmd=scope_cmd)
  1143. if restart.returncode != 0:
  1144. failed_or_stale_units.append(svc_name)
  1145. print(f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}")
  1146. return
  1147. # restart returns 0 even if the new process crashes at once — verify.
  1148. if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
  1149. restarted_services.append(svc_name)
  1150. return
  1151. # Retry once — transient startup failures (stale module cache,
  1152. # import race) often clear; reset-failed so the retry isn't blocked.
  1153. print(f" ⚠ {svc_name} died after restart, retrying...")
  1154. _systemctl_reset_and_restart(_manage_cmd, svc_name, scope_cmd=scope_cmd)
  1155. if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
  1156. restarted_services.append(svc_name)
  1157. print(f" ✓ {svc_name} recovered on retry")
  1158. return
  1159. failed_or_stale_units.append(svc_name)
  1160. _scope_flag = "--user " if scope == "user" else ""
  1161. _sudo_hint = "sudo " if scope == "system" else ""
  1162. print(
  1163. f" ✗ {svc_name} failed to stay running after restart.\n"
  1164. f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n"
  1165. f" Recover manually:\n"
  1166. f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n"
  1167. f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}"
  1168. )
  1169. def _restart_systemd_gateway_units(
  1170. restarted_services, failed_or_stale_units, restarted_scoped_units, drain_budget, self_restart_pending=None,
  1171. ):
  1172. """Restart every active hermes-gateway*/hermes-serve* systemd unit (user + system).
  1173. Settled units → ``restarted_services`` (bare) and ``restarted_scoped_units``
  1174. (``scope/name``); failures → ``failed_or_stale_units``. Per-unit timeouts isolated.
  1175. """
  1176. from hermes_cli.gateway import supports_systemd_services, _ensure_user_systemd_env
  1177. if not supports_systemd_services():
  1178. return
  1179. _manage_cmd_cache: dict = {}
  1180. with suppress(Exception):
  1181. _ensure_user_systemd_env()
  1182. def _on_list_timeout(scope: str, exc: subprocess.TimeoutExpired) -> None:
  1183. # Discovery timeout — skip this scope, keep the other.
  1184. print(
  1185. f" ⚠ systemctl timed out listing {scope}-scope "
  1186. f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). "
  1187. f"Check the gateway with: hermes gateway status"
  1188. )
  1189. def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None:
  1190. # Isolate to this unit; a scope-wide handler used to abort every
  1191. # later gateway and leave the fleet on mixed code.
  1192. failed_or_stale_units.append(svc_name)
  1193. print(
  1194. # See #68523.
  1195. f" ⚠ systemctl timed out restarting {svc_name} "
  1196. f"({exc.cmd if exc.cmd else 'unknown command'}); "
  1197. f"continuing with remaining gateways"
  1198. )
  1199. # Enumerate every scope first: leftover per-profile units (``hermes-gateway-<profile>``)
  1200. # all point at the SAME live MainPID on a multiplexed host, and restarting each in turn
  1201. # is an N-fold outage from one update. Units sharing a live PID collapse to one restart;
  1202. # the others are named as legacy units to migrate, never silently dropped.
  1203. from hermes_cli.update_host_obligation import collapse_units_to_host_processes
  1204. targets: dict[str, tuple[str, list, str]] = {} # "<scope>/<unit>" -> (scope, scope_cmd, unit)
  1205. for scope, scope_cmd, result in _systemd_gateway_unit_listings(_on_list_timeout):
  1206. _for_each_systemd_gateway_unit(
  1207. result.stdout,
  1208. process_unit=lambda svc_name, _scope=scope, _cmd=scope_cmd: targets.setdefault(
  1209. f"{_scope}/{svc_name}", (_scope, _cmd, svc_name)),
  1210. on_unit_timeout=_on_unit_timeout,
  1211. )
  1212. keys = list(targets)
  1213. covered: dict[str, str] = {}
  1214. if len(keys) > 1:
  1215. # Only worth a `systemctl show` round when several units could be one process.
  1216. keys, covered = collapse_units_to_host_processes(
  1217. keys, lambda key: _unit_main_pid(targets[key][1], targets[key][2]))
  1218. for unit_key, owner_key in covered.items():
  1219. print(
  1220. f" • {unit_key} is a legacy per-profile unit sharing one host gateway process with "
  1221. f"{owner_key}; restarting it again would restart that process twice. Fold the units "
  1222. "together with: hermes gateway migrate"
  1223. )
  1224. from hermes_cli.update_cmd_posix_pause import already_restarted
  1225. for key in keys:
  1226. scope, scope_cmd, svc_name = targets[key]
  1227. if key in already_restarted()["units"]: # paused for the update, restarted on the new code
  1228. restarted_scoped_units.add(key)
  1229. continue
  1230. # Scope-qualify before the next unit; ``finally`` so a mid-pass abort keeps settled units.
  1231. _scope_mark = len(restarted_services)
  1232. try:
  1233. _restart_one_systemd_gateway_unit(
  1234. svc_name,
  1235. scope=scope,
  1236. scope_cmd=scope_cmd,
  1237. drain_budget=drain_budget,
  1238. _manage_cmd_cache=_manage_cmd_cache,
  1239. restarted_services=restarted_services,
  1240. failed_or_stale_units=failed_or_stale_units,
  1241. self_restart_pending=self_restart_pending,
  1242. )
  1243. except subprocess.TimeoutExpired as exc:
  1244. _on_unit_timeout(svc_name, exc)
  1245. finally:
  1246. restarted_scoped_units.update(f"{scope}/{name}" for name in restarted_services[_scope_mark:])
  1247. def _unit_main_pid(scope_cmd: list, svc_name: str) -> int:
  1248. """Live ``MainPID`` of a unit; ``0`` when inactive, unprivileged or unreadable.
  1249. Property reads need no manage-units privileges, and an unreadable PID is never collapsed:
  1250. identity that cannot be proved keeps its own restart.
  1251. """
  1252. try:
  1253. result = _systemctl(list(scope_cmd) + ["show", svc_name, "--property=MainPID", "--value"], timeout=10)
  1254. except (OSError, subprocess.TimeoutExpired):
  1255. return 0
  1256. if getattr(result, "returncode", 1) != 0:
  1257. return 0
  1258. try:
  1259. return int((getattr(result, "stdout", "") or "").strip() or 0)
  1260. except ValueError:
  1261. return 0
  1262. @dataclass
  1263. class _GatewayRestartOutcome:
  1264. """Restart-phase bookkeeping. ``restarted_services`` keeps bare unit names (fleet
  1265. probe, receipt, summary read it); ``incomplete`` ⇒ a gateway may still be stale."""
  1266. incomplete: bool
  1267. phase_errors: list
  1268. pre_restart_gateway_pids: list | None
  1269. restarted_services: list
  1270. failed_or_stale_units: list
  1271. relaunched_profiles: list
  1272. externally_supervised_profiles: list
  1273. killed_pids: set
  1274. #: Gateways stopped with NO successor (no profile mapping / relaunch could not be armed);
  1275. #: the summary tells the user to restart them by hand, so the fleet probe must not expect
  1276. #: a row for them.
  1277. stopped_unmapped_pids: set = field(default_factory=set)
  1278. #: ``scope/name`` of every settled systemd unit; the fleet probe stops waiting for a state stamp
  1279. #: once none of them is active or activating any more (the successor died, nothing will publish).
  1280. restarted_scoped_units: set = field(default_factory=set)
  1281. #: Gateways that are ANCESTORS of this updater and accepted a self-restart request
  1282. #: (``_drain_or_signal_gateway_for_update`` branch 1): they restart only after this process
  1283. #: exits, so the fleet matrix renders them as pending instead of STALE (#119597).
  1284. self_restart_pending_pids: set = field(default_factory=set)
  1285. #: Units the restart phase itself could not restart. Verification warns about the ones its fleet
  1286. #: probe does not prove current: a timed-out ``systemctl`` call can still bring its unit up. Not
  1287. #: ``failed_or_stale_units``: the Windows resume merge and an aborted phase add to that list, and
  1288. #: neither gets this systemctl/launchctl advice.
  1289. unrestarted_units: list = field(default_factory=list)
  1290. def fleet_probe_signals(self) -> tuple:
  1291. """``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that
  1292. legitimately predict a fleet-matrix row."""
  1293. pre = self.pre_restart_gateway_pids
  1294. if pre is not None:
  1295. pre = [pid for pid in pre if pid not in self.stopped_unmapped_pids]
  1296. return pre, self.killed_pids - self.stopped_unmapped_pids
  1297. def record_receipt(self, **extra) -> None:
  1298. """Best-effort ``record_gateway_restart`` from the current bookkeeping."""
  1299. with suppress(Exception):
  1300. from hermes_cli.update_receipt import record_gateway_restart
  1301. record_gateway_restart(
  1302. restarted_services=self.restarted_services, relaunched_profiles=self.relaunched_profiles,
  1303. externally_supervised_profiles=self.externally_supervised_profiles,
  1304. killed_pids=sorted(self.killed_pids), failed_units=self.failed_or_stale_units,
  1305. incomplete=self.incomplete, **extra,
  1306. )
  1307. def _restart_manual_gateways(out: _GatewayRestartOutcome, _drain_budget) -> None:
  1308. """Drain/stop every manual (non-service) gateway and print the restart summary.
  1309. Mutates ``out`` in place; raises so the caller's abort recovery fires.
  1310. """
  1311. import signal as _signal
  1312. from hermes_cli.gateway import (
  1313. find_gateway_pids, find_profile_gateway_processes, _prepare_profile_gateway_update_restart, _get_service_pids,
  1314. _wait_for_gateway_exit,
  1315. )
  1316. # Exclude just-restarted service PIDs so we don't kill what systemd/launchd spawned.
  1317. from hermes_cli.update_cmd_posix_pause import already_restarted
  1318. # Plus the gateways this update paused and already restarted on the new code.
  1319. service_pids = _get_service_pids(all_profiles=True) | already_restarted()["pids"]
  1320. manual_pids = find_gateway_pids(exclude_pids=service_pids, all_profiles=True)
  1321. profile_processes = {
  1322. proc.pid: proc
  1323. for proc in find_profile_gateway_processes(exclude_pids=service_pids)
  1324. if proc.pid in manual_pids
  1325. }
  1326. # ``all_profiles`` is host-wide: a sibling install's gateway matches too. Only this update's
  1327. # homes are stopped; the profile-mapped PIDs come from this install's own PID files (#93349).
  1328. manual_pids = _scoped_manual_gateway_pids(manual_pids, keep=profile_processes)
  1329. # Profile gateways we couldn't arm a relaunch for must NOT keep running stale:
  1330. # the unmapped sweep below stops them and lists them under "Restart manually".
  1331. # These must NOT be left running: their modules are the pre-update ones and every lazy import from here
  1332. # on mixes versions against the new code on disk (#88654). Handing them to the unmapped sweep below
  1333. # stops them and surfaces them in the "Stopped N manual gateway process(es) / Restart manually" summary,
  1334. # which is the contract already used for gateways with no profile mapping.
  1335. unrestartable_pids = set()
  1336. for pid, proc in profile_processes.items():
  1337. restart_mode = _prepare_profile_gateway_update_restart(proc.profile, pid)
  1338. if restart_mode is None:
  1339. # A bare ``continue`` here left it serving stale modules with no signal.
  1340. print(
  1341. f" ⚠ {proc.profile}: could not arm an automatic "
  1342. f"gateway restart for PID {pid} — stopping it instead "
  1343. "so it cannot keep running pre-update code"
  1344. )
  1345. unrestartable_pids.add(pid)
  1346. continue
  1347. # SIGUSR1 drain first, SIGTERM fallback if unsupported/over budget — the watcher
  1348. # relaunches either way. The helper announces its choice first because a silent
  1349. # full-budget wait reads as a hung update.
  1350. if not _drain_or_signal_gateway_for_update(
  1351. pid, _drain_budget, proc.profile, self_restart_pending=out.self_restart_pending_pids):
  1352. with suppress(ProcessLookupError, PermissionError):
  1353. os.kill(pid, _signal.SIGTERM)
  1354. # Wait ≤5s for exit: Telegram keeps the old getUpdates session ~30s; a new gateway
  1355. # inside that window gets a 409 (_handle_polling_conflict retries, but a brief
  1356. # wait avoids it on fast machines).
  1357. _wait_for_gateway_exit(timeout=5.0, force_after=None)
  1358. out.killed_pids.add(pid)
  1359. if restart_mode == "external-supervisor":
  1360. out.externally_supervised_profiles.append(proc.profile)
  1361. else:
  1362. out.relaunched_profiles.append(proc.profile)
  1363. for pid in manual_pids:
  1364. if pid in profile_processes and pid not in unrestartable_pids:
  1365. continue
  1366. with suppress(ProcessLookupError, PermissionError):
  1367. os.kill(pid, _signal.SIGTERM)
  1368. out.killed_pids.add(pid)
  1369. out.stopped_unmapped_pids.add(pid)
  1370. if out.restarted_services or out.killed_pids:
  1371. print()
  1372. for svc in out.restarted_services:
  1373. print(f" ✓ Restarted {svc}")
  1374. if out.relaunched_profiles:
  1375. print(f" ✓ Restarting manual gateway profile(s): {', '.join(out.relaunched_profiles)}")
  1376. if out.externally_supervised_profiles:
  1377. names = ", ".join(out.externally_supervised_profiles)
  1378. print(f" ✓ Handed gateway profile(s) back to their external supervisor: {names}")
  1379. unmapped_count = (len(out.killed_pids) - len(out.relaunched_profiles) - len(out.externally_supervised_profiles))
  1380. if unmapped_count:
  1381. print(f" → Stopped {unmapped_count} manual gateway process(es)")
  1382. print(" Restart manually: hermes gateway run")
  1383. if unmapped_count > 1:
  1384. print(" (or: hermes -p <profile> gateway run for each profile)")
  1385. def _force_kill_stuck_gateways(killed_pids) -> None:
  1386. """Survivor sweep: gateways ignoring SIGTERM (stuck drain, blocked I/O, zombie) never
  1387. exit, so the watcher never respawns and ImportErrors persist. Give graceful paths a
  1388. moment, then SIGKILL remaining pre-update PIDs."""
  1389. with _best_effort('Post-restart survivor sweep failed: %s'):
  1390. from hermes_cli.gateway import find_gateway_pids, _get_service_pids
  1391. # --- Post-restart survivor sweep ----------------------------- Issue #17648: some gateways ignore
  1392. # SIGTERM (stuck drain, blocked I/O, PID dead but zombie). The detached profile watchers wait 120s
  1393. # for the old PID to exit — if it never does, no respawn happens and the user keeps hitting
  1394. # ImportError against a stale sys.modules.
  1395. _time.sleep(3.0)
  1396. _surviving = find_gateway_pids(exclude_pids=_get_service_pids(all_profiles=True), all_profiles=True)
  1397. # Only PIDs we already tried to kill; newer ones are left alone.
  1398. _stuck = [pid for pid in _surviving if pid in killed_pids]
  1399. if _stuck:
  1400. print()
  1401. print(f" ⚠ {len(_stuck)} gateway process(es) ignored SIGTERM — force-killing")
  1402. from gateway.status import get_process_start_time, terminate_pid
  1403. for pid in _stuck:
  1404. with suppress(ProcessLookupError, PermissionError, OSError):
  1405. # taskkill /T /F on Windows (no SIGKILL there), SIGKILL on POSIX.
  1406. terminate_pid(pid, force=True, expected_start_time=get_process_start_time(pid))
  1407. # Let the OS reap so watchers see the exit and respawn.
  1408. _time.sleep(1.5)
  1409. def _recover_after_restart_phase_abort(
  1410. e, _pre_update_plan, out: _GatewayRestartOutcome, *, gateway_mode, restarted_scoped_units
  1411. ) -> None:
  1412. """Phase-abort recovery: fresh-child restart + fail-closed verdict; updates ``out`` in place."""
  1413. from hermes_cli.update_abort_recovery import _owed_stale_serve_rows
  1414. from hermes_cli.update_cmd import (
  1415. _abort_recovery_is_complete, _recover_gateway_restart_after_abort, _surviving_pre_update_serve_runtimes,
  1416. _warn_stale_serve_runtimes,
  1417. )
  1418. logger.debug("Gateway restart during update failed: %s", e)
  1419. out.phase_errors.append(str(e))
  1420. # Restart output never printed: assume stale unless provably no gateway runs.
  1421. # Empty ``_surviving`` proves safety only if nothing ran beforehand; a gone
  1422. # pre-restart gateway was stopped without verified replacement → fail closed.
  1423. # An exception escaping the whole phase means the drain/restart output the user relies on never printed.
  1424. # Don't let that pass for a clean update: surface it and treat the fleet as stale unless we can
  1425. # positively prove no gateway is running (#78574). A positive-empty ``_surviving`` is only
  1426. # proof-of-safety when nothing was running before we touched anything. If a gateway was discovered
  1427. # pre-restart and none survive now, it was stopped and its replacement was never verified — the same
  1428. # fail-open contract this fix closes — so we must still fail closed on ``[]``.
  1429. _surviving = _surviving_gateway_pids_after_failed_restart()
  1430. _planned_gateway_profiles = {
  1431. runtime.profile
  1432. for runtime in getattr(_pre_update_plan, "runtimes", ()) or ()
  1433. if getattr(runtime, "kind", None) == "gateway"
  1434. and isinstance(getattr(runtime, "profile", None), str)
  1435. }
  1436. _already_restarted_profiles = set(out.relaunched_profiles) | set(out.externally_supervised_profiles)
  1437. _already_restarted_profiles.update(
  1438. profile
  1439. for profile in _planned_gateway_profiles
  1440. if any(_gateway_service_matches_profile(profile, service) for service in out.restarted_services)
  1441. )
  1442. _recovery_result = _recover_gateway_restart_after_abort(
  1443. _pre_update_plan, gateway_mode=gateway_mode, skip_profiles=_already_restarted_profiles,
  1444. skip_units=set(restarted_scoped_units),
  1445. )
  1446. _serve_units_failed = list((_recovery_result.get("serve_units") or {}).get("failed") or [])
  1447. # Deliberately NOT merged into ``restarted_services`` (gateway vocabulary feeding the
  1448. # fleet probe); serve coverage lives in the recovery result/receipt. A serve/dashboard
  1449. # still the SAME pre-update process is live on old code (unreachable by `gateway
  1450. # restart`): recovery may not claim success while one remains, and must never kill
  1451. # one (manual/Desktop serves have no relaunch authority).
  1452. _stale_runtime_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan)
  1453. _recovery_result["stale_runtimes"] = _stale_runtime_rows
  1454. # Only systemd-VERIFIED outcomes claim coverage; a relaunch that merely exited 0
  1455. # ("relaunch_attempted") was never observed and must not clear incomplete.
  1456. _recovery_verified = set(_recovery_result.get("verified") or [])
  1457. out.relaunched_profiles.extend(
  1458. profile for profile in sorted(_recovery_verified) if profile not in out.relaunched_profiles
  1459. )
  1460. if _abort_recovery_is_complete(
  1461. planned_gateway_profiles=_planned_gateway_profiles,
  1462. covered_gateway_profiles=_already_restarted_profiles | _recovery_verified,
  1463. recovery_result=_recovery_result,
  1464. stale_runtime_rows=_stale_runtime_rows,
  1465. ):
  1466. # Fresh child is terminal; the fleet-version matrix stays the authoritative
  1467. # read-back before success is declared. Desktop-owned survivors (the only rows
  1468. # that can remain here) are named, not owed. See #111494.
  1469. out.incomplete = False
  1470. _warn_stale_serve_runtimes(_stale_runtime_rows)
  1471. elif (
  1472. _restart_phase_failure_is_incomplete(_surviving, out.pre_restart_gateway_pids)
  1473. or _owed_stale_serve_rows(_stale_runtime_rows)
  1474. or _serve_units_failed
  1475. ):
  1476. # Owed restart (C3): verification records the follow-up; the /update marker stays committed.
  1477. out.incomplete = True
  1478. _warn_gateway_restart_phase_aborted(e, _surviving)
  1479. _warn_stale_serve_runtimes(_stale_runtime_rows)
  1480. out.record_receipt(phase_error=str(e), fresh_recovery=_recovery_result)
  1481. def _gateway_drain_budget() -> float:
  1482. """Seconds a drain-first (SIGUSR1) restart may wait for a gateway to exit; 45s floor."""
  1483. try:
  1484. from hermes_cli.gateway import _get_restart_exit_wait_budget
  1485. return max(float(_get_restart_exit_wait_budget()), 45.0)
  1486. except Exception:
  1487. return 45.0
  1488. def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):
  1489. """Restart every running gateway (systemd, launchd, manual) onto the pulled code.
  1490. Never raises: a phase abort runs fresh-child recovery and fails closed unless
  1491. every planned gateway is verifiably covered.
  1492. """
  1493. from hermes_cli.update_cmd import _m
  1494. from hermes_cli.update_stage import publish_stage
  1495. publish_stage("Restarting Hermes services") # also after a failed build: the code is committed
  1496. # All bookkeeping is declared before the try so abort recovery and fleet reconciliation
  1497. # can read it even if the phase raises early. ``pre_restart_gateway_pids`` stays empty
  1498. # until we are about to stop/drain, so an early exception has nothing to fail closed on,
  1499. # while a failure after stopping a discovered gateway fails closed on an empty survivor probe.
  1500. out = _GatewayRestartOutcome(
  1501. incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], failed_or_stale_units=[],
  1502. relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(),
  1503. )
  1504. # Scope-qualified twin (``user/hermes-serve`` vs ``system/hermes-serve`` are different
  1505. # processes; abort recovery needs WHICH settled). Bare names stay in
  1506. # ``restarted_services`` for the fleet probe, receipt and summary.
  1507. # Snapshot of gateways running before we touch anything. Stays empty until we successfully import the
  1508. # probe and are about to stop/drain — so an exception raised before we touch any gateway keeps this
  1509. # empty (nothing to fail closed on), while a failure after we have stopped a discovered gateway lets the
  1510. # handler fail closed on an empty survivor probe rather than reporting a clean update (#78574).
  1511. # Declared outside the restart try/except below (and never reset to None) so it's always safe to read
  1512. # afterwards even if that block raises before reaching its own restart bookkeeping — needed to forward
  1513. # already-restarted units to ``_refresh_dashboard_after_update`` (review on #83595).
  1514. restarted_scoped_units: set = set()
  1515. try:
  1516. # Every gateway helper the phase needs is imported up front so a broken gateway
  1517. # module aborts into recovery BEFORE any unit is touched.
  1518. from hermes_cli.gateway import (
  1519. is_macos,
  1520. find_gateway_pids,
  1521. find_profile_gateway_processes,
  1522. _prepare_profile_gateway_update_restart,
  1523. _get_service_pids,
  1524. _wait_for_gateway_exit,
  1525. )
  1526. # Drain budget covers ``restart_after_turn_timeout`` and stop()'s
  1527. # ``restart_drain_timeout`` so a gateway waiting on a turn isn't hard-killed;
  1528. # units without SIGUSR1 wiring just time out into ``systemctl restart``.
  1529. _drain_budget = _gateway_drain_budget()
  1530. # Snapshot before any stop/drain so an empty survivor probe reads as "stopped
  1531. # and never came back", not "nothing was running"; None fails closed.
  1532. try:
  1533. out.pre_restart_gateway_pids = _scoped_manual_gateway_pids(find_gateway_pids(all_profiles=True), quiet=True)
  1534. except Exception:
  1535. out.pre_restart_gateway_pids = None
  1536. _restart_systemd_gateway_units(
  1537. out.restarted_services, out.failed_or_stale_units, restarted_scoped_units, _drain_budget,
  1538. out.self_restart_pending_pids,
  1539. )
  1540. # macOS: EVERY ai.hermes.gateway* LaunchAgent (systemd parity).
  1541. if is_macos():
  1542. with suppress(FileNotFoundError, ImportError):
  1543. _restart_macos_launchd_gateways(
  1544. out.restarted_services, out.failed_or_stale_units, _drain_budget,
  1545. self_restart_pending=out.self_restart_pending_pids,
  1546. )
  1547. _restart_manual_gateways(out, _drain_budget)
  1548. if out.failed_or_stale_units:
  1549. # Owed restart (C3): verification records the follow-up; the /update marker stays committed.
  1550. out.incomplete = True
  1551. out.unrestarted_units = list(out.failed_or_stale_units)
  1552. out.record_receipt()
  1553. _force_kill_stuck_gateways(out.killed_pids)
  1554. except Exception as e:
  1555. _recover_after_restart_phase_abort(
  1556. e, _pre_update_plan, out, gateway_mode=gateway_mode, restarted_scoped_units=restarted_scoped_units
  1557. )
  1558. out.restarted_scoped_units = set(restarted_scoped_units)
  1559. if not out.incomplete:
  1560. # Stamp the HOST obligation so every other profile's CLI knows this update's restart
  1561. # already happened; without it each profile re-kills the one shared multiplexer.
  1562. from hermes_cli.update_host_obligation import mark_host_restart_completed
  1563. mark_host_restart_completed(_restart_identity_sha())
  1564. return out
  1565. def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool:
  1566. """Whether an escaped restart-phase exception must fail the update.
  1567. Fail closed unless provably safe: ``surviving`` None (unprobeable) or non-empty →
  1568. stale. ``[]`` proves safety ONLY if nothing ran beforehand; a pre-restart gateway
  1569. (``pre_restart_pids`` non-empty or None) now gone was stopped unverified.
  1570. * ``surviving is None`` — the survivor probe could not determine state (typically the freshly-pulled
  1571. ``hermes_cli.gateway`` no longer imports, one of the ways the phase aborts). That is proof-of-safety
  1572. ONLY when nothing was running before we touched anything. If a gateway was discovered pre-restart
  1573. (``pre_restart_pids`` non-empty, or ``None`` meaning the pre-state could not be read), it was stopped
  1574. without a verified replacement, so we still fail closed (#78574).
  1575. """
  1576. if surviving is None or surviving:
  1577. return True
  1578. return pre_restart_pids is None or bool(pre_restart_pids)
  1579. def _fleet_probe_expected_runtimes(
  1580. pre_update_plan, pre_restart_pids, windows_resume_token, restarted_services, killed_pids,
  1581. ) -> bool:
  1582. """Whether the post-update fleet probe should have produced rows.
  1583. ``collect_fleet_versions()`` swallows every failure and an empty matrix prints as
  1584. healthy, so zero rows is only proof-of-safety when NOTHING says a gateway existed
  1585. pre-update. Signals: ``restarted_services``/``killed_pids``; ``pre_restart_pids``
  1586. non-empty or None (same contract as ``_restart_phase_failure_is_incomplete``); plan
  1587. inventoried ≥1 ``kind == "gateway"`` runtime. ``windows_resume_token`` is deliberately EXCLUDED: it is
  1588. pause/resume bookkeeping, not an inventory, and its entries don't map to probe rows
  1589. (``unmapped`` Scheduled-Task gateways never publish gateway_state.json; a paused
  1590. profile resumes DETACHED). Counting it made every Windows update that paused a
  1591. gateway exit 1 after a long silent wait; a live pre-update Windows gateway is already
  1592. covered by ``pre_restart_pids`` and the plan. The same condition gates the settle sleep.
  1593. See #93406.
  1594. See #78574.
  1595. See #93406.
  1596. """
  1597. del windows_resume_token # excluded on purpose — see docstring
  1598. # See #93406.
  1599. if restarted_services or killed_pids:
  1600. return True
  1601. if pre_restart_pids is None or pre_restart_pids:
  1602. return True
  1603. with suppress(Exception):
  1604. # Gateway-kind only: serve/dashboard plan records never publish a gateway_state.json
  1605. # row, so a dashboard-only plan cannot ground a rows-expected verdict (#97332).
  1606. if pre_update_plan is not None and any(
  1607. getattr(runtime, "kind", None) == "gateway" for runtime in pre_update_plan.runtimes
  1608. ):
  1609. return True
  1610. return False
  1611. def _wait_for_service_active(scope_cmd_: list, svc_name_: str, timeout: float = 10.0) -> bool:
  1612. """Poll ``systemctl is-active`` (0.5s) up to ``timeout``: the Stopped -> Started
  1613. transition isn't instantaneous, so a one-shot check falsely reports down."""
  1614. deadline = _time.monotonic() + max(timeout, 0.5)
  1615. while True:
  1616. with suppress(FileNotFoundError, subprocess.TimeoutExpired):
  1617. _verify = _systemctl(scope_cmd_ + ["is-active", svc_name_], timeout=5)
  1618. if _verify.stdout.strip() == "active":
  1619. return True
  1620. if _time.monotonic() >= deadline:
  1621. return False
  1622. _time.sleep(0.5)