| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842 |
- """Gateway fleet restart + post-update verification for ``hermes update``.
- Split out of ``hermes_cli/update_cmd.py``; every name is re-imported there so
- ``hermes_cli.update_cmd.<name>`` keeps resolving/monkeypatching. Origin helpers are
- imported lazily inside each function (no import cycle; test patches stay effective).
- """
- import json
- import logging
- import re
- from contextlib import suppress
- import os
- import subprocess
- import sys
- import time as _time
- from dataclasses import dataclass, field
- from pathlib import Path
- from hermes_cli.update_cmd_common import _best_effort
- from hermes_cli.update_host_obligation import PROFILE_MARKER_NAME
- from hermes_cli.update_inventory import _gateway_service_matches_profile
- # Log-record parity with the origin module.
- logger = logging.getLogger("hermes_cli.update_cmd")
- # Under HERMES_HOME (not next to the venv): records the fleet-restart obligation
- # after a pull advanced HEAD; cleared only when the restart completes or nothing ran.
- # The existing ``.update-incomplete`` / ``.lazy-refresh-incomplete`` markers gate dependency/venv repair;
- # this one is the fleet-restart obligation after a git pull that advanced HEAD (#95294).
- _FLEET_RESTART_PENDING_NAME = PROFILE_MARKER_NAME
- _FRESH_RESTART_SUPERVISORS = frozenset({"systemd", "launchd", "service", "s6"})
- _SYSTEMD_SCOPES = (("user", ["systemctl", "--user"]), ("system", ["systemctl"]))
- _LIST_GATEWAY_UNITS = [
- "list-units", "hermes-gateway*", "hermes-serve*", "hermes-dashboard*",
- "--plain", "--no-legend", "--no-pager",
- ]
- def _write_gateway_update_exit_code(ok: bool) -> None:
- from hermes_cli.update_cmd import get_hermes_home
- from hermes_cli.update_receipt import committed_success
- from pm.filesystem import durable_write_bytes
- # A run whose receipt already closed as a success is never reported failed (C3, review
- # regression 3): the command boundary's catch-alls call this with False.
- ok = ok or committed_success()
- path = get_hermes_home() / ".update_exit_code"
- with suppress(OSError):
- # Atomic: the gateway watcher polls this file and reads an empty one as exit 1.
- durable_write_bytes(path, b"0" if ok else b"1")
- def _fleet_restart_pending_marker_path() -> Path:
- """LEGACY per-``HERMES_HOME`` breadcrumb. Read-compat only — nothing writes it any more.
- One host runs one multiplexing gateway, so the pull→restart obligation is host-scoped
- (``hermes_cli/update_host_obligation.py``). An obligation armed by the old per-profile code
- is still read and cleared here so an in-flight update is discharged after the upgrade.
- """
- from hermes_cli.update_cmd import get_hermes_home
- return get_hermes_home() / _FLEET_RESTART_PENDING_NAME
- def _write_fleet_restart_pending_marker(*, expected_sha: str = "", runtimes: list[dict] | None = None, owner: str = "") -> bool:
- """Arm the HOST pull→restart obligation. Never raises; False when no store every profile reads took it.
- An unwritable host state dir falls back to this profile's per-home marker, which settles the
- debt only on an install proven to have no other profile; otherwise the arm returns False and
- the commit point refuses to move (``update_host_obligation.arm_host_obligation``, review S2).
- """
- if runtimes == []:
- # An explicit empty inventory owes no restart (e.g. Desktop-hosted `serve` with no
- # gateway services). Arming the marker here leaves a breadcrumb nothing can discharge:
- # a no-gateway host would then fail every later ``hermes update`` (#115311).
- return True
- from hermes_cli.update_cmd import _m
- from hermes_cli.update_host_obligation import arm_host_obligation
- if _m()._pytest_owns_live_checkout(_fleet_restart_pending_marker_path().parent):
- logger.debug("Skipping fleet-restart-pending obligation under pytest (live checkout)")
- return True
- return arm_host_obligation(_fleet_restart_pending_marker_path(), expected_sha=expected_sha, runtimes=runtimes,
- profile=_current_profile_name(), owner=owner)
- def _current_profile_name() -> str:
- """Profile whose CLI armed the obligation (diagnostics only — the record is host-scoped)."""
- try:
- from hermes_cli.profiles import get_active_profile_name
- return get_active_profile_name() or "default"
- except Exception:
- return ""
- def _clear_fleet_restart_pending_marker() -> None:
- """Discharge the obligation for the whole host (legacy per-home marker included). Never raises."""
- from hermes_cli.update_cmd import _m
- from hermes_cli.update_host_obligation import clear_host_obligation
- clear_host_obligation()
- _m()._clear_marker_file(_fleet_restart_pending_marker_path(), label="fleet-restart-pending")
- def _fleet_restart_obligation_armed() -> bool:
- """True when this HOST owes a fleet restart — from any profile's CLI."""
- from hermes_cli.update_host_obligation import host_obligation_present
- if host_obligation_present():
- return True
- with suppress(OSError):
- return _fleet_restart_pending_marker_path().is_file()
- return False
- def _obligation_fields() -> dict[str, str] | None:
- """Armed obligation as ``key=value`` fields: HOST record first, then the legacy marker.
- ``None`` means nothing armed OR a malformed record; both must leave the obligation standing.
- """
- from hermes_cli.update_host_obligation import host_obligation_present, obligation_fields
- fields = obligation_fields()
- if fields is not None:
- return fields
- if host_obligation_present():
- # The record exists but its terms are unknown (corrupt, or a NEWER CLI's version). An
- # unrelated legacy marker's inventory cannot discharge terms nobody can read: fail closed.
- return None
- try:
- text = _fleet_restart_pending_marker_path().read_text(encoding="utf-8-sig")
- except (OSError, UnicodeError):
- return None
- legacy: dict[str, str] = {}
- for line in text.splitlines():
- key, sep, value = line.partition("=")
- if not sep or key in legacy:
- return None
- legacy[key] = value
- return legacy
- def _current_checkout_sha() -> str | None:
- """Current on-disk checkout HEAD, or None if it cannot be resolved."""
- from hermes_cli.update_cmd import _capture_head_sha, _m
- try:
- from hermes_cli.version_info import read_checkout_sha
- return read_checkout_sha()
- except Exception:
- return _capture_head_sha(["git"], _m().PROJECT_ROOT)
- def _receipt_looks_unfinished(receipt: dict) -> bool:
- """True when *receipt* is from an update that did not finish cleanly.
- The command boundary stamps a ``stop_reason`` on every receipt, including clean
- ones (``completed at command boundary``, ``sys.exit(0)``); it must not make a
- successful receipt look unfinished, or the next ``hermes update`` retriggers
- ``fleet_restart_pending`` from pre-pull plan SHAs (#98022).
- """
- exit_code = receipt.get("exit_code")
- outcome = receipt.get("outcome")
- if exit_code not in (0, None) or outcome in ("failed", "partial", "running"):
- return True
- gateway_restart = receipt.get("gateway_restart")
- if isinstance(gateway_restart, dict) and gateway_restart.get("incomplete"):
- return True
- # A stop_reason alone (update_contract refusals: outcome="refused", no exit_code)
- # counts only when nothing else vouched for success.
- succeeded = exit_code == 0 or outcome == "success"
- return bool(receipt.get("stop_reason")) and not succeeded
- def _receipt_reports_stale_runtime(receipt: dict, expected_sha: str | None = None) -> bool:
- """True when ``update_receipts/latest.json`` records a runtime SHA skew.
- Prefer the post-restart ``fleet`` matrix. ``plan.runtimes[].code_sha`` is captured
- *before* the pull, so a finished update's plan always looks stale and must not
- retrigger a restart; consult it only for an unfinished receipt.
- See #95294.
- """
- from hermes_cli.update_cmd import _current_checkout_sha
- if not isinstance(receipt, dict):
- return False
- expected_sha = expected_sha or _current_checkout_sha()
- if not expected_sha:
- return False
- def _sha_mismatch(code_sha) -> bool:
- return bool(code_sha) and str(code_sha) != str(expected_sha)
- from hermes_cli.update_receipt import row_is_external
- fleet = receipt.get("fleet")
- if isinstance(fleet, list) and fleet:
- return any(
- isinstance(entry, dict)
- and not row_is_external(entry)
- and (entry.get("state") == "stale" or _sha_mismatch(entry.get("code_sha")))
- for entry in fleet
- )
- if not _receipt_looks_unfinished(receipt):
- return False
- plan = receipt.get("plan")
- if not isinstance(plan, dict):
- return False
- return any(
- isinstance(runtime, dict) and _sha_mismatch(runtime.get("code_sha"))
- for runtime in plan.get("runtimes") or []
- )
- _SUPERVISED_SERVE_BACKENDS = frozenset(
- {"manual-serve", "desktop", "desktop-ssh", "systemd", "launchd", "windows-service", "service"}
- )
- # Backends whose supervisor restarts the process without any updater bookkeeping. ``manual-serve``
- # is excluded: it owes a durable handoff (``defer_manual_serve``) before it stops counting.
- # The inventory writer classifies a serve/dashboard row as launchd, desktop, desktop-ssh, systemd (a unit's
- # MainPID, #141021) or manual-serve (``update_inventory._collect_ledger_runtimes``); ``windows-service`` and
- # ``service`` mirror ``_SUPERVISED_SERVE_BACKENDS`` for parity only — they are set for gateway rows alone.
- _SUPERVISOR_OWNED_SERVE_BACKENDS = _SUPERVISED_SERVE_BACKENDS - {"manual-serve"}
- def _receipt_owed_gateways(receipt: dict, pending_manual: list[dict]) -> set[tuple[str, str]] | None:
- """Pure coverage classification after manual retention of this receipt snapshot.
- Empty means this receipt owes no gateways, not that an independent marker owes none. Unknown identities, unclassified serve backends and failed manual transfers make coverage unverified.
- """
- plan = receipt.get("plan") or {}
- entries: list[tuple[object, str | None]] = [(entry, None) for entry in plan.get("runtimes") or []]
- entries.extend((entry, None) for entry in receipt.get("pending_manual_serves") or [])
- entries.extend((entry, None) for entry in receipt.get("carried_manual_serves") or [])
- entries.extend((entry, "gateway") for entry in receipt.get("fleet") or [])
- owed: set[tuple[str, str]] = set()
- unverified = False
- for entry, default_kind in entries:
- if not isinstance(entry, dict):
- unverified = True
- continue
- kind = entry.get("kind", default_kind)
- profile = entry.get("profile")
- # A serve/dashboard row is outside the gateway matrix's evidence, not evidence against the
- # gateways it does cover: a supervised backend (desktop, systemd, launchd) is its
- # supervisor's to restart, and a manual-serve row outside the retention list is the serve
- # obligation mechanism's — a host running a dashboard carries such a row in every receipt,
- # and a blanket veto made the gateway warning permanently undischargeable there (#115090).
- # Only an unclassified backend or a failed manual transfer still makes coverage unverified.
- if kind in ("serve", "dashboard") and entry.get("supervisor") in _SUPERVISED_SERVE_BACKENDS and entry not in pending_manual:
- continue
- if kind != "gateway" or not profile or profile == "unknown":
- unverified = True
- continue
- owed.add((kind, profile))
- return None if unverified else owed
- def _fleet_covered_gateways(fleet: list) -> set[tuple[str, str]] | None:
- """``(kind, profile)`` identities the live rows vouch for; ``None`` when any row is unidentified.
- A multiplexer's row carries the ``served_profiles`` its runtime status records (``_fleet_row``
- keeps the field only when well-formed), so one live process covers every profile it serves.
- """
- covered: set[tuple[str, str]] = set()
- for row in fleet:
- profile = row.get("profile") if isinstance(row, dict) else None
- if not profile or profile == "unknown":
- return None # unidentified runtime: the matrix cannot vouch for it
- covered.add(("gateway", profile))
- covered.update(("gateway", served) for served in row.get("served_profiles") or [])
- return covered
- def _live_fleet_covers_receipt(expected_sha: str | None, receipt: dict, owed: set[tuple[str, str]] | None, *, accept_states: tuple = ("current",)) -> bool:
- """Require a successor at the expected SHA for every owed gateway identity."""
- if not expected_sha:
- return False
- from hermes_cli.update_receipt import collect_fleet_versions, row_is_external
- try:
- if owed is None:
- return False
- if not owed:
- return bool((receipt.get("plan") or {}).get("runtimes"))
- fleet = collect_fleet_versions()
- # State labels are checkout-relative; completed restarts may accept stale rows at the pulled SHA.
- if not fleet or any(
- row.get("state") not in accept_states or row.get("code_sha") != expected_sha
- for row in fleet if not row_is_external(row)
- ):
- return False
- covered = _fleet_covered_gateways(fleet)
- return covered is not None and owed <= covered
- except Exception as exc:
- logger.debug("Could not reconcile pending fleet identities: %s", exc)
- return False
- # A gateway the restart phase stopped without knowing its profile: no live row can be matched to it,
- # so only a live, current gateway of this checkout settles it, never an empty host (""=no profile).
- _UNMAPPED_GATEWAY = ("gateway", "")
- def _marker_owed_gateways(inventory: object) -> set[tuple[str, str]] | None:
- """The ``("gateway", profile)`` set a marker's inventory owes; None when it recorded none.
- Raises ValueError for a malformed or unsupported inventory, which keeps the marker.
- """
- from hermes_cli.update_cmd_fleet_gatewayless import runtime_outside_gateway_evidence
- if inventory is None:
- return None
- if not isinstance(inventory, dict) or inventory.get("version") != 1:
- raise ValueError("unsupported fleet-restart inventory")
- runtimes = inventory.get("runtimes")
- if not isinstance(runtimes, list):
- raise ValueError("fleet-restart inventory has no runtime list")
- owed: set[tuple[str, str]] = set()
- for runtime in runtimes:
- if not isinstance(runtime, dict):
- raise ValueError("fleet-restart inventory row is not an object")
- if runtime_outside_gateway_evidence(runtime):
- continue
- if runtime.get("kind") == "gateway" and runtime.get("stopped_unmapped"):
- owed.add(_UNMAPPED_GATEWAY)
- continue
- profile = runtime.get("profile")
- if runtime.get("kind") != "gateway" or not isinstance(profile, str) or not profile.strip() or profile == "unknown":
- raise ValueError("fleet-restart inventory row is not an identified gateway")
- owed.add(("gateway", profile))
- return owed
- def _discharge_gatewayless_marker(checkout_sha: str, expected_sha: str) -> bool:
- """Settle an inventory-less marker on a host with no live gateway (#118742).
- Only when the host itself shows nothing the update could still owe a restart to, and HEAD
- still holds the code it pulled.
- """
- from hermes_cli.update_cmd_fleet_checkout import checkout_contains
- from hermes_cli.update_cmd_fleet_gatewayless import host_owes_no_gateway_restart
- try:
- gatewayless = (checkout_sha == expected_sha or checkout_contains(expected_sha)) and host_owes_no_gateway_restart()
- except Exception as exc:
- logger.debug("Gateway-less host probe failed; keeping fleet-restart-pending marker: %s", exc)
- return False
- if not gatewayless:
- return False
- _clear_fleet_restart_pending_marker()
- logger.debug("Fleet-restart-pending marker discharged: host runs no gateway at %s", checkout_sha[:10])
- return True
- def _marker_only_restart_obsolete() -> bool:
- """Settle only the inventory stored with this marker's target SHA.
- Historical receipts cannot narrow this obligation. Malformed or unsupported inventories stay
- fail-closed; empty discovery never proves a stopped gateway recovered. Two shapes record no
- obligation and settle without one: an explicit empty inventory (a pull that found no gateway,
- #115311) clears outright, and an inventory-less marker (the pre-inventory writer, or a tail
- that died before its inventory was recorded, #115638) clears once every live gateway is
- current on the checkout — there is no recorded owed set, so the fleet running the code on disk
- is the whole of the evidence the marker's warning can be about, even after HEAD moved past
- ``expected_sha`` by an out-of-band pull — and so does an inventory-less record armed with no
- SHA at all (a no-op update whose head capture failed, #125952): with no owed set and no SHA,
- the checkout is the only code it can be held to. With no live gateway at all, the inventory-less marker
- asks the host instead (``update_cmd_fleet_gatewayless``): it clears when no profile left a
- gateway that should be running and every live runtime is supervisor-owned or handed off, so a
- Desktop-only install stops failing every later update (#118742).
- A serve/dashboard row whose supervisor owns the restart (Desktop backend, systemd/launchd
- unit, Windows service) is outside the gateway matrix's evidence, not evidence against it —
- the same boundary ``_receipt_owed_gateways`` draws for receipts (#115090) and the restart
- phase draws for the Desktop backend (#111494). Counting it made the warning permanently
- undischargeable on every host that runs a dashboard. A manual-serve row still needs its
- durable handoff (``defer_manual_serve``), and an unclassified backend stays fail-closed.
- Discharging here strands nobody: the same row is still accounted at update time by
- ``update_inventory.report_unaccounted_runtimes``, which prints it and records an owed restart when the restart
- phase never touched it — this marker only stops re-warning about it on every later startup.
- """
- from hermes_cli.update_cmd_fleet_checkout import checkout_contains
- try:
- fields = _obligation_fields()
- if fields is None:
- return False
- expected_sha = fields.get("expected_sha", "").strip()
- owed = _marker_owed_gateways(json.loads(fields.get("inventory", "null")))
- except (OSError, UnicodeError, ValueError):
- return False
- if owed is not None and not owed:
- # A pull that recorded no gateway runtime owes no restart; clearing avoids the
- # stuck "Fleet restart incomplete" loop on Desktop-hosted (no-service) installs.
- _clear_fleet_restart_pending_marker()
- logger.debug("Fleet-restart-pending marker discharged: no gateway obligation recorded")
- return True
- if owed is not None and not expected_sha:
- return False # an inventoried obligation without its SHA can never be proven
- checkout_sha = _current_checkout_sha()
- if owed is not None and checkout_sha != expected_sha and not checkout_contains(expected_sha):
- return False # a newer pull moved HEAD; it owns a fresh obligation
- # HEAD may sit past ``expected_sha`` by a carried local commit (a cherry-picked hotfix) that no
- # pull made and no fresh obligation covers; the fleet is held to the code it actually runs, which
- # is what an equality gate on ``expected_sha`` could never discharge (#119367).
- target_sha = checkout_sha
- if not target_sha:
- return False
- try:
- from hermes_cli.update_receipt import collect_fleet_versions, row_is_external
- fleet = collect_fleet_versions()
- except Exception as exc:
- logger.debug("Fleet probe failed; keeping fleet-restart-pending marker: %s", exc)
- return False
- if not fleet or (not expected_sha and all(row_is_external(row) for row in fleet)):
- if owed is not None or not expected_sha:
- # Absence cannot prove recovery of the recorded inventory / unnamed code; a fleet
- # whose every row serves ANOTHER checkout root is absence too, not evidence.
- return False
- return _discharge_gatewayless_marker(checkout_sha, expected_sha)
- covered = _fleet_covered_gateways(fleet)
- if covered is None:
- return False # unidentified runtime: the matrix cannot vouch for it
- for row in fleet:
- if row_is_external(row):
- continue
- if row.get("state") != "current" or str(row.get("code_sha")) != target_sha:
- return False # stale / down / unknown-identity row still owes the restart
- if owed is not None and not owed - {_UNMAPPED_GATEWAY} <= covered:
- return False # A gateway this marker owns is absent (down) or unidentifiable.
- if owed and _UNMAPPED_GATEWAY in owed and not (_fleet_covered_gateways(
- [row for row in fleet if not row_is_external(row)]) or set()) - owed:
- # An unmapped stopped gateway has no identity to match, so only a local gateway that no
- # mapped owed one explains can be its successor; another checkout's gateway never is.
- return False
- _clear_fleet_restart_pending_marker()
- logger.debug(
- "Fleet-restart-pending marker discharged: %d gateway(s) already serve %s",
- len(fleet), target_sha[:10],
- )
- return True
- def _receipt_restart_phase_completed(receipt: dict) -> str | None:
- """Return the pulled SHA when the restart phase completed, even if a later step failed."""
- gateway_restart = receipt.get("gateway_restart")
- if not isinstance(gateway_restart, dict) or not gateway_restart:
- return None
- if gateway_restart.get("incomplete") or gateway_restart.get("phase_error"):
- return None
- post_sha = (receipt.get("post_update") or {}).get("sha")
- return str(post_sha) if post_sha else None
- def _pending_fleet_restart_needed(*, receipt: dict | None = None, pending_manual: list[dict] | None = None) -> bool:
- """Require identity-matched gateways at checkout HEAD for update catch-up."""
- from hermes_cli.update_cmd import _current_checkout_sha
- from hermes_cli.update_receipt import read_latest_receipt
- from hermes_cli.update_serve_obligations import retain_receipt_manual_serves
- if receipt is None:
- receipt = read_latest_receipt() or {}
- if pending_manual is None:
- pending_manual = retain_receipt_manual_serves(receipt)
- # The HOST obligation owns its inventory; latest.json can belong to an older update.
- if _fleet_restart_obligation_armed():
- return not _marker_only_restart_obsolete()
- owed = _receipt_owed_gateways(receipt, pending_manual)
- if not _receipt_reports_stale_runtime(receipt):
- return False
- return not _live_fleet_covers_receipt(_current_checkout_sha(), receipt, owed)
- def _update_owes_fleet_restart(*, receipt: dict | None = None, pending_manual: list[dict] | None = None) -> bool:
- """Hold a completed restart to the code it pulled, not a later checkout HEAD."""
- from hermes_cli.update_cmd import _current_checkout_sha
- from hermes_cli.update_receipt import read_latest_receipt
- from hermes_cli.update_serve_obligations import retain_receipt_manual_serves
- if receipt is None:
- receipt = read_latest_receipt() or {}
- if pending_manual is None:
- pending_manual = retain_receipt_manual_serves(receipt)
- # A completed older receipt cannot discharge an independent host obligation's inventory.
- if _fleet_restart_obligation_armed():
- return not _marker_only_restart_obsolete()
- owed = _receipt_owed_gateways(receipt, pending_manual)
- if not _receipt_reports_stale_runtime(receipt):
- return False
- restarted_to = _receipt_restart_phase_completed(receipt)
- # A fleet an operator has since restarted onto a moved checkout (``hermes gateway restart`` —
- # the remedy this warning names) has nothing of the update left to owe either.
- if restarted_to and _live_fleet_covers_receipt(restarted_to, receipt, owed, accept_states=("current", "stale")):
- return False
- return not _live_fleet_covers_receipt(_current_checkout_sha(), receipt, owed)
- def _warn_pending_fleet_restart(*, startup: bool = False) -> None:
- """Print the specific interrupted-update fleet-restart warning."""
- stream = sys.stderr if startup else sys.stdout
- print("⚠ A previous `hermes update` pulled new code but did not restart running gateways.", file=stream)
- print(" Gateways may still be serving pre-update modules (mixed sys.modules).", file=stream)
- if startup:
- print(" Run `hermes update` or `hermes gateway restart`.", file=stream)
- def _warn_pending_fleet_restart_on_startup() -> None:
- """Cheap CLI-startup hint. Never restarts; never raises."""
- from hermes_cli.update_receipt import read_latest_receipt
- from hermes_cli.update_serve_obligations import retain_receipt_manual_serves, warn_pending_manual_serves
- receipt = read_latest_receipt() or {}
- pending_manual = None
- with suppress(Exception):
- pending_manual = retain_receipt_manual_serves(receipt)
- with suppress(Exception):
- if _update_owes_fleet_restart(receipt=receipt, pending_manual=pending_manual):
- _warn_pending_fleet_restart(startup=True)
- with suppress(Exception):
- warn_pending_manual_serves(startup=True, pending_manual=pending_manual)
- def _systemd_gateway_unit_listings(on_list_timeout=None):
- """Yield ``(scope, scope_cmd, list-units CompletedProcess)`` per systemd scope that answered.
- A missing systemctl skips the scope silently; a listing timeout skips it after
- ``on_list_timeout(scope, exc)`` (when given) so the other scope is still processed.
- """
- for scope, scope_cmd in _SYSTEMD_SCOPES:
- try:
- result = _systemctl(scope_cmd + _LIST_GATEWAY_UNITS, timeout=10)
- except FileNotFoundError:
- continue
- except subprocess.TimeoutExpired as exc:
- if on_list_timeout is not None:
- on_list_timeout(scope, exc)
- continue
- yield scope, scope_cmd, result
- def _needs_sudo(scope: str) -> bool:
- return (
- scope == "system"
- and hasattr(os, "geteuid")
- and os.geteuid() != 0 # windows-footgun: ok — systemd path, Linux-only
- )
- def _unit_main_pid(scope_cmd: list, svc_name: str) -> int:
- """Live ``MainPID`` of a unit; ``0`` when inactive, unprivileged or unreadable.
- Property reads need no manage-units privileges, and an unreadable PID is never collapsed:
- identity that cannot be proved keeps its own restart.
- """
- try:
- result = _systemctl(list(scope_cmd) + ["show", svc_name, "--property=MainPID", "--value"], timeout=10)
- except (OSError, subprocess.TimeoutExpired):
- return 0
- if getattr(result, "returncode", 1) != 0:
- return 0
- try:
- return int((getattr(result, "stdout", "") or "").strip() or 0)
- except ValueError:
- return 0
- def _restart_systemd_gateway_units_best_effort(failed: list, listings) -> None:
- """Restart every hermes-gateway/serve unit ONCE PER LIVE HOST PROCESS.
- One host runs one multiplexing gateway, so leftover per-profile units
- (``hermes-gateway-<profile>.service``) all point at the SAME live ``MainPID``; restarting
- each in turn restarts the host gateway N times — a self-inflicted N-fold outage triggered
- by one update. Units that share a live main PID are collapsed to one representative and the
- others are named as LEGACY units to migrate, never silently dropped.
- """
- from hermes_cli.update_host_obligation import collapse_units_to_host_processes
- answered = set()
- targets: dict[str, tuple[str, list, str]] = {} # "<scope>/<unit>" -> (scope, scope_cmd, unit)
- for scope, scope_cmd, result in listings:
- answered.add(scope)
- if result.returncode != 0:
- failed.append(f"systemd-{scope} (listing failed)")
- continue
- _for_each_systemd_gateway_unit(
- result.stdout,
- process_unit=lambda svc_name, _scope=scope, _cmd=scope_cmd: targets.setdefault(
- f"{_scope}/{svc_name}", (_scope, _cmd, svc_name)),
- on_unit_timeout=lambda svc_name, exc: failed.append(svc_name),
- )
- keys = list(targets)
- covered: dict[str, str] = {}
- if len(keys) > 1:
- # Only worth a `systemctl show` round when several units could be one process.
- keys, covered = collapse_units_to_host_processes(
- keys, lambda key: _unit_main_pid(targets[key][1], targets[key][2]))
- for unit_key, owner_key in covered.items():
- print(
- f" • {unit_key} is a legacy per-profile unit sharing one host gateway process with "
- f"{owner_key}; restarting it again would restart that process twice. Fold the units "
- "together with: hermes gateway migrate"
- )
- for key in keys:
- scope, scope_cmd, svc_name = targets[key]
- if not _systemd_unit_owned_by_update(scope_cmd, svc_name):
- continue
- manage_cmd = list(scope_cmd) + ["--no-ask-password"]
- if _needs_sudo(scope):
- manage_cmd = ["sudo", "-n"] + manage_cmd
- try:
- result = _systemctl_reset_and_restart(manage_cmd, svc_name, scope_cmd=scope_cmd)
- if result.returncode != 0 or not _wait_for_service_active(scope_cmd, svc_name):
- failed.append(svc_name)
- except subprocess.TimeoutExpired:
- failed.append(svc_name)
- # A timeout or missing executable is not an empty scope.
- failed.extend(f"systemd-{scope} (listing unavailable)" for scope, _ in _SYSTEMD_SCOPES if scope not in answered)
- def _live_fleet_current_rows() -> list[dict] | None:
- """The fleet matrix when the probe finds at least one gateway and every row is ``current``
- at the checkout SHA (identity known); ``None`` on any unknown/stale/down row or a failed
- probe (restart)."""
- checkout_sha = _current_checkout_sha()
- if not checkout_sha:
- return None
- try:
- from hermes_cli.update_receipt import collect_fleet_versions
- fleet = collect_fleet_versions()
- except Exception as exc:
- logger.debug("Pending fleet restart: fleet probe failed: %s", exc)
- return None
- if not fleet or _fleet_covered_gateways(fleet) is None:
- return None
- if all(row.get("state") == "current" and str(row.get("code_sha")) == checkout_sha for row in fleet):
- return fleet
- return None
- def _restart_identity_sha() -> str:
- """The SHA a completed host restart is stamped with; ``""`` when nothing names the code.
- ``_current_checkout_sha()`` is ``None`` on every non-git install (zip, pip, Docker), and an
- empty stamp can never match, so the per-host restart-once guard would be inert exactly on the
- installs it exists for: each profile's ``hermes update`` would re-kill the one shared
- multiplexer. The obligation's own ``expected_sha`` — else the receipt's post-update identity —
- names the same pulled code.
- """
- sha = _current_checkout_sha()
- if sha:
- return str(sha)
- sha = ((_obligation_fields() or {}).get("expected_sha") or "").strip()
- if sha:
- return sha
- with suppress(Exception):
- from hermes_cli.update_receipt import read_latest_receipt
- post_update = (read_latest_receipt() or {}).get("post_update")
- if isinstance(post_update, dict):
- return str(post_update.get("sha") or "")
- return ""
- def _fleet_restart_skip_reason(plan) -> str | None:
- """Why the completion tail may leave the fleet alone, or ``None`` when a restart is owed.
- Every route (pulled, already-current, ZIP) now finishes through the same completion
- tail, so the guards the old catch-up path carried live here: one host runs ONE
- multiplexing gateway, so a second profile's ``hermes update`` attaches to the restart the
- first one already stamped (#95294), and a fleet already serving the checkout code (a
- no-op update, a manual ``hermes gateway restart`` seconds ago) is not re-killed (#117051).
- The second guard needs BOTH the pre-update plan and the live probe: the live matrix only
- lists gateways, so a planned ``serve`` still on pre-update code (or any runtime without a
- stamped identity) keeps the restart — the reconciliation there is what surfaces it.
- """
- from hermes_cli.update_host_obligation import host_restart_already_completed
- checkout_sha = _restart_identity_sha()
- if host_restart_already_completed(checkout_sha):
- return "this host's gateway was already restarted for this update"
- if (checkout_sha and plan is not None and plan.runtimes
- and all(str(runtime.code_sha) == checkout_sha for runtime in plan.runtimes)
- and _live_fleet_current_rows() is not None):
- return "every running gateway already serves the checkout code"
- return None
- def _run_pending_fleet_restart() -> bool:
- """Historical retry hook; new retries use the ordinary completion owner."""
- from hermes_cli._old_updater import stop_for_relaunch
- stop_for_relaunch(incomplete=True)
- def _systemctl(cmd: list, *, timeout: float):
- """Run a systemctl (or sudo systemctl) invocation, capturing utf-8 text with a timeout."""
- return subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout, check=False)
- def _systemctl_reset_and_restart(manage_cmd: list, svc_name: str, *, scope_cmd: list | None = None):
- """``reset-failed`` then ``restart``: a unit parked in failed state by systemd's own
- auto-restart can wedge a plain ``restart`` against RestartSec backoff and stay dead."""
- # Property reads need no manage-units privileges: narrow sudoers may permit
- # restart/reset-failed but deny show. Keep the same user/system manager scope.
- from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job, systemd_restart_timeout
- scope_cmd = scope_cmd if scope_cmd is not None else manage_cmd
- timeout = systemd_restart_timeout(scope_cmd, svc_name)
- _systemctl(manage_cmd + ["reset-failed", svc_name], timeout=10)
- return run_unit_job(manage_cmd, scope_cmd, "restart", svc_name, timeout=timeout)
- def _systemd_unit_owned_by_update(scope_cmd: list, svc_name: str) -> bool:
- """Gate a unit restart on the unit's home being one this update owns (#93349).
- ``hermes-gateway*`` is an account-wide namespace: a second install's ``hermes update`` used
- to drain and restart the account's real ``hermes-gateway.service`` because the unit was
- listed, not because it ran the updated code. Foreign or unreadable ownership prints a notice
- and leaves the unit alone; it is not a failed restart.
- """
- from hermes_cli.update_fleet_scope import announce_skipped_runtime, systemd_unit_hermes_home, home_in_update_scope
- home = systemd_unit_hermes_home(scope_cmd, svc_name)
- if home is not None and home_in_update_scope(home):
- return True
- announce_skipped_runtime("systemd unit", svc_name, home)
- return False
- def _scoped_manual_gateway_pids(pids, *, keep=(), quiet: bool = False) -> list[int]:
- """*pids* whose live home this update owns (plus *keep*, PIDs already mapped to this
- install's profile PID files); every other gateway process is named and left running."""
- from hermes_cli.update_fleet_scope import announce_skipped_runtime, partition_gateway_pids_by_scope
- keep = set(keep)
- owned, foreign = partition_gateway_pids_by_scope([pid for pid in pids if pid not in keep])
- if not quiet:
- for pid, home in foreign:
- announce_skipped_runtime("gateway process", f"PID {pid}", home)
- return [pid for pid in pids if pid in keep or pid in owned]
- def _is_serve_or_dashboard_unit(unit: str) -> bool:
- """``hermes-serve*``/``hermes-dashboard*`` (``.service``) by the same exact-or-hyphenated shape."""
- return (
- unit == "hermes-serve.service"
- or unit.startswith("hermes-serve-")
- # #125297: ``hermes-dashboard*`` units are systemd-supervised dashboard backends — the
- # same fleet this pass restarts. Leaving them out meant a successful update reported
- # the dashboard ``deferred`` (still on pre-update code) while nothing ever restarted it.
- or unit == "hermes-dashboard.service"
- or unit.startswith("hermes-dashboard-")
- )
- def _is_hermes_gateway_unit(unit: str) -> bool:
- """Exact base unit or hyphenated profile family only: ``startswith("hermes-serve")``
- would accept ``hermes-server.service``."""
- return (
- # list-units is already pattern-filtered, but keep the name gate so a stray non-gateway/serve line
- # cannot enter the restart path. See #83595.
- unit == "hermes-gateway.service"
- or unit.startswith("hermes-gateway-")
- or _is_serve_or_dashboard_unit(unit)
- )
- def _for_each_systemd_gateway_unit(list_units_stdout: str, *, process_unit, on_unit_timeout) -> None:
- """Process each hermes-gateway*/hermes-serve* unit from ``systemctl list-units``.
- ``TimeoutExpired`` from ``process_unit`` is isolated per unit via ``on_unit_timeout``
- so one wedged systemctl call cannot abort the rest of the fleet.
- See #68523.
- """
- for line in (list_units_stdout or "").strip().splitlines():
- parts = line.split()
- if not parts:
- continue
- unit = parts[0]
- if not unit.endswith(".service") or not _is_hermes_gateway_unit(unit):
- continue
- svc_name = unit.removesuffix(".service")
- try:
- process_unit(svc_name)
- except subprocess.TimeoutExpired as exc:
- on_unit_timeout(svc_name, exc)
- def _service_unit_supports_graceful_sigusr1_restart(svc_name: str) -> bool:
- """Whether *svc_name* wires SIGUSR1 to a graceful drain-then-restart.
- Only ``hermes-gateway*`` runs ``gateway/run.py`` (the handler); SIGUSR1 would just
- kill ``hermes-serve*`` and burn the drain budget, so those go straight to the blunt
- restart. Same exact/hyphenated shape as ``_for_each_systemd_gateway_unit`` so a
- near-prefix unit like ``hermes-gatewayd`` is never signalled.
- See #83438.
- """
- return svc_name == "hermes-gateway" or svc_name.startswith("hermes-gateway-")
- def _warn_incomplete_gateway_fleet_restart(failed_units: list) -> None:
- """Print an explicit incomplete-update warning for unrestarted units."""
- from hermes_cli.gateway import is_macos
- if not failed_units:
- return
- ordered = list(dict.fromkeys(failed_units)) # de-dup, discovery order
- print()
- print("⚠ Update incomplete — some units were not restarted:")
- for name in ordered:
- print(f" - {name}")
- if is_macos():
- # A label lands here when launchd wasn't supervising a live process after
- # the restart — likely deregistered, which `launchctl kickstart` can't revive.
- # See #88848.
- print(" Listed services may be deregistered from launchd, or still")
- print(" running pre-update code (mixed sys.modules). Recover with:")
- print(" hermes gateway status")
- print(" launchctl list | grep <label>")
- print(" launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/<label>.plist")
- return
- print(" Skipped units may still be running pre-update code (mixed")
- print(" sys.modules). Restart them manually, then verify:")
- print(" hermes gateway status")
- if any(not name.startswith("ai.hermes.") for name in ordered):
- print(" systemctl --user restart <unit> # user-scope")
- print(" sudo systemctl restart <unit> # system-scope")
- if any(name.startswith("ai.hermes.") for name in ordered):
- print(" launchctl kickstart -k gui/$UID/<label> # macOS (or user/$UID)")
- def _restart_launchd_gateway_after_update(
- *, supervision_verify: bool = True, self_restart_pending: set | None = None,
- ) -> tuple[list, list]:
- """Restart the invoking profile's launchd gateway after an update.
- No ``launchctl list`` gating: a booted-out job (plist present, definition
- deregistered) fails it, and it can exit non-zero while the job is alive — gating on it
- silently skipped the restart yet printed "Update complete!". When the plist exists
- ``launchd_restart()`` always runs; every failure path is loud with a manual recovery
- command. Returns ``(restarted_labels, failed_labels)``; with ``supervision_verify``
- success also requires a fresh supervised PID ("the call returned" is not "supervised").
- 74973 (salvage #75021 by @jeff-mettel): the restart used to be gated on ``launchctl list <label>``
- exiting 0. A *booted-out* job — plist present, definition deregistered from launchd (crashed helper,
- manual bootout, failed prior update) — fails that check, so the whole branch silently skipped: no
- restart, no message, ``KeepAlive`` unable to revive a definition launchd no longer knows, and the update
- still printed "Update complete!".
- See #88848.
- """
- from hermes_cli.gateway import (
- get_launchd_label, get_launchd_plist_path, launchd_restart, wait_for_launchd_gateway_supervision,
- _is_pid_ancestor_of_current_process, _launchctl_supervised_pid,
- )
- current_label = get_launchd_label()
- old_pid = None
- try:
- if not get_launchd_plist_path().exists():
- return [], [] # not a launchd install — nothing to do or warn
- # Snapshot BEFORE the restart: "supervising some pid" was true before too, so only a pid that
- # actually changed distinguishes a restart from a no-op (the sibling loop's contract). Read-only
- # and verification-only — the restart itself is never gated on `launchctl list` (#74973).
- old_pid = _launchctl_supervised_pid(current_label) if supervision_verify else None
- try:
- launchd_restart()
- except subprocess.CalledProcessError as e:
- stderr = (getattr(e, "stderr", "") or "").strip()
- print(
- f" ⚠ Gateway restart failed: {stderr}\n"
- " The gateway may be DOWN on pre-update code. "
- "Recover manually: hermes gateway restart"
- )
- return [], [current_label]
- except (FileNotFoundError, subprocess.TimeoutExpired) as e:
- # A plist exists, so a gateway is SUPPOSED to be supervised; a broken/wedged
- # launchctl is not proof nothing needs restarting. Count it, tell the operator.
- print(
- # The old code `pass`ed here (#74973's second silent variant); count it and tell the operator.
- " ⚠ Could not restart the gateway "
- f"({e.__class__.__name__}: {e}).\n"
- " Recover manually: hermes gateway restart"
- )
- return [], [current_label]
- if not supervision_verify:
- return [current_label], []
- if old_pid is not None and _is_pid_ancestor_of_current_process(old_pid):
- # launchd_restart() handed the restart to the gateway this updater runs INSIDE (cron job in
- # the gateway tree, #100179): it exits only after this process does, so no fresh supervised
- # pid can appear while we wait. Record it as pending for the fleet matrix (#119597).
- if self_restart_pending is not None:
- self_restart_pending.add(old_pid)
- return [current_label], []
- # launchd_restart() returning only means "restart REQUESTED" (async). A helper dying
- # before first bootstrap, or a bootstrap exiting 0 without registering (macOS 26.6.1),
- # would otherwise reach "Update complete!" unsupervised. Verified domain-agnostically:
- # domain locate fails on macOS-26 per-user domains.
- # launchd_restart() returning is only "restart REQUESTED" — the self-restart branch hands work to the
- # running gateway, a plist reload to a detached helper; both asynchronous. See #88848.
- if wait_for_launchd_gateway_supervision(label=current_label, old_pid=old_pid):
- return [current_label], []
- print(
- f" ✗ {current_label} restarted but launchd is not supervising a new process for it.\n"
- " Check logs, then: hermes gateway restart"
- )
- return [], [current_label]
- def _restart_macos_launchd_gateways(
- restarted_services: list, failed_or_stale_units: list, drain_budget: float, *, require_supervision: bool = False,
- self_restart_pending: set | None = None,
- ) -> None:
- """Restart every launchd-managed gateway after an update (macOS).
- The pull is shared across profiles, so every ``ai.hermes.gateway*`` LaunchAgent
- must reload it or siblings stay on pre-update ``sys.modules`` (systemd parity).
- Invoking profile uses ``launchd_restart()``; siblings get the same drain-first
- sequence with their domain (``gui/<uid>`` vs ``user/<uid>``) resolved per label so
- none is kickstarted in the wrong domain. ``TimeoutExpired`` is isolated per label.
- See #41403.
- The invoking profile keeps the existing ``launchd_restart()`` treatment (self-restart request → graceful
- drain → kickstart). ``subprocess.TimeoutExpired`` is isolated per label so one wedged launchctl call
- cannot leave the rest of the fleet on old code (#68523).
- """
- from hermes_cli.gateway import (
- get_launchd_label, get_launchd_plist_path, launchd_gateway_labels_for_install, legacy_launchd_labels_for_install,
- _graceful_restart_via_sigusr1, _launchd_kickstart,
- _locate_launchd_gateway_service, _wait_for_launchd_service_pid,
- )
- if require_supervision:
- listing = subprocess.run(["launchctl", "list"], capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=10, check=False)
- if listing.returncode != 0:
- failed_or_stale_units.append("launchd (listing failed)")
- return
- from hermes_cli.update_cmd_posix_pause import already_restarted
- current_label = get_launchd_label()
- resumed = already_restarted()["labels"] # paused for the update, restarted on the new code
- _restarted, _failed = ([], []) if current_label in resumed else _restart_launchd_gateway_after_update(
- supervision_verify=True, self_restart_pending=self_restart_pending)
- restarted_services.extend(_restarted)
- failed_or_stale_units.extend(_failed)
- derived_labels = launchd_gateway_labels_for_install()
- # Units labelled before the profile-name suffix scheme (ai.hermes.gateway-<hash>) are invisible
- # to the derivation; legacy_launchd_labels_for_install() credits one only when its plist is
- # provably this install's, so the #41403 boundary (never touch another install's fleet) holds.
- # See #115254.
- legacy_labels = legacy_launchd_labels_for_install(exclude=set(derived_labels) | {current_label})
- if legacy_labels:
- print(f" ↻ legacy-labelled units of this install join the restart: {', '.join(legacy_labels)}")
- from hermes_cli.update_fleet_scope import describe_skipped_runtime, launchd_label_foreign_home
- for label in derived_labels + legacy_labels:
- if label == current_label or label in resumed:
- continue
- # Labels are account-global: root B's default profile derives the same bare label root A
- # installed. A plist pinning a foreign HERMES_HOME is another install's job (#93349).
- if (foreign_home := launchd_label_foreign_home(label)) is not None:
- print(describe_skipped_runtime("launchd job", label, foreign_home))
- continue
- try:
- # Locate = liveness + domain in one probe; kickstart and fresh-PID checks
- # reuse that domain so a sibling is never probed in one and restarted in another.
- domain, old_pid = _locate_launchd_gateway_service(label)
- if domain is None:
- if require_supervision and get_launchd_plist_path().with_name(f"{label}.plist").exists():
- failed_or_stale_units.append(label)
- continue # A profile without an installed job has no restart target.
- graceful_ok = False
- if old_pid is not None and old_pid > 0:
- print(f" → {label}: draining (up to {drain_budget:.0f}s)...")
- from hermes_cli.update_cmd_drain_report import drain_progress_reporter
- graceful_ok = _graceful_restart_via_sigusr1(
- old_pid, drain_timeout=drain_budget,
- on_progress=drain_progress_reporter(_gateway_home_for_pid(old_pid), budget_s=drain_budget))
- if graceful_ok and _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=10.0, domain=domain):
- # KeepAlive already respawned it on new code — a kickstart would kill it.
- restarted_services.append(label)
- continue
- try:
- _launchd_kickstart(label, domain)
- except subprocess.CalledProcessError as e:
- stderr = (getattr(e, "stderr", "") or "").strip()
- failed_or_stale_units.append(label)
- print(
- f" ⚠ Failed to restart {label}: {stderr}\n"
- f" Recover manually: launchctl kickstart -k {domain}/{label}"
- )
- continue
- if _wait_for_launchd_service_pid(label, old_pid=old_pid, timeout=15.0, domain=domain):
- restarted_services.append(label)
- else:
- failed_or_stale_units.append(label)
- print(
- f" ✗ {label} failed to come back after restart.\n"
- f" Check logs, then: launchctl kickstart -k {domain}/{label}"
- )
- except subprocess.TimeoutExpired:
- failed_or_stale_units.append(label)
- print(f" ⚠ launchctl timed out restarting {label}; continuing with remaining gateways")
- def _surviving_gateway_pids_after_failed_restart():
- """Best-effort PIDs of gateways still running after the restart phase died.
- ``None`` when undeterminable (notably ``hermes_cli.gateway`` no longer importing
- under the replaced checkout). Callers treat ``None`` and non-empty as "assume
- stale"; only a positive empty result proves nothing needs restarting.
- """
- try:
- from hermes_cli.gateway import find_gateway_pids
- return _scoped_manual_gateway_pids(find_gateway_pids(all_profiles=True), quiet=True)
- except Exception as exc: # pragma: no cover - defensive
- logger.debug("Could not probe for surviving gateways after update: %s", exc)
- return None
- _MANUAL_GATEWAY_SKIP_REASON = (
- "manual gateway has no supervisor relaunch authority; left running for explicit operator restart"
- )
- _DESKTOP_SERVE_SKIP_REASON = (
- "desktop app owns and respawns this serve backend;"
- " the recovery pass must not restart it out from under its supervisor"
- )
- # NOT a claim that no supervisor exists: a unit's MainPID row is classified "systemd" (#141021),
- # and "manual-serve" still covers a process inside some unit's cgroup it is not MainPID of, or
- # unreadable ownership. Unit-backed serves are recovered by the fresh child's systemd pass;
- # survivors reported by _surviving_pre_update_serve_runtimes.
- _SERVE_SKIP_REASON = (
- "no per-profile relaunch command reaches a serve/dashboard runtime; recovered by the fresh"
- " systemd unit pass when a hermes-serve*/hermes-dashboard* unit owns it, else left running for explicit"
- " operator restart"
- )
- # A launchd-owned backend (#116503): the fresh child has no per-label kickstart for serve/dashboard
- # jobs, but the post-update dashboard cleanup pass kickstarts the loaded job — never a detached
- # argv respawn, which would fight the job's own KeepAlive.
- _LAUNCHD_SERVE_SKIP_REASON = (
- "launchd job owns this backend; the post-update dashboard cleanup kickstarts the job through"
- " launchd, never a detached argv respawn that would fight its KeepAlive"
- )
- def _gateway_recovery_partition(plan, *, skip_profiles: set[str] | None = None) -> tuple[dict[str, str], list[dict]]:
- """Partition pre-update runtimes into fresh-restart candidates and skips.
- Uses only the pre-checkout inventory: re-importing ``hermes_cli.gateway`` in the
- failing interpreter is what raises the original ``ImportError``. Returns
- ``(candidates, skipped)``: profile → supervisor for supervised gateways the fresh
- process may restart; every other inventoried runtime with an explicit reason so
- nothing vanishes silently. Skipped serve/dashboard is NOT unrecoverable: the fresh
- child's ``hermes-serve*`` systemd pass enumerates units from systemd; leftovers are
- caught by :func:`_surviving_pre_update_serve_runtimes`.
- """
- skip_profiles = skip_profiles or set()
- candidates: dict[str, str] = {}
- skipped: list[dict] = []
- with _best_effort('Could not prepare fresh gateway restart profiles: %s'):
- for runtime in getattr(plan, "runtimes", ()) or ():
- kind = getattr(runtime, "kind", None)
- profile = getattr(runtime, "profile", None)
- supervisor = getattr(runtime, "supervisor", None)
- if not isinstance(profile, str) or not profile:
- continue
- if kind == "gateway":
- if profile in skip_profiles:
- continue
- if supervisor in _FRESH_RESTART_SUPERVISORS:
- candidates.setdefault(profile, str(supervisor))
- continue
- reason = _MANUAL_GATEWAY_SKIP_REASON
- elif kind in ("serve", "dashboard"):
- from hermes_cli.update_inventory import CLIENT_OWNED_SERVE_SUPERVISORS
- if supervisor in CLIENT_OWNED_SERVE_SUPERVISORS:
- reason = _DESKTOP_SERVE_SKIP_REASON
- elif supervisor == "launchd":
- reason = _LAUNCHD_SERVE_SKIP_REASON
- else:
- reason = _SERVE_SKIP_REASON
- else:
- continue
- skipped.append({"profile": profile, "kind": str(kind), "supervisor": str(supervisor), "reason": reason})
- return candidates, skipped
- def _warn_gateway_restart_phase_aborted(exc: BaseException, pids) -> None:
- """Print a recovery warning when the whole restart phase raised.
- Previously a blanket debug-logged ``except Exception`` erased every drain/restart
- line, so "Update complete!" exited 0 while the gateway kept serving pre-update
- modules and died on the next turn with an ImportError.
- Issue #78574: the gateway auto-restart phase was wrapped in a blanket ``except Exception`` that only
- logged at debug level, so an early failure (e.g. importing ``hermes_cli.gateway`` from the freshly
- pulled checkout) erased every drain/restart line from the update output.
- """
- print()
- print(f"⚠ Update incomplete — gateway auto-restart failed: {exc}")
- if pids:
- listed = ", ".join(str(pid) for pid in pids)
- print(f" Gateway process(es) still running pre-update code: {listed}")
- else:
- print(" Any gateway still running is serving pre-update code")
- print(" (mixed sys.modules) against the updated checkout.")
- print(" Restart it manually, then verify:")
- print(" hermes gateway restart")
- print(" hermes gateway status")
- def _drain_or_signal_gateway_for_update(
- pid: int, drain_budget: float, label: str, *, self_restart_pending: set | None = None,
- ) -> bool:
- """Three-way triage (shared by systemd and bare-process paths) for handing a
- running gateway over to new code. Returns True when signalled/stopped.
- 1. Gateway is an ancestor of this process (auto-update cron inside the gateway
- tree): waiting is circular (gateway waits on in-flight work → cron session
- waits on update → update waits on gateway) and the 1800s force-drain cap burns.
- So fire-and-forget: signal restart and return; it completes once THIS process exits.
- The pid lands in ``self_restart_pending`` so the fleet matrix can tell "restart
- deferred until the updater exits" from "restart never happened" (#119597): the
- ancestor is still serving the old code when the matrix runs, by construction.
- 2. Event loop provably wedged: SIGUSR1 can never drain it; bounded SIGTERM→SIGKILL.
- 3. Live out-of-tree gateway: graceful SIGUSR1 drain up to ``drain_budget``.
- The wedged-loop probe cannot break it: the cron session posts activity every ~180s (process-tool poll
- return), so it is "actively waiting forever" and never marked wedged — the gateway burns the full
- force-drain cap (1800s) before killing its own updater's session. See #86684.
- """
- from hermes_cli.gateway import (
- GATEWAY_LOOP_WEDGED, _escalate_wedged_gateway, _graceful_restart_via_sigusr1,
- _is_pid_ancestor_of_current_process, _request_gateway_self_restart, probe_gateway_loop_liveness,
- )
- if _is_pid_ancestor_of_current_process(pid):
- print(
- f" → {label}: update is running inside this gateway's "
- "process tree — signalling restart and letting the gateway "
- "drain itself (avoids the cron-update deadlock, #100179)"
- )
- accepted = _request_gateway_self_restart(pid)
- if accepted and self_restart_pending is not None:
- self_restart_pending.add(pid)
- return accepted
- if probe_gateway_loop_liveness(pid) == GATEWAY_LOOP_WEDGED:
- print(f" ⚠ {label}: gateway event loop is unresponsive — skipping drain, forcing a bounded stop...")
- _escalate_wedged_gateway(pid)
- return True
- print(f" → {label}: draining (up to {drain_budget:.0f}s)...")
- from hermes_cli.update_cmd_drain_report import drain_progress_reporter
- return _graceful_restart_via_sigusr1(
- pid, drain_timeout=drain_budget,
- on_progress=drain_progress_reporter(_gateway_home_for_pid(pid), budget_s=drain_budget))
- def _gateway_home_for_pid(pid: int):
- """HERMES_HOME of the gateway ``pid`` per the fleet inventory, else None (own profile's file)."""
- with suppress(Exception):
- from hermes_cli.update_receipt import _profile_homes
- from gateway.status import read_runtime_status
- for _profile, home in _profile_homes():
- record = read_runtime_status(home / "gateway_state.json") or {}
- if record.get("pid") == pid:
- return home
- return None
- def _sudo_noninteractive_ok(targeted_probe: list) -> bool:
- """True when this user can elevate without a prompt.
- ``sudo -n true`` first; a refusal is inconclusive because a NOPASSWD sudoers entry scoped
- to one command (the hardened shape) rejects the blanket probe, so fall back to running
- ``sudo -n <targeted_probe>`` — callers pass a non-destructive stand-in for the argv they
- are about to elevate.
- """
- try:
- if subprocess.run(["sudo", "-n", "true"], capture_output=True, timeout=5, check=False).returncode == 0:
- return True
- # Blanket sudo refused — a targeted NOPASSWD sudoers entry may still work.
- return subprocess.run(["sudo", "-n", *targeted_probe], capture_output=True, timeout=5, check=False).returncode == 0
- except (OSError, subprocess.TimeoutExpired):
- return False
- def _resolve_manage_cmd(cache: dict, scope_: str, scope_cmd_: list, svc_name_: str):
- """Resolve the command prefix for manage-units verbs (None ⇒ no privilege path).
- Manage-units verbs on a *system* service trigger a polkit prompt for non-root
- users, which flashes and dies inside our captured 10-15s subprocess. Root → plain
- systemctl; else ``sudo -n`` blanket probe, then a targeted ``reset-failed`` probe
- so a least-privilege sudoers entry scoped to hermes-gateway* qualifies (idempotent
- no-op we run before every privileged restart anyway). On None the caller must SKIP
- the restart (without draining first!). ``--no-ask-password`` prevents polkit hangs.
- """
- if scope_ in cache:
- return cache[scope_]
- cmd = scope_cmd_ + ["--no-ask-password"]
- if _needs_sudo(scope_):
- sudo_cmd = ["sudo", "-n"] + cmd
- cmd = sudo_cmd if _sudo_noninteractive_ok(cmd + ["reset-failed", svc_name_]) else None
- cache[scope_] = cmd
- return cmd
- def _repair_unit_without_fatal_exit_park(svc_name: str, scope: str) -> None:
- """A unit whose restart policy predates ``RestartPreventExitStatus=78`` crash-loops on the PERMANENT
- exit: a ``Restart=on-failure`` system unit restarted ~180x on a host-attach refusal while the
- regenerated user units parked (#118282). The gateway rewrites its USER unit at boot; a SYSTEM unit
- lives in /etc, so rewrite it here when we are root, else name the repair."""
- from hermes_cli.gateway import (
- _SYSTEM_UNIT_DIR, GATEWAY_FATAL_CONFIG_EXIT_CODE, get_service_name,
- refresh_systemd_unit_if_needed, user_systemd_unit_dir,
- )
- system = scope == "system"
- unit_path = (_SYSTEM_UNIT_DIR if system else user_systemd_unit_dir()) / f"{svc_name}.service"
- try:
- parked = re.search(rf"^RestartPreventExitStatus=.*\b{GATEWAY_FATAL_CONFIG_EXIT_CODE}\b", unit_path.read_text(encoding="utf-8-sig"), re.MULTILINE)
- except OSError:
- return
- if parked:
- return
- if system and not _needs_sudo(scope) and svc_name == get_service_name():
- # The refresh adopts the unit's HERMES_HOME into os.environ (sudo strips it); the rest of the
- # update keeps running for the invoking profile.
- launch_home = os.environ.get("HERMES_HOME")
- try:
- refresh_systemd_unit_if_needed(system=True)
- finally:
- if launch_home is None:
- os.environ.pop("HERMES_HOME", None)
- else:
- os.environ["HERMES_HOME"] = launch_home
- return
- print(
- f" ⚠ {svc_name} lacks RestartPreventExitStatus={GATEWAY_FATAL_CONFIG_EXIT_CODE}: a permanent refusal "
- f"(exit {GATEWAY_FATAL_CONFIG_EXIT_CODE}) would crash-loop it instead of parking.\n"
- f" Repair: {'sudo ' if system else ''}hermes gateway install{' --system' if system else ''}"
- )
- def _restart_one_systemd_gateway_unit(
- svc_name: str, *, scope: str, scope_cmd: list, drain_budget: float, _manage_cmd_cache: dict,
- restarted_services: list, failed_or_stale_units: list, self_restart_pending: set | None = None,
- ) -> None:
- """Restart one active systemd gateway/serve unit: graceful SIGUSR1 drain, then forced restart.
- Appends settled names to ``restarted_services`` and failures to ``failed_or_stale_units``.
- """
- check = _systemctl(scope_cmd + ["is-active", svc_name], timeout=5)
- if check.stdout.strip() != "active":
- return
- if not _systemd_unit_owned_by_update(scope_cmd, svc_name):
- return
- _repair_unit_without_fatal_exit_park(svc_name, scope)
- # None ⇒ no non-interactive privilege path; avoid manage-units verbs
- # entirely or polkit prompts inside the captured subprocess.
- _manage_cmd = _resolve_manage_cmd(_manage_cmd_cache, scope, scope_cmd, svc_name)
- # Graceful SIGUSR1 first so in-flight runs drain: handler → request_restart(via_service=True)
- # → drain → exit, Restart=always respawns. hermes-serve has no handler → blunt restart below.
- _main_pid = 0
- if _service_unit_supports_graceful_sigusr1_restart(svc_name):
- try:
- _show = _systemctl(scope_cmd + ["show", svc_name, "--property=MainPID", "--value"], timeout=5)
- _main_pid = int((_show.stdout or "").strip() or 0)
- except (ValueError, subprocess.TimeoutExpired, FileNotFoundError):
- _main_pid = 0
- # Three-way triage (ancestor / wedged / graceful drain).
- _graceful_ok = _main_pid > 0 and _drain_or_signal_gateway_for_update(
- _main_pid, drain_budget, svc_name, self_restart_pending=self_restart_pending)
- if _graceful_ok:
- from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job, service_restart_sec, systemd_restart_timeout
- # ``Restart=always`` respawns only after RestartSec (60s in our unit; dead time for a
- # voluntary restart). ``reset-failed`` + ``start`` skips it (~1-3s); if RestartSec already
- # elapsed, ``start`` is a no-op and we fall through to the poll. Needs manage-units
- # privileges; without them auto-restart still fires after RestartSec.
- if _manage_cmd is not None:
- _systemctl(_manage_cmd + ["reset-failed", svc_name], timeout=10)
- run_unit_job(_manage_cmd, scope_cmd, "start", svc_name,
- timeout=systemd_restart_timeout(scope_cmd, svc_name, start_only=True))
- if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
- restarted_services.append(svc_name)
- return
- # Passive poll: auto-restart fires after RestartSec regardless of
- # privileges — primary when _manage_cmd is None, fallback otherwise.
- _restart_sec = service_restart_sec(scope_cmd, svc_name, default=0.0)
- if _manage_cmd is None and _restart_sec > 5.0:
- print(
- f" → {svc_name}: waiting for systemd "
- f"auto-restart (~{int(_restart_sec)}s; "
- "no root for an immediate restart)..."
- )
- if _wait_for_service_active(scope_cmd, svc_name, timeout=max(10.0, _restart_sec + 10.0)):
- restarted_services.append(svc_name)
- return
- # Exited but not respawned (older unit without Restart=on-failure /
- # RestartForceExitStatus=75); fall through to forced restart.
- print(f" ⚠ {svc_name} drained but didn't relaunch — forcing restart")
- # Forcing needs manage-units privileges; without a non-interactive path
- # polkit would prompt inside the captured subprocess — skip, instruct.
- if _manage_cmd is None:
- failed_or_stale_units.append(svc_name)
- print(
- f" ⚠ {svc_name} is a system service and restarting it needs root.\n"
- f" Restart it manually to load the new version:\n"
- f" sudo systemctl restart {svc_name}\n"
- f" To let `hermes update` restart it automatically, allow\n"
- f" passwordless sudo for systemctl, or run updates with sudo."
- )
- return
- # Blunt restart — only when the graceful path failed (no SIGUSR1 wiring, drain over
- # budget, restart-policy mismatch). Mirrors `hermes gateway restart` (`systemd_restart()`).
- restart = _systemctl_reset_and_restart(_manage_cmd, svc_name, scope_cmd=scope_cmd)
- if restart.returncode != 0:
- failed_or_stale_units.append(svc_name)
- print(f" ⚠ Failed to restart {svc_name}: {restart.stderr.strip()}")
- return
- # restart returns 0 even if the new process crashes at once — verify.
- if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
- restarted_services.append(svc_name)
- return
- # Retry once — transient startup failures (stale module cache,
- # import race) often clear; reset-failed so the retry isn't blocked.
- print(f" ⚠ {svc_name} died after restart, retrying...")
- _systemctl_reset_and_restart(_manage_cmd, svc_name, scope_cmd=scope_cmd)
- if _wait_for_service_active(scope_cmd, svc_name, timeout=10.0):
- restarted_services.append(svc_name)
- print(f" ✓ {svc_name} recovered on retry")
- return
- failed_or_stale_units.append(svc_name)
- _scope_flag = "--user " if scope == "user" else ""
- _sudo_hint = "sudo " if scope == "system" else ""
- print(
- f" ✗ {svc_name} failed to stay running after restart.\n"
- f" Check logs: {_sudo_hint}journalctl {_scope_flag}-u {svc_name} --since '2 min ago'\n"
- f" Recover manually:\n"
- f" {_sudo_hint}systemctl {_scope_flag}reset-failed {svc_name}\n"
- f" {_sudo_hint}systemctl {_scope_flag}restart {svc_name}"
- )
- def _restart_systemd_gateway_units(
- restarted_services, failed_or_stale_units, restarted_scoped_units, drain_budget, self_restart_pending=None,
- ):
- """Restart every active hermes-gateway*/hermes-serve* systemd unit (user + system).
- Settled units → ``restarted_services`` (bare) and ``restarted_scoped_units``
- (``scope/name``); failures → ``failed_or_stale_units``. Per-unit timeouts isolated.
- """
- from hermes_cli.gateway import supports_systemd_services, _ensure_user_systemd_env
- if not supports_systemd_services():
- return
- _manage_cmd_cache: dict = {}
- with suppress(Exception):
- _ensure_user_systemd_env()
- def _on_list_timeout(scope: str, exc: subprocess.TimeoutExpired) -> None:
- # Discovery timeout — skip this scope, keep the other.
- print(
- f" ⚠ systemctl timed out listing {scope}-scope "
- f"gateway units ({exc.cmd if exc.cmd else 'unknown command'}). "
- f"Check the gateway with: hermes gateway status"
- )
- def _on_unit_timeout(svc_name: str, exc: subprocess.TimeoutExpired) -> None:
- # Isolate to this unit; a scope-wide handler used to abort every
- # later gateway and leave the fleet on mixed code.
- failed_or_stale_units.append(svc_name)
- print(
- # See #68523.
- f" ⚠ systemctl timed out restarting {svc_name} "
- f"({exc.cmd if exc.cmd else 'unknown command'}); "
- f"continuing with remaining gateways"
- )
- # Enumerate every scope first: leftover per-profile units (``hermes-gateway-<profile>``)
- # all point at the SAME live MainPID on a multiplexed host, and restarting each in turn
- # is an N-fold outage from one update. Units sharing a live PID collapse to one restart;
- # the others are named as legacy units to migrate, never silently dropped.
- from hermes_cli.update_host_obligation import collapse_units_to_host_processes
- targets: dict[str, tuple[str, list, str]] = {} # "<scope>/<unit>" -> (scope, scope_cmd, unit)
- for scope, scope_cmd, result in _systemd_gateway_unit_listings(_on_list_timeout):
- _for_each_systemd_gateway_unit(
- result.stdout,
- process_unit=lambda svc_name, _scope=scope, _cmd=scope_cmd: targets.setdefault(
- f"{_scope}/{svc_name}", (_scope, _cmd, svc_name)),
- on_unit_timeout=_on_unit_timeout,
- )
- keys = list(targets)
- covered: dict[str, str] = {}
- if len(keys) > 1:
- # Only worth a `systemctl show` round when several units could be one process.
- keys, covered = collapse_units_to_host_processes(
- keys, lambda key: _unit_main_pid(targets[key][1], targets[key][2]))
- for unit_key, owner_key in covered.items():
- print(
- f" • {unit_key} is a legacy per-profile unit sharing one host gateway process with "
- f"{owner_key}; restarting it again would restart that process twice. Fold the units "
- "together with: hermes gateway migrate"
- )
- from hermes_cli.update_cmd_posix_pause import already_restarted
- for key in keys:
- scope, scope_cmd, svc_name = targets[key]
- if key in already_restarted()["units"]: # paused for the update, restarted on the new code
- restarted_scoped_units.add(key)
- continue
- # Scope-qualify before the next unit; ``finally`` so a mid-pass abort keeps settled units.
- _scope_mark = len(restarted_services)
- try:
- _restart_one_systemd_gateway_unit(
- svc_name,
- scope=scope,
- scope_cmd=scope_cmd,
- drain_budget=drain_budget,
- _manage_cmd_cache=_manage_cmd_cache,
- restarted_services=restarted_services,
- failed_or_stale_units=failed_or_stale_units,
- self_restart_pending=self_restart_pending,
- )
- except subprocess.TimeoutExpired as exc:
- _on_unit_timeout(svc_name, exc)
- finally:
- restarted_scoped_units.update(f"{scope}/{name}" for name in restarted_services[_scope_mark:])
- def _unit_main_pid(scope_cmd: list, svc_name: str) -> int:
- """Live ``MainPID`` of a unit; ``0`` when inactive, unprivileged or unreadable.
- Property reads need no manage-units privileges, and an unreadable PID is never collapsed:
- identity that cannot be proved keeps its own restart.
- """
- try:
- result = _systemctl(list(scope_cmd) + ["show", svc_name, "--property=MainPID", "--value"], timeout=10)
- except (OSError, subprocess.TimeoutExpired):
- return 0
- if getattr(result, "returncode", 1) != 0:
- return 0
- try:
- return int((getattr(result, "stdout", "") or "").strip() or 0)
- except ValueError:
- return 0
- @dataclass
- class _GatewayRestartOutcome:
- """Restart-phase bookkeeping. ``restarted_services`` keeps bare unit names (fleet
- probe, receipt, summary read it); ``incomplete`` ⇒ a gateway may still be stale."""
- incomplete: bool
- phase_errors: list
- pre_restart_gateway_pids: list | None
- restarted_services: list
- failed_or_stale_units: list
- relaunched_profiles: list
- externally_supervised_profiles: list
- killed_pids: set
- #: Gateways stopped with NO successor (no profile mapping / relaunch could not be armed);
- #: the summary tells the user to restart them by hand, so the fleet probe must not expect
- #: a row for them.
- stopped_unmapped_pids: set = field(default_factory=set)
- #: ``scope/name`` of every settled systemd unit; the fleet probe stops waiting for a state stamp
- #: once none of them is active or activating any more (the successor died, nothing will publish).
- restarted_scoped_units: set = field(default_factory=set)
- #: Gateways that are ANCESTORS of this updater and accepted a self-restart request
- #: (``_drain_or_signal_gateway_for_update`` branch 1): they restart only after this process
- #: exits, so the fleet matrix renders them as pending instead of STALE (#119597).
- self_restart_pending_pids: set = field(default_factory=set)
- #: Units the restart phase itself could not restart. Verification warns about the ones its fleet
- #: probe does not prove current: a timed-out ``systemctl`` call can still bring its unit up. Not
- #: ``failed_or_stale_units``: the Windows resume merge and an aborted phase add to that list, and
- #: neither gets this systemctl/launchctl advice.
- unrestarted_units: list = field(default_factory=list)
- def fleet_probe_signals(self) -> tuple:
- """``(pre_restart_pids, killed_pids)`` with the unmapped stops removed — the signals that
- legitimately predict a fleet-matrix row."""
- pre = self.pre_restart_gateway_pids
- if pre is not None:
- pre = [pid for pid in pre if pid not in self.stopped_unmapped_pids]
- return pre, self.killed_pids - self.stopped_unmapped_pids
- def record_receipt(self, **extra) -> None:
- """Best-effort ``record_gateway_restart`` from the current bookkeeping."""
- with suppress(Exception):
- from hermes_cli.update_receipt import record_gateway_restart
- record_gateway_restart(
- restarted_services=self.restarted_services, relaunched_profiles=self.relaunched_profiles,
- externally_supervised_profiles=self.externally_supervised_profiles,
- killed_pids=sorted(self.killed_pids), failed_units=self.failed_or_stale_units,
- incomplete=self.incomplete, **extra,
- )
- def _restart_manual_gateways(out: _GatewayRestartOutcome, _drain_budget) -> None:
- """Drain/stop every manual (non-service) gateway and print the restart summary.
- Mutates ``out`` in place; raises so the caller's abort recovery fires.
- """
- import signal as _signal
- from hermes_cli.gateway import (
- find_gateway_pids, find_profile_gateway_processes, _prepare_profile_gateway_update_restart, _get_service_pids,
- _wait_for_gateway_exit,
- )
- # Exclude just-restarted service PIDs so we don't kill what systemd/launchd spawned.
- from hermes_cli.update_cmd_posix_pause import already_restarted
- # Plus the gateways this update paused and already restarted on the new code.
- service_pids = _get_service_pids(all_profiles=True) | already_restarted()["pids"]
- manual_pids = find_gateway_pids(exclude_pids=service_pids, all_profiles=True)
- profile_processes = {
- proc.pid: proc
- for proc in find_profile_gateway_processes(exclude_pids=service_pids)
- if proc.pid in manual_pids
- }
- # ``all_profiles`` is host-wide: a sibling install's gateway matches too. Only this update's
- # homes are stopped; the profile-mapped PIDs come from this install's own PID files (#93349).
- manual_pids = _scoped_manual_gateway_pids(manual_pids, keep=profile_processes)
- # Profile gateways we couldn't arm a relaunch for must NOT keep running stale:
- # the unmapped sweep below stops them and lists them under "Restart manually".
- # These must NOT be left running: their modules are the pre-update ones and every lazy import from here
- # on mixes versions against the new code on disk (#88654). Handing them to the unmapped sweep below
- # stops them and surfaces them in the "Stopped N manual gateway process(es) / Restart manually" summary,
- # which is the contract already used for gateways with no profile mapping.
- unrestartable_pids = set()
- for pid, proc in profile_processes.items():
- restart_mode = _prepare_profile_gateway_update_restart(proc.profile, pid)
- if restart_mode is None:
- # A bare ``continue`` here left it serving stale modules with no signal.
- print(
- f" ⚠ {proc.profile}: could not arm an automatic "
- f"gateway restart for PID {pid} — stopping it instead "
- "so it cannot keep running pre-update code"
- )
- unrestartable_pids.add(pid)
- continue
- # SIGUSR1 drain first, SIGTERM fallback if unsupported/over budget — the watcher
- # relaunches either way. The helper announces its choice first because a silent
- # full-budget wait reads as a hung update.
- if not _drain_or_signal_gateway_for_update(
- pid, _drain_budget, proc.profile, self_restart_pending=out.self_restart_pending_pids):
- with suppress(ProcessLookupError, PermissionError):
- os.kill(pid, _signal.SIGTERM)
- # Wait ≤5s for exit: Telegram keeps the old getUpdates session ~30s; a new gateway
- # inside that window gets a 409 (_handle_polling_conflict retries, but a brief
- # wait avoids it on fast machines).
- _wait_for_gateway_exit(timeout=5.0, force_after=None)
- out.killed_pids.add(pid)
- if restart_mode == "external-supervisor":
- out.externally_supervised_profiles.append(proc.profile)
- else:
- out.relaunched_profiles.append(proc.profile)
- for pid in manual_pids:
- if pid in profile_processes and pid not in unrestartable_pids:
- continue
- with suppress(ProcessLookupError, PermissionError):
- os.kill(pid, _signal.SIGTERM)
- out.killed_pids.add(pid)
- out.stopped_unmapped_pids.add(pid)
- if out.restarted_services or out.killed_pids:
- print()
- for svc in out.restarted_services:
- print(f" ✓ Restarted {svc}")
- if out.relaunched_profiles:
- print(f" ✓ Restarting manual gateway profile(s): {', '.join(out.relaunched_profiles)}")
- if out.externally_supervised_profiles:
- names = ", ".join(out.externally_supervised_profiles)
- print(f" ✓ Handed gateway profile(s) back to their external supervisor: {names}")
- unmapped_count = (len(out.killed_pids) - len(out.relaunched_profiles) - len(out.externally_supervised_profiles))
- if unmapped_count:
- print(f" → Stopped {unmapped_count} manual gateway process(es)")
- print(" Restart manually: hermes gateway run")
- if unmapped_count > 1:
- print(" (or: hermes -p <profile> gateway run for each profile)")
- def _force_kill_stuck_gateways(killed_pids) -> None:
- """Survivor sweep: gateways ignoring SIGTERM (stuck drain, blocked I/O, zombie) never
- exit, so the watcher never respawns and ImportErrors persist. Give graceful paths a
- moment, then SIGKILL remaining pre-update PIDs."""
- with _best_effort('Post-restart survivor sweep failed: %s'):
- from hermes_cli.gateway import find_gateway_pids, _get_service_pids
- # --- Post-restart survivor sweep ----------------------------- Issue #17648: some gateways ignore
- # SIGTERM (stuck drain, blocked I/O, PID dead but zombie). The detached profile watchers wait 120s
- # for the old PID to exit — if it never does, no respawn happens and the user keeps hitting
- # ImportError against a stale sys.modules.
- _time.sleep(3.0)
- _surviving = find_gateway_pids(exclude_pids=_get_service_pids(all_profiles=True), all_profiles=True)
- # Only PIDs we already tried to kill; newer ones are left alone.
- _stuck = [pid for pid in _surviving if pid in killed_pids]
- if _stuck:
- print()
- print(f" ⚠ {len(_stuck)} gateway process(es) ignored SIGTERM — force-killing")
- from gateway.status import get_process_start_time, terminate_pid
- for pid in _stuck:
- with suppress(ProcessLookupError, PermissionError, OSError):
- # taskkill /T /F on Windows (no SIGKILL there), SIGKILL on POSIX.
- terminate_pid(pid, force=True, expected_start_time=get_process_start_time(pid))
- # Let the OS reap so watchers see the exit and respawn.
- _time.sleep(1.5)
- def _recover_after_restart_phase_abort(
- e, _pre_update_plan, out: _GatewayRestartOutcome, *, gateway_mode, restarted_scoped_units
- ) -> None:
- """Phase-abort recovery: fresh-child restart + fail-closed verdict; updates ``out`` in place."""
- from hermes_cli.update_abort_recovery import _owed_stale_serve_rows
- from hermes_cli.update_cmd import (
- _abort_recovery_is_complete, _recover_gateway_restart_after_abort, _surviving_pre_update_serve_runtimes,
- _warn_stale_serve_runtimes,
- )
- logger.debug("Gateway restart during update failed: %s", e)
- out.phase_errors.append(str(e))
- # Restart output never printed: assume stale unless provably no gateway runs.
- # Empty ``_surviving`` proves safety only if nothing ran beforehand; a gone
- # pre-restart gateway was stopped without verified replacement → fail closed.
- # An exception escaping the whole phase means the drain/restart output the user relies on never printed.
- # Don't let that pass for a clean update: surface it and treat the fleet as stale unless we can
- # positively prove no gateway is running (#78574). A positive-empty ``_surviving`` is only
- # proof-of-safety when nothing was running before we touched anything. If a gateway was discovered
- # pre-restart and none survive now, it was stopped and its replacement was never verified — the same
- # fail-open contract this fix closes — so we must still fail closed on ``[]``.
- _surviving = _surviving_gateway_pids_after_failed_restart()
- _planned_gateway_profiles = {
- runtime.profile
- for runtime in getattr(_pre_update_plan, "runtimes", ()) or ()
- if getattr(runtime, "kind", None) == "gateway"
- and isinstance(getattr(runtime, "profile", None), str)
- }
- _already_restarted_profiles = set(out.relaunched_profiles) | set(out.externally_supervised_profiles)
- _already_restarted_profiles.update(
- profile
- for profile in _planned_gateway_profiles
- if any(_gateway_service_matches_profile(profile, service) for service in out.restarted_services)
- )
- _recovery_result = _recover_gateway_restart_after_abort(
- _pre_update_plan, gateway_mode=gateway_mode, skip_profiles=_already_restarted_profiles,
- skip_units=set(restarted_scoped_units),
- )
- _serve_units_failed = list((_recovery_result.get("serve_units") or {}).get("failed") or [])
- # Deliberately NOT merged into ``restarted_services`` (gateway vocabulary feeding the
- # fleet probe); serve coverage lives in the recovery result/receipt. A serve/dashboard
- # still the SAME pre-update process is live on old code (unreachable by `gateway
- # restart`): recovery may not claim success while one remains, and must never kill
- # one (manual/Desktop serves have no relaunch authority).
- _stale_runtime_rows = _surviving_pre_update_serve_runtimes(_pre_update_plan)
- _recovery_result["stale_runtimes"] = _stale_runtime_rows
- # Only systemd-VERIFIED outcomes claim coverage; a relaunch that merely exited 0
- # ("relaunch_attempted") was never observed and must not clear incomplete.
- _recovery_verified = set(_recovery_result.get("verified") or [])
- out.relaunched_profiles.extend(
- profile for profile in sorted(_recovery_verified) if profile not in out.relaunched_profiles
- )
- if _abort_recovery_is_complete(
- planned_gateway_profiles=_planned_gateway_profiles,
- covered_gateway_profiles=_already_restarted_profiles | _recovery_verified,
- recovery_result=_recovery_result,
- stale_runtime_rows=_stale_runtime_rows,
- ):
- # Fresh child is terminal; the fleet-version matrix stays the authoritative
- # read-back before success is declared. Desktop-owned survivors (the only rows
- # that can remain here) are named, not owed. See #111494.
- out.incomplete = False
- _warn_stale_serve_runtimes(_stale_runtime_rows)
- elif (
- _restart_phase_failure_is_incomplete(_surviving, out.pre_restart_gateway_pids)
- or _owed_stale_serve_rows(_stale_runtime_rows)
- or _serve_units_failed
- ):
- # Owed restart (C3): verification records the follow-up; the /update marker stays committed.
- out.incomplete = True
- _warn_gateway_restart_phase_aborted(e, _surviving)
- _warn_stale_serve_runtimes(_stale_runtime_rows)
- out.record_receipt(phase_error=str(e), fresh_recovery=_recovery_result)
- def _gateway_drain_budget() -> float:
- """Seconds a drain-first (SIGUSR1) restart may wait for a gateway to exit; 45s floor."""
- try:
- from hermes_cli.gateway import _get_restart_exit_wait_budget
- return max(float(_get_restart_exit_wait_budget()), 45.0)
- except Exception:
- return 45.0
- def _restart_gateway_fleet_after_update(_pre_update_plan, gateway_mode: bool):
- """Restart every running gateway (systemd, launchd, manual) onto the pulled code.
- Never raises: a phase abort runs fresh-child recovery and fails closed unless
- every planned gateway is verifiably covered.
- """
- from hermes_cli.update_cmd import _m
- from hermes_cli.update_stage import publish_stage
- publish_stage("Restarting Hermes services") # also after a failed build: the code is committed
- # All bookkeeping is declared before the try so abort recovery and fleet reconciliation
- # can read it even if the phase raises early. ``pre_restart_gateway_pids`` stays empty
- # until we are about to stop/drain, so an early exception has nothing to fail closed on,
- # while a failure after stopping a discovered gateway fails closed on an empty survivor probe.
- out = _GatewayRestartOutcome(
- incomplete=False, phase_errors=[], pre_restart_gateway_pids=[], restarted_services=[], failed_or_stale_units=[],
- relaunched_profiles=[], externally_supervised_profiles=[], killed_pids=set(),
- )
- # Scope-qualified twin (``user/hermes-serve`` vs ``system/hermes-serve`` are different
- # processes; abort recovery needs WHICH settled). Bare names stay in
- # ``restarted_services`` for the fleet probe, receipt and summary.
- # Snapshot of gateways running before we touch anything. Stays empty until we successfully import the
- # probe and are about to stop/drain — so an exception raised before we touch any gateway keeps this
- # empty (nothing to fail closed on), while a failure after we have stopped a discovered gateway lets the
- # handler fail closed on an empty survivor probe rather than reporting a clean update (#78574).
- # Declared outside the restart try/except below (and never reset to None) so it's always safe to read
- # afterwards even if that block raises before reaching its own restart bookkeeping — needed to forward
- # already-restarted units to ``_refresh_dashboard_after_update`` (review on #83595).
- restarted_scoped_units: set = set()
- try:
- # Every gateway helper the phase needs is imported up front so a broken gateway
- # module aborts into recovery BEFORE any unit is touched.
- from hermes_cli.gateway import (
- is_macos,
- find_gateway_pids,
- find_profile_gateway_processes,
- _prepare_profile_gateway_update_restart,
- _get_service_pids,
- _wait_for_gateway_exit,
- )
- # Drain budget covers ``restart_after_turn_timeout`` and stop()'s
- # ``restart_drain_timeout`` so a gateway waiting on a turn isn't hard-killed;
- # units without SIGUSR1 wiring just time out into ``systemctl restart``.
- _drain_budget = _gateway_drain_budget()
- # Snapshot before any stop/drain so an empty survivor probe reads as "stopped
- # and never came back", not "nothing was running"; None fails closed.
- try:
- out.pre_restart_gateway_pids = _scoped_manual_gateway_pids(find_gateway_pids(all_profiles=True), quiet=True)
- except Exception:
- out.pre_restart_gateway_pids = None
- _restart_systemd_gateway_units(
- out.restarted_services, out.failed_or_stale_units, restarted_scoped_units, _drain_budget,
- out.self_restart_pending_pids,
- )
- # macOS: EVERY ai.hermes.gateway* LaunchAgent (systemd parity).
- if is_macos():
- with suppress(FileNotFoundError, ImportError):
- _restart_macos_launchd_gateways(
- out.restarted_services, out.failed_or_stale_units, _drain_budget,
- self_restart_pending=out.self_restart_pending_pids,
- )
- _restart_manual_gateways(out, _drain_budget)
- if out.failed_or_stale_units:
- # Owed restart (C3): verification records the follow-up; the /update marker stays committed.
- out.incomplete = True
- out.unrestarted_units = list(out.failed_or_stale_units)
- out.record_receipt()
- _force_kill_stuck_gateways(out.killed_pids)
- except Exception as e:
- _recover_after_restart_phase_abort(
- e, _pre_update_plan, out, gateway_mode=gateway_mode, restarted_scoped_units=restarted_scoped_units
- )
- out.restarted_scoped_units = set(restarted_scoped_units)
- if not out.incomplete:
- # Stamp the HOST obligation so every other profile's CLI knows this update's restart
- # already happened; without it each profile re-kills the one shared multiplexer.
- from hermes_cli.update_host_obligation import mark_host_restart_completed
- mark_host_restart_completed(_restart_identity_sha())
- return out
- def _restart_phase_failure_is_incomplete(surviving, pre_restart_pids) -> bool:
- """Whether an escaped restart-phase exception must fail the update.
- Fail closed unless provably safe: ``surviving`` None (unprobeable) or non-empty →
- stale. ``[]`` proves safety ONLY if nothing ran beforehand; a pre-restart gateway
- (``pre_restart_pids`` non-empty or None) now gone was stopped unverified.
- * ``surviving is None`` — the survivor probe could not determine state (typically the freshly-pulled
- ``hermes_cli.gateway`` no longer imports, one of the ways the phase aborts). That is proof-of-safety
- ONLY when nothing was running before we touched anything. If a gateway was discovered pre-restart
- (``pre_restart_pids`` non-empty, or ``None`` meaning the pre-state could not be read), it was stopped
- without a verified replacement, so we still fail closed (#78574).
- """
- if surviving is None or surviving:
- return True
- return pre_restart_pids is None or bool(pre_restart_pids)
- def _fleet_probe_expected_runtimes(
- pre_update_plan, pre_restart_pids, windows_resume_token, restarted_services, killed_pids,
- ) -> bool:
- """Whether the post-update fleet probe should have produced rows.
- ``collect_fleet_versions()`` swallows every failure and an empty matrix prints as
- healthy, so zero rows is only proof-of-safety when NOTHING says a gateway existed
- pre-update. Signals: ``restarted_services``/``killed_pids``; ``pre_restart_pids``
- non-empty or None (same contract as ``_restart_phase_failure_is_incomplete``); plan
- inventoried ≥1 ``kind == "gateway"`` runtime. ``windows_resume_token`` is deliberately EXCLUDED: it is
- pause/resume bookkeeping, not an inventory, and its entries don't map to probe rows
- (``unmapped`` Scheduled-Task gateways never publish gateway_state.json; a paused
- profile resumes DETACHED). Counting it made every Windows update that paused a
- gateway exit 1 after a long silent wait; a live pre-update Windows gateway is already
- covered by ``pre_restart_pids`` and the plan. The same condition gates the settle sleep.
- See #93406.
- See #78574.
- See #93406.
- """
- del windows_resume_token # excluded on purpose — see docstring
- # See #93406.
- if restarted_services or killed_pids:
- return True
- if pre_restart_pids is None or pre_restart_pids:
- return True
- with suppress(Exception):
- # Gateway-kind only: serve/dashboard plan records never publish a gateway_state.json
- # row, so a dashboard-only plan cannot ground a rows-expected verdict (#97332).
- if pre_update_plan is not None and any(
- getattr(runtime, "kind", None) == "gateway" for runtime in pre_update_plan.runtimes
- ):
- return True
- return False
- def _wait_for_service_active(scope_cmd_: list, svc_name_: str, timeout: float = 10.0) -> bool:
- """Poll ``systemctl is-active`` (0.5s) up to ``timeout``: the Stopped -> Started
- transition isn't instantaneous, so a one-shot check falsely reports down."""
- deadline = _time.monotonic() + max(timeout, 0.5)
- while True:
- with suppress(FileNotFoundError, subprocess.TimeoutExpired):
- _verify = _systemctl(scope_cmd_ + ["is-active", svc_name_], timeout=5)
- if _verify.stdout.strip() == "active":
- return True
- if _time.monotonic() >= deadline:
- return False
- _time.sleep(0.5)
|