update_cmd_windows.py 91 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683
  1. """Windows gateway lifecycle for ``hermes update``: pause/resume/cold-start the service, sweep venv holders, reap orphaned backends.
  2. Split out of ``update_cmd.py``; names are re-imported there so ``hermes_cli.update_cmd.<name>`` still resolves/monkeypatches.
  3. Origin helpers are imported lazily per function (no cycle; test patches on the origin stay effective).
  4. """
  5. import logging
  6. from contextlib import contextmanager, nullcontext, redirect_stdout, suppress
  7. import os
  8. import re
  9. import shlex
  10. import subprocess
  11. import sys
  12. import time as _time
  13. from datetime import datetime, timezone, UTC
  14. from pathlib import Path
  15. from agent.deadline import poll_until
  16. from hermes_cli.update_cmd_common import _best_effort
  17. logger = logging.getLogger("hermes_cli.update_cmd") # log-record parity with the origin module
  18. _BACKEND_PURPOSES = ("serve", "dashboard")
  19. def _try_call(fn, log_message: str, *log_args, default=None):
  20. """``fn()``, or *default* after logging the exception at debug (``log_message`` gets ``*log_args, exc``)."""
  21. try:
  22. return fn()
  23. except Exception as exc:
  24. logger.debug(log_message, *log_args, exc)
  25. return default
  26. @contextmanager
  27. def _abort_on_error(prefix: str):
  28. """Re-raise any failure of a mandatory step as ``RuntimeError(f"{prefix}: {exc}")`` (chained)."""
  29. try:
  30. yield
  31. except Exception as exc:
  32. raise RuntimeError(f"{prefix}: {exc}") from exc
  33. def _planned_stop_marker_path(profile_path: Path) -> Path:
  34. return Path(profile_path) / ".gateway-planned-stop.json"
  35. def _planned_stop_markers(running_pids, profile_processes, service_gateway_pids) -> dict:
  36. """``{pid: marker path}`` for each gateway the socket pause will ask (profile-mapped, not SCM)."""
  37. markers = {}
  38. for pid in running_pids:
  39. home = None if pid in service_gateway_pids else getattr(profile_processes.get(pid), "path", None)
  40. if home:
  41. markers[pid] = _planned_stop_marker_path(Path(home))
  42. return markers
  43. def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool:
  44. """Write a planned-stop marker into a specific profile home."""
  45. try:
  46. from gateway.status import _get_process_start_time
  47. from utils import atomic_json_write
  48. atomic_json_write(
  49. _planned_stop_marker_path(profile_path),
  50. {"target_pid": pid, "target_start_time": _get_process_start_time(pid), "stopper_pid": os.getpid(),
  51. "written_at": datetime.now(UTC).isoformat()},
  52. indent=None, separators=(",", ":"),
  53. )
  54. return True
  55. except (OSError, PermissionError):
  56. return False
  57. def _wait_for_windows_update_gateway_exit(pids: list[int], *, timeout: float) -> set[int]:
  58. """Wait for the given gateway PIDs to exit, returning survivors."""
  59. if not pids:
  60. return set()
  61. from gateway.status import _pid_exists
  62. def _alive(pid: int) -> bool:
  63. try:
  64. return bool(_pid_exists(pid))
  65. except Exception:
  66. return False
  67. remaining = set(pids)
  68. def _all_gone() -> bool:
  69. nonlocal remaining
  70. remaining = {pid for pid in remaining if _alive(pid)}
  71. return not remaining
  72. poll_until(_all_gone, max(timeout, 0.0), 0.25)
  73. return {pid for pid in remaining if _alive(pid)}
  74. def _self_and_non_gateway_ancestor_pids(psutil) -> set[int]:
  75. """PIDs a venv-holder scan must never nominate: this process and its non-gateway ancestry.
  76. Do NOT blanket-exclude ancestors: under ``/update`` the updater is a CHILD of the gateway, and hiding it
  77. dead-ends the update on ``venv-blocked``. Gateway ancestors stay visible (the pause path stops them
  78. gracefully; a detached child survives on Windows); interactive ancestry is never a blocker."""
  79. _is_gw = None
  80. with suppress(Exception):
  81. # Never return ourselves or our own ancestry: a CLI ``hermes update`` runs from the venv python and
  82. # would otherwise nominate itself. Same #87594 carve-out as _detect_venv_python_processes: a GATEWAY
  83. # ancestor is not "our own ancestry" in the interactive sense — it is the process the pause
  84. # machinery must see (the /update-from-gateway topology makes the updater the gateway's child).
  85. from gateway.status import looks_like_gateway_command_line as _is_gw
  86. skip: set[int] = {os.getpid()}
  87. with suppress(Exception):
  88. for anc in psutil.Process().parents():
  89. anc_cmdline = _cmdline_or_empty(anc)
  90. if not (_is_gw is not None and anc_cmdline and _is_gw(anc_cmdline)):
  91. skip.add(int(anc.pid))
  92. return skip
  93. def _cmdline_or_empty(proc) -> str:
  94. """Joined argv of a psutil process, ``""`` when it can't be read."""
  95. try:
  96. return " ".join(proc.cmdline() or [])
  97. except Exception:
  98. return ""
  99. def _lower_dir_prefix(path: Path) -> str:
  100. """``str(path)`` lower-cased with one trailing separator, resolved when possible (prefix matching)."""
  101. try:
  102. raw = str(path.resolve())
  103. except OSError:
  104. raw = str(path)
  105. return raw.lower().rstrip(os.sep) + os.sep
  106. def _psutil():
  107. """The ``psutil`` module, or ``None`` when it can't be imported (callers degrade, never raise)."""
  108. try:
  109. import psutil # type: ignore
  110. except Exception:
  111. return None
  112. return psutil
  113. def _parent_is_live(proc) -> bool:
  114. """True when *proc* has a running parent that is not a recycled PID (a "parent" created after its child)."""
  115. parent = proc.parent()
  116. return parent is not None and parent.is_running() and parent.create_time() <= proc.create_time()
  117. def _detect_venv_python_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str, str]]:
  118. """Live processes running from the project venv's interpreter as ``(pid, name, cmdline)``; never raises.
  119. The hermes.exe shim guard misses the Desktop backend and anything off ``venv\\Scripts\\python(w).exe``;
  120. they keep ``.pyd`` files mapped so a mid-update dependency sync dies half-way. Empty off-Windows / without
  121. psutil; self + non-gateway ancestors excluded. cmdline/cwd are expensive per process on Windows (500+
  122. procs can blow the Desktop preflight watchdog), so they are fetched lazily for plausible candidates only.
  123. The FULL cmdline is kept: callers parse it (the pausable-gateway exemption looks for ``gateway run``).
  124. """
  125. from hermes_cli.update_cmd import _m
  126. psutil = _psutil()
  127. if not _m()._is_windows() or psutil is None:
  128. return []
  129. from hermes_constants import project_venv_dir
  130. venv_dir = project_venv_dir(_m().PROJECT_ROOT) or _m().PROJECT_ROOT / "venv"
  131. venv_prefix = _lower_dir_prefix(venv_dir)
  132. root_prefix = _lower_dir_prefix(_m().PROJECT_ROOT)
  133. skip = set(exclude_pids or set()) | _self_and_non_gateway_ancestor_pids(psutil)
  134. matches: list[tuple[int, str, str]] = []
  135. try:
  136. proc_iter = psutil.process_iter(["pid", "exe", "name"])
  137. except Exception:
  138. return []
  139. for proc in proc_iter:
  140. try:
  141. info = proc.info
  142. except Exception:
  143. continue
  144. pid, exe = info.get("pid"), info.get("exe")
  145. if not exe or pid is None or int(pid) in skip:
  146. continue
  147. try:
  148. exe_norm = str(Path(exe).resolve()).lower()
  149. except (OSError, ValueError):
  150. exe_norm = str(exe).lower()
  151. # Primary match: exe lives under this venv (desktop backend / gateway case).
  152. in_venv = exe_norm.startswith(venv_prefix)
  153. name = str(info.get("name") or Path(exe).name)
  154. name_low = name.lower()
  155. if not (in_venv or name_low.startswith(("python", "pypy")) or name_low in {"uv.exe", "uvx.exe", "hermes.exe"}):
  156. continue
  157. cmdline_raw = _cmdline_or_empty(proc)
  158. cmdline_low = cmdline_raw.lower()
  159. # Fallback: uv/base-interpreter trampolines have an exe OUTSIDE the venv yet hold
  160. # its .pyd files — match cmdline (venv path, or `-m hermes_cli.main` + root/cwd).
  161. if in_venv or venv_prefix in cmdline_low or (
  162. "hermes_cli.main" in cmdline_low and (root_prefix in cmdline_low or _cwd_prefix(proc).startswith(root_prefix))
  163. ):
  164. matches.append((int(pid), name, cmdline_raw))
  165. return matches
  166. def _cwd_prefix(proc) -> str:
  167. """Lower-cased cwd of *proc* with one trailing separator; bare ``os.sep`` when unreadable."""
  168. try:
  169. return str(proc.cwd() or "").lower().rstrip(os.sep) + os.sep
  170. except Exception:
  171. return os.sep
  172. _HOLDER_VALUE_FLAGS_FALLBACK = frozenset({
  173. "--profile", "-p", "--config", "--model", "-m", "--provider", "--reasoning", "--toolsets", "-t",
  174. "--skills", "-s", "--continue", "-c", "--resume", "-r", "--oneshot", "-z", "--in", "--usage-file", "--cpu-profile",
  175. })
  176. _holder_value_flags_cache: frozenset | None = None
  177. def _holder_value_flags() -> frozenset:
  178. """Top-level CLI flags that consume a value, from the REAL parser (nargs != 0); cached per process.
  179. Derived so the holder classifier can't drift from argparse (a handwritten subset misparsed ``--reasoning high
  180. serve``). Pre-argparse profile selectors are added explicitly (stripped before argparse sees argv). Falls back
  181. to a static snapshot when the parser can't import — the updater must classify holders even on a broken tree.
  182. Introspects ``build_top_level_parser()`` (every option with nargs != 0) so the holder classifier can
  183. never drift from the argparse surface (#91869 review: a handwritten subset misparsed ``--reasoning high
  184. serve`` as subcommand ``high`` and ``-m dashboard serve`` as ``dashboard`` — recreating the wrong-hint
  185. class).
  186. """
  187. global _holder_value_flags_cache
  188. if _holder_value_flags_cache is not None:
  189. return _holder_value_flags_cache
  190. flags: set[str] = {"--profile", "-p", "--config"}
  191. try:
  192. from hermes_cli._parser import build_top_level_parser
  193. for action in build_top_level_parser()[0]._actions:
  194. if action.option_strings and action.nargs != 0:
  195. flags.update(action.option_strings)
  196. _holder_value_flags_cache = frozenset(flags)
  197. except Exception:
  198. _holder_value_flags_cache = _HOLDER_VALUE_FLAGS_FALLBACK
  199. return _holder_value_flags_cache
  200. def _hermes_holder_subcommand(cmdline: str) -> str | None:
  201. """The actual Hermes SUBCOMMAND a venv-holder argv runs, or None (callers must NOT guess a label).
  202. Token-based, never substring (``kanban --preserve-cache`` contains "serve"): find the ``hermes_cli.main`` /
  203. ``hermes(.exe)`` entry token, return the first following token that isn't a flag or a flag's value.
  204. Profile selectors (``--profile X``, ``-p X``) are skipped like the canonical gateway matcher does. See
  205. #90778.
  206. """
  207. try:
  208. tokens = shlex.split(cmdline, posix=False)
  209. except Exception:
  210. tokens = cmdline.split()
  211. # ``python -c <src> … -m hermes_cli.main <subcommand>``: the entry token belongs to the argv the
  212. # inline source carries for a LATER spawn, not to this holder (#107002) -- unless the source is a
  213. # Hermes bootstrap running the entry point in this process (#124318).
  214. from gateway.status_inline_source import command_line_runs_inline_source, inline_bootstrap_argv
  215. normalized = [t.strip("\"'").replace("\\", "/") for t in tokens]
  216. if command_line_runs_inline_source(normalized):
  217. tokens = inline_bootstrap_argv(normalized)
  218. if tokens is None:
  219. return None
  220. def _is_entry(i: int, token: str) -> bool:
  221. low = token.lower().strip("\"'").replace("\\", "/")
  222. return (low.endswith("hermes_cli.main") and i > 0 and tokens[i - 1] == "-m") or (
  223. low.rsplit("/", 1)[-1] in ("hermes", "hermes.exe")) or low.endswith("hermes_cli/main.py")
  224. entry_idx = next((i for i, token in enumerate(tokens) if _is_entry(i, token)), None)
  225. if entry_idx is None:
  226. return None
  227. value_flags = _holder_value_flags()
  228. i = entry_idx + 1
  229. while i < len(tokens):
  230. token = tokens[i]
  231. if token in value_flags or token.split("=", 1)[0] in value_flags:
  232. i += 1 if "=" in token else 2 # --flag value consumes two tokens; --flag=value one.
  233. elif token.startswith("-"):
  234. i += 1
  235. else:
  236. return token.lower()
  237. return None
  238. def _venv_launcher_ancestors(pids: list[int]) -> list[int]:
  239. """Venv-interpreter parents of *pids* that hold the install open; never raises.
  240. A shim-started gateway is a chain: ``venv\\Scripts\\python.exe`` launcher (keeps ``.pyd`` mapped) -> uv
  241. CPython worker (writes the PID file). The pause set sees the worker, the venv scan sees the launcher, so a
  242. paused gateway still tripped the guard. One hop up only, venv-prefixed only (bounds blast radius)."""
  243. from hermes_cli.update_cmd import _m
  244. psutil = _psutil()
  245. if not _m()._is_windows() or not pids or psutil is None:
  246. return []
  247. from hermes_constants import project_venv_dir
  248. venv_dir = project_venv_dir(_m().PROJECT_ROOT) or _m().PROJECT_ROOT / "venv"
  249. venv_prefix = _lower_dir_prefix(venv_dir)
  250. skip = _self_and_non_gateway_ancestor_pids(psutil) | set(pids)
  251. found: list[int] = []
  252. for pid in pids:
  253. with suppress(Exception):
  254. parent = psutil.Process(int(pid)).parent()
  255. if parent is None:
  256. continue
  257. ppid = int(parent.pid)
  258. if ppid not in skip and ppid not in found and (parent.exe() or "").lower().startswith(venv_prefix):
  259. found.append(ppid)
  260. return found
  261. def _venv_holder_kind(cmdline: str) -> str:
  262. """Machine-readable class of one venv holder for ``--list-venv-holders``.
  263. ``gateway`` (the pausable gateway matcher), ``backend`` (``serve``/``dashboard`` -- the Desktop
  264. app's backend shape), ``hermes:<subcommand>`` for any other Hermes entry, else ``python``.
  265. Derived from the same classifiers the refusal path uses so automation stops exactly what the
  266. guard would refuse on."""
  267. from hermes_cli._scan_venv_blockers import _is_pausable_gateway
  268. if _is_pausable_gateway(cmdline):
  269. return "gateway"
  270. subcommand = _hermes_holder_subcommand(cmdline)
  271. if subcommand in _BACKEND_PURPOSES:
  272. return "backend"
  273. if subcommand:
  274. return f"hermes:{subcommand}"
  275. return "python"
  276. VENV_HOLDERS_EXIT = 3 # ``hermes update --list-venv-holders``: holders present (distinct from refusal 2)
  277. def list_venv_holders() -> list[dict]:
  278. """``[{pid, exe, argv, kind}]`` for every process the venv-holder guard would refuse on, read-only.
  279. Off Windows (or without psutil) the guard never fires, so the list is empty. ``exe``/``argv`` are the
  280. live psutil values when readable (the scan may carry only a cmdline prefix)."""
  281. from hermes_cli.update_cmd import _m
  282. psutil = _psutil()
  283. holders: list[dict] = []
  284. for pid, name, cmdline in _m()._detect_venv_python_processes():
  285. exe, argv = name, cmdline
  286. if psutil is not None:
  287. with suppress(Exception):
  288. proc = psutil.Process(int(pid))
  289. exe = proc.exe() or name
  290. argv = " ".join(proc.cmdline()) or cmdline
  291. holders.append({"pid": int(pid), "exe": exe, "argv": argv, "kind": _venv_holder_kind(argv)})
  292. return holders
  293. def _leftover_pausable_gateway_pids(matches: list[tuple[int, str, str]]) -> list[int] | None:
  294. """PIDs from *matches* when EVERY remaining venv holder is a pausable gateway, else ``None`` (keep refusing).
  295. A gateway respawned inside the pause->guard window (or via an unmapped spawn path) still holds ``.pyd`` files.
  296. Uses the Desktop preflight's ``_is_pausable_gateway`` so exemption and tolerance cannot drift; live argv is
  297. re-read via psutil when possible since the scan may hold only a cmdline prefix."""
  298. from hermes_cli._scan_venv_blockers import _is_pausable_gateway
  299. psutil = _psutil()
  300. pids: list[int] = []
  301. for pid, _name, cmdline in matches:
  302. argv = cmdline
  303. if psutil is not None:
  304. with suppress(Exception):
  305. argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline
  306. if not _is_pausable_gateway(argv):
  307. return None
  308. pids.append(int(pid))
  309. return pids
  310. def _refuse_gateway_ancestor_tree_kill(pids: list[int], *, gateway_mode: bool) -> bool:
  311. """Refuse a plain Windows update that would tree-kill its own ancestry (a chat agent's ``hermes update`` is
  312. a gateway child; ``taskkill /T /F`` kills the updater first). ``--gateway`` is exempt (detached delivery).
  313. Refuse only when a nominated gateway is positively an ancestor; unknown ancestry keeps existing recovery.
  314. The leftover holder recovery below uses ``taskkill /T /F`` on Windows, so force-stopping that gateway
  315. also kills the updater before it can mutate the checkout (#98814).
  316. """
  317. if gateway_mode or not pids:
  318. return False
  319. def _ancestors():
  320. from hermes_cli.gateway import _is_pid_ancestor_of_current_process
  321. return [int(pid) for pid in pids if _is_pid_ancestor_of_current_process(int(pid))]
  322. ancestors = _try_call(_ancestors, "Could not inspect gateway ancestry before tree-kill: %s")
  323. if not ancestors:
  324. return False
  325. print(
  326. "✗ Refusing to stop the gateway process tree because this updater "
  327. f"is running inside it (gateway PID(s): {', '.join(str(pid) for pid in ancestors)}).\n"
  328. " On Windows, taskkill /T would terminate the updater before the update can run.\n"
  329. " From a chat platform, use `/update` instead.\n Otherwise, run `hermes update` from a separate terminal."
  330. )
  331. return True
  332. def _ledger_manual_serve_holders(matches: list[tuple[int, str, str]]) -> list[dict]:
  333. """Full ledger entries for venv holders that are MANUAL serve/dashboard backends.
  334. Positive identity only: self-registered purpose serve/dashboard, live (pid, create_time), recorded spawner
  335. NOT alive (a Desktop-owned backend keeps its live Electron spawner and must keep the refusal — the app would
  336. respawn what we kill). Full entries let the relauncher rebuild from host/port/profile, not argv."""
  337. try:
  338. from hermes_cli.process_identity import ledger_entries, spawner_is_dead
  339. except Exception:
  340. return []
  341. holder_pids = {int(pid) for pid, _name, _cmd in matches}
  342. return [
  343. entry for entry in ledger_entries()
  344. if entry.get("purpose") in _BACKEND_PURPOSES and isinstance(entry.get("pid"), int) and entry["pid"] in holder_pids
  345. and spawner_is_dead(entry) is not False # False = live Desktop supervisor owns it; keep refusing
  346. ]
  347. def _serve_relaunch_commands(entries: list[dict]) -> list[list[str]]:
  348. """Rebuild launch commands for stopped serves from ledger host/port/profile — never argv parsing
  349. (joined argv cannot round-trip Windows paths with spaces). Entries without a port are skipped."""
  350. from hermes_cli.update_cmd import _m
  351. hermes = "hermes"
  352. with suppress(Exception):
  353. scripts_dir = _m()._venv_scripts_dir()
  354. if scripts_dir is not None:
  355. hermes = next((str(scripts_dir / n) for n in ("hermes.exe", "hermes") if (scripts_dir / n).is_file()), hermes)
  356. commands: list[list[str]] = []
  357. for entry in entries:
  358. port = entry.get("port")
  359. if not isinstance(port, int) or port <= 0:
  360. continue
  361. profile, host = str(entry.get("profile") or ""), str(entry.get("host") or "")
  362. commands.append(
  363. [hermes] + (["--profile", profile] if profile and profile != "default" else [])
  364. + [str(entry.get("purpose"))] + (["--host", host] if host else []) + ["--port", str(port)]
  365. )
  366. return commands
  367. def _relaunch_stopped_serves(token: dict) -> None:
  368. """Idempotent atexit relaunch of manual serves stopped by the venv guard.
  369. `pending` flips False on first invocation so explicit call + atexit registration cannot double-spawn."""
  370. from hermes_cli.update_cmd import _m, _record_update_step
  371. if not token.get("pending"):
  372. return
  373. token["pending"] = False
  374. entries = token.get("entries") or []
  375. if not entries:
  376. return
  377. commands = _serve_relaunch_commands(entries)
  378. skipped = len(entries) - len(commands)
  379. failed: list = []
  380. if commands:
  381. print(" ⟲ Relaunching stopped serve/dashboard backend(s)")
  382. failed = _m()._respawn_dashboard_processes(commands)
  383. if skipped or failed:
  384. print(" ⚠ Some stopped backends could not be relaunched automatically; restart them manually (hermes serve --host <ip> --port <port>).")
  385. _record_update_step(
  386. "serve_relaunch", not failed and not skipped,
  387. f"relaunched={len(commands) - len(failed)} failed={len(failed)} skipped={skipped}",
  388. )
  389. def _is_backend_argv(argv_low: str) -> bool:
  390. """Whether an argv is a DESKTOP backend — feeds ``taskkill /T`` via ``_orphaned_desktop_backend_pids``.
  391. Same predicate as ``_looks_like_desktop_control_plane``: ``-m hermes_cli.main`` entry shape (the
  392. Desktop's only spawn shape, ``apps/desktop/electron/main.ts``) AND the canonical holder classifier says
  393. ``serve``/``dashboard``. A user-launched ``hermes.exe serve`` / ``hermes dashboard`` is NOT the
  394. Desktop's: the guard refuses on it, never reaps it.
  395. """
  396. return _looks_like_desktop_control_plane(argv_low)
  397. def _live_argv_low(psutil, pid, cmdline: str) -> str | None:
  398. """Current lower-cased argv of *pid* (falls back to the scanned *cmdline*); ``None`` if it exited."""
  399. argv = cmdline
  400. try:
  401. argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline
  402. except psutil.NoSuchProcess:
  403. return None
  404. except Exception:
  405. pass
  406. return argv.lower()
  407. def _orphaned_desktop_backend_pids(matches: list[tuple[int, str, str]]) -> list[tuple[int, int]] | None:
  408. """``(pid, start_time)`` roots from *matches* when every remaining holder is an ORPHANED backend, else ``None``.
  409. Killing a Desktop-owned ``serve`` is futile (the app respawns it), but a straggler whose Desktop is gone
  410. would dead-end the update with "Hermes is still running" and zero open windows. Qualifies only if cmdline
  411. is a Hermes backend AND the parent is demonstrably gone (PID missing or reused). Tree-aware: holders inside
  412. an accepted root's tree fold into it; only roots are returned (``taskkill /T`` reaps descendants). Any
  413. live-parent backend, unjustified non-backend, unprovable case, or no psutil -> ``None``. Never raises.
  414. The venv-holder guard refuses on the Desktop app's ``serve`` backend by design: while the Desktop is
  415. open, killing its backend is futile (the app supervises and respawns it within seconds), so the user
  416. must close the app. But in the GUI-updater handoff path the Desktop has *already exited* — by contract
  417. it tree-kills its backends and waits for the venv shim before spawning hermes-setup, and the
  418. update-in-progress marker parks any relaunched Desktop from spawning a fresh backend (#50238). A
  419. ``serve`` backend still holding the venv at that point is a straggler whose supervisor is gone: SIGTERM
  420. raced its spawn, or it belongs to a crashed window. Nothing will respawn it, and refusing on it
  421. dead-ends the update with "Hermes is still running" while the user stares at zero open windows (ryanc's
  422. 2026-08-09 01:59/02:17 failures).
  423. """
  424. psutil = _psutil()
  425. if psutil is None:
  426. return None
  427. # Pass 1: find orphaned backend ROOTS among the holders.
  428. roots: list[tuple[int, int]] = []
  429. remaining: list[int] = [] # holders still to justify
  430. for pid, _name, cmdline in matches:
  431. low = _live_argv_low(psutil, pid, cmdline)
  432. if low is None:
  433. continue # exited between scan and classification — nothing to reap
  434. if not _is_backend_argv(low):
  435. remaining.append(int(pid))
  436. continue
  437. try:
  438. proc = psutil.Process(int(pid))
  439. # Fingerprint from the SAME psutil handle, centisecond-quantized like
  440. # gateway.status.get_process_start_time so pid_is_hermes round-trips at kill time.
  441. process_start_time = round(proc.create_time() * 100)
  442. except psutil.NoSuchProcess:
  443. continue # exited during classification — nothing to reap
  444. except Exception:
  445. return None
  446. try:
  447. ppid = proc.ppid()
  448. parent = psutil.Process(ppid) if ppid else None
  449. # PID-reuse check: a "parent" created after its child is a recycled PID. A live parent is
  450. # not a root but may be an orphan root's descendant (the venv trampoline re-execs uv
  451. # python with the SAME argv) — defer to pass 2.
  452. if parent is not None and parent.is_running() and parent.create_time() <= proc.create_time():
  453. remaining.append(int(pid))
  454. continue
  455. except psutil.NoSuchProcess:
  456. pass # parent gone → orphan
  457. except Exception:
  458. return None
  459. roots.append((int(pid), process_start_time))
  460. # Pass 2: every non-backend holder must descend from an accepted orphan root
  461. # (dies with the tree reap); anything else keeps the refusal.
  462. root_set = {pid for pid, _start_time in roots}
  463. for pid in remaining:
  464. try:
  465. if not root_set or not root_set & {int(a.pid) for a in psutil.Process(pid).parents()}:
  466. return None
  467. except psutil.NoSuchProcess:
  468. continue # exited already
  469. except Exception:
  470. return None
  471. return roots
  472. def _ledger_reapable_backend_pids(matches: list[tuple[int, str, str]]) -> list[int]:
  473. """PIDs the spawn ledger positively identifies as orphaned backends; never raises.
  474. Strongest rung (no PPID/cmdline inference): qualifies when ``(pid, create_time)`` matches a live ledger entry
  475. (PID reuse can't forge it), purpose is a REAPABLE kind (never interactive), and the recorded SPAWNER is
  476. provably dead. Safe in ANY context. Unlisted holders fall to later rungs and never disqualify identified ones."""
  477. try:
  478. from hermes_cli.process_identity import REAPABLE_PURPOSES, ledger_entries, spawner_is_dead
  479. entries = ledger_entries()
  480. except Exception:
  481. return []
  482. by_pid = {e.get("pid"): e for e in entries if isinstance(e.get("pid"), int)}
  483. return [
  484. int(pid) for pid, _name, _cmdline in matches
  485. if (entry := by_pid.get(int(pid))) and entry.get("purpose") in REAPABLE_PURPOSES and spawner_is_dead(entry) is True
  486. ]
  487. def _handoff_reapable_backend_pids(matches: list[tuple[int, str, str]]) -> list[int] | None:
  488. """Backend PIDs safe to tree-reap during a GUI-updater hand-off, INCLUDING ones with a live parent; never raises.
  489. The orphan-only rung bails on ANY live parent (mid-teardown Electron, launcher->worker chain) and hung a
  490. hand-off. Inside the hand-off gate (marker + ``--gateway`` + no live ``hermes.exe`` shim) nothing legitimate
  491. supervises a ``serve`` from this venv, so survivors are leaks. Any non-backend holder or no psutil ->
  492. ``None``. The CALLER must have confirmed the gate; outside it the stricter orphan-only path stands.
  493. Any ``serve`` backend still holding the venv here is therefore a leak, live parent or not, and reaping
  494. its tree is correct rather than a race. See #50238.
  495. """
  496. psutil = _psutil()
  497. if psutil is None:
  498. return None
  499. roots: list[int] = []
  500. for pid, _name, cmdline in matches:
  501. low = _live_argv_low(psutil, pid, cmdline)
  502. if low is None:
  503. continue # exited — nothing to reap
  504. if not _is_backend_argv(low):
  505. return None # unexpected non-backend holder: refuse the whole set
  506. roots.append(int(pid))
  507. return roots or None
  508. def _stop_process_trees(pids: list[int] | list[tuple[int, int]]) -> None:
  509. """Force-stop each PID with its full child tree (Windows); best effort, never raises.
  510. ``taskkill /T /F``: stopping only the parent can leave a ``.hermes-runtime`` child holding the install open.
  511. See #70026.
  512. """
  513. from gateway.status import get_process_start_time
  514. from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags
  515. for entry in pids:
  516. pid, expected_start_time = entry if isinstance(entry, tuple) else (int(entry), get_process_start_time(int(entry)))
  517. try:
  518. if expected_start_time is None:
  519. logger.debug("Skipping taskkill of PID %s: process identity unavailable", pid)
  520. continue
  521. if not pid_is_hermes(pid, expected_start_time=expected_start_time):
  522. logger.debug("Skipping taskkill of non-Hermes or changed PID %s", pid)
  523. continue
  524. subprocess.run(
  525. ["taskkill", "/PID", str(pid), "/T", "/F"], check=False,
  526. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
  527. creationflags=windows_hide_flags(),
  528. )
  529. except Exception as exc:
  530. logger.debug("Could not stop process tree %s: %s", pid, exc)
  531. def _looks_like_desktop_control_plane(cmdline: str) -> bool:
  532. """True for this-install ``hermes serve`` / ``hermes dashboard`` argv (Desktop control plane).
  533. Not the messaging gateway — don't feed into ``looks_like_gateway_command_line``. Token-based via the
  534. parser-derived classifier, never substring (``kanban --preserve-cache``, ``-m dashboard chat``).
  535. Undeterminable subcommand is NOT a control plane.
  536. See #92091.
  537. A cmdline whose subcommand cannot be determined is NOT a control plane — callers must not guess
  538. ownership. See #90778, #91869.
  539. """
  540. return "hermes_cli.main" in (cmdline or "").lower() and _hermes_holder_subcommand(cmdline) in _BACKEND_PURPOSES
  541. def _desktop_owns_gateway_lifecycle() -> bool:
  542. """True when Desktop currently supervises this install's control plane (updater must not steal gateway start).
  543. Not proof messaging is served: serve is the control plane, the gateway a detached sibling. Prefer the spawn
  544. ledger; fall back to the venv-holder scan. An orphaned control plane (supervisor gone) does not count;
  545. without psutil orphanhood is unprovable and a live control plane suffices.
  546. See #76129, #92091.
  547. """
  548. from hermes_cli.update_cmd import _m
  549. with _best_effort('Desktop-lifecycle ledger probe failed: %s'):
  550. from hermes_cli.process_identity import ledger_entries, spawner_is_dead
  551. if any(e.get("purpose") in _BACKEND_PURPOSES and spawner_is_dead(e) is False for e in ledger_entries()):
  552. return True
  553. psutil = _psutil()
  554. for pid, _name, cmdline in _try_call(_detect_venv_python_processes, "Desktop-lifecycle holder scan failed: %s") or []:
  555. if not _looks_like_desktop_control_plane(cmdline):
  556. continue
  557. if psutil is None:
  558. return True
  559. with suppress(Exception):
  560. if _parent_is_live(psutil.Process(int(pid))):
  561. return True
  562. return False
  563. def _win_service(name: str):
  564. """``(psutil, service)`` for the named SCM service (psutil imported here so tests can stub the module)."""
  565. import psutil
  566. return psutil, psutil.win_service_get(name)
  567. def _sc_exe(verb: str, name: str, service, settled_status: str) -> None:
  568. """``sc.exe <verb> <name>``; a non-zero exit is only an error when SCM doesn't already report *settled_status*."""
  569. result = subprocess.run(
  570. ["sc.exe", verb, name], capture_output=True, text=True, encoding="utf-8", errors="replace",
  571. timeout=10, check=False,
  572. )
  573. if result.returncode != 0 and service.status() != settled_status:
  574. detail = (result.stderr or result.stdout).strip()
  575. raise RuntimeError(detail or f"sc.exe {verb} failed with {result.returncode}")
  576. def _original_process_is_alive(psutil, pid: int, create_time: float) -> bool:
  577. """Is the process with this exact ``(pid, create_time)`` identity still alive? AccessDenied/unknown reads
  578. True (fail closed: the venv may still be locked)."""
  579. try:
  580. current = float(psutil.Process(pid).create_time())
  581. except (psutil.NoSuchProcess, psutil.ZombieProcess):
  582. return False
  583. except Exception:
  584. return True
  585. return abs(current - create_time) <= 0.001
  586. def _process_create_time(psutil, pid: int, label: str) -> float:
  587. """``create_time`` of *pid*; unreadable identity aborts the stop (RuntimeError)."""
  588. try:
  589. return float(psutil.Process(int(pid)).create_time())
  590. except Exception as exc:
  591. raise RuntimeError(f"Windows {label} process identity is unavailable before stop") from exc
  592. def _verify_service_identities(psutil, name: str, service, expected_service_identity, expected_gateway_identity) -> None:
  593. """Refuse to stop *name* unless SCM state, service/gateway process identities and ancestry all still match."""
  594. if expected_service_identity is not None:
  595. try:
  596. current_status, current_service_pid = str(service.status()), int(service.pid() or 0)
  597. except Exception as exc:
  598. raise RuntimeError(f"Windows service {name} SCM identity is unavailable before stop") from exc
  599. if current_status != "running":
  600. raise RuntimeError(f"Windows service {name} is not stably running before stop: {current_status}")
  601. if current_service_pid != int(expected_service_identity[0]):
  602. raise RuntimeError(f"Windows service {name} SCM process identity changed before stop")
  603. for label, identity in (("service", expected_service_identity), ("gateway", expected_gateway_identity)):
  604. if identity is not None and abs(_process_create_time(psutil, identity[0], label) - float(identity[1])) > 0.001:
  605. raise RuntimeError(f"Windows {label} process identity changed before stop")
  606. if expected_service_identity is not None and expected_gateway_identity is not None:
  607. try:
  608. ancestor_pids = {int(parent.pid) for parent in psutil.Process(int(expected_gateway_identity[0])).parents()}
  609. except Exception as exc:
  610. raise RuntimeError("Windows gateway ancestry is unavailable before service stop") from exc
  611. if int(expected_service_identity[0]) not in ancestor_pids:
  612. raise RuntimeError(f"Windows gateway is no longer owned by service {name}")
  613. def _stop_windows_gateway_service(
  614. name: str, *, expected_processes: tuple[tuple[int, float], ...] = (), expected_service_identity: tuple[int, float] | None = None,
  615. expected_gateway_identity: tuple[int, float] | None = None, timeout: float = 30.0,
  616. ) -> None:
  617. """Stop one verified Windows service and wait until SCM reports it down.
  618. Lingering matching-identity processes after SCM says stopped make venv mutation unsafe — fail closed."""
  619. psutil, service = _win_service(name)
  620. _verify_service_identities(psutil, name, service, expected_service_identity, expected_gateway_identity)
  621. _sc_exe("stop", name, service, "stopped")
  622. def _alive() -> list[int]:
  623. return [pid for pid, create_time in expected_processes if _original_process_is_alive(psutil, pid, create_time)]
  624. if poll_until(lambda: service.status() == "stopped" and not _alive(), timeout, 0.2):
  625. return
  626. if service.status() != "stopped":
  627. raise RuntimeError(f"Windows service {name} did not stop within {timeout:.0f}s; venv mutation unsafe.")
  628. if alive_after_stop := _alive():
  629. raise RuntimeError(f"Windows service {name} stopped but its process tree is still alive: {alive_after_stop}")
  630. def _start_windows_gateway_service(name: str, *, timeout: float = 30.0) -> None:
  631. """Start one previously paused Windows service and verify it is running."""
  632. service = _win_service(name)[1]
  633. _sc_exe("start", name, service, "running")
  634. if not poll_until(lambda: service.status() == "running", timeout, 0.2):
  635. raise RuntimeError(f"Windows service {name} did not start within {timeout:.0f}s")
  636. def _restore_windows_gateway_service(name: str, *, timeout: float = 60.0) -> None:
  637. """Restore a service after an uncertain stop, including STOP_PENDING."""
  638. from hermes_cli.update_cmd import _start_windows_gateway_service
  639. service = _win_service(name)[1]
  640. def _settled() -> bool:
  641. status = service.status()
  642. if status == "stopped":
  643. _start_windows_gateway_service(name)
  644. return status in ("running", "stopped")
  645. if not poll_until(_settled, timeout, 0.2):
  646. raise RuntimeError(f"Windows service {name} did not reach a restorable state within {timeout:.0f}s")
  647. def _windows_cold_start_plan() -> dict | None:
  648. """Pause token for the no-running-gateway case: cold-start after update when an autostart entry exists.
  649. An installed autostart entry is an explicit "I want a gateway" signal; a gateway that died between
  650. updates would otherwise stay down until next login (resume only relaunches what was running).
  651. Desktop-owned lifecycle -> ``None`` (spawning ``gateway run`` beside Desktop races ports/state);
  652. the skip is ownership, not liveness — except when the start attestation reports a vouched-for
  653. gateway that died without a clean exit. The Desktop hand-off exits the app before the updater starts
  654. and can kill the running gateway in those same seconds, so discovery finds no live PID to pause
  655. (#109538) — the dead attestation is the only surviving "a gateway was up" evidence, and the Desktop
  656. only restarts gateways it stopped itself (its hand-off script, after the update verifies; #119809).
  657. Keep the plan, and record the attestation
  658. *generation* that authorized it on the token: the marker is a mutable one-shot that any concurrent
  659. ``hermes gateway status``/``start`` consumes, so execution authorizes the spawn from the token and
  660. consumes only that generation (#110020 review)."""
  661. from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle
  662. from hermes_cli import gateway_windows
  663. with _best_effort('Could not check Windows gateway autostart state before update: %s'):
  664. if not gateway_windows.is_installed():
  665. return None
  666. token = {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": [], "cold_start_if_installed": True}
  667. with _best_effort('Could not check Desktop gateway-lifecycle ownership before update: %s'):
  668. if _desktop_owns_gateway_lifecycle():
  669. generation = gateway_windows.attested_death_generation(current_pids=[])
  670. if generation is None:
  671. logger.debug("Skipping Windows gateway cold-start plan: Desktop owns gateway lifecycle")
  672. return None
  673. token["attested_generation"] = generation
  674. return token
  675. return None
  676. class ServicePauseFailed(RuntimeError):
  677. """A gateway service could not be stopped; ``rollback_failures`` lists what the in-line
  678. rollback could not restart. Empty = everything stopped was restarted, so the pause is void;
  679. otherwise its record must stay, since gateways are still down."""
  680. def __init__(self, detail: str, rollback_failures: list[str]):
  681. super().__init__(detail)
  682. self.rollback_failures = rollback_failures
  683. def _pause_windows_gateway_services(service_gateways, token: dict, profiles: dict, unmapped: list) -> dict:
  684. """Stop each SCM gateway service, recording them on *token*; roll everything back on failure.
  685. Runs after every fallible ordinary-gateway step so a failure here restores the attempted
  686. services AND the already-paused ordinary gateways before re-raising."""
  687. from hermes_cli.update_cmd import _restore_windows_gateway_service, _stop_windows_gateway_service
  688. paused_services = []
  689. current_service_name = None
  690. try:
  691. for service in service_gateways:
  692. current_service_name = str(service.name)
  693. _stop_windows_gateway_service(
  694. current_service_name, expected_processes=tuple(getattr(service, "descendant_identities", ())),
  695. expected_service_identity=(int(service.service_pid), float(service.service_create_time)),
  696. expected_gateway_identity=(int(service.gateway_pid), float(service.gateway_create_time)),
  697. )
  698. paused_services.append(current_service_name)
  699. current_service_name = None
  700. if paused_services:
  701. token.update(services=paused_services, expected_services=list(paused_services), restarted_services=[])
  702. token["service_profiles"] = {str(s.name): str(s.profile) for s in service_gateways if str(s.name) in paused_services}
  703. print(" ✓ Paused Windows gateway service(s): " + ", ".join(paused_services))
  704. return token
  705. except Exception as exc:
  706. restore_names = ([current_service_name] if current_service_name else []) + list(reversed(paused_services))
  707. rollback: list[tuple[str, object]] = [(n, lambda n=n: _restore_windows_gateway_service(n)) for n in dict.fromkeys(restore_names)]
  708. if profiles or unmapped:
  709. rollback.append(("ordinary gateways", lambda: _resume_windows_gateways_after_update(token)))
  710. rollback_failures = []
  711. for label, restore in rollback:
  712. try:
  713. restore()
  714. except Exception as restore_exc:
  715. rollback_failures.append(f"{label}: {restore_exc}")
  716. detail = f"Could not stop Windows gateway service {current_service_name or 'unknown'}: {exc}"
  717. if rollback_failures:
  718. detail += "; rollback failures: " + "; ".join(rollback_failures)
  719. raise ServicePauseFailed(detail, rollback_failures) from exc
  720. def _owned_gateway_pids(pids, *, keep=(), quiet: bool = True) -> list[int]:
  721. """*pids* whose live home this update owns, plus *keep* (PIDs mapped to this install's profile
  722. PID files / services). The same home scope the POSIX fleet restart uses (#93349): a gateway of
  723. another Hermes install, or one whose home cannot be read, is named (unless *quiet*) and left
  724. running, never paused, force-killed or replayed (#124659)."""
  725. from hermes_cli.update_cmd_fleet import _scoped_manual_gateway_pids
  726. return _scoped_manual_gateway_pids(list(pids), keep=keep, quiet=quiet)
  727. def _discover_windows_gateways():
  728. """``(profile_processes, service_gateways, service_gateway_pids, running_pids)`` for the pause; any indeterminate probe aborts."""
  729. from hermes_cli.gateway import find_gateway_pids, find_profile_gateway_processes, find_windows_gateway_services
  730. with _abort_on_error("Could not map Windows gateway PIDs to profiles"):
  731. profile_process_list = find_profile_gateway_processes(strict=True)
  732. profile_processes = {proc.pid: proc for proc in profile_process_list}
  733. with _abort_on_error("Could not determine Windows gateway service ownership"):
  734. service_gateways = find_windows_gateway_services(profile_processes=profile_process_list)
  735. service_gateway_pids = {int(service.gateway_pid) for service in service_gateways}
  736. with _abort_on_error("Could not discover Windows gateway PIDs before update"):
  737. # find_gateway_pids(all_profiles=True) is a HOST-wide scan; only this install's fleet is paused.
  738. running_pids = _owned_gateway_pids(dict.fromkeys(
  739. [*find_gateway_pids(all_profiles=True), *sorted(profile_processes), *sorted(service_gateway_pids)]
  740. ), keep=set(profile_processes) | service_gateway_pids, quiet=False)
  741. return profile_processes, service_gateways, service_gateway_pids, running_pids
  742. def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids, on_request=None):
  743. """Marker + socket-first pause for every profile-mapped gateway; ``(profiles, mapped_pids, socket_acks)``.
  744. Socket ACK = the gateway drains and exits by its own graceful path. No answer (older
  745. gateway) -> the marker poll / force-kill ladder in the caller. ``on_request(pid)`` runs once the
  746. request is issued (marker on disk, else a positive socket ACK) and before the socket call: from
  747. then on a crash leaves that gateway owed while it drains, never before."""
  748. profiles: dict[str, int] = {}
  749. mapped_pids = []
  750. socket_acks: list[dict] = []
  751. for pid in running_pids:
  752. proc = None if pid in service_gateway_pids else profile_processes.get(pid)
  753. if proc is None:
  754. continue
  755. profiles[str(proc.profile)] = int(pid)
  756. mapped_pids.append(int(pid))
  757. requested = _write_update_planned_stop_marker(Path(proc.path), int(pid))
  758. if requested and on_request is not None:
  759. on_request(int(pid))
  760. try:
  761. # Socket-first pause (#92091 step 2): ask the gateway to drain and exit itself instead of
  762. # relying on the marker poll + force-kill ladder. A positive ACK means the gateway is running
  763. # its own graceful restart path (same drain as SIGUSR1/service restarts) and will release its
  764. # venv handles on the way out. No answer (older gateway, no socket) → the marker watcher /
  765. # force-kill fallback below behaves exactly as before this verb existed.
  766. from gateway.control_socket import pause_gateway_for_update
  767. ack = pause_gateway_for_update(Path(proc.path))
  768. if ack and (ack.get("pausing") or ack.get("already_stopping")):
  769. socket_acks.append(ack)
  770. if not requested and on_request is not None:
  771. on_request(int(pid))
  772. except Exception as exc:
  773. logger.debug("Socket pause unavailable for gateway %s: %s", pid, exc)
  774. return profiles, mapped_pids, socket_acks
  775. def _gateway_drain_timeout(socket_acks: list[dict]) -> float:
  776. """Drain budget: configured restart drain (>= 1s), raised to a socket-paused gateway's declared
  777. ACTIVE-TURN budget + teardown grace so it isn't force-killed mid-turn."""
  778. from hermes_cli.gateway import _get_restart_drain_timeout
  779. try:
  780. drain_timeout = max(float(_get_restart_drain_timeout()), 1.0)
  781. except Exception:
  782. drain_timeout = 10.0
  783. if socket_acks:
  784. with suppress(Exception):
  785. declared = max(float(a.get("drain_timeout") or 0.0) for a in socket_acks)
  786. drain_timeout = max(drain_timeout, declared + 10.0)
  787. print(f" → {len(socket_acks)} gateway(s) ACKed socket pause; waiting up to {int(drain_timeout)}s for graceful exit")
  788. return drain_timeout
  789. def _pause_windows_gateways_for_update() -> dict | None:
  790. """Stop running Windows gateways before mutating the checkout or venv.
  791. Scheduled/startup gateways run via pythonw.exe, invisible to the hermes.exe instance guard, yet keep files
  792. locked during ``git``/``uv``. Stop only PIDs the gateway discovery code identifies. The paused set is on
  793. disk (``update_pause_record``) before the first stop, with any set a killed earlier update orphaned."""
  794. from hermes_cli.update_cmd import _m
  795. if not _m()._is_windows():
  796. return None
  797. with _abort_on_error("Could not prepare Windows gateway pause for update"):
  798. import gateway.status
  799. from hermes_cli.gateway import _capture_gateway_argv
  800. from hermes_cli import update_pause_record as pause_record
  801. with _abort_on_error("Could not read the gateways an earlier update paused"):
  802. adopted, claims = pause_record.adopt_orphans()
  803. profile_processes, service_gateways, service_gateway_pids, running_pids = _discover_windows_gateways()
  804. if not running_pids:
  805. return _cold_start_pause_token(adopted, claims)
  806. unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids]
  807. from gateway.status import get_process_start_time
  808. from hermes_cli.update_fleet_scope import gateway_pid_home
  809. born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered
  810. # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it.
  811. # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). Two homes can
  812. # run identical selectorless argv, so the home and birth identify the runtime the replay must restore:
  813. # read with discovery's own resolver (environment, else the exact-birth ledger witness), and a home
  814. # that no longer proves refuses the pause before any stop. ``home: null`` would read as a pre-home
  815. # record, whose argv-only match lets another home's gateway retire this debt.
  816. homes = {int(pid): gateway_pid_home(int(pid)) for pid in unmapped_pids}
  817. if None in homes.values():
  818. raise RuntimeError("Could not prove the Hermes home of unmapped gateway PID(s) "
  819. + ", ".join(str(p) for p, h in homes.items() if h is None) + "; nothing was stopped")
  820. unmapped = [
  821. {"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p),
  822. "Could not capture argv for unmapped gateway %s: %s", int(pid)),
  823. "home": homes[int(pid)], "ct": born[int(pid)]}
  824. for pid in unmapped_pids
  825. ]
  826. intended = {
  827. "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped,
  828. "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids
  829. if pid in profile_processes and pid not in service_gateway_pids},
  830. "identities": {str(pid): pause_record.identity(pid)["ct"] for pid in running_pids},
  831. }
  832. if service_gateways:
  833. names = [str(s.name) for s in service_gateways]
  834. intended.update(services=names, expected_services=list(names), restarted_services=[],
  835. service_profiles={str(s.name): str(s.profile) for s in service_gateways})
  836. with _abort_on_error("Could not record the gateways this update pauses"):
  837. pause_record.record_pause(intended, adopted, claims)
  838. # The intent (and where each request goes), right before the first request. A live entry is owed
  839. # only once its request was issued (mark_stop_sent, or this update's marker still on disk): a
  840. # crash before that leaves it "never asked" (still serving, dropped), after it, draining (owed).
  841. pause_record.mark_stop_requested(
  842. intended, running_pids, markers=_planned_stop_markers(running_pids, profile_processes, service_gateway_pids))
  843. profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped,
  844. on_request=lambda pid: pause_record.mark_stop_sent(intended, pid), born=born)
  845. token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped}
  846. # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the
  847. # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start.
  848. running_profiles = set(profiles) | {str(p.profile) for p in profile_processes.values()} | {str(s.profile) for s in service_gateways}
  849. _record_attested_cold_start_profiles(token, running_profiles)
  850. try:
  851. token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped)
  852. except ServicePauseFailed as exc:
  853. if not exc.rollback_failures: # everything stopped was restarted in-line
  854. pause_record.abandon_pause(intended, adopted)
  855. raise
  856. return pause_record.finish_pause(token, intended, adopted)
  857. def _cold_start_pause_token(adopted: dict | None, claims: list) -> dict | None:
  858. """No gateway runs: the cold-start plan, plus any orphaned set adopted from a killed update."""
  859. from hermes_cli import update_pause_record as pause_record
  860. token = _windows_cold_start_plan()
  861. # Other profiles may hold a dead attestation even when the active profile owes nothing
  862. # (clean exit, autostart not installed): give them a token to ride on.
  863. probe = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
  864. _record_attested_cold_start_profiles(probe, set())
  865. token = probe if probe.get("cold_start_profiles") else token
  866. if adopted is None:
  867. return pause_record.stamp_tree(token) if token is not None else None
  868. token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
  869. with _abort_on_error("Could not record the adopted paused-gateway set"):
  870. return pause_record.record_pause(token, adopted, claims)
  871. def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped,
  872. on_request=None, born=None) -> dict:
  873. """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``.
  874. A force-killed process needs no stop record: dead, it is owed; alive, it was never stopped.
  875. The kill is authorized by each process's birth as discovered (*born*, a mapped gateway's
  876. strict PID-file identity first), never re-read after the drain: a PID that exited and was
  877. reused meanwhile reads its replacement's own birth. No discovered birth refuses the kill."""
  878. from gateway.status import get_process_start_time, terminate_pid
  879. from hermes_cli.update_cmd import _m
  880. mapped = [int(p) for p in running_pids if int(p) not in service_gateway_pids and int(p) in profile_processes]
  881. born = {**{int(k): v for k, v in (born or {}).items()},
  882. **{p: round(float(profile_processes[p].create_time) * 100) for p in mapped
  883. if float(getattr(profile_processes[p], "create_time", 0) or 0) > 0}}
  884. # Resolve venv-side launchers before any request: the launcher keeps ``.pyd`` mapped and would trip the
  885. # venv-holder guard, so it is killed with the survivors, under the identity it has now.
  886. launcher_pids = _m()._venv_launcher_ancestors(mapped)
  887. born.update({int(p): get_process_start_time(int(p)) for p in launcher_pids if int(p) not in born})
  888. profiles, mapped_pids, socket_acks = _request_socket_pauses(
  889. running_pids, profile_processes, service_gateway_pids, on_request=on_request)
  890. print("→ Stopping Windows gateway process(es) before updating Hermes...")
  891. drain_timeout = _gateway_drain_timeout(socket_acks)
  892. survivors = _m()._wait_for_windows_update_gateway_exit(mapped_pids, timeout=drain_timeout)
  893. # Tree-kill survivors, unmapped gateways, and pre-request launchers; one already gone raises
  894. # ProcessLookupError, one whose birth differs (or was never read) is refused, and both are skipped.
  895. force_killed = []
  896. for pid in sorted(set(survivors).union(unmapped_pids).union(launcher_pids)):
  897. with suppress(ProcessLookupError, PermissionError, OSError):
  898. terminate_pid(int(pid), force=True, expected_start_time=born.get(int(pid)))
  899. force_killed.append(int(pid))
  900. if profiles:
  901. print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}")
  902. if force_killed:
  903. print(f" → Force-stopped {len(force_killed)} gateway process(es)")
  904. if unmapped_pids:
  905. print(f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping")
  906. if any(not u.get("argv") for u in unmapped): # no recoverable cmdline (psutil missing, denied, gone)
  907. print(" Restart manually after update: hermes gateway run")
  908. return profiles
  909. def _record_attested_cold_start_profiles(token: dict, running_profiles: set) -> None:
  910. """Record ``token["cold_start_profiles"] = {name: generation}`` for every known profile that is not
  911. running yet holds a start attestation for a gateway that died without a clean exit (#110959).
  912. The all-or-nothing plan only ever probed the ACTIVE profile and only when NO gateway ran at all, so a
  913. dead-but-attested default beside a still-running ``beta`` never got a cold-start obligation. Only
  914. Desktop-owned installs need this (elsewhere autostart brings the profile back); the active profile is
  915. left to the existing plan so it is never spawned twice. Best-effort: never blocks the pause."""
  916. from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle
  917. from hermes_cli import gateway_windows
  918. with _best_effort("Could not evaluate per-profile attested cold-starts before update: %s"):
  919. if not _desktop_owns_gateway_lifecycle():
  920. return
  921. from hermes_cli.profiles import get_active_profile_name, profiles_to_serve
  922. active = get_active_profile_name() or "default"
  923. cold: dict[str, str] = {}
  924. # An activation list, not inventory: parked profiles stay offline.
  925. for name, home in profiles_to_serve(multiplex=True, include_standalone=True):
  926. if name in running_profiles or (name == active and token.get("cold_start_if_installed")):
  927. continue
  928. generation = gateway_windows.attested_death_generation([], home=Path(home))
  929. if generation:
  930. cold[name] = generation
  931. if cold:
  932. token["cold_start_profiles"] = cold
  933. def _cold_start_attested_profiles(token: dict) -> None:
  934. """Spawn each ``cold_start_profiles`` entry under its own HERMES_HOME and consume exactly the
  935. generation that authorized it; one profile's failure never aborts the others (#110959)."""
  936. from hermes_cli import gateway_windows
  937. from hermes_cli.profiles import get_profile_dir
  938. pending = dict(token.get("cold_start_profiles") or {})
  939. if not pending:
  940. return
  941. for name, generation in sorted(pending.items()):
  942. home = Path(get_profile_dir(name))
  943. with _best_effort(f"Could not cold-start Windows gateway profile {name} after update: %s"):
  944. if not gateway_windows._live_gateway_pids(home=home): # a concurrent autostart must not be doubled
  945. pid = gateway_windows._spawn_detached(home=home)
  946. if not pid:
  947. raise RuntimeError("cold-start did not return a process ID")
  948. ready_pids = gateway_windows._wait_for_gateway_ready(home=home)
  949. if not ready_pids:
  950. raise RuntimeError(f"gateway profile {name} did not become ready")
  951. gateway_windows._consume_start_attestation(generation, home=home)
  952. # Keep the attestation chain: a death after this CLI exits must be visible to the next update.
  953. gateway_windows._write_start_attestation(ready_pids, f"cold-start after update (profile {name})", home=home)
  954. print(f"\n✓ Gateway profile {name} started via cold-start after update (PID: {ready_pids[0]})")
  955. token["cold_start_profiles"].pop(name, None)
  956. if token["cold_start_profiles"]:
  957. # Surface the miss like the active-profile cold-start does: the merged outcome marks the update
  958. # incomplete instead of reporting success with a profile still down.
  959. raise RuntimeError("Windows gateway cold-start was not verified for profile(s): "
  960. + ", ".join(sorted(token["cold_start_profiles"])))
  961. token.pop("cold_start_profiles", None)
  962. def _cold_start_windows_gateway_after_update(token: dict | None = None) -> bool:
  963. """Direct-spawn a detached gateway after update for the ``cold_start_if_installed`` case (installed but down).
  964. Idempotent: re-checks nothing is running so a concurrent autostart can't duplicate. A successful Popen
  965. doesn't prove survival (a job object denying breakaway kills it), so success is gated on the liveness poll.
  966. Vouched PIDs are attested so a death AFTER updater exit is reported by the next CLI invocation.
  967. A successful ``Popen`` only proves the process was created, not that it survived (e.g. a Windows job
  968. object denying breakaway kills it before it logs anything — #84185). So the success line is gated on the
  969. same post-spawn liveness poll every other ``_spawn_detached`` caller uses
  970. (``gateway_windows._report_gateway_start``), instead of being printed unconditionally from the returned
  971. PID.
  972. Desktop-owned lifecycle suppresses the spawn only while nothing attests a gateway is expected: an
  973. attested gateway that died without a clean exit is restored even then (#109538) — the Desktop does
  974. not restart the messaging gateway itself. That authority is the ``attested_generation`` the plan
  975. recorded on ``token``, not the marker on disk: the marker is a mutable one-shot a concurrent
  976. ``hermes gateway status``/``start`` consumes, which would otherwise skip this spawn and clear the
  977. token (#110020 review). Only that generation is consumed afterwards — never a newer marker.
  978. """
  979. from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle, _m
  980. if not _m()._is_windows():
  981. return True
  982. with _abort_on_error("Could not load Windows gateway cold-start helpers"):
  983. from hermes_cli import gateway_windows
  984. from hermes_cli.gateway import find_gateway_pids
  985. with _abort_on_error("Could not re-check gateway liveness before cold-start"):
  986. # Another install's live gateway must not suppress this install's cold start (#124659).
  987. if _owned_gateway_pids(find_gateway_pids(all_profiles=True)):
  988. return True
  989. token = token or {}
  990. generation = token.get("attested_generation")
  991. if generation is None and "attested_generation" not in token:
  992. # Token written by pre-generation code and resumed across this very update: probe the marker.
  993. with _abort_on_error("Could not re-read the start attestation before cold-start"):
  994. generation = gateway_windows.attested_death_generation(current_pids=[])
  995. with _abort_on_error("Could not re-check Desktop gateway-lifecycle ownership before cold-start"):
  996. if _desktop_owns_gateway_lifecycle() and not generation:
  997. logger.debug("Skipping Windows gateway cold-start: Desktop owns gateway lifecycle")
  998. return True
  999. with _abort_on_error("Could not cold-start Windows gateway after update"):
  1000. pid = gateway_windows._spawn_detached()
  1001. if not pid:
  1002. raise RuntimeError("Windows gateway cold-start did not return a process ID")
  1003. ready_pids = gateway_windows._wait_for_gateway_ready()
  1004. if not ready_pids:
  1005. raise RuntimeError(f"Windows gateway cold-start PID {pid} did not become ready")
  1006. # The dead attestation has done its job (it authorized this spawn under Desktop ownership). Consume
  1007. # it only now: a spawn that never became ready leaves it in place, so the registered retry still
  1008. # holds its recovery obligation instead of seeing Desktop ownership with no marker and returning
  1009. # success without a gateway (#110020 review).
  1010. if generation:
  1011. gateway_windows._consume_start_attestation(generation)
  1012. print(f"\n✓ Gateway started via cold-start after update (PID: {', '.join(map(str, ready_pids))})")
  1013. with suppress(Exception):
  1014. gateway_windows._write_start_attestation(ready_pids, "cold-start after update")
  1015. return True
  1016. def _refresh_windows_gateway_launchers() -> None:
  1017. """Regenerate installed Windows gateway launcher scripts after update; best-effort, never fails the update.
  1018. Launchers are written once at install, so old installs kept launching via ``pythonw.exe`` (``sys.stderr is
  1019. None`` death). The task's /TR points at a stable path, so rewriting in place retargets it without UAC.
  1020. The Scheduled Task / Startup-folder launchers (``gateway.cmd`` + ``gateway.vbs``) are persistence
  1021. artifacts written once at install time — ``hermes update`` never touched them, so installs created
  1022. before the hidden-console rework (aa2ae36c3f) kept launching the gateway through ``pythonw.exe``
  1023. forever: every descendant spawn flashed a conhost (#54220/#56747) and, since #70344, the console-less
  1024. gateway died at startup with ``RuntimeError: sys.stderr is None`` (#71671).
  1025. """
  1026. from hermes_cli.update_cmd import _m
  1027. if not _m()._is_windows():
  1028. return
  1029. with _best_effort('Could not refresh Windows gateway launchers after update: %s'):
  1030. from hermes_cli import gateway_windows
  1031. if gateway_windows.is_installed():
  1032. gateway_windows._write_task_script()
  1033. print(" ✓ Refreshed Windows gateway launcher scripts")
  1034. # Installs from before #80569 can carry a Startup entry beside the task: both fire at logon.
  1035. done, warnings = gateway_windows.reconcile_autostart_launchers()
  1036. for message in done:
  1037. print(f" ✓ {message}")
  1038. for message in warnings:
  1039. print(f" ⚠ {message}")
  1040. if gateway_windows.is_task_registered():
  1041. # A task registered by an older build never picks up template hardening otherwise (#113670).
  1042. gateway_windows.reconcile_scheduled_task(gateway_windows.get_task_name())
  1043. def _refresh_bootstrap_cache_scripts(branch: str = "main") -> None:
  1044. """Overwrite ``$HERMES_HOME/bootstrap-cache/install-<ref>.{ps1,sh}`` for *branch* from the fresh checkout.
  1045. Old ``hermes-setup.exe`` builds NEVER re-download a cached branch-ref script, so a stale one runs
  1046. months-old code forever. Guards mirror ``install_script.rs``: only the sanitized *branch* key is rewritten;
  1047. commit-SHA pins (7-40 hex) are immutable and skipped. Best-effort: never fails the update.
  1048. Installer binaries built before the #67193 cache-refresh fix (June 2026 and earlier) NEVER re-download a
  1049. cached branch-ref script — ``install-main.ps1`` cached at install time is reused forever, executing
  1050. months-stale code with long-fixed bugs (the 2026-08-09 incident: a June 4 cached script's venv stage
  1051. lacked the 81327 process-tree sweep and died on ``Access denied``). The binary has no self-update path,
  1052. so the poisoned cache outlives every ``hermes update``.
  1053. Overwriting the cached script for *branch* with the freshly pulled ``scripts/install.ps1`` /
  1054. ``scripts/install.sh`` on every update turns the stale binary's unconditional reuse into a feature: it
  1055. "reuses" a file this function keeps permanently current. Post-#67193 installers re-download on each run
  1056. anyway, so for them this is a harmless pre-seed of the same bytes.
  1057. The .ps1 copy gets a UTF-8 BOM to match the installer's cache format (#67193 encoding fix).
  1058. """
  1059. from hermes_cli.update_cmd import _m
  1060. with _best_effort('Could not refresh bootstrap-cache scripts after update: %s'):
  1061. cache_dir = Path(_m().get_hermes_home()) / "bootstrap-cache"
  1062. if not cache_dir.is_dir():
  1063. return
  1064. safe_ref = re.sub(r"[^A-Za-z0-9._-]", "_", str(branch or "main")) # install_script.rs::sanitize_ref()
  1065. if re.fullmatch(r"[0-9a-fA-F]{7,40}", safe_ref): # install_script.rs::is_valid_commit(): immutable pin
  1066. return
  1067. refreshed = []
  1068. for kind, src_name in (("ps1", "install.ps1"), ("sh", "install.sh")):
  1069. src = _m().PROJECT_ROOT / "scripts" / src_name
  1070. cached = cache_dir / f"install-{safe_ref}.{kind}"
  1071. if not src.is_file() or not cached.is_file():
  1072. continue # this ref was never bootstrap-cached — nothing to heal
  1073. data = src.read_bytes()
  1074. if kind == "ps1" and not data.startswith(b"\xef\xbb\xbf"):
  1075. data = b"\xef\xbb\xbf" + data # PowerShell needs the BOM or localized/em-dash text mis-decodes.
  1076. # See #67193.
  1077. if cached.read_bytes() == data:
  1078. continue
  1079. tmp = cached.with_suffix(cached.suffix + ".tmp")
  1080. tmp.write_bytes(data)
  1081. os.replace(tmp, cached)
  1082. refreshed.append(cached.name)
  1083. if refreshed:
  1084. print(" ✓ Refreshed installer bootstrap-cache script(s): " + ", ".join(sorted(refreshed)))
  1085. _SERVICE_READY_TIMEOUT_S = 60.0
  1086. # A launch recovering an orphaned pause (``update_pause_record.recover``, before every command)
  1087. # only probes briefly, and a service whose gateway stays unready is re-probed with backoff; the
  1088. # full wait belongs to ``hermes update`` itself.
  1089. _SERVICE_READY_PROBE_S = 5.0
  1090. _SERVICE_RETRY_BASE_S = 60.0
  1091. _SERVICE_RETRY_MAX_S = 3600.0
  1092. def _service_gateway_ready(name: str, profile: str | None, timeout_s: float | None = None) -> list[int]:
  1093. """The stable gateway *name*'s service process supervises (in *profile*'s home when known): SCM
  1094. ``running`` proves only the wrapper started, not that the gateway it hosts came up."""
  1095. from hermes_cli import gateway_windows
  1096. from hermes_cli.profiles import get_profile_dir
  1097. psutil, service = _win_service(name)
  1098. def under_service(pids):
  1099. try:
  1100. service_pid = int(service.pid() or 0)
  1101. except (psutil.Error, OSError, ValueError, TypeError): # SCM/psutil failure or a non-int pid
  1102. return []
  1103. owned = []
  1104. for pid in pids:
  1105. # A gateway that exited or a process tree psutil cannot read is not the service's; any
  1106. # other error is a bug the resume boundary reports, not a silent "not under service".
  1107. with suppress(psutil.Error, OSError):
  1108. if service_pid and service_pid in {int(p.pid) for p in psutil.Process(int(pid)).parents()}:
  1109. owned.append(int(pid))
  1110. return owned
  1111. home = Path(get_profile_dir(profile)) if profile else None
  1112. return gateway_windows._wait_for_gateway_ready(
  1113. timeout_s=_SERVICE_READY_TIMEOUT_S if timeout_s is None else timeout_s, home=home,
  1114. all_profiles=home is None, pid_filter=under_service)
  1115. def _service_running(name: str) -> bool:
  1116. try:
  1117. import psutil
  1118. except ImportError:
  1119. return False
  1120. try:
  1121. return _win_service(name)[1].status() == "running"
  1122. except (psutil.Error, OSError, AttributeError): # missing service, SCM access, non-Windows psutil
  1123. return False
  1124. def _resume_windows_services(token: dict) -> None:
  1125. """Restart the SCM services recorded on *token*; one leaves the token only once the gateway it
  1126. supervises is ready. Failed ones stay on the token so a retry sees them.
  1127. A recovering launch (``token["recovery"]``) probes readiness briefly, and a running service
  1128. whose gateway was not ready is not re-probed before its backoff (``service_retry``) elapses:
  1129. the debt stays owed without stalling every command."""
  1130. from hermes_cli.update_cmd import _start_windows_gateway_service
  1131. services = list(token.get("services") or [])
  1132. token.setdefault("expected_services", list(services))
  1133. verified_restarts = list(token.get("restarted_services") or [])
  1134. retry = {name: state for name, state in (token.get("service_retry") or {}).items() if name in services}
  1135. recovering = bool(token.get("recovery"))
  1136. restarted_services = []
  1137. failed_services = []
  1138. for service_name in map(str, services):
  1139. backoff = retry.get(service_name) or {}
  1140. try:
  1141. if recovering and _time.time() < float(backoff.get("next_at") or 0) and _service_running(service_name):
  1142. failed_services.append(service_name)
  1143. print(f" ⚠ Windows gateway service {service_name}: its gateway was not ready; "
  1144. f"next check after {int(float(backoff['next_at']) - _time.time())}s")
  1145. continue
  1146. _start_windows_gateway_service(service_name)
  1147. if not _service_gateway_ready(service_name, (token.get("service_profiles") or {}).get(service_name),
  1148. _SERVICE_READY_PROBE_S if recovering else None):
  1149. attempts = int(backoff.get("attempts") or 0) + 1
  1150. retry[service_name] = {"attempts": attempts, "next_at": _time.time() + min(
  1151. _SERVICE_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _SERVICE_RETRY_MAX_S)}
  1152. raise RuntimeError(f"Windows service {service_name} is running but its gateway did not become ready")
  1153. retry.pop(service_name, None)
  1154. restarted_services.append(service_name)
  1155. if service_name not in verified_restarts:
  1156. verified_restarts.append(service_name)
  1157. except Exception as exc:
  1158. logger.warning("Could not restart Windows gateway service %s after update: %s", service_name, exc)
  1159. print(f" ⚠ Could not restart Windows gateway service: {service_name}")
  1160. failed_services.append(service_name)
  1161. token["restarted_services"] = verified_restarts
  1162. token["services"] = failed_services
  1163. if retry:
  1164. token["service_retry"] = retry
  1165. else:
  1166. token.pop("service_retry", None)
  1167. if failed_services:
  1168. raise RuntimeError("Could not restart Windows gateway service(s): " + ", ".join(failed_services))
  1169. if restarted_services:
  1170. print("\n ✓ Restarted Windows gateway service(s): " + ", ".join(restarted_services))
  1171. def _relaunch_paused_gateways(profiles: dict, unmapped: list) -> tuple[dict, list]:
  1172. """Spawn the restart watcher for each paused profile gateway and unmapped argv; ``(launched
  1173. profiles, launched unmapped entries)``. Nothing leaves the token here: a watcher only proves a
  1174. relaunch was scheduled, so every entry stays owed until ITS gateway is verified ready."""
  1175. with _abort_on_error("Could not load Windows gateway restart helper"):
  1176. from hermes_cli.gateway import launch_detached_gateway_restart_by_cmdline, launch_detached_profile_gateway_restart
  1177. # An exception from a launch (incl. bad pid/argv coercion) logs at debug and reads as a failed relaunch.
  1178. launched = {}
  1179. for profile, old_pid in sorted(profiles.items()):
  1180. if _try_call(lambda p=profile, o=old_pid: launch_detached_profile_gateway_restart(str(p), int(o)),
  1181. "Could not restart Windows gateway profile %s after update: %s", profile):
  1182. launched[str(profile)] = int(old_pid)
  1183. launched_unmapped = []
  1184. for entry in unmapped:
  1185. argv, old_pid = entry.get("argv"), entry.get("pid")
  1186. # On the paused runtime's home (None on a pre-home record: the updater's own home, as before).
  1187. if argv and old_pid and _try_call(lambda o=old_pid, a=argv, h=entry.get("home"): launch_detached_gateway_restart_by_cmdline(
  1188. int(o), list(a), home=h),
  1189. "Could not restart unmapped Windows gateway (pid %s) after update: %s", old_pid):
  1190. launched_unmapped.append(entry)
  1191. return launched, launched_unmapped
  1192. _RELAUNCH_VERIFY_TIMEOUT_S = 30.0
  1193. def _pending_relaunch_pids(profiles: dict, unmapped: list, pid_exists) -> list[int]:
  1194. """Old PIDs a restart watcher is still waiting on, sorted; empty when every relaunch can have run.
  1195. ``_spawn_gateway_restart_watcher`` respawns the gateway only once the PID it was handed is gone,
  1196. so while any of them is alive the relaunch has provably not started yet. Pure function of data
  1197. (``pid_exists`` is injected) so the decision is testable off Windows.
  1198. """
  1199. candidates = [int(pid) for pid in profiles.values()]
  1200. candidates += [int(entry["pid"]) for entry in unmapped if entry.get("argv") and entry.get("pid")]
  1201. return sorted({pid for pid in candidates if pid > 0 and pid_exists(pid)})
  1202. def _relaunch_verify_timeout_s(profiles: dict, unmapped: list, pid_exists) -> float:
  1203. """Liveness budget for the post-relaunch poll.
  1204. The base window assumes the watchers respawn immediately. When an old PID is still alive the
  1205. watcher is still in its wait loop, so the poll must reach at least the watcher's own deadline —
  1206. otherwise ``hermes update`` declares "no stable gateway process appeared" for a gateway that was
  1207. never scheduled to appear inside the window (#107002).
  1208. """
  1209. from hermes_cli.gateway import GATEWAY_RESTART_WATCHER_TIMEOUT_S
  1210. if not _pending_relaunch_pids(profiles, unmapped, pid_exists):
  1211. return _RELAUNCH_VERIFY_TIMEOUT_S
  1212. return float(GATEWAY_RESTART_WATCHER_TIMEOUT_S) + _RELAUNCH_VERIFY_TIMEOUT_S
  1213. def _unmapped_ready_filter(entry: dict, taken: set):
  1214. """``pid_filter`` matching the gateway an unmapped entry's replay started: a gateway running the
  1215. same arguments after the (normalized) interpreter on the entry's home, born after the process it
  1216. replaces, not one another entry already vouched for. Two homes can run identical selectorless argv,
  1217. so argv alone would let home B's gateway retire home A's debt (a service gateway on another home
  1218. included; on A's home only one gateway can run). Only an entry recorded before ``home`` was
  1219. captured (no key: a pre-durable-pause updater's token) keeps the argv-only match; a recorded
  1220. ``home`` that is empty proves no runtime and is never matched."""
  1221. import psutil
  1222. from gateway.status import _looks_like_gateway_process, _same_hermes_home, get_process_start_time
  1223. from hermes_cli.update_fleet_scope import gateway_pid_home
  1224. tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0)
  1225. home, old_ct = entry.get("home"), entry.get("ct")
  1226. def same_runtime(pid: int) -> bool:
  1227. if "home" not in entry:
  1228. return True
  1229. if not home:
  1230. return False
  1231. live_home, ct = gateway_pid_home(pid), get_process_start_time(pid)
  1232. return live_home is not None and _same_hermes_home(live_home, home) and (old_ct is None or ct is None or ct > old_ct)
  1233. def matches(pid: int) -> bool:
  1234. if pid == old or pid in taken:
  1235. return False
  1236. try:
  1237. return (list(psutil.Process(pid).cmdline() or [])[1:] == tail and _looks_like_gateway_process(pid)
  1238. and same_runtime(pid))
  1239. except (psutil.Error, OSError, ValueError): # exited, access denied, zombie, invalid pid
  1240. return False
  1241. return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))]
  1242. _READY_INTERVAL_S = 0.4 # gateway_windows._wait_for_gateway_ready's poll interval and
  1243. _READY_CONFIRM_S = 2.0 # confirmation window: one hit proves a spawn, not a survived startup
  1244. def _poll_until_ready(targets: list, deadline: float, taken: set, respawn_window: float | None = None) -> dict:
  1245. """``{key: pids}`` for each ``(key, probe, claim_all)`` target whose gateway stayed visible for
  1246. a whole confirmation window. All targets share one loop, so N targets cost one window, not N:
  1247. one fleet process scan per tick (``probe(scan)`` calls ``scan()`` only if it needs the fleet),
  1248. and each target is credited only on its own hits. *taken* holds, while a target is probed, the
  1249. PIDs earlier targets vouched for (all of a profile's, the first of an unmapped entry's) so one
  1250. gateway never retires two debts. A hit provisional at the deadline still finishes its window;
  1251. a probe that raises leaves its target unready.
  1252. *respawn_window*: a target's own liveness budget, in two parts. The restart watcher only spawns
  1253. once the paused PID is gone, and that wait is charged to the *old* process — 25–35 s of a shared
  1254. 30 s deadline on a busy update — while a cold gateway then needs >11 s before its control socket
  1255. answers. So (a) a target first seen late gets ``respawn_window`` counted from its sighting, and
  1256. (b) a target the deadline expires on without ever being seen gets ONE extra window, because the
  1257. spawn it is waiting for may not have happened yet. Charging that wait to the new process is what
  1258. made an update report "relaunch was not verified alive" for a gateway that was up seconds later;
  1259. the verdict is committed and never revisited, so the stale debt surfaced as a manual follow-up on
  1260. the next launch. Both parts are bounded, so a relaunch that truly never happens still fails
  1261. closed. ``None`` keeps the single shared budget. Each target earns its sighting extension once,
  1262. and the whole poll stops at the entry deadline plus both windows and one confirmation tick: a
  1263. replacement that flaps in and out must not renew its budget forever.
  1264. """
  1265. from hermes_cli.gateway import find_gateway_pids
  1266. ready: dict = {}
  1267. since: dict = {}
  1268. failed: set = set()
  1269. graced: set = set()
  1270. first_seen: set = set()
  1271. hard_stop = deadline + 2 * (respawn_window or 0.0) + _READY_CONFIRM_S + _READY_INTERVAL_S
  1272. while True:
  1273. now = _time.monotonic()
  1274. if now >= hard_stop:
  1275. return ready
  1276. pending = [t for t in targets if t[0] not in ready and t[0] not in failed and (now < deadline or t[0] in since)]
  1277. if not pending:
  1278. starved = [key for key, _probe, _all in targets
  1279. if key not in ready and key not in failed and key not in first_seen and key not in graced]
  1280. if respawn_window is not None and starved:
  1281. graced.update(starved)
  1282. deadline = now + respawn_window
  1283. continue
  1284. return ready
  1285. fleet: list = []
  1286. def scan() -> list:
  1287. if not fleet:
  1288. fleet.append(list(find_gateway_pids(all_profiles=True)))
  1289. return fleet[0]
  1290. taken.clear()
  1291. taken.update(pid for pids in ready.values() for pid in pids)
  1292. for key, probe, claim_all in pending:
  1293. try:
  1294. pids = list(probe(scan))
  1295. except Exception as exc: # health: allow BLE001 -- one target's probe failure must not stall the others
  1296. logger.warning("Could not verify Windows gateway %s after update: %s", key, exc)
  1297. failed.add(key)
  1298. continue
  1299. if not pids:
  1300. since.pop(key, None)
  1301. continue
  1302. vouched = pids if claim_all else pids[:1]
  1303. taken.update(vouched)
  1304. if respawn_window is not None and key not in first_seen:
  1305. # The replacement did not exist before this tick, so the wait that preceded it belongs
  1306. # to the process it replaced: this target's budget starts now, not at the shared
  1307. # deadline's origin.
  1308. first_seen.add(key)
  1309. deadline = max(deadline, now + respawn_window)
  1310. if now - since.setdefault(key, now) >= _READY_CONFIRM_S:
  1311. ready[key] = vouched
  1312. if len(ready) + len(failed) < len(targets):
  1313. _time.sleep(_READY_INTERVAL_S)
  1314. def _verify_relaunched_gateways_alive(token: dict, profiles: dict, unmapped: list) -> None:
  1315. """Retire each relaunched runtime on ITS OWN readiness: a profile on a stable gateway in its
  1316. profile home, an unmapped entry on a stable gateway running its argv. A watcher only proves a
  1317. relaunch was scheduled, and another target's readiness proves nothing about this one.
  1318. A parent Job Object denying CREATE_BREAKAWAY_FROM_JOB can kill the gateway on updater teardown.
  1319. Vouched PIDs are persisted so a death AFTER updater exit is reported by the next CLI invocation."""
  1320. with _abort_on_error("Could not load Windows gateway liveness helpers"):
  1321. from gateway.status import _pid_exists
  1322. from hermes_cli import gateway_windows
  1323. from hermes_cli.profiles import get_profile_dir
  1324. deadline = _time.monotonic() + _relaunch_verify_timeout_s(profiles, unmapped, _pid_exists)
  1325. taken: set = set()
  1326. # Profiles first: a gateway on a profile's home is that profile's, never an unmapped entry's.
  1327. targets = [(("profile", name), lambda _scan, h=Path(get_profile_dir(name)), o=int(old_pid): gateway_windows._live_gateway_pids(
  1328. home=h, pid_filter=lambda ps: [p for p in ps if int(p) != o]), True) for name, old_pid in sorted(profiles.items())]
  1329. targets += [(("unmapped", i), lambda scan, f=_unmapped_ready_filter(entry, taken): f(scan()), False)
  1330. for i, entry in enumerate(unmapped)]
  1331. ready = _poll_until_ready(targets, deadline, taken, respawn_window=_RELAUNCH_VERIFY_TIMEOUT_S)
  1332. ready_profiles = [name for name in sorted(profiles) if ("profile", name) in ready]
  1333. ready_unmapped = [entry for i, entry in enumerate(unmapped) if ("unmapped", i) in ready]
  1334. ready_pids = [pid for pids in ready.values() for pid in pids]
  1335. # Fleet reconciliation (#91277) cross-checks every planned runtime against relaunched_profiles:
  1336. # only a verified one is listed, so an unready one still surfaces as unaccounted.
  1337. token["relaunched_profiles"] = ready_profiles
  1338. token["profiles"] = {n: pid for n, pid in (token.get("profiles") or {}).items() if n not in ready_profiles}
  1339. token["unmapped"] = [u for u in (token.get("unmapped") or []) if u not in ready_unmapped]
  1340. if ready_pids:
  1341. with suppress(Exception):
  1342. gateway_windows._write_start_attestation(sorted(set(ready_pids)), "post-update relaunch")
  1343. if ready_profiles:
  1344. print(f"\n ✓ Restarting Windows gateway profile(s): {', '.join(ready_profiles)}")
  1345. if ready_unmapped:
  1346. print(f" ✓ Restarting {len(ready_unmapped)} unmapped Windows gateway process(es)")
  1347. unready = sorted(set(profiles) - set(ready_profiles)) + [f"pid {u.get('pid')}" for u in unmapped if u not in ready_unmapped]
  1348. if unready:
  1349. print(
  1350. "\n ⚠ Windows gateway restart could not be verified for: " + ", ".join(unready) + "\n"
  1351. " (The respawned gateway may have been killed by a parent Job Object during updater teardown, #48820.)\n"
  1352. " Recover with: hermes gateway restart"
  1353. )
  1354. raise RuntimeError("Windows gateway relaunch after update was not verified alive: " + ", ".join(unready))
  1355. def _resume_windows_gateways_after_update(token: dict | None) -> None:
  1356. """Restart Windows profile gateways previously paused for update."""
  1357. from hermes_cli.update_cmd import _m
  1358. if not token or not token.get("resume_needed"):
  1359. return
  1360. # The foreground call sites register this same function via atexit as a safety net for
  1361. # process death before they get a chance to run it themselves (#115563). Once execution
  1362. # actually reaches here — foreground or the atexit fallback itself — ownership is taken:
  1363. # unregister immediately so a failure below (or the foreground caller failing after this
  1364. # returns) cannot replay the same RuntimeError a second time at interpreter teardown.
  1365. # ``unregister`` is a no-op when this function was never registered.
  1366. import atexit
  1367. atexit.unregister(_resume_windows_gateways_after_update)
  1368. if not _m()._is_windows() and token.get("platform") != "posix":
  1369. token["resume_needed"] = False
  1370. return
  1371. # Startup recovery runs ahead of an unrelated command whose stdout may be parsed (``--json``):
  1372. # its notices go to stderr, as ``recover()``'s own do.
  1373. with redirect_stdout(sys.stderr) if token.get("recovery") else nullcontext():
  1374. _resume_unless_torn(token)
  1375. def _resume_unless_torn(token: dict) -> None:
  1376. """Resume the paused set, or leave it owed when the checkout is not whole."""
  1377. from hermes_cli import update_pause_record as pause_record
  1378. if token.get("pause_id"):
  1379. whole, why = pause_record.tree_is_whole(token)
  1380. if not whole:
  1381. # Starting gateways now would boot them on a half-written tree or the pre-sync
  1382. # dependencies. The durable record keeps the obligation for the next launch, which
  1383. # runs after the interrupted-pull restore and the dependency sync.
  1384. print(f" ⚠ Paused gateway(s) left stopped: {why}.\n"
  1385. " The next `hermes` command (or `hermes update`) restarts them.")
  1386. token["resume_needed"] = False
  1387. token["resume_deferred"] = why
  1388. pause_record.sync(token)
  1389. return
  1390. try:
  1391. _resume_paused_set(token)
  1392. finally:
  1393. pause_record.sync(token)
  1394. def _resume_paused_set(token: dict) -> None:
  1395. """Bring every paused runtime back; each one is retired from *token* only on its own verified
  1396. start, and one failure (an SCM service, a profile) never keeps the others stopped."""
  1397. from hermes_cli.update_cmd import _m
  1398. if token.get("platform") == "posix": # Linux/macOS pre-swap pause (update_cmd_posix_pause)
  1399. from hermes_cli.update_cmd_posix_pause import resume_paused_set
  1400. return resume_paused_set(token)
  1401. # Regenerate launcher scripts before respawning so a legacy pythonw-era
  1402. # autostart entry comes back on the current design at next login too.
  1403. _m()._refresh_windows_gateway_launchers()
  1404. failures: list[str] = []
  1405. def attempt(step) -> None:
  1406. try:
  1407. step()
  1408. except Exception as exc: # health: allow BLE001 -- per-step isolation: any failed step is recorded and re-raised as a summary so it never keeps the other paused runtimes stopped
  1409. failures.append(str(exc))
  1410. attempt(lambda: _resume_windows_services(token))
  1411. profiles = dict(token.get("profiles") or {})
  1412. unmapped = list(token.get("unmapped") or [])
  1413. if profiles or any(u.get("argv") for u in unmapped):
  1414. launched, launched_unmapped = _relaunch_paused_gateways(profiles, unmapped)
  1415. if launched or launched_unmapped:
  1416. attempt(lambda: _verify_relaunched_gateways_alive(token, launched, launched_unmapped))
  1417. else:
  1418. token["relaunched_profiles"] = []
  1419. not_launched = sorted(set(profiles) - set(launched)) + [
  1420. f"pid {u.get('pid')}" for u in unmapped if u not in launched_unmapped]
  1421. if not_launched:
  1422. failures.append("Could not restart every paused Windows gateway: " + ", ".join(not_launched))
  1423. elif token.get("cold_start_if_installed"):
  1424. # Before the per-profile spawns: this guard is fleet-wide (any live gateway ⇒ done).
  1425. def cold_start() -> None:
  1426. if not _m()._cold_start_windows_gateway_after_update(token):
  1427. raise RuntimeError("Windows gateway cold-start was not verified")
  1428. token["cold_start_if_installed"] = False
  1429. attempt(cold_start)
  1430. # After the paused profiles are back: a dead-attested sibling that fails to cold-start must not
  1431. # keep the profiles that WERE running from being relaunched.
  1432. attempt(lambda: _cold_start_attested_profiles(token))
  1433. if failures:
  1434. raise RuntimeError("; ".join(failures))
  1435. token["resume_needed"] = False
  1436. def _resume_windows_gateways_and_merge_outcome(outcome, _windows_gateway_resume, gateway_mode: bool):
  1437. """Resume gateways paused for a Windows update and fold the token into ``outcome``'s systemd/launchd-style
  1438. bookkeeping so reconciliation never reports a healthy gateway as unaccounted. Must never abort the update."""
  1439. from hermes_cli.update_cmd import _m
  1440. try:
  1441. _m()._resume_windows_gateways_after_update(_windows_gateway_resume)
  1442. except Exception as _windows_resume_exc:
  1443. # Owed restart, not a failed update (C3): verification records the ``gateway_restart``
  1444. # follow-up; the gateway's /update marker keeps the committed result.
  1445. outcome.incomplete = True
  1446. outcome.phase_errors.append(str(_windows_resume_exc))
  1447. print(f" ⚠ Windows gateway service restart incomplete: {_windows_resume_exc}")
  1448. if not isinstance(_windows_gateway_resume, dict):
  1449. return
  1450. def _extend_unique(target: list, items) -> None:
  1451. target.extend(item for item in dict.fromkeys(items) if item not in target)
  1452. token = _windows_gateway_resume
  1453. # Failed relaunches are absent from the token so they still surface. Best-effort.
  1454. with _best_effort('Could not merge Windows relaunch outcome into fleet reconciliation bookkeeping: %s'):
  1455. _extend_unique(outcome.relaunched_profiles, token.get("relaunched_profiles") or [])
  1456. windows_restarted = list(token.get("restarted_services") or [])
  1457. service_profiles = token.get("service_profiles") or {}
  1458. _extend_unique(outcome.restarted_services, windows_restarted)
  1459. _extend_unique(outcome.relaunched_profiles, (p for p in (service_profiles.get(n) for n in windows_restarted) if p))
  1460. _extend_unique(outcome.failed_or_stale_units, (str(service_profiles.get(n) or n) for n in (token.get("services") or [])))
  1461. with suppress(Exception):
  1462. from hermes_cli.update_receipt import record_gateway_restart
  1463. record_gateway_restart(
  1464. restarted_services=outcome.restarted_services, relaunched_profiles=outcome.relaunched_profiles,
  1465. externally_supervised_profiles=outcome.externally_supervised_profiles, killed_pids=sorted(outcome.killed_pids),
  1466. failed_units=outcome.failed_or_stale_units, incomplete=outcome.incomplete or bool(outcome.failed_or_stale_units),
  1467. phase_error="; ".join(outcome.phase_errors) or None,
  1468. )