| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683 |
- """Windows gateway lifecycle for ``hermes update``: pause/resume/cold-start the service, sweep venv holders, reap orphaned backends.
- Split out of ``update_cmd.py``; names are re-imported there so ``hermes_cli.update_cmd.<name>`` still resolves/monkeypatches.
- Origin helpers are imported lazily per function (no cycle; test patches on the origin stay effective).
- """
- import logging
- from contextlib import contextmanager, nullcontext, redirect_stdout, suppress
- import os
- import re
- import shlex
- import subprocess
- import sys
- import time as _time
- from datetime import datetime, timezone, UTC
- from pathlib import Path
- from agent.deadline import poll_until
- from hermes_cli.update_cmd_common import _best_effort
- logger = logging.getLogger("hermes_cli.update_cmd") # log-record parity with the origin module
- _BACKEND_PURPOSES = ("serve", "dashboard")
- def _try_call(fn, log_message: str, *log_args, default=None):
- """``fn()``, or *default* after logging the exception at debug (``log_message`` gets ``*log_args, exc``)."""
- try:
- return fn()
- except Exception as exc:
- logger.debug(log_message, *log_args, exc)
- return default
- @contextmanager
- def _abort_on_error(prefix: str):
- """Re-raise any failure of a mandatory step as ``RuntimeError(f"{prefix}: {exc}")`` (chained)."""
- try:
- yield
- except Exception as exc:
- raise RuntimeError(f"{prefix}: {exc}") from exc
- def _planned_stop_marker_path(profile_path: Path) -> Path:
- return Path(profile_path) / ".gateway-planned-stop.json"
- def _planned_stop_markers(running_pids, profile_processes, service_gateway_pids) -> dict:
- """``{pid: marker path}`` for each gateway the socket pause will ask (profile-mapped, not SCM)."""
- markers = {}
- for pid in running_pids:
- home = None if pid in service_gateway_pids else getattr(profile_processes.get(pid), "path", None)
- if home:
- markers[pid] = _planned_stop_marker_path(Path(home))
- return markers
- def _write_update_planned_stop_marker(profile_path: Path, pid: int) -> bool:
- """Write a planned-stop marker into a specific profile home."""
- try:
- from gateway.status import _get_process_start_time
- from utils import atomic_json_write
- atomic_json_write(
- _planned_stop_marker_path(profile_path),
- {"target_pid": pid, "target_start_time": _get_process_start_time(pid), "stopper_pid": os.getpid(),
- "written_at": datetime.now(UTC).isoformat()},
- indent=None, separators=(",", ":"),
- )
- return True
- except (OSError, PermissionError):
- return False
- def _wait_for_windows_update_gateway_exit(pids: list[int], *, timeout: float) -> set[int]:
- """Wait for the given gateway PIDs to exit, returning survivors."""
- if not pids:
- return set()
- from gateway.status import _pid_exists
- def _alive(pid: int) -> bool:
- try:
- return bool(_pid_exists(pid))
- except Exception:
- return False
- remaining = set(pids)
- def _all_gone() -> bool:
- nonlocal remaining
- remaining = {pid for pid in remaining if _alive(pid)}
- return not remaining
- poll_until(_all_gone, max(timeout, 0.0), 0.25)
- return {pid for pid in remaining if _alive(pid)}
- def _self_and_non_gateway_ancestor_pids(psutil) -> set[int]:
- """PIDs a venv-holder scan must never nominate: this process and its non-gateway ancestry.
- Do NOT blanket-exclude ancestors: under ``/update`` the updater is a CHILD of the gateway, and hiding it
- dead-ends the update on ``venv-blocked``. Gateway ancestors stay visible (the pause path stops them
- gracefully; a detached child survives on Windows); interactive ancestry is never a blocker."""
- _is_gw = None
- with suppress(Exception):
- # Never return ourselves or our own ancestry: a CLI ``hermes update`` runs from the venv python and
- # would otherwise nominate itself. Same #87594 carve-out as _detect_venv_python_processes: a GATEWAY
- # ancestor is not "our own ancestry" in the interactive sense — it is the process the pause
- # machinery must see (the /update-from-gateway topology makes the updater the gateway's child).
- from gateway.status import looks_like_gateway_command_line as _is_gw
- skip: set[int] = {os.getpid()}
- with suppress(Exception):
- for anc in psutil.Process().parents():
- anc_cmdline = _cmdline_or_empty(anc)
- if not (_is_gw is not None and anc_cmdline and _is_gw(anc_cmdline)):
- skip.add(int(anc.pid))
- return skip
- def _cmdline_or_empty(proc) -> str:
- """Joined argv of a psutil process, ``""`` when it can't be read."""
- try:
- return " ".join(proc.cmdline() or [])
- except Exception:
- return ""
- def _lower_dir_prefix(path: Path) -> str:
- """``str(path)`` lower-cased with one trailing separator, resolved when possible (prefix matching)."""
- try:
- raw = str(path.resolve())
- except OSError:
- raw = str(path)
- return raw.lower().rstrip(os.sep) + os.sep
- def _psutil():
- """The ``psutil`` module, or ``None`` when it can't be imported (callers degrade, never raise)."""
- try:
- import psutil # type: ignore
- except Exception:
- return None
- return psutil
- def _parent_is_live(proc) -> bool:
- """True when *proc* has a running parent that is not a recycled PID (a "parent" created after its child)."""
- parent = proc.parent()
- return parent is not None and parent.is_running() and parent.create_time() <= proc.create_time()
- def _detect_venv_python_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str, str]]:
- """Live processes running from the project venv's interpreter as ``(pid, name, cmdline)``; never raises.
- The hermes.exe shim guard misses the Desktop backend and anything off ``venv\\Scripts\\python(w).exe``;
- they keep ``.pyd`` files mapped so a mid-update dependency sync dies half-way. Empty off-Windows / without
- psutil; self + non-gateway ancestors excluded. cmdline/cwd are expensive per process on Windows (500+
- procs can blow the Desktop preflight watchdog), so they are fetched lazily for plausible candidates only.
- The FULL cmdline is kept: callers parse it (the pausable-gateway exemption looks for ``gateway run``).
- """
- from hermes_cli.update_cmd import _m
- psutil = _psutil()
- if not _m()._is_windows() or psutil is None:
- return []
- from hermes_constants import project_venv_dir
- venv_dir = project_venv_dir(_m().PROJECT_ROOT) or _m().PROJECT_ROOT / "venv"
- venv_prefix = _lower_dir_prefix(venv_dir)
- root_prefix = _lower_dir_prefix(_m().PROJECT_ROOT)
- skip = set(exclude_pids or set()) | _self_and_non_gateway_ancestor_pids(psutil)
- matches: list[tuple[int, str, str]] = []
- try:
- proc_iter = psutil.process_iter(["pid", "exe", "name"])
- except Exception:
- return []
- for proc in proc_iter:
- try:
- info = proc.info
- except Exception:
- continue
- pid, exe = info.get("pid"), info.get("exe")
- if not exe or pid is None or int(pid) in skip:
- continue
- try:
- exe_norm = str(Path(exe).resolve()).lower()
- except (OSError, ValueError):
- exe_norm = str(exe).lower()
- # Primary match: exe lives under this venv (desktop backend / gateway case).
- in_venv = exe_norm.startswith(venv_prefix)
- name = str(info.get("name") or Path(exe).name)
- name_low = name.lower()
- if not (in_venv or name_low.startswith(("python", "pypy")) or name_low in {"uv.exe", "uvx.exe", "hermes.exe"}):
- continue
- cmdline_raw = _cmdline_or_empty(proc)
- cmdline_low = cmdline_raw.lower()
- # Fallback: uv/base-interpreter trampolines have an exe OUTSIDE the venv yet hold
- # its .pyd files — match cmdline (venv path, or `-m hermes_cli.main` + root/cwd).
- if in_venv or venv_prefix in cmdline_low or (
- "hermes_cli.main" in cmdline_low and (root_prefix in cmdline_low or _cwd_prefix(proc).startswith(root_prefix))
- ):
- matches.append((int(pid), name, cmdline_raw))
- return matches
- def _cwd_prefix(proc) -> str:
- """Lower-cased cwd of *proc* with one trailing separator; bare ``os.sep`` when unreadable."""
- try:
- return str(proc.cwd() or "").lower().rstrip(os.sep) + os.sep
- except Exception:
- return os.sep
- _HOLDER_VALUE_FLAGS_FALLBACK = frozenset({
- "--profile", "-p", "--config", "--model", "-m", "--provider", "--reasoning", "--toolsets", "-t",
- "--skills", "-s", "--continue", "-c", "--resume", "-r", "--oneshot", "-z", "--in", "--usage-file", "--cpu-profile",
- })
- _holder_value_flags_cache: frozenset | None = None
- def _holder_value_flags() -> frozenset:
- """Top-level CLI flags that consume a value, from the REAL parser (nargs != 0); cached per process.
- Derived so the holder classifier can't drift from argparse (a handwritten subset misparsed ``--reasoning high
- serve``). Pre-argparse profile selectors are added explicitly (stripped before argparse sees argv). Falls back
- to a static snapshot when the parser can't import — the updater must classify holders even on a broken tree.
- Introspects ``build_top_level_parser()`` (every option with nargs != 0) so the holder classifier can
- never drift from the argparse surface (#91869 review: a handwritten subset misparsed ``--reasoning high
- serve`` as subcommand ``high`` and ``-m dashboard serve`` as ``dashboard`` — recreating the wrong-hint
- class).
- """
- global _holder_value_flags_cache
- if _holder_value_flags_cache is not None:
- return _holder_value_flags_cache
- flags: set[str] = {"--profile", "-p", "--config"}
- try:
- from hermes_cli._parser import build_top_level_parser
- for action in build_top_level_parser()[0]._actions:
- if action.option_strings and action.nargs != 0:
- flags.update(action.option_strings)
- _holder_value_flags_cache = frozenset(flags)
- except Exception:
- _holder_value_flags_cache = _HOLDER_VALUE_FLAGS_FALLBACK
- return _holder_value_flags_cache
- def _hermes_holder_subcommand(cmdline: str) -> str | None:
- """The actual Hermes SUBCOMMAND a venv-holder argv runs, or None (callers must NOT guess a label).
- Token-based, never substring (``kanban --preserve-cache`` contains "serve"): find the ``hermes_cli.main`` /
- ``hermes(.exe)`` entry token, return the first following token that isn't a flag or a flag's value.
- Profile selectors (``--profile X``, ``-p X``) are skipped like the canonical gateway matcher does. See
- #90778.
- """
- try:
- tokens = shlex.split(cmdline, posix=False)
- except Exception:
- tokens = cmdline.split()
- # ``python -c <src> … -m hermes_cli.main <subcommand>``: the entry token belongs to the argv the
- # inline source carries for a LATER spawn, not to this holder (#107002) -- unless the source is a
- # Hermes bootstrap running the entry point in this process (#124318).
- from gateway.status_inline_source import command_line_runs_inline_source, inline_bootstrap_argv
- normalized = [t.strip("\"'").replace("\\", "/") for t in tokens]
- if command_line_runs_inline_source(normalized):
- tokens = inline_bootstrap_argv(normalized)
- if tokens is None:
- return None
- def _is_entry(i: int, token: str) -> bool:
- low = token.lower().strip("\"'").replace("\\", "/")
- return (low.endswith("hermes_cli.main") and i > 0 and tokens[i - 1] == "-m") or (
- low.rsplit("/", 1)[-1] in ("hermes", "hermes.exe")) or low.endswith("hermes_cli/main.py")
- entry_idx = next((i for i, token in enumerate(tokens) if _is_entry(i, token)), None)
- if entry_idx is None:
- return None
- value_flags = _holder_value_flags()
- i = entry_idx + 1
- while i < len(tokens):
- token = tokens[i]
- if token in value_flags or token.split("=", 1)[0] in value_flags:
- i += 1 if "=" in token else 2 # --flag value consumes two tokens; --flag=value one.
- elif token.startswith("-"):
- i += 1
- else:
- return token.lower()
- return None
- def _venv_launcher_ancestors(pids: list[int]) -> list[int]:
- """Venv-interpreter parents of *pids* that hold the install open; never raises.
- A shim-started gateway is a chain: ``venv\\Scripts\\python.exe`` launcher (keeps ``.pyd`` mapped) -> uv
- CPython worker (writes the PID file). The pause set sees the worker, the venv scan sees the launcher, so a
- paused gateway still tripped the guard. One hop up only, venv-prefixed only (bounds blast radius)."""
- from hermes_cli.update_cmd import _m
- psutil = _psutil()
- if not _m()._is_windows() or not pids or psutil is None:
- return []
- from hermes_constants import project_venv_dir
- venv_dir = project_venv_dir(_m().PROJECT_ROOT) or _m().PROJECT_ROOT / "venv"
- venv_prefix = _lower_dir_prefix(venv_dir)
- skip = _self_and_non_gateway_ancestor_pids(psutil) | set(pids)
- found: list[int] = []
- for pid in pids:
- with suppress(Exception):
- parent = psutil.Process(int(pid)).parent()
- if parent is None:
- continue
- ppid = int(parent.pid)
- if ppid not in skip and ppid not in found and (parent.exe() or "").lower().startswith(venv_prefix):
- found.append(ppid)
- return found
- def _venv_holder_kind(cmdline: str) -> str:
- """Machine-readable class of one venv holder for ``--list-venv-holders``.
- ``gateway`` (the pausable gateway matcher), ``backend`` (``serve``/``dashboard`` -- the Desktop
- app's backend shape), ``hermes:<subcommand>`` for any other Hermes entry, else ``python``.
- Derived from the same classifiers the refusal path uses so automation stops exactly what the
- guard would refuse on."""
- from hermes_cli._scan_venv_blockers import _is_pausable_gateway
- if _is_pausable_gateway(cmdline):
- return "gateway"
- subcommand = _hermes_holder_subcommand(cmdline)
- if subcommand in _BACKEND_PURPOSES:
- return "backend"
- if subcommand:
- return f"hermes:{subcommand}"
- return "python"
- VENV_HOLDERS_EXIT = 3 # ``hermes update --list-venv-holders``: holders present (distinct from refusal 2)
- def list_venv_holders() -> list[dict]:
- """``[{pid, exe, argv, kind}]`` for every process the venv-holder guard would refuse on, read-only.
- Off Windows (or without psutil) the guard never fires, so the list is empty. ``exe``/``argv`` are the
- live psutil values when readable (the scan may carry only a cmdline prefix)."""
- from hermes_cli.update_cmd import _m
- psutil = _psutil()
- holders: list[dict] = []
- for pid, name, cmdline in _m()._detect_venv_python_processes():
- exe, argv = name, cmdline
- if psutil is not None:
- with suppress(Exception):
- proc = psutil.Process(int(pid))
- exe = proc.exe() or name
- argv = " ".join(proc.cmdline()) or cmdline
- holders.append({"pid": int(pid), "exe": exe, "argv": argv, "kind": _venv_holder_kind(argv)})
- return holders
- def _leftover_pausable_gateway_pids(matches: list[tuple[int, str, str]]) -> list[int] | None:
- """PIDs from *matches* when EVERY remaining venv holder is a pausable gateway, else ``None`` (keep refusing).
- A gateway respawned inside the pause->guard window (or via an unmapped spawn path) still holds ``.pyd`` files.
- Uses the Desktop preflight's ``_is_pausable_gateway`` so exemption and tolerance cannot drift; live argv is
- re-read via psutil when possible since the scan may hold only a cmdline prefix."""
- from hermes_cli._scan_venv_blockers import _is_pausable_gateway
- psutil = _psutil()
- pids: list[int] = []
- for pid, _name, cmdline in matches:
- argv = cmdline
- if psutil is not None:
- with suppress(Exception):
- argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline
- if not _is_pausable_gateway(argv):
- return None
- pids.append(int(pid))
- return pids
- def _refuse_gateway_ancestor_tree_kill(pids: list[int], *, gateway_mode: bool) -> bool:
- """Refuse a plain Windows update that would tree-kill its own ancestry (a chat agent's ``hermes update`` is
- a gateway child; ``taskkill /T /F`` kills the updater first). ``--gateway`` is exempt (detached delivery).
- Refuse only when a nominated gateway is positively an ancestor; unknown ancestry keeps existing recovery.
- The leftover holder recovery below uses ``taskkill /T /F`` on Windows, so force-stopping that gateway
- also kills the updater before it can mutate the checkout (#98814).
- """
- if gateway_mode or not pids:
- return False
- def _ancestors():
- from hermes_cli.gateway import _is_pid_ancestor_of_current_process
- return [int(pid) for pid in pids if _is_pid_ancestor_of_current_process(int(pid))]
- ancestors = _try_call(_ancestors, "Could not inspect gateway ancestry before tree-kill: %s")
- if not ancestors:
- return False
- print(
- "✗ Refusing to stop the gateway process tree because this updater "
- f"is running inside it (gateway PID(s): {', '.join(str(pid) for pid in ancestors)}).\n"
- " On Windows, taskkill /T would terminate the updater before the update can run.\n"
- " From a chat platform, use `/update` instead.\n Otherwise, run `hermes update` from a separate terminal."
- )
- return True
- def _ledger_manual_serve_holders(matches: list[tuple[int, str, str]]) -> list[dict]:
- """Full ledger entries for venv holders that are MANUAL serve/dashboard backends.
- Positive identity only: self-registered purpose serve/dashboard, live (pid, create_time), recorded spawner
- NOT alive (a Desktop-owned backend keeps its live Electron spawner and must keep the refusal — the app would
- respawn what we kill). Full entries let the relauncher rebuild from host/port/profile, not argv."""
- try:
- from hermes_cli.process_identity import ledger_entries, spawner_is_dead
- except Exception:
- return []
- holder_pids = {int(pid) for pid, _name, _cmd in matches}
- return [
- entry for entry in ledger_entries()
- if entry.get("purpose") in _BACKEND_PURPOSES and isinstance(entry.get("pid"), int) and entry["pid"] in holder_pids
- and spawner_is_dead(entry) is not False # False = live Desktop supervisor owns it; keep refusing
- ]
- def _serve_relaunch_commands(entries: list[dict]) -> list[list[str]]:
- """Rebuild launch commands for stopped serves from ledger host/port/profile — never argv parsing
- (joined argv cannot round-trip Windows paths with spaces). Entries without a port are skipped."""
- from hermes_cli.update_cmd import _m
- hermes = "hermes"
- with suppress(Exception):
- scripts_dir = _m()._venv_scripts_dir()
- if scripts_dir is not None:
- hermes = next((str(scripts_dir / n) for n in ("hermes.exe", "hermes") if (scripts_dir / n).is_file()), hermes)
- commands: list[list[str]] = []
- for entry in entries:
- port = entry.get("port")
- if not isinstance(port, int) or port <= 0:
- continue
- profile, host = str(entry.get("profile") or ""), str(entry.get("host") or "")
- commands.append(
- [hermes] + (["--profile", profile] if profile and profile != "default" else [])
- + [str(entry.get("purpose"))] + (["--host", host] if host else []) + ["--port", str(port)]
- )
- return commands
- def _relaunch_stopped_serves(token: dict) -> None:
- """Idempotent atexit relaunch of manual serves stopped by the venv guard.
- `pending` flips False on first invocation so explicit call + atexit registration cannot double-spawn."""
- from hermes_cli.update_cmd import _m, _record_update_step
- if not token.get("pending"):
- return
- token["pending"] = False
- entries = token.get("entries") or []
- if not entries:
- return
- commands = _serve_relaunch_commands(entries)
- skipped = len(entries) - len(commands)
- failed: list = []
- if commands:
- print(" ⟲ Relaunching stopped serve/dashboard backend(s)")
- failed = _m()._respawn_dashboard_processes(commands)
- if skipped or failed:
- print(" ⚠ Some stopped backends could not be relaunched automatically; restart them manually (hermes serve --host <ip> --port <port>).")
- _record_update_step(
- "serve_relaunch", not failed and not skipped,
- f"relaunched={len(commands) - len(failed)} failed={len(failed)} skipped={skipped}",
- )
- def _is_backend_argv(argv_low: str) -> bool:
- """Whether an argv is a DESKTOP backend — feeds ``taskkill /T`` via ``_orphaned_desktop_backend_pids``.
- Same predicate as ``_looks_like_desktop_control_plane``: ``-m hermes_cli.main`` entry shape (the
- Desktop's only spawn shape, ``apps/desktop/electron/main.ts``) AND the canonical holder classifier says
- ``serve``/``dashboard``. A user-launched ``hermes.exe serve`` / ``hermes dashboard`` is NOT the
- Desktop's: the guard refuses on it, never reaps it.
- """
- return _looks_like_desktop_control_plane(argv_low)
- def _live_argv_low(psutil, pid, cmdline: str) -> str | None:
- """Current lower-cased argv of *pid* (falls back to the scanned *cmdline*); ``None`` if it exited."""
- argv = cmdline
- try:
- argv = " ".join(psutil.Process(int(pid)).cmdline()) or cmdline
- except psutil.NoSuchProcess:
- return None
- except Exception:
- pass
- return argv.lower()
- def _orphaned_desktop_backend_pids(matches: list[tuple[int, str, str]]) -> list[tuple[int, int]] | None:
- """``(pid, start_time)`` roots from *matches* when every remaining holder is an ORPHANED backend, else ``None``.
- Killing a Desktop-owned ``serve`` is futile (the app respawns it), but a straggler whose Desktop is gone
- would dead-end the update with "Hermes is still running" and zero open windows. Qualifies only if cmdline
- is a Hermes backend AND the parent is demonstrably gone (PID missing or reused). Tree-aware: holders inside
- an accepted root's tree fold into it; only roots are returned (``taskkill /T`` reaps descendants). Any
- live-parent backend, unjustified non-backend, unprovable case, or no psutil -> ``None``. Never raises.
- The venv-holder guard refuses on the Desktop app's ``serve`` backend by design: while the Desktop is
- open, killing its backend is futile (the app supervises and respawns it within seconds), so the user
- must close the app. But in the GUI-updater handoff path the Desktop has *already exited* — by contract
- it tree-kills its backends and waits for the venv shim before spawning hermes-setup, and the
- update-in-progress marker parks any relaunched Desktop from spawning a fresh backend (#50238). A
- ``serve`` backend still holding the venv at that point is a straggler whose supervisor is gone: SIGTERM
- raced its spawn, or it belongs to a crashed window. Nothing will respawn it, and refusing on it
- dead-ends the update with "Hermes is still running" while the user stares at zero open windows (ryanc's
- 2026-08-09 01:59/02:17 failures).
- """
- psutil = _psutil()
- if psutil is None:
- return None
- # Pass 1: find orphaned backend ROOTS among the holders.
- roots: list[tuple[int, int]] = []
- remaining: list[int] = [] # holders still to justify
- for pid, _name, cmdline in matches:
- low = _live_argv_low(psutil, pid, cmdline)
- if low is None:
- continue # exited between scan and classification — nothing to reap
- if not _is_backend_argv(low):
- remaining.append(int(pid))
- continue
- try:
- proc = psutil.Process(int(pid))
- # Fingerprint from the SAME psutil handle, centisecond-quantized like
- # gateway.status.get_process_start_time so pid_is_hermes round-trips at kill time.
- process_start_time = round(proc.create_time() * 100)
- except psutil.NoSuchProcess:
- continue # exited during classification — nothing to reap
- except Exception:
- return None
- try:
- ppid = proc.ppid()
- parent = psutil.Process(ppid) if ppid else None
- # PID-reuse check: a "parent" created after its child is a recycled PID. A live parent is
- # not a root but may be an orphan root's descendant (the venv trampoline re-execs uv
- # python with the SAME argv) — defer to pass 2.
- if parent is not None and parent.is_running() and parent.create_time() <= proc.create_time():
- remaining.append(int(pid))
- continue
- except psutil.NoSuchProcess:
- pass # parent gone → orphan
- except Exception:
- return None
- roots.append((int(pid), process_start_time))
- # Pass 2: every non-backend holder must descend from an accepted orphan root
- # (dies with the tree reap); anything else keeps the refusal.
- root_set = {pid for pid, _start_time in roots}
- for pid in remaining:
- try:
- if not root_set or not root_set & {int(a.pid) for a in psutil.Process(pid).parents()}:
- return None
- except psutil.NoSuchProcess:
- continue # exited already
- except Exception:
- return None
- return roots
- def _ledger_reapable_backend_pids(matches: list[tuple[int, str, str]]) -> list[int]:
- """PIDs the spawn ledger positively identifies as orphaned backends; never raises.
- Strongest rung (no PPID/cmdline inference): qualifies when ``(pid, create_time)`` matches a live ledger entry
- (PID reuse can't forge it), purpose is a REAPABLE kind (never interactive), and the recorded SPAWNER is
- provably dead. Safe in ANY context. Unlisted holders fall to later rungs and never disqualify identified ones."""
- try:
- from hermes_cli.process_identity import REAPABLE_PURPOSES, ledger_entries, spawner_is_dead
- entries = ledger_entries()
- except Exception:
- return []
- by_pid = {e.get("pid"): e for e in entries if isinstance(e.get("pid"), int)}
- return [
- int(pid) for pid, _name, _cmdline in matches
- if (entry := by_pid.get(int(pid))) and entry.get("purpose") in REAPABLE_PURPOSES and spawner_is_dead(entry) is True
- ]
- def _handoff_reapable_backend_pids(matches: list[tuple[int, str, str]]) -> list[int] | None:
- """Backend PIDs safe to tree-reap during a GUI-updater hand-off, INCLUDING ones with a live parent; never raises.
- The orphan-only rung bails on ANY live parent (mid-teardown Electron, launcher->worker chain) and hung a
- hand-off. Inside the hand-off gate (marker + ``--gateway`` + no live ``hermes.exe`` shim) nothing legitimate
- supervises a ``serve`` from this venv, so survivors are leaks. Any non-backend holder or no psutil ->
- ``None``. The CALLER must have confirmed the gate; outside it the stricter orphan-only path stands.
- Any ``serve`` backend still holding the venv here is therefore a leak, live parent or not, and reaping
- its tree is correct rather than a race. See #50238.
- """
- psutil = _psutil()
- if psutil is None:
- return None
- roots: list[int] = []
- for pid, _name, cmdline in matches:
- low = _live_argv_low(psutil, pid, cmdline)
- if low is None:
- continue # exited — nothing to reap
- if not _is_backend_argv(low):
- return None # unexpected non-backend holder: refuse the whole set
- roots.append(int(pid))
- return roots or None
- def _stop_process_trees(pids: list[int] | list[tuple[int, int]]) -> None:
- """Force-stop each PID with its full child tree (Windows); best effort, never raises.
- ``taskkill /T /F``: stopping only the parent can leave a ``.hermes-runtime`` child holding the install open.
- See #70026.
- """
- from gateway.status import get_process_start_time
- from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags
- for entry in pids:
- pid, expected_start_time = entry if isinstance(entry, tuple) else (int(entry), get_process_start_time(int(entry)))
- try:
- if expected_start_time is None:
- logger.debug("Skipping taskkill of PID %s: process identity unavailable", pid)
- continue
- if not pid_is_hermes(pid, expected_start_time=expected_start_time):
- logger.debug("Skipping taskkill of non-Hermes or changed PID %s", pid)
- continue
- subprocess.run(
- ["taskkill", "/PID", str(pid), "/T", "/F"], check=False,
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, stdin=subprocess.DEVNULL,
- creationflags=windows_hide_flags(),
- )
- except Exception as exc:
- logger.debug("Could not stop process tree %s: %s", pid, exc)
- def _looks_like_desktop_control_plane(cmdline: str) -> bool:
- """True for this-install ``hermes serve`` / ``hermes dashboard`` argv (Desktop control plane).
- Not the messaging gateway — don't feed into ``looks_like_gateway_command_line``. Token-based via the
- parser-derived classifier, never substring (``kanban --preserve-cache``, ``-m dashboard chat``).
- Undeterminable subcommand is NOT a control plane.
- See #92091.
- A cmdline whose subcommand cannot be determined is NOT a control plane — callers must not guess
- ownership. See #90778, #91869.
- """
- return "hermes_cli.main" in (cmdline or "").lower() and _hermes_holder_subcommand(cmdline) in _BACKEND_PURPOSES
- def _desktop_owns_gateway_lifecycle() -> bool:
- """True when Desktop currently supervises this install's control plane (updater must not steal gateway start).
- Not proof messaging is served: serve is the control plane, the gateway a detached sibling. Prefer the spawn
- ledger; fall back to the venv-holder scan. An orphaned control plane (supervisor gone) does not count;
- without psutil orphanhood is unprovable and a live control plane suffices.
- See #76129, #92091.
- """
- from hermes_cli.update_cmd import _m
- with _best_effort('Desktop-lifecycle ledger probe failed: %s'):
- from hermes_cli.process_identity import ledger_entries, spawner_is_dead
- if any(e.get("purpose") in _BACKEND_PURPOSES and spawner_is_dead(e) is False for e in ledger_entries()):
- return True
- psutil = _psutil()
- for pid, _name, cmdline in _try_call(_detect_venv_python_processes, "Desktop-lifecycle holder scan failed: %s") or []:
- if not _looks_like_desktop_control_plane(cmdline):
- continue
- if psutil is None:
- return True
- with suppress(Exception):
- if _parent_is_live(psutil.Process(int(pid))):
- return True
- return False
- def _win_service(name: str):
- """``(psutil, service)`` for the named SCM service (psutil imported here so tests can stub the module)."""
- import psutil
- return psutil, psutil.win_service_get(name)
- def _sc_exe(verb: str, name: str, service, settled_status: str) -> None:
- """``sc.exe <verb> <name>``; a non-zero exit is only an error when SCM doesn't already report *settled_status*."""
- result = subprocess.run(
- ["sc.exe", verb, name], capture_output=True, text=True, encoding="utf-8", errors="replace",
- timeout=10, check=False,
- )
- if result.returncode != 0 and service.status() != settled_status:
- detail = (result.stderr or result.stdout).strip()
- raise RuntimeError(detail or f"sc.exe {verb} failed with {result.returncode}")
- def _original_process_is_alive(psutil, pid: int, create_time: float) -> bool:
- """Is the process with this exact ``(pid, create_time)`` identity still alive? AccessDenied/unknown reads
- True (fail closed: the venv may still be locked)."""
- try:
- current = float(psutil.Process(pid).create_time())
- except (psutil.NoSuchProcess, psutil.ZombieProcess):
- return False
- except Exception:
- return True
- return abs(current - create_time) <= 0.001
- def _process_create_time(psutil, pid: int, label: str) -> float:
- """``create_time`` of *pid*; unreadable identity aborts the stop (RuntimeError)."""
- try:
- return float(psutil.Process(int(pid)).create_time())
- except Exception as exc:
- raise RuntimeError(f"Windows {label} process identity is unavailable before stop") from exc
- def _verify_service_identities(psutil, name: str, service, expected_service_identity, expected_gateway_identity) -> None:
- """Refuse to stop *name* unless SCM state, service/gateway process identities and ancestry all still match."""
- if expected_service_identity is not None:
- try:
- current_status, current_service_pid = str(service.status()), int(service.pid() or 0)
- except Exception as exc:
- raise RuntimeError(f"Windows service {name} SCM identity is unavailable before stop") from exc
- if current_status != "running":
- raise RuntimeError(f"Windows service {name} is not stably running before stop: {current_status}")
- if current_service_pid != int(expected_service_identity[0]):
- raise RuntimeError(f"Windows service {name} SCM process identity changed before stop")
- for label, identity in (("service", expected_service_identity), ("gateway", expected_gateway_identity)):
- if identity is not None and abs(_process_create_time(psutil, identity[0], label) - float(identity[1])) > 0.001:
- raise RuntimeError(f"Windows {label} process identity changed before stop")
- if expected_service_identity is not None and expected_gateway_identity is not None:
- try:
- ancestor_pids = {int(parent.pid) for parent in psutil.Process(int(expected_gateway_identity[0])).parents()}
- except Exception as exc:
- raise RuntimeError("Windows gateway ancestry is unavailable before service stop") from exc
- if int(expected_service_identity[0]) not in ancestor_pids:
- raise RuntimeError(f"Windows gateway is no longer owned by service {name}")
- def _stop_windows_gateway_service(
- name: str, *, expected_processes: tuple[tuple[int, float], ...] = (), expected_service_identity: tuple[int, float] | None = None,
- expected_gateway_identity: tuple[int, float] | None = None, timeout: float = 30.0,
- ) -> None:
- """Stop one verified Windows service and wait until SCM reports it down.
- Lingering matching-identity processes after SCM says stopped make venv mutation unsafe — fail closed."""
- psutil, service = _win_service(name)
- _verify_service_identities(psutil, name, service, expected_service_identity, expected_gateway_identity)
- _sc_exe("stop", name, service, "stopped")
- def _alive() -> list[int]:
- return [pid for pid, create_time in expected_processes if _original_process_is_alive(psutil, pid, create_time)]
- if poll_until(lambda: service.status() == "stopped" and not _alive(), timeout, 0.2):
- return
- if service.status() != "stopped":
- raise RuntimeError(f"Windows service {name} did not stop within {timeout:.0f}s; venv mutation unsafe.")
- if alive_after_stop := _alive():
- raise RuntimeError(f"Windows service {name} stopped but its process tree is still alive: {alive_after_stop}")
- def _start_windows_gateway_service(name: str, *, timeout: float = 30.0) -> None:
- """Start one previously paused Windows service and verify it is running."""
- service = _win_service(name)[1]
- _sc_exe("start", name, service, "running")
- if not poll_until(lambda: service.status() == "running", timeout, 0.2):
- raise RuntimeError(f"Windows service {name} did not start within {timeout:.0f}s")
- def _restore_windows_gateway_service(name: str, *, timeout: float = 60.0) -> None:
- """Restore a service after an uncertain stop, including STOP_PENDING."""
- from hermes_cli.update_cmd import _start_windows_gateway_service
- service = _win_service(name)[1]
- def _settled() -> bool:
- status = service.status()
- if status == "stopped":
- _start_windows_gateway_service(name)
- return status in ("running", "stopped")
- if not poll_until(_settled, timeout, 0.2):
- raise RuntimeError(f"Windows service {name} did not reach a restorable state within {timeout:.0f}s")
- def _windows_cold_start_plan() -> dict | None:
- """Pause token for the no-running-gateway case: cold-start after update when an autostart entry exists.
- An installed autostart entry is an explicit "I want a gateway" signal; a gateway that died between
- updates would otherwise stay down until next login (resume only relaunches what was running).
- Desktop-owned lifecycle -> ``None`` (spawning ``gateway run`` beside Desktop races ports/state);
- the skip is ownership, not liveness — except when the start attestation reports a vouched-for
- gateway that died without a clean exit. The Desktop hand-off exits the app before the updater starts
- and can kill the running gateway in those same seconds, so discovery finds no live PID to pause
- (#109538) — the dead attestation is the only surviving "a gateway was up" evidence, and the Desktop
- only restarts gateways it stopped itself (its hand-off script, after the update verifies; #119809).
- Keep the plan, and record the attestation
- *generation* that authorized it on the token: the marker is a mutable one-shot that any concurrent
- ``hermes gateway status``/``start`` consumes, so execution authorizes the spawn from the token and
- consumes only that generation (#110020 review)."""
- from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle
- from hermes_cli import gateway_windows
- with _best_effort('Could not check Windows gateway autostart state before update: %s'):
- if not gateway_windows.is_installed():
- return None
- token = {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": [], "cold_start_if_installed": True}
- with _best_effort('Could not check Desktop gateway-lifecycle ownership before update: %s'):
- if _desktop_owns_gateway_lifecycle():
- generation = gateway_windows.attested_death_generation(current_pids=[])
- if generation is None:
- logger.debug("Skipping Windows gateway cold-start plan: Desktop owns gateway lifecycle")
- return None
- token["attested_generation"] = generation
- return token
- return None
- class ServicePauseFailed(RuntimeError):
- """A gateway service could not be stopped; ``rollback_failures`` lists what the in-line
- rollback could not restart. Empty = everything stopped was restarted, so the pause is void;
- otherwise its record must stay, since gateways are still down."""
- def __init__(self, detail: str, rollback_failures: list[str]):
- super().__init__(detail)
- self.rollback_failures = rollback_failures
- def _pause_windows_gateway_services(service_gateways, token: dict, profiles: dict, unmapped: list) -> dict:
- """Stop each SCM gateway service, recording them on *token*; roll everything back on failure.
- Runs after every fallible ordinary-gateway step so a failure here restores the attempted
- services AND the already-paused ordinary gateways before re-raising."""
- from hermes_cli.update_cmd import _restore_windows_gateway_service, _stop_windows_gateway_service
- paused_services = []
- current_service_name = None
- try:
- for service in service_gateways:
- current_service_name = str(service.name)
- _stop_windows_gateway_service(
- current_service_name, expected_processes=tuple(getattr(service, "descendant_identities", ())),
- expected_service_identity=(int(service.service_pid), float(service.service_create_time)),
- expected_gateway_identity=(int(service.gateway_pid), float(service.gateway_create_time)),
- )
- paused_services.append(current_service_name)
- current_service_name = None
- if paused_services:
- token.update(services=paused_services, expected_services=list(paused_services), restarted_services=[])
- token["service_profiles"] = {str(s.name): str(s.profile) for s in service_gateways if str(s.name) in paused_services}
- print(" ✓ Paused Windows gateway service(s): " + ", ".join(paused_services))
- return token
- except Exception as exc:
- restore_names = ([current_service_name] if current_service_name else []) + list(reversed(paused_services))
- rollback: list[tuple[str, object]] = [(n, lambda n=n: _restore_windows_gateway_service(n)) for n in dict.fromkeys(restore_names)]
- if profiles or unmapped:
- rollback.append(("ordinary gateways", lambda: _resume_windows_gateways_after_update(token)))
- rollback_failures = []
- for label, restore in rollback:
- try:
- restore()
- except Exception as restore_exc:
- rollback_failures.append(f"{label}: {restore_exc}")
- detail = f"Could not stop Windows gateway service {current_service_name or 'unknown'}: {exc}"
- if rollback_failures:
- detail += "; rollback failures: " + "; ".join(rollback_failures)
- raise ServicePauseFailed(detail, rollback_failures) from exc
- def _owned_gateway_pids(pids, *, keep=(), quiet: bool = True) -> list[int]:
- """*pids* whose live home this update owns, plus *keep* (PIDs mapped to this install's profile
- PID files / services). The same home scope the POSIX fleet restart uses (#93349): a gateway of
- another Hermes install, or one whose home cannot be read, is named (unless *quiet*) and left
- running, never paused, force-killed or replayed (#124659)."""
- from hermes_cli.update_cmd_fleet import _scoped_manual_gateway_pids
- return _scoped_manual_gateway_pids(list(pids), keep=keep, quiet=quiet)
- def _discover_windows_gateways():
- """``(profile_processes, service_gateways, service_gateway_pids, running_pids)`` for the pause; any indeterminate probe aborts."""
- from hermes_cli.gateway import find_gateway_pids, find_profile_gateway_processes, find_windows_gateway_services
- with _abort_on_error("Could not map Windows gateway PIDs to profiles"):
- profile_process_list = find_profile_gateway_processes(strict=True)
- profile_processes = {proc.pid: proc for proc in profile_process_list}
- with _abort_on_error("Could not determine Windows gateway service ownership"):
- service_gateways = find_windows_gateway_services(profile_processes=profile_process_list)
- service_gateway_pids = {int(service.gateway_pid) for service in service_gateways}
- with _abort_on_error("Could not discover Windows gateway PIDs before update"):
- # find_gateway_pids(all_profiles=True) is a HOST-wide scan; only this install's fleet is paused.
- running_pids = _owned_gateway_pids(dict.fromkeys(
- [*find_gateway_pids(all_profiles=True), *sorted(profile_processes), *sorted(service_gateway_pids)]
- ), keep=set(profile_processes) | service_gateway_pids, quiet=False)
- return profile_processes, service_gateways, service_gateway_pids, running_pids
- def _request_socket_pauses(running_pids, profile_processes, service_gateway_pids, on_request=None):
- """Marker + socket-first pause for every profile-mapped gateway; ``(profiles, mapped_pids, socket_acks)``.
- Socket ACK = the gateway drains and exits by its own graceful path. No answer (older
- gateway) -> the marker poll / force-kill ladder in the caller. ``on_request(pid)`` runs once the
- request is issued (marker on disk, else a positive socket ACK) and before the socket call: from
- then on a crash leaves that gateway owed while it drains, never before."""
- profiles: dict[str, int] = {}
- mapped_pids = []
- socket_acks: list[dict] = []
- for pid in running_pids:
- proc = None if pid in service_gateway_pids else profile_processes.get(pid)
- if proc is None:
- continue
- profiles[str(proc.profile)] = int(pid)
- mapped_pids.append(int(pid))
- requested = _write_update_planned_stop_marker(Path(proc.path), int(pid))
- if requested and on_request is not None:
- on_request(int(pid))
- try:
- # Socket-first pause (#92091 step 2): ask the gateway to drain and exit itself instead of
- # relying on the marker poll + force-kill ladder. A positive ACK means the gateway is running
- # its own graceful restart path (same drain as SIGUSR1/service restarts) and will release its
- # venv handles on the way out. No answer (older gateway, no socket) → the marker watcher /
- # force-kill fallback below behaves exactly as before this verb existed.
- from gateway.control_socket import pause_gateway_for_update
- ack = pause_gateway_for_update(Path(proc.path))
- if ack and (ack.get("pausing") or ack.get("already_stopping")):
- socket_acks.append(ack)
- if not requested and on_request is not None:
- on_request(int(pid))
- except Exception as exc:
- logger.debug("Socket pause unavailable for gateway %s: %s", pid, exc)
- return profiles, mapped_pids, socket_acks
- def _gateway_drain_timeout(socket_acks: list[dict]) -> float:
- """Drain budget: configured restart drain (>= 1s), raised to a socket-paused gateway's declared
- ACTIVE-TURN budget + teardown grace so it isn't force-killed mid-turn."""
- from hermes_cli.gateway import _get_restart_drain_timeout
- try:
- drain_timeout = max(float(_get_restart_drain_timeout()), 1.0)
- except Exception:
- drain_timeout = 10.0
- if socket_acks:
- with suppress(Exception):
- declared = max(float(a.get("drain_timeout") or 0.0) for a in socket_acks)
- drain_timeout = max(drain_timeout, declared + 10.0)
- print(f" → {len(socket_acks)} gateway(s) ACKed socket pause; waiting up to {int(drain_timeout)}s for graceful exit")
- return drain_timeout
- def _pause_windows_gateways_for_update() -> dict | None:
- """Stop running Windows gateways before mutating the checkout or venv.
- Scheduled/startup gateways run via pythonw.exe, invisible to the hermes.exe instance guard, yet keep files
- locked during ``git``/``uv``. Stop only PIDs the gateway discovery code identifies. The paused set is on
- disk (``update_pause_record``) before the first stop, with any set a killed earlier update orphaned."""
- from hermes_cli.update_cmd import _m
- if not _m()._is_windows():
- return None
- with _abort_on_error("Could not prepare Windows gateway pause for update"):
- import gateway.status
- from hermes_cli.gateway import _capture_gateway_argv
- from hermes_cli import update_pause_record as pause_record
- with _abort_on_error("Could not read the gateways an earlier update paused"):
- adopted, claims = pause_record.adopt_orphans()
- profile_processes, service_gateways, service_gateway_pids, running_pids = _discover_windows_gateways()
- if not running_pids:
- return _cold_start_pause_token(adopted, claims)
- unmapped_pids = [pid for pid in running_pids if pid not in profile_processes and pid not in service_gateway_pids]
- from gateway.status import get_process_start_time
- from hermes_cli.update_fleet_scope import gateway_pid_home
- born = {int(pid): get_process_start_time(int(pid)) for pid in running_pids} # the processes discovered
- # Snapshot unmapped gateways' argv *before* anything stops so resume (or a later launch) can replay it.
- # Unmapped = no profile->PID-file mapping (e.g. Scheduled Task ``pythonw.exe -m ...``). Two homes can
- # run identical selectorless argv, so the home and birth identify the runtime the replay must restore:
- # read with discovery's own resolver (environment, else the exact-birth ledger witness), and a home
- # that no longer proves refuses the pause before any stop. ``home: null`` would read as a pre-home
- # record, whose argv-only match lets another home's gateway retire this debt.
- homes = {int(pid): gateway_pid_home(int(pid)) for pid in unmapped_pids}
- if None in homes.values():
- raise RuntimeError("Could not prove the Hermes home of unmapped gateway PID(s) "
- + ", ".join(str(p) for p, h in homes.items() if h is None) + "; nothing was stopped")
- unmapped = [
- {"pid": int(pid), "argv": _try_call(lambda p=int(pid): _capture_gateway_argv(p),
- "Could not capture argv for unmapped gateway %s: %s", int(pid)),
- "home": homes[int(pid)], "ct": born[int(pid)]}
- for pid in unmapped_pids
- ]
- intended = {
- "resume_needed": True, "unmapped_pids": unmapped_pids, "unmapped": unmapped,
- "profiles": {str(profile_processes[pid].profile): int(pid) for pid in running_pids
- if pid in profile_processes and pid not in service_gateway_pids},
- "identities": {str(pid): pause_record.identity(pid)["ct"] for pid in running_pids},
- }
- if service_gateways:
- names = [str(s.name) for s in service_gateways]
- intended.update(services=names, expected_services=list(names), restarted_services=[],
- service_profiles={str(s.name): str(s.profile) for s in service_gateways})
- with _abort_on_error("Could not record the gateways this update pauses"):
- pause_record.record_pause(intended, adopted, claims)
- # The intent (and where each request goes), right before the first request. A live entry is owed
- # only once its request was issued (mark_stop_sent, or this update's marker still on disk): a
- # crash before that leaves it "never asked" (still serving, dropped), after it, draining (owed).
- pause_record.mark_stop_requested(
- intended, running_pids, markers=_planned_stop_markers(running_pids, profile_processes, service_gateway_pids))
- profiles = _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped,
- on_request=lambda pid: pause_record.mark_stop_sent(intended, pid), born=born)
- token = {"resume_needed": True, "profiles": profiles, "unmapped_pids": unmapped_pids, "unmapped": unmapped}
- # Every profile with ANY live gateway at discovery counts as running: service-supervised ones skip the
- # socket pause (absent from ``profiles``) but the SCM restart brings them back, not a cold-start.
- running_profiles = set(profiles) | {str(p.profile) for p in profile_processes.values()} | {str(s.profile) for s in service_gateways}
- _record_attested_cold_start_profiles(token, running_profiles)
- try:
- token = _pause_windows_gateway_services(service_gateways, token, profiles, unmapped)
- except ServicePauseFailed as exc:
- if not exc.rollback_failures: # everything stopped was restarted in-line
- pause_record.abandon_pause(intended, adopted)
- raise
- return pause_record.finish_pause(token, intended, adopted)
- def _cold_start_pause_token(adopted: dict | None, claims: list) -> dict | None:
- """No gateway runs: the cold-start plan, plus any orphaned set adopted from a killed update."""
- from hermes_cli import update_pause_record as pause_record
- token = _windows_cold_start_plan()
- # Other profiles may hold a dead attestation even when the active profile owes nothing
- # (clean exit, autostart not installed): give them a token to ride on.
- probe = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
- _record_attested_cold_start_profiles(probe, set())
- token = probe if probe.get("cold_start_profiles") else token
- if adopted is None:
- return pause_record.stamp_tree(token) if token is not None else None
- token = token if token is not None else {"resume_needed": True, "profiles": {}, "unmapped_pids": [], "unmapped": []}
- with _abort_on_error("Could not record the adopted paused-gateway set"):
- return pause_record.record_pause(token, adopted, claims)
- def _stop_windows_gateways(running_pids, profile_processes, service_gateway_pids, unmapped_pids, unmapped,
- on_request=None, born=None) -> dict:
- """Socket-pause the mapped gateways, drain, then tree-kill survivors; the paused ``profiles``.
- A force-killed process needs no stop record: dead, it is owed; alive, it was never stopped.
- The kill is authorized by each process's birth as discovered (*born*, a mapped gateway's
- strict PID-file identity first), never re-read after the drain: a PID that exited and was
- reused meanwhile reads its replacement's own birth. No discovered birth refuses the kill."""
- from gateway.status import get_process_start_time, terminate_pid
- from hermes_cli.update_cmd import _m
- mapped = [int(p) for p in running_pids if int(p) not in service_gateway_pids and int(p) in profile_processes]
- born = {**{int(k): v for k, v in (born or {}).items()},
- **{p: round(float(profile_processes[p].create_time) * 100) for p in mapped
- if float(getattr(profile_processes[p], "create_time", 0) or 0) > 0}}
- # Resolve venv-side launchers before any request: the launcher keeps ``.pyd`` mapped and would trip the
- # venv-holder guard, so it is killed with the survivors, under the identity it has now.
- launcher_pids = _m()._venv_launcher_ancestors(mapped)
- born.update({int(p): get_process_start_time(int(p)) for p in launcher_pids if int(p) not in born})
- profiles, mapped_pids, socket_acks = _request_socket_pauses(
- running_pids, profile_processes, service_gateway_pids, on_request=on_request)
- print("→ Stopping Windows gateway process(es) before updating Hermes...")
- drain_timeout = _gateway_drain_timeout(socket_acks)
- survivors = _m()._wait_for_windows_update_gateway_exit(mapped_pids, timeout=drain_timeout)
- # Tree-kill survivors, unmapped gateways, and pre-request launchers; one already gone raises
- # ProcessLookupError, one whose birth differs (or was never read) is refused, and both are skipped.
- force_killed = []
- for pid in sorted(set(survivors).union(unmapped_pids).union(launcher_pids)):
- with suppress(ProcessLookupError, PermissionError, OSError):
- terminate_pid(int(pid), force=True, expected_start_time=born.get(int(pid)))
- force_killed.append(int(pid))
- if profiles:
- print(f" ✓ Paused gateway profile(s): {', '.join(sorted(profiles))}")
- if force_killed:
- print(f" → Force-stopped {len(force_killed)} gateway process(es)")
- if unmapped_pids:
- print(f" → Stopped {len(unmapped_pids)} gateway process(es) without profile mapping")
- if any(not u.get("argv") for u in unmapped): # no recoverable cmdline (psutil missing, denied, gone)
- print(" Restart manually after update: hermes gateway run")
- return profiles
- def _record_attested_cold_start_profiles(token: dict, running_profiles: set) -> None:
- """Record ``token["cold_start_profiles"] = {name: generation}`` for every known profile that is not
- running yet holds a start attestation for a gateway that died without a clean exit (#110959).
- The all-or-nothing plan only ever probed the ACTIVE profile and only when NO gateway ran at all, so a
- dead-but-attested default beside a still-running ``beta`` never got a cold-start obligation. Only
- Desktop-owned installs need this (elsewhere autostart brings the profile back); the active profile is
- left to the existing plan so it is never spawned twice. Best-effort: never blocks the pause."""
- from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle
- from hermes_cli import gateway_windows
- with _best_effort("Could not evaluate per-profile attested cold-starts before update: %s"):
- if not _desktop_owns_gateway_lifecycle():
- return
- from hermes_cli.profiles import get_active_profile_name, profiles_to_serve
- active = get_active_profile_name() or "default"
- cold: dict[str, str] = {}
- # An activation list, not inventory: parked profiles stay offline.
- for name, home in profiles_to_serve(multiplex=True, include_standalone=True):
- if name in running_profiles or (name == active and token.get("cold_start_if_installed")):
- continue
- generation = gateway_windows.attested_death_generation([], home=Path(home))
- if generation:
- cold[name] = generation
- if cold:
- token["cold_start_profiles"] = cold
- def _cold_start_attested_profiles(token: dict) -> None:
- """Spawn each ``cold_start_profiles`` entry under its own HERMES_HOME and consume exactly the
- generation that authorized it; one profile's failure never aborts the others (#110959)."""
- from hermes_cli import gateway_windows
- from hermes_cli.profiles import get_profile_dir
- pending = dict(token.get("cold_start_profiles") or {})
- if not pending:
- return
- for name, generation in sorted(pending.items()):
- home = Path(get_profile_dir(name))
- with _best_effort(f"Could not cold-start Windows gateway profile {name} after update: %s"):
- if not gateway_windows._live_gateway_pids(home=home): # a concurrent autostart must not be doubled
- pid = gateway_windows._spawn_detached(home=home)
- if not pid:
- raise RuntimeError("cold-start did not return a process ID")
- ready_pids = gateway_windows._wait_for_gateway_ready(home=home)
- if not ready_pids:
- raise RuntimeError(f"gateway profile {name} did not become ready")
- gateway_windows._consume_start_attestation(generation, home=home)
- # Keep the attestation chain: a death after this CLI exits must be visible to the next update.
- gateway_windows._write_start_attestation(ready_pids, f"cold-start after update (profile {name})", home=home)
- print(f"\n✓ Gateway profile {name} started via cold-start after update (PID: {ready_pids[0]})")
- token["cold_start_profiles"].pop(name, None)
- if token["cold_start_profiles"]:
- # Surface the miss like the active-profile cold-start does: the merged outcome marks the update
- # incomplete instead of reporting success with a profile still down.
- raise RuntimeError("Windows gateway cold-start was not verified for profile(s): "
- + ", ".join(sorted(token["cold_start_profiles"])))
- token.pop("cold_start_profiles", None)
- def _cold_start_windows_gateway_after_update(token: dict | None = None) -> bool:
- """Direct-spawn a detached gateway after update for the ``cold_start_if_installed`` case (installed but down).
- Idempotent: re-checks nothing is running so a concurrent autostart can't duplicate. A successful Popen
- doesn't prove survival (a job object denying breakaway kills it), so success is gated on the liveness poll.
- Vouched PIDs are attested so a death AFTER updater exit is reported by the next CLI invocation.
- A successful ``Popen`` only proves the process was created, not that it survived (e.g. a Windows job
- object denying breakaway kills it before it logs anything — #84185). So the success line is gated on the
- same post-spawn liveness poll every other ``_spawn_detached`` caller uses
- (``gateway_windows._report_gateway_start``), instead of being printed unconditionally from the returned
- PID.
- Desktop-owned lifecycle suppresses the spawn only while nothing attests a gateway is expected: an
- attested gateway that died without a clean exit is restored even then (#109538) — the Desktop does
- not restart the messaging gateway itself. That authority is the ``attested_generation`` the plan
- recorded on ``token``, not the marker on disk: the marker is a mutable one-shot a concurrent
- ``hermes gateway status``/``start`` consumes, which would otherwise skip this spawn and clear the
- token (#110020 review). Only that generation is consumed afterwards — never a newer marker.
- """
- from hermes_cli.update_cmd import _desktop_owns_gateway_lifecycle, _m
- if not _m()._is_windows():
- return True
- with _abort_on_error("Could not load Windows gateway cold-start helpers"):
- from hermes_cli import gateway_windows
- from hermes_cli.gateway import find_gateway_pids
- with _abort_on_error("Could not re-check gateway liveness before cold-start"):
- # Another install's live gateway must not suppress this install's cold start (#124659).
- if _owned_gateway_pids(find_gateway_pids(all_profiles=True)):
- return True
- token = token or {}
- generation = token.get("attested_generation")
- if generation is None and "attested_generation" not in token:
- # Token written by pre-generation code and resumed across this very update: probe the marker.
- with _abort_on_error("Could not re-read the start attestation before cold-start"):
- generation = gateway_windows.attested_death_generation(current_pids=[])
- with _abort_on_error("Could not re-check Desktop gateway-lifecycle ownership before cold-start"):
- if _desktop_owns_gateway_lifecycle() and not generation:
- logger.debug("Skipping Windows gateway cold-start: Desktop owns gateway lifecycle")
- return True
- with _abort_on_error("Could not cold-start Windows gateway after update"):
- pid = gateway_windows._spawn_detached()
- if not pid:
- raise RuntimeError("Windows gateway cold-start did not return a process ID")
- ready_pids = gateway_windows._wait_for_gateway_ready()
- if not ready_pids:
- raise RuntimeError(f"Windows gateway cold-start PID {pid} did not become ready")
- # The dead attestation has done its job (it authorized this spawn under Desktop ownership). Consume
- # it only now: a spawn that never became ready leaves it in place, so the registered retry still
- # holds its recovery obligation instead of seeing Desktop ownership with no marker and returning
- # success without a gateway (#110020 review).
- if generation:
- gateway_windows._consume_start_attestation(generation)
- print(f"\n✓ Gateway started via cold-start after update (PID: {', '.join(map(str, ready_pids))})")
- with suppress(Exception):
- gateway_windows._write_start_attestation(ready_pids, "cold-start after update")
- return True
- def _refresh_windows_gateway_launchers() -> None:
- """Regenerate installed Windows gateway launcher scripts after update; best-effort, never fails the update.
- Launchers are written once at install, so old installs kept launching via ``pythonw.exe`` (``sys.stderr is
- None`` death). The task's /TR points at a stable path, so rewriting in place retargets it without UAC.
- The Scheduled Task / Startup-folder launchers (``gateway.cmd`` + ``gateway.vbs``) are persistence
- artifacts written once at install time — ``hermes update`` never touched them, so installs created
- before the hidden-console rework (aa2ae36c3f) kept launching the gateway through ``pythonw.exe``
- forever: every descendant spawn flashed a conhost (#54220/#56747) and, since #70344, the console-less
- gateway died at startup with ``RuntimeError: sys.stderr is None`` (#71671).
- """
- from hermes_cli.update_cmd import _m
- if not _m()._is_windows():
- return
- with _best_effort('Could not refresh Windows gateway launchers after update: %s'):
- from hermes_cli import gateway_windows
- if gateway_windows.is_installed():
- gateway_windows._write_task_script()
- print(" ✓ Refreshed Windows gateway launcher scripts")
- # Installs from before #80569 can carry a Startup entry beside the task: both fire at logon.
- done, warnings = gateway_windows.reconcile_autostart_launchers()
- for message in done:
- print(f" ✓ {message}")
- for message in warnings:
- print(f" ⚠ {message}")
- if gateway_windows.is_task_registered():
- # A task registered by an older build never picks up template hardening otherwise (#113670).
- gateway_windows.reconcile_scheduled_task(gateway_windows.get_task_name())
- def _refresh_bootstrap_cache_scripts(branch: str = "main") -> None:
- """Overwrite ``$HERMES_HOME/bootstrap-cache/install-<ref>.{ps1,sh}`` for *branch* from the fresh checkout.
- Old ``hermes-setup.exe`` builds NEVER re-download a cached branch-ref script, so a stale one runs
- months-old code forever. Guards mirror ``install_script.rs``: only the sanitized *branch* key is rewritten;
- commit-SHA pins (7-40 hex) are immutable and skipped. Best-effort: never fails the update.
- Installer binaries built before the #67193 cache-refresh fix (June 2026 and earlier) NEVER re-download a
- cached branch-ref script — ``install-main.ps1`` cached at install time is reused forever, executing
- months-stale code with long-fixed bugs (the 2026-08-09 incident: a June 4 cached script's venv stage
- lacked the 81327 process-tree sweep and died on ``Access denied``). The binary has no self-update path,
- so the poisoned cache outlives every ``hermes update``.
- Overwriting the cached script for *branch* with the freshly pulled ``scripts/install.ps1`` /
- ``scripts/install.sh`` on every update turns the stale binary's unconditional reuse into a feature: it
- "reuses" a file this function keeps permanently current. Post-#67193 installers re-download on each run
- anyway, so for them this is a harmless pre-seed of the same bytes.
- The .ps1 copy gets a UTF-8 BOM to match the installer's cache format (#67193 encoding fix).
- """
- from hermes_cli.update_cmd import _m
- with _best_effort('Could not refresh bootstrap-cache scripts after update: %s'):
- cache_dir = Path(_m().get_hermes_home()) / "bootstrap-cache"
- if not cache_dir.is_dir():
- return
- safe_ref = re.sub(r"[^A-Za-z0-9._-]", "_", str(branch or "main")) # install_script.rs::sanitize_ref()
- if re.fullmatch(r"[0-9a-fA-F]{7,40}", safe_ref): # install_script.rs::is_valid_commit(): immutable pin
- return
- refreshed = []
- for kind, src_name in (("ps1", "install.ps1"), ("sh", "install.sh")):
- src = _m().PROJECT_ROOT / "scripts" / src_name
- cached = cache_dir / f"install-{safe_ref}.{kind}"
- if not src.is_file() or not cached.is_file():
- continue # this ref was never bootstrap-cached — nothing to heal
- data = src.read_bytes()
- if kind == "ps1" and not data.startswith(b"\xef\xbb\xbf"):
- data = b"\xef\xbb\xbf" + data # PowerShell needs the BOM or localized/em-dash text mis-decodes.
- # See #67193.
- if cached.read_bytes() == data:
- continue
- tmp = cached.with_suffix(cached.suffix + ".tmp")
- tmp.write_bytes(data)
- os.replace(tmp, cached)
- refreshed.append(cached.name)
- if refreshed:
- print(" ✓ Refreshed installer bootstrap-cache script(s): " + ", ".join(sorted(refreshed)))
- _SERVICE_READY_TIMEOUT_S = 60.0
- # A launch recovering an orphaned pause (``update_pause_record.recover``, before every command)
- # only probes briefly, and a service whose gateway stays unready is re-probed with backoff; the
- # full wait belongs to ``hermes update`` itself.
- _SERVICE_READY_PROBE_S = 5.0
- _SERVICE_RETRY_BASE_S = 60.0
- _SERVICE_RETRY_MAX_S = 3600.0
- def _service_gateway_ready(name: str, profile: str | None, timeout_s: float | None = None) -> list[int]:
- """The stable gateway *name*'s service process supervises (in *profile*'s home when known): SCM
- ``running`` proves only the wrapper started, not that the gateway it hosts came up."""
- from hermes_cli import gateway_windows
- from hermes_cli.profiles import get_profile_dir
- psutil, service = _win_service(name)
- def under_service(pids):
- try:
- service_pid = int(service.pid() or 0)
- except (psutil.Error, OSError, ValueError, TypeError): # SCM/psutil failure or a non-int pid
- return []
- owned = []
- for pid in pids:
- # A gateway that exited or a process tree psutil cannot read is not the service's; any
- # other error is a bug the resume boundary reports, not a silent "not under service".
- with suppress(psutil.Error, OSError):
- if service_pid and service_pid in {int(p.pid) for p in psutil.Process(int(pid)).parents()}:
- owned.append(int(pid))
- return owned
- home = Path(get_profile_dir(profile)) if profile else None
- return gateway_windows._wait_for_gateway_ready(
- timeout_s=_SERVICE_READY_TIMEOUT_S if timeout_s is None else timeout_s, home=home,
- all_profiles=home is None, pid_filter=under_service)
- def _service_running(name: str) -> bool:
- try:
- import psutil
- except ImportError:
- return False
- try:
- return _win_service(name)[1].status() == "running"
- except (psutil.Error, OSError, AttributeError): # missing service, SCM access, non-Windows psutil
- return False
- def _resume_windows_services(token: dict) -> None:
- """Restart the SCM services recorded on *token*; one leaves the token only once the gateway it
- supervises is ready. Failed ones stay on the token so a retry sees them.
- A recovering launch (``token["recovery"]``) probes readiness briefly, and a running service
- whose gateway was not ready is not re-probed before its backoff (``service_retry``) elapses:
- the debt stays owed without stalling every command."""
- from hermes_cli.update_cmd import _start_windows_gateway_service
- services = list(token.get("services") or [])
- token.setdefault("expected_services", list(services))
- verified_restarts = list(token.get("restarted_services") or [])
- retry = {name: state for name, state in (token.get("service_retry") or {}).items() if name in services}
- recovering = bool(token.get("recovery"))
- restarted_services = []
- failed_services = []
- for service_name in map(str, services):
- backoff = retry.get(service_name) or {}
- try:
- if recovering and _time.time() < float(backoff.get("next_at") or 0) and _service_running(service_name):
- failed_services.append(service_name)
- print(f" ⚠ Windows gateway service {service_name}: its gateway was not ready; "
- f"next check after {int(float(backoff['next_at']) - _time.time())}s")
- continue
- _start_windows_gateway_service(service_name)
- if not _service_gateway_ready(service_name, (token.get("service_profiles") or {}).get(service_name),
- _SERVICE_READY_PROBE_S if recovering else None):
- attempts = int(backoff.get("attempts") or 0) + 1
- retry[service_name] = {"attempts": attempts, "next_at": _time.time() + min(
- _SERVICE_RETRY_BASE_S * 2 ** min(attempts - 1, 6), _SERVICE_RETRY_MAX_S)}
- raise RuntimeError(f"Windows service {service_name} is running but its gateway did not become ready")
- retry.pop(service_name, None)
- restarted_services.append(service_name)
- if service_name not in verified_restarts:
- verified_restarts.append(service_name)
- except Exception as exc:
- logger.warning("Could not restart Windows gateway service %s after update: %s", service_name, exc)
- print(f" ⚠ Could not restart Windows gateway service: {service_name}")
- failed_services.append(service_name)
- token["restarted_services"] = verified_restarts
- token["services"] = failed_services
- if retry:
- token["service_retry"] = retry
- else:
- token.pop("service_retry", None)
- if failed_services:
- raise RuntimeError("Could not restart Windows gateway service(s): " + ", ".join(failed_services))
- if restarted_services:
- print("\n ✓ Restarted Windows gateway service(s): " + ", ".join(restarted_services))
- def _relaunch_paused_gateways(profiles: dict, unmapped: list) -> tuple[dict, list]:
- """Spawn the restart watcher for each paused profile gateway and unmapped argv; ``(launched
- profiles, launched unmapped entries)``. Nothing leaves the token here: a watcher only proves a
- relaunch was scheduled, so every entry stays owed until ITS gateway is verified ready."""
- with _abort_on_error("Could not load Windows gateway restart helper"):
- from hermes_cli.gateway import launch_detached_gateway_restart_by_cmdline, launch_detached_profile_gateway_restart
- # An exception from a launch (incl. bad pid/argv coercion) logs at debug and reads as a failed relaunch.
- launched = {}
- for profile, old_pid in sorted(profiles.items()):
- if _try_call(lambda p=profile, o=old_pid: launch_detached_profile_gateway_restart(str(p), int(o)),
- "Could not restart Windows gateway profile %s after update: %s", profile):
- launched[str(profile)] = int(old_pid)
- launched_unmapped = []
- for entry in unmapped:
- argv, old_pid = entry.get("argv"), entry.get("pid")
- # On the paused runtime's home (None on a pre-home record: the updater's own home, as before).
- if argv and old_pid and _try_call(lambda o=old_pid, a=argv, h=entry.get("home"): launch_detached_gateway_restart_by_cmdline(
- int(o), list(a), home=h),
- "Could not restart unmapped Windows gateway (pid %s) after update: %s", old_pid):
- launched_unmapped.append(entry)
- return launched, launched_unmapped
- _RELAUNCH_VERIFY_TIMEOUT_S = 30.0
- def _pending_relaunch_pids(profiles: dict, unmapped: list, pid_exists) -> list[int]:
- """Old PIDs a restart watcher is still waiting on, sorted; empty when every relaunch can have run.
- ``_spawn_gateway_restart_watcher`` respawns the gateway only once the PID it was handed is gone,
- so while any of them is alive the relaunch has provably not started yet. Pure function of data
- (``pid_exists`` is injected) so the decision is testable off Windows.
- """
- candidates = [int(pid) for pid in profiles.values()]
- candidates += [int(entry["pid"]) for entry in unmapped if entry.get("argv") and entry.get("pid")]
- return sorted({pid for pid in candidates if pid > 0 and pid_exists(pid)})
- def _relaunch_verify_timeout_s(profiles: dict, unmapped: list, pid_exists) -> float:
- """Liveness budget for the post-relaunch poll.
- The base window assumes the watchers respawn immediately. When an old PID is still alive the
- watcher is still in its wait loop, so the poll must reach at least the watcher's own deadline —
- otherwise ``hermes update`` declares "no stable gateway process appeared" for a gateway that was
- never scheduled to appear inside the window (#107002).
- """
- from hermes_cli.gateway import GATEWAY_RESTART_WATCHER_TIMEOUT_S
- if not _pending_relaunch_pids(profiles, unmapped, pid_exists):
- return _RELAUNCH_VERIFY_TIMEOUT_S
- return float(GATEWAY_RESTART_WATCHER_TIMEOUT_S) + _RELAUNCH_VERIFY_TIMEOUT_S
- def _unmapped_ready_filter(entry: dict, taken: set):
- """``pid_filter`` matching the gateway an unmapped entry's replay started: a gateway running the
- same arguments after the (normalized) interpreter on the entry's home, born after the process it
- replaces, not one another entry already vouched for. Two homes can run identical selectorless argv,
- so argv alone would let home B's gateway retire home A's debt (a service gateway on another home
- included; on A's home only one gateway can run). Only an entry recorded before ``home`` was
- captured (no key: a pre-durable-pause updater's token) keeps the argv-only match; a recorded
- ``home`` that is empty proves no runtime and is never matched."""
- import psutil
- from gateway.status import _looks_like_gateway_process, _same_hermes_home, get_process_start_time
- from hermes_cli.update_fleet_scope import gateway_pid_home
- tail, old = list(entry.get("argv") or [])[1:], int(entry.get("pid") or 0)
- home, old_ct = entry.get("home"), entry.get("ct")
- def same_runtime(pid: int) -> bool:
- if "home" not in entry:
- return True
- if not home:
- return False
- live_home, ct = gateway_pid_home(pid), get_process_start_time(pid)
- return live_home is not None and _same_hermes_home(live_home, home) and (old_ct is None or ct is None or ct > old_ct)
- def matches(pid: int) -> bool:
- if pid == old or pid in taken:
- return False
- try:
- return (list(psutil.Process(pid).cmdline() or [])[1:] == tail and _looks_like_gateway_process(pid)
- and same_runtime(pid))
- except (psutil.Error, OSError, ValueError): # exited, access denied, zombie, invalid pid
- return False
- return lambda pids: [pid for pid in _owned_gateway_pids(pids) if matches(int(pid))]
- _READY_INTERVAL_S = 0.4 # gateway_windows._wait_for_gateway_ready's poll interval and
- _READY_CONFIRM_S = 2.0 # confirmation window: one hit proves a spawn, not a survived startup
- def _poll_until_ready(targets: list, deadline: float, taken: set, respawn_window: float | None = None) -> dict:
- """``{key: pids}`` for each ``(key, probe, claim_all)`` target whose gateway stayed visible for
- a whole confirmation window. All targets share one loop, so N targets cost one window, not N:
- one fleet process scan per tick (``probe(scan)`` calls ``scan()`` only if it needs the fleet),
- and each target is credited only on its own hits. *taken* holds, while a target is probed, the
- PIDs earlier targets vouched for (all of a profile's, the first of an unmapped entry's) so one
- gateway never retires two debts. A hit provisional at the deadline still finishes its window;
- a probe that raises leaves its target unready.
- *respawn_window*: a target's own liveness budget, in two parts. The restart watcher only spawns
- once the paused PID is gone, and that wait is charged to the *old* process — 25–35 s of a shared
- 30 s deadline on a busy update — while a cold gateway then needs >11 s before its control socket
- answers. So (a) a target first seen late gets ``respawn_window`` counted from its sighting, and
- (b) a target the deadline expires on without ever being seen gets ONE extra window, because the
- spawn it is waiting for may not have happened yet. Charging that wait to the new process is what
- made an update report "relaunch was not verified alive" for a gateway that was up seconds later;
- the verdict is committed and never revisited, so the stale debt surfaced as a manual follow-up on
- the next launch. Both parts are bounded, so a relaunch that truly never happens still fails
- closed. ``None`` keeps the single shared budget. Each target earns its sighting extension once,
- and the whole poll stops at the entry deadline plus both windows and one confirmation tick: a
- replacement that flaps in and out must not renew its budget forever.
- """
- from hermes_cli.gateway import find_gateway_pids
- ready: dict = {}
- since: dict = {}
- failed: set = set()
- graced: set = set()
- first_seen: set = set()
- hard_stop = deadline + 2 * (respawn_window or 0.0) + _READY_CONFIRM_S + _READY_INTERVAL_S
- while True:
- now = _time.monotonic()
- if now >= hard_stop:
- return ready
- pending = [t for t in targets if t[0] not in ready and t[0] not in failed and (now < deadline or t[0] in since)]
- if not pending:
- starved = [key for key, _probe, _all in targets
- if key not in ready and key not in failed and key not in first_seen and key not in graced]
- if respawn_window is not None and starved:
- graced.update(starved)
- deadline = now + respawn_window
- continue
- return ready
- fleet: list = []
- def scan() -> list:
- if not fleet:
- fleet.append(list(find_gateway_pids(all_profiles=True)))
- return fleet[0]
- taken.clear()
- taken.update(pid for pids in ready.values() for pid in pids)
- for key, probe, claim_all in pending:
- try:
- pids = list(probe(scan))
- except Exception as exc: # health: allow BLE001 -- one target's probe failure must not stall the others
- logger.warning("Could not verify Windows gateway %s after update: %s", key, exc)
- failed.add(key)
- continue
- if not pids:
- since.pop(key, None)
- continue
- vouched = pids if claim_all else pids[:1]
- taken.update(vouched)
- if respawn_window is not None and key not in first_seen:
- # The replacement did not exist before this tick, so the wait that preceded it belongs
- # to the process it replaced: this target's budget starts now, not at the shared
- # deadline's origin.
- first_seen.add(key)
- deadline = max(deadline, now + respawn_window)
- if now - since.setdefault(key, now) >= _READY_CONFIRM_S:
- ready[key] = vouched
- if len(ready) + len(failed) < len(targets):
- _time.sleep(_READY_INTERVAL_S)
- def _verify_relaunched_gateways_alive(token: dict, profiles: dict, unmapped: list) -> None:
- """Retire each relaunched runtime on ITS OWN readiness: a profile on a stable gateway in its
- profile home, an unmapped entry on a stable gateway running its argv. A watcher only proves a
- relaunch was scheduled, and another target's readiness proves nothing about this one.
- A parent Job Object denying CREATE_BREAKAWAY_FROM_JOB can kill the gateway on updater teardown.
- Vouched PIDs are persisted so a death AFTER updater exit is reported by the next CLI invocation."""
- with _abort_on_error("Could not load Windows gateway liveness helpers"):
- from gateway.status import _pid_exists
- from hermes_cli import gateway_windows
- from hermes_cli.profiles import get_profile_dir
- deadline = _time.monotonic() + _relaunch_verify_timeout_s(profiles, unmapped, _pid_exists)
- taken: set = set()
- # Profiles first: a gateway on a profile's home is that profile's, never an unmapped entry's.
- targets = [(("profile", name), lambda _scan, h=Path(get_profile_dir(name)), o=int(old_pid): gateway_windows._live_gateway_pids(
- home=h, pid_filter=lambda ps: [p for p in ps if int(p) != o]), True) for name, old_pid in sorted(profiles.items())]
- targets += [(("unmapped", i), lambda scan, f=_unmapped_ready_filter(entry, taken): f(scan()), False)
- for i, entry in enumerate(unmapped)]
- ready = _poll_until_ready(targets, deadline, taken, respawn_window=_RELAUNCH_VERIFY_TIMEOUT_S)
- ready_profiles = [name for name in sorted(profiles) if ("profile", name) in ready]
- ready_unmapped = [entry for i, entry in enumerate(unmapped) if ("unmapped", i) in ready]
- ready_pids = [pid for pids in ready.values() for pid in pids]
- # Fleet reconciliation (#91277) cross-checks every planned runtime against relaunched_profiles:
- # only a verified one is listed, so an unready one still surfaces as unaccounted.
- token["relaunched_profiles"] = ready_profiles
- token["profiles"] = {n: pid for n, pid in (token.get("profiles") or {}).items() if n not in ready_profiles}
- token["unmapped"] = [u for u in (token.get("unmapped") or []) if u not in ready_unmapped]
- if ready_pids:
- with suppress(Exception):
- gateway_windows._write_start_attestation(sorted(set(ready_pids)), "post-update relaunch")
- if ready_profiles:
- print(f"\n ✓ Restarting Windows gateway profile(s): {', '.join(ready_profiles)}")
- if ready_unmapped:
- print(f" ✓ Restarting {len(ready_unmapped)} unmapped Windows gateway process(es)")
- unready = sorted(set(profiles) - set(ready_profiles)) + [f"pid {u.get('pid')}" for u in unmapped if u not in ready_unmapped]
- if unready:
- print(
- "\n ⚠ Windows gateway restart could not be verified for: " + ", ".join(unready) + "\n"
- " (The respawned gateway may have been killed by a parent Job Object during updater teardown, #48820.)\n"
- " Recover with: hermes gateway restart"
- )
- raise RuntimeError("Windows gateway relaunch after update was not verified alive: " + ", ".join(unready))
- def _resume_windows_gateways_after_update(token: dict | None) -> None:
- """Restart Windows profile gateways previously paused for update."""
- from hermes_cli.update_cmd import _m
- if not token or not token.get("resume_needed"):
- return
- # The foreground call sites register this same function via atexit as a safety net for
- # process death before they get a chance to run it themselves (#115563). Once execution
- # actually reaches here — foreground or the atexit fallback itself — ownership is taken:
- # unregister immediately so a failure below (or the foreground caller failing after this
- # returns) cannot replay the same RuntimeError a second time at interpreter teardown.
- # ``unregister`` is a no-op when this function was never registered.
- import atexit
- atexit.unregister(_resume_windows_gateways_after_update)
- if not _m()._is_windows() and token.get("platform") != "posix":
- token["resume_needed"] = False
- return
- # Startup recovery runs ahead of an unrelated command whose stdout may be parsed (``--json``):
- # its notices go to stderr, as ``recover()``'s own do.
- with redirect_stdout(sys.stderr) if token.get("recovery") else nullcontext():
- _resume_unless_torn(token)
- def _resume_unless_torn(token: dict) -> None:
- """Resume the paused set, or leave it owed when the checkout is not whole."""
- from hermes_cli import update_pause_record as pause_record
- if token.get("pause_id"):
- whole, why = pause_record.tree_is_whole(token)
- if not whole:
- # Starting gateways now would boot them on a half-written tree or the pre-sync
- # dependencies. The durable record keeps the obligation for the next launch, which
- # runs after the interrupted-pull restore and the dependency sync.
- print(f" ⚠ Paused gateway(s) left stopped: {why}.\n"
- " The next `hermes` command (or `hermes update`) restarts them.")
- token["resume_needed"] = False
- token["resume_deferred"] = why
- pause_record.sync(token)
- return
- try:
- _resume_paused_set(token)
- finally:
- pause_record.sync(token)
- def _resume_paused_set(token: dict) -> None:
- """Bring every paused runtime back; each one is retired from *token* only on its own verified
- start, and one failure (an SCM service, a profile) never keeps the others stopped."""
- from hermes_cli.update_cmd import _m
- if token.get("platform") == "posix": # Linux/macOS pre-swap pause (update_cmd_posix_pause)
- from hermes_cli.update_cmd_posix_pause import resume_paused_set
- return resume_paused_set(token)
- # Regenerate launcher scripts before respawning so a legacy pythonw-era
- # autostart entry comes back on the current design at next login too.
- _m()._refresh_windows_gateway_launchers()
- failures: list[str] = []
- def attempt(step) -> None:
- try:
- step()
- except Exception as exc: # health: allow BLE001 -- per-step isolation: any failed step is recorded and re-raised as a summary so it never keeps the other paused runtimes stopped
- failures.append(str(exc))
- attempt(lambda: _resume_windows_services(token))
- profiles = dict(token.get("profiles") or {})
- unmapped = list(token.get("unmapped") or [])
- if profiles or any(u.get("argv") for u in unmapped):
- launched, launched_unmapped = _relaunch_paused_gateways(profiles, unmapped)
- if launched or launched_unmapped:
- attempt(lambda: _verify_relaunched_gateways_alive(token, launched, launched_unmapped))
- else:
- token["relaunched_profiles"] = []
- not_launched = sorted(set(profiles) - set(launched)) + [
- f"pid {u.get('pid')}" for u in unmapped if u not in launched_unmapped]
- if not_launched:
- failures.append("Could not restart every paused Windows gateway: " + ", ".join(not_launched))
- elif token.get("cold_start_if_installed"):
- # Before the per-profile spawns: this guard is fleet-wide (any live gateway ⇒ done).
- def cold_start() -> None:
- if not _m()._cold_start_windows_gateway_after_update(token):
- raise RuntimeError("Windows gateway cold-start was not verified")
- token["cold_start_if_installed"] = False
- attempt(cold_start)
- # After the paused profiles are back: a dead-attested sibling that fails to cold-start must not
- # keep the profiles that WERE running from being relaunched.
- attempt(lambda: _cold_start_attested_profiles(token))
- if failures:
- raise RuntimeError("; ".join(failures))
- token["resume_needed"] = False
- def _resume_windows_gateways_and_merge_outcome(outcome, _windows_gateway_resume, gateway_mode: bool):
- """Resume gateways paused for a Windows update and fold the token into ``outcome``'s systemd/launchd-style
- bookkeeping so reconciliation never reports a healthy gateway as unaccounted. Must never abort the update."""
- from hermes_cli.update_cmd import _m
- try:
- _m()._resume_windows_gateways_after_update(_windows_gateway_resume)
- except Exception as _windows_resume_exc:
- # Owed restart, not a failed update (C3): verification records the ``gateway_restart``
- # follow-up; the gateway's /update marker keeps the committed result.
- outcome.incomplete = True
- outcome.phase_errors.append(str(_windows_resume_exc))
- print(f" ⚠ Windows gateway service restart incomplete: {_windows_resume_exc}")
- if not isinstance(_windows_gateway_resume, dict):
- return
- def _extend_unique(target: list, items) -> None:
- target.extend(item for item in dict.fromkeys(items) if item not in target)
- token = _windows_gateway_resume
- # Failed relaunches are absent from the token so they still surface. Best-effort.
- with _best_effort('Could not merge Windows relaunch outcome into fleet reconciliation bookkeeping: %s'):
- _extend_unique(outcome.relaunched_profiles, token.get("relaunched_profiles") or [])
- windows_restarted = list(token.get("restarted_services") or [])
- service_profiles = token.get("service_profiles") or {}
- _extend_unique(outcome.restarted_services, windows_restarted)
- _extend_unique(outcome.relaunched_profiles, (p for p in (service_profiles.get(n) for n in windows_restarted) if p))
- _extend_unique(outcome.failed_or_stale_units, (str(service_profiles.get(n) or n) for n in (token.get("services") or [])))
- with suppress(Exception):
- from hermes_cli.update_receipt import record_gateway_restart
- record_gateway_restart(
- restarted_services=outcome.restarted_services, relaunched_profiles=outcome.relaunched_profiles,
- externally_supervised_profiles=outcome.externally_supervised_profiles, killed_pids=sorted(outcome.killed_pids),
- failed_units=outcome.failed_or_stale_units, incomplete=outcome.incomplete or bool(outcome.failed_or_stale_units),
- phase_error="; ".join(outcome.phase_errors) or None,
- )
|