update_cmd_posix_pause.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690
  1. """Pre-swap gateway pause for ``hermes update`` on Linux and macOS.
  2. Windows pauses its gateways before the fetch (``update_cmd_windows``) because they lock venv
  3. files. On POSIX nothing is locked, but a gateway that keeps running while git rewrites the tree
  4. loads new-tree modules into an old process on its next lazy import. So every gateway running
  5. from this install is stopped at the commit point, immediately before the first checkout move
  6. (``update_cmd_commit.arm_commit_point``), and the same set is restarted once the dependencies
  7. are synced and the launchers published, before the product builds
  8. (``source_completion.complete_source_checkout``'s ``before_build``).
  9. The set rides the Windows pause contract: one token in ``update_pause_record`` (identity = pid
  10. + start time; supervisor unit, or argv + home for bare processes), written before the first stop,
  11. resumed by ``_resume_windows_gateways_after_update`` (which dispatches here on
  12. ``platform == "posix"``), adopted by the next ``hermes update`` and resumed by the next launch
  13. when this process dies.
  14. * every gateway is drained first: the control socket's ``pause-for-update`` runs the same
  15. ``request_restart`` drain as the post-update restart's SIGUSR1 (refuse new turns, let the
  16. in-flight ones finish within ``agent.restart_after_turn_timeout``, then exit), waited for within
  17. that restart's budget. The request is recorded before it is sent, and carries no planned-stop
  18. marker (the gateway's marker watcher would take the immediate stop path instead).
  19. * systemd ``hermes-gateway*`` units and launchd ``ai.hermes.gateway*`` jobs are then stopped and
  20. later started through their supervisor, so it cannot respawn them on old code mid-update; one
  21. that does not take the drain gets its supervisor's stop at once.
  22. * bare ``gateway run`` processes that do not answer get marker + SIGTERM, and are replayed from
  23. their recorded argv + home.
  24. * the updater never kills itself: a unit whose cgroup holds this process (``/update`` from
  25. chat: the update runs in the gateway unit's cgroup, which ``systemctl stop`` kills) is
  26. stopped only after this process moved into a transient scope of its own. A launchd job
  27. whose coalition holds it, or any gateway whose stop would take the updater down, is left
  28. running for the post-commit restart.
  29. """
  30. from __future__ import annotations
  31. import copy
  32. import os
  33. import signal
  34. import subprocess
  35. import sys
  36. import time
  37. from contextlib import suppress
  38. from pathlib import Path
  39. #: This run's token: armed in ``_cmd_update_impl``, filled at the commit point. Same dict object
  40. #: the completion request carries, so the paused set reaches the completion child.
  41. _RUN: dict | None = None
  42. #: Runtimes the early resume already restarted on the new code: the late fleet restart in the
  43. #: same completion process must not restart them a second time.
  44. _RESTARTED: dict = {"units": set(), "labels": set(), "pids": set()}
  45. _STOP_TIMEOUT_S = 120.0
  46. _READY_TIMEOUT_S = 30.0
  47. class PauseRefused(RuntimeError):
  48. """The pause could not be completed; everything it stopped was started again."""
  49. def _empty_token() -> dict:
  50. return {"platform": "posix", "resume_needed": False, "profiles": {}, "unmapped_pids": [],
  51. "unmapped": [], "posix_units": []}
  52. def arm_pause(*, no_gateway_restart: bool) -> dict | None:
  53. """Arm this run's pause (no stop yet) and adopt any set a killed update left paused.
  54. ``None`` on Windows (its own pause) and with ``--no-gateway-restart`` (the user manages
  55. gateways, nothing is stopped). An adopted orphan set is recorded under this process now, so
  56. even an update that never moves the checkout restarts it at exit."""
  57. global _RUN
  58. if sys.platform == "win32" or no_gateway_restart:
  59. return None
  60. from hermes_cli import update_pause_record as pause_record
  61. token = _empty_token()
  62. try:
  63. adopted, claims = pause_record.adopt_orphans()
  64. if adopted is not None:
  65. token = pause_record.record_pause({**token, "resume_needed": True}, adopted, claims)
  66. token["platform"] = "posix"
  67. except Exception as exc: # health: allow BLE001 -- an unreadable record must not block the update; recovery retries it
  68. print(f" ⚠ Could not adopt gateways an earlier update paused: {exc}")
  69. _RUN = token
  70. return token
  71. def gateways_paused() -> bool:
  72. """True while this run holds gateways stopped (no gateway can answer a chat prompt)."""
  73. return bool(_RUN and _RUN.get("posix_stopped") and _RUN.get("resume_needed"))
  74. # --------------------------------------------------------------------------- discovery
  75. def _unit_cgroup(scope_cmd: list, unit: str) -> str | None:
  76. from hermes_cli.update_cmd_fleet import _systemctl
  77. with suppress(OSError, subprocess.TimeoutExpired):
  78. result = _systemctl(scope_cmd + ["show", unit, "--property=ControlGroup", "--value"], timeout=10)
  79. if result.returncode == 0 and result.stdout.strip():
  80. return result.stdout.strip()
  81. return None
  82. def _pid_cgroup(pid: int | str = "self") -> str | None:
  83. """The systemd cgroup path of *pid*: the unified (v2) line, else the v1 ``name=systemd``
  84. hierarchy, which systemd names units in identically. ``None``: not Linux or unreadable, which
  85. callers must treat as unknown, never as outside a unit."""
  86. with suppress(OSError):
  87. lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="utf-8").splitlines()
  88. unified = next((line[3:].strip() for line in lines if line.startswith("0::")), "")
  89. legacy = next((line.split(":", 2)[2].strip() for line in lines if ":name=systemd:" in line), "")
  90. return unified or legacy or None
  91. return None
  92. def _inside(cgroup: str | None, unit_cgroup: str | None) -> bool:
  93. if not cgroup or not unit_cgroup:
  94. return False
  95. return cgroup == unit_cgroup or cgroup.startswith(unit_cgroup.rstrip("/") + "/")
  96. def _discover_systemd() -> list[dict]:
  97. """Active ``hermes-gateway*`` units of this install, one per live MainPID."""
  98. from hermes_cli.gateway import _ensure_user_systemd_env, supports_systemd_services
  99. from hermes_cli.update_cmd_fleet import (
  100. _for_each_systemd_gateway_unit, _systemd_gateway_unit_listings, _systemd_unit_owned_by_update,
  101. _unit_main_pid,
  102. )
  103. if not supports_systemd_services():
  104. return []
  105. with suppress(Exception):
  106. _ensure_user_systemd_env()
  107. units: list[dict] = []
  108. seen_pids: set[int] = set()
  109. for scope, scope_cmd, listing in _systemd_gateway_unit_listings():
  110. names: list[str] = []
  111. _for_each_systemd_gateway_unit(listing.stdout, process_unit=names.append,
  112. on_unit_timeout=lambda name, exc: None)
  113. for name in names:
  114. if not name.startswith("hermes-gateway"):
  115. continue # serve/dashboard units are not gateways: the post-commit restart owns them
  116. pid = _unit_main_pid(scope_cmd, name)
  117. if pid <= 0 or pid in seen_pids or not _systemd_unit_owned_by_update(scope_cmd, name):
  118. continue
  119. seen_pids.add(pid) # a legacy per-profile unit sharing the multiplexer's PID is one process
  120. units.append({"kind": "systemd", "scope": scope, "unit": name, "pid": pid,
  121. "cgroup": _unit_cgroup(scope_cmd, name), "home": _home_of(pid)})
  122. return units
  123. def _discover_launchd() -> list[dict]:
  124. """Loaded ``ai.hermes.gateway*`` jobs of this install with a live process (macOS)."""
  125. if sys.platform != "darwin":
  126. return []
  127. from hermes_cli.gateway import (
  128. _locate_launchd_gateway_service, get_launchd_plist_path, launchd_gateway_labels_for_install,
  129. legacy_launchd_labels_for_install,
  130. )
  131. from hermes_cli.update_fleet_scope import launchd_label_foreign_home
  132. derived = launchd_gateway_labels_for_install()
  133. jobs = []
  134. for label in derived + legacy_launchd_labels_for_install(exclude=set(derived)):
  135. if launchd_label_foreign_home(label) is not None:
  136. continue
  137. domain, pid = _locate_launchd_gateway_service(label)
  138. if domain is None or not pid or pid <= 0:
  139. continue
  140. plist = get_launchd_plist_path().with_name(f"{label}.plist")
  141. jobs.append({"kind": "launchd", "label": label, "domain": domain, "plist": str(plist), "pid": int(pid),
  142. "home": _home_of(int(pid))})
  143. return jobs
  144. def _home_of(pid: int) -> str | None:
  145. """The Hermes home *pid* serves (its control socket lives there), or ``None``."""
  146. from hermes_cli.update_fleet_scope import gateway_pid_home
  147. with suppress(Exception):
  148. if home := gateway_pid_home(pid):
  149. return str(home)
  150. return None
  151. def _supervisor_markers(pid: int) -> str | None:
  152. """Why *pid* answers to a supervisor other than a hermes unit/job (it would respawn it on old
  153. code), or ``None``."""
  154. from hermes_cli.gateway import _capture_gateway_argv
  155. if "--external-supervisor" in (_capture_gateway_argv(pid) or []):
  156. return "--external-supervisor"
  157. try:
  158. import psutil # type: ignore
  159. env = psutil.Process(pid).environ()
  160. except Exception: # health: allow BLE001 -- unreadable environment: no marker proven
  161. return None
  162. if env.get("HERMES_S6_SUPERVISED_CHILD"):
  163. return "s6"
  164. if unit := _respawning_service(pid):
  165. return f"systemd unit {unit}"
  166. # launchd names terminal apps' shells application.<bundle id>; those are not KeepAlive jobs.
  167. xpc = env.get("XPC_SERVICE_NAME", "0")
  168. if xpc not in ("", "0") and not xpc.startswith("application."):
  169. return f"launchd job {xpc}"
  170. return None
  171. def _respawning_service(pid: int) -> str | None:
  172. """The systemd service that would respawn *pid* when it exits, or ``None``.
  173. systemd restarts a unit only when its main process exits, so the gateway must BE the
  174. MainPID or its direct child (a non-exec wrapper that exits with it). The inherited
  175. INVOCATION_ID is no evidence: a terminal tab's shell carries one (its .scope is never
  176. respawned), and so does anything deep under a CI runner's service."""
  177. from hermes_cli.update_cmd_fleet import _systemctl, _unit_main_pid
  178. cgroup = _pid_cgroup(pid) or ""
  179. leaf = cgroup.rsplit("/", 1)[-1]
  180. if not leaf.endswith(".service"):
  181. return None
  182. scope_cmd = ["systemctl", "--user"] if "/user@" in cgroup else ["systemctl"]
  183. with suppress(OSError, subprocess.SubprocessError):
  184. main = _unit_main_pid(scope_cmd, leaf)
  185. if main > 0 and main in (pid, _ppid(pid)):
  186. restart = _systemctl(scope_cmd + ["show", leaf, "--property=Restart", "--value"], timeout=10)
  187. return leaf if (restart.stdout or "").strip() not in ("", "no") else None
  188. return None
  189. def _discover_bare(service_pids: set[int]) -> tuple[list[dict], list[str]]:
  190. """``(bare gateways, notices)``: this install's ``gateway run`` processes outside any unit."""
  191. from gateway.status import get_process_start_time
  192. from hermes_cli.gateway import _capture_gateway_argv, _get_service_pids, find_gateway_pids
  193. from hermes_cli.update_cmd_fleet import _scoped_manual_gateway_pids
  194. from hermes_cli.update_fleet_scope import gateway_pid_home
  195. exclude = set(service_pids) | set(_get_service_pids(all_profiles=True))
  196. pids = _scoped_manual_gateway_pids(find_gateway_pids(exclude_pids=exclude, all_profiles=True))
  197. bare, notices = [], []
  198. for pid in pids:
  199. pid = int(pid)
  200. marker = _supervisor_markers(pid)
  201. argv, home = _capture_gateway_argv(pid), gateway_pid_home(pid)
  202. if marker or not argv or not home:
  203. why = f"supervised by {marker}" if marker else "its command line could not be read" if not argv \
  204. else "its Hermes home could not be proved"
  205. notices.append(f" ↷ gateway PID {pid} keeps running until after the update ({why})")
  206. continue
  207. bare.append({"pid": pid, "argv": list(argv), "home": str(home), "ct": get_process_start_time(pid)})
  208. return bare, notices
  209. # --------------------------------------------------------------------------- self-preservation
  210. def _ppid(pid: int) -> int:
  211. with suppress(OSError, ValueError, IndexError):
  212. stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
  213. return int(stat.rsplit(")", 1)[1].split()[1])
  214. return 0
  215. def _escape_cgroup(unit: dict) -> bool:
  216. """Move this process (and its ancestors in the same unit, e.g. ``/update``'s bash wrapper)
  217. into a transient scope so stopping *unit* cannot kill it. True only when proven out: a
  218. membership that cannot be read (either side) keeps the unit running, since its stop would take
  219. this update down with it."""
  220. own, unit_cgroup = _pid_cgroup(), unit.get("cgroup")
  221. if own is None or not unit_cgroup:
  222. return False
  223. if not _inside(own, unit_cgroup):
  224. return True
  225. from gateway.status import looks_like_gateway_command_line
  226. pids, pid = [], os.getpid()
  227. while pid > 1 and pid != unit["pid"] and _inside(_pid_cgroup(pid), unit["cgroup"]):
  228. with suppress(OSError):
  229. cmdline = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace")
  230. if pid != os.getpid() and looks_like_gateway_command_line(cmdline):
  231. break
  232. pids.append(pid)
  233. pid = _ppid(pid)
  234. bus = "--user" if unit["scope"] == "user" else "--system"
  235. cmd = ["busctl", bus, "call", "org.freedesktop.systemd1", "/org/freedesktop/systemd1",
  236. "org.freedesktop.systemd1.Manager", "StartTransientUnit", "ssa(sv)a(sa(sv))",
  237. f"hermes-update-{os.getpid()}.scope", "fail", "2", "PIDs", "au", str(len(pids)), *map(str, pids),
  238. "Description", "s", f"hermes update (outside {unit['unit']} while it is paused)", "0"]
  239. with suppress(OSError, subprocess.SubprocessError):
  240. subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace",
  241. stdin=subprocess.DEVNULL, timeout=15, check=False)
  242. deadline = time.monotonic() + 5.0
  243. while time.monotonic() < deadline:
  244. if (now := _pid_cgroup()) is not None and not _inside(now, unit_cgroup):
  245. return True
  246. time.sleep(0.1)
  247. return False
  248. def _left_running(units: list[dict], jobs: list[dict]) -> tuple[list[dict], list[dict], list[str]]:
  249. """Drop the runtimes whose stop would kill this updater; name each one."""
  250. from hermes_cli.gateway import _is_pid_ancestor_of_current_process
  251. keep_units, keep_jobs, notices = [], [], []
  252. for unit in units:
  253. if _manage_cmd(unit["scope"]) is None:
  254. notices.append(f" ↷ {unit['unit']} keeps running until after the update (stopping a system "
  255. "unit needs root or passwordless sudo)")
  256. elif _escape_cgroup(unit):
  257. keep_units.append(unit)
  258. else:
  259. notices.append(f" ↷ {unit['unit']} keeps running until after the update (this update could "
  260. "not prove it runs outside that unit's cgroup)")
  261. for job in jobs:
  262. if (_is_pid_ancestor_of_current_process(job["pid"])
  263. or os.environ.get("XPC_SERVICE_NAME") == job["label"]):
  264. notices.append(f" ↷ {job['label']} keeps running until after the update (this update runs "
  265. "inside its launchd job)")
  266. else:
  267. keep_jobs.append(job)
  268. return keep_units, keep_jobs, notices
  269. # --------------------------------------------------------------------------- stop / start
  270. def _manage_cmd(scope: str) -> list | None:
  271. from hermes_cli.update_cmd_fleet import _needs_sudo, _sudo_noninteractive_ok
  272. cmd = ["systemctl", "--user"] if scope == "user" else ["systemctl"]
  273. cmd = cmd + ["--no-ask-password"]
  274. if scope == "system" and _needs_sudo(scope):
  275. sudo = ["sudo", "-n", *cmd]
  276. return sudo if _sudo_noninteractive_ok(cmd + ["show", "-p", "Id", "hermes-gateway"]) else None
  277. return cmd
  278. def _scope_cmd(unit: dict) -> list:
  279. return ["systemctl", "--user"] if unit["scope"] == "user" else ["systemctl"]
  280. def _stop_unit(unit: dict) -> None:
  281. """Stop *unit* through systemd. A stop job systemd is still running holds the pause unless
  282. the gateway is provably still up: its job outlives our client timeout (a cgroup child that
  283. ignores SIGTERM keeps it in ``deactivating`` for up to twice TimeoutStopSec), it keeps the
  284. supervisor from respawning the gateway, and a rollback ``start`` would only queue behind it
  285. and boot the gateway seconds before the checkout moves."""
  286. from hermes_cli.update_cmd_fleet import _unit_main_pid
  287. from hermes_cli.update_cmd_fleet_unit_budget import UNIT_JOB_STILL_RUNNING, run_unit_job
  288. cmd = _manage_cmd(unit["scope"])
  289. if cmd is None:
  290. raise PauseRefused(f"no privilege to stop system unit {unit['unit']}")
  291. result = run_unit_job(cmd, _scope_cmd(unit), "stop", unit["unit"], timeout=_STOP_TIMEOUT_S)
  292. if result.returncode == UNIT_JOB_STILL_RUNNING and not _alive(unit["pid"], unit.get("ct")) \
  293. and _unit_main_pid(_scope_cmd(unit), unit["unit"]) <= 0:
  294. return
  295. if result.returncode != 0:
  296. raise PauseRefused(f"systemctl stop {unit['unit']} failed: {(result.stderr or '').strip()}")
  297. def _start_unit(unit: dict) -> int:
  298. """Start *unit* through systemd; the new MainPID (raises when it does not come up). The unit
  299. gets the definition repair the restart it replaces ran first (``_restart_one_systemd_gateway_unit``)."""
  300. from hermes_cli.update_cmd_fleet import (
  301. _repair_unit_without_fatal_exit_park, _systemctl, _unit_main_pid, _wait_for_service_active,
  302. )
  303. from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job
  304. cmd = _manage_cmd(unit["scope"])
  305. if cmd is None:
  306. raise RuntimeError(f"no privilege to start system unit {unit['unit']}")
  307. _repair_unit_without_fatal_exit_park(unit["unit"], unit["scope"])
  308. _systemctl(cmd + ["reset-failed", unit["unit"]], timeout=10)
  309. scope_cmd = _scope_cmd(unit)
  310. result = run_unit_job(cmd, scope_cmd, "start", unit["unit"], timeout=_STOP_TIMEOUT_S)
  311. if result.returncode != 0 or not _wait_for_service_active(scope_cmd, unit["unit"], timeout=15.0):
  312. raise RuntimeError(f"{unit['unit']} did not start: {(result.stderr or '').strip() or 'not active'}")
  313. return _unit_main_pid(scope_cmd, unit["unit"])
  314. def _stop_job(job: dict, *, drained: bool = False) -> None:
  315. """Boot *job* out of launchd. *drained*: its gateway already exited through the drain; the
  316. bootout only keeps KeepAlive from serving the update."""
  317. from hermes_cli.gateway import _locate_launchd_gateway_service
  318. from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker
  319. if not drained and job.get("home"):
  320. _write_update_planned_stop_marker(Path(job["home"]), job["pid"])
  321. result = subprocess.run(["launchctl", "bootout", f"{job['domain']}/{job['label']}"],
  322. capture_output=True, text=True, encoding="utf-8", errors="replace",
  323. stdin=subprocess.DEVNULL, timeout=90, check=False)
  324. _wait_gone([job["pid"]], 30.0)
  325. # KeepAlive may have respawned the drained gateway already: the job, not the old PID, decides.
  326. if result.returncode != 0 and (_locate_launchd_gateway_service(job["label"])[1] or 0) > 0:
  327. raise PauseRefused(f"launchctl bootout {job['label']} failed: {(result.stderr or '').strip()}")
  328. def _start_job(job: dict) -> None:
  329. """Load and start *job*. The invoking profile's plist is regenerated first, as the restart it
  330. replaces does (``launchd_restart``); that refresh also loads it."""
  331. from hermes_cli.gateway import (
  332. _launchctl_bootstrap, _wait_for_launchd_service_pid, get_launchd_label, refresh_launchd_plist_if_needed,
  333. )
  334. if not (job["label"] == get_launchd_label() and refresh_launchd_plist_if_needed()):
  335. _launchctl_bootstrap(job["domain"], job["plist"], job["label"])
  336. subprocess.run(["launchctl", "kickstart", f"{job['domain']}/{job['label']}"],
  337. capture_output=True, text=True, encoding="utf-8", errors="replace",
  338. stdin=subprocess.DEVNULL, timeout=30, check=False)
  339. if not _wait_for_launchd_service_pid(job["label"], old_pid=job["pid"], timeout=15.0, domain=job["domain"]):
  340. raise RuntimeError(f"{job['label']} did not come back under launchd")
  341. def _alive(pid: int, ct) -> bool:
  342. from gateway.status import _pid_exists, get_process_start_time
  343. try:
  344. if not _pid_exists(int(pid)):
  345. return False
  346. except Exception: # health: allow BLE001 -- unknown liveness counts as alive (never relaunch a twin)
  347. return True
  348. return ct is None or get_process_start_time(int(pid)) in (None, ct)
  349. def _wait_gone(pids, timeout: float, born: dict | None = None) -> set[int]:
  350. deadline = time.monotonic() + timeout
  351. live = set(pids)
  352. while live and time.monotonic() < deadline:
  353. live = {pid for pid in live if _alive(pid, (born or {}).get(pid))}
  354. if live:
  355. time.sleep(0.2)
  356. return live
  357. def _ask_to_drain(entry: dict) -> bool:
  358. """Ask the gateway over its control socket to finish its in-flight turns and exit; True on its
  359. ACK. That is the ``request_restart`` drain the post-update restart uses (SIGUSR1); SIGTERM,
  360. ``systemctl stop`` and ``launchctl bootout`` enter ``stop()`` at once and cut a turn off. No
  361. planned-stop marker on this path: the gateway's marker watcher takes that same immediate
  362. signal-stop path, and could fire before the request lands. The caller records the request
  363. (``mark_stop_sent``) BEFORE sending it, so a request that reaches the gateway after this update
  364. died still leaves the evidence that keeps the drained gateway's restart debt."""
  365. from gateway.control_socket import pause_gateway_for_update
  366. if not entry.get("home"):
  367. return False
  368. try:
  369. ack = pause_gateway_for_update(Path(entry["home"]))
  370. except Exception: # health: allow BLE001 -- no answer is the pre-verb gateway: the fallback stop handles it
  371. return False
  372. return bool(ack and (ack.get("pausing") or ack.get("already_stopping")))
  373. def _stop_at_once(entry: dict) -> None:
  374. """The stop for a gateway that did not take the drain: its supervisor's, else marker + SIGTERM."""
  375. from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker
  376. if entry.get("kind") == "systemd":
  377. _stop_unit(entry)
  378. elif entry.get("kind") == "launchd":
  379. _stop_job(entry)
  380. else:
  381. _write_update_planned_stop_marker(Path(entry["home"]), entry["pid"])
  382. with suppress(ProcessLookupError, PermissionError):
  383. os.kill(entry["pid"], signal.SIGTERM)
  384. def _drain_budget() -> float:
  385. """The post-update restart's wait for a drained gateway (after-turn wait + stop drain + headroom)."""
  386. from hermes_cli.update_cmd_fleet import _gateway_drain_budget
  387. return _gateway_drain_budget()
  388. def _stop_gateways(token: dict, entries: list[dict]) -> None:
  389. """Drain every gateway (bare ones SIGTERMed when they do not answer), then wait for all of them
  390. within one drain budget. A supervised gateway that exited through the drain is stopped through
  391. its supervisor at once: its exit 75 asks for a restart, which systemd honours after
  392. ``RestartSec=5`` and launchd immediately, still on the old tree. Past the budget a supervised
  393. survivor gets its supervisor's stop and a bare one SIGKILL, as the restart path escalates."""
  394. from hermes_cli import update_pause_record as pause_record
  395. from hermes_cli.update_cmd_drain_report import drain_progress_reporter
  396. budget, waiting = _drain_budget(), []
  397. for entry in entries:
  398. pause_record.mark_stop_sent(token, entry["pid"]) # before the request: see _ask_to_drain
  399. if _ask_to_drain(entry):
  400. waiting.append(entry)
  401. else:
  402. _stop_at_once(entry)
  403. if not entry.get("kind"):
  404. waiting.append(entry) # a SIGTERMed bare gateway is waited for like a drained one
  405. if not waiting:
  406. return
  407. print(f" ⏳ Waiting up to {budget:.0f}s for in-flight turns on {_names(waiting)} to finish before the update...")
  408. ticks = [drain_progress_reporter(Path(e["home"]) if e.get("home") else None, budget_s=budget) for e in waiting]
  409. deadline = time.monotonic() + budget
  410. while waiting and time.monotonic() < deadline:
  411. for entry in [e for e in waiting if not _alive(e["pid"], e.get("ct"))]:
  412. waiting.remove(entry)
  413. _disarm(entry)
  414. for tick in ticks if waiting else ():
  415. tick()
  416. time.sleep(0.2)
  417. for entry in waiting:
  418. if entry.get("kind"):
  419. _stop_at_once(entry)
  420. else:
  421. with suppress(ProcessLookupError, PermissionError):
  422. os.kill(entry["pid"], getattr(signal, "SIGKILL", signal.SIGTERM))
  423. bare = [e for e in waiting if not e.get("kind")]
  424. if left := _wait_gone([e["pid"] for e in bare], 10.0, {e["pid"]: e.get("ct") for e in bare}):
  425. raise PauseRefused("gateway PID(s) " + ", ".join(map(str, sorted(left))) + " did not stop")
  426. def _disarm(entry: dict) -> None:
  427. """A drained gateway exited: keep its supervisor from bringing it back during the update."""
  428. if entry.get("kind") == "systemd":
  429. _stop_unit(entry)
  430. elif entry.get("kind") == "launchd":
  431. _stop_job(entry, drained=True)
  432. def _names(entries: list[dict]) -> str:
  433. return ", ".join(e.get("unit") or e.get("label") or f"gateway PID {e['pid']}" for e in entries)
  434. def _gateway_on_home(home: str, exclude: set[int]) -> int | None:
  435. """The gateway serving *home*, by the canonical identity reader (PID file + runtime lock,
  436. then the runtime status record), never a process scan: a replayed ``gateway run`` matches the
  437. process matcher for the moment before it finds the home already served and exits."""
  438. from gateway.status import live_gateway_pid_for_home
  439. try:
  440. pid = live_gateway_pid_for_home(Path(home))
  441. except Exception: # health: allow BLE001 -- unreadable identity files: nothing proven to adopt
  442. return None
  443. return int(pid) if pid and int(pid) not in exclude else None
  444. def _replay_env(home: str) -> dict:
  445. """The environment a replayed gateway starts with, as the restart watcher it replaces chose it
  446. (``_spawn_gateway_restart_watcher``): a named profile's gateway, replayed by an update launched
  447. under that same profile, inherits the launcher's environment (an exported bot token may be its
  448. only credential). The multiplexing root, and any home other than the launcher's, get that home's
  449. own secrets and none of the launcher's (``served_profile_child_env``)."""
  450. from hermes_constants import get_default_hermes_root, get_routing_process_hermes_home
  451. from tools.environments.local import build_subprocess_env, served_profile_child_env
  452. target = Path(home).resolve()
  453. if target != get_default_hermes_root().resolve() and target == get_routing_process_hermes_home().resolve():
  454. return build_subprocess_env(scrub_secrets=False, inherit_profile_home=False, extra={"HERMES_HOME": home})
  455. return served_profile_child_env(target_home=home, inherit_credentials=True)
  456. def _relaunch_bare(entry: dict) -> int:
  457. """Replay a bare gateway from its recorded argv under its recorded home; the new PID once it
  458. is serving. A gateway already serving that home (an earlier attempt's) is adopted instead."""
  459. if (existing := _gateway_on_home(entry["home"], {int(entry["pid"])})) is not None:
  460. return existing
  461. logs = Path(entry["home"]) / "logs"
  462. logs.mkdir(parents=True, exist_ok=True)
  463. env = _replay_env(entry["home"])
  464. for leaked in ("INVOCATION_ID", "JOURNAL_STREAM", "HERMES_UPDATE_PAUSED", "_HERMES_GATEWAY"):
  465. env.pop(leaked, None)
  466. with open(logs / "gateway-update-resume.log", "ab") as log:
  467. proc = subprocess.Popen(list(entry["argv"]), env=env, stdin=subprocess.DEVNULL, stdout=log,
  468. # detached-spawn: ok — resumes the paused gateway after an update; it must outlive the updater
  469. stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
  470. deadline = time.monotonic() + _READY_TIMEOUT_S
  471. while time.monotonic() < deadline:
  472. if (pid := _gateway_on_home(entry["home"], {int(entry["pid"])})) is not None:
  473. return pid # the replay, or a gateway someone started meanwhile (the replay then exits 0)
  474. if proc.poll() is not None:
  475. raise RuntimeError(f"replayed gateway for {entry['home']} exited {proc.returncode} "
  476. f"(see {logs / 'gateway-update-resume.log'})")
  477. time.sleep(0.5)
  478. raise RuntimeError(f"replayed gateway for {entry['home']} did not come up within {int(_READY_TIMEOUT_S)}s")
  479. def _describe(token: dict) -> str:
  480. names = [u.get("unit") or u.get("label") for u in token.get("posix_units") or []]
  481. names += [f"gateway PID {u['pid']}" for u in token.get("unmapped") or [] if u.get("argv")]
  482. return ", ".join(names)
  483. # --------------------------------------------------------------------------- the commit point
  484. def pause_at_commit_point() -> str | None:
  485. """Stop this install's gateways before the first checkout move; once per run.
  486. Never refuses the update: the pause is protection, not a gate. When it cannot complete
  487. (discovery or the durable record fails, a stop fails), the gateways run through the swap and
  488. the post-commit restart refreshes them, which is how POSIX updated before the pause existed.
  489. Always ``None``; the caller's refusal branch stays for the contract it shares with Windows."""
  490. token = _RUN
  491. if token is None:
  492. return None
  493. if "posix_pause_verdict" not in token:
  494. token["posix_pause_verdict"] = _pause(token)
  495. return token["posix_pause_verdict"]
  496. def _pause(token: dict) -> str | None:
  497. from hermes_cli import update_pause_record as pause_record
  498. try:
  499. units, jobs = _discover_systemd(), _discover_launchd()
  500. units, jobs, notices = _left_running(units, jobs)
  501. bare, more = _discover_bare({u["pid"] for u in units} | {j["pid"] for j in jobs})
  502. except Exception as exc: # health: allow BLE001 -- a discovery that cannot finish stops nothing
  503. print(f" ⚠ Could not list the running gateways ({exc}); they keep running and are restarted after the update")
  504. return None
  505. for line in notices + more:
  506. print(line)
  507. supervised = units + jobs
  508. if not supervised and not bare:
  509. return None
  510. from gateway.status import get_process_start_time
  511. from hermes_cli.update_cmd_windows import _planned_stop_marker_path
  512. before = copy.deepcopy(token) # what this attempt may add; a failed record puts it back
  513. for entry in supervised + bare:
  514. entry["ct"] = get_process_start_time(entry["pid"]) # native birth stamp: what _alive compares
  515. token["posix_units"] = [*token.get("posix_units", []), *supervised]
  516. token["unmapped"] = [*token.get("unmapped", []), *bare]
  517. token["unmapped_pids"] = [*token.get("unmapped_pids", []), *(e["pid"] for e in bare)]
  518. # Recovery judges liveness by the canonical identity (``ct:<unix seconds>``), never the native stamp.
  519. token["identities"] = {**token.get("identities", {}),
  520. **{str(e["pid"]): pause_record.identity(e["pid"])["ct"] for e in supervised + bare}}
  521. token["resume_needed"] = True
  522. try:
  523. pause_record.record_pause(token, None, [])
  524. pause_record.mark_stop_requested(
  525. token, [e["pid"] for e in supervised + bare],
  526. markers={e["pid"]: _planned_stop_marker_path(Path(e["home"])) for e in supervised + bare if e.get("home")})
  527. except Exception as exc: # health: allow BLE001 -- no durable record, no stop
  528. # Nothing was stopped: this attempt's id must not arm the checkout's tree gate, whose
  529. # required record write would then refuse the update. An adopted obligation stays as it was.
  530. if token.get("pause_id") != before.get("pause_id"):
  531. with suppress(Exception):
  532. pause_record.discharge(token)
  533. token.clear()
  534. token.update(before)
  535. print(f" ⚠ Could not record the gateways to pause ({exc}); they keep running and are restarted after the update")
  536. return None
  537. # From here a refusal or a crash leaves the set owed: the command's own exit resume (registered
  538. # in ``_cmd_update_impl`` for this token), else the next launch's recovery, restarts it.
  539. token["posix_stopped"] = True
  540. try:
  541. _stop_gateways(token, supervised + bare)
  542. except Exception as exc: # health: allow BLE001 -- roll back: restart the stopped set, update unpaused
  543. print(f" ⚠ Could not pause every gateway ({exc}); restarting them, the update continues without the pause")
  544. try:
  545. resume_paused_set(token)
  546. except Exception as again: # health: allow BLE001 -- still owed: completion restarts them after the deps
  547. print(f" ⚠ {again}; they restart once the dependencies are synced")
  548. for restarted in _RESTARTED.values():
  549. restarted.clear() # back on the OLD code: the post-commit fleet restart must refresh them
  550. pause_record.sync(token)
  551. return None
  552. pause_record.sync(token)
  553. print(f" ⏸ Paused for the update: {_describe(token)} (restarted once the dependencies are synced)")
  554. return None
  555. # --------------------------------------------------------------------------- resume
  556. def resume_paused_set(token: dict) -> None:
  557. """Start every paused runtime again; each leaves *token* only on its own verified start."""
  558. failures: list[str] = []
  559. restarted: list[str] = []
  560. kept_units = []
  561. for entry in token.get("posix_units") or []:
  562. try:
  563. if entry.get("kind") == "launchd":
  564. _start_job(entry)
  565. _RESTARTED["labels"].add(entry["label"])
  566. name = entry["label"]
  567. else:
  568. _RESTARTED["pids"].add(_start_unit(entry))
  569. _RESTARTED["units"].add(f"{entry['scope']}/{entry['unit']}")
  570. name = entry["unit"]
  571. restarted.append(name)
  572. token.setdefault("restarted_services", []).append(name)
  573. except Exception as exc: # health: allow BLE001 -- one failed unit never keeps the others stopped
  574. failures.append(str(exc))
  575. kept_units.append(entry)
  576. token["posix_units"] = kept_units
  577. kept_bare = []
  578. for entry in token.get("unmapped") or []:
  579. if not entry.get("argv"):
  580. continue
  581. try:
  582. new_pid = _relaunch_bare(entry)
  583. _RESTARTED["pids"].add(new_pid)
  584. restarted.append(f"gateway PID {entry['pid']} → {new_pid}")
  585. except Exception as exc: # health: allow BLE001 -- one failed replay never keeps the others stopped
  586. failures.append(str(exc))
  587. kept_bare.append(entry)
  588. token["unmapped"] = kept_bare
  589. token["unmapped_pids"] = [e["pid"] for e in kept_bare]
  590. token["profiles"] = {}
  591. if restarted:
  592. print(f" ▶ Restarted paused gateway(s): {', '.join(restarted)}")
  593. if failures:
  594. raise RuntimeError("Could not restart every paused gateway: " + "; ".join(failures))
  595. token["resume_needed"] = False
  596. token["posix_stopped"] = False
  597. def already_restarted() -> dict:
  598. """What the early resume restarted on the new code in this process (for the fleet restart)."""
  599. return _RESTARTED