| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690 |
- """Pre-swap gateway pause for ``hermes update`` on Linux and macOS.
- Windows pauses its gateways before the fetch (``update_cmd_windows``) because they lock venv
- files. On POSIX nothing is locked, but a gateway that keeps running while git rewrites the tree
- loads new-tree modules into an old process on its next lazy import. So every gateway running
- from this install is stopped at the commit point, immediately before the first checkout move
- (``update_cmd_commit.arm_commit_point``), and the same set is restarted once the dependencies
- are synced and the launchers published, before the product builds
- (``source_completion.complete_source_checkout``'s ``before_build``).
- The set rides the Windows pause contract: one token in ``update_pause_record`` (identity = pid
- + start time; supervisor unit, or argv + home for bare processes), written before the first stop,
- resumed by ``_resume_windows_gateways_after_update`` (which dispatches here on
- ``platform == "posix"``), adopted by the next ``hermes update`` and resumed by the next launch
- when this process dies.
- * every gateway is drained first: the control socket's ``pause-for-update`` runs the same
- ``request_restart`` drain as the post-update restart's SIGUSR1 (refuse new turns, let the
- in-flight ones finish within ``agent.restart_after_turn_timeout``, then exit), waited for within
- that restart's budget. The request is recorded before it is sent, and carries no planned-stop
- marker (the gateway's marker watcher would take the immediate stop path instead).
- * systemd ``hermes-gateway*`` units and launchd ``ai.hermes.gateway*`` jobs are then stopped and
- later started through their supervisor, so it cannot respawn them on old code mid-update; one
- that does not take the drain gets its supervisor's stop at once.
- * bare ``gateway run`` processes that do not answer get marker + SIGTERM, and are replayed from
- their recorded argv + home.
- * the updater never kills itself: a unit whose cgroup holds this process (``/update`` from
- chat: the update runs in the gateway unit's cgroup, which ``systemctl stop`` kills) is
- stopped only after this process moved into a transient scope of its own. A launchd job
- whose coalition holds it, or any gateway whose stop would take the updater down, is left
- running for the post-commit restart.
- """
- from __future__ import annotations
- import copy
- import os
- import signal
- import subprocess
- import sys
- import time
- from contextlib import suppress
- from pathlib import Path
- #: This run's token: armed in ``_cmd_update_impl``, filled at the commit point. Same dict object
- #: the completion request carries, so the paused set reaches the completion child.
- _RUN: dict | None = None
- #: Runtimes the early resume already restarted on the new code: the late fleet restart in the
- #: same completion process must not restart them a second time.
- _RESTARTED: dict = {"units": set(), "labels": set(), "pids": set()}
- _STOP_TIMEOUT_S = 120.0
- _READY_TIMEOUT_S = 30.0
- class PauseRefused(RuntimeError):
- """The pause could not be completed; everything it stopped was started again."""
- def _empty_token() -> dict:
- return {"platform": "posix", "resume_needed": False, "profiles": {}, "unmapped_pids": [],
- "unmapped": [], "posix_units": []}
- def arm_pause(*, no_gateway_restart: bool) -> dict | None:
- """Arm this run's pause (no stop yet) and adopt any set a killed update left paused.
- ``None`` on Windows (its own pause) and with ``--no-gateway-restart`` (the user manages
- gateways, nothing is stopped). An adopted orphan set is recorded under this process now, so
- even an update that never moves the checkout restarts it at exit."""
- global _RUN
- if sys.platform == "win32" or no_gateway_restart:
- return None
- from hermes_cli import update_pause_record as pause_record
- token = _empty_token()
- try:
- adopted, claims = pause_record.adopt_orphans()
- if adopted is not None:
- token = pause_record.record_pause({**token, "resume_needed": True}, adopted, claims)
- token["platform"] = "posix"
- except Exception as exc: # health: allow BLE001 -- an unreadable record must not block the update; recovery retries it
- print(f" ⚠ Could not adopt gateways an earlier update paused: {exc}")
- _RUN = token
- return token
- def gateways_paused() -> bool:
- """True while this run holds gateways stopped (no gateway can answer a chat prompt)."""
- return bool(_RUN and _RUN.get("posix_stopped") and _RUN.get("resume_needed"))
- # --------------------------------------------------------------------------- discovery
- def _unit_cgroup(scope_cmd: list, unit: str) -> str | None:
- from hermes_cli.update_cmd_fleet import _systemctl
- with suppress(OSError, subprocess.TimeoutExpired):
- result = _systemctl(scope_cmd + ["show", unit, "--property=ControlGroup", "--value"], timeout=10)
- if result.returncode == 0 and result.stdout.strip():
- return result.stdout.strip()
- return None
- def _pid_cgroup(pid: int | str = "self") -> str | None:
- """The systemd cgroup path of *pid*: the unified (v2) line, else the v1 ``name=systemd``
- hierarchy, which systemd names units in identically. ``None``: not Linux or unreadable, which
- callers must treat as unknown, never as outside a unit."""
- with suppress(OSError):
- lines = Path(f"/proc/{pid}/cgroup").read_text(encoding="utf-8").splitlines()
- unified = next((line[3:].strip() for line in lines if line.startswith("0::")), "")
- legacy = next((line.split(":", 2)[2].strip() for line in lines if ":name=systemd:" in line), "")
- return unified or legacy or None
- return None
- def _inside(cgroup: str | None, unit_cgroup: str | None) -> bool:
- if not cgroup or not unit_cgroup:
- return False
- return cgroup == unit_cgroup or cgroup.startswith(unit_cgroup.rstrip("/") + "/")
- def _discover_systemd() -> list[dict]:
- """Active ``hermes-gateway*`` units of this install, one per live MainPID."""
- from hermes_cli.gateway import _ensure_user_systemd_env, supports_systemd_services
- from hermes_cli.update_cmd_fleet import (
- _for_each_systemd_gateway_unit, _systemd_gateway_unit_listings, _systemd_unit_owned_by_update,
- _unit_main_pid,
- )
- if not supports_systemd_services():
- return []
- with suppress(Exception):
- _ensure_user_systemd_env()
- units: list[dict] = []
- seen_pids: set[int] = set()
- for scope, scope_cmd, listing in _systemd_gateway_unit_listings():
- names: list[str] = []
- _for_each_systemd_gateway_unit(listing.stdout, process_unit=names.append,
- on_unit_timeout=lambda name, exc: None)
- for name in names:
- if not name.startswith("hermes-gateway"):
- continue # serve/dashboard units are not gateways: the post-commit restart owns them
- pid = _unit_main_pid(scope_cmd, name)
- if pid <= 0 or pid in seen_pids or not _systemd_unit_owned_by_update(scope_cmd, name):
- continue
- seen_pids.add(pid) # a legacy per-profile unit sharing the multiplexer's PID is one process
- units.append({"kind": "systemd", "scope": scope, "unit": name, "pid": pid,
- "cgroup": _unit_cgroup(scope_cmd, name), "home": _home_of(pid)})
- return units
- def _discover_launchd() -> list[dict]:
- """Loaded ``ai.hermes.gateway*`` jobs of this install with a live process (macOS)."""
- if sys.platform != "darwin":
- return []
- from hermes_cli.gateway import (
- _locate_launchd_gateway_service, get_launchd_plist_path, launchd_gateway_labels_for_install,
- legacy_launchd_labels_for_install,
- )
- from hermes_cli.update_fleet_scope import launchd_label_foreign_home
- derived = launchd_gateway_labels_for_install()
- jobs = []
- for label in derived + legacy_launchd_labels_for_install(exclude=set(derived)):
- if launchd_label_foreign_home(label) is not None:
- continue
- domain, pid = _locate_launchd_gateway_service(label)
- if domain is None or not pid or pid <= 0:
- continue
- plist = get_launchd_plist_path().with_name(f"{label}.plist")
- jobs.append({"kind": "launchd", "label": label, "domain": domain, "plist": str(plist), "pid": int(pid),
- "home": _home_of(int(pid))})
- return jobs
- def _home_of(pid: int) -> str | None:
- """The Hermes home *pid* serves (its control socket lives there), or ``None``."""
- from hermes_cli.update_fleet_scope import gateway_pid_home
- with suppress(Exception):
- if home := gateway_pid_home(pid):
- return str(home)
- return None
- def _supervisor_markers(pid: int) -> str | None:
- """Why *pid* answers to a supervisor other than a hermes unit/job (it would respawn it on old
- code), or ``None``."""
- from hermes_cli.gateway import _capture_gateway_argv
- if "--external-supervisor" in (_capture_gateway_argv(pid) or []):
- return "--external-supervisor"
- try:
- import psutil # type: ignore
- env = psutil.Process(pid).environ()
- except Exception: # health: allow BLE001 -- unreadable environment: no marker proven
- return None
- if env.get("HERMES_S6_SUPERVISED_CHILD"):
- return "s6"
- if unit := _respawning_service(pid):
- return f"systemd unit {unit}"
- # launchd names terminal apps' shells application.<bundle id>; those are not KeepAlive jobs.
- xpc = env.get("XPC_SERVICE_NAME", "0")
- if xpc not in ("", "0") and not xpc.startswith("application."):
- return f"launchd job {xpc}"
- return None
- def _respawning_service(pid: int) -> str | None:
- """The systemd service that would respawn *pid* when it exits, or ``None``.
- systemd restarts a unit only when its main process exits, so the gateway must BE the
- MainPID or its direct child (a non-exec wrapper that exits with it). The inherited
- INVOCATION_ID is no evidence: a terminal tab's shell carries one (its .scope is never
- respawned), and so does anything deep under a CI runner's service."""
- from hermes_cli.update_cmd_fleet import _systemctl, _unit_main_pid
- cgroup = _pid_cgroup(pid) or ""
- leaf = cgroup.rsplit("/", 1)[-1]
- if not leaf.endswith(".service"):
- return None
- scope_cmd = ["systemctl", "--user"] if "/user@" in cgroup else ["systemctl"]
- with suppress(OSError, subprocess.SubprocessError):
- main = _unit_main_pid(scope_cmd, leaf)
- if main > 0 and main in (pid, _ppid(pid)):
- restart = _systemctl(scope_cmd + ["show", leaf, "--property=Restart", "--value"], timeout=10)
- return leaf if (restart.stdout or "").strip() not in ("", "no") else None
- return None
- def _discover_bare(service_pids: set[int]) -> tuple[list[dict], list[str]]:
- """``(bare gateways, notices)``: this install's ``gateway run`` processes outside any unit."""
- from gateway.status import get_process_start_time
- from hermes_cli.gateway import _capture_gateway_argv, _get_service_pids, find_gateway_pids
- from hermes_cli.update_cmd_fleet import _scoped_manual_gateway_pids
- from hermes_cli.update_fleet_scope import gateway_pid_home
- exclude = set(service_pids) | set(_get_service_pids(all_profiles=True))
- pids = _scoped_manual_gateway_pids(find_gateway_pids(exclude_pids=exclude, all_profiles=True))
- bare, notices = [], []
- for pid in pids:
- pid = int(pid)
- marker = _supervisor_markers(pid)
- argv, home = _capture_gateway_argv(pid), gateway_pid_home(pid)
- if marker or not argv or not home:
- why = f"supervised by {marker}" if marker else "its command line could not be read" if not argv \
- else "its Hermes home could not be proved"
- notices.append(f" ↷ gateway PID {pid} keeps running until after the update ({why})")
- continue
- bare.append({"pid": pid, "argv": list(argv), "home": str(home), "ct": get_process_start_time(pid)})
- return bare, notices
- # --------------------------------------------------------------------------- self-preservation
- def _ppid(pid: int) -> int:
- with suppress(OSError, ValueError, IndexError):
- stat = Path(f"/proc/{pid}/stat").read_text(encoding="utf-8")
- return int(stat.rsplit(")", 1)[1].split()[1])
- return 0
- def _escape_cgroup(unit: dict) -> bool:
- """Move this process (and its ancestors in the same unit, e.g. ``/update``'s bash wrapper)
- into a transient scope so stopping *unit* cannot kill it. True only when proven out: a
- membership that cannot be read (either side) keeps the unit running, since its stop would take
- this update down with it."""
- own, unit_cgroup = _pid_cgroup(), unit.get("cgroup")
- if own is None or not unit_cgroup:
- return False
- if not _inside(own, unit_cgroup):
- return True
- from gateway.status import looks_like_gateway_command_line
- pids, pid = [], os.getpid()
- while pid > 1 and pid != unit["pid"] and _inside(_pid_cgroup(pid), unit["cgroup"]):
- with suppress(OSError):
- cmdline = Path(f"/proc/{pid}/cmdline").read_bytes().replace(b"\0", b" ").decode("utf-8", "replace")
- if pid != os.getpid() and looks_like_gateway_command_line(cmdline):
- break
- pids.append(pid)
- pid = _ppid(pid)
- bus = "--user" if unit["scope"] == "user" else "--system"
- cmd = ["busctl", bus, "call", "org.freedesktop.systemd1", "/org/freedesktop/systemd1",
- "org.freedesktop.systemd1.Manager", "StartTransientUnit", "ssa(sv)a(sa(sv))",
- f"hermes-update-{os.getpid()}.scope", "fail", "2", "PIDs", "au", str(len(pids)), *map(str, pids),
- "Description", "s", f"hermes update (outside {unit['unit']} while it is paused)", "0"]
- with suppress(OSError, subprocess.SubprocessError):
- subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace",
- stdin=subprocess.DEVNULL, timeout=15, check=False)
- deadline = time.monotonic() + 5.0
- while time.monotonic() < deadline:
- if (now := _pid_cgroup()) is not None and not _inside(now, unit_cgroup):
- return True
- time.sleep(0.1)
- return False
- def _left_running(units: list[dict], jobs: list[dict]) -> tuple[list[dict], list[dict], list[str]]:
- """Drop the runtimes whose stop would kill this updater; name each one."""
- from hermes_cli.gateway import _is_pid_ancestor_of_current_process
- keep_units, keep_jobs, notices = [], [], []
- for unit in units:
- if _manage_cmd(unit["scope"]) is None:
- notices.append(f" ↷ {unit['unit']} keeps running until after the update (stopping a system "
- "unit needs root or passwordless sudo)")
- elif _escape_cgroup(unit):
- keep_units.append(unit)
- else:
- notices.append(f" ↷ {unit['unit']} keeps running until after the update (this update could "
- "not prove it runs outside that unit's cgroup)")
- for job in jobs:
- if (_is_pid_ancestor_of_current_process(job["pid"])
- or os.environ.get("XPC_SERVICE_NAME") == job["label"]):
- notices.append(f" ↷ {job['label']} keeps running until after the update (this update runs "
- "inside its launchd job)")
- else:
- keep_jobs.append(job)
- return keep_units, keep_jobs, notices
- # --------------------------------------------------------------------------- stop / start
- def _manage_cmd(scope: str) -> list | None:
- from hermes_cli.update_cmd_fleet import _needs_sudo, _sudo_noninteractive_ok
- cmd = ["systemctl", "--user"] if scope == "user" else ["systemctl"]
- cmd = cmd + ["--no-ask-password"]
- if scope == "system" and _needs_sudo(scope):
- sudo = ["sudo", "-n", *cmd]
- return sudo if _sudo_noninteractive_ok(cmd + ["show", "-p", "Id", "hermes-gateway"]) else None
- return cmd
- def _scope_cmd(unit: dict) -> list:
- return ["systemctl", "--user"] if unit["scope"] == "user" else ["systemctl"]
- def _stop_unit(unit: dict) -> None:
- """Stop *unit* through systemd. A stop job systemd is still running holds the pause unless
- the gateway is provably still up: its job outlives our client timeout (a cgroup child that
- ignores SIGTERM keeps it in ``deactivating`` for up to twice TimeoutStopSec), it keeps the
- supervisor from respawning the gateway, and a rollback ``start`` would only queue behind it
- and boot the gateway seconds before the checkout moves."""
- from hermes_cli.update_cmd_fleet import _unit_main_pid
- from hermes_cli.update_cmd_fleet_unit_budget import UNIT_JOB_STILL_RUNNING, run_unit_job
- cmd = _manage_cmd(unit["scope"])
- if cmd is None:
- raise PauseRefused(f"no privilege to stop system unit {unit['unit']}")
- result = run_unit_job(cmd, _scope_cmd(unit), "stop", unit["unit"], timeout=_STOP_TIMEOUT_S)
- if result.returncode == UNIT_JOB_STILL_RUNNING and not _alive(unit["pid"], unit.get("ct")) \
- and _unit_main_pid(_scope_cmd(unit), unit["unit"]) <= 0:
- return
- if result.returncode != 0:
- raise PauseRefused(f"systemctl stop {unit['unit']} failed: {(result.stderr or '').strip()}")
- def _start_unit(unit: dict) -> int:
- """Start *unit* through systemd; the new MainPID (raises when it does not come up). The unit
- gets the definition repair the restart it replaces ran first (``_restart_one_systemd_gateway_unit``)."""
- from hermes_cli.update_cmd_fleet import (
- _repair_unit_without_fatal_exit_park, _systemctl, _unit_main_pid, _wait_for_service_active,
- )
- from hermes_cli.update_cmd_fleet_unit_budget import run_unit_job
- cmd = _manage_cmd(unit["scope"])
- if cmd is None:
- raise RuntimeError(f"no privilege to start system unit {unit['unit']}")
- _repair_unit_without_fatal_exit_park(unit["unit"], unit["scope"])
- _systemctl(cmd + ["reset-failed", unit["unit"]], timeout=10)
- scope_cmd = _scope_cmd(unit)
- result = run_unit_job(cmd, scope_cmd, "start", unit["unit"], timeout=_STOP_TIMEOUT_S)
- if result.returncode != 0 or not _wait_for_service_active(scope_cmd, unit["unit"], timeout=15.0):
- raise RuntimeError(f"{unit['unit']} did not start: {(result.stderr or '').strip() or 'not active'}")
- return _unit_main_pid(scope_cmd, unit["unit"])
- def _stop_job(job: dict, *, drained: bool = False) -> None:
- """Boot *job* out of launchd. *drained*: its gateway already exited through the drain; the
- bootout only keeps KeepAlive from serving the update."""
- from hermes_cli.gateway import _locate_launchd_gateway_service
- from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker
- if not drained and job.get("home"):
- _write_update_planned_stop_marker(Path(job["home"]), job["pid"])
- result = subprocess.run(["launchctl", "bootout", f"{job['domain']}/{job['label']}"],
- capture_output=True, text=True, encoding="utf-8", errors="replace",
- stdin=subprocess.DEVNULL, timeout=90, check=False)
- _wait_gone([job["pid"]], 30.0)
- # KeepAlive may have respawned the drained gateway already: the job, not the old PID, decides.
- if result.returncode != 0 and (_locate_launchd_gateway_service(job["label"])[1] or 0) > 0:
- raise PauseRefused(f"launchctl bootout {job['label']} failed: {(result.stderr or '').strip()}")
- def _start_job(job: dict) -> None:
- """Load and start *job*. The invoking profile's plist is regenerated first, as the restart it
- replaces does (``launchd_restart``); that refresh also loads it."""
- from hermes_cli.gateway import (
- _launchctl_bootstrap, _wait_for_launchd_service_pid, get_launchd_label, refresh_launchd_plist_if_needed,
- )
- if not (job["label"] == get_launchd_label() and refresh_launchd_plist_if_needed()):
- _launchctl_bootstrap(job["domain"], job["plist"], job["label"])
- subprocess.run(["launchctl", "kickstart", f"{job['domain']}/{job['label']}"],
- capture_output=True, text=True, encoding="utf-8", errors="replace",
- stdin=subprocess.DEVNULL, timeout=30, check=False)
- if not _wait_for_launchd_service_pid(job["label"], old_pid=job["pid"], timeout=15.0, domain=job["domain"]):
- raise RuntimeError(f"{job['label']} did not come back under launchd")
- def _alive(pid: int, ct) -> bool:
- from gateway.status import _pid_exists, get_process_start_time
- try:
- if not _pid_exists(int(pid)):
- return False
- except Exception: # health: allow BLE001 -- unknown liveness counts as alive (never relaunch a twin)
- return True
- return ct is None or get_process_start_time(int(pid)) in (None, ct)
- def _wait_gone(pids, timeout: float, born: dict | None = None) -> set[int]:
- deadline = time.monotonic() + timeout
- live = set(pids)
- while live and time.monotonic() < deadline:
- live = {pid for pid in live if _alive(pid, (born or {}).get(pid))}
- if live:
- time.sleep(0.2)
- return live
- def _ask_to_drain(entry: dict) -> bool:
- """Ask the gateway over its control socket to finish its in-flight turns and exit; True on its
- ACK. That is the ``request_restart`` drain the post-update restart uses (SIGUSR1); SIGTERM,
- ``systemctl stop`` and ``launchctl bootout`` enter ``stop()`` at once and cut a turn off. No
- planned-stop marker on this path: the gateway's marker watcher takes that same immediate
- signal-stop path, and could fire before the request lands. The caller records the request
- (``mark_stop_sent``) BEFORE sending it, so a request that reaches the gateway after this update
- died still leaves the evidence that keeps the drained gateway's restart debt."""
- from gateway.control_socket import pause_gateway_for_update
- if not entry.get("home"):
- return False
- try:
- ack = pause_gateway_for_update(Path(entry["home"]))
- except Exception: # health: allow BLE001 -- no answer is the pre-verb gateway: the fallback stop handles it
- return False
- return bool(ack and (ack.get("pausing") or ack.get("already_stopping")))
- def _stop_at_once(entry: dict) -> None:
- """The stop for a gateway that did not take the drain: its supervisor's, else marker + SIGTERM."""
- from hermes_cli.update_cmd_windows import _write_update_planned_stop_marker
- if entry.get("kind") == "systemd":
- _stop_unit(entry)
- elif entry.get("kind") == "launchd":
- _stop_job(entry)
- else:
- _write_update_planned_stop_marker(Path(entry["home"]), entry["pid"])
- with suppress(ProcessLookupError, PermissionError):
- os.kill(entry["pid"], signal.SIGTERM)
- def _drain_budget() -> float:
- """The post-update restart's wait for a drained gateway (after-turn wait + stop drain + headroom)."""
- from hermes_cli.update_cmd_fleet import _gateway_drain_budget
- return _gateway_drain_budget()
- def _stop_gateways(token: dict, entries: list[dict]) -> None:
- """Drain every gateway (bare ones SIGTERMed when they do not answer), then wait for all of them
- within one drain budget. A supervised gateway that exited through the drain is stopped through
- its supervisor at once: its exit 75 asks for a restart, which systemd honours after
- ``RestartSec=5`` and launchd immediately, still on the old tree. Past the budget a supervised
- survivor gets its supervisor's stop and a bare one SIGKILL, as the restart path escalates."""
- from hermes_cli import update_pause_record as pause_record
- from hermes_cli.update_cmd_drain_report import drain_progress_reporter
- budget, waiting = _drain_budget(), []
- for entry in entries:
- pause_record.mark_stop_sent(token, entry["pid"]) # before the request: see _ask_to_drain
- if _ask_to_drain(entry):
- waiting.append(entry)
- else:
- _stop_at_once(entry)
- if not entry.get("kind"):
- waiting.append(entry) # a SIGTERMed bare gateway is waited for like a drained one
- if not waiting:
- return
- print(f" ⏳ Waiting up to {budget:.0f}s for in-flight turns on {_names(waiting)} to finish before the update...")
- ticks = [drain_progress_reporter(Path(e["home"]) if e.get("home") else None, budget_s=budget) for e in waiting]
- deadline = time.monotonic() + budget
- while waiting and time.monotonic() < deadline:
- for entry in [e for e in waiting if not _alive(e["pid"], e.get("ct"))]:
- waiting.remove(entry)
- _disarm(entry)
- for tick in ticks if waiting else ():
- tick()
- time.sleep(0.2)
- for entry in waiting:
- if entry.get("kind"):
- _stop_at_once(entry)
- else:
- with suppress(ProcessLookupError, PermissionError):
- os.kill(entry["pid"], getattr(signal, "SIGKILL", signal.SIGTERM))
- bare = [e for e in waiting if not e.get("kind")]
- if left := _wait_gone([e["pid"] for e in bare], 10.0, {e["pid"]: e.get("ct") for e in bare}):
- raise PauseRefused("gateway PID(s) " + ", ".join(map(str, sorted(left))) + " did not stop")
- def _disarm(entry: dict) -> None:
- """A drained gateway exited: keep its supervisor from bringing it back during the update."""
- if entry.get("kind") == "systemd":
- _stop_unit(entry)
- elif entry.get("kind") == "launchd":
- _stop_job(entry, drained=True)
- def _names(entries: list[dict]) -> str:
- return ", ".join(e.get("unit") or e.get("label") or f"gateway PID {e['pid']}" for e in entries)
- def _gateway_on_home(home: str, exclude: set[int]) -> int | None:
- """The gateway serving *home*, by the canonical identity reader (PID file + runtime lock,
- then the runtime status record), never a process scan: a replayed ``gateway run`` matches the
- process matcher for the moment before it finds the home already served and exits."""
- from gateway.status import live_gateway_pid_for_home
- try:
- pid = live_gateway_pid_for_home(Path(home))
- except Exception: # health: allow BLE001 -- unreadable identity files: nothing proven to adopt
- return None
- return int(pid) if pid and int(pid) not in exclude else None
- def _replay_env(home: str) -> dict:
- """The environment a replayed gateway starts with, as the restart watcher it replaces chose it
- (``_spawn_gateway_restart_watcher``): a named profile's gateway, replayed by an update launched
- under that same profile, inherits the launcher's environment (an exported bot token may be its
- only credential). The multiplexing root, and any home other than the launcher's, get that home's
- own secrets and none of the launcher's (``served_profile_child_env``)."""
- from hermes_constants import get_default_hermes_root, get_routing_process_hermes_home
- from tools.environments.local import build_subprocess_env, served_profile_child_env
- target = Path(home).resolve()
- if target != get_default_hermes_root().resolve() and target == get_routing_process_hermes_home().resolve():
- return build_subprocess_env(scrub_secrets=False, inherit_profile_home=False, extra={"HERMES_HOME": home})
- return served_profile_child_env(target_home=home, inherit_credentials=True)
- def _relaunch_bare(entry: dict) -> int:
- """Replay a bare gateway from its recorded argv under its recorded home; the new PID once it
- is serving. A gateway already serving that home (an earlier attempt's) is adopted instead."""
- if (existing := _gateway_on_home(entry["home"], {int(entry["pid"])})) is not None:
- return existing
- logs = Path(entry["home"]) / "logs"
- logs.mkdir(parents=True, exist_ok=True)
- env = _replay_env(entry["home"])
- for leaked in ("INVOCATION_ID", "JOURNAL_STREAM", "HERMES_UPDATE_PAUSED", "_HERMES_GATEWAY"):
- env.pop(leaked, None)
- with open(logs / "gateway-update-resume.log", "ab") as log:
- proc = subprocess.Popen(list(entry["argv"]), env=env, stdin=subprocess.DEVNULL, stdout=log,
- # detached-spawn: ok — resumes the paused gateway after an update; it must outlive the updater
- stderr=subprocess.STDOUT, start_new_session=True, close_fds=True)
- deadline = time.monotonic() + _READY_TIMEOUT_S
- while time.monotonic() < deadline:
- if (pid := _gateway_on_home(entry["home"], {int(entry["pid"])})) is not None:
- return pid # the replay, or a gateway someone started meanwhile (the replay then exits 0)
- if proc.poll() is not None:
- raise RuntimeError(f"replayed gateway for {entry['home']} exited {proc.returncode} "
- f"(see {logs / 'gateway-update-resume.log'})")
- time.sleep(0.5)
- raise RuntimeError(f"replayed gateway for {entry['home']} did not come up within {int(_READY_TIMEOUT_S)}s")
- def _describe(token: dict) -> str:
- names = [u.get("unit") or u.get("label") for u in token.get("posix_units") or []]
- names += [f"gateway PID {u['pid']}" for u in token.get("unmapped") or [] if u.get("argv")]
- return ", ".join(names)
- # --------------------------------------------------------------------------- the commit point
- def pause_at_commit_point() -> str | None:
- """Stop this install's gateways before the first checkout move; once per run.
- Never refuses the update: the pause is protection, not a gate. When it cannot complete
- (discovery or the durable record fails, a stop fails), the gateways run through the swap and
- the post-commit restart refreshes them, which is how POSIX updated before the pause existed.
- Always ``None``; the caller's refusal branch stays for the contract it shares with Windows."""
- token = _RUN
- if token is None:
- return None
- if "posix_pause_verdict" not in token:
- token["posix_pause_verdict"] = _pause(token)
- return token["posix_pause_verdict"]
- def _pause(token: dict) -> str | None:
- from hermes_cli import update_pause_record as pause_record
- try:
- units, jobs = _discover_systemd(), _discover_launchd()
- units, jobs, notices = _left_running(units, jobs)
- bare, more = _discover_bare({u["pid"] for u in units} | {j["pid"] for j in jobs})
- except Exception as exc: # health: allow BLE001 -- a discovery that cannot finish stops nothing
- print(f" ⚠ Could not list the running gateways ({exc}); they keep running and are restarted after the update")
- return None
- for line in notices + more:
- print(line)
- supervised = units + jobs
- if not supervised and not bare:
- return None
- from gateway.status import get_process_start_time
- from hermes_cli.update_cmd_windows import _planned_stop_marker_path
- before = copy.deepcopy(token) # what this attempt may add; a failed record puts it back
- for entry in supervised + bare:
- entry["ct"] = get_process_start_time(entry["pid"]) # native birth stamp: what _alive compares
- token["posix_units"] = [*token.get("posix_units", []), *supervised]
- token["unmapped"] = [*token.get("unmapped", []), *bare]
- token["unmapped_pids"] = [*token.get("unmapped_pids", []), *(e["pid"] for e in bare)]
- # Recovery judges liveness by the canonical identity (``ct:<unix seconds>``), never the native stamp.
- token["identities"] = {**token.get("identities", {}),
- **{str(e["pid"]): pause_record.identity(e["pid"])["ct"] for e in supervised + bare}}
- token["resume_needed"] = True
- try:
- pause_record.record_pause(token, None, [])
- pause_record.mark_stop_requested(
- token, [e["pid"] for e in supervised + bare],
- markers={e["pid"]: _planned_stop_marker_path(Path(e["home"])) for e in supervised + bare if e.get("home")})
- except Exception as exc: # health: allow BLE001 -- no durable record, no stop
- # Nothing was stopped: this attempt's id must not arm the checkout's tree gate, whose
- # required record write would then refuse the update. An adopted obligation stays as it was.
- if token.get("pause_id") != before.get("pause_id"):
- with suppress(Exception):
- pause_record.discharge(token)
- token.clear()
- token.update(before)
- print(f" ⚠ Could not record the gateways to pause ({exc}); they keep running and are restarted after the update")
- return None
- # From here a refusal or a crash leaves the set owed: the command's own exit resume (registered
- # in ``_cmd_update_impl`` for this token), else the next launch's recovery, restarts it.
- token["posix_stopped"] = True
- try:
- _stop_gateways(token, supervised + bare)
- except Exception as exc: # health: allow BLE001 -- roll back: restart the stopped set, update unpaused
- print(f" ⚠ Could not pause every gateway ({exc}); restarting them, the update continues without the pause")
- try:
- resume_paused_set(token)
- except Exception as again: # health: allow BLE001 -- still owed: completion restarts them after the deps
- print(f" ⚠ {again}; they restart once the dependencies are synced")
- for restarted in _RESTARTED.values():
- restarted.clear() # back on the OLD code: the post-commit fleet restart must refresh them
- pause_record.sync(token)
- return None
- pause_record.sync(token)
- print(f" ⏸ Paused for the update: {_describe(token)} (restarted once the dependencies are synced)")
- return None
- # --------------------------------------------------------------------------- resume
- def resume_paused_set(token: dict) -> None:
- """Start every paused runtime again; each leaves *token* only on its own verified start."""
- failures: list[str] = []
- restarted: list[str] = []
- kept_units = []
- for entry in token.get("posix_units") or []:
- try:
- if entry.get("kind") == "launchd":
- _start_job(entry)
- _RESTARTED["labels"].add(entry["label"])
- name = entry["label"]
- else:
- _RESTARTED["pids"].add(_start_unit(entry))
- _RESTARTED["units"].add(f"{entry['scope']}/{entry['unit']}")
- name = entry["unit"]
- restarted.append(name)
- token.setdefault("restarted_services", []).append(name)
- except Exception as exc: # health: allow BLE001 -- one failed unit never keeps the others stopped
- failures.append(str(exc))
- kept_units.append(entry)
- token["posix_units"] = kept_units
- kept_bare = []
- for entry in token.get("unmapped") or []:
- if not entry.get("argv"):
- continue
- try:
- new_pid = _relaunch_bare(entry)
- _RESTARTED["pids"].add(new_pid)
- restarted.append(f"gateway PID {entry['pid']} → {new_pid}")
- except Exception as exc: # health: allow BLE001 -- one failed replay never keeps the others stopped
- failures.append(str(exc))
- kept_bare.append(entry)
- token["unmapped"] = kept_bare
- token["unmapped_pids"] = [e["pid"] for e in kept_bare]
- token["profiles"] = {}
- if restarted:
- print(f" ▶ Restarted paused gateway(s): {', '.join(restarted)}")
- if failures:
- raise RuntimeError("Could not restart every paused gateway: " + "; ".join(failures))
- token["resume_needed"] = False
- token["posix_stopped"] = False
- def already_restarted() -> dict:
- """What the early resume restarted on the new code in this process (for the fleet restart)."""
- return _RESTARTED
|