restart.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371
  1. """Shared gateway restart constants and supervisor detection helpers."""
  2. import math
  3. import os
  4. import re
  5. import subprocess
  6. import sys
  7. from collections.abc import Callable, Mapping
  8. from hermes_cli.config import DEFAULT_CONFIG
  9. # EX_TEMPFAIL (sysexits.h): ask the service manager to restart after a graceful drain/reload.
  10. GATEWAY_SERVICE_RESTART_EXIT_CODE = 75
  11. # EX_CONFIG (sysexits.h): fatal configuration error (token collision, no platforms);
  12. # the s6 finish script maps it to exit 125 so the supervisor stops restarting.
  13. # See #51228.
  14. GATEWAY_FATAL_CONFIG_EXIT_CODE = 78
  15. def map_fatal_config_exit_for_launchd(returncode: int) -> int:
  16. """Translate gateway EX_CONFIG so launchd can park the job.
  17. systemd uses ``RestartPreventExitStatus=78``; s6 maps 78→125. launchd cannot
  18. gate on a specific status, and unconditional ``KeepAlive=true`` respawns 78
  19. forever (#89477). The generated plist uses ``KeepAlive.SuccessfulExit=false``;
  20. mapping 78→0 is a deliberate stop. Exit 75 (please restart) and other
  21. failures pass through so KeepAlive still relaunches them. Negative ``wait()``
  22. codes (signals) are left to the caller.
  23. """
  24. if returncode == GATEWAY_FATAL_CONFIG_EXIT_CODE:
  25. return 0
  26. return returncode
  27. # Set by ``hermes gateway run --external-supervisor``. Unlike systemd's INVOCATION_ID
  28. # and launchd's XPC_SERVICE_NAME, this survives wrappers that replace the child
  29. # environment (e.g. ``sudo env -i``).
  30. EXTERNAL_GATEWAY_SUPERVISOR_ENV = "HERMES_GATEWAY_EXTERNAL_SUPERVISOR"
  31. # Forwarded by the stderr-timestamp launchd wrapper (hermes_cli/stderr_timestamp.py) to the gateway
  32. # grandchild, which sees ``XPC_SERVICE_NAME=0``. Read only via :func:`launchd_job_label`.
  33. LAUNCHD_LABEL_ENV = "HERMES_LAUNCHD_LABEL"
  34. DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["restart_drain_timeout"])
  35. DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT = float(DEFAULT_CONFIG["gateway"]["signal_interrupt_grace_timeout"])
  36. DEFAULT_GATEWAY_POST_INTERRUPT_GRACE_TIMEOUT = 5.0
  37. # In-band restart waits for active turns to finish *before* ``stop()`` begins; distinct from
  38. # ``restart_drain_timeout``, the force-interrupt budget once ``stop()`` runs (short under TimeoutStopSec).
  39. DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["restart_after_turn_timeout"])
  40. # Floor under the ``stop()`` drain for cron jobs and api_server (/v1) runs. ``restart_drain_timeout``
  41. # defaults to 0 because interrupting a *chat* turn is cheap and recoverable (user told, session
  42. # resume_pending); an interrupted *cron* run is a permanent failure in jobs.json and an interrupted
  43. # /v1 run fails its waiting caller — a 0s drain silently destroys work.
  44. DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT = float(DEFAULT_CONFIG["agent"]["cron_drain_timeout"])
  45. # Watchdog leash held back for post-drain work (interrupt agents, kill subprocesses, mark jobs,
  46. # disconnect adapters). Waiting for cron past that trades a job killed *and recorded* for one
  47. # SIGKILLed mid-write and wedged at ``last_status=running`` forever.
  48. CRON_DRAIN_CLEANUP_RESERVE_S = 10.0
  49. # systemd TimeoutStopSec headroom after the stop-path drain budget, and the floor when that
  50. # budget is still the default immediate (0s) chat drain. Keep in lockstep with generate_systemd_unit().
  51. # See #94759.
  52. SYSTEMD_STOP_HEADROOM_S = 30.0
  53. SYSTEMD_TIMEOUT_STOP_SEC_FLOOR = 60.0
  54. # launchd is the one supervisor whose stop budget the gateway cannot size:
  55. # ``ExitTimeOut`` lives in the plist, and the per-user (gui) domain CLAMPS
  56. # it — measured on macOS 26.6.1: plist 215 -> live 60, 90 -> 60, 60 -> 60,
  57. # 30 -> 30. The gateway can only *read* the live value (``launchctl print
  58. # gui/<uid>/<label>`` -> ``exit timeout = N``) and fit its SIGTERM-driven
  59. # stop inside it. Draining past it is not "a longer drain" — launchd
  60. # SIGKILLs at N seconds, mid-SQLite-write on a busy restart, which is the
  61. # unclean-exit half of the state.db corruption class.
  62. LAUNCHD_GUI_EXIT_TIMEOUT_CLAMP_S = 60
  63. LAUNCHD_STOP_CLEANUP_RESERVE_S = 10.0
  64. # How far before launchd's SIGKILL the thread watchdog must fire so its
  65. # faulthandler dump + os._exit actually land (the dump is sub-second; the
  66. # margin covers a slow disk).
  67. LAUNCHD_WATCHDOG_DUMP_MARGIN_S = 2.0
  68. _LAUNCHD_EXIT_TIMEOUT_RE = re.compile(r"^\s*exit timeout\s*=\s*(\d+)\s*$", re.MULTILINE)
  69. def parse_launchd_exit_timeout(print_output: object) -> float | None:
  70. """Extract ``exit timeout = N`` from ``launchctl print`` output."""
  71. match = _LAUNCHD_EXIT_TIMEOUT_RE.search(str(print_output or ""))
  72. if match is None:
  73. return None
  74. return float(match.group(1))
  75. def launchd_job_label(environ: Mapping[str, str] | None = None) -> str | None:
  76. """The ``ai.hermes.*`` launchd job label in *environ*, or ``None`` (no darwin gate).
  77. launchd stamps ``XPC_SERVICE_NAME`` only on the job process it spawns. The generated plist
  78. runs the stderr-timestamp wrapper there, so the gateway grandchild reads ``XPC_SERVICE_NAME=0``
  79. and finds its label only in the wrapper's re-export, ``HERMES_LAUNCHD_LABEL``. Both go through
  80. the same ``ai.hermes`` predicate: app-coalition labels (``application.<bundle>…``, exported
  81. into IDE integrated terminals) are not our job. ONE seam for every reader of launchd identity
  82. (drain cap, restart route, control-socket supervisor declaration).
  83. """
  84. env = os.environ if environ is None else environ
  85. for var in ("XPC_SERVICE_NAME", LAUNCHD_LABEL_ENV):
  86. label = str(env.get(var, "") or "").strip()
  87. if label.startswith("ai.hermes"):
  88. return label
  89. return None
  90. def launchd_service_label(environ: Mapping[str, str] | None = None, *, platform: str = sys.platform) -> str | None:
  91. """Return this process's ``ai.hermes.*`` launchd job label, or ``None`` off darwin.
  92. ``launchctl print`` reports ``exit timeout = 1`` for app-coalition labels — treating those as
  93. a budget would cap the drain to 0 for a gateway Ctrl+C'd in an IDE terminal, hence the
  94. predicate in :func:`launchd_job_label`. ``platform`` is data so the mapping logic is testable
  95. on any host.
  96. """
  97. if platform != "darwin":
  98. return None
  99. return launchd_job_label(environ)
  100. def read_launchd_exit_timeout_s(
  101. label: str | None = None,
  102. *,
  103. environ: Mapping[str, str] | None = None,
  104. uid: int | None = None,
  105. run: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
  106. platform: str = sys.platform,
  107. ) -> float | None:
  108. """Live ``ExitTimeOut`` (seconds) launchd enforces for this gateway's job.
  109. Returns ``None`` — meaning "no launchd budget applies" — when the process
  110. is not launchd-owned (non-darwin, or no ``ai.hermes`` job label — see :func:`launchd_job_label`), ``launchctl`` is missing
  111. or fails, or the print output carries no ``exit timeout`` line. Callers
  112. must treat ``None`` as fail-open: the configured drain stands unchanged.
  113. """
  114. label = label or launchd_service_label(environ, platform=platform)
  115. if not label:
  116. return None
  117. if uid is None:
  118. getuid = getattr(os, "getuid", None) # absent on Windows; label is None there anyway
  119. if getuid is None:
  120. return None
  121. uid = getuid()
  122. domain = "system" if uid == 0 else f"gui/{uid}"
  123. try:
  124. proc = run(
  125. ["launchctl", "print", f"{domain}/{label}"],
  126. capture_output=True,
  127. text=True,
  128. encoding="utf-8",
  129. errors="replace",
  130. timeout=5,
  131. )
  132. except (OSError, subprocess.SubprocessError, ValueError):
  133. return None
  134. if proc.returncode != 0:
  135. return None
  136. return parse_launchd_exit_timeout(proc.stdout)
  137. def resolve_launchd_capped_drain(
  138. drain_timeout: float,
  139. launchd_exit_timeout_s: float | None,
  140. *,
  141. cleanup_reserve_s: float = LAUNCHD_STOP_CLEANUP_RESERVE_S,
  142. ) -> float:
  143. """Clamp a SIGTERM-driven stop drain to what launchd will actually allow.
  144. ``launchd_exit_timeout_s`` is the live ``exit timeout`` for this job (see
  145. :func:`read_launchd_exit_timeout_s`); ``None`` means no launchd budget
  146. applies and the configured drain is returned untouched. Otherwise the
  147. drain may use at most ``exit_timeout - cleanup_reserve_s`` so the
  148. post-drain teardown (interrupt agents, disconnect adapters, checkpoint
  149. and close SQLite) still completes before launchd escalates to SIGKILL.
  150. Never *extends* the drain — an operator who configured a short one
  151. keeps it.
  152. """
  153. drain = _seconds(drain_timeout)
  154. budget = _seconds(launchd_exit_timeout_s) # None / non-numeric → 0.0 → no budget applies
  155. if budget <= 0.0:
  156. return drain
  157. return min(drain, max(budget - _seconds(cleanup_reserve_s), 0.0))
  158. def effective_stop_drain_timeout(runner: object) -> float:
  159. """Drain budget for the stop in progress on ``runner``.
  160. Signal-driven stops under launchd are timed by launchd's live
  161. ``ExitTimeOut`` (``runner._launchd_exit_timeout_s``, set at boot);
  162. everything else — in-band SIGUSR1 restart after the turn, ``--replace``
  163. takeover, tests — keeps the configured drain. Duck-typed and
  164. getattr-guarded on purpose: shutdown-path tests drive ``_stop_impl``
  165. from bare doubles that are not ``GatewayRunner`` instances.
  166. """
  167. drain = getattr(runner, "_restart_drain_timeout", DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT)
  168. if not getattr(runner, "_stop_requested_by_signal", False):
  169. return drain
  170. return resolve_launchd_capped_drain(drain, getattr(runner, "_launchd_exit_timeout_s", None))
  171. def effective_stop_watchdog_delay(runner: object, watchdog_delay: float) -> float:
  172. """Thread-watchdog leash for the stop in progress on ``runner``.
  173. ``watchdog_delay`` is the supervisor-agnostic leash (effective drain +
  174. grace). Under launchd a signal-driven stop is SIGKILLed at the live
  175. ``ExitTimeOut`` — with a 60s budget the default leash (50 + 60 = 110s)
  176. can never fire, so the forensic stack dump and ``os._exit`` the watchdog
  177. exists for are lost. Clamp the leash to ``ExitTimeOut -
  178. LAUNCHD_WATCHDOG_DUMP_MARGIN_S`` so the dump lands before SIGKILL. Same
  179. duck-typing / fail-open rules as :func:`effective_stop_drain_timeout`.
  180. """
  181. leash = _seconds(watchdog_delay)
  182. if not getattr(runner, "_stop_requested_by_signal", False):
  183. return leash
  184. return resolve_launchd_capped_drain(
  185. leash, getattr(runner, "_launchd_exit_timeout_s", None), cleanup_reserve_s=LAUNCHD_WATCHDOG_DUMP_MARGIN_S,
  186. )
  187. _TRUTHY = {"1", "true", "yes", "on"}
  188. def is_global_startup_conflict(error_code: str | None) -> bool:
  189. """True when an adapter's fatal error is a single-writer ownership conflict.
  190. Adapters emit ``{scope}_lock`` with ``retryable=True`` so a *mid-run* reconnect can
  191. recover; at startup a live foreign holder is a configuration conflict (two gateways
  192. cannot poll one token), not a transient blip. Matches by error CODE only, never text.
  193. ``BasePlatformAdapter._acquire_platform_lock`` emits ``{scope}_lock`` with ``retryable=True`` on
  194. purpose: a *mid-run* reconnect must be able to recover once the live holder exits or a stale record is
  195. cleared (#54167). This matches by error CODE only (the ``{scope}_lock`` / ``lock_conflict`` families
  196. every adapter emits for scoped-lock and identity conflicts), never by message text.
  197. """
  198. code = (error_code or "").strip().lower()
  199. return bool(code) and (code == "lock_conflict" or code.endswith("_lock"))
  200. def is_gateway_supervisor_process(environ: Mapping[str, str] | None = None) -> bool:
  201. """Return whether this gateway process is owned by a supervisor that RESTARTS it.
  202. Selects the exit-75 restart route, so only markers of a manager with a restart policy count:
  203. systemd ``INVOCATION_ID``, launchd ``XPC_SERVICE_NAME`` (or the wrapper-forwarded
  204. ``HERMES_LAUNCHD_LABEL`` the grandchild sees), the s6 sentinel, or the explicit
  205. ``--external-supervisor`` opt-in. The generalized ``HERMES_SUPERVISED_CHILD`` launcher marker is
  206. deliberately NOT read here: the Windows Scheduled-Task launcher sets it without a restart policy
  207. (#113670), and routing its ``/restart`` through exit 75 would leave the gateway dead.
  208. """
  209. env = os.environ if environ is None else environ
  210. xpc_service = env.get("XPC_SERVICE_NAME", "")
  211. return bool(env.get("INVOCATION_ID") or env.get("HERMES_S6_SUPERVISED_CHILD") or (xpc_service and xpc_service != "0")
  212. or launchd_job_label(env)
  213. or str(env.get(EXTERNAL_GATEWAY_SUPERVISOR_ENV, "")).strip().lower() in _TRUTHY)
  214. def is_supervised_gateway_launch(environ: Mapping[str, str] | None = None) -> bool:
  215. """Return whether this gateway was launched by a generated service/launcher rather than a shell.
  216. Superset of :func:`is_gateway_supervisor_process` that also honours ``HERMES_SUPERVISED_CHILD``,
  217. the marker every generated launcher exports (systemd unit, launchd plist, s6 run script, Windows
  218. Scheduled Task — see ``hermes_cli.main._apply_profile_override``). This is the identity the
  219. self-targeting guards key on: a kill or lifecycle command issued from inside such a gateway takes
  220. down the process hosting the caller with nobody at a terminal to bring it back (#113667).
  221. """
  222. env = os.environ if environ is None else environ
  223. if env.get("HERMES_SUPERVISED_CHILD"):
  224. return True
  225. return is_gateway_supervisor_process(environ)
  226. def is_container_restart_context() -> bool:
  227. """In a container (Docker/Podman) the detached setsid restart path dies with the cgroup,
  228. so exit-75 service restart is the only viable path. Own function so tests can mock it."""
  229. return os.path.exists("/.dockerenv") or os.path.exists("/run/.containerenv")
  230. def _seconds(value: object, fallback: float = 0.0) -> float:
  231. """Non-negative float, or ``fallback`` on non-numeric input."""
  232. try:
  233. return max(float(value), 0.0) # type: ignore[arg-type]
  234. except (TypeError, ValueError):
  235. return fallback
  236. def _parse_timeout_keeping_zero(raw: object, default: float, *, finite: bool = False) -> float:
  237. """Parse a timeout where ``0`` is a deliberate disable (must NOT fall through
  238. to ``default``), unlike None / blank / non-numeric (/ non-finite) input."""
  239. if raw is None or (isinstance(raw, str) and not raw.strip()):
  240. return default
  241. try:
  242. value = float(raw) # type: ignore[arg-type]
  243. except (TypeError, ValueError):
  244. return default
  245. return default if finite and not math.isfinite(value) else max(0.0, value)
  246. def parse_restart_drain_timeout(raw: object) -> float:
  247. """Parse a configured drain timeout; falsy (incl. ``0``) falls back to the shared default."""
  248. return _parse_timeout_keeping_zero(raw or None, DEFAULT_GATEWAY_RESTART_DRAIN_TIMEOUT)
  249. def parse_restart_after_turn_timeout(raw: object) -> float:
  250. """Parse the after-turn wait cap for in-band restart (``0`` = legacy immediate drain)."""
  251. return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_RESTART_AFTER_TURN_TIMEOUT)
  252. def parse_cron_drain_timeout(raw: object) -> float:
  253. """Parse the cron/api_server drain floor (``0`` = opt out; both interrupted on the chat budget).
  254. ``0`` is a deliberate opt-out — cron and api_server (/v1) work is then interrupted on the same budget as chat work, the
  255. pre-#82161 behaviour — and must not fall through to the default, unlike empty/missing input.
  256. """
  257. return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT)
  258. def parse_signal_interrupt_grace_timeout(raw: object) -> float:
  259. """Parse the unexpected-signal post-interrupt grace timeout."""
  260. return _parse_timeout_keeping_zero(raw, DEFAULT_GATEWAY_SIGNAL_INTERRUPT_GRACE_TIMEOUT, finite=True)
  261. def resolve_cron_drain_budget(
  262. drain_timeout: float, cron_drain_timeout: float, *, watchdog_delay: float, elapsed: float = 0.0,
  263. cleanup_reserve_s: float = CRON_DRAIN_CLEANUP_RESERVE_S,
  264. ) -> float:
  265. """Seconds the stop drain may wait on in-flight cron and api_server (/v1) work.
  266. Clamped to what this process can honour: the watchdog hard-exits at ``watchdog_delay``,
  267. so waiting past that leash minus ``cleanup_reserve_s`` swaps a cleanly-interrupted job
  268. for a SIGKILL that leaves it wedged. Never less than ``drain_timeout`` (only extends).
  269. """
  270. drain = _seconds(drain_timeout)
  271. floor = _seconds(cron_drain_timeout)
  272. if floor <= 0.0:
  273. return drain
  274. ceiling = _seconds(watchdog_delay) - _seconds(elapsed) - _seconds(cleanup_reserve_s, CRON_DRAIN_CLEANUP_RESERVE_S)
  275. return max(drain, min(floor, ceiling))
  276. def resolve_systemd_timeout_stop_sec(
  277. drain_timeout: float, cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, *,
  278. cleanup_reserve_s: float = CRON_DRAIN_CLEANUP_RESERVE_S, headroom_s: float = SYSTEMD_STOP_HEADROOM_S,
  279. floor_s: float = SYSTEMD_TIMEOUT_STOP_SEC_FLOOR,
  280. ) -> int:
  281. """Seconds systemd ``TimeoutStopSec`` must cover: the stop path may first wait
  282. ``cron_drain_timeout`` + ``cleanup_reserve_s`` for cron and api_server work, so sizing from the chat drain
  283. alone lets systemd SIGKILL an in-budget drain. A zero cron timeout is an opt-out.
  284. ``restart_drain_timeout`` is only the chat-turn interrupt budget (default 0). See #94759.
  285. """
  286. drain = _seconds(drain_timeout)
  287. cron = _seconds(cron_drain_timeout)
  288. cron_budget = (cron + _seconds(cleanup_reserve_s)) if cron > 0.0 else 0.0
  289. return int(max(_seconds(floor_s), max(drain, cron_budget) + _seconds(headroom_s)))
  290. def resolve_restart_exit_wait_budget(
  291. drain_timeout: float, after_turn_timeout: float, cron_drain_timeout: float, *, headroom: float = 15.0,
  292. ) -> float:
  293. """Seconds a CLI waits for the gateway PID to exit after SIGUSR1: the in-band after-turn
  294. deferral, then the longest supervisor stop envelope — the one systemd's ``TimeoutStopSec`` is
  295. sized from (cron drain + cleanup reserve, supervisor headroom, floor; #94759) — then
  296. observer ``headroom``.
  297. A non-finite drain means "wait indefinitely", not a crash in the integer envelope.
  298. """
  299. if not all(math.isfinite(_seconds(value)) for value in (drain_timeout, cron_drain_timeout)):
  300. return math.inf
  301. return _seconds(after_turn_timeout) + resolve_systemd_timeout_stop_sec(drain_timeout, cron_drain_timeout) + _seconds(headroom)