shutdown_watchdog.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403
  1. """Out-of-loop shutdown and event-loop liveness backstops.
  2. A frozen asyncio loop takes every asyncio-based recovery path down with it, and launchd/systemd
  3. KeepAlive only restarts a *dead* process. Hence: (1) an OS-thread shutdown watchdog that dumps
  4. stacks and ``os._exit``s past ``restart_drain_timeout + grace``; (2) a heartbeat file at
  5. ``<HERMES_HOME>/state/gateway.heartbeat`` so supervisors can tell "process alive" from "loop
  6. frozen"; (3) a lifetime thread watchdog that hard-exits when the loop is too frozen to run its
  7. own callbacks; (4) a self-rescheduling floor timer that keeps the selector timeout finite."""
  8. from __future__ import annotations
  9. import asyncio
  10. import contextlib
  11. import faulthandler
  12. import json
  13. import logging
  14. import os
  15. import sys
  16. import threading
  17. import time
  18. from datetime import datetime, timezone, UTC
  19. from pathlib import Path
  20. from typing import Any, Callable, Dict, Optional
  21. from gateway.restart import GATEWAY_SERVICE_RESTART_EXIT_CODE
  22. from hermes_constants import get_hermes_home, get_process_hermes_home
  23. from utils import atomic_json_write
  24. logger = logging.getLogger(__name__)
  25. # Extra leash beyond ``agent.restart_drain_timeout`` so a slow-but-progressing drain survives.
  26. # Matches the issue #66892 suggested hardening.
  27. DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S = 60.0
  28. DEFAULT_HEARTBEAT_INTERVAL_S = 30.0
  29. DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S = 5.0
  30. DEFAULT_LOOP_WATCHDOG_INTERVAL_S = 30.0
  31. DEFAULT_LOOP_WATCHDOG_TIMEOUT_S = 10.0
  32. # 3 sustained misses (~90-120s of loop block) escalate; stays tight because the heartbeat write is
  33. # off-loop. Slow loops tune gateway.loop_watchdog_* in config.yaml.
  34. # The false-positive class that motivated raising this (the watchdog's own on-loop heartbeat fsync stalling
  35. # the loop it monitors) is fixed at the root by the off-loop heartbeat write + two-witness probe (#90502),
  36. # so the default stays tight for genuine wedges.
  37. DEFAULT_LOOP_WATCHDOG_MAX_STRIKES = 3
  38. _HEARTBEAT_RELATIVE = ("state", "gateway.heartbeat")
  39. _WATCHDOG_DUMP_RELATIVE = ("logs", "gateway-shutdown-watchdog.log")
  40. def _coerce_float(value: Any, default: float, floor: float = 0.0) -> float:
  41. """``max(float(value), floor)``, or ``default`` when not coercible."""
  42. try:
  43. return max(float(value), floor)
  44. except (TypeError, ValueError):
  45. return default
  46. class _LoopFloorTimerHandle:
  47. """Cancelable owner for the currently scheduled selector floor timer."""
  48. def __init__(self, loop: asyncio.AbstractEventLoop, interval: float):
  49. self._loop, self._interval, self._cancelled = loop, interval, False
  50. self._timer: Optional[asyncio.TimerHandle] = None
  51. self._tick()
  52. def _tick(self) -> None:
  53. if not self._cancelled:
  54. self._timer = self._loop.call_later(self._interval, self._tick)
  55. def cancel(self) -> None:
  56. self._cancelled = True
  57. if self._timer is not None:
  58. self._timer.cancel()
  59. class _LoopLivenessWatchdogHandle:
  60. def __init__(self, stop_event: threading.Event, thread: threading.Thread):
  61. self._stop_event = stop_event
  62. self.stop, self.join, self.is_alive = stop_event.set, thread.join, thread.is_alive
  63. def _arm_loop_floor_timer(
  64. loop: asyncio.AbstractEventLoop, interval: float = DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S
  65. ) -> _LoopFloorTimerHandle:
  66. """Keep at least one timer pending so selector waits remain bounded."""
  67. iv = _coerce_float(interval, 0.0)
  68. return _LoopFloorTimerHandle(loop, iv if iv > 0 else DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S)
  69. def start_loop_liveness_watchdog(
  70. loop: asyncio.AbstractEventLoop, *, probe_interval: float = DEFAULT_LOOP_WATCHDOG_INTERVAL_S,
  71. probe_timeout: float = DEFAULT_LOOP_WATCHDOG_TIMEOUT_S,
  72. max_strikes: int = DEFAULT_LOOP_WATCHDOG_MAX_STRIKES,
  73. exit_code: int = GATEWAY_SERVICE_RESTART_EXIT_CODE,
  74. ) -> Optional[_LoopLivenessWatchdogHandle]:
  75. """Start an out-of-loop watchdog that hard-exits after missed probes. The caller
  76. (``GatewayRunner._start_loop_liveness_guards``) enforces the ``gateway.loop_watchdog: false``
  77. opt-out."""
  78. stop_event = threading.Event()
  79. def _watchdog() -> None:
  80. strikes = 0
  81. while not stop_event.wait(timeout=probe_interval):
  82. probe_event = threading.Event()
  83. try:
  84. loop.call_soon_threadsafe(probe_event.set)
  85. except RuntimeError: # normally closed loop: nothing left to backstop
  86. return
  87. except Exception:
  88. logger.debug("Failed to schedule gateway loop liveness probe", exc_info=True)
  89. return
  90. deadline = time.monotonic() + probe_timeout
  91. while not stop_event.is_set(): # poll so a stop() mid-wait is honoured within ~50ms
  92. remaining = deadline - time.monotonic()
  93. if remaining <= 0 or probe_event.wait(timeout=min(remaining, 0.05)):
  94. break
  95. else:
  96. return
  97. if probe_event.is_set():
  98. strikes = 0
  99. continue
  100. if stop_event.is_set(): # re-checked before each irreversible step: a late stop() wins
  101. return
  102. strikes += 1
  103. if strikes < max_strikes:
  104. continue
  105. if stop_event.is_set():
  106. return
  107. with contextlib.suppress(Exception):
  108. logger.critical(
  109. "Gateway event loop missed %d consecutive liveness probes; dumping all thread "
  110. "stacks and exiting with code %d so the service supervisor can restart it.",
  111. strikes, exit_code)
  112. # File + stderr: a console-less launcher (Windows .vbs login item) discards stderr, so a
  113. # liveness death used to leave no stack anywhere (#120356, #91097).
  114. _write_watchdog_dump(
  115. get_shutdown_watchdog_dump_path(), delay_s=probe_timeout * strikes,
  116. snapshot={"strikes": strikes, "probe_interval_s": probe_interval},
  117. event="loop_liveness_watchdog_fired")
  118. if stop_event.is_set():
  119. return
  120. _mark_exited_quietly(exit_code, "loop_liveness_watchdog")
  121. _hard_exit(exit_code)
  122. thread = threading.Thread(target=_watchdog, daemon=True, name="gateway-loop-liveness-watchdog")
  123. try:
  124. thread.start()
  125. except Exception:
  126. logger.debug("Failed to start gateway loop liveness watchdog", exc_info=True)
  127. return None
  128. return _LoopLivenessWatchdogHandle(stop_event, thread)
  129. def _hard_exit(exit_code: int) -> None:
  130. """``os._exit`` skips every cleanup: SIGKILL in-flight foreground commands first, they run in their
  131. own process group and would outlive the gateway, reparented to init."""
  132. with contextlib.suppress(Exception):
  133. from tools.environments.base import kill_live_foreground_processes
  134. kill_live_foreground_processes(now=True)
  135. os._exit(exit_code)
  136. def _mark_exited_quietly(exit_code: int, reason: str) -> None:
  137. """Best-effort terminal stamp on BOTH lifecycle records before ``os._exit`` skips teardown:
  138. the lifecycle ledger (so the next boot names the watchdog, not SIGKILL/OOM) and
  139. ``gateway_state.json`` (so ``hermes gateway status`` and every other reader of that file stop
  140. seeing ``running`` for a process the watchdog killed — #113372). The runtime-status write goes
  141. LAST: it is the record housekeeping refreshes, so nothing may overwrite it after we stamp it."""
  142. with contextlib.suppress(Exception):
  143. from gateway.lifecycle_ledger import mark_exited
  144. mark_exited(exit_code, reason=reason)
  145. with contextlib.suppress(Exception): # os._exit skips atexit: stamp the exit-metrics marker now
  146. from hermes_cli.observability.shared_metrics_process import stamp_exit
  147. stamp_exit("watchdog")
  148. with contextlib.suppress(Exception):
  149. from gateway.status import write_runtime_status
  150. # Only the supervisor-restart code asserts a restart; other codes leave the recorded
  151. # operator intent (a restart-drain that wedged is still a requested restart) untouched.
  152. restart = {"restart_requested": True} if exit_code == GATEWAY_SERVICE_RESTART_EXIT_CODE else {}
  153. write_runtime_status(
  154. gateway_state="degraded", exit_reason=reason, wait_timeout=0.25, **restart)
  155. def _process_hermes_home() -> Path:
  156. """HERMES_HOME for process-level identity files (ignore profile overrides)."""
  157. return get_process_hermes_home() if os.environ.get("HERMES_HOME", "").strip() else get_hermes_home()
  158. def _home(home: Optional[Path]) -> Path:
  159. return home if home is not None else _process_hermes_home()
  160. def get_loop_heartbeat_path(home: Optional[Path] = None) -> Path:
  161. return _home(home).joinpath(*_HEARTBEAT_RELATIVE)
  162. def get_loop_tick_socket_path(home: Optional[Path] = None, pid: Optional[int] = None) -> Path:
  163. """``<HERMES_HOME>/state/gateway.loop-tick.<pid>.sock`` — PID-suffixed so a stale node from a
  164. dead process is never mistaken for this gateway's witness. Served by the loop itself
  165. (``_tick_socket_handler``), so an answer proves the loop dispatches; the heartbeat cannot.
  166. Served by the gateway loop itself (see ``_tick_socket_handler``): an answer is direct proof that the
  167. loop is dispatching, which is exactly the property the heartbeat file lost when its write moved off-loop
  168. (#90502).
  169. """
  170. pid = int(pid if pid is not None else os.getpid())
  171. return _home(home) / "state" / f"gateway.loop-tick.{pid}.sock"
  172. def get_shutdown_watchdog_dump_path(home: Optional[Path] = None) -> Path:
  173. return _home(home).joinpath(*_WATCHDOG_DUMP_RELATIVE)
  174. def write_loop_heartbeat(
  175. *, pid: Optional[int] = None, start_time: Optional[float] = None,
  176. home: Optional[Path] = None, extra: Optional[dict[str, Any]] = None) -> Path:
  177. """Atomically rewrite the loop-liveness heartbeat file; never raises.
  178. ``start_time`` (process start, epoch seconds) lets supervisors detect PID reuse."""
  179. path = get_loop_heartbeat_path(home)
  180. payload: dict[str, Any] = {"pid": int(pid if pid is not None else os.getpid()),
  181. "updated_at": datetime.now(UTC).isoformat(),
  182. "monotonic": time.monotonic()}
  183. if start_time is not None:
  184. payload["start_time"] = float(start_time)
  185. with contextlib.suppress(Exception): # after an unclean death this is the last memory record
  186. from gateway.lifecycle_ledger import sample_memory
  187. if mem := sample_memory():
  188. payload["mem"] = mem
  189. if extra:
  190. payload.update(extra)
  191. try:
  192. atomic_json_write(path, payload, indent=None)
  193. except Exception:
  194. logger.debug("Failed to write gateway loop heartbeat", exc_info=True)
  195. return path
  196. def resolve_shutdown_watchdog_delay(
  197. drain_timeout: float, *, grace_s: float = DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S) -> float:
  198. """Return the wall-clock leash for the shutdown watchdog thread."""
  199. grace = _coerce_float(grace_s, DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S)
  200. return _coerce_float(drain_timeout, 0.0) + grace
  201. def _write_watchdog_dump(dump_path: Path, *, delay_s: float, snapshot: Optional[dict[str, Any]],
  202. event: str = "shutdown_watchdog_fired") -> None:
  203. """Best-effort faulthandler + metadata dump before hard-exit (file, then stderr)."""
  204. header = {"event": event, "pid": os.getpid(), "delay_s": delay_s,
  205. "fired_at": datetime.now(UTC).isoformat(), "snapshot": snapshot or {}}
  206. with contextlib.suppress(OSError): # unwritable dir: skip only the file copy, stderr still runs
  207. dump_path.parent.mkdir(parents=True, exist_ok=True)
  208. with contextlib.suppress(Exception), open(dump_path, "a", encoding="utf-8") as fh:
  209. fh.write(json.dumps(header, default=str) + "\n--- faulthandler dump (all threads) ---\n")
  210. fh.flush()
  211. try:
  212. faulthandler.dump_traceback(file=fh, all_threads=True)
  213. except Exception:
  214. fh.write("(faulthandler.dump_traceback failed)\n")
  215. fh.write("--- end dump ---\n")
  216. fh.flush()
  217. with contextlib.suppress(Exception): # stderr too: journald/launchd get it if disk is wedged
  218. sys.stderr.write(f"Gateway {event.replace('_', ' ')} after {delay_s:.0f}s "
  219. f"(pid={os.getpid()}); dumping all thread stacks.\n")
  220. sys.stderr.flush()
  221. faulthandler.dump_traceback(all_threads=True)
  222. def arm_shutdown_watchdog(
  223. delay_s: float, *, done_event: Optional[threading.Event] = None,
  224. snapshot_fn: Optional[Callable[[], dict[str, Any]]] = None, exit_code: int = 1,
  225. dump_path: Optional[Path] = None, name: str = "gateway-shutdown-watchdog") -> threading.Event:
  226. """Arm a daemon-thread hard-exit backstop for a wedged shutdown path: exits quietly if
  227. ``done_event`` is set within ``delay_s``, else dumps diagnostics and ``os._exit(exit_code)``.
  228. Never raises; returns ``done_event`` for disarming."""
  229. done = done_event if done_event is not None else threading.Event()
  230. delay = _coerce_float(delay_s, DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S)
  231. if delay <= 0:
  232. return done
  233. def _watchdog() -> None:
  234. deadline = time.monotonic() + delay # chunked wait so a late disarm is observed within ~1s
  235. while time.monotonic() < deadline:
  236. if done.wait(timeout=min(deadline - time.monotonic(), 1.0)):
  237. return
  238. if done.is_set():
  239. return
  240. try:
  241. snapshot = snapshot_fn() if snapshot_fn is not None else None
  242. except Exception as exc:
  243. snapshot = {"snapshot_error": repr(exc)}
  244. target = dump_path if dump_path is not None else get_shutdown_watchdog_dump_path()
  245. _write_watchdog_dump(target, delay_s=delay, snapshot=snapshot)
  246. with contextlib.suppress(Exception):
  247. logger.critical("Shutdown watchdog fired after %.0fs — forcing process exit "
  248. "(asyncio drain path appears wedged; see %s)", delay, target)
  249. for stream in (sys.stdout, sys.stderr):
  250. with contextlib.suppress(Exception):
  251. stream.flush()
  252. # Mirror _exit_after_graceful_shutdown: release PID file + runtime lock BEFORE the log drain
  253. # (never strand locks), then drain the log queue so logger.critical lands before os._exit.
  254. with contextlib.suppress(Exception):
  255. # Mirror _exit_after_graceful_shutdown: release PID file + runtime lock BEFORE the log drain
  256. # (locks must never be stranded), then drain the async log queue so the logger.critical above
  257. # actually reaches the file before os._exit bypasses atexit. (#66892)
  258. from gateway.status import remove_pid_file, release_gateway_runtime_lock
  259. remove_pid_file()
  260. release_gateway_runtime_lock()
  261. with contextlib.suppress(Exception):
  262. from hermes_logging import drain_log_queue
  263. drain_log_queue(timeout=1.0)
  264. _mark_exited_quietly(exit_code, "shutdown_watchdog")
  265. _hard_exit(exit_code)
  266. try:
  267. threading.Thread(target=_watchdog, daemon=True, name=name).start()
  268. except Exception:
  269. logger.debug("Failed to arm shutdown watchdog", exc_info=True)
  270. return done
  271. async def _tick_socket_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
  272. """Answer a liveness ping with one byte; never raises. Runs on the gateway loop, so a reply
  273. witnesses loop schedulability; the write is a socket-buffer copy (no fsync), immune to the
  274. stalls that age the heartbeat."""
  275. try:
  276. writer.write(b"1")
  277. await writer.drain()
  278. except Exception:
  279. pass
  280. finally: # close even on CancelledError (BaseException), as on BASE
  281. with contextlib.suppress(Exception):
  282. writer.close()
  283. def _sweep_stale_tick_sockets(own_path: Path) -> None:
  284. """Unlink loop-tick socket nodes left by dead PIDs (POSIX only; never raises).
  285. create_unix_server removes a leftover node at OUR path (os._exit / SIGKILL skip the
  286. finally-unlink) but not SIBLING nodes from other dead PIDs. os.kill(pid, 0) is a liveness
  287. probe only on POSIX (Windows would TerminateProcess)."""
  288. try:
  289. for stale in (p for p in own_path.parent.glob("gateway.loop-tick.*.sock") if p != own_path):
  290. try:
  291. os.kill(int(stale.name.split(".")[-2]), 0) # windows-footgun: ok — POSIX-only
  292. except (ValueError, IndexError, OSError):
  293. stale.unlink(missing_ok=True)
  294. except Exception:
  295. logger.debug("stale loop-tick socket sweep failed", exc_info=True)
  296. async def loop_heartbeat_forever(
  297. *, interval_s: float = DEFAULT_HEARTBEAT_INTERVAL_S, start_time: Optional[float] = None,
  298. home: Optional[Path] = None, should_continue: Optional[Callable[[], bool]] = None) -> None:
  299. """Rewrite the loop heartbeat file on a cadence until cancelled / gated off. Runs on the
  300. gateway loop so a frozen loop lets the file age for monitors. The fsync write goes to a thread
  301. (inline, a stalled filesystem blocked the loop inside its own heartbeat and the liveness
  302. watchdog killed it: WSL2 fsync stalls p99 31s / max 112s vs a ~90-120s budget) but is *awaited*,
  303. never fire-and-forget: an unawaited task would keep the file fresh while the loop was wedged,
  304. and one in-flight write at a time keeps a long stall from queuing a thread per interval.
  305. Off-loop, file freshness no longer proves loop schedulability, so this task also arms a
  306. loop-scheduling witness (``_tick_socket_handler``, flagged as ``loop_tick_socket``); probes must
  307. require the witness to agree before classifying WEDGED."""
  308. interval = _coerce_float(interval_s, DEFAULT_HEARTBEAT_INTERVAL_S, floor=1.0)
  309. # The witness exposes only a scheduling ping, not control/auth operations. Reuse its
  310. # loopback transport when AF_UNIX is unavailable or the encoded address will not fit;
  311. # consumers already discover this port in the heartbeat, independent of their TMPDIR.
  312. from gateway.control_socket import _fits_sun_path
  313. tick_server = tick_socket_path = tick_tcp_port = None
  314. try:
  315. unix_path = get_loop_tick_socket_path(home) if os.name == "posix" else None
  316. if unix_path is not None and _fits_sun_path(unix_path):
  317. tick_socket_path = unix_path
  318. tick_socket_path.parent.mkdir(parents=True, exist_ok=True)
  319. _sweep_stale_tick_sockets(tick_socket_path)
  320. tick_server = await asyncio.start_unix_server(_tick_socket_handler,
  321. path=str(tick_socket_path))
  322. else:
  323. tick_server = await asyncio.start_server(_tick_socket_handler, host="127.0.0.1", port=0)
  324. for _s in tick_server.sockets or []:
  325. with contextlib.suppress(Exception):
  326. if isinstance(_sname := _s.getsockname(), tuple) and len(_sname) >= 2:
  327. tick_tcp_port = int(_sname[1])
  328. break
  329. except Exception:
  330. tick_server = tick_tcp_port = None
  331. logger.warning("Loop tick socket unavailable — liveness probes will have no "
  332. "loop-scheduling witness and will not escalate on a stale heartbeat",
  333. exc_info=True)
  334. extra = {"loop_tick_socket": tick_server is not None, "loop_tick_tcp_port": tick_tcp_port}
  335. try:
  336. while True: # first write is immediate so monitors see a fresh file at once
  337. try:
  338. await asyncio.to_thread(write_loop_heartbeat, start_time=start_time, home=home,
  339. extra=extra)
  340. except Exception: # write_loop_heartbeat never raises: executor problem, keep the task
  341. logger.debug("Loop heartbeat write failed off-loop", exc_info=True)
  342. if should_continue is not None and not should_continue():
  343. return
  344. await asyncio.sleep(interval)
  345. if should_continue is not None and not should_continue():
  346. return
  347. finally:
  348. if tick_server is not None:
  349. tick_server.close()
  350. with contextlib.suppress(Exception):
  351. await tick_server.wait_closed()
  352. if tick_socket_path is not None:
  353. with contextlib.suppress(Exception):
  354. tick_socket_path.unlink(missing_ok=True)