hermes_startup_watchdog.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. """Startup-liveness watchdog — respawn a gateway that wedges before its loop runs (OOF-298).
  2. Every other liveness backstop (loop-liveness watchdog, shutdown watchdog, heartbeat file)
  3. assumes startup succeeded; none can fire if the process deadlocks before the event loop
  4. is alive (OOF-298: ~30h with every thread in ``futex_wait_queue``, zero logs, s6 saw a
  5. live PID). A daemon thread armed at process entry and disarmed once the loop is confirmed
  6. live dumps all-thread stacks (``faulthandler``), records the exit in the lifecycle ledger
  7. (NS-608) and ``os._exit``\\ s with the service-restart code so s6/systemd respawn.
  8. Slow-but-alive startups are not killed. Order of authority: (1) phase-owned progress
  9. leases (:func:`report_startup_progress`) — authoritative, prove the *startup path itself*
  10. is alive, work for I/O-bound phases (schema migration, corruption repair in
  11. ``SessionDB.__init__``) with ~zero CPU; (2) process-wide CPU progress, capped at
  12. ``_MAX_CPU_EXTENSIONS`` since an unrelated thread burning CPU must not hide a parked
  13. startup thread forever. Known limitation: a *spinning* startup deadlock earns the capped
  14. extensions before firing; the observed class is parked-thread deadlocks. Idle-by-design
  15. waits call :func:`kick_startup_watchdog` (respawn-storm backoff, up to 300s); MCP
  16. discovery's 120s wait sits inside the 300s default.
  17. IMPORT-LIGHTNESS IS A CORRECTNESS PROPERTY: top-level module, stdlib only. Arming must
  18. precede importing ``gateway`` (hundreds of modules; an import-time deadlock is in scope),
  19. and at fire time the wedged main thread may hold the import lock — so the fire path does
  20. no imports on its own thread; the ledger write runs on a helper thread joined with a
  21. timeout. Config is env-only (``HERMES_STARTUP_WATCHDOG=0``,
  22. ``HERMES_STARTUP_WATCHDOG_TIMEOUT_S``) because config.yaml parsing is itself in scope.
  23. Everything is best-effort: a watchdog failure must never affect the startup it observes.
  24. """
  25. from __future__ import annotations
  26. import faulthandler
  27. import json
  28. import logging
  29. import os
  30. import threading
  31. import time
  32. from datetime import datetime, timezone, UTC
  33. from pathlib import Path
  34. from typing import Any, Dict, Optional
  35. logger = logging.getLogger(__name__)
  36. DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S = 300.0
  37. _MIN_TIMEOUT_S = 30.0
  38. # Mirrors gateway.restart.GATEWAY_SERVICE_RESTART_EXIT_CODE (parity test in
  39. # tests/gateway/test_startup_watchdog.py) — this module must not import gateway.
  40. SERVICE_RESTART_EXIT_CODE = 75
  41. ENV_STARTUP_WATCHDOG = "HERMES_STARTUP_WATCHDOG"
  42. ENV_STARTUP_WATCHDOG_TIMEOUT_S = "HERMES_STARTUP_WATCHDOG_TIMEOUT_S"
  43. _DUMP_RELATIVE = ("logs", "gateway-startup-watchdog.log")
  44. _FALSEY = frozenset({"0", "false", "no", "off"})
  45. # The waiter re-reads its deadline at most this often so kicks/extensions
  46. # take effect promptly without busy-waiting.
  47. _POLL_SLICE_S = 5.0
  48. # Minimum process CPU-time delta within one expired window to count as
  49. # progress. A parked futex deadlock accrues microseconds; a schema migration
  50. # accrues orders of magnitude more per window even on slow disks.
  51. _CPU_PROGRESS_MIN_S = 1.0
  52. # Hard cap on CPU-fallback extensions: CPU is process-wide evidence, so it may
  53. # only stretch the runway to (1 + cap) x timeout (3 x 300s = 20min); anything
  54. # longer must hold an explicit phase lease.
  55. _MAX_CPU_EXTENSIONS = 3
  56. # Per-call clamp on progress leases; a phase that needs longer renews (the
  57. # renewal is the liveness evidence). 15min covers the observed worst-case
  58. # single migration step on multi-GB state.db files with margin.
  59. _MAX_LEASE_S = 900.0
  60. # Bounded wait for the lifecycle-ledger helper thread (import lock may be
  61. # held by the wedged main thread).
  62. _LEDGER_JOIN_TIMEOUT_S = 5.0
  63. # Upper bound on the ENTIRE forensic fire path. A sibling escort thread that
  64. # touches no logging/filesystem/locks hard-exits if forensics wedge (e.g. the
  65. # wedged main thread holds the logging handler lock, disk full/hung). Must
  66. # exceed _LEDGER_JOIN_TIMEOUT_S.
  67. _FIRE_EXIT_BOUND_S = 10.0
  68. # Handle states. Transitions are guarded by the handle's state lock so a
  69. # disarm and a fire can never both "win": armed -> disarmed or armed -> firing.
  70. # See #89750.
  71. _ARMED = "armed"
  72. _DISARMED = "disarmed"
  73. _FIRING = "firing"
  74. # Module singleton: the arm sites (hermes_cli.main / hermes_cli.gateway /
  75. # gateway.run.main / cli.py --gateway) and the disarm site (GatewayRunner)
  76. # share no object, and only one gateway startup ever runs per process.
  77. _handle_lock = threading.Lock()
  78. _handle: Optional[StartupWatchdogHandle] = None
  79. def _process_hermes_home() -> Path:
  80. """Use the stdlib-only process resolver before application startup."""
  81. from hermes_constants import get_process_hermes_home
  82. return get_process_hermes_home()
  83. def get_startup_watchdog_dump_path(home: Optional[Path] = None) -> Path:
  84. """Return ``<HERMES_HOME>/logs/gateway-startup-watchdog.log``."""
  85. base = home if home is not None else _process_hermes_home()
  86. return base.joinpath(*_DUMP_RELATIVE)
  87. def startup_watchdog_disabled() -> bool:
  88. """True when ``HERMES_STARTUP_WATCHDOG`` opts out explicitly."""
  89. return os.environ.get(ENV_STARTUP_WATCHDOG, "").strip().lower() in _FALSEY
  90. def resolve_startup_watchdog_timeout() -> float:
  91. """Deadline in seconds; env override, floor-clamped, default on garbage."""
  92. raw = os.environ.get(ENV_STARTUP_WATCHDOG_TIMEOUT_S, "").strip()
  93. if not raw:
  94. return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
  95. try:
  96. value = float(raw)
  97. except ValueError:
  98. logger.warning(
  99. "Ignoring non-numeric %s=%r; using default %.0fs",
  100. ENV_STARTUP_WATCHDOG_TIMEOUT_S, raw, DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S,
  101. )
  102. return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
  103. if value <= 0:
  104. return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
  105. return max(value, _MIN_TIMEOUT_S)
  106. def _append_dump(write, failure_msg: str) -> None:
  107. """Open the dump file for append and hand it to *write*; failures only log at DEBUG."""
  108. try:
  109. path = get_startup_watchdog_dump_path()
  110. path.parent.mkdir(parents=True, exist_ok=True)
  111. with open(path, "a", encoding="utf-8") as fh:
  112. write(fh)
  113. except Exception:
  114. logger.debug(failure_msg, exc_info=True)
  115. def _write_dump_record(record: dict[str, Any]) -> None:
  116. """Append a one-line JSON metadata record beside the faulthandler dump."""
  117. _append_dump(
  118. lambda fh: fh.write(json.dumps(record, default=str) + "\n"),
  119. "Failed to write startup watchdog dump record",
  120. )
  121. def _log_quietly(level: str, msg: str, *args) -> None:
  122. """Log at *level*; never raises (the wedged main thread may hold the logging lock)."""
  123. try:
  124. getattr(logger, level)(msg, *args)
  125. except Exception:
  126. pass
  127. def _mark_lifecycle_exit(exit_code: int) -> None:
  128. """Record the watchdog exit in the NS-608 lifecycle sentinel.
  129. Runs on a helper thread (see ``_fire``): the import can block on the
  130. interpreter import lock, and the fire path must reach ``os._exit`` regardless.
  131. """
  132. try:
  133. from gateway.lifecycle_ledger import mark_exited
  134. mark_exited(exit_code, reason="startup_liveness_watchdog")
  135. except Exception:
  136. pass
  137. class StartupWatchdogHandle:
  138. """Disarm/inspect handle for the armed startup watchdog thread."""
  139. def __init__(self, timeout_s: float, exit_code: int):
  140. self.timeout_s = timeout_s
  141. self.exit_code = exit_code
  142. self.armed_at = time.monotonic()
  143. self._state = _ARMED
  144. self._state_lock = threading.Lock()
  145. self._deadline = self.armed_at + timeout_s
  146. self._disarmed_event = threading.Event()
  147. self._thread: Optional[threading.Thread] = None
  148. self._extensions = 0
  149. # Phase-owned progress lease (see lease()): monotonic deadline the
  150. # current startup phase has claimed for legitimately long sync work.
  151. self._lease_until = 0.0
  152. self._lease_phase: Optional[str] = None
  153. self._lease_count = 0
  154. # Set by _fire() once forensics complete so the exit escort stands down.
  155. self._fire_done = threading.Event()
  156. def disarm(self) -> None:
  157. """Startup reached a live event loop — stand down. Idempotent.
  158. Atomic with respect to firing: whichever of disarm/fire takes the state
  159. lock first wins, so a disarm landing before the fire sequence is never lost.
  160. """
  161. with self._state_lock:
  162. if self._state == _ARMED:
  163. self._state = _DISARMED
  164. self._disarmed_event.set()
  165. def kick(self, extra_s: float = 0.0) -> None:
  166. """Push the deadline out to ``now + timeout + extra_s``.
  167. For call sites about to block intentionally with ~zero CPU (the
  168. respawn-storm backoff sleep), otherwise indistinguishable from a parked deadlock.
  169. """
  170. try:
  171. extra = max(0.0, float(extra_s))
  172. except (TypeError, ValueError):
  173. extra = 0.0
  174. with self._state_lock:
  175. self._deadline = time.monotonic() + self.timeout_s + extra
  176. def lease(self, expected_s: float, phase: str = "") -> None:
  177. """Claim a progress lease: this phase expects up to ``expected_s`` more seconds of work.
  178. The authoritative "still making progress" signal — owned by the startup path,
  179. so it works for I/O-bound phases and cannot be counterfeited by unrelated
  180. threads. Clamped to ``_MAX_LEASE_S`` per call so one buggy caller cannot
  181. silence the watchdog indefinitely; long phases renew. Never raises.
  182. """
  183. try:
  184. expected = float(expected_s)
  185. except (TypeError, ValueError):
  186. return
  187. if expected <= 0:
  188. return
  189. expected = min(expected, _MAX_LEASE_S)
  190. with self._state_lock:
  191. self._lease_until = max(self._lease_until, time.monotonic() + expected)
  192. if phase:
  193. self._lease_phase = str(phase)
  194. self._lease_count += 1
  195. @property
  196. def disarmed(self) -> bool:
  197. return self._state == _DISARMED
  198. def is_alive(self) -> bool:
  199. return self._thread is not None and self._thread.is_alive()
  200. def join(self, timeout: Optional[float] = None) -> None:
  201. if self._thread is not None:
  202. self._thread.join(timeout=timeout)
  203. # ── internals ────────────────────────────────────────────────────────
  204. @staticmethod
  205. def _process_cpu_seconds() -> Optional[float]:
  206. """Process-wide CPU time (user+system, all threads); None on failure."""
  207. try:
  208. return time.process_time()
  209. except Exception:
  210. return None
  211. def _fire(self) -> None:
  212. """Forensics, then exit — with the exit itself independently bounded.
  213. Everything producing forensics (logging, dump record, faulthandler, ledger)
  214. can block: the wedged main thread may hold the logging handler lock, the disk
  215. may be hung. None of that may stop the respawn, so the escort thread starts
  216. FIRST. ``os._exit`` is async-signal-safe and lock-free by design.
  217. """
  218. try:
  219. threading.Thread(
  220. target=self._exit_escort, daemon=True, name="gateway-startup-watchdog-exit-escort"
  221. ).start()
  222. except Exception:
  223. pass
  224. elapsed = time.monotonic() - self.armed_at
  225. _log_quietly(
  226. "critical",
  227. "Gateway startup did not reach a live event loop within %.0fs "
  228. "(elapsed %.0fs, %d extension(s)), holds no progress lease "
  229. "and shows no CPU progress; dumping all thread stacks and "
  230. "exiting with code %d so the service supervisor can restart "
  231. "it (OOF-298).",
  232. self.timeout_s, elapsed, self._extensions, self.exit_code,
  233. )
  234. _write_dump_record(
  235. {
  236. "ts": datetime.now(UTC).isoformat(),
  237. "tag": "startup_watchdog.fired",
  238. "pid": os.getpid(),
  239. "timeout_s": self.timeout_s,
  240. "elapsed_s": round(elapsed, 3),
  241. "extensions": self._extensions,
  242. "lease_count": self._lease_count,
  243. "last_lease_phase": self._lease_phase,
  244. "exit_code": self.exit_code,
  245. }
  246. )
  247. # Write the durable copy first. A detached service can have a blocked
  248. # stderr, and the exit escort bounds the whole forensic path to 10s.
  249. _append_dump(
  250. lambda fh: faulthandler.dump_traceback(file=fh, all_threads=True),
  251. "Startup watchdog file-based faulthandler dump failed",
  252. )
  253. try:
  254. faulthandler.dump_traceback(all_threads=True)
  255. except Exception:
  256. logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
  257. # Ledger write on a helper thread (it imports application code; the
  258. # wedged main thread may hold the import lock). Bounded join, then exit
  259. # regardless — NS-608 classification is best-effort; the respawn is not.
  260. try:
  261. ledger_thread = threading.Thread(
  262. target=_mark_lifecycle_exit, args=(self.exit_code,), daemon=True,
  263. name="gateway-startup-watchdog-ledger",
  264. )
  265. ledger_thread.start()
  266. ledger_thread.join(timeout=_LEDGER_JOIN_TIMEOUT_S)
  267. except Exception:
  268. pass
  269. self._fire_done.set()
  270. self._exit(self.exit_code)
  271. def _exit_escort(self) -> None:
  272. """Hard-exit if the forensic fire path wedges (bounded-exit seam).
  273. Deliberately free of log handlers, filesystem access, module loads and any
  274. lock shared with application code: only a sleep, an Event check and the exit seam.
  275. """
  276. self._sleep(_FIRE_EXIT_BOUND_S)
  277. if self._fire_done.is_set():
  278. return
  279. self._exit(self.exit_code)
  280. @staticmethod
  281. def _sleep(seconds: float) -> None:
  282. """Seam for tests; production is a bare ``time.sleep``."""
  283. time.sleep(seconds)
  284. @staticmethod
  285. def _exit(code: int) -> None:
  286. """Seam for tests; production is a bare ``os._exit``."""
  287. os._exit(code)
  288. def _extend_if_armed(self, deadline: float) -> bool:
  289. """Set a new deadline under the state lock; False when no longer armed."""
  290. with self._state_lock:
  291. if self._state != _ARMED:
  292. return False
  293. self._deadline = deadline
  294. return True
  295. def _run(self) -> None:
  296. last_cpu = self._process_cpu_seconds()
  297. while True:
  298. with self._state_lock:
  299. if self._state != _ARMED:
  300. return
  301. deadline = self._deadline
  302. remaining = deadline - time.monotonic()
  303. if remaining > 0:
  304. if self._disarmed_event.wait(timeout=min(remaining, _POLL_SLICE_S)):
  305. return
  306. continue
  307. # Deadline expired. Order of authority: (1) a phase lease is honored
  308. # outright; (2) CPU progress extends at most _MAX_CPU_EXTENSIONS
  309. # times (process-wide CPU proves activity, not startup progress).
  310. now = time.monotonic()
  311. with self._state_lock:
  312. lease_until = self._lease_until
  313. lease_phase = self._lease_phase
  314. if lease_until > now:
  315. if not self._extend_if_armed(max(lease_until, now + min(_POLL_SLICE_S, self.timeout_s))):
  316. return
  317. _log_quietly(
  318. "warning",
  319. "Gateway startup exceeded %.0fs but phase %r holds a "
  320. "progress lease for another %.0fs — honoring it.",
  321. self.timeout_s, lease_phase or "unknown", lease_until - now,
  322. )
  323. # Leased work may be I/O-bound; reset the CPU baseline so the
  324. # post-lease window is judged on its own activity.
  325. last_cpu = self._process_cpu_seconds()
  326. continue
  327. cpu = self._process_cpu_seconds()
  328. window_delta = cpu - last_cpu if cpu is not None and last_cpu is not None else None
  329. if window_delta is not None and window_delta >= _CPU_PROGRESS_MIN_S and self._extensions < _MAX_CPU_EXTENSIONS:
  330. last_cpu = cpu
  331. self._extensions += 1
  332. if not self._extend_if_armed(time.monotonic() + self.timeout_s):
  333. return
  334. _log_quietly(
  335. "warning",
  336. "Gateway startup exceeded %.0fs but is consuming CPU "
  337. "(%.1fs this window); extending the startup watchdog "
  338. "deadline (CPU-fallback extension %d of %d — phases "
  339. "doing long legitimate work should call "
  340. "report_startup_progress instead).",
  341. self.timeout_s, window_delta, self._extensions, _MAX_CPU_EXTENSIONS,
  342. )
  343. continue
  344. # No progress: claim the fire transition atomically so a disarm
  345. # racing this exact moment can still win if it gets there first.
  346. with self._state_lock:
  347. if self._state != _ARMED:
  348. return
  349. self._state = _FIRING
  350. self._fire()
  351. return
  352. def _start(self) -> bool:
  353. thread = threading.Thread(target=self._run, daemon=True, name="gateway-startup-watchdog")
  354. try:
  355. thread.start()
  356. except Exception:
  357. logger.debug("Failed to start gateway startup watchdog", exc_info=True)
  358. return False
  359. self._thread = thread
  360. return True
  361. def arm_startup_watchdog(
  362. timeout_s: Optional[float] = None,
  363. *,
  364. exit_code: int = SERVICE_RESTART_EXIT_CODE,
  365. ) -> Optional[StartupWatchdogHandle]:
  366. """Arm the process-wide startup watchdog. Idempotent; never raises.
  367. Returns the (possibly pre-existing) handle, or ``None`` when disabled via
  368. ``HERMES_STARTUP_WATCHDOG=0`` or when the thread could not be started.
  369. """
  370. global _handle
  371. try:
  372. if startup_watchdog_disabled():
  373. return None
  374. with _handle_lock:
  375. if _handle is not None and _handle.is_alive():
  376. return _handle
  377. resolved = (
  378. float(timeout_s)
  379. if timeout_s is not None and float(timeout_s) > 0
  380. else resolve_startup_watchdog_timeout()
  381. )
  382. handle = StartupWatchdogHandle(resolved, exit_code)
  383. if not handle._start():
  384. return None
  385. _handle = handle
  386. return handle
  387. except Exception:
  388. logger.debug("Failed to arm gateway startup watchdog", exc_info=True)
  389. return None
  390. def disarm_startup_watchdog() -> None:
  391. """Disarm the process-wide startup watchdog, if armed. Never raises.
  392. ``disarm()`` runs while still holding the singleton lock — it is non-blocking,
  393. and this closes the window where a concurrent re-arm could swap in a new
  394. handle that the disarm then misses.
  395. """
  396. global _handle
  397. try:
  398. with _handle_lock:
  399. handle = _handle
  400. _handle = None
  401. if handle is not None:
  402. handle.disarm()
  403. except Exception:
  404. logger.debug("Failed to disarm gateway startup watchdog", exc_info=True)
  405. def _with_armed_handle(method: str, failure_msg: str, *args) -> None:
  406. """Call ``handle.<method>(*args)`` on the armed handle; no-op when unarmed, never raises."""
  407. try:
  408. with _handle_lock:
  409. handle = _handle
  410. if handle is not None:
  411. getattr(handle, method)(*args)
  412. except Exception:
  413. logger.debug(failure_msg, exc_info=True)
  414. def kick_startup_watchdog(extra_s: float = 0.0) -> None:
  415. """Extend the armed watchdog's deadline. No-op when not armed; never raises.
  416. Call before intentionally blocking with ~zero CPU activity (e.g. the
  417. respawn-storm backoff sleep) so the idle wait is not mistaken for a parked deadlock.
  418. """
  419. _with_armed_handle("kick", "Failed to kick gateway startup watchdog", extra_s)
  420. def report_startup_progress(expected_s: float, phase: str = "") -> None:
  421. """Declare a phase-owned progress lease on the armed startup watchdog.
  422. Call from startup phases about to do legitimately long synchronous work — most
  423. importantly ``state.db`` schema migrations and corruption repair/backup inside
  424. ``SessionDB.__init__`` — with an honest worst case, renewing for multi-step phases.
  425. Per-call duration is clamped to ``_MAX_LEASE_S``. No-op when not armed; never
  426. raises — safe to call unconditionally from application code.
  427. """
  428. _with_armed_handle("lease", "Failed to report startup progress", expected_s, phase)
  429. def _reset_for_tests() -> None:
  430. """Drop the module singleton (test isolation only)."""
  431. global _handle
  432. with _handle_lock:
  433. handle = _handle
  434. _handle = None
  435. if handle is not None:
  436. handle.disarm()