| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513 |
- """Startup-liveness watchdog — respawn a gateway that wedges before its loop runs (OOF-298).
- Every other liveness backstop (loop-liveness watchdog, shutdown watchdog, heartbeat file)
- assumes startup succeeded; none can fire if the process deadlocks before the event loop
- is alive (OOF-298: ~30h with every thread in ``futex_wait_queue``, zero logs, s6 saw a
- live PID). A daemon thread armed at process entry and disarmed once the loop is confirmed
- live dumps all-thread stacks (``faulthandler``), records the exit in the lifecycle ledger
- (NS-608) and ``os._exit``\\ s with the service-restart code so s6/systemd respawn.
- Slow-but-alive startups are not killed. Order of authority: (1) phase-owned progress
- leases (:func:`report_startup_progress`) — authoritative, prove the *startup path itself*
- is alive, work for I/O-bound phases (schema migration, corruption repair in
- ``SessionDB.__init__``) with ~zero CPU; (2) process-wide CPU progress, capped at
- ``_MAX_CPU_EXTENSIONS`` since an unrelated thread burning CPU must not hide a parked
- startup thread forever. Known limitation: a *spinning* startup deadlock earns the capped
- extensions before firing; the observed class is parked-thread deadlocks. Idle-by-design
- waits call :func:`kick_startup_watchdog` (respawn-storm backoff, up to 300s); MCP
- discovery's 120s wait sits inside the 300s default.
- IMPORT-LIGHTNESS IS A CORRECTNESS PROPERTY: top-level module, stdlib only. Arming must
- precede importing ``gateway`` (hundreds of modules; an import-time deadlock is in scope),
- and at fire time the wedged main thread may hold the import lock — so the fire path does
- no imports on its own thread; the ledger write runs on a helper thread joined with a
- timeout. Config is env-only (``HERMES_STARTUP_WATCHDOG=0``,
- ``HERMES_STARTUP_WATCHDOG_TIMEOUT_S``) because config.yaml parsing is itself in scope.
- Everything is best-effort: a watchdog failure must never affect the startup it observes.
- """
- from __future__ import annotations
- import faulthandler
- import json
- import logging
- import os
- import threading
- import time
- from datetime import datetime, timezone, UTC
- from pathlib import Path
- from typing import Any, Dict, Optional
- logger = logging.getLogger(__name__)
- DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S = 300.0
- _MIN_TIMEOUT_S = 30.0
- # Mirrors gateway.restart.GATEWAY_SERVICE_RESTART_EXIT_CODE (parity test in
- # tests/gateway/test_startup_watchdog.py) — this module must not import gateway.
- SERVICE_RESTART_EXIT_CODE = 75
- ENV_STARTUP_WATCHDOG = "HERMES_STARTUP_WATCHDOG"
- ENV_STARTUP_WATCHDOG_TIMEOUT_S = "HERMES_STARTUP_WATCHDOG_TIMEOUT_S"
- _DUMP_RELATIVE = ("logs", "gateway-startup-watchdog.log")
- _FALSEY = frozenset({"0", "false", "no", "off"})
- # The waiter re-reads its deadline at most this often so kicks/extensions
- # take effect promptly without busy-waiting.
- _POLL_SLICE_S = 5.0
- # Minimum process CPU-time delta within one expired window to count as
- # progress. A parked futex deadlock accrues microseconds; a schema migration
- # accrues orders of magnitude more per window even on slow disks.
- _CPU_PROGRESS_MIN_S = 1.0
- # Hard cap on CPU-fallback extensions: CPU is process-wide evidence, so it may
- # only stretch the runway to (1 + cap) x timeout (3 x 300s = 20min); anything
- # longer must hold an explicit phase lease.
- _MAX_CPU_EXTENSIONS = 3
- # Per-call clamp on progress leases; a phase that needs longer renews (the
- # renewal is the liveness evidence). 15min covers the observed worst-case
- # single migration step on multi-GB state.db files with margin.
- _MAX_LEASE_S = 900.0
- # Bounded wait for the lifecycle-ledger helper thread (import lock may be
- # held by the wedged main thread).
- _LEDGER_JOIN_TIMEOUT_S = 5.0
- # Upper bound on the ENTIRE forensic fire path. A sibling escort thread that
- # touches no logging/filesystem/locks hard-exits if forensics wedge (e.g. the
- # wedged main thread holds the logging handler lock, disk full/hung). Must
- # exceed _LEDGER_JOIN_TIMEOUT_S.
- _FIRE_EXIT_BOUND_S = 10.0
- # Handle states. Transitions are guarded by the handle's state lock so a
- # disarm and a fire can never both "win": armed -> disarmed or armed -> firing.
- # See #89750.
- _ARMED = "armed"
- _DISARMED = "disarmed"
- _FIRING = "firing"
- # Module singleton: the arm sites (hermes_cli.main / hermes_cli.gateway /
- # gateway.run.main / cli.py --gateway) and the disarm site (GatewayRunner)
- # share no object, and only one gateway startup ever runs per process.
- _handle_lock = threading.Lock()
- _handle: Optional[StartupWatchdogHandle] = None
- def _process_hermes_home() -> Path:
- """Use the stdlib-only process resolver before application startup."""
- from hermes_constants import get_process_hermes_home
- return get_process_hermes_home()
- def get_startup_watchdog_dump_path(home: Optional[Path] = None) -> Path:
- """Return ``<HERMES_HOME>/logs/gateway-startup-watchdog.log``."""
- base = home if home is not None else _process_hermes_home()
- return base.joinpath(*_DUMP_RELATIVE)
- def startup_watchdog_disabled() -> bool:
- """True when ``HERMES_STARTUP_WATCHDOG`` opts out explicitly."""
- return os.environ.get(ENV_STARTUP_WATCHDOG, "").strip().lower() in _FALSEY
- def resolve_startup_watchdog_timeout() -> float:
- """Deadline in seconds; env override, floor-clamped, default on garbage."""
- raw = os.environ.get(ENV_STARTUP_WATCHDOG_TIMEOUT_S, "").strip()
- if not raw:
- return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
- try:
- value = float(raw)
- except ValueError:
- logger.warning(
- "Ignoring non-numeric %s=%r; using default %.0fs",
- ENV_STARTUP_WATCHDOG_TIMEOUT_S, raw, DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S,
- )
- return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
- if value <= 0:
- return DEFAULT_STARTUP_WATCHDOG_TIMEOUT_S
- return max(value, _MIN_TIMEOUT_S)
- def _append_dump(write, failure_msg: str) -> None:
- """Open the dump file for append and hand it to *write*; failures only log at DEBUG."""
- try:
- path = get_startup_watchdog_dump_path()
- path.parent.mkdir(parents=True, exist_ok=True)
- with open(path, "a", encoding="utf-8") as fh:
- write(fh)
- except Exception:
- logger.debug(failure_msg, exc_info=True)
- def _write_dump_record(record: dict[str, Any]) -> None:
- """Append a one-line JSON metadata record beside the faulthandler dump."""
- _append_dump(
- lambda fh: fh.write(json.dumps(record, default=str) + "\n"),
- "Failed to write startup watchdog dump record",
- )
- def _log_quietly(level: str, msg: str, *args) -> None:
- """Log at *level*; never raises (the wedged main thread may hold the logging lock)."""
- try:
- getattr(logger, level)(msg, *args)
- except Exception:
- pass
- def _mark_lifecycle_exit(exit_code: int) -> None:
- """Record the watchdog exit in the NS-608 lifecycle sentinel.
- Runs on a helper thread (see ``_fire``): the import can block on the
- interpreter import lock, and the fire path must reach ``os._exit`` regardless.
- """
- try:
- from gateway.lifecycle_ledger import mark_exited
- mark_exited(exit_code, reason="startup_liveness_watchdog")
- except Exception:
- pass
- class StartupWatchdogHandle:
- """Disarm/inspect handle for the armed startup watchdog thread."""
- def __init__(self, timeout_s: float, exit_code: int):
- self.timeout_s = timeout_s
- self.exit_code = exit_code
- self.armed_at = time.monotonic()
- self._state = _ARMED
- self._state_lock = threading.Lock()
- self._deadline = self.armed_at + timeout_s
- self._disarmed_event = threading.Event()
- self._thread: Optional[threading.Thread] = None
- self._extensions = 0
- # Phase-owned progress lease (see lease()): monotonic deadline the
- # current startup phase has claimed for legitimately long sync work.
- self._lease_until = 0.0
- self._lease_phase: Optional[str] = None
- self._lease_count = 0
- # Set by _fire() once forensics complete so the exit escort stands down.
- self._fire_done = threading.Event()
- def disarm(self) -> None:
- """Startup reached a live event loop — stand down. Idempotent.
- Atomic with respect to firing: whichever of disarm/fire takes the state
- lock first wins, so a disarm landing before the fire sequence is never lost.
- """
- with self._state_lock:
- if self._state == _ARMED:
- self._state = _DISARMED
- self._disarmed_event.set()
- def kick(self, extra_s: float = 0.0) -> None:
- """Push the deadline out to ``now + timeout + extra_s``.
- For call sites about to block intentionally with ~zero CPU (the
- respawn-storm backoff sleep), otherwise indistinguishable from a parked deadlock.
- """
- try:
- extra = max(0.0, float(extra_s))
- except (TypeError, ValueError):
- extra = 0.0
- with self._state_lock:
- self._deadline = time.monotonic() + self.timeout_s + extra
- def lease(self, expected_s: float, phase: str = "") -> None:
- """Claim a progress lease: this phase expects up to ``expected_s`` more seconds of work.
- The authoritative "still making progress" signal — owned by the startup path,
- so it works for I/O-bound phases and cannot be counterfeited by unrelated
- threads. Clamped to ``_MAX_LEASE_S`` per call so one buggy caller cannot
- silence the watchdog indefinitely; long phases renew. Never raises.
- """
- try:
- expected = float(expected_s)
- except (TypeError, ValueError):
- return
- if expected <= 0:
- return
- expected = min(expected, _MAX_LEASE_S)
- with self._state_lock:
- self._lease_until = max(self._lease_until, time.monotonic() + expected)
- if phase:
- self._lease_phase = str(phase)
- self._lease_count += 1
- @property
- def disarmed(self) -> bool:
- return self._state == _DISARMED
- def is_alive(self) -> bool:
- return self._thread is not None and self._thread.is_alive()
- def join(self, timeout: Optional[float] = None) -> None:
- if self._thread is not None:
- self._thread.join(timeout=timeout)
- # ── internals ────────────────────────────────────────────────────────
- @staticmethod
- def _process_cpu_seconds() -> Optional[float]:
- """Process-wide CPU time (user+system, all threads); None on failure."""
- try:
- return time.process_time()
- except Exception:
- return None
- def _fire(self) -> None:
- """Forensics, then exit — with the exit itself independently bounded.
- Everything producing forensics (logging, dump record, faulthandler, ledger)
- can block: the wedged main thread may hold the logging handler lock, the disk
- may be hung. None of that may stop the respawn, so the escort thread starts
- FIRST. ``os._exit`` is async-signal-safe and lock-free by design.
- """
- try:
- threading.Thread(
- target=self._exit_escort, daemon=True, name="gateway-startup-watchdog-exit-escort"
- ).start()
- except Exception:
- pass
- elapsed = time.monotonic() - self.armed_at
- _log_quietly(
- "critical",
- "Gateway startup did not reach a live event loop within %.0fs "
- "(elapsed %.0fs, %d extension(s)), holds no progress lease "
- "and shows no CPU progress; dumping all thread stacks and "
- "exiting with code %d so the service supervisor can restart "
- "it (OOF-298).",
- self.timeout_s, elapsed, self._extensions, self.exit_code,
- )
- _write_dump_record(
- {
- "ts": datetime.now(UTC).isoformat(),
- "tag": "startup_watchdog.fired",
- "pid": os.getpid(),
- "timeout_s": self.timeout_s,
- "elapsed_s": round(elapsed, 3),
- "extensions": self._extensions,
- "lease_count": self._lease_count,
- "last_lease_phase": self._lease_phase,
- "exit_code": self.exit_code,
- }
- )
- # Write the durable copy first. A detached service can have a blocked
- # stderr, and the exit escort bounds the whole forensic path to 10s.
- _append_dump(
- lambda fh: faulthandler.dump_traceback(file=fh, all_threads=True),
- "Startup watchdog file-based faulthandler dump failed",
- )
- try:
- faulthandler.dump_traceback(all_threads=True)
- except Exception:
- logger.debug("Startup watchdog faulthandler dump failed", exc_info=True)
- # Ledger write on a helper thread (it imports application code; the
- # wedged main thread may hold the import lock). Bounded join, then exit
- # regardless — NS-608 classification is best-effort; the respawn is not.
- try:
- ledger_thread = threading.Thread(
- target=_mark_lifecycle_exit, args=(self.exit_code,), daemon=True,
- name="gateway-startup-watchdog-ledger",
- )
- ledger_thread.start()
- ledger_thread.join(timeout=_LEDGER_JOIN_TIMEOUT_S)
- except Exception:
- pass
- self._fire_done.set()
- self._exit(self.exit_code)
- def _exit_escort(self) -> None:
- """Hard-exit if the forensic fire path wedges (bounded-exit seam).
- Deliberately free of log handlers, filesystem access, module loads and any
- lock shared with application code: only a sleep, an Event check and the exit seam.
- """
- self._sleep(_FIRE_EXIT_BOUND_S)
- if self._fire_done.is_set():
- return
- self._exit(self.exit_code)
- @staticmethod
- def _sleep(seconds: float) -> None:
- """Seam for tests; production is a bare ``time.sleep``."""
- time.sleep(seconds)
- @staticmethod
- def _exit(code: int) -> None:
- """Seam for tests; production is a bare ``os._exit``."""
- os._exit(code)
- def _extend_if_armed(self, deadline: float) -> bool:
- """Set a new deadline under the state lock; False when no longer armed."""
- with self._state_lock:
- if self._state != _ARMED:
- return False
- self._deadline = deadline
- return True
- def _run(self) -> None:
- last_cpu = self._process_cpu_seconds()
- while True:
- with self._state_lock:
- if self._state != _ARMED:
- return
- deadline = self._deadline
- remaining = deadline - time.monotonic()
- if remaining > 0:
- if self._disarmed_event.wait(timeout=min(remaining, _POLL_SLICE_S)):
- return
- continue
- # Deadline expired. Order of authority: (1) a phase lease is honored
- # outright; (2) CPU progress extends at most _MAX_CPU_EXTENSIONS
- # times (process-wide CPU proves activity, not startup progress).
- now = time.monotonic()
- with self._state_lock:
- lease_until = self._lease_until
- lease_phase = self._lease_phase
- if lease_until > now:
- if not self._extend_if_armed(max(lease_until, now + min(_POLL_SLICE_S, self.timeout_s))):
- return
- _log_quietly(
- "warning",
- "Gateway startup exceeded %.0fs but phase %r holds a "
- "progress lease for another %.0fs — honoring it.",
- self.timeout_s, lease_phase or "unknown", lease_until - now,
- )
- # Leased work may be I/O-bound; reset the CPU baseline so the
- # post-lease window is judged on its own activity.
- last_cpu = self._process_cpu_seconds()
- continue
- cpu = self._process_cpu_seconds()
- window_delta = cpu - last_cpu if cpu is not None and last_cpu is not None else None
- if window_delta is not None and window_delta >= _CPU_PROGRESS_MIN_S and self._extensions < _MAX_CPU_EXTENSIONS:
- last_cpu = cpu
- self._extensions += 1
- if not self._extend_if_armed(time.monotonic() + self.timeout_s):
- return
- _log_quietly(
- "warning",
- "Gateway startup exceeded %.0fs but is consuming CPU "
- "(%.1fs this window); extending the startup watchdog "
- "deadline (CPU-fallback extension %d of %d — phases "
- "doing long legitimate work should call "
- "report_startup_progress instead).",
- self.timeout_s, window_delta, self._extensions, _MAX_CPU_EXTENSIONS,
- )
- continue
- # No progress: claim the fire transition atomically so a disarm
- # racing this exact moment can still win if it gets there first.
- with self._state_lock:
- if self._state != _ARMED:
- return
- self._state = _FIRING
- self._fire()
- return
- def _start(self) -> bool:
- thread = threading.Thread(target=self._run, daemon=True, name="gateway-startup-watchdog")
- try:
- thread.start()
- except Exception:
- logger.debug("Failed to start gateway startup watchdog", exc_info=True)
- return False
- self._thread = thread
- return True
- def arm_startup_watchdog(
- timeout_s: Optional[float] = None,
- *,
- exit_code: int = SERVICE_RESTART_EXIT_CODE,
- ) -> Optional[StartupWatchdogHandle]:
- """Arm the process-wide startup watchdog. Idempotent; never raises.
- Returns the (possibly pre-existing) handle, or ``None`` when disabled via
- ``HERMES_STARTUP_WATCHDOG=0`` or when the thread could not be started.
- """
- global _handle
- try:
- if startup_watchdog_disabled():
- return None
- with _handle_lock:
- if _handle is not None and _handle.is_alive():
- return _handle
- resolved = (
- float(timeout_s)
- if timeout_s is not None and float(timeout_s) > 0
- else resolve_startup_watchdog_timeout()
- )
- handle = StartupWatchdogHandle(resolved, exit_code)
- if not handle._start():
- return None
- _handle = handle
- return handle
- except Exception:
- logger.debug("Failed to arm gateway startup watchdog", exc_info=True)
- return None
- def disarm_startup_watchdog() -> None:
- """Disarm the process-wide startup watchdog, if armed. Never raises.
- ``disarm()`` runs while still holding the singleton lock — it is non-blocking,
- and this closes the window where a concurrent re-arm could swap in a new
- handle that the disarm then misses.
- """
- global _handle
- try:
- with _handle_lock:
- handle = _handle
- _handle = None
- if handle is not None:
- handle.disarm()
- except Exception:
- logger.debug("Failed to disarm gateway startup watchdog", exc_info=True)
- def _with_armed_handle(method: str, failure_msg: str, *args) -> None:
- """Call ``handle.<method>(*args)`` on the armed handle; no-op when unarmed, never raises."""
- try:
- with _handle_lock:
- handle = _handle
- if handle is not None:
- getattr(handle, method)(*args)
- except Exception:
- logger.debug(failure_msg, exc_info=True)
- def kick_startup_watchdog(extra_s: float = 0.0) -> None:
- """Extend the armed watchdog's deadline. No-op when not armed; never raises.
- Call before intentionally blocking with ~zero CPU activity (e.g. the
- respawn-storm backoff sleep) so the idle wait is not mistaken for a parked deadlock.
- """
- _with_armed_handle("kick", "Failed to kick gateway startup watchdog", extra_s)
- def report_startup_progress(expected_s: float, phase: str = "") -> None:
- """Declare a phase-owned progress lease on the armed startup watchdog.
- Call from startup phases about to do legitimately long synchronous work — most
- importantly ``state.db`` schema migrations and corruption repair/backup inside
- ``SessionDB.__init__`` — with an honest worst case, renewing for multi-step phases.
- Per-call duration is clamped to ``_MAX_LEASE_S``. No-op when not armed; never
- raises — safe to call unconditionally from application code.
- """
- _with_armed_handle("lease", "Failed to report startup progress", expected_s, phase)
- def _reset_for_tests() -> None:
- """Drop the module singleton (test isolation only)."""
- global _handle
- with _handle_lock:
- handle = _handle
- _handle = None
- if handle is not None:
- handle.disarm()
|