lifecycle_ledger.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439
  1. """Gateway lifecycle ledger — durable termination-reason evidence.
  2. Graceful shutdowns leave forensics (``gateway-exit-diag.log``); an unclean
  3. death (SIGKILL, kernel OOM, VM death) runs no handler. A sentinel at
  4. ``<HERMES_HOME>/state/gateway.lifecycle.json`` closes the gap:
  5. :func:`record_startup` finds ``phase == "running"`` from the previous life →
  6. unclean death, appended to the exit-diag log as ``gateway.previous_unclean_exit``
  7. and logged at WARNING; :func:`mark_exited` rewrites ``phase=exited`` on every
  8. clean exit path. Best-effort: forensics must never affect the lifecycle.
  9. """
  10. from __future__ import annotations
  11. import json
  12. import logging
  13. import os
  14. import sqlite3
  15. import time
  16. from contextlib import closing
  17. from datetime import datetime, timezone, UTC
  18. from pathlib import Path
  19. from typing import Any, Dict, Optional
  20. logger = logging.getLogger(__name__)
  21. def _process_hermes_home() -> Path:
  22. """HERMES_HOME for process-level identity files (ignore task overrides)."""
  23. from hermes_constants import get_hermes_home, get_process_hermes_home
  24. # get_process_hermes_home expands ``~``/``$VAR`` (python -m gateway.run skips the CLI normalizer).
  25. return get_process_hermes_home() if os.environ.get("HERMES_HOME", "").strip() else get_hermes_home()
  26. def _home_path(home: Optional[Path], *relative: str) -> Path:
  27. return (_process_hermes_home() if home is None else home).joinpath(*relative)
  28. def get_lifecycle_sentinel_path(home: Optional[Path] = None) -> Path:
  29. """Return ``<HERMES_HOME>/state/gateway.lifecycle.json``."""
  30. return _home_path(home, "state", "gateway.lifecycle.json")
  31. def _now_iso() -> str:
  32. return datetime.now(UTC).isoformat()
  33. def _proc_fields(path: str, wanted: dict[str, str]) -> dict[str, int]:
  34. """``{dst: int}`` for each ``src: dst`` key found in a ``Key: value`` /proc file."""
  35. found: dict[str, int] = {}
  36. try:
  37. with open(path, encoding="utf-8-sig") as fh:
  38. for line in fh:
  39. key = line.split(":", 1)[0]
  40. if key in wanted:
  41. found[wanted[key]] = int(line.split()[1])
  42. if len(found) == len(wanted):
  43. break
  44. except (OSError, ValueError, IndexError):
  45. return {}
  46. return found
  47. def sample_memory() -> dict[str, Any]:
  48. """Cheap /proc snapshot (KiB): own RSS + MemTotal/MemAvailable + swap used. Linux-only
  49. (``{}`` elsewhere), never raises; the 30s heartbeat embeds it so OOM cycles are classifiable."""
  50. sample = _proc_fields("/proc/self/status", {"VmRSS": "rss_kib"})
  51. mem = _proc_fields("/proc/meminfo", {"MemTotal": "mem_total_kib", "MemAvailable": "mem_available_kib",
  52. "SwapTotal": "SwapTotal", "SwapFree": "SwapFree"})
  53. swap_total, swap_free = mem.pop("SwapTotal", None), mem.pop("SwapFree", None)
  54. sample.update(mem)
  55. if swap_total is not None and swap_free is not None:
  56. sample["swap_used_kib"] = swap_total - swap_free
  57. return sample
  58. def _read_json(path: Path) -> Optional[dict[str, Any]]:
  59. try:
  60. data = json.loads(path.read_text(encoding="utf-8-sig"))
  61. except (OSError, ValueError):
  62. return None
  63. return data if isinstance(data, dict) else None
  64. def _write_sentinel(payload: dict[str, Any], home: Optional[Path]) -> None:
  65. try:
  66. from utils import atomic_json_write
  67. path = get_lifecycle_sentinel_path(home)
  68. from hermes_constants import mkdir_under_hermes_home
  69. mkdir_under_hermes_home(path.parent)
  70. atomic_json_write(path, payload, indent=None)
  71. except Exception:
  72. logger.debug("Failed to write lifecycle sentinel", exc_info=True)
  73. def _append_exit_diag(record: dict[str, Any], home: Optional[Path]) -> None:
  74. """Append a JSON line to gateway-exit-diag.log (same format as the CLI's ``_exit_diag``)."""
  75. try:
  76. path = _home_path(home, "logs", "gateway-exit-diag.log")
  77. from hermes_constants import mkdir_under_hermes_home
  78. mkdir_under_hermes_home(path.parent)
  79. with path.open("a", encoding="utf-8") as fh:
  80. fh.write(json.dumps(record, default=str) + "\n")
  81. except OSError:
  82. logger.debug("Failed to append unclean-exit record", exc_info=True)
  83. def _pid_is_sentinel_owner(pid: Any, start_time: Any, create_time: Any) -> bool:
  84. """True when ``pid`` is a live process that is the sentinel's incarnation — guards the
  85. ``--replace`` race: a live matching owner mid-teardown is a handover, not a death.
  86. Identity is the psutil ``create_time`` the sentinel stamps at claim (epoch seconds, same
  87. producer on both sides). The sentinel's ``start_time`` is the ledger claim time and is NOT
  88. comparable with ``gateway.status.get_process_start_time`` (proc ticks on Linux, centiseconds
  89. elsewhere) — the old comparison never matched, so every ``--replace`` handover read as an
  90. unclean death. A pre-stamp sentinel (no ``create_time``) still has ``start_time`` in epoch
  91. seconds: the owner was born BEFORE it claimed, a PID reuser AFTER the owner died, so a birth
  92. later than the claim is a reuser."""
  93. try:
  94. pid_int = int(pid)
  95. # NOT os.kill(pid, 0): on Windows that sends CTRL_C_EVENT to the target's console group.
  96. from gateway.status import _pid_exists
  97. if pid_int <= 0 or not _pid_exists(pid_int):
  98. return False
  99. except Exception:
  100. return False
  101. from hermes_cli.process_identity import _process_create_time
  102. actual = _process_create_time(pid_int)
  103. if actual is None:
  104. return True # can't disambiguate PID reuse
  105. if type(create_time) in (int, float):
  106. return abs(actual - float(create_time)) <= 2.0
  107. if type(start_time) in (int, float):
  108. return actual <= float(start_time) + 2.0
  109. return True
  110. def _suspected_oom(mem: dict[str, Any]) -> bool:
  111. """Heuristic only (classification stays with the reader); thresholds are
  112. memory_status' "critical" tier so a live warning and a post-mortem verdict agree."""
  113. from gateway.memory_status import _CRITICAL_AVAILABLE_FRACTION, _CRITICAL_AVAILABLE_KIB
  114. total, avail = mem.get("mem_total_kib"), mem.get("mem_available_kib")
  115. if not isinstance(avail, int):
  116. return False
  117. return avail < _CRITICAL_AVAILABLE_KIB or (
  118. isinstance(total, int) and total > 0 and avail / total < _CRITICAL_AVAILABLE_FRACTION
  119. )
  120. def detect_unclean_exit(home: Optional[Path] = None) -> Optional[dict[str, Any]]:
  121. """Evidence dict when the previous life died uncleanly, else ``None``. Read-only."""
  122. sentinel = _read_json(get_lifecycle_sentinel_path(home))
  123. if not sentinel or sentinel.get("phase") != "running":
  124. return None
  125. if _pid_is_sentinel_owner(sentinel.get("pid"), sentinel.get("start_time"), sentinel.get("create_time")):
  126. return None # live owner — planned takeover in flight, not a death
  127. evidence: dict[str, Any] = {
  128. "prior_pid": sentinel.get("pid"), "prior_started_at": sentinel.get("started_at"),
  129. "prior_start_time": sentinel.get("start_time"),
  130. }
  131. # Enrich with the last heartbeat: last proven liveness and memory at that moment.
  132. try:
  133. from gateway.shutdown_watchdog import get_loop_heartbeat_path
  134. hb = _read_json(get_loop_heartbeat_path(home)) or {}
  135. except Exception:
  136. hb = {}
  137. if hb:
  138. evidence["last_heartbeat_at"] = hb.get("updated_at")
  139. mem = hb.get("mem")
  140. if isinstance(mem, dict):
  141. evidence["last_heartbeat_mem"] = mem
  142. if _suspected_oom(mem):
  143. evidence["suspected_oom"] = True
  144. return evidence
  145. # Startup-watchdog lease held while the unclean-exit integrity check runs (#115542):
  146. # ``PRAGMA quick_check(1)`` is one monolithic SQLite operation and per-call leases
  147. # clamp at 900s, so a single entry lease cannot cover a multi-thousand-second healthy
  148. # check on a huge store — without renewal the watchdog kills the attempt with exit 75
  149. # and the restart loop never reaches the cron ticker. The progress handler below renews
  150. # a phase-owned lease for as long as SQLite is making progress; without a deadline it
  151. # always returns 0 so it never aborts the check, and the ok/absent/first-complaint
  152. # verdicts stay fail-closed. When an explicit ``deadline_at`` is supplied the
  153. # handler returns non-zero once it passes, so SQLite aborts the scan and the caller
  154. # can bound a pathological store instead of pinning the boot path.
  155. _INTEGRITY_CHECK_LEASE_PHASE = "state_db_unclean_integrity_check"
  156. _INTEGRITY_CHECK_LEASE_S = 900.0
  157. # VM instructions between progress-handler invocations; each invocation is a
  158. # monotonic-clock compare, renewing at most every _INTEGRITY_CHECK_LEASE_RENEW_S.
  159. _INTEGRITY_CHECK_PROGRESS_OPS = 100_000
  160. _INTEGRITY_CHECK_LEASE_RENEW_S = 60.0
  161. # Bound on the deferred post-connect scan: a pathological store cannot pin its thread forever.
  162. STATE_DB_INTEGRITY_DEADLINE_S = 1800.0
  163. def _install_integrity_check_lease(conn: sqlite3.Connection, *, deadline_at: Optional[float] = None) -> None:
  164. """Claim the integrity-check lease and renew it from a SQLite progress handler.
  165. ``report_startup_progress`` is a no-op when no watchdog is armed and never raises.
  166. Without ``deadline_at`` the handler always returns 0 -- it must never abort the verdict
  167. PRAGMA. With ``deadline_at`` (a ``time.monotonic`` timestamp) it returns non-zero once
  168. that instant is reached, so SQLite raises ``OperationalError: interrupted`` and the
  169. caller can bound the scan. Synchronous by design: no checker worker thread can outlive
  170. the check.
  171. """
  172. from hermes_startup_watchdog import report_startup_progress
  173. report_startup_progress(_INTEGRITY_CHECK_LEASE_S, phase=_INTEGRITY_CHECK_LEASE_PHASE)
  174. last_renew = time.monotonic()
  175. def _on_progress() -> int:
  176. nonlocal last_renew
  177. now = time.monotonic()
  178. if now - last_renew >= _INTEGRITY_CHECK_LEASE_RENEW_S:
  179. last_renew = now
  180. report_startup_progress(_INTEGRITY_CHECK_LEASE_S, phase=_INTEGRITY_CHECK_LEASE_PHASE)
  181. if deadline_at is not None and now >= deadline_at:
  182. return 1
  183. return 0
  184. conn.set_progress_handler(_on_progress, _INTEGRITY_CHECK_PROGRESS_OPS)
  185. def check_state_db_integrity(home: Optional[Path] = None, *, deadline_s: Optional[float] = None) -> str:
  186. """``"ok"``, ``"absent"``, or the first ``quick_check`` complaint. Never raises.
  187. Only after an unclean death — SIGKILL mid-WAL-checkpoint can leave half-written
  188. b-tree pages. ``quick_check(1)`` stops at the first problem (~2s on a healthy
  189. 500MB store): cheap once per unclean boot, too costly every boot. Opened
  190. normally: a WAL store needs its -shm sidecar for read-only, and the PRAGMA writes nothing.
  191. Holds a phase-owned startup-watchdog lease for the check duration
  192. (renewed from a SQLite progress handler, #115542) so a long healthy check
  193. on a huge store is not mistaken for a parked deadlock.
  194. ``deadline_s`` (optional): abort the scan once that many seconds elapse and
  195. return ``"check-incomplete: exceeded <N>s"``. ``deadline_s=None`` (default) is the
  196. historic unbounded behaviour -- the progress handler never aborts the PRAGMA.
  197. """
  198. path = _home_path(home, "state.db")
  199. if not path.exists():
  200. return "absent"
  201. deadline_at: Optional[float] = None
  202. if deadline_s is not None:
  203. try:
  204. deadline_at = time.monotonic() + max(0.0, float(deadline_s))
  205. except (TypeError, ValueError):
  206. deadline_at = None
  207. try:
  208. with closing(sqlite3.connect(str(path))) as conn:
  209. _install_integrity_check_lease(conn, deadline_at=deadline_at)
  210. row = conn.execute("PRAGMA quick_check(1)").fetchone()
  211. except Exception as exc:
  212. if (deadline_at is not None and isinstance(exc, sqlite3.OperationalError)
  213. and "interrupt" in str(exc).lower()):
  214. limit = max(0.0, float(deadline_s)) if deadline_s is not None else 0.0
  215. return f"check-incomplete: exceeded {limit:g}s"
  216. return f"check-failed: {exc}"
  217. return "check-failed: no result" if not row or row[0] is None else str(row[0])
  218. def _state_db_journal_mode(path: Path) -> Optional[str]:
  219. """On-disk journal mode of ``path`` via one ``PRAGMA journal_mode`` read on a short-lived
  220. connection (no lock outlives it); ``None`` when undeterminable. Never raises."""
  221. try:
  222. from hermes_state_wal import _on_disk_journal_mode
  223. with closing(sqlite3.connect(str(path), timeout=1.0)) as conn:
  224. return _on_disk_journal_mode(conn) or None
  225. except Exception:
  226. logger.debug("Could not read state.db journal mode", exc_info=True)
  227. return None
  228. def _report_unclean_exit(evidence: dict[str, Any], home: Optional[Path], *, defer_integrity_check: bool = False) -> None:
  229. """Persist the exit-diag record and log at WARNING.
  230. When ``defer_integrity_check`` is set, the forensic store scan is NOT run here (it would
  231. gate platform startup); the record carries ``state_db_integrity: "deferred"`` and
  232. :func:`run_deferred_integrity_check` runs the scan post-connect.
  233. """
  234. if defer_integrity_check:
  235. evidence["state_db_integrity"] = "deferred"
  236. else:
  237. # The death may have torn the store; this is the only moment we know to look.
  238. verdict = evidence["state_db_integrity"] = check_state_db_integrity(home=home)
  239. if verdict not in ("ok", "absent"):
  240. logger.error(
  241. "state.db FAILED integrity check after an unclean gateway exit: %s — sessions may read as "
  242. "missing until it is repaired. Run `hermes doctor`.",
  243. verdict,
  244. )
  245. _append_exit_diag({"ts": _now_iso(), "tag": "gateway.previous_unclean_exit", "pid": os.getpid(), **evidence}, home)
  246. logger.warning(
  247. "Previous gateway life (pid=%s, started_at=%s) exited UNCLEANLY (no exit path ran — SIGKILL / OOM / "
  248. "VM death, or a process kill issued by the agent or one of its descendants, e.g. a pkill/taskkill of "
  249. "the host interpreter image; see #113667). last_heartbeat_at=%s last_mem=%s suspected_oom=%s",
  250. evidence.get("prior_pid"), evidence.get("prior_started_at"), evidence.get("last_heartbeat_at"),
  251. evidence.get("last_heartbeat_mem"), evidence.get("suspected_oom", False),
  252. )
  253. def record_startup(home: Optional[Path] = None, *, defer_integrity_check: bool = True) -> Optional[dict[str, Any]]:
  254. """Boot entry point: report any unclean previous exit (evidence dict, also persisted
  255. to ``gateway-exit-diag.log`` and logged at WARNING) then claim the sentinel. Never raises.
  256. ``defer_integrity_check`` defaults to True: the state.db integrity scan is forensic and
  257. must NOT gate platform startup (a large store's ``quick_check`` blocked the ``:8644`` fleet
  258. peer-DM transport for minutes). In deferred mode the exit-diag record is written
  259. immediately with ``state_db_integrity: "deferred"`` and the scan runs post-connect via
  260. :func:`run_deferred_integrity_check`. Pass False for the historic INLINE scan.
  261. """
  262. evidence: Optional[dict[str, Any]] = None
  263. try:
  264. evidence = detect_unclean_exit(home)
  265. if evidence is not None:
  266. _report_unclean_exit(evidence, home, defer_integrity_check=defer_integrity_check)
  267. except Exception:
  268. logger.debug("Unclean-exit detection failed", exc_info=True)
  269. try:
  270. claim: dict[str, Any] = {"phase": "running", "pid": os.getpid(), "start_time": time.time(), "started_at": _now_iso()}
  271. # Process birth (psutil), distinct from ``start_time`` (the ledger claim, seconds later once
  272. # imports finish): the Windows start attestation binds PIDs to birth time (#110020 review).
  273. from hermes_cli.process_identity import _process_create_time
  274. create_time = _process_create_time(os.getpid())
  275. if create_time is not None:
  276. claim["create_time"] = create_time
  277. # Carry the verdict on the PREVIOUS life on the new sentinel: it is the only
  278. # machine-readable copy (/api/status reads it to report an OOM restart).
  279. # Scoped to this life — the next clean exit or boot rewrites the sentinel.
  280. if evidence is not None:
  281. claim["prior_unclean_exit"] = True
  282. if evidence.get("suspected_oom"):
  283. claim["prior_suspected_oom"] = True
  284. _write_sentinel(claim, home)
  285. except Exception:
  286. logger.debug("Failed to claim lifecycle sentinel", exc_info=True)
  287. return evidence
  288. def run_deferred_integrity_check(
  289. evidence: Optional[dict[str, Any]], *, home: Optional[Path] = None,
  290. deadline_s: float = STATE_DB_INTEGRITY_DEADLINE_S,
  291. ) -> str:
  292. """Run the deferred post-connect state.db scan and append a second diag record.
  293. Companion to :func:`record_startup` with ``defer_integrity_check=True``: the forensic scan
  294. is moved off the startup path (where, on a multi-GB store, it gated every platform adapter
  295. — including the ``:8644`` fleet peer-DM transport — for minutes) into a daemon thread
  296. started only after platforms connected. ``deadline_s`` bounds a pathological store so the
  297. thread cannot be pinned forever.
  298. Appends ``{"tag": "gateway.state_db_integrity", "deferred": true, "state_db_integrity":
  299. <verdict>, "prior_pid": ...}`` and logs the existing ERROR text on a bad verdict. NEVER
  300. raises -- a forensic scan must not disturb a running gateway (the alerting path is the
  301. bounded per-poll readiness probe ``gateway/readiness.py::_probe_state_db``).
  302. Only a WAL store is scanned: in rollback-journal mode (``database.journal_mode: delete``,
  303. or a silent WAL fallback) the scan's SHARED lock would block every live commit for its
  304. whole run, so the scan is skipped with a ``skipped: ...`` verdict and one WARNING. The
  305. on-disk mode decides, not config; an undeterminable mode is skipped too.
  306. """
  307. try:
  308. path = _home_path(home, "state.db")
  309. mode = _state_db_journal_mode(path) if path.exists() else "wal"
  310. if mode != "wal":
  311. verdict = (f"skipped: rollback-journal store (journal_mode={mode or 'unknown'}), "
  312. "scan would block live writes")
  313. logger.warning(
  314. "Previous gateway exit was unclean, but the state.db integrity check was skipped: %s is a "
  315. "rollback-journal store (journal_mode=%s) and a background scan would block live writes. "
  316. "To check it, stop the gateway and run: sqlite3 %s 'PRAGMA quick_check'",
  317. path, mode or "unknown", path,
  318. )
  319. else:
  320. verdict = check_state_db_integrity(home=home, deadline_s=deadline_s)
  321. if verdict not in ("ok", "absent") and not verdict.startswith("skipped:"):
  322. logger.error(
  323. "state.db FAILED integrity check after an unclean gateway exit: %s — sessions may read as "
  324. "missing until it is repaired. Run `hermes doctor`.",
  325. verdict,
  326. )
  327. record: dict[str, Any] = {
  328. "ts": _now_iso(),
  329. "tag": "gateway.state_db_integrity",
  330. "deferred": True,
  331. "state_db_integrity": verdict,
  332. }
  333. if isinstance(evidence, dict) and evidence.get("prior_pid") is not None:
  334. record["prior_pid"] = evidence.get("prior_pid")
  335. _append_exit_diag(record, home)
  336. if isinstance(evidence, dict):
  337. evidence["state_db_integrity"] = verdict
  338. return verdict
  339. except Exception:
  340. logger.debug("Deferred state.db integrity check failed", exc_info=True)
  341. return "check-failed: deferred scan raised"
  342. def mark_exited(exit_code: Optional[int] = None, reason: str = "graceful_shutdown", home: Optional[Path] = None) -> None:
  343. """Mark the current life as cleanly exited. Idempotent, never raises.
  344. Only rewrites a sentinel provably owned by this process: during ``--replace``
  345. the replacement claims it before the old process finishes teardown and must not
  346. be clobbered. ``pid=None`` / malformed sentinels have unknown ownership → left alone.
  347. """
  348. try:
  349. sentinel = _read_json(get_lifecycle_sentinel_path(home))
  350. if sentinel is not None and sentinel.get("pid") != os.getpid():
  351. return
  352. exited: dict[str, Any] = {"phase": "exited", "pid": os.getpid(), "exit_code": exit_code, "exit_reason": reason,
  353. "exited_at": _now_iso()}
  354. # Carry the incarnation identity: the Windows start attestation matches a clean exit by
  355. # PID *and* start time so a reused PID's exit cannot vouch for a different life (#110020).
  356. for key in ("start_time", "create_time"):
  357. if sentinel is not None and sentinel.get(key) is not None:
  358. exited[key] = sentinel[key]
  359. _write_sentinel(exited, home)
  360. except Exception:
  361. logger.debug("Failed to mark lifecycle sentinel exited", exc_info=True)
  362. def read_prior_exit_label(profile_home: Path) -> str:
  363. """``clean``/``unclean``/``unknown`` for the profile's last gateway life; annotates
  364. ``container-boot.log``. ``running`` is unclean: the old PID namespace is gone at container boot."""
  365. try:
  366. phase = (_read_json(get_lifecycle_sentinel_path(profile_home)) or {}).get("phase")
  367. return {"exited": "clean", "running": "unclean"}.get(phase, "unknown")
  368. except Exception:
  369. return "unknown"