shutdown_forensics.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. """Shutdown forensics — capture context when the gateway receives SIGTERM/SIGINT.
  2. ``shutdown_signal_handler`` runs synchronously inside the asyncio loop, so
  3. :func:`snapshot_shutdown_context` is a fast (<10ms) non-blocking probe and
  4. :func:`spawn_async_diagnostic` is a fire-and-forget ``ps`` walk in a detached
  5. subprocess. Anything that waits belongs in the async helper, never in the probe.
  6. """
  7. from __future__ import annotations
  8. import json
  9. import os
  10. import shutil
  11. import signal
  12. import subprocess
  13. import sys
  14. import time
  15. from pathlib import Path
  16. from typing import Any, Dict, List, Optional
  17. from gateway.restart import DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT, resolve_systemd_timeout_stop_sec
  18. import contextlib
  19. _SIGNAL_NAME_BY_NUM: dict[int, str] = {
  20. int(getattr(signal, _name)): _name
  21. for _name in ("SIGTERM", "SIGINT", "SIGHUP", "SIGQUIT", "SIGUSR1", "SIGUSR2")
  22. if getattr(signal, _name, None) is not None
  23. }
  24. def _signal_name(sig: Any) -> str:
  25. """Human-readable signal name (``str(sig)`` as fallback)."""
  26. if sig is None:
  27. return "UNKNOWN"
  28. try:
  29. sig_int = int(sig)
  30. except (TypeError, ValueError):
  31. return str(sig)
  32. return _SIGNAL_NAME_BY_NUM.get(sig_int, f"signal#{sig_int}")
  33. def _read_proc_field(pid: int, key: str) -> Optional[str]:
  34. """Read a single field from /proc/<pid>/status. Linux only; None elsewhere."""
  35. with contextlib.suppress(OSError), open(f"/proc/{pid}/status", encoding="utf-8") as fh:
  36. for line in fh:
  37. if line.startswith(key + ":"):
  38. return line.split(":", 1)[1].strip()
  39. return None
  40. def _proc_summary(pid: int) -> dict[str, Any]:
  41. """Compact /proc/<pid> identity (pid, name, state, ppid, uid). Never reads cmdline/argv —
  42. those bytes are not safe to persist (tokens, URIs, ``-e KEY=`` overlays)."""
  43. summary: dict[str, Any] = {"pid": pid}
  44. if pid <= 0:
  45. return summary
  46. for out_key, proc_key in (("name", "Name"), ("state", "State")):
  47. if (value := _read_proc_field(pid, proc_key)) is not None:
  48. summary[out_key] = value
  49. if (ppid := _read_proc_field(pid, "PPid")) is not None:
  50. with contextlib.suppress(ValueError):
  51. summary["ppid"] = int(ppid)
  52. if (uid := _read_proc_field(pid, "Uid")) is not None:
  53. summary["uid"] = uid.split()[0] if uid else uid # "real effective saved fs"
  54. return summary
  55. def _read_marker(path: Path) -> Optional[str]:
  56. """Return the marker file's text, or None if absent/unreadable."""
  57. try:
  58. return path.read_text(encoding="utf-8-sig")
  59. except OSError:
  60. return None
  61. def snapshot_shutdown_context(received_signal: Any = None) -> dict[str, Any]:
  62. """Fast (<10ms) snapshot of who/what is asking us to shut down: signal name/number, own + parent
  63. /proc summaries, systemd parentage, takeover/planned-stop markers, TracerPid, 1-min load,
  64. timestamps. Pure stdlib, never raises, never blocks."""
  65. pid, ppid = os.getpid(), os.getppid()
  66. ctx: dict[str, Any] = {
  67. "ts": time.time(), "ts_monotonic": time.monotonic(),
  68. "signal": _signal_name(received_signal),
  69. "signal_num": int(received_signal) if received_signal is not None else None,
  70. "pid": pid, "ppid": ppid, "parent": _proc_summary(ppid), "self": _proc_summary(pid),
  71. }
  72. # INVOCATION_ID is set by systemd units; ppid==1 also suggests systemd forwarded the SIGTERM.
  73. for ctx_key, env_key in (("systemd_invocation_id", "INVOCATION_ID"),
  74. ("systemd_journal_stream", "JOURNAL_STREAM")):
  75. if os.environ.get(env_key):
  76. ctx[ctx_key] = os.environ[env_key]
  77. ctx["under_systemd"] = bool(os.environ.get("INVOCATION_ID")) or ppid == 1
  78. # High load points at "something crushing the box" rather than an external killer.
  79. with contextlib.suppress(OSError, AttributeError):
  80. ctx["loadavg_1m"] = os.getloadavg()[0]
  81. # Nonzero TracerPid means a debugger/strace is attached.
  82. with contextlib.suppress(TypeError, ValueError):
  83. if (tracer := _read_proc_field(pid, "TracerPid")) is not None and tracer != "0":
  84. ctx["tracer_pid"] = int(tracer) if tracer.isdigit() else tracer
  85. ctx["tracer"] = _proc_summary(int(tracer)) if tracer.isdigit() else None
  86. # Race hint: a takeover marker on disk that does NOT name us is a smoking gun for "another
  87. # --replace instance is killing us". Filenames mirror gateway.status; literals keep the signal-
  88. # handler path import-light.
  89. with contextlib.suppress(Exception):
  90. hermes_home_str = os.path.expanduser(os.environ.get("HERMES_HOME", ""))
  91. if hermes_home_str:
  92. raw = _read_marker(Path(hermes_home_str) / ".gateway-takeover.json")
  93. if raw is not None:
  94. ctx["takeover_marker"] = raw[:300]
  95. ctx["takeover_marker_for_self"] = (f'"target_pid": {pid}' in raw
  96. or f"'target_pid': {pid}" in raw)
  97. raw = _read_marker(Path(hermes_home_str) / ".gateway-planned-stop.json")
  98. if raw is not None:
  99. ctx["planned_stop_marker"] = raw[:300]
  100. return ctx
  101. def _async_diagnostic_script(signal_name: str, self_pid: int) -> str:
  102. """POSIX listing used by the detached diagnostic. Columns are identity/resource only — no argv."""
  103. return (
  104. f"echo '=== shutdown diagnostic @ {signal_name} ==='; "
  105. "echo '--- date ---'; date -u +%Y-%m-%dT%H:%M:%SZ; "
  106. "echo '--- ps (top 60 by cpu, comm only) ---'; "
  107. # ``sort`` instead of GNU ``--sort=-pcpu`` so BSD ps (macOS) produces a listing too; the header
  108. # line is echoed first so ``sort`` does not bury it among the 0.0-cpu rows.
  109. "ps -eo pid,ppid,user,pcpu,pmem,stat,comm 2>/dev/null | { IFS= read -r h; echo \"$h\"; sort -nrk4; } | head -60; "
  110. f"echo '--- pstree of self ---'; pstree -pl {self_pid} 2>/dev/null | head -40 || true; "
  111. "echo '--- loadavg ---'; cat /proc/loadavg 2>/dev/null || sysctl -n vm.loadavg 2>/dev/null || true; "
  112. "echo '--- recent dmesg (oom/killed) ---'; "
  113. "dmesg -T 2>/dev/null | tail -20 || journalctl --user -n 20 --no-pager 2>/dev/null | tail -20 || true; "
  114. "echo '=== end ==='"
  115. )
  116. def spawn_async_diagnostic(log_path: Path, signal_name: str, *,
  117. timeout_seconds: float = 5.0) -> Optional[int]:
  118. """Fire-and-forget ``ps``-style snapshot appended to ``log_path``: a detached subprocess (own
  119. ``timeout`` so a wedged ``ps`` self-cleans) rather than a blocking process listing in the signal
  120. handler, which can freeze the loop >2s on a busy host. Returns the subprocess PID, or ``None``
  121. on failure / Windows (bash -c is available on every POSIX target; Windows has no ps anyway).
  122. The listing is comm-only: full argv is not persisted.
  123. """
  124. try:
  125. log_path.parent.mkdir(parents=True, exist_ok=True)
  126. except OSError:
  127. return None
  128. if sys.platform == "win32":
  129. return None
  130. script = _async_diagnostic_script(signal_name, os.getpid())
  131. try: # O_APPEND so concurrent diagnostics from rapid signals don't trample each other
  132. fd = os.open(str(log_path), os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o600)
  133. except OSError:
  134. return None
  135. with contextlib.suppress(OSError): # tighten logs created 0644 by earlier releases
  136. os.fchmod(fd, 0o600)
  137. # GNU ``timeout`` (Homebrew: ``gtimeout``) is absent from stock macOS; without it the detached
  138. # script still cannot block teardown, so run it unbounded rather than skip the diagnostic.
  139. timeout_bin = shutil.which("timeout") or shutil.which("gtimeout")
  140. bound = [timeout_bin, f"{timeout_seconds:.0f}"] if timeout_bin else []
  141. try: # start_new_session: outlive systemd killing our cgroup (KillMode=control-group) to flush
  142. return subprocess.Popen(
  143. [*bound, "bash", "-c", script], stdout=fd,
  144. # detached-spawn: ok — forensics dump runs while the gateway dies; bounded by timeout(1)
  145. stderr=subprocess.STDOUT, stdin=subprocess.DEVNULL, start_new_session=True,
  146. close_fds=True).pid
  147. except OSError:
  148. return None
  149. finally:
  150. with contextlib.suppress(OSError): # subprocess inherited the fd; drop our handle
  151. os.close(fd)
  152. def format_context_for_log(ctx: dict[str, Any]) -> str:
  153. """Render a shutdown context dict as one scannable log line (parent identity, never argv)."""
  154. parent = ctx.get("parent") or {}
  155. load_str = f"{load:.2f}" if isinstance(load := ctx.get("loadavg_1m"), (int, float)) else "?"
  156. extras: list[str] = []
  157. if ctx.get("takeover_marker") is not None:
  158. who = 'self' if ctx.get('takeover_marker_for_self') else 'other'
  159. extras.append(f"takeover_marker_present={who}")
  160. if ctx.get("planned_stop_marker") is not None:
  161. extras.append("planned_stop_marker_present=yes")
  162. if ctx.get("tracer_pid"):
  163. extras.append(f"tracer_pid={ctx['tracer_pid']}")
  164. extras_str = (" " + " ".join(extras)) if extras else ""
  165. return (
  166. f"signal={ctx.get('signal', '?')} under_systemd={'yes' if ctx.get('under_systemd') else 'no'} "
  167. f"parent_pid={parent.get('pid') or '?'} parent_name={parent.get('name') or '?'} "
  168. f"loadavg_1m={load_str}{extras_str}"
  169. )
  170. def context_as_json(ctx: dict[str, Any]) -> str:
  171. """JSON-serialise a context dict for structured ingestion. Never raises."""
  172. try:
  173. return json.dumps(ctx, default=str, sort_keys=True)
  174. except (TypeError, ValueError):
  175. return "{}"
  176. def check_systemd_timing_alignment(
  177. drain_timeout: float, cron_drain_timeout: float = DEFAULT_GATEWAY_CRON_DRAIN_TIMEOUT
  178. ) -> Optional[dict[str, Any]]:
  179. """At startup, sanity-check that systemd's TimeoutStopSec covers stop. A stale unit file
  180. (upgraded without re-running ``hermes setup``) can have ``TimeoutStopSec`` below the stop
  181. budget, so systemd SIGKILLs the cgroup mid-drain (a phantom ``code=killed status=9`` in the
  182. journal). ``None`` when aligned OR undeterminable (not under systemd, no ``systemctl``);
  183. otherwise a dict with ``timeout_stop_sec``/``drain_timeout``/``expected_min``/``mismatch``.
  184. """
  185. if not os.environ.get("INVOCATION_ID"):
  186. return None # Not running under systemd (or at least not directly)
  187. # /proc/self/cgroup: "0::/user.slice/.../hermes-gateway.service"
  188. unit_name: Optional[str] = None
  189. with contextlib.suppress(OSError), open("/proc/self/cgroup", encoding="utf-8") as fh:
  190. for line in fh:
  191. parts = reversed(line.strip().split("/"))
  192. unit_name = next((p for p in parts if p.endswith(".service")), None)
  193. if unit_name:
  194. break
  195. if (timeout_us := _systemd_timeout_stop_us(unit_name) if unit_name else None) is None:
  196. return None
  197. timeout_stop_sec = timeout_us / 1_000_000.0
  198. expected = float(resolve_systemd_timeout_stop_sec(drain_timeout, cron_drain_timeout))
  199. return {"unit": unit_name, "timeout_stop_sec": timeout_stop_sec, "drain_timeout": drain_timeout,
  200. "cron_drain_timeout": cron_drain_timeout, "expected_min": expected,
  201. "mismatch": timeout_stop_sec < expected}
  202. def _systemd_timeout_stop_us(unit_name: str) -> Optional[int]:
  203. """``TimeoutStopUSec`` of ``unit_name`` in microseconds; ``--user`` first (hermes' usual)."""
  204. for flag in (["--user"], []):
  205. try:
  206. result = subprocess.run(
  207. ["systemctl", *flag, "show", unit_name, "--property=TimeoutStopUSec"],
  208. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=2.0,
  209. check=False,
  210. )
  211. except (subprocess.TimeoutExpired, OSError):
  212. continue
  213. # Output: "TimeoutStopUSec=1min 30s" or "TimeoutStopUSec=90000000"
  214. for line in result.stdout.splitlines() if result.returncode == 0 else ():
  215. if line.startswith("TimeoutStopUSec="):
  216. value = line.split("=", 1)[1].strip()
  217. timeout_us = int(value) if value.isdigit() else parse_systemd_duration_to_us(value)
  218. if timeout_us is not None:
  219. return timeout_us
  220. return None
  221. def parse_systemd_duration_to_us(raw: str) -> Optional[int]:
  222. """Parse 'TimeoutStopUSec=1min 30s' / '90s' style values to microseconds. Covers us, ms, s, min,
  223. h, d, w, month, y; a bare number is seconds. None on anything unexpected; never raises. Public: also consumed by
  224. hermes_cli.gateway's restart-wait sizing.
  225. """
  226. if not raw:
  227. return None
  228. units = {"us": 1, "ms": 1_000, "s": 1_000_000, "sec": 1_000_000,
  229. "min": 60_000_000, "h": 3_600_000_000, "hr": 3_600_000_000,
  230. # Fixed systemd time-util.h constants, not variable calendar months/years.
  231. "d": 86_400_000_000, "w": 604_800_000_000,
  232. "month": 2_629_800_000_000, "y": 31_557_600_000_000}
  233. total_us, token, digits = 0, "", ""
  234. def _flush() -> bool: # fold the pending digits/token pair into total_us
  235. nonlocal total_us, token, digits
  236. multiplier = units.get(token.lower()) if token else 1_000_000
  237. if multiplier is None or not digits:
  238. return False
  239. try:
  240. total_us += int(float(digits) * multiplier)
  241. except (ValueError, OverflowError):
  242. return False
  243. digits = token = ""
  244. return True
  245. for ch in raw + " ":
  246. if ch.isdigit() or ch == ".":
  247. if token and not _flush(): # a digit after a unit ends the previous number
  248. return None
  249. digits += ch
  250. elif ch.isalpha():
  251. token += ch
  252. elif digits and not _flush():
  253. return None
  254. return total_us if total_us > 0 else None