memory_status.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112
  1. """Memory status rollup for ``/api/status``.
  2. Read side for signals the gateway already persists: the 30s ``state/gateway.heartbeat``
  3. (RSS + MemAvailable/MemTotal + swap) and the lifecycle sentinel's ``suspected_oom``
  4. flag — two small file reads, no IPC. ``/api/status`` is unauthenticated, so only
  5. coarse numbers (MB), enums and booleans. A missing/corrupt file degrades to
  6. ``pressure="unknown"`` rather than raising into the status endpoint.
  7. """
  8. from __future__ import annotations
  9. from datetime import datetime, timezone, UTC
  10. from pathlib import Path
  11. from typing import Any, Dict, Optional
  12. # Thresholds on system MemAvailable. ``critical`` doubles as the lifecycle
  13. # ledger's OOM-suspicion heuristic: a level that makes a later unclean death
  14. # "suspected OOM" already warns while the process is alive.
  15. _CRITICAL_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available
  16. _CRITICAL_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal
  17. _ELEVATED_AVAILABLE_KIB = 128 * 1024 # < 128 MiB available
  18. _ELEVATED_AVAILABLE_FRACTION = 0.15 # < 15% of MemTotal
  19. _PRESSURE_TIERS = ( # order-sensitive: worst first
  20. ("critical", _CRITICAL_AVAILABLE_KIB, _CRITICAL_AVAILABLE_FRACTION),
  21. ("elevated", _ELEVATED_AVAILABLE_KIB, _ELEVATED_AVAILABLE_FRACTION),
  22. )
  23. # Writer cadence is 30s; 150s tolerates a briefly stalled loop without letting
  24. # a long-dead gateway's last sample pose as current.
  25. _HEARTBEAT_FRESH_TTL_S = 150.0
  26. def _nonneg_int(value: Any) -> Optional[int]:
  27. """Return *value* if it is a non-negative int (bools rejected), else None."""
  28. return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
  29. def _mb(kib: Any) -> Optional[int]:
  30. return None if _nonneg_int(kib) is None else kib // 1024
  31. def _parse_iso(value: Any) -> Optional[datetime]:
  32. try:
  33. parsed = datetime.fromisoformat(value) if isinstance(value, str) and value else None
  34. except ValueError:
  35. return None
  36. return parsed.replace(tzinfo=UTC) if parsed is not None and parsed.tzinfo is None else parsed
  37. def classify_pressure(available_kib: Any, total_kib: Any) -> str:
  38. """``ok``/``elevated``/``critical`` from MemAvailable/MemTotal; ``unknown`` when the
  39. sample is missing/malformed — "could not read it" must never read as "fine"."""
  40. available, total = _nonneg_int(available_kib), _nonneg_int(total_kib)
  41. if available is None:
  42. return "unknown"
  43. fraction = available / total if total else None
  44. for level, kib_floor, frac_floor in _PRESSURE_TIERS:
  45. if available < kib_floor or (fraction is not None and fraction < frac_floor):
  46. return level
  47. return "ok"
  48. def _read_state_files(home: Optional[Path]) -> tuple:
  49. """``(heartbeat, sentinel)`` dicts, each ``None`` when unreadable."""
  50. try:
  51. from gateway.lifecycle_ledger import _read_json, get_lifecycle_sentinel_path
  52. from gateway.shutdown_watchdog import get_loop_heartbeat_path
  53. return _read_json(get_loop_heartbeat_path(home)), _read_json(get_lifecycle_sentinel_path(home))
  54. except Exception:
  55. return None, None
  56. def collect_memory_status(
  57. home: Optional[Path] = None,
  58. *,
  59. now: Optional[datetime] = None,
  60. ) -> dict[str, Any]:
  61. """``memory`` block for ``/api/status``; ``home`` scopes to a profile (``None`` =
  62. active), ``now`` is injectable. Never raises — a down gateway or corrupt files
  63. yield ``pressure="unknown"`` plus whatever fields could be recovered."""
  64. moment = now or datetime.now(UTC)
  65. status: dict[str, Any] = {
  66. "pressure": "unknown", "gateway_rss_mb": None, "system_total_mb": None, "system_available_mb": None,
  67. "swap_used_mb": None, "sampled_at": None, "last_boot_unclean": False, "last_boot_suspected_oom": False,
  68. # Identity of the CURRENT life (sentinel started_at): the dashboard keys
  69. # banner dismissal on it so acknowledging one OOM restart does not mute the NEXT.
  70. "boot_id": None,
  71. }
  72. heartbeat, sentinel = _read_state_files(home)
  73. if heartbeat:
  74. sampled_at, mem = _parse_iso(heartbeat.get("updated_at")), heartbeat.get("mem")
  75. if isinstance(mem, dict):
  76. for dst, src in (("gateway_rss_mb", "rss_kib"), ("system_total_mb", "mem_total_kib"),
  77. ("system_available_mb", "mem_available_kib"), ("swap_used_mb", "swap_used_kib")):
  78. status[dst] = _mb(mem.get(src))
  79. if sampled_at is not None:
  80. status["sampled_at"] = sampled_at.isoformat()
  81. # Stale sample: numbers still reported (sampled_at says when) but
  82. # pressure stays "unknown" so a dead gateway's final gasp cannot
  83. # render a live "critical" banner forever.
  84. if 0 <= (moment - sampled_at).total_seconds() <= _HEARTBEAT_FRESH_TTL_S:
  85. status["pressure"] = classify_pressure(mem.get("mem_available_kib"), mem.get("mem_total_kib"))
  86. if sentinel:
  87. status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit"))
  88. status["last_boot_suspected_oom"] = bool(sentinel.get("prior_suspected_oom"))
  89. started_at = sentinel.get("started_at")
  90. status["boot_id"] = started_at if isinstance(started_at, str) and started_at else None
  91. return status