memory_monitor.py 4.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111
  1. """Periodic process memory usage logging for the gateway.
  2. Emits one grep-friendly ``[MEMORY] ...`` line every N seconds (default 300)
  3. from a daemon thread so slow leaks show up as an RSS time series in the logs.
  4. A baseline snapshot is logged on start and a final one on stop. Uses stdlib
  5. ``resource`` first, ``psutil`` as fallback (Windows); if neither works the
  6. monitor warns once and stays disabled.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import gc
  11. import logging
  12. import os
  13. import sys
  14. import threading
  15. import time
  16. from typing import Optional
  17. logger = logging.getLogger(__name__)
  18. _BYTES_TO_MB = 1024 * 1024
  19. _monitor_thread: Optional[threading.Thread] = None
  20. _stop_event: Optional[threading.Event] = None
  21. _start_time: Optional[float] = None
  22. _lock = threading.Lock()
  23. def _get_rss_mb() -> Optional[int]:
  24. """Return process RSS high-water mark in MB, or None if unavailable."""
  25. try:
  26. import resource
  27. # ru_maxrss is KB on Linux but bytes on macOS.
  28. return int(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / (_BYTES_TO_MB if sys.platform == "darwin" else 1024))
  29. except Exception:
  30. pass
  31. try:
  32. import psutil # type: ignore
  33. return int(psutil.Process(os.getpid()).memory_info().rss / _BYTES_TO_MB)
  34. except Exception:
  35. return None
  36. def log_memory_usage(prefix: str = "") -> None:
  37. """Log ``[MEMORY] [<prefix> ]rss=... gc=... threads=... uptime=...``; safe from any thread."""
  38. rss = _get_rss_mb()
  39. logger.info(
  40. "[MEMORY] %srss=%s gc=%s threads=%d uptime=%ds", f"{prefix} " if prefix else "",
  41. "unavailable" if rss is None else f"{rss}MB", gc.get_count(), threading.active_count(),
  42. int(time.monotonic() - _start_time) if _start_time else 0,
  43. )
  44. def _monitor_loop(stop_event: threading.Event, interval: float) -> None:
  45. """Background thread body — log every ``interval`` seconds until stopped."""
  46. while not stop_event.wait(interval):
  47. try:
  48. log_memory_usage()
  49. except Exception as e: # never let the monitor crash the gateway
  50. logger.debug("Memory monitor iteration failed: %s", e)
  51. def start_memory_monitoring(interval_seconds: float = 300.0) -> bool:
  52. """Start periodic logging in a daemon thread (baseline logged immediately). False if
  53. already running or RSS introspection is unavailable (warned once)."""
  54. global _monitor_thread, _stop_event, _start_time
  55. with _lock:
  56. if _monitor_thread is not None and _monitor_thread.is_alive():
  57. return False
  58. if _get_rss_mb() is None:
  59. logger.warning(
  60. "[MEMORY] Memory monitoring unavailable: neither resource.getrusage nor psutil could read process RSS "
  61. "— skipping periodic logging.",
  62. )
  63. return False
  64. _start_time = time.monotonic()
  65. _stop_event = threading.Event()
  66. log_memory_usage(prefix="baseline")
  67. _monitor_thread = threading.Thread(
  68. target=_monitor_loop, args=(_stop_event, float(interval_seconds)), name="gateway-memory-monitor", daemon=True
  69. )
  70. _monitor_thread.start()
  71. logger.info("[MEMORY] Periodic memory monitoring started (interval: %ds)", int(interval_seconds))
  72. return True
  73. def stop_memory_monitoring(timeout: float = 2.0) -> None:
  74. """Stop the monitor thread and log a final ``shutdown`` snapshot. No-op if never started."""
  75. global _monitor_thread, _stop_event
  76. with _lock:
  77. if _stop_event is None or _monitor_thread is None:
  78. return
  79. with contextlib.suppress(Exception):
  80. log_memory_usage(prefix="shutdown")
  81. _stop_event.set()
  82. thread, _monitor_thread, _stop_event = _monitor_thread, None, None
  83. with contextlib.suppress(Exception): # join outside the lock so a stuck log call can't deadlock the stop path
  84. thread.join(timeout=timeout)
  85. logger.info("[MEMORY] Periodic memory monitoring stopped")
  86. def is_running() -> bool:
  87. """True if the background monitor thread is alive."""
  88. with _lock:
  89. return _monitor_thread is not None and _monitor_thread.is_alive()