mem_trim.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. """Rate-limited heap release for long-lived Hermes gateway processes.
  2. On Linux/glibc, ``malloc_trim(0)`` can return pages from freed Python/C allocations to the OS. Every
  3. platform runs the rate-limited ``gc.collect()``; only the page release is glibc-only. Behavior is configured
  4. under ``context.memory_trim`` in ``config.yaml``.
  5. """
  6. from __future__ import annotations
  7. import ctypes
  8. import gc
  9. import logging
  10. import platform
  11. import sys
  12. import threading
  13. import time
  14. from collections.abc import Callable
  15. from pathlib import Path
  16. from typing import Any
  17. logger = logging.getLogger(__name__)
  18. _DEFAULT_COOLDOWN_SECONDS = 60.0
  19. _DEFAULT_LOG_EVERY_N = 1
  20. _DEFAULT_INFO_LOG_MIN_DELTA_MB = 0.0
  21. # Even forced trims honor a short floor: AIAgent.close() forces a trim, and delegate
  22. # batches close N child subagents back-to-back in the SAME process — without a floor
  23. # that stacks N+1 uncooled full gc.collect() passes (50-500ms each in a large gateway
  24. # process). 5s coalesces the burst while keeping the parent's final close-trim effective.
  25. _FORCE_FLOOR_SECONDS = 5.0
  26. _trim_lock = threading.Lock()
  27. _last_trim_monotonic = 0.0
  28. # ``_last_trim_monotonic`` stamps collecting trims only, so an allocator-only trim (housekeeping tick, agent
  29. # close) never pushes a later forced collection (cache-pressure eviction) inside its floor.
  30. _last_alloc_trim_monotonic = 0.0
  31. _probe_done = False
  32. _malloc_trim: Callable[[int], int] | None = None
  33. _trim_call_count = 0
  34. def _config_settings() -> tuple[bool, float, int, float]:
  35. """Return fail-open ``(enabled, cooldown, log_every_n, info_log_min_delta_mb)`` from config."""
  36. settings: Any = None
  37. try:
  38. # Read-only, no-deepcopy variant: this runs on EVERY trim attempt (before the
  39. # cooldown check), and a full-config deepcopy per attempt is exactly the
  40. # allocator garbage this module exists to release.
  41. from hermes_cli.config import load_config_readonly
  42. config = load_config_readonly() or {}
  43. context = config.get("context") if isinstance(config, dict) else None
  44. settings = context.get("memory_trim") if isinstance(context, dict) else None
  45. except Exception:
  46. pass
  47. if not isinstance(settings, dict):
  48. settings = {}
  49. enabled = settings["enabled"] if isinstance(settings.get("enabled"), bool) else True
  50. return (
  51. enabled,
  52. _cooldown_seconds(settings.get("cooldown_seconds")),
  53. _coerce(settings.get("log_every_n"), _DEFAULT_LOG_EVERY_N, int, 1),
  54. _coerce(settings.get("info_log_min_delta_mb"), _DEFAULT_INFO_LOG_MIN_DELTA_MB, float, 0.0))
  55. def _coerce(value: Any, default, cast, floor):
  56. """``cast(value)`` clamped to ``floor``; bools and unparseable values fall back to ``default``."""
  57. if isinstance(value, bool):
  58. return default
  59. try:
  60. return max(floor, cast(value))
  61. except (TypeError, ValueError):
  62. return default
  63. def _cooldown_seconds(value: Any) -> float:
  64. return _coerce(value, _DEFAULT_COOLDOWN_SECONDS, float, 0.0)
  65. def _read_proc_status() -> str | None:
  66. """Read Linux process status without making non-Linux callers special-case."""
  67. if sys.platform != "linux":
  68. return None
  69. try:
  70. return Path("/proc/self/status").read_text(encoding="utf-8")
  71. except OSError:
  72. return None
  73. def collect_memory_snapshot(history_bytes: int | None = None) -> dict[str, int | None]:
  74. """Lightweight process-memory telemetry for trim logs and canaries.
  75. ``VmRSS`` / ``RssAnon`` are Linux-only best effort; deliberately psutil-free.
  76. """
  77. snapshot: dict[str, int | None] = {
  78. "rss_kib": None, "rss_anon_kib": None, "thread_count": threading.active_count()}
  79. status = _read_proc_status()
  80. if status:
  81. for line in status.splitlines():
  82. key, separator, raw_value = line.partition(":")
  83. if not separator or key not in {"VmRSS", "RssAnon"}:
  84. continue
  85. value = raw_value.strip().split(maxsplit=1)
  86. if value and value[0].isdigit():
  87. snapshot["rss_kib" if key == "VmRSS" else "rss_anon_kib"] = int(value[0])
  88. if isinstance(history_bytes, int) and history_bytes >= 0:
  89. snapshot["history_bytes"] = history_bytes
  90. return snapshot
  91. def _should_log_trim(
  92. *, force: bool, log_every_n: int, call_count: int, before: dict[str, int | None],
  93. after: dict[str, int | None], info_log_min_delta_mb: float) -> bool:
  94. # Called only after malloc_trim reported success; a forced successful trim is an
  95. # explicit observability event regardless of RSS.
  96. if force:
  97. return True
  98. if call_count % log_every_n:
  99. return False
  100. before_rss = before.get("rss_kib")
  101. after_rss = after.get("rss_kib")
  102. if before_rss is None or after_rss is None:
  103. return True
  104. return abs(after_rss - before_rss) >= info_log_min_delta_mb * 1024
  105. def _probe_glibc_malloc_trim() -> Callable[[int], int] | None:
  106. """Resolve glibc's malloc_trim once; return None on unsupported systems."""
  107. global _malloc_trim, _probe_done
  108. if _probe_done:
  109. return _malloc_trim
  110. _probe_done = True
  111. if sys.platform != "linux":
  112. return None
  113. try:
  114. if platform.libc_ver()[0].lower() != "glibc":
  115. return None
  116. trim = ctypes.CDLL(None).malloc_trim
  117. trim.argtypes = [ctypes.c_size_t]
  118. trim.restype = ctypes.c_int
  119. _malloc_trim = trim
  120. except Exception as exc:
  121. logger.debug("malloc_trim unavailable: %s", exc)
  122. return _malloc_trim
  123. def trim_memory(
  124. *, force: bool = False, reason: str = "", cooldown_seconds: float | None = None,
  125. collect: bool = True) -> bool:
  126. """Collect cycles and ask glibc to release free heap pages.
  127. Returns ``True`` only when ``malloc_trim(0)`` ran and reported success. Unsupported allocators,
  128. the config kill switch, cooldown suppression, and all runtime errors return ``False`` without
  129. affecting the caller.
  130. ``collect=False`` drops the forced full ``gc.collect()`` and only asks the allocator to return
  131. already-free pages. Every call site on a busy path of a long-lived process passes it (gateway
  132. housekeeping tick, agent close, TUI turn completion, post-compression): a full collection walks
  133. the whole shared heap with the GIL held, 250-750 ms on a multi-million-object gateway, freezing
  134. every thread (adapters, streams, API requests) while it runs. Most of what those paths free goes
  135. by reference counting; cycles wait for ``trim_memory_when_idle``, the timer lanes' one full
  136. collection per quiet period. The forced collection otherwise stays only where memory is the
  137. emergency (agent-cache pressure eviction) or the process is single-session (slash worker).
  138. """
  139. enabled, configured_cooldown, log_every_n, info_log_min_delta_mb = _config_settings()
  140. if not enabled:
  141. return False
  142. global _last_trim_monotonic, _last_alloc_trim_monotonic, _trim_call_count
  143. with _trim_lock:
  144. trim = _probe_glibc_malloc_trim()
  145. now = time.monotonic()
  146. cooldown = configured_cooldown if cooldown_seconds is None else _cooldown_seconds(cooldown_seconds)
  147. last = _last_trim_monotonic if collect else max(_last_trim_monotonic, _last_alloc_trim_monotonic)
  148. if last and now - last < (_FORCE_FLOOR_SECONDS if force else cooldown):
  149. return False
  150. # Record the attempt before calling into libc so repeated failures do not
  151. # turn every turn boundary into an expensive full collection.
  152. _last_alloc_trim_monotonic = now
  153. if collect:
  154. _last_trim_monotonic = now
  155. try:
  156. before = collect_memory_snapshot()
  157. started = time.perf_counter()
  158. if collect:
  159. gc.collect()
  160. # Cycle collection is allocator-independent; only the page release needs glibc.
  161. trim_result = trim(0) if trim is not None else 0
  162. released = bool(trim_result)
  163. after = collect_memory_snapshot()
  164. duration_ms = (time.perf_counter() - started) * 1000
  165. _trim_call_count += 1
  166. if released and _should_log_trim(
  167. force=force, log_every_n=log_every_n, call_count=_trim_call_count,
  168. before=before, after=after, info_log_min_delta_mb=info_log_min_delta_mb):
  169. logger.info(
  170. "memory trim: reason=%s malloc_trim=%s collected=%s rss_kib=%s->%s "
  171. "rss_anon_kib=%s->%s threads=%s duration_ms=%.1f",
  172. reason or "cleanup", trim_result, collect,
  173. before.get("rss_kib"), after.get("rss_kib"),
  174. before.get("rss_anon_kib"), after.get("rss_anon_kib"),
  175. after.get("thread_count"), duration_ms)
  176. return released
  177. except Exception as exc:
  178. logger.warning(
  179. "memory trim failed after %s: %s: %s", reason or "cleanup", type(exc).__name__, exc)
  180. return False
  181. # A gateway that has been turn-free this long has nobody waiting on it, so the full collection the busy-path
  182. # trims skip can run here without freezing anyone. Python 3.14's incremental collector advances only on new
  183. # allocations, so a quiet process otherwise keeps closed agents' reference cycles (and their pages) forever.
  184. IDLE_COLLECT_AFTER_SECONDS = 300.0
  185. _idle_collect_lock = threading.Lock()
  186. _idle_collected_for: float | None = None # quiet-period stamp the last idle collection covered
  187. _agent_closed_since_idle_collect = False
  188. def note_agent_closed() -> None:
  189. """A closed agent may leave reference cycles: re-arm the idle collection even within one quiet period."""
  190. global _agent_closed_since_idle_collect
  191. _agent_closed_since_idle_collect = True
  192. def trim_memory_when_idle(*, reason: str) -> bool:
  193. """Timer-lane trim: allocator release only, plus ONE full collection per quiet period once no turn has run
  194. in this process for ``IDLE_COLLECT_AFTER_SECONDS`` (again only after a later agent close)."""
  195. global _idle_collected_for, _agent_closed_since_idle_collect
  196. from agent.review_idle_queue import QUEUE
  197. quiet_for, quiet_since = QUEUE.quiet_period()
  198. with _idle_collect_lock:
  199. due = quiet_since is not None and quiet_for >= IDLE_COLLECT_AFTER_SECONDS and (
  200. quiet_since != _idle_collected_for or _agent_closed_since_idle_collect)
  201. if due:
  202. _idle_collected_for, _agent_closed_since_idle_collect = quiet_since, False
  203. if not due or not _config_settings()[0]:
  204. return trim_memory(reason=reason, collect=False)
  205. started = time.perf_counter()
  206. freed = gc.collect()
  207. logger.info("idle full collection: reason=%s quiet_s=%d unreachable=%d duration_ms=%.1f",
  208. reason, quiet_for, freed, (time.perf_counter() - started) * 1000)
  209. return trim_memory(force=True, reason=f"{reason} after idle collection", collect=False)