agent_cache_pressure.py 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. """Memory-pressure bounds for the gateway's per-session AIAgent cache.
  2. Each cached ``AIAgent`` pins its full live transcript (tens of MB on a tool-heavy
  3. session); the LRU cap counts entries, not bytes, and the idle TTL defers eviction
  4. for busy sessions, so neither sees actual memory use. This module supplies that
  5. signal — own anonymous RSS against a budget derived from the cgroup limit — and
  6. ``GatewayRunner`` sheds LRU transcripts via soft eviction (rebuilt from the
  7. persisted session next turn). Pure/read-only; config under ``agent.agent_cache``.
  8. """
  9. from __future__ import annotations
  10. import os
  11. import sys
  12. from dataclasses import dataclass
  13. from pathlib import Path
  14. from typing import Any, Callable, Iterable, List, Optional, Tuple
  15. # Shed well under the limit: once cgroup ``memory.high`` throttling kicks in (swap full),
  16. # a SIGTERM flush cannot finish inside systemd's stop timeout.
  17. _AUTO_BUDGET_FRACTION = 0.65
  18. # Below this a budget is noise — small containers would evict every pass and never keep a warm prefix.
  19. _AUTO_BUDGET_FLOOR_MB = 512
  20. _DEFAULT_MAX_EVICTIONS_PER_PASS = 16
  21. # Never shed the hottest sessions: their prompt cache is worth the most; evicting them
  22. # just moves the cost to the next turn.
  23. _DEFAULT_PROTECT_RECENT = 8
  24. _BYTES_PER_MB = 1024 * 1024
  25. _OFF_WORDS = frozenset({"", "off", "none", "false", "disabled"})
  26. @dataclass(frozen=True)
  27. class AgentCacheBounds:
  28. """Operator-facing bounds. ``max_size``/``idle_ttl_secs`` are ``None`` when unset
  29. so ``gateway/run.py`` keeps its defaults; ``memory_high_mb`` ``None`` = pressure eviction off."""
  30. max_size: Optional[int] = None
  31. idle_ttl_secs: Optional[float] = None
  32. memory_high_mb: Optional[int] = None
  33. max_evictions_per_pass: int = _DEFAULT_MAX_EVICTIONS_PER_PASS
  34. protect_recent: int = _DEFAULT_PROTECT_RECENT
  35. def _is_int(value: Any) -> bool:
  36. return isinstance(value, int) and not isinstance(value, bool)
  37. def _positive(value: Any, cast: Callable[[Any], Any] = int) -> Any:
  38. """``cast(value)`` if it is a positive number (bools rejected), else None."""
  39. try:
  40. parsed = None if isinstance(value, bool) or value is None else cast(value)
  41. except (TypeError, ValueError):
  42. return None
  43. return parsed if parsed is not None and parsed > 0 else None
  44. def _finite_limit(path: Path) -> Optional[int]:
  45. """A cgroup memory limit file's value when it is a real cap; None for unreadable, empty,
  46. ``max``, or the v1 near-2^63 sentinel (all mean unlimited)."""
  47. try:
  48. limit = int(path.read_text(encoding="utf-8-sig").strip())
  49. except (OSError, ValueError):
  50. return None
  51. return limit if 0 < limit < (1 << 62) else None
  52. def _cgroup_limit_bytes() -> Optional[int]:
  53. """Memory limit this process runs under, if cgroup-capped.
  54. Prefers v2 ``memory.high`` (the throttling point) over ``memory.max``, then v1.
  55. Own cgroup first (where a systemd unit's ``MemoryHigh=``/``MemoryMax=`` lands —
  56. root reads ``max`` there), then root for container-style limits.
  57. """
  58. if sys.platform != "linux":
  59. return None
  60. try:
  61. from gateway.cgroup_cleanup import _own_cgroup_path
  62. own = _own_cgroup_path()
  63. except Exception:
  64. own = None
  65. roots = ([f"/sys/fs/cgroup{own}"] if own and own != "/" else []) + ["/sys/fs/cgroup"]
  66. for candidate in [f"{r}/memory.{f}" for r in roots for f in ("high", "max")] + ["/sys/fs/cgroup/memory/memory.limit_in_bytes"]:
  67. limit = _finite_limit(Path(candidate))
  68. if limit is not None:
  69. return limit
  70. return None
  71. def _total_memory_bytes() -> Optional[int]:
  72. try:
  73. return int(os.sysconf("SC_PAGE_SIZE")) * int(os.sysconf("SC_PHYS_PAGES"))
  74. except (OSError, ValueError, AttributeError):
  75. pass
  76. try:
  77. import psutil # type: ignore
  78. return int(psutil.virtual_memory().total)
  79. except Exception:
  80. return None
  81. def resolve_memory_high_mb(setting: Any) -> Optional[int]:
  82. """Absolute MB budget: ``"auto"`` derives from the cgroup limit (or total RAM when
  83. uncapped); a positive number is literal; anything falsy/off disables the pass."""
  84. if isinstance(setting, str):
  85. normalized = setting.strip().lower()
  86. if normalized != "auto":
  87. return None if normalized in _OFF_WORDS else _positive(normalized)
  88. elif setting is False:
  89. return None
  90. elif setting is not True:
  91. return _positive(setting)
  92. limit = _cgroup_limit_bytes() or _total_memory_bytes()
  93. if not limit:
  94. return None
  95. budget = int(limit * _AUTO_BUDGET_FRACTION / _BYTES_PER_MB)
  96. return budget if budget >= _AUTO_BUDGET_FLOOR_MB else None
  97. def resolve_agent_cache_bounds(config: Any) -> AgentCacheBounds:
  98. """Read ``agent.agent_cache`` from the *raw* config: the gateway loader does not
  99. deep-merge ``DEFAULT_CONFIG``, so callers can tell "operator chose 128" from "unset"."""
  100. section = (config.get("agent") or {}).get("agent_cache") if isinstance(config, dict) else None
  101. if not isinstance(section, dict):
  102. section = {}
  103. protect_recent = section.get("protect_recent")
  104. protect_parsed = _positive(protect_recent)
  105. # 0 means "shed anything" — distinct from unset. The bool guard keeps `protect_recent: false`
  106. # (False == 0) on the default instead of silently disabling MRU protection.
  107. if protect_parsed is None and _is_int(protect_recent) and protect_recent == 0:
  108. protect_parsed = 0
  109. return AgentCacheBounds(
  110. max_size=_positive(section.get("max_size")),
  111. idle_ttl_secs=_positive(section.get("idle_ttl_secs"), float),
  112. memory_high_mb=resolve_memory_high_mb(section.get("memory_high_mb", "auto")),
  113. max_evictions_per_pass=_positive(section.get("max_evictions_per_pass")) or _DEFAULT_MAX_EVICTIONS_PER_PASS,
  114. protect_recent=_DEFAULT_PROTECT_RECENT if protect_parsed is None else protect_parsed,
  115. )
  116. def _cgroup_anon_bytes() -> Optional[int]:
  117. """Anonymous memory charged to this process's own cgroup v2 (``memory.stat`` ``anon``), or None.
  118. The budget is derived from the same cgroup's ``memory.high``/``memory.max``, and the kernel
  119. charges every process in the unit against it — execute_code kernels, terminal children — so a
  120. self-only reading under-counts by exactly the children's share (#110549). Anon, not
  121. ``memory.current``: the module's signal is heap, and reclaimable page cache is noise.
  122. Only for a *capped* cgroup: an uncapped one (a plain login session) is the whole user
  123. slice, and its budget is total RAM — self RSS stays the right scope there.
  124. """
  125. if sys.platform != "linux":
  126. return None
  127. try:
  128. from gateway.cgroup_cleanup import _own_cgroup_path
  129. own = _own_cgroup_path()
  130. if not own or own == "/":
  131. return None
  132. root = Path(f"/sys/fs/cgroup{own}")
  133. capped = any(_finite_limit(root / f"memory.{f}") for f in ("high", "max"))
  134. text = root.joinpath("memory.stat").read_text(encoding="utf-8-sig") if capped else ""
  135. except (OSError, ValueError, ImportError):
  136. return None
  137. for line in text.splitlines():
  138. key, _, value = line.partition(" ")
  139. if key == "anon" and value.strip().isdigit():
  140. return int(value)
  141. return None
  142. def read_anon_rss_mb() -> Optional[int]:
  143. """Anonymous memory in MB (where cached transcripts live; file-backed pages are noise),
  144. or None. Own cgroup's ``memory.stat`` anon first — the scope the budget is charged
  145. against, so same-unit child processes count; then ``/proc/self/status``; psutil covers
  146. other platforms (total RSS only)."""
  147. charged = _cgroup_anon_bytes()
  148. if charged:
  149. return charged // _BYTES_PER_MB
  150. try:
  151. from hermes_cli.mem_trim import collect_memory_snapshot
  152. snapshot = collect_memory_snapshot()
  153. for key in ("rss_anon_kib", "rss_kib"):
  154. kib = snapshot.get(key)
  155. if isinstance(kib, int) and kib > 0:
  156. return kib // 1024
  157. except Exception:
  158. pass
  159. try:
  160. import psutil # type: ignore
  161. return int(psutil.Process(os.getpid()).memory_info().rss / _BYTES_PER_MB)
  162. except Exception:
  163. return None
  164. def transcript_persistence_caught_up(agent: Any) -> bool:
  165. """True when the live transcript is fully on disk.
  166. Soft eviction rebuilds from the persisted session, so it is only safe once
  167. ``_last_flushed_db_idx`` (advanced only on a fully successful write) has caught
  168. up. Unknown shapes are *not* caught up: a skipped eviction costs memory, a
  169. wrong one costs the conversation.
  170. """
  171. messages, flushed = getattr(agent, "_session_messages", None), getattr(agent, "_last_flushed_db_idx", None)
  172. return isinstance(messages, list) and _is_int(flushed) and flushed >= len(messages)
  173. def plan_pressure_evictions(
  174. ordered_entries: Iterable[tuple[str, Any]], *, is_evictable: Callable[[str, Any], bool],
  175. max_evictions: int, protect_recent: int = 0,
  176. ) -> list[tuple[str, Any]]:
  177. """Choose which cached sessions to shed, least-recently-used first.
  178. ``ordered_entries`` must be LRU→MRU (the cache OrderedDict ``move_to_end``s on
  179. every hit). The batch is capped so one pass cannot stall the gateway.
  180. ``protect_recent`` is clamped to half the cache: a few huge transcripts can
  181. exhaust the budget alone, and a fixed guard would leave nothing to shed.
  182. """
  183. entries = list(ordered_entries)
  184. if max_evictions <= 0 or not entries:
  185. return []
  186. protect = min(max(protect_recent, 0), len(entries) // 2)
  187. if protect:
  188. entries = entries[:-protect]
  189. plan: list[tuple[str, Any]] = []
  190. for key, agent in entries:
  191. if len(plan) >= max_evictions:
  192. break
  193. if is_evictable(key, agent):
  194. plan.append((key, agent))
  195. return plan