api_server_memory_sessions.py 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146
  1. """Memory-provider continuity across api_server requests (#120116).
  2. The api_server adapter builds a fresh ``AIAgent`` per request (per-request callbacks, model
  3. route, ephemeral prompt), unlike the messaging platforms, whose cached agent — and with it the
  4. memory provider — lives for the whole session. External providers deliver recall as the
  5. PREVIOUS turn's background prefetch held on the provider instance, so a provider that is
  6. re-initialised per request never has anything to inject, and each init re-runs the provider's
  7. startup (for an embedded daemon: a restart that also kills the retain still in flight).
  8. This registry keeps one initialised ``MemoryManager`` per (profile home, session id): a request
  9. checks the session's manager out before building its agent (``AIAgent(memory_manager=...)``
  10. skips provider init) and checks it back in when the turn ends. Check-out is exclusive, so two
  11. concurrent requests on one session never share a manager; the loser's fresh manager is shut down
  12. when it checks in behind the winner. Idle entries and LRU overflow are shut down under the owning
  13. profile's scope, like the gateway agent cache's eviction.
  14. """
  15. from __future__ import annotations
  16. import logging
  17. import threading
  18. import time
  19. from collections import OrderedDict
  20. from contextlib import nullcontext, suppress
  21. from pathlib import Path
  22. from typing import Any, Dict, List, Optional, Tuple
  23. logger = logging.getLogger(__name__)
  24. class ApiServerMemorySessions:
  25. """Session-keyed ``MemoryManager`` registry with exclusive check-out/check-in."""
  26. def __init__(self, *, max_size: Optional[int] = None, idle_ttl_secs: Optional[float] = None) -> None:
  27. # (manager, owning home, last check-in on the monotonic clock, the same on the wall clock)
  28. self._entries: OrderedDict[tuple[str, str], tuple[Any, Optional[Path], float, float]] = OrderedDict()
  29. self._lock = threading.Lock()
  30. self._max_size = max_size
  31. self._idle_ttl_secs = idle_ttl_secs
  32. # -- bounds (same knobs as the gateway agent cache, resolved lazily) -------------------------
  33. def _bounds(self) -> tuple[int, float]:
  34. if self._max_size is None or self._idle_ttl_secs is None:
  35. from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS, _AGENT_CACHE_MAX_SIZE, _load_gateway_config
  36. from gateway.agent_cache_pressure import resolve_agent_cache_bounds
  37. configured = None
  38. with suppress(Exception):
  39. configured = resolve_agent_cache_bounds(_load_gateway_config())
  40. if self._max_size is None:
  41. self._max_size = getattr(configured, "max_size", None) or _AGENT_CACHE_MAX_SIZE
  42. if self._idle_ttl_secs is None:
  43. self._idle_ttl_secs = getattr(configured, "idle_ttl_secs", None) or _AGENT_CACHE_IDLE_TTL_SECS
  44. return self._max_size, self._idle_ttl_secs
  45. @staticmethod
  46. def _owner_home() -> tuple[str, Optional[Path]]:
  47. """(registry key, profile home to re-enter on eviction) for the CURRENT scope. Callers run
  48. inside ``_profile_scope`` (or a single-profile gateway), so the ambient home is the owner's."""
  49. from hermes_constants import get_hermes_home, hermes_home_key
  50. home = Path(get_hermes_home())
  51. return hermes_home_key(home), home
  52. # -- check-out / check-in ----------------------------------------------------------------
  53. def checkout(self, session_id: Optional[str]) -> Optional[Any]:
  54. """The manager a previous request on ``session_id`` checked in, or None (build a new one).
  55. One checked in before the retry horizon is shut down instead: a housekeeping sweep's fence
  56. on an id ages out at that horizon, so the id may now name a new session, which must not
  57. get its previous session's recall."""
  58. if not session_id:
  59. return None
  60. home_key, _home = self._owner_home()
  61. with self._lock:
  62. entry = self._entries.pop((home_key, session_id), None)
  63. if entry is None:
  64. return None
  65. from hermes_state_mutation_retirement import RETRY_HORIZON_S
  66. if time.time() - entry[3] > RETRY_HORIZON_S:
  67. self._shutdown_async(entry[0], entry[1])
  68. return None
  69. return entry[0]
  70. def checkin(self, agent: Any) -> None:
  71. """Park ``agent``'s manager under the session the turn ended on (``agent.session_id`` carries a
  72. mid-turn compression rotation) and shut down whatever this displaces or has gone idle."""
  73. manager = getattr(agent, "_memory_manager", None)
  74. session_id = str(getattr(agent, "session_id", "") or "")
  75. if manager is None or not session_id:
  76. return
  77. home_key, home = self._owner_home()
  78. max_size, idle_ttl = self._bounds()
  79. now = time.monotonic()
  80. doomed: list[tuple[Any, Optional[Path]]] = []
  81. with self._lock:
  82. displaced = self._entries.pop((home_key, session_id), None)
  83. if displaced is not None and displaced[0] is not manager:
  84. doomed.append((displaced[0], displaced[1]))
  85. self._entries[(home_key, session_id)] = (manager, home, now, time.time())
  86. for key, (mgr, owner, last_used, _wall) in list(self._entries.items()):
  87. if mgr is manager:
  88. continue
  89. if now - last_used > idle_ttl or len(self._entries) > max_size:
  90. del self._entries[key]
  91. doomed.append((mgr, owner))
  92. for mgr, owner in doomed:
  93. self._shutdown_async(mgr, owner)
  94. def close_all(self) -> None:
  95. """Adapter shutdown: drain and shut down every parked manager (inline: the process is ending)."""
  96. with self._lock:
  97. entries = list(self._entries.values())
  98. self._entries.clear()
  99. for mgr, owner, *_ in entries:
  100. self._shutdown(mgr, owner)
  101. # -- teardown -------------------------------------------------------------------------------
  102. def _shutdown_async(self, manager: Any, owner: Optional[Path]) -> None:
  103. """Eviction runs inside a request's own turn: never make that reply wait on a provider drain."""
  104. from agent.memory_provider import spawn_context_thread
  105. spawn_context_thread(self._shutdown, args=(manager, owner), name="api-server-memory-evict").start()
  106. @staticmethod
  107. def _shutdown(manager: Any, owner: Optional[Path]) -> None:
  108. """Bounded drain then provider shutdown, under the OWNING profile's scope: eviction runs inside
  109. whichever request happened to trigger it, and a provider reads its home/credentials at call time."""
  110. scope: Any = nullcontext()
  111. with suppress(Exception):
  112. from agent.secret_scope import is_multiplex_active
  113. if owner is not None and is_multiplex_active():
  114. from gateway.run import _profile_runtime_scope
  115. scope = _profile_runtime_scope(owner)
  116. try:
  117. with scope:
  118. with suppress(Exception):
  119. manager.flush_pending(timeout=10)
  120. manager.shutdown_all()
  121. except Exception:
  122. logger.debug("api_server memory manager shutdown failed", exc_info=True)
  123. # -- introspection (tests) --------------------------------------------------------------------
  124. def parked(self) -> dict[tuple[str, str], Any]:
  125. with self._lock:
  126. return {key: entry[0] for key, entry in self._entries.items()}