run_watchers.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. """Session housekeeping / stall / catalog-refresh watcher loops, bound onto ``GatewayRunner`` via the MRO.
  2. ``gateway.run`` internals are imported lazily inside method bodies (import cycle), so
  3. ``patch("gateway.run.X")`` keeps intercepting them at call time.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import contextlib
  8. import logging
  9. import time
  10. from collections import Counter
  11. from typing import Any, Dict, Optional
  12. from gateway.session_stall import (
  13. format_session_stall_notification,
  14. resolve_session_idle_seconds_from_activity,
  15. should_clear_session_stall_notification,
  16. should_emit_session_stall_notification,
  17. )
  18. # Log-record parity with the origin module.
  19. logger = logging.getLogger("gateway.run")
  20. _SESSION_STORE_PRUNE_INTERVAL = 3600.0 # once per hour
  21. async def _interruptible_sleep(runner, seconds: int) -> None:
  22. """Sleep in 1s increments so the watcher stops quickly when ``runner._running`` flips."""
  23. for _ in range(seconds):
  24. if not runner._running:
  25. break
  26. await asyncio.sleep(1)
  27. class GatewaySessionWatchersMixin:
  28. """Session housekeeping / stall / catalog-refresh watcher loops for GatewayRunner."""
  29. async def _session_housekeeping_watcher(self, interval: int = 300):
  30. """Reclaim resources without ending durable conversations."""
  31. await asyncio.sleep(60)
  32. while self._running:
  33. try:
  34. await self._session_housekeeping()
  35. except Exception as e:
  36. logger.debug("Session housekeeping error: %s", e)
  37. await _interruptible_sleep(self, interval)
  38. async def _session_housekeeping(self) -> None:
  39. """Idle/pressure agent-cache sweeps plus the hourly SessionStore prune."""
  40. try:
  41. if evicted := self._sweep_idle_cached_agents():
  42. logger.info("Agent cache idle sweep: evicted %d agent(s)", evicted)
  43. except Exception as e:
  44. logger.debug("Idle agent sweep failed: %s", e)
  45. # Neither LRU cap nor idle TTL knows what a cached transcript costs in memory.
  46. try:
  47. # Neither the LRU cap nor the idle TTL is aware of how much memory a cached transcript costs, so
  48. # a busy gateway keeps every warm session's tool output resident until RSS hits the cgroup limit
  49. # (#80764). Shed LRU transcripts once the heap is over budget; they reload from the persisted
  50. # session on the next turn.
  51. self._sweep_agent_cache_under_pressure()
  52. except Exception as e:
  53. logger.debug("Agent cache pressure sweep failed: %s", e)
  54. # Prune stale SessionStore entries: the dict + sessions.json otherwise grow unbounded.
  55. prune_ts = getattr(self, "_last_session_store_prune_ts", 0.0) # tests may omit
  56. if time.time() - prune_ts > _SESSION_STORE_PRUNE_INTERVAL:
  57. try:
  58. max_age = int(getattr(self.config, "session_store_max_age_days", 0) or 0)
  59. if max_age > 0 and (n := await self.async_session_store.prune_old_entries(max_age)):
  60. logger.info("SessionStore prune: dropped %d stale entries", n)
  61. except Exception as e:
  62. logger.debug("SessionStore prune failed: %s", e)
  63. self._last_session_store_prune_ts = time.time()
  64. def _session_stall_timeout_seconds(self) -> float:
  65. """Return configured stall timeout (seconds); 0 disables the watchdog."""
  66. from gateway.run import _float_env
  67. return _float_env("HERMES_SESSION_STALL_TIMEOUT", 300)
  68. def _session_activity_for_stall(self, session_key: str) -> Optional[dict]:
  69. """Stall-progress snapshot from ``AIAgent.get_activity_summary()`` only; no other clocks.
  70. See #72039.
  71. """
  72. from gateway.run import _AGENT_PENDING_SENTINEL
  73. agent = (getattr(self, "_running_agents", None) or {}).get(session_key)
  74. if agent is None or agent is _AGENT_PENDING_SENTINEL:
  75. return None
  76. try:
  77. summary = agent.get_activity_summary()
  78. except Exception: # incl. AttributeError: agent without an activity summary
  79. return None
  80. return summary if isinstance(summary, dict) else None
  81. def _stall_candidates(self) -> dict[str, tuple[Any, Any]]:
  82. """session_key -> (adapter, pending event) from every live adapter's pending slot (default
  83. + multiplex profiles, deduped by identity), then the overflow queues; first one wins."""
  84. candidates: dict[str, tuple[Any, Any]] = {}
  85. maps = (getattr(self, "adapters", {}), *getattr(self, "_profile_adapters", {}).values())
  86. adapters = {id(a): a for m in maps for a in list(m.values()) if a is not None}
  87. for adapter in adapters.values():
  88. pending = getattr(adapter, "_pending_messages", None) or {}
  89. for session_key, event in list(pending.items()):
  90. if session_key and session_key not in candidates and event is not None:
  91. candidates[session_key] = (adapter, event)
  92. for session_key, overflow in list((getattr(self, "_queued_events", None) or {}).items()):
  93. if not session_key or session_key in candidates or not overflow:
  94. continue
  95. source = getattr(overflow[0], "source", None)
  96. if source is not None and (adapter := self._delivery_adapter_for(source)) is not None:
  97. candidates[session_key] = (adapter, overflow[0])
  98. return candidates
  99. async def _check_session_stalls(self, timeout_seconds: float) -> int:
  100. """Notify once per stall episode for pending inbound sessions; returns notices sent."""
  101. if getattr(self, "_session_stall_notified", None) is None: # tests may build bare runners
  102. self._session_stall_notified = {}
  103. notified_map = self._session_stall_notified
  104. sent, now, candidates = 0, time.time(), self._stall_candidates()
  105. # Every candidate carries a non-None pending event, so has_pending_inbound is always True.
  106. for session_key, (adapter, pending_event) in list(candidates.items()):
  107. activity = self._session_activity_for_stall(session_key)
  108. idle_seconds = resolve_session_idle_seconds_from_activity(activity, now=now)
  109. if should_clear_session_stall_notification(
  110. timeout_seconds=timeout_seconds, idle_seconds=idle_seconds, has_pending_inbound=True
  111. ):
  112. notified_map.pop(session_key, None)
  113. if idle_seconds is None or not should_emit_session_stall_notification(
  114. timeout_seconds=timeout_seconds, idle_seconds=idle_seconds,
  115. has_pending_inbound=True, already_notified=bool(notified_map.get(session_key)),
  116. ):
  117. continue
  118. if await self._notify_session_stall(
  119. session_key, adapter, pending_event, idle_seconds, activity or {},
  120. timeout_seconds, notified_map,
  121. ):
  122. sent += 1
  123. # Drop latches for sessions that no longer appear in any pending map.
  124. for key in [k for k in notified_map if k not in candidates]:
  125. notified_map.pop(key, None)
  126. return sent
  127. async def _notify_session_stall(self, session_key: str, adapter, pending_event,
  128. idle_seconds: float, activity: dict, timeout_seconds: float,
  129. notified_map: dict) -> bool:
  130. """Log one stall episode and deliver the notice. True only when sent (latched);
  131. undeliverable (no chat_id) latches without sending; send failures never latch."""
  132. from gateway.run import _STALL_NOTIFY_SEND_TIMEOUT_SECONDS
  133. logger.warning(
  134. "Session stall detected: session=%s idle=%.0fs (timeout=%.0fs, ~%d min); pending "
  135. "inbound present | last_activity=%s | provenance=%s (agent.session_stall_timeout)",
  136. session_key, idle_seconds, timeout_seconds, max(1, int(idle_seconds // 60)),
  137. activity.get("last_activity_desc") or activity.get("last_activity_description")
  138. or "unknown",
  139. activity.get("provenance") or activity.get("last_activity_provenance") or "unknown",
  140. )
  141. source = getattr(pending_event, "source", None)
  142. if not getattr(source, "chat_id", None):
  143. logger.warning("Session stall notify skipped (no chat_id): session=%s", session_key)
  144. notified_map[session_key] = True # cannot deliver; latch to avoid log spam every tick
  145. return False
  146. # Re-read pending state + activity IMMEDIATELY before delivery: the snapshot ages while
  147. # earlier candidates await sends; an agent that progressed (or drained its queue) must not
  148. # get a false stall notice. Abort with the latch un-set so the next tick re-evaluates.
  149. # See #76354.
  150. still_pending = (
  151. (getattr(adapter, "_pending_messages", None) or {}).get(session_key) is not None
  152. or bool((getattr(self, "_queued_events", None) or {}).get(session_key))
  153. )
  154. fresh_idle = resolve_session_idle_seconds_from_activity(
  155. self._session_activity_for_stall(session_key), now=time.time()
  156. )
  157. if not still_pending or (fresh_idle is not None and fresh_idle < timeout_seconds):
  158. logger.info("Session stall notify aborted (no longer stale): session=%s pending=%s "
  159. "fresh_idle=%s", session_key, still_pending, fresh_idle)
  160. notified_map.pop(session_key, None) # re-arm so a FUTURE genuine stall notifies again
  161. return False
  162. from gateway.warning_notifications import present_notification
  163. from gateway.run import _async_profile_runtime_scope
  164. try:
  165. metadata = self._thread_metadata_for_source(source)
  166. notice = format_session_stall_notification(idle_seconds)
  167. result = None
  168. async def send_notice():
  169. nonlocal result
  170. # Bound the send: a wedged adapter transport (network hang, dead websocket) must not
  171. # block the watcher pass — siblings would go unevaluated and the watcher stop.
  172. result = await asyncio.wait_for(
  173. adapter.send(str(source.chat_id), notice, metadata=metadata),
  174. timeout=_STALL_NOTIFY_SEND_TIMEOUT_SECONDS,
  175. )
  176. async with _async_profile_runtime_scope(self._resolve_profile_home_for_source(source)):
  177. presented = await present_notification(send_notice, platform=source.platform)
  178. if not presented:
  179. notified_map[session_key] = True # suppressed: latch so the stall is not re-evaluated every tick
  180. return False
  181. # Adapters often return SendResult(success=False) instead of raising.
  182. if result is not None and getattr(result, "success", True) is False:
  183. raise RuntimeError(getattr(result, "error", "send returned success=False"))
  184. except TimeoutError:
  185. logger.warning(
  186. "Session stall notify send timed out after %.0fs for %s; will retry next tick",
  187. _STALL_NOTIFY_SEND_TIMEOUT_SECONDS, session_key,
  188. )
  189. return False
  190. except Exception as exc:
  191. logger.warning("Session stall notify failed for %s: %s", session_key, exc)
  192. return False
  193. notified_map[session_key] = True
  194. return True
  195. async def _model_catalog_refresh_watcher(self) -> None:
  196. """Refresh the /model picker's remote catalogs every TTL window. The picker itself only
  197. refreshes on a cold/stale open, so if nobody opens ``/model`` the cache never updates."""
  198. from hermes_cli.model_catalog import refresh_catalogs, refresh_interval_seconds
  199. await asyncio.sleep(30) # let startup settle
  200. while self._running:
  201. try:
  202. await asyncio.to_thread(refresh_catalogs)
  203. except Exception as exc:
  204. logger.debug("Model catalog refresh failed: %s", exc)
  205. try:
  206. interval = refresh_interval_seconds()
  207. except Exception:
  208. interval = 1200.0
  209. deadline = time.monotonic() + interval
  210. while self._running and time.monotonic() < deadline:
  211. await asyncio.sleep(min(30.0, max(0.0, deadline - time.monotonic())))
  212. async def _session_stall_watcher(self, interval: float = 30.0):
  213. """Pending-inbound + stale-activity stall watchdog. Progress comes only from
  214. ``get_activity_summary()``; pending inbound is a notify policy gate, not a progress clock.
  215. Notify-only: never kills the turn (contrast ``gateway_timeout`` / ``shutdown_watchdog``).
  216. See #72016.
  217. See #72039.
  218. """
  219. # Short initial delay so startup reconnect noise does not false-fire.
  220. await asyncio.sleep(min(30.0, max(1.0, float(interval))))
  221. while self._running:
  222. try:
  223. if (timeout := self._session_stall_timeout_seconds()) > 0:
  224. await self._check_session_stalls(timeout)
  225. except Exception as exc:
  226. logger.debug("Session stall watcher error: %s", exc)
  227. await _interruptible_sleep(self, max(1, int(float(interval))))