run_agent_cache.py 68 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157
  1. """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset
  2. for GatewayRunner (MRO mixin). ``gateway.run`` internals are imported lazily inside method bodies
  3. (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them at call time."""
  4. from __future__ import annotations
  5. import importlib
  6. import logging
  7. import threading
  8. import time
  9. from contextlib import nullcontext, suppress
  10. from pathlib import Path
  11. from typing import TYPE_CHECKING, Any, Dict, List, Optional
  12. from agent.interrupt_compat import _accepts_keyword
  13. from gateway.config import Platform
  14. from gateway.session import SessionSource, build_session_context_prompt
  15. from gateway.session_prompt_pin import PROMPT_PIN_VERSION, sanitize_prompt_pin
  16. from gateway.run_shutdown import _log_suppressed
  17. from hermes_cli.config import DEFAULT_CONFIG, cfg_get
  18. from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES
  19. from hermes_cli.observability.shared_metrics_busy import observe_running_chats
  20. if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
  21. from gateway.run import GatewayRunner
  22. from gateway.run_turn_runner import TurnRunner
  23. # Log-record parity with the origin module.
  24. logger = logging.getLogger("gateway.run")
  25. # Override fields layered onto runtime kwargs when non-None (partial overrides don't clobber defaults).
  26. _OVERRIDE_APPLY_KEYS = (
  27. "provider", "requested_provider", "api_key", "base_url", "api_mode", "credential_pool", "capabilities", "max_tokens",
  28. )
  29. def _first_agent(entry: Any) -> Any:
  30. """Unwrap a cache entry (``(agent, sig, ...)`` tuple or bare agent) to its agent."""
  31. return entry[0] if isinstance(entry, tuple) and entry else entry
  32. def _tuple_agent(entry: Any) -> Any:
  33. """Agent of a ``(agent, sig, ...)`` cache tuple; None for any other entry shape."""
  34. return entry[0] if isinstance(entry, tuple) and entry else None
  35. class GatewayAgentCacheMixin:
  36. """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset for GatewayRunner."""
  37. @classmethod
  38. def _extract_cache_busting_config(cls, user_config: dict | None) -> dict:
  39. """Values that must bust the cached agent, as a flat dict keyed by 'section.key'. ``user_config``
  40. is the raw file (no DEFAULT_CONFIG merge), so absent keys and non-dict sections take the
  41. DEFAULT_CONFIG value — what the agent was actually built with — while an explicit ``null`` stays
  42. None so opting out of a non-None default still rebuilds. Includes the live tool registry
  43. generation: MCP reloads mutate the registry without touching config.yaml."""
  44. out: dict[str, Any] = {}
  45. cfg = user_config if isinstance(user_config, dict) else {}
  46. for section, key in cls._CACHE_BUSTING_CONFIG_KEYS:
  47. default = cfg_get(DEFAULT_CONFIG, section, key)
  48. section_val = cfg.get(section)
  49. if section == "checkpoints" and isinstance(section_val, bool):
  50. # Legacy ``checkpoints: true``: a live toggle must still rebuild the cached agent.
  51. out[f"{section}.{key}"] = section_val if key == "enabled" else default
  52. else:
  53. out[f"{section}.{key}"] = cfg_get(cfg, section, key, default=default)
  54. try:
  55. from tools.registry import registry
  56. out["tools.registry_generation"] = getattr(registry, "_generation", None)
  57. except Exception:
  58. out["tools.registry_generation"] = None
  59. for key, value in cls._memory_provider_identity_signature(cfg_get(cfg, "memory", "provider")).items():
  60. out[f"memory.{key}"] = value
  61. return out
  62. # Kept for the process lifetime: loading a provider imports its plugin module, and this runs on every inbound message.
  63. _MEMORY_IDENTITY_PROVIDER_MEMO: dict[str, Any] = {}
  64. @classmethod
  65. def _memory_provider_identity_signature(cls, provider_name: Any) -> dict[str, Any]:
  66. """The active memory provider's ``identity_signature()``. ``{}`` when there is no provider,
  67. it fails to load, or the hook raises."""
  68. if not isinstance(provider_name, str) or not provider_name.strip():
  69. return {}
  70. name = provider_name.strip()
  71. try:
  72. instance = cls._MEMORY_IDENTITY_PROVIDER_MEMO.get(name)
  73. if instance is None:
  74. from plugins.memory import load_memory_provider
  75. from plugins.plugin_loader import bounded_load_wait
  76. with bounded_load_wait(): # runs every turn: never stall on another thread's hung import
  77. instance = load_memory_provider(name, register_skills=False)
  78. if instance is None:
  79. return {}
  80. cls._MEMORY_IDENTITY_PROVIDER_MEMO[name] = instance
  81. signature = instance.identity_signature()
  82. return dict(signature) if isinstance(signature, dict) else {}
  83. except Exception:
  84. return {}
  85. @staticmethod
  86. def _agent_config_signature(
  87. model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str,
  88. cache_keys: dict | None = None, user_id: str | None = None, user_id_alt: str | None = None,
  89. skip_context_files: bool = False,
  90. ) -> str:
  91. """Stable key from agent config: change → cached AIAgent rebuilt; unchanged → reused (frozen
  92. prompt + schemas for cache hits). ``user_id`` / ``user_id_alt`` participate because Honcho
  93. freezes them at init; omitting them in shared-thread keys would cross-attribute messages.
  94. ``user_id`` and ``user_id_alt`` are the runtime user identities carried by the current message's
  95. gateway source. They participate in the cache key because memory providers freeze them at
  96. first-message init (the Honcho plugin resolves its user peer from them once). Without them in the signature, a
  97. shared-thread session_key (one in which ``build_session_key`` intentionally omits the participant
  98. ID, e.g. ``thread_sessions_per_user=False``) would reuse the cached AIAgent across distinct users,
  99. causing the second user's messages to be attributed to the first user's resolved Honcho peer. This
  100. broke #27371's per-user-peer contract in multi-user gateways. Per-user agent rebuilds in shared
  101. threads trade prompt-cache warmth for correct memory attribution.
  102. """
  103. import hashlib, json as _j
  104. # Fingerprint the FULL credential, not a short prefix: OAuth/JWT-style tokens often share a
  105. # common prefix (e.g. "eyJhbGci"), so a prefix would give false cache hits across auth switches.
  106. _api_key = str(runtime.get("api_key", "") or "")
  107. blob = _j.dumps(
  108. [
  109. model,
  110. hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "",
  111. runtime.get("base_url", ""), runtime.get("provider", ""),
  112. runtime.get("requested_provider", ""), runtime.get("api_mode", ""),
  113. sorted((runtime.get("capabilities") or {}).items()),
  114. sorted(enabled_toolsets) if enabled_toolsets else [],
  115. # reasoning_config excluded — set per-message on the cached agent; no prompt/tool effect.
  116. ephemeral_prompt or "",
  117. sorted((cache_keys or {}).items()),
  118. str(user_id or ""), str(user_id_alt or ""),
  119. # skip_context_files changes the agent's frozen system prompt (context files in vs out):
  120. # a toggled edit must rebuild the cached agent, not silently reuse it.
  121. bool(skip_context_files),
  122. ],
  123. sort_keys=True, default=str,
  124. )
  125. return hashlib.sha256(blob.encode()).hexdigest()[:16]
  126. def _session_model_override(self, session_key: str) -> Optional[dict]:
  127. """Current in-memory /model override for ``session_key`` (None when absent)."""
  128. state = self._peek_session_state(session_key)
  129. return state.conversation.model_override if state else None
  130. def _rehydrate_session_model_override(self, session_key: str) -> None:
  131. """Lazily restore a persisted /model override after a gateway restart: non-secret parts
  132. (model/provider/base_url) are written through on /model and read back on first use; api_key
  133. is never persisted and is re-resolved. No-op when an in-memory override or nothing exists."""
  134. from gateway.run import _resolve_runtime_agent_kwargs_for_provider
  135. store = getattr(self, "session_store", None)
  136. if self._session_model_override(session_key) is not None or store is None:
  137. return
  138. try:
  139. persisted = store.get_model_override(session_key)
  140. except Exception:
  141. logger.debug("Failed to read persisted session model override", exc_info=True)
  142. return
  143. if not persisted:
  144. return
  145. override: dict[str, Any] = {k: persisted.get(k) for k in ("model", "provider", "base_url")}
  146. provider = persisted.get("provider")
  147. from hermes_cli.runtime_provider import is_foreign_provider_endpoint
  148. if is_foreign_provider_endpoint(provider, override.get("base_url")):
  149. override["base_url"] = None # left over from a switch that kept the previous provider's URL
  150. if provider:
  151. # Re-resolve credentials for the persisted provider. On failure (e.g. credentials removed
  152. # since the switch) keep the credential-less override — _resolve_session_agent_runtime
  153. # retries the resolution for that provider on each turn (default route + notice meanwhile).
  154. try:
  155. runtime = _resolve_runtime_agent_kwargs_for_provider(provider, target_model=persisted.get("model") or None)
  156. for k in ("api_key", "api_mode", "credential_pool", "requested_provider", "max_tokens"):
  157. override[k] = runtime.get(k)
  158. override["request_overrides"] = dict(runtime.get("request_overrides") or {})
  159. override["capabilities"] = dict(runtime.get("capabilities") or {})
  160. if not override.get("base_url") or provider.strip().lower() in LLAMACPP_ALIASES:
  161. # The managed llama.cpp supervisor owns its live port; a persisted loopback URL from a
  162. # boot that fell back to an ephemeral port would strand the session on a dead endpoint.
  163. override["base_url"] = runtime.get("base_url")
  164. from hermes_cli.models import normalize_opencode_base_url, opencode_provider_family
  165. if opencode_provider_family(provider) is not None and override.get("base_url"):
  166. # api_mode was just re-derived from the target model; a relay URL persisted by an older
  167. # build for another wire (/v1-stripped) or the other family is healed to match (#96066).
  168. override["base_url"] = normalize_opencode_base_url(provider, override.get("api_mode"), override["base_url"])
  169. except Exception:
  170. logger.debug(
  171. "Credential re-resolution failed for persisted override "
  172. "(provider=%s); using credential-less override", provider, exc_info=True,
  173. )
  174. self._session_state(session_key).conversation.model_override = override
  175. logger.info(
  176. "Rehydrated persisted /model override for session=%s: model=%s provider=%s",
  177. session_key, override.get("model"), provider or "",
  178. )
  179. def _apply_session_model_override(self, session_key: str, model: str, runtime_kwargs: dict) -> tuple:
  180. """Apply /model session overrides (precedence over config.yaml defaults; ``None`` fields skipped
  181. so partial overrides don't clobber defaults), returning (model, runtime_kwargs)."""
  182. from gateway.run import _credential_pool_for_provider
  183. override = self._session_model_override(session_key)
  184. if not override:
  185. return model, runtime_kwargs
  186. model = override.get("model", model)
  187. for key in _OVERRIDE_APPLY_KEYS:
  188. val = override.get(key)
  189. if val is not None:
  190. runtime_kwargs[key] = val
  191. # request_overrides reflects the switched-to provider; apply whenever the override recorded
  192. # it (even as None) so switching to a provider without configured overrides clears a stale
  193. # value left by the default provider's runtime resolution.
  194. if "request_overrides" in override:
  195. ro = override.get("request_overrides")
  196. runtime_kwargs["request_overrides"] = dict(ro) if isinstance(ro, dict) and ro else ro
  197. if (
  198. runtime_kwargs.get("api_key")
  199. and runtime_kwargs.get("credential_pool") is None
  200. and override.get("provider")
  201. ):
  202. runtime_kwargs["credential_pool"] = _credential_pool_for_provider(override.get("provider"))
  203. return model, runtime_kwargs
  204. def _snapshot_session_model_override(self, session_key: str) -> dict:
  205. """Capture a gateway session override before a one-turn switch."""
  206. override = self._session_model_override(session_key)
  207. return {"had_override": override is not None, "override": dict(override) if override is not None else None}
  208. def _claim_one_turn_restore(self, session_key: str, snapshot: Optional[dict] = None) -> None:
  209. """Arm the one-shot restore snapshot for ``/model --once`` / ``/moa``. A repeated one-shot
  210. command before the turn runs keeps the EARLIEST snapshot: the later command's snapshot is
  211. the first temporary model, not the user's standing override. Pass *snapshot* when the
  212. caller captured the pre-switch state earlier (``/model --once`` applies its override before
  213. arming); omit it to snapshot now."""
  214. conv = self._session_state(session_key).conversation
  215. if not conv.one_turn_restore:
  216. conv.one_turn_restore = dict(snapshot) if snapshot is not None else self._snapshot_session_model_override(session_key)
  217. def _restore_session_model_override(self, session_key: str, snapshot: dict) -> None:
  218. """Restore the session override captured before a one-turn switch."""
  219. if not session_key:
  220. return
  221. if snapshot.get("had_override"):
  222. self._session_state(session_key).conversation.model_override = dict(snapshot.get("override") or {})
  223. elif (state := self._peek_session_state(session_key)) is not None:
  224. state.conversation.model_override = None
  225. self._evict_cached_agent(session_key)
  226. def _is_intentional_model_switch(self, session_key: str, agent: Any, config_model: str) -> bool:
  227. """True when *agent* running a model other than *config_model* is deliberate: a /model session
  228. override names that model, or the Nous gateway moved the session off the ``nous/welcome``
  229. alias that *config_model* still carries (``anon_auth.apply_model_switch``)."""
  230. override = self._session_model_override(session_key)
  231. if override is not None and override.get("model") == agent.model:
  232. return True
  233. # Exactly the recorded move (alias -> backing): a later fallback onto some other model is
  234. # ordinary drift and still evicts.
  235. return getattr(agent, "_nous_model_switch", None) == (config_model, agent.model)
  236. def _fallback_baseline_model(self, session_key: str, source: Optional[SessionSource], agent: Any,
  237. *, policy_model: Optional[str] = None) -> str:
  238. """The model *agent* was built to run, for the post-turn fallback check: what
  239. ``_resolve_session_agent_runtime`` chose (session /model, then a LOCAL session's frozen launch
  240. policy model, then channel_overrides on the chat, thread or parent, then the global model),
  241. canonicalized as ``AIAgent.__init__`` did on the agent's PRIMARY route. The route a fallback
  242. moved it to would make that fallback look primary."""
  243. from gateway.run import _resolve_gateway_model
  244. primary = getattr(agent, "_primary_runtime", None) or {}
  245. provider = primary.get("provider", getattr(agent, "provider", "")) or ""
  246. # An override whose provider was unavailable this turn ran on the default route instead.
  247. override = self._session_model_override(session_key) or {}
  248. if override.get("model") and override.get("provider") in (None, "", provider):
  249. model = override["model"]
  250. elif policy_model:
  251. model = policy_model
  252. elif source is not None:
  253. model = self._resolve_model_for_channel(
  254. source.platform, str(source.chat_id) if source.chat_id else "",
  255. thread_id=str(source.thread_id) if getattr(source, "thread_id", None) else None,
  256. parent_id=str(source.parent_chat_id) if getattr(source, "parent_chat_id", None) else None,
  257. )
  258. else:
  259. model = _resolve_gateway_model()
  260. # Vendor prefix stripped on native providers, else the cached agent is evicted every turn,
  261. # destroying prompt caching.
  262. with suppress(Exception):
  263. from hermes_cli.model_normalize import _AGGREGATOR_PROVIDERS, normalize_model_for_provider
  264. if provider and provider not in _AGGREGATOR_PROVIDERS:
  265. model = normalize_model_for_provider(model, provider)
  266. # The Nous welcome host runs its one model whatever the chat configured (pin_model_for_route).
  267. with suppress(Exception):
  268. from hermes_cli.anon_auth import pin_model_for_route
  269. model = pin_model_for_route(provider, primary.get("base_url", getattr(agent, "base_url", None)), model)
  270. return model
  271. def _persist_active_agents(self) -> None:
  272. """Persist the live in-flight agent count to ``gateway_state.json`` at every turn boundary.
  273. Passes ONLY ``active_agents`` so the read-merge-write keeps lifecycle state (gateway_state=None
  274. would clobber it). Best-effort: a failed write must never disrupt a turn. The running-turn
  275. set also feeds the day's peak of chats running at once (in memory, see shared_metrics_busy)."""
  276. from gateway.run import _write_runtime_status_quiet
  277. _write_runtime_status_quiet(active_agents=self._active_work_count())
  278. observe_running_chats(key for key, _ in self._running_agent_items())
  279. def _release_running_agent_state(
  280. self, session_key: str, *, run_generation: Optional[int] = None
  281. ) -> bool:
  282. """Pop ALL per-running-agent state for ``session_key`` (call at every site that ends a running
  283. turn); True when cleared. Persistent state (model overrides, voice mode, approvals) is NOT
  284. touched. With ``run_generation``, only clear if still current — a stale async unwind bumped
  285. by /stop or /new must not clobber a newer run (returns False)."""
  286. if not session_key or (
  287. run_generation is not None and not self._is_session_run_current(session_key, run_generation)
  288. ):
  289. return False
  290. state = self._peek_session_state(session_key)
  291. if state is not None:
  292. if state.turn.lease is not None:
  293. from gateway.active_lease_io import release_soon
  294. release_soon(state.turn.lease)
  295. # One structured reset instead of a drifting pop-list. Turn-lease tokens are deliberately NOT
  296. # cleared here — _release_turn_lease owns them.
  297. state.turn.clear()
  298. retired, state.turn.retired_agents = state.turn.retired_agents, []
  299. for agent in retired:
  300. # Evicted mid-turn (a one-turn agent): its turn no longer uses it.
  301. self._spawn_release_thread(self._release_evicted_agent_soft, (agent,),
  302. f"agent-retire-{str(session_key)[:24]}", inline_fallback=True,
  303. session_key=session_key)
  304. # Turn boundary: a running-agent slot was just released; persist the new (lower) in-flight count
  305. # so the dashboard readout stays current. Preserves gateway_state (see _persist_active_agents).
  306. self._persist_active_agents()
  307. return True
  308. def _drop_turn_slot(self, session_key: str, *, run_generation: Optional[int] = None) -> None:
  309. """Release the running-agent slot and evict the cached instance (/stop, eviction, reaper).
  310. ``_interrupt_requested`` is cleared only by the turn finalizer, so on a hung/still-draining
  311. run the flag would survive and silently kill the session's NEXT message (interrupted=True,
  312. api_calls=0, empty response); the next message rebuilds from history while the old agent
  313. keeps its flag so a hung drain still dies (#44212). With ``run_generation`` (the post-bump
  314. value ``_interrupt_running_turn`` returns), the release is generation-guarded: an async
  315. path awaits between bump and release, so a successor claiming the slot in that window must
  316. not have its sentinel/lease wiped by the displaced path's tail. Then sweep lease tokens
  317. from generations OLDER than the current one: a hung evicted turn's finalizer may never run,
  318. and each such generation would otherwise pin its token (and its ``_SessionLease``) forever.
  319. Identity-checked + idempotent, so a live successor's token is never affected."""
  320. self._release_running_agent_state(session_key, run_generation=run_generation)
  321. self._evict_cached_agent(session_key)
  322. state = self._peek_session_state(session_key)
  323. registry = getattr(self, "_turn_leases", None)
  324. if state is None or registry is None:
  325. return
  326. current = int(state.persistent.run_generation or 0)
  327. tokens = state.turn.lease_tokens
  328. for gen in [g for g in tokens if int(g) < current]:
  329. token = tokens.pop(gen)
  330. try:
  331. registry.release(token)
  332. except Exception:
  333. logger.debug("Failed to release displaced turn lease gen %s for %s", gen, session_key,
  334. exc_info=True)
  335. def _held_turn_lease(self, session_key: str, run_generation: int):
  336. """Return ``(registry, lease_tokens)`` when ``session_key`` holds a lease token for
  337. ``run_generation``, else None. Callers ``get``/``pop`` the token from the map themselves."""
  338. registry = getattr(self, "_turn_leases", None)
  339. state = self._peek_session_state(session_key) if session_key and registry is not None else None
  340. tokens = state.turn.lease_tokens if state is not None else None
  341. if tokens is None or run_generation not in tokens:
  342. return None
  343. return registry, tokens
  344. def _release_turn_lease(self, session_key: str, run_generation: int) -> bool:
  345. """Release the turn lease acquired by (``session_key``, ``run_generation``). Keyed by (routing
  346. key, run generation) so a stale unwind pops only ITS token; the registry's identity check
  347. refuses it if a newer turn holds the lease. Idempotent."""
  348. held = self._held_turn_lease(session_key, run_generation)
  349. if held is None:
  350. return False
  351. registry, tokens = held
  352. token = tokens.pop(run_generation)
  353. try:
  354. return registry.release(token)
  355. except Exception:
  356. logger.debug("Failed to release turn lease", exc_info=True)
  357. return False
  358. def _rebind_turn_lease(self, session_key: str, run_generation: int, new_session_id: str) -> bool:
  359. """Follow a mid-turn session_id rotation (compression) with the held turn lease, or an alias
  360. key resolving the new id could start a concurrent turn the lease never sees. Call at every
  361. mid-turn reassignment; no-op if no token."""
  362. held = self._held_turn_lease(session_key, run_generation) if new_session_id else None
  363. if held is None:
  364. return False
  365. registry, tokens = held
  366. try:
  367. return registry.rebind(tokens[run_generation], new_session_id)
  368. except Exception:
  369. logger.debug("Failed to rebind turn lease", exc_info=True)
  370. return False
  371. def _clear_conversation_scope(self, session_key: str, *, reason: str) -> None:
  372. """THE single conversation-boundary funnel (/new, /resume, suspension replacement,
  373. compression-exhausted reset). New conversation-scoped dicts go in _CONVERSATION_SCOPED_STATE
  374. so every boundary picks them up. Turn-scoped state (_running_agents/_ts, slot leases, turn-
  375. lease tokens) is owned by _release_running_agent_state and NOT cleared. Idle agent-cache
  376. eviction is NOT a boundary (a resumed turn rebuilds from these). getattr-guarded.
  377. Why a funnel: these boundaries used to each carry a hand-copied pop-list of the per-session dicts,
  378. and the lists drifted every time a new dict was added (#48031, #58403, #10702, #35809 were all
  379. "boundary X forgot dict Y" bugs — e.g. /new cleared the /model override but not the /model --once
  380. restore snapshot). Adding a new conversation-scoped dict now means adding its attribute name to
  381. _CONVERSATION_SCOPED_STATE below; every boundary picks it up automatically.
  382. """
  383. from gateway.run import _CONVERSATION_SCOPED_STATE
  384. if not session_key:
  385. return
  386. state = self._peek_session_state(session_key)
  387. if state is not None:
  388. state.conversation.clear()
  389. # Legacy plain-dict stores still in _CONVERSATION_SCOPED_STATE (not yet folded into
  390. # SessionState), e.g. _pending_model_notes. SessionState-backed names resolve to MutableMapping
  391. # views (not dict), so the isinstance(dict) guard skips them — already handled above.
  392. for attr in _CONVERSATION_SCOPED_STATE:
  393. store = getattr(self, attr, None)
  394. if isinstance(store, dict):
  395. store.pop(session_key, None)
  396. self._clear_session_boundary_security_state(session_key)
  397. logger.debug("Cleared conversation scope for %s (%s)", session_key, reason)
  398. def _clear_session_boundary_security_state(self, session_key: str) -> None:
  399. """Clear per-session control state that must not survive a boundary switch."""
  400. if not session_key:
  401. return
  402. pending_skills_reload_notes = getattr(self, "_pending_skills_reload_notes", None)
  403. if isinstance(pending_skills_reload_notes, dict):
  404. pending_skills_reload_notes.pop(session_key, None)
  405. state = self._peek_session_state(session_key)
  406. if state is not None:
  407. state.persistent.approvals = None
  408. state.persistent.update_prompt_pending = False
  409. for mod, attr, what in (
  410. ("tools.slash_confirm", "clear", "slash-confirm"), ("tools.approval", "clear_session", "approval"),
  411. ):
  412. try:
  413. clear = getattr(importlib.import_module(mod), attr)
  414. except Exception:
  415. continue
  416. try:
  417. clear(session_key)
  418. except Exception as e:
  419. logger.debug("Failed to clear %s state for session boundary %s: %s", what, session_key, e)
  420. # The persisted /yolo copy dies with the in-memory one, or the next turn's restore revives it. None, not
  421. # False: the next conversation follows its launch policy again (seeded once per boundary), and a
  422. # toggle-less chat stays off.
  423. store = getattr(self, "session_store", None)
  424. if store is not None:
  425. try:
  426. store.set_session_yolo(session_key, None)
  427. except Exception:
  428. # Never fails the boundary itself, but a surviving ON would come back after a restart.
  429. logger.warning("Failed to clear persisted yolo for session boundary %s", session_key, exc_info=True)
  430. def _begin_session_run_generation(self, session_key: str) -> int:
  431. """Claim a fresh, monotonically increasing run generation token (NEVER reset): a late result
  432. from a worker /stop or /new invalidated is recognized and dropped."""
  433. if not session_key:
  434. return 0
  435. persistent = self._session_state(session_key).persistent
  436. # Monotonic by design (#28686): incremented here, NEVER reset.
  437. persistent.run_generation = int(persistent.run_generation) + 1
  438. return persistent.run_generation
  439. def _invalidate_session_run_generation(self, session_key: str, *, reason: str = "") -> int:
  440. """Invalidate any in-flight run token for ``session_key``.
  441. Settles a pending one-shot model override first: the displaced turn's finalizer is
  442. generation-guarded and would otherwise leave ``/moa`` / ``/model --once`` in force."""
  443. self._restore_pending_one_turn_model_override(session_key)
  444. generation = self._begin_session_run_generation(session_key)
  445. if reason:
  446. logger.info("Invalidated run generation for %s → %d (%s)", session_key, generation, reason)
  447. return generation
  448. def _current_session_run_generation(self, session_key: str) -> int:
  449. """Current run generation for ``session_key`` (0 when the key tracks no run)."""
  450. state = self._peek_session_state(session_key)
  451. return int(state.persistent.run_generation) if state is not None else 0
  452. def _is_session_run_current(self, session_key: str, generation: int) -> bool:
  453. """Return True when ``generation`` is still current for ``session_key``."""
  454. if not session_key:
  455. return True
  456. return self._current_session_run_generation(session_key) == int(generation)
  457. def _bind_adapter_run_generation(self, adapter: Any, session_key: str, generation: int | None) -> None:
  458. """Bind a gateway run generation to the adapter's active-session event."""
  459. if not adapter or not session_key or generation is None:
  460. return
  461. with suppress(Exception):
  462. interrupt_event = getattr(adapter, "_active_sessions", {}).get(session_key)
  463. if interrupt_event is not None:
  464. interrupt_event._hermes_run_generation = int(generation)
  465. def _interrupt_running_turn(
  466. self, session_key: str, *, interrupt_reason: str, invalidation_reason: str, tool_reason: str | None = None,
  467. ) -> int:
  468. """Sync core shared by /stop, /new and eviction: request a hard interrupt on the in-flight
  469. agent, invalidate its run generation, and reap the tool processes that turn spawned.
  470. ``tool_reason`` names a system issuer (eviction); ``None`` keeps the user attribution of /stop and /new.
  471. Returns the post-bump generation."""
  472. from contextvars import copy_context
  473. from gateway.run import _AGENT_PENDING_SENTINEL, _reap_gateway_turn_processes, request_hard_interrupt
  474. state = self._peek_session_state(session_key)
  475. running_agent = state.turn.agent if state else None
  476. _process_task_id, _process_baseline = "", None
  477. if running_agent and running_agent is not _AGENT_PENDING_SENTINEL:
  478. # A raising interrupt implementation must not leave the slot unroutable: the generation
  479. # bump and release below are the cleanup that matters.
  480. with _log_suppressed(logging.WARNING, "Failed to interrupt running agent for %s; continuing",
  481. session_key, exc_info=True):
  482. request_hard_interrupt(running_agent, interrupt_reason, tool_reason=tool_reason)
  483. _process_task_id = getattr(running_agent, "_gateway_turn_process_task_id", "")
  484. _process_baseline = getattr(running_agent, "_gateway_turn_process_baseline", None)
  485. # Bump the generation BEFORE scheduling the reap thread and capture the post-bump value:
  486. # task_id is session-scoped, so a replacement turn spawning before the reap runs bumps it
  487. # again and the closure sees a stale generation and skips — the replacement's own baseline
  488. # covers its cleanup, so nothing stays unreaped.
  489. _generation_at_interrupt = self._invalidate_session_run_generation(session_key, reason=invalidation_reason)
  490. if _process_task_id and _process_baseline is not None:
  491. threading.Thread(
  492. target=copy_context().run,
  493. args=(_reap_gateway_turn_processes, _process_task_id, _process_baseline),
  494. kwargs={
  495. "source": "gateway_turn_interrupt",
  496. "is_still_current": lambda: self._is_session_run_current(session_key, _generation_at_interrupt),
  497. },
  498. name=f"gateway-turn-reaper-{_process_task_id[:12]}",
  499. daemon=True,
  500. ).start()
  501. return _generation_at_interrupt
  502. async def _interrupt_and_clear_session(
  503. self, session_key: str, source: SessionSource, *, interrupt_reason: str,
  504. invalidation_reason: str, release_running_state: bool = True,
  505. ) -> None:
  506. """Interrupt the current run and clear queued session state consistently."""
  507. if not session_key:
  508. return
  509. state = self._peek_session_state(session_key)
  510. running_agent = state.turn.agent if state else None
  511. _generation_at_interrupt = self._interrupt_running_turn(
  512. session_key, interrupt_reason=interrupt_reason, invalidation_reason=invalidation_reason,
  513. )
  514. from gateway.run import _AGENT_PENDING_SENTINEL
  515. # The turn's hard interrupt reaches only its in-turn children; background delegations were
  516. # detached at dispatch and would otherwise run to completion and wake the session later.
  517. # Each interrupted unit still returns as a completion (status=interrupted, partial output).
  518. from tools.async_delegation import interrupt_for_session
  519. interrupt_for_session(
  520. session_key=session_key, reason=invalidation_reason,
  521. parent_session_id=str(getattr(running_agent, "session_id", "") or ""))
  522. if running_agent and running_agent is not _AGENT_PENDING_SENTINEL:
  523. # Plugins holding a per-turn external resource (an outbound RPC blocked on a tool result
  524. # the loop will never consume) learn the turn is gone. Fires for /stop and the /new
  525. # running-agent fast path; the pending-sentinel /stop has no in-flight work, so it stays
  526. # silent. Dispatch failures are swallowed so a misbehaving plugin cannot break an interrupt.
  527. try:
  528. from hermes_cli.plugins import invoke_hook as _invoke_hook
  529. _invoke_hook(
  530. "agent_loop_stopped",
  531. session_key=session_key,
  532. platform=source.platform.value if source.platform else "",
  533. reason=interrupt_reason,
  534. invalidation_reason=invalidation_reason,
  535. )
  536. except Exception:
  537. logger.debug("agent_loop_stopped hook dispatch failed", exc_info=True)
  538. adapter = self._delivery_adapter_for(source)
  539. interrupt_session_activity = getattr(type(adapter), "interrupt_session_activity", None)
  540. if adapter and callable(interrupt_session_activity):
  541. metadata = self._thread_metadata_for_source(source)
  542. if _accepts_keyword(interrupt_session_activity, "metadata"):
  543. await adapter.interrupt_session_activity(session_key, source.chat_id, metadata=metadata)
  544. else:
  545. await adapter.interrupt_session_activity(session_key, source.chat_id)
  546. if adapter and hasattr(adapter, "get_pending_message"):
  547. # Discard a stale human follow-up (the slot held only user text when /stop started doing
  548. # this, 59575d6a917) — but an internal wake (async-delegation completion, notify+wake)
  549. # shares the slot now and was claim-settled on admission, so dropping it loses it for
  550. # good and the session idles until the next user message (#114456). Leave it parked for
  551. # the adapter's post-command drain; a wake queued behind a discarded human head is
  552. # promoted out of the overflow FIFO for the same reason. Whether a wake may still run
  553. # against a session /new just closed is decided where it is processed
  554. # (_resolve_async_delegation_session fails closed), not here.
  555. parked = adapter.get_pending_message(session_key)
  556. wake = parked if getattr(parked, "internal", False) else None
  557. if wake is None:
  558. overflow = self._overflow_queue(session_key) or []
  559. wake = next((e for e in overflow if getattr(e, "internal", False)), None)
  560. if wake is not None:
  561. overflow.remove(wake)
  562. if wake is not None:
  563. adapter._pending_messages[session_key] = wake
  564. if state is not None:
  565. state.persistent.pending_command_text = None
  566. if release_running_state:
  567. # Guarded release: a message that arrived during the awaits above may already run as
  568. # the successor generation — the displaced /stop tail must not wipe its slot.
  569. self._drop_turn_slot(session_key, run_generation=_generation_at_interrupt)
  570. async def _refresh_agent_cache_message_count(self, session_key: str, session_id: Optional[str]) -> None:
  571. """Re-baseline a cached agent's stored message_count after THIS turn — the coherence guard
  572. rebuilds on mismatch, so without this every turn would rebuild and destroy prompt caching.
  573. Only the count is refreshed, only if the same agent is still cached. DB errors leave the
  574. snapshot as-is (one spare rebuild).
  575. But the snapshot is taken at agent-BUILD time — before this turn writes its own user + assistant (+
  576. tool) rows — and the cache entry is never rewritten on a reuse. See #45966.
  577. """
  578. from gateway.run import _AGENT_PENDING_SENTINEL
  579. _cache_lock = getattr(self, "_agent_cache_lock", None)
  580. _cache = getattr(self, "_agent_cache", None)
  581. if self._session_db is None or not session_id or not _cache_lock or _cache is None:
  582. return
  583. try:
  584. _sess_row = await self._session_db.get_session(session_id)
  585. _live = _sess_row.get("message_count", 0) if _sess_row else None
  586. except Exception:
  587. return
  588. if _live is None:
  589. return
  590. with _cache_lock:
  591. cached = _cache.get(session_key)
  592. # Only re-baseline a live 3-tuple entry; skip pending sentinels, legacy 2-tuples (they opt
  593. # out of the guard), and entries evicted/rebuilt mid-turn. A snapshot taken for a different
  594. # session_id (same session_key, different conversation) is a different DB row — leave it.
  595. if not (isinstance(cached, tuple) and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL):
  596. return
  597. _snapshot_sid = cached[3] if len(cached) > 3 else None
  598. if (_snapshot_sid is not None and _snapshot_sid != session_id) or cached[2] == _live:
  599. return
  600. # Legacy 3-tuple keeps its 3-element shape for callers indexing ``cached[2]``.
  601. _cache[session_key] = (cached[0], cached[1], _live) + (() if _snapshot_sid is None else (_snapshot_sid,))
  602. def _set_pending_turn_sidecar_notes(self, session_key: str, notes: list[str]) -> None:
  603. """Stage per-turn must-deliver notes for the next agent run (one-shot)."""
  604. if not session_key or not notes:
  605. return
  606. self._session_state(session_key).conversation.sidecar_notes = list(notes)
  607. def _consume_pending_turn_sidecar_notes(self, session_key: str) -> list[str]:
  608. state = self._peek_session_state(session_key) if session_key else None
  609. if state is None:
  610. return []
  611. staged, state.conversation.sidecar_notes = state.conversation.sidecar_notes, []
  612. return list(staged) if isinstance(staged, list) else []
  613. def _voice_channel_sidecar_note(self, event, source: SessionSource, session_key: str) -> Optional[str]:
  614. """``[Voice channel now: ...]`` note when VC state changed; ``None`` when unchanged so per-turn
  615. member/speaking churn can't touch the prompt."""
  616. if source.platform != Platform.DISCORD:
  617. return None
  618. adapter = self._delivery_adapter_for(source)
  619. guild_id = self._get_guild_id(event)
  620. if not (guild_id and adapter and hasattr(adapter, "get_voice_channel_context")):
  621. return None
  622. try:
  623. vc_now = adapter.get_voice_channel_context(guild_id) or ""
  624. except Exception:
  625. logger.debug("voice-channel context read failed", exc_info=True)
  626. return None
  627. vc_prev = None
  628. if session_key:
  629. _vc_state = self._session_state(session_key)
  630. vc_prev, _vc_state.conversation.vc_last = _vc_state.conversation.vc_last, vc_now
  631. if vc_now == (vc_prev if vc_prev is not None else ""):
  632. return None
  633. return f"[Voice channel now: {vc_now or 'not connected to a voice channel'}]"
  634. async def _rehydrate_prompt_pins(self, session_key: str, expected_session_id: Optional[str]) -> None:
  635. """Adopt the durable pin snapshot for an internal turn when this process holds no context pin
  636. for *session_key*: a restart, or an eviction (/stop, /undo, /model), which clears only
  637. ``ephemeral_pin``. Without it the wake renders from its origin-only source (no chat or user
  638. names) and the next human turn re-keys back, rewriting the system bytes twice. Human turns
  639. never call this, so an evicted agent's next human turn still re-renders."""
  640. state = self._peek_session_state(session_key)
  641. if state is not None and state.conversation.ephemeral_pin is not None:
  642. return
  643. try:
  644. pin = sanitize_prompt_pin(await self.async_session_store.get_prompt_pin(
  645. session_key, expected_session_id=expected_session_id))
  646. except Exception:
  647. # Cache continuity only: a routing-store fault must not fail the turn.
  648. logger.debug("Failed to read persisted prompt pin for %s", session_key, exc_info=True)
  649. return
  650. if pin is None:
  651. return
  652. conversation = self._session_state(session_key).conversation
  653. conversation.ephemeral_pin = (pin["context_key"], pin["context_prompt"], pin["redact_pii"])
  654. if conversation.channel_pin is None:
  655. conversation.channel_pin = (pin["channel_prompt"], pin["parent_chat_id"])
  656. async def _persist_prompt_pins(self, session_key: Optional[str], expected_session_id: Optional[str]) -> None:
  657. """Persist this conversation's pins before the agent runs; the store no-ops an unchanged
  658. snapshot and refuses a session that has moved on."""
  659. state = self._peek_session_state(session_key) if session_key else None
  660. if state is None:
  661. return
  662. ephemeral_pin, channel_pin = state.conversation.ephemeral_pin, state.conversation.channel_pin
  663. if ephemeral_pin is None or channel_pin is None:
  664. return
  665. snapshot = {
  666. "version": PROMPT_PIN_VERSION, "context_key": ephemeral_pin[0],
  667. "context_prompt": ephemeral_pin[1], "redact_pii": ephemeral_pin[2],
  668. "channel_prompt": channel_pin[0], "parent_chat_id": channel_pin[1],
  669. }
  670. try:
  671. await self.async_session_store.set_prompt_pin(
  672. session_key, snapshot, expected_session_id=expected_session_id)
  673. except Exception:
  674. # Durability protects cache continuity; a store outage must not block the user turn.
  675. logger.debug("Failed to persist prompt pin for %s", session_key, exc_info=True)
  676. def _pinned_session_context_prompt(
  677. self, context, redact_pii: bool, session_key: Optional[str], *, internal: bool = False,
  678. ) -> str:
  679. """Session-context prompt pinned per session: key hit → pinned bytes reused VERBATIM (immune
  680. to renderer nondeterminism); key miss → re-render and re-pin (rename, topic edit, /sethome).
  681. ``internal`` events (kanban wakes, delegation completions, watch notifications) carry a
  682. source rebuilt from the persisted origin, without chat_name/user_name/message_id. Rendering
  683. from it re-keyed the pin, and the next human turn re-keyed it back (A→B→A), rewriting
  684. already-sent system bytes each time. An internal event is never a real metadata change, so
  685. it reuses an existing pin verbatim (restored by ``_rehydrate_prompt_pins`` after a restart or an eviction); with
  686. no pin yet it renders and pins as usual. The pin records the ``privacy.redact_pii`` it was
  687. rendered under: bytes from another privacy policy are never reused, even by an internal
  688. event. Callers pass ``event.internal or event.inherit_channel_pin`` (synthetic
  689. goal/heartbeat/resume turns strip ``message_id``) so this pin and ``_pinned_channel_inputs``
  690. reuse on the same decision."""
  691. _pin_state = self._peek_session_state(session_key) if session_key else None
  692. _eph_pin = _pin_state.conversation.ephemeral_pin if _pin_state else None
  693. if _eph_pin is not None and _eph_pin[2] != redact_pii:
  694. _eph_pin = None
  695. if internal and _eph_pin is not None:
  696. return _eph_pin[1]
  697. _eph_key = self._ephemeral_change_key(context, redact_pii)
  698. if _eph_pin is not None and _eph_pin[0] == _eph_key:
  699. return _eph_pin[1]
  700. text = build_session_context_prompt(context, redact_pii=redact_pii)
  701. if session_key:
  702. self._session_state(session_key).conversation.ephemeral_pin = (_eph_key, text, redact_pii)
  703. return text
  704. def _pinned_channel_inputs(
  705. self, session_key: Optional[str], channel_prompt: Optional[str], source: SessionSource, *, internal: bool,
  706. ):
  707. """``(channel_prompt, source)`` for this turn's agent run.
  708. The ephemeral system prompt also appends ``channel_prompt`` and the ``channel_overrides``
  709. prompt (looked up by chat/thread/``parent_chat_id``). Events with no channel inputs of
  710. their own (internal events, and synthetic goal/heartbeat/resume turns — see
  711. ``MessageEvent.inherit_channel_pin``) carry ``channel_prompt=None`` and a source without
  712. ``parent_chat_id``, so they dropped both and toggled the system prompt like the context pin
  713. did. Human turns record their inputs; turns that ask to reuse the pin get them back
  714. (``_rehydrate_prompt_pins`` restores both after a restart)."""
  715. if not session_key:
  716. return channel_prompt, source
  717. if not internal:
  718. self._session_state(session_key).conversation.channel_pin = (channel_prompt, source.parent_chat_id)
  719. return channel_prompt, source
  720. state = self._peek_session_state(session_key)
  721. pin = state.conversation.channel_pin if state else None
  722. if pin is None:
  723. return channel_prompt, source
  724. pinned_prompt, pinned_parent = pin
  725. if pinned_parent and not source.parent_chat_id:
  726. from gateway.session_identity import replace_source
  727. source = replace_source(source, parent_chat_id=pinned_parent)
  728. return pinned_prompt, source
  729. @staticmethod
  730. def _ephemeral_change_key(context, redact_pii: bool) -> str:
  731. """Hash the exact inputs ``build_session_context_prompt`` renders. Invariant
  732. (test_prompt_tail_freeze.py): any input whose change alters the rendered bytes MUST appear
  733. here — omission means a stale pinned prompt; extras only re-render."""
  734. import hashlib
  735. src = context.source
  736. def _s(v) -> str:
  737. return str(v or "")
  738. discord_ids: tuple = ()
  739. discord_tools = ""
  740. if src.platform == Platform.DISCORD:
  741. from gateway.session import _discord_tools_loaded
  742. discord_tools = "1" if _discord_tools_loaded() else "0"
  743. # message_id is not rendered (value nor presence): it arrives per-turn in the user message.
  744. discord_ids = (
  745. _s(src.guild_id), _s(src.parent_chat_id), _s(src.thread_id), _s(src.chat_id),
  746. )
  747. # Slack's capability-aware platform note is gated on _slack_tools_loaded() — the gate state must
  748. # be in the key (same parity contract as the Discord gate above) so a config / MCP-registration
  749. # flip re-renders once instead of serving a stale pinned note for the rest of the session.
  750. slack_tools = ""
  751. if src.platform == Platform.SLACK:
  752. from gateway.session import _slack_tools_loaded
  753. slack_tools = "1" if _slack_tools_loaded() else "0"
  754. try:
  755. from hermes_constants import display_hermes_home
  756. home_display = str(display_hermes_home())
  757. except Exception:
  758. home_display = ""
  759. key_tuple = (
  760. src.platform.value if src.platform else "",
  761. _s(src.chat_id), _s(src.thread_id), _s(src.chat_type), _s(src.chat_name), _s(src.chat_topic),
  762. _s(src.user_name), _s(src.user_id), _s(getattr(src, "profile", None)),
  763. bool(context.shared_multi_user_session), discord_ids, discord_tools, slack_tools,
  764. tuple(p.value for p in context.connected_platforms),
  765. tuple(
  766. (p.value, _s(getattr(hc, "name", "")), _s(getattr(hc, "chat_id", "")))
  767. for p, hc in context.home_channels.items()
  768. ),
  769. bool(redact_pii), home_display,
  770. )
  771. return hashlib.sha256(repr(key_tuple).encode("utf-8")).hexdigest()
  772. def _evict_cached_agent(self, session_key: str, *, commit_memory: bool = False,
  773. release_after_turn: bool = False) -> None:
  774. """Remove a cached agent (/new, /model, ...) and soft-release its LLM client pool (AIAgent
  775. holds reference cycles; without it RSS grows across /new). Soft = frees clients and child
  776. subagents but PRESERVES terminal sandbox / browser / bg processes since the session may
  777. resume; true boundaries call ``_cleanup_agent_resources`` first. Cleanup runs on a daemon
  778. thread so ``_agent_cache_lock`` never spans slow socket teardown.
  779. Pops the entry AND soft-releases the evicted agent's LLM client pool so the httpx connection
  780. (sockets + held buffers) is freed promptly rather than waiting on CPython GC — AIAgent holds
  781. reference cycles (callbacks, tool state) that delay refcount collection, so a manual release is
  782. required to keep gateway RSS flat across many /new, /model, undo and reset operations (#29298, same
  783. leak class as #25315).
  784. ``release_after_turn``: an agent still mid-turn is released when that turn's running slot
  785. clears (``_release_running_agent_state``) instead of being left to the garbage collector.
  786. """
  787. from gateway.run import _AGENT_PENDING_SENTINEL
  788. # Prompt-stability state rides the agent-cache lifecycle: a fresh agent must re-render its
  789. # session-context bytes (the pin) and re-see the current voice-channel state once.
  790. state = self._peek_session_state(session_key)
  791. if state is not None:
  792. state.conversation.ephemeral_pin = None
  793. state.conversation.vc_last = None
  794. # Tests build runners with ``_agent_cache_lock = None``; evict lock-free then. With the lock
  795. # present ``_agent_cache`` is read directly (an initialized runner always has it).
  796. _lock = getattr(self, "_agent_cache_lock", None)
  797. evicted = None
  798. if _lock:
  799. with _lock:
  800. evicted = self._agent_cache.pop(session_key, None)
  801. else:
  802. _cache = getattr(self, "_agent_cache", None)
  803. if _cache is not None:
  804. evicted = _cache.pop(session_key, None)
  805. agent = _first_agent(evicted)
  806. # Never tear down an agent that's mid-turn — its client, sandbox and child subagents are in use.
  807. if agent is None or agent is _AGENT_PENDING_SENTINEL:
  808. return
  809. if id(agent) in self._running_agent_ids():
  810. if release_after_turn and state is not None:
  811. state.turn.retired_agents.append(agent)
  812. return
  813. if commit_memory:
  814. target, args = self._commit_then_release_soft, (agent, session_key, True)
  815. else:
  816. target, args = self._release_evicted_agent_soft, (agent,)
  817. self._spawn_release_thread(target, args, f"agent-evict-{str(session_key)[:24]}", inline_fallback=True,
  818. session_key=session_key)
  819. def _evict_cached_agent_at_boundary(self, session_key: str) -> None:
  820. """Reset and compress end the conversation the provider was tracking, so the old agent's
  821. transcript is committed (``on_session_end``) before its clients go; a plain evict is not."""
  822. self._evict_cached_agent(session_key, commit_memory=True)
  823. def _spawn_release_thread(self, target, args: tuple, name: str, *, inline_fallback: bool,
  824. session_key: Optional[str] = None) -> None:
  825. """Run a release on a daemon thread. ``inline_fallback`` runs it inline (best-effort) when no
  826. thread can start (interpreter shutdown); otherwise a spawn failure propagates, as on main.
  827. The thread runs inside the owning profile's scope (see ``_run_release_in_profile_scope``)."""
  828. import contextvars
  829. ctx = contextvars.copy_context()
  830. try:
  831. threading.Thread(target=ctx.run, args=(self._run_release_in_profile_scope, target, args, session_key),
  832. daemon=True, name=name).start()
  833. except Exception:
  834. if not inline_fallback:
  835. raise
  836. with suppress(Exception):
  837. ctx.run(self._run_release_in_profile_scope, target, args, session_key)
  838. def _run_release_in_profile_scope(self, target, args: tuple, session_key: Optional[str]) -> None:
  839. """Call ``target(*args)`` under the profile that OWNS ``session_key``.
  840. Threads start with an EMPTY context, so a bare thread would commit end-of-session memory
  841. (provider ``on_session_end`` reads credentials/home at call time) under the launch profile.
  842. And the LRU-cap eviction runs inside the REQUESTING turn, whose agent may belong to another
  843. profile — so "some scope is present" is not enough either. The owner comes from the session
  844. key: a named profile's home, else the DEFAULT profile (``agent:main:`` keys), which is the
  845. root Hermes dir even when the gateway was launched under a named profile. Its scope is
  846. entered unless the current one already is the owner's."""
  847. from agent.secret_scope import current_secret_scope, is_multiplex_active
  848. scope = nullcontext()
  849. if is_multiplex_active():
  850. from gateway.run import _profile_runtime_scope
  851. from hermes_constants import get_default_hermes_root, get_hermes_home, hermes_home_key
  852. owner = None
  853. store = getattr(self, "session_store", None)
  854. if session_key and store is not None:
  855. try:
  856. owner = store._profile_home_for_key(session_key)
  857. except Exception:
  858. logger.warning("Could not resolve the owning profile for %s; releasing under the default profile",
  859. session_key, exc_info=True)
  860. owner_home = Path(owner) if owner else get_default_hermes_root()
  861. if current_secret_scope() is None or hermes_home_key(get_hermes_home()) != hermes_home_key(owner_home):
  862. scope = _profile_runtime_scope(owner_home)
  863. with scope:
  864. target(*args)
  865. def _commit_memory_before_soft_evict(self, agent: Any, key: str, boundary: bool = False) -> None:
  866. """Commit the live transcript to memory providers before resource-only eviction. A session
  867. boundary (reset, compress) also ends the context engine's session; that engine can run without a provider."""
  868. if agent is None or not hasattr(agent, "commit_memory_session"):
  869. return
  870. has_provider = getattr(agent, "_memory_manager", None) is not None
  871. has_engine = boundary and getattr(agent, "context_compressor", None) is not None
  872. if not has_provider and not has_engine:
  873. return
  874. try:
  875. messages = getattr(agent, "_session_messages", None)
  876. agent.commit_memory_session(messages if isinstance(messages, list) else None)
  877. logger.debug(
  878. "Committed on_session_end extraction before soft-evicting "
  879. "session=%s (resource eviction)", key,
  880. )
  881. except Exception as _e:
  882. logger.debug("Pre-evict memory commit failed for %s: %s", key, _e)
  883. def _commit_then_release_soft(self, agent: Any, key: str, boundary: bool = False) -> None:
  884. """Commit end-of-session memory (if warranted), then soft-release — on the daemon eviction
  885. thread. Order matters: commit needs the live memory manager before ``release_clients``."""
  886. self._commit_memory_before_soft_evict(agent, key, boundary)
  887. self._release_evicted_agent_soft(agent)
  888. def _release_evicted_agent_soft(self, agent: Any) -> None:
  889. """Soft cleanup for cache-evicted agents: unlike _cleanup_agent_resources, the session may
  890. resume, so terminal sandbox, browser daemon and bg processes outlive the AIAgent instance."""
  891. if agent is None:
  892. return
  893. with suppress(Exception):
  894. if hasattr(agent, "release_clients"):
  895. agent.release_clients()
  896. else:
  897. # Older agent instance (shouldn't happen in practice) — legacy full-close path.
  898. self._cleanup_agent_resources(agent)
  899. # Free conversation history — tens of MB of tool output on heavy 100+-tool-call sessions.
  900. # release_clients() preserves session tool state for resume, but the message list is rebuilt from
  901. # persisted session JSON on the next turn, so dropping it here is safe.
  902. if hasattr(agent, "_session_messages"):
  903. agent._session_messages = []
  904. # _db_flush_scan_prefix (run_agent.py, stamped on every successful flush) is a shallow copy
  905. # sharing every message dict of the flushed transcript, so leaving it pins the multi-MB strings
  906. # this eviction frees. Pressure-evictable agents have flushed by definition, so it's populated.
  907. if hasattr(agent, "_db_flush_scan_prefix"):
  908. agent._db_flush_scan_prefix = None
  909. def _agent_cache_bounds(self):
  910. """Operator-configured agent-cache bounds, resolved once per process (lazily, not in
  911. ``__init__``, so ``__new__``-constructed test / slash-command runners work too)."""
  912. from gateway.run import _load_gateway_config
  913. bounds = getattr(self, "_agent_cache_bounds_cache", None)
  914. if bounds is None:
  915. from gateway.agent_cache_pressure import resolve_agent_cache_bounds
  916. try:
  917. bounds = resolve_agent_cache_bounds(_load_gateway_config())
  918. except Exception as _e:
  919. logger.debug("Agent cache bounds config read failed: %s", _e)
  920. # Resolve from an empty config rather than bare AgentCacheBounds(): the dataclass default
  921. # has memory_high_mb=None (pressure pass OFF) but an *absent* section means "auto" — a
  922. # transient config read failure must not permanently disable the OOM valve.
  923. bounds = resolve_agent_cache_bounds({})
  924. self._agent_cache_bounds_cache = bounds
  925. return bounds
  926. def _agent_cache_cap(self) -> int:
  927. """Effective LRU cap — the configured override, else the default."""
  928. from gateway.run import _AGENT_CACHE_MAX_SIZE
  929. return self._agent_cache_bounds().max_size or _AGENT_CACHE_MAX_SIZE
  930. def _agent_cache_idle_ttl(self) -> float:
  931. """Effective idle TTL in seconds — configured override, else default."""
  932. from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS
  933. return self._agent_cache_bounds().idle_ttl_secs or _AGENT_CACHE_IDLE_TTL_SECS
  934. def _sweep_agent_cache_under_pressure(self) -> int:
  935. """Shed cached transcripts once the gateway heap nears its budget; returns count evicted.
  936. The LRU cap counts entries and the idle sweep counts seconds; neither knows one cached agent
  937. pins a full ``_session_messages`` transcript (tens of MB), so RSS climbs until the cgroup
  938. throttles. Above the anonymous-RSS budget this soft-evicts LRU agents (transcript rebuilt
  939. from the persisted session next turn). Never touched: agents mid-turn, the most recently
  940. used sessions, and transcripts not yet on disk.
  941. A gateway serving many chats can hold every warm transcript for the TTL window.
  942. Pressure eviction bounds that heap before the cgroup throttles and SIGTERM can
  943. no longer flush inside systemd's stop timeout (#80764).
  944. """
  945. from gateway.run import _AGENT_PENDING_SENTINEL
  946. from gateway.agent_cache_pressure import (
  947. plan_pressure_evictions, read_anon_rss_mb, transcript_persistence_caught_up
  948. )
  949. bounds = self._agent_cache_bounds()
  950. _cache = getattr(self, "_agent_cache", None)
  951. _lock = getattr(self, "_agent_cache_lock", None)
  952. # Nothing cached — whatever is using the heap, it isn't us, and warning about it every tick
  953. # would point at the wrong subsystem.
  954. if not bounds.memory_high_mb or not _cache or _lock is None:
  955. return 0
  956. rss_mb = read_anon_rss_mb()
  957. if rss_mb is None or rss_mb < bounds.memory_high_mb:
  958. return 0
  959. running_ids = self._running_agent_ids()
  960. def _is_live(agent: Any) -> bool:
  961. return agent is not None and agent is not _AGENT_PENDING_SENTINEL and id(agent) not in running_ids
  962. def _is_evictable(key: str, agent: Any) -> bool:
  963. return _is_live(agent) and transcript_persistence_caught_up(agent)
  964. with _lock:
  965. ordered = [(key, _first_agent(entry)) for key, entry in _cache.items()]
  966. plan = plan_pressure_evictions(
  967. ordered, is_evictable=_is_evictable, max_evictions=bounds.max_evictions_per_pass,
  968. protect_recent=bounds.protect_recent,
  969. )
  970. for key, _ in plan:
  971. _cache.pop(key, None)
  972. if not plan:
  973. _mid_turn = sum(1 for _, a in ordered if a is not None and id(a) in running_ids)
  974. _unflushed = sum(1 for _, a in ordered if _is_live(a) and not transcript_persistence_caught_up(a))
  975. logger.warning(
  976. "Agent cache pressure: anon RSS %dMB over budget %dMB but no "
  977. "evictable session (%d cached, %d mid-turn, %d blocked on "
  978. "un-flushed persistence)%s",
  979. rss_mb, bounds.memory_high_mb, len(ordered), _mid_turn, _unflushed,
  980. (
  981. " — transcripts are not reaching the session DB "
  982. "(session persistence disabled or failing?); the memory "
  983. "valve cannot shed sessions until they persist."
  984. if _unflushed and not _mid_turn
  985. else " — memory will keep climbing until those turns finish."
  986. ),
  987. )
  988. return 0
  989. evicted_count = len(plan)
  990. logger.warning(
  991. "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting %d LRU session(s): %s",
  992. rss_mb, bounds.memory_high_mb, evicted_count, ", ".join(key for key, _ in plan),
  993. )
  994. try:
  995. threading.Thread(target=self._release_pressure_batch, args=(plan,), daemon=True,
  996. name="agent-cache-pressure").start()
  997. except Exception:
  998. # Thread spawn failed (interpreter shutdown): release inline, unguarded (as on main).
  999. self._release_pressure_batch(plan)
  1000. # _release_pressure_batch drains `plan` in place (so the trim runs with no lingering agent
  1001. # refs) — len(plan) is 0 once the daemon thread finishes, hence the pre-captured count.
  1002. return evicted_count
  1003. def _release_pressure_batch(self, plan: list[tuple]) -> None:
  1004. """Release a pressure-evicted batch sequentially on one daemon thread, then ``malloc_trim`` so
  1005. RSS actually falls. The plan is drained (``pop`` + ``del``), not iterated, so no local
  1006. reference pins evicted agents during ``gc.collect`` + trim (else the valve over-evicts)."""
  1007. while plan:
  1008. key, agent = plan.pop(0) # FIFO — evict LRU-first order preserved
  1009. try:
  1010. # Pressure sweeps run from the unscoped housekeeping watcher: enter each owner's scope.
  1011. self._run_release_in_profile_scope(self._commit_then_release_soft, (agent, key), key)
  1012. except Exception as _e:
  1013. logger.debug("Pressure release failed for %s: %s", key, _e)
  1014. del agent
  1015. with suppress(Exception):
  1016. from hermes_cli.mem_trim import trim_memory
  1017. trim_memory(force=True, reason="agent_cache_pressure")
  1018. def _enforce_agent_cache_cap(self) -> None:
  1019. """Evict oldest cached agents past the LRU cap (requires _agent_cache_lock); cleanup on a
  1020. daemon thread. Mid-turn agents are SKIPPED, so the cache may stay over cap until the next
  1021. insert."""
  1022. _cache = getattr(self, "_agent_cache", None)
  1023. # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the arg so skip enforcement
  1024. # if a test fixture swapped the cache type.
  1025. if _cache is None or not hasattr(_cache, "move_to_end"):
  1026. return
  1027. # Snapshot of agent instances mid-turn, keyed by id() so lookup is O(1) and independent of
  1028. # AIAgent.__eq__ (which MagicMock overrides in tests).
  1029. running_ids = self._running_agent_ids()
  1030. # Walk LRU → MRU; only the first (size - cap) LRU positions are candidates. An active slot is
  1031. # SKIPPED rather than evicting a newer entry — that would penalise a fresh session (no cache
  1032. # history) to protect a long-running one. Cache may stay over cap until the next insert.
  1033. cap = self._agent_cache_cap()
  1034. candidates = [(key, _tuple_agent(_cache.get(key))) for key in list(_cache.keys())[:max(0, len(_cache) - cap)]]
  1035. evict_plan = [(key, agent) for key, agent in candidates if agent is None or id(agent) not in running_ids]
  1036. for key, _ in evict_plan:
  1037. _cache.pop(key, None)
  1038. remaining_over_cap = len(_cache) - cap
  1039. if remaining_over_cap > 0:
  1040. logger.warning(
  1041. "Agent cache over cap (%d > %d); %d excess slot(s) held by "
  1042. "mid-turn agents — will re-check on next insert.",
  1043. len(_cache), cap, remaining_over_cap,
  1044. )
  1045. for key, agent in evict_plan:
  1046. logger.info("Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache))
  1047. if agent is not None:
  1048. # Commit end-of-session memory, then soft-release, both on the daemon thread so the
  1049. # (possibly network-bound) provider call never blocks the held cache lock.
  1050. self._spawn_release_thread(self._commit_then_release_soft, (agent, key), f"agent-cache-evict-{key[:24]}",
  1051. inline_fallback=False, session_key=key)
  1052. def _sweep_idle_cached_agents(self) -> int:
  1053. """Evict cached agents idle past the idle TTL (lock acquired internally; cleanup on daemon
  1054. threads; mid-turn agents SKIPPED); returns the number evicted."""
  1055. _cache = getattr(self, "_agent_cache", None)
  1056. _lock = getattr(self, "_agent_cache_lock", None)
  1057. if _cache is None or _lock is None:
  1058. return 0
  1059. now = time.time()
  1060. idle_ttl = self._agent_cache_idle_ttl()
  1061. to_evict: list[tuple] = []
  1062. running_ids = self._running_agent_ids()
  1063. with _lock:
  1064. for key, entry in list(_cache.items()):
  1065. agent = _tuple_agent(entry)
  1066. if agent is None or id(agent) in running_ids:
  1067. continue # mid-turn — don't tear it down
  1068. last_activity = getattr(agent, "_last_activity_ts", None)
  1069. if last_activity is None or (now - last_activity) <= idle_ttl:
  1070. continue
  1071. to_evict.append((key, agent))
  1072. for key, _ in to_evict:
  1073. _cache.pop(key, None)
  1074. for key, agent in to_evict:
  1075. logger.info("Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now))
  1076. self._spawn_release_thread(self._commit_then_release_soft, (agent, key), f"agent-cache-idle-{key[:24]}",
  1077. inline_fallback=False, session_key=key)
  1078. return len(to_evict)