| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157 |
- """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset
- for GatewayRunner (MRO mixin). ``gateway.run`` internals are imported lazily inside method bodies
- (import cycle), so ``patch("gateway.run.X")`` keeps intercepting them at call time."""
- from __future__ import annotations
- import importlib
- import logging
- import threading
- import time
- from contextlib import nullcontext, suppress
- from pathlib import Path
- from typing import TYPE_CHECKING, Any, Dict, List, Optional
- from agent.interrupt_compat import _accepts_keyword
- from gateway.config import Platform
- from gateway.session import SessionSource, build_session_context_prompt
- from gateway.session_prompt_pin import PROMPT_PIN_VERSION, sanitize_prompt_pin
- from gateway.run_shutdown import _log_suppressed
- from hermes_cli.config import DEFAULT_CONFIG, cfg_get
- from hermes_cli.local_runtime.endpoint import LLAMACPP_ALIASES
- from hermes_cli.observability.shared_metrics_busy import observe_running_chats
- if TYPE_CHECKING: # string annotations only; never imported at runtime (cycle)
- from gateway.run import GatewayRunner
- from gateway.run_turn_runner import TurnRunner
- # Log-record parity with the origin module.
- logger = logging.getLogger("gateway.run")
- # Override fields layered onto runtime kwargs when non-None (partial overrides don't clobber defaults).
- _OVERRIDE_APPLY_KEYS = (
- "provider", "requested_provider", "api_key", "base_url", "api_mode", "credential_pool", "capabilities", "max_tokens",
- )
- def _first_agent(entry: Any) -> Any:
- """Unwrap a cache entry (``(agent, sig, ...)`` tuple or bare agent) to its agent."""
- return entry[0] if isinstance(entry, tuple) and entry else entry
- def _tuple_agent(entry: Any) -> Any:
- """Agent of a ``(agent, sig, ...)`` cache tuple; None for any other entry shape."""
- return entry[0] if isinstance(entry, tuple) and entry else None
- class GatewayAgentCacheMixin:
- """Agent cache, session model overrides, turn leases, run generations and conversation-scope reset for GatewayRunner."""
- @classmethod
- def _extract_cache_busting_config(cls, user_config: dict | None) -> dict:
- """Values that must bust the cached agent, as a flat dict keyed by 'section.key'. ``user_config``
- is the raw file (no DEFAULT_CONFIG merge), so absent keys and non-dict sections take the
- DEFAULT_CONFIG value — what the agent was actually built with — while an explicit ``null`` stays
- None so opting out of a non-None default still rebuilds. Includes the live tool registry
- generation: MCP reloads mutate the registry without touching config.yaml."""
- out: dict[str, Any] = {}
- cfg = user_config if isinstance(user_config, dict) else {}
- for section, key in cls._CACHE_BUSTING_CONFIG_KEYS:
- default = cfg_get(DEFAULT_CONFIG, section, key)
- section_val = cfg.get(section)
- if section == "checkpoints" and isinstance(section_val, bool):
- # Legacy ``checkpoints: true``: a live toggle must still rebuild the cached agent.
- out[f"{section}.{key}"] = section_val if key == "enabled" else default
- else:
- out[f"{section}.{key}"] = cfg_get(cfg, section, key, default=default)
- try:
- from tools.registry import registry
- out["tools.registry_generation"] = getattr(registry, "_generation", None)
- except Exception:
- out["tools.registry_generation"] = None
- for key, value in cls._memory_provider_identity_signature(cfg_get(cfg, "memory", "provider")).items():
- out[f"memory.{key}"] = value
- return out
- # Kept for the process lifetime: loading a provider imports its plugin module, and this runs on every inbound message.
- _MEMORY_IDENTITY_PROVIDER_MEMO: dict[str, Any] = {}
- @classmethod
- def _memory_provider_identity_signature(cls, provider_name: Any) -> dict[str, Any]:
- """The active memory provider's ``identity_signature()``. ``{}`` when there is no provider,
- it fails to load, or the hook raises."""
- if not isinstance(provider_name, str) or not provider_name.strip():
- return {}
- name = provider_name.strip()
- try:
- instance = cls._MEMORY_IDENTITY_PROVIDER_MEMO.get(name)
- if instance is None:
- from plugins.memory import load_memory_provider
- from plugins.plugin_loader import bounded_load_wait
- with bounded_load_wait(): # runs every turn: never stall on another thread's hung import
- instance = load_memory_provider(name, register_skills=False)
- if instance is None:
- return {}
- cls._MEMORY_IDENTITY_PROVIDER_MEMO[name] = instance
- signature = instance.identity_signature()
- return dict(signature) if isinstance(signature, dict) else {}
- except Exception:
- return {}
- @staticmethod
- def _agent_config_signature(
- model: str, runtime: dict, enabled_toolsets: list, ephemeral_prompt: str,
- cache_keys: dict | None = None, user_id: str | None = None, user_id_alt: str | None = None,
- skip_context_files: bool = False,
- ) -> str:
- """Stable key from agent config: change → cached AIAgent rebuilt; unchanged → reused (frozen
- prompt + schemas for cache hits). ``user_id`` / ``user_id_alt`` participate because Honcho
- freezes them at init; omitting them in shared-thread keys would cross-attribute messages.
- ``user_id`` and ``user_id_alt`` are the runtime user identities carried by the current message's
- gateway source. They participate in the cache key because memory providers freeze them at
- first-message init (the Honcho plugin resolves its user peer from them once). Without them in the signature, a
- shared-thread session_key (one in which ``build_session_key`` intentionally omits the participant
- ID, e.g. ``thread_sessions_per_user=False``) would reuse the cached AIAgent across distinct users,
- causing the second user's messages to be attributed to the first user's resolved Honcho peer. This
- broke #27371's per-user-peer contract in multi-user gateways. Per-user agent rebuilds in shared
- threads trade prompt-cache warmth for correct memory attribution.
- """
- import hashlib, json as _j
- # Fingerprint the FULL credential, not a short prefix: OAuth/JWT-style tokens often share a
- # common prefix (e.g. "eyJhbGci"), so a prefix would give false cache hits across auth switches.
- _api_key = str(runtime.get("api_key", "") or "")
- blob = _j.dumps(
- [
- model,
- hashlib.sha256(_api_key.encode()).hexdigest() if _api_key else "",
- runtime.get("base_url", ""), runtime.get("provider", ""),
- runtime.get("requested_provider", ""), runtime.get("api_mode", ""),
- sorted((runtime.get("capabilities") or {}).items()),
- sorted(enabled_toolsets) if enabled_toolsets else [],
- # reasoning_config excluded — set per-message on the cached agent; no prompt/tool effect.
- ephemeral_prompt or "",
- sorted((cache_keys or {}).items()),
- str(user_id or ""), str(user_id_alt or ""),
- # skip_context_files changes the agent's frozen system prompt (context files in vs out):
- # a toggled edit must rebuild the cached agent, not silently reuse it.
- bool(skip_context_files),
- ],
- sort_keys=True, default=str,
- )
- return hashlib.sha256(blob.encode()).hexdigest()[:16]
- def _session_model_override(self, session_key: str) -> Optional[dict]:
- """Current in-memory /model override for ``session_key`` (None when absent)."""
- state = self._peek_session_state(session_key)
- return state.conversation.model_override if state else None
- def _rehydrate_session_model_override(self, session_key: str) -> None:
- """Lazily restore a persisted /model override after a gateway restart: non-secret parts
- (model/provider/base_url) are written through on /model and read back on first use; api_key
- is never persisted and is re-resolved. No-op when an in-memory override or nothing exists."""
- from gateway.run import _resolve_runtime_agent_kwargs_for_provider
- store = getattr(self, "session_store", None)
- if self._session_model_override(session_key) is not None or store is None:
- return
- try:
- persisted = store.get_model_override(session_key)
- except Exception:
- logger.debug("Failed to read persisted session model override", exc_info=True)
- return
- if not persisted:
- return
- override: dict[str, Any] = {k: persisted.get(k) for k in ("model", "provider", "base_url")}
- provider = persisted.get("provider")
- from hermes_cli.runtime_provider import is_foreign_provider_endpoint
- if is_foreign_provider_endpoint(provider, override.get("base_url")):
- override["base_url"] = None # left over from a switch that kept the previous provider's URL
- if provider:
- # Re-resolve credentials for the persisted provider. On failure (e.g. credentials removed
- # since the switch) keep the credential-less override — _resolve_session_agent_runtime
- # retries the resolution for that provider on each turn (default route + notice meanwhile).
- try:
- runtime = _resolve_runtime_agent_kwargs_for_provider(provider, target_model=persisted.get("model") or None)
- for k in ("api_key", "api_mode", "credential_pool", "requested_provider", "max_tokens"):
- override[k] = runtime.get(k)
- override["request_overrides"] = dict(runtime.get("request_overrides") or {})
- override["capabilities"] = dict(runtime.get("capabilities") or {})
- if not override.get("base_url") or provider.strip().lower() in LLAMACPP_ALIASES:
- # The managed llama.cpp supervisor owns its live port; a persisted loopback URL from a
- # boot that fell back to an ephemeral port would strand the session on a dead endpoint.
- override["base_url"] = runtime.get("base_url")
- from hermes_cli.models import normalize_opencode_base_url, opencode_provider_family
- if opencode_provider_family(provider) is not None and override.get("base_url"):
- # api_mode was just re-derived from the target model; a relay URL persisted by an older
- # build for another wire (/v1-stripped) or the other family is healed to match (#96066).
- override["base_url"] = normalize_opencode_base_url(provider, override.get("api_mode"), override["base_url"])
- except Exception:
- logger.debug(
- "Credential re-resolution failed for persisted override "
- "(provider=%s); using credential-less override", provider, exc_info=True,
- )
- self._session_state(session_key).conversation.model_override = override
- logger.info(
- "Rehydrated persisted /model override for session=%s: model=%s provider=%s",
- session_key, override.get("model"), provider or "",
- )
- def _apply_session_model_override(self, session_key: str, model: str, runtime_kwargs: dict) -> tuple:
- """Apply /model session overrides (precedence over config.yaml defaults; ``None`` fields skipped
- so partial overrides don't clobber defaults), returning (model, runtime_kwargs)."""
- from gateway.run import _credential_pool_for_provider
- override = self._session_model_override(session_key)
- if not override:
- return model, runtime_kwargs
- model = override.get("model", model)
- for key in _OVERRIDE_APPLY_KEYS:
- val = override.get(key)
- if val is not None:
- runtime_kwargs[key] = val
- # request_overrides reflects the switched-to provider; apply whenever the override recorded
- # it (even as None) so switching to a provider without configured overrides clears a stale
- # value left by the default provider's runtime resolution.
- if "request_overrides" in override:
- ro = override.get("request_overrides")
- runtime_kwargs["request_overrides"] = dict(ro) if isinstance(ro, dict) and ro else ro
- if (
- runtime_kwargs.get("api_key")
- and runtime_kwargs.get("credential_pool") is None
- and override.get("provider")
- ):
- runtime_kwargs["credential_pool"] = _credential_pool_for_provider(override.get("provider"))
- return model, runtime_kwargs
- def _snapshot_session_model_override(self, session_key: str) -> dict:
- """Capture a gateway session override before a one-turn switch."""
- override = self._session_model_override(session_key)
- return {"had_override": override is not None, "override": dict(override) if override is not None else None}
- def _claim_one_turn_restore(self, session_key: str, snapshot: Optional[dict] = None) -> None:
- """Arm the one-shot restore snapshot for ``/model --once`` / ``/moa``. A repeated one-shot
- command before the turn runs keeps the EARLIEST snapshot: the later command's snapshot is
- the first temporary model, not the user's standing override. Pass *snapshot* when the
- caller captured the pre-switch state earlier (``/model --once`` applies its override before
- arming); omit it to snapshot now."""
- conv = self._session_state(session_key).conversation
- if not conv.one_turn_restore:
- conv.one_turn_restore = dict(snapshot) if snapshot is not None else self._snapshot_session_model_override(session_key)
- def _restore_session_model_override(self, session_key: str, snapshot: dict) -> None:
- """Restore the session override captured before a one-turn switch."""
- if not session_key:
- return
- if snapshot.get("had_override"):
- self._session_state(session_key).conversation.model_override = dict(snapshot.get("override") or {})
- elif (state := self._peek_session_state(session_key)) is not None:
- state.conversation.model_override = None
- self._evict_cached_agent(session_key)
- def _is_intentional_model_switch(self, session_key: str, agent: Any, config_model: str) -> bool:
- """True when *agent* running a model other than *config_model* is deliberate: a /model session
- override names that model, or the Nous gateway moved the session off the ``nous/welcome``
- alias that *config_model* still carries (``anon_auth.apply_model_switch``)."""
- override = self._session_model_override(session_key)
- if override is not None and override.get("model") == agent.model:
- return True
- # Exactly the recorded move (alias -> backing): a later fallback onto some other model is
- # ordinary drift and still evicts.
- return getattr(agent, "_nous_model_switch", None) == (config_model, agent.model)
- def _fallback_baseline_model(self, session_key: str, source: Optional[SessionSource], agent: Any,
- *, policy_model: Optional[str] = None) -> str:
- """The model *agent* was built to run, for the post-turn fallback check: what
- ``_resolve_session_agent_runtime`` chose (session /model, then a LOCAL session's frozen launch
- policy model, then channel_overrides on the chat, thread or parent, then the global model),
- canonicalized as ``AIAgent.__init__`` did on the agent's PRIMARY route. The route a fallback
- moved it to would make that fallback look primary."""
- from gateway.run import _resolve_gateway_model
- primary = getattr(agent, "_primary_runtime", None) or {}
- provider = primary.get("provider", getattr(agent, "provider", "")) or ""
- # An override whose provider was unavailable this turn ran on the default route instead.
- override = self._session_model_override(session_key) or {}
- if override.get("model") and override.get("provider") in (None, "", provider):
- model = override["model"]
- elif policy_model:
- model = policy_model
- elif source is not None:
- model = self._resolve_model_for_channel(
- source.platform, str(source.chat_id) if source.chat_id else "",
- thread_id=str(source.thread_id) if getattr(source, "thread_id", None) else None,
- parent_id=str(source.parent_chat_id) if getattr(source, "parent_chat_id", None) else None,
- )
- else:
- model = _resolve_gateway_model()
- # Vendor prefix stripped on native providers, else the cached agent is evicted every turn,
- # destroying prompt caching.
- with suppress(Exception):
- from hermes_cli.model_normalize import _AGGREGATOR_PROVIDERS, normalize_model_for_provider
- if provider and provider not in _AGGREGATOR_PROVIDERS:
- model = normalize_model_for_provider(model, provider)
- # The Nous welcome host runs its one model whatever the chat configured (pin_model_for_route).
- with suppress(Exception):
- from hermes_cli.anon_auth import pin_model_for_route
- model = pin_model_for_route(provider, primary.get("base_url", getattr(agent, "base_url", None)), model)
- return model
- def _persist_active_agents(self) -> None:
- """Persist the live in-flight agent count to ``gateway_state.json`` at every turn boundary.
- Passes ONLY ``active_agents`` so the read-merge-write keeps lifecycle state (gateway_state=None
- would clobber it). Best-effort: a failed write must never disrupt a turn. The running-turn
- set also feeds the day's peak of chats running at once (in memory, see shared_metrics_busy)."""
- from gateway.run import _write_runtime_status_quiet
- _write_runtime_status_quiet(active_agents=self._active_work_count())
- observe_running_chats(key for key, _ in self._running_agent_items())
- def _release_running_agent_state(
- self, session_key: str, *, run_generation: Optional[int] = None
- ) -> bool:
- """Pop ALL per-running-agent state for ``session_key`` (call at every site that ends a running
- turn); True when cleared. Persistent state (model overrides, voice mode, approvals) is NOT
- touched. With ``run_generation``, only clear if still current — a stale async unwind bumped
- by /stop or /new must not clobber a newer run (returns False)."""
- if not session_key or (
- run_generation is not None and not self._is_session_run_current(session_key, run_generation)
- ):
- return False
- state = self._peek_session_state(session_key)
- if state is not None:
- if state.turn.lease is not None:
- from gateway.active_lease_io import release_soon
- release_soon(state.turn.lease)
- # One structured reset instead of a drifting pop-list. Turn-lease tokens are deliberately NOT
- # cleared here — _release_turn_lease owns them.
- state.turn.clear()
- retired, state.turn.retired_agents = state.turn.retired_agents, []
- for agent in retired:
- # Evicted mid-turn (a one-turn agent): its turn no longer uses it.
- self._spawn_release_thread(self._release_evicted_agent_soft, (agent,),
- f"agent-retire-{str(session_key)[:24]}", inline_fallback=True,
- session_key=session_key)
- # Turn boundary: a running-agent slot was just released; persist the new (lower) in-flight count
- # so the dashboard readout stays current. Preserves gateway_state (see _persist_active_agents).
- self._persist_active_agents()
- return True
- def _drop_turn_slot(self, session_key: str, *, run_generation: Optional[int] = None) -> None:
- """Release the running-agent slot and evict the cached instance (/stop, eviction, reaper).
- ``_interrupt_requested`` is cleared only by the turn finalizer, so on a hung/still-draining
- run the flag would survive and silently kill the session's NEXT message (interrupted=True,
- api_calls=0, empty response); the next message rebuilds from history while the old agent
- keeps its flag so a hung drain still dies (#44212). With ``run_generation`` (the post-bump
- value ``_interrupt_running_turn`` returns), the release is generation-guarded: an async
- path awaits between bump and release, so a successor claiming the slot in that window must
- not have its sentinel/lease wiped by the displaced path's tail. Then sweep lease tokens
- from generations OLDER than the current one: a hung evicted turn's finalizer may never run,
- and each such generation would otherwise pin its token (and its ``_SessionLease``) forever.
- Identity-checked + idempotent, so a live successor's token is never affected."""
- self._release_running_agent_state(session_key, run_generation=run_generation)
- self._evict_cached_agent(session_key)
- state = self._peek_session_state(session_key)
- registry = getattr(self, "_turn_leases", None)
- if state is None or registry is None:
- return
- current = int(state.persistent.run_generation or 0)
- tokens = state.turn.lease_tokens
- for gen in [g for g in tokens if int(g) < current]:
- token = tokens.pop(gen)
- try:
- registry.release(token)
- except Exception:
- logger.debug("Failed to release displaced turn lease gen %s for %s", gen, session_key,
- exc_info=True)
- def _held_turn_lease(self, session_key: str, run_generation: int):
- """Return ``(registry, lease_tokens)`` when ``session_key`` holds a lease token for
- ``run_generation``, else None. Callers ``get``/``pop`` the token from the map themselves."""
- registry = getattr(self, "_turn_leases", None)
- state = self._peek_session_state(session_key) if session_key and registry is not None else None
- tokens = state.turn.lease_tokens if state is not None else None
- if tokens is None or run_generation not in tokens:
- return None
- return registry, tokens
- def _release_turn_lease(self, session_key: str, run_generation: int) -> bool:
- """Release the turn lease acquired by (``session_key``, ``run_generation``). Keyed by (routing
- key, run generation) so a stale unwind pops only ITS token; the registry's identity check
- refuses it if a newer turn holds the lease. Idempotent."""
- held = self._held_turn_lease(session_key, run_generation)
- if held is None:
- return False
- registry, tokens = held
- token = tokens.pop(run_generation)
- try:
- return registry.release(token)
- except Exception:
- logger.debug("Failed to release turn lease", exc_info=True)
- return False
- def _rebind_turn_lease(self, session_key: str, run_generation: int, new_session_id: str) -> bool:
- """Follow a mid-turn session_id rotation (compression) with the held turn lease, or an alias
- key resolving the new id could start a concurrent turn the lease never sees. Call at every
- mid-turn reassignment; no-op if no token."""
- held = self._held_turn_lease(session_key, run_generation) if new_session_id else None
- if held is None:
- return False
- registry, tokens = held
- try:
- return registry.rebind(tokens[run_generation], new_session_id)
- except Exception:
- logger.debug("Failed to rebind turn lease", exc_info=True)
- return False
- def _clear_conversation_scope(self, session_key: str, *, reason: str) -> None:
- """THE single conversation-boundary funnel (/new, /resume, suspension replacement,
- compression-exhausted reset). New conversation-scoped dicts go in _CONVERSATION_SCOPED_STATE
- so every boundary picks them up. Turn-scoped state (_running_agents/_ts, slot leases, turn-
- lease tokens) is owned by _release_running_agent_state and NOT cleared. Idle agent-cache
- eviction is NOT a boundary (a resumed turn rebuilds from these). getattr-guarded.
- Why a funnel: these boundaries used to each carry a hand-copied pop-list of the per-session dicts,
- and the lists drifted every time a new dict was added (#48031, #58403, #10702, #35809 were all
- "boundary X forgot dict Y" bugs — e.g. /new cleared the /model override but not the /model --once
- restore snapshot). Adding a new conversation-scoped dict now means adding its attribute name to
- _CONVERSATION_SCOPED_STATE below; every boundary picks it up automatically.
- """
- from gateway.run import _CONVERSATION_SCOPED_STATE
- if not session_key:
- return
- state = self._peek_session_state(session_key)
- if state is not None:
- state.conversation.clear()
- # Legacy plain-dict stores still in _CONVERSATION_SCOPED_STATE (not yet folded into
- # SessionState), e.g. _pending_model_notes. SessionState-backed names resolve to MutableMapping
- # views (not dict), so the isinstance(dict) guard skips them — already handled above.
- for attr in _CONVERSATION_SCOPED_STATE:
- store = getattr(self, attr, None)
- if isinstance(store, dict):
- store.pop(session_key, None)
- self._clear_session_boundary_security_state(session_key)
- logger.debug("Cleared conversation scope for %s (%s)", session_key, reason)
- def _clear_session_boundary_security_state(self, session_key: str) -> None:
- """Clear per-session control state that must not survive a boundary switch."""
- if not session_key:
- return
- pending_skills_reload_notes = getattr(self, "_pending_skills_reload_notes", None)
- if isinstance(pending_skills_reload_notes, dict):
- pending_skills_reload_notes.pop(session_key, None)
- state = self._peek_session_state(session_key)
- if state is not None:
- state.persistent.approvals = None
- state.persistent.update_prompt_pending = False
- for mod, attr, what in (
- ("tools.slash_confirm", "clear", "slash-confirm"), ("tools.approval", "clear_session", "approval"),
- ):
- try:
- clear = getattr(importlib.import_module(mod), attr)
- except Exception:
- continue
- try:
- clear(session_key)
- except Exception as e:
- logger.debug("Failed to clear %s state for session boundary %s: %s", what, session_key, e)
- # The persisted /yolo copy dies with the in-memory one, or the next turn's restore revives it. None, not
- # False: the next conversation follows its launch policy again (seeded once per boundary), and a
- # toggle-less chat stays off.
- store = getattr(self, "session_store", None)
- if store is not None:
- try:
- store.set_session_yolo(session_key, None)
- except Exception:
- # Never fails the boundary itself, but a surviving ON would come back after a restart.
- logger.warning("Failed to clear persisted yolo for session boundary %s", session_key, exc_info=True)
- def _begin_session_run_generation(self, session_key: str) -> int:
- """Claim a fresh, monotonically increasing run generation token (NEVER reset): a late result
- from a worker /stop or /new invalidated is recognized and dropped."""
- if not session_key:
- return 0
- persistent = self._session_state(session_key).persistent
- # Monotonic by design (#28686): incremented here, NEVER reset.
- persistent.run_generation = int(persistent.run_generation) + 1
- return persistent.run_generation
- def _invalidate_session_run_generation(self, session_key: str, *, reason: str = "") -> int:
- """Invalidate any in-flight run token for ``session_key``.
- Settles a pending one-shot model override first: the displaced turn's finalizer is
- generation-guarded and would otherwise leave ``/moa`` / ``/model --once`` in force."""
- self._restore_pending_one_turn_model_override(session_key)
- generation = self._begin_session_run_generation(session_key)
- if reason:
- logger.info("Invalidated run generation for %s → %d (%s)", session_key, generation, reason)
- return generation
- def _current_session_run_generation(self, session_key: str) -> int:
- """Current run generation for ``session_key`` (0 when the key tracks no run)."""
- state = self._peek_session_state(session_key)
- return int(state.persistent.run_generation) if state is not None else 0
- def _is_session_run_current(self, session_key: str, generation: int) -> bool:
- """Return True when ``generation`` is still current for ``session_key``."""
- if not session_key:
- return True
- return self._current_session_run_generation(session_key) == int(generation)
- def _bind_adapter_run_generation(self, adapter: Any, session_key: str, generation: int | None) -> None:
- """Bind a gateway run generation to the adapter's active-session event."""
- if not adapter or not session_key or generation is None:
- return
- with suppress(Exception):
- interrupt_event = getattr(adapter, "_active_sessions", {}).get(session_key)
- if interrupt_event is not None:
- interrupt_event._hermes_run_generation = int(generation)
- def _interrupt_running_turn(
- self, session_key: str, *, interrupt_reason: str, invalidation_reason: str, tool_reason: str | None = None,
- ) -> int:
- """Sync core shared by /stop, /new and eviction: request a hard interrupt on the in-flight
- agent, invalidate its run generation, and reap the tool processes that turn spawned.
- ``tool_reason`` names a system issuer (eviction); ``None`` keeps the user attribution of /stop and /new.
- Returns the post-bump generation."""
- from contextvars import copy_context
- from gateway.run import _AGENT_PENDING_SENTINEL, _reap_gateway_turn_processes, request_hard_interrupt
- state = self._peek_session_state(session_key)
- running_agent = state.turn.agent if state else None
- _process_task_id, _process_baseline = "", None
- if running_agent and running_agent is not _AGENT_PENDING_SENTINEL:
- # A raising interrupt implementation must not leave the slot unroutable: the generation
- # bump and release below are the cleanup that matters.
- with _log_suppressed(logging.WARNING, "Failed to interrupt running agent for %s; continuing",
- session_key, exc_info=True):
- request_hard_interrupt(running_agent, interrupt_reason, tool_reason=tool_reason)
- _process_task_id = getattr(running_agent, "_gateway_turn_process_task_id", "")
- _process_baseline = getattr(running_agent, "_gateway_turn_process_baseline", None)
- # Bump the generation BEFORE scheduling the reap thread and capture the post-bump value:
- # task_id is session-scoped, so a replacement turn spawning before the reap runs bumps it
- # again and the closure sees a stale generation and skips — the replacement's own baseline
- # covers its cleanup, so nothing stays unreaped.
- _generation_at_interrupt = self._invalidate_session_run_generation(session_key, reason=invalidation_reason)
- if _process_task_id and _process_baseline is not None:
- threading.Thread(
- target=copy_context().run,
- args=(_reap_gateway_turn_processes, _process_task_id, _process_baseline),
- kwargs={
- "source": "gateway_turn_interrupt",
- "is_still_current": lambda: self._is_session_run_current(session_key, _generation_at_interrupt),
- },
- name=f"gateway-turn-reaper-{_process_task_id[:12]}",
- daemon=True,
- ).start()
- return _generation_at_interrupt
- async def _interrupt_and_clear_session(
- self, session_key: str, source: SessionSource, *, interrupt_reason: str,
- invalidation_reason: str, release_running_state: bool = True,
- ) -> None:
- """Interrupt the current run and clear queued session state consistently."""
- if not session_key:
- return
- state = self._peek_session_state(session_key)
- running_agent = state.turn.agent if state else None
- _generation_at_interrupt = self._interrupt_running_turn(
- session_key, interrupt_reason=interrupt_reason, invalidation_reason=invalidation_reason,
- )
- from gateway.run import _AGENT_PENDING_SENTINEL
- # The turn's hard interrupt reaches only its in-turn children; background delegations were
- # detached at dispatch and would otherwise run to completion and wake the session later.
- # Each interrupted unit still returns as a completion (status=interrupted, partial output).
- from tools.async_delegation import interrupt_for_session
- interrupt_for_session(
- session_key=session_key, reason=invalidation_reason,
- parent_session_id=str(getattr(running_agent, "session_id", "") or ""))
- if running_agent and running_agent is not _AGENT_PENDING_SENTINEL:
- # Plugins holding a per-turn external resource (an outbound RPC blocked on a tool result
- # the loop will never consume) learn the turn is gone. Fires for /stop and the /new
- # running-agent fast path; the pending-sentinel /stop has no in-flight work, so it stays
- # silent. Dispatch failures are swallowed so a misbehaving plugin cannot break an interrupt.
- try:
- from hermes_cli.plugins import invoke_hook as _invoke_hook
- _invoke_hook(
- "agent_loop_stopped",
- session_key=session_key,
- platform=source.platform.value if source.platform else "",
- reason=interrupt_reason,
- invalidation_reason=invalidation_reason,
- )
- except Exception:
- logger.debug("agent_loop_stopped hook dispatch failed", exc_info=True)
- adapter = self._delivery_adapter_for(source)
- interrupt_session_activity = getattr(type(adapter), "interrupt_session_activity", None)
- if adapter and callable(interrupt_session_activity):
- metadata = self._thread_metadata_for_source(source)
- if _accepts_keyword(interrupt_session_activity, "metadata"):
- await adapter.interrupt_session_activity(session_key, source.chat_id, metadata=metadata)
- else:
- await adapter.interrupt_session_activity(session_key, source.chat_id)
- if adapter and hasattr(adapter, "get_pending_message"):
- # Discard a stale human follow-up (the slot held only user text when /stop started doing
- # this, 59575d6a917) — but an internal wake (async-delegation completion, notify+wake)
- # shares the slot now and was claim-settled on admission, so dropping it loses it for
- # good and the session idles until the next user message (#114456). Leave it parked for
- # the adapter's post-command drain; a wake queued behind a discarded human head is
- # promoted out of the overflow FIFO for the same reason. Whether a wake may still run
- # against a session /new just closed is decided where it is processed
- # (_resolve_async_delegation_session fails closed), not here.
- parked = adapter.get_pending_message(session_key)
- wake = parked if getattr(parked, "internal", False) else None
- if wake is None:
- overflow = self._overflow_queue(session_key) or []
- wake = next((e for e in overflow if getattr(e, "internal", False)), None)
- if wake is not None:
- overflow.remove(wake)
- if wake is not None:
- adapter._pending_messages[session_key] = wake
- if state is not None:
- state.persistent.pending_command_text = None
- if release_running_state:
- # Guarded release: a message that arrived during the awaits above may already run as
- # the successor generation — the displaced /stop tail must not wipe its slot.
- self._drop_turn_slot(session_key, run_generation=_generation_at_interrupt)
- async def _refresh_agent_cache_message_count(self, session_key: str, session_id: Optional[str]) -> None:
- """Re-baseline a cached agent's stored message_count after THIS turn — the coherence guard
- rebuilds on mismatch, so without this every turn would rebuild and destroy prompt caching.
- Only the count is refreshed, only if the same agent is still cached. DB errors leave the
- snapshot as-is (one spare rebuild).
- But the snapshot is taken at agent-BUILD time — before this turn writes its own user + assistant (+
- tool) rows — and the cache entry is never rewritten on a reuse. See #45966.
- """
- from gateway.run import _AGENT_PENDING_SENTINEL
- _cache_lock = getattr(self, "_agent_cache_lock", None)
- _cache = getattr(self, "_agent_cache", None)
- if self._session_db is None or not session_id or not _cache_lock or _cache is None:
- return
- try:
- _sess_row = await self._session_db.get_session(session_id)
- _live = _sess_row.get("message_count", 0) if _sess_row else None
- except Exception:
- return
- if _live is None:
- return
- with _cache_lock:
- cached = _cache.get(session_key)
- # Only re-baseline a live 3-tuple entry; skip pending sentinels, legacy 2-tuples (they opt
- # out of the guard), and entries evicted/rebuilt mid-turn. A snapshot taken for a different
- # session_id (same session_key, different conversation) is a different DB row — leave it.
- if not (isinstance(cached, tuple) and len(cached) > 2 and cached[0] is not _AGENT_PENDING_SENTINEL):
- return
- _snapshot_sid = cached[3] if len(cached) > 3 else None
- if (_snapshot_sid is not None and _snapshot_sid != session_id) or cached[2] == _live:
- return
- # Legacy 3-tuple keeps its 3-element shape for callers indexing ``cached[2]``.
- _cache[session_key] = (cached[0], cached[1], _live) + (() if _snapshot_sid is None else (_snapshot_sid,))
- def _set_pending_turn_sidecar_notes(self, session_key: str, notes: list[str]) -> None:
- """Stage per-turn must-deliver notes for the next agent run (one-shot)."""
- if not session_key or not notes:
- return
- self._session_state(session_key).conversation.sidecar_notes = list(notes)
- def _consume_pending_turn_sidecar_notes(self, session_key: str) -> list[str]:
- state = self._peek_session_state(session_key) if session_key else None
- if state is None:
- return []
- staged, state.conversation.sidecar_notes = state.conversation.sidecar_notes, []
- return list(staged) if isinstance(staged, list) else []
- def _voice_channel_sidecar_note(self, event, source: SessionSource, session_key: str) -> Optional[str]:
- """``[Voice channel now: ...]`` note when VC state changed; ``None`` when unchanged so per-turn
- member/speaking churn can't touch the prompt."""
- if source.platform != Platform.DISCORD:
- return None
- adapter = self._delivery_adapter_for(source)
- guild_id = self._get_guild_id(event)
- if not (guild_id and adapter and hasattr(adapter, "get_voice_channel_context")):
- return None
- try:
- vc_now = adapter.get_voice_channel_context(guild_id) or ""
- except Exception:
- logger.debug("voice-channel context read failed", exc_info=True)
- return None
- vc_prev = None
- if session_key:
- _vc_state = self._session_state(session_key)
- vc_prev, _vc_state.conversation.vc_last = _vc_state.conversation.vc_last, vc_now
- if vc_now == (vc_prev if vc_prev is not None else ""):
- return None
- return f"[Voice channel now: {vc_now or 'not connected to a voice channel'}]"
- async def _rehydrate_prompt_pins(self, session_key: str, expected_session_id: Optional[str]) -> None:
- """Adopt the durable pin snapshot for an internal turn when this process holds no context pin
- for *session_key*: a restart, or an eviction (/stop, /undo, /model), which clears only
- ``ephemeral_pin``. Without it the wake renders from its origin-only source (no chat or user
- names) and the next human turn re-keys back, rewriting the system bytes twice. Human turns
- never call this, so an evicted agent's next human turn still re-renders."""
- state = self._peek_session_state(session_key)
- if state is not None and state.conversation.ephemeral_pin is not None:
- return
- try:
- pin = sanitize_prompt_pin(await self.async_session_store.get_prompt_pin(
- session_key, expected_session_id=expected_session_id))
- except Exception:
- # Cache continuity only: a routing-store fault must not fail the turn.
- logger.debug("Failed to read persisted prompt pin for %s", session_key, exc_info=True)
- return
- if pin is None:
- return
- conversation = self._session_state(session_key).conversation
- conversation.ephemeral_pin = (pin["context_key"], pin["context_prompt"], pin["redact_pii"])
- if conversation.channel_pin is None:
- conversation.channel_pin = (pin["channel_prompt"], pin["parent_chat_id"])
- async def _persist_prompt_pins(self, session_key: Optional[str], expected_session_id: Optional[str]) -> None:
- """Persist this conversation's pins before the agent runs; the store no-ops an unchanged
- snapshot and refuses a session that has moved on."""
- state = self._peek_session_state(session_key) if session_key else None
- if state is None:
- return
- ephemeral_pin, channel_pin = state.conversation.ephemeral_pin, state.conversation.channel_pin
- if ephemeral_pin is None or channel_pin is None:
- return
- snapshot = {
- "version": PROMPT_PIN_VERSION, "context_key": ephemeral_pin[0],
- "context_prompt": ephemeral_pin[1], "redact_pii": ephemeral_pin[2],
- "channel_prompt": channel_pin[0], "parent_chat_id": channel_pin[1],
- }
- try:
- await self.async_session_store.set_prompt_pin(
- session_key, snapshot, expected_session_id=expected_session_id)
- except Exception:
- # Durability protects cache continuity; a store outage must not block the user turn.
- logger.debug("Failed to persist prompt pin for %s", session_key, exc_info=True)
- def _pinned_session_context_prompt(
- self, context, redact_pii: bool, session_key: Optional[str], *, internal: bool = False,
- ) -> str:
- """Session-context prompt pinned per session: key hit → pinned bytes reused VERBATIM (immune
- to renderer nondeterminism); key miss → re-render and re-pin (rename, topic edit, /sethome).
- ``internal`` events (kanban wakes, delegation completions, watch notifications) carry a
- source rebuilt from the persisted origin, without chat_name/user_name/message_id. Rendering
- from it re-keyed the pin, and the next human turn re-keyed it back (A→B→A), rewriting
- already-sent system bytes each time. An internal event is never a real metadata change, so
- it reuses an existing pin verbatim (restored by ``_rehydrate_prompt_pins`` after a restart or an eviction); with
- no pin yet it renders and pins as usual. The pin records the ``privacy.redact_pii`` it was
- rendered under: bytes from another privacy policy are never reused, even by an internal
- event. Callers pass ``event.internal or event.inherit_channel_pin`` (synthetic
- goal/heartbeat/resume turns strip ``message_id``) so this pin and ``_pinned_channel_inputs``
- reuse on the same decision."""
- _pin_state = self._peek_session_state(session_key) if session_key else None
- _eph_pin = _pin_state.conversation.ephemeral_pin if _pin_state else None
- if _eph_pin is not None and _eph_pin[2] != redact_pii:
- _eph_pin = None
- if internal and _eph_pin is not None:
- return _eph_pin[1]
- _eph_key = self._ephemeral_change_key(context, redact_pii)
- if _eph_pin is not None and _eph_pin[0] == _eph_key:
- return _eph_pin[1]
- text = build_session_context_prompt(context, redact_pii=redact_pii)
- if session_key:
- self._session_state(session_key).conversation.ephemeral_pin = (_eph_key, text, redact_pii)
- return text
- def _pinned_channel_inputs(
- self, session_key: Optional[str], channel_prompt: Optional[str], source: SessionSource, *, internal: bool,
- ):
- """``(channel_prompt, source)`` for this turn's agent run.
- The ephemeral system prompt also appends ``channel_prompt`` and the ``channel_overrides``
- prompt (looked up by chat/thread/``parent_chat_id``). Events with no channel inputs of
- their own (internal events, and synthetic goal/heartbeat/resume turns — see
- ``MessageEvent.inherit_channel_pin``) carry ``channel_prompt=None`` and a source without
- ``parent_chat_id``, so they dropped both and toggled the system prompt like the context pin
- did. Human turns record their inputs; turns that ask to reuse the pin get them back
- (``_rehydrate_prompt_pins`` restores both after a restart)."""
- if not session_key:
- return channel_prompt, source
- if not internal:
- self._session_state(session_key).conversation.channel_pin = (channel_prompt, source.parent_chat_id)
- return channel_prompt, source
- state = self._peek_session_state(session_key)
- pin = state.conversation.channel_pin if state else None
- if pin is None:
- return channel_prompt, source
- pinned_prompt, pinned_parent = pin
- if pinned_parent and not source.parent_chat_id:
- from gateway.session_identity import replace_source
- source = replace_source(source, parent_chat_id=pinned_parent)
- return pinned_prompt, source
- @staticmethod
- def _ephemeral_change_key(context, redact_pii: bool) -> str:
- """Hash the exact inputs ``build_session_context_prompt`` renders. Invariant
- (test_prompt_tail_freeze.py): any input whose change alters the rendered bytes MUST appear
- here — omission means a stale pinned prompt; extras only re-render."""
- import hashlib
- src = context.source
- def _s(v) -> str:
- return str(v or "")
- discord_ids: tuple = ()
- discord_tools = ""
- if src.platform == Platform.DISCORD:
- from gateway.session import _discord_tools_loaded
- discord_tools = "1" if _discord_tools_loaded() else "0"
- # message_id is not rendered (value nor presence): it arrives per-turn in the user message.
- discord_ids = (
- _s(src.guild_id), _s(src.parent_chat_id), _s(src.thread_id), _s(src.chat_id),
- )
- # Slack's capability-aware platform note is gated on _slack_tools_loaded() — the gate state must
- # be in the key (same parity contract as the Discord gate above) so a config / MCP-registration
- # flip re-renders once instead of serving a stale pinned note for the rest of the session.
- slack_tools = ""
- if src.platform == Platform.SLACK:
- from gateway.session import _slack_tools_loaded
- slack_tools = "1" if _slack_tools_loaded() else "0"
- try:
- from hermes_constants import display_hermes_home
- home_display = str(display_hermes_home())
- except Exception:
- home_display = ""
- key_tuple = (
- src.platform.value if src.platform else "",
- _s(src.chat_id), _s(src.thread_id), _s(src.chat_type), _s(src.chat_name), _s(src.chat_topic),
- _s(src.user_name), _s(src.user_id), _s(getattr(src, "profile", None)),
- bool(context.shared_multi_user_session), discord_ids, discord_tools, slack_tools,
- tuple(p.value for p in context.connected_platforms),
- tuple(
- (p.value, _s(getattr(hc, "name", "")), _s(getattr(hc, "chat_id", "")))
- for p, hc in context.home_channels.items()
- ),
- bool(redact_pii), home_display,
- )
- return hashlib.sha256(repr(key_tuple).encode("utf-8")).hexdigest()
- def _evict_cached_agent(self, session_key: str, *, commit_memory: bool = False,
- release_after_turn: bool = False) -> None:
- """Remove a cached agent (/new, /model, ...) and soft-release its LLM client pool (AIAgent
- holds reference cycles; without it RSS grows across /new). Soft = frees clients and child
- subagents but PRESERVES terminal sandbox / browser / bg processes since the session may
- resume; true boundaries call ``_cleanup_agent_resources`` first. Cleanup runs on a daemon
- thread so ``_agent_cache_lock`` never spans slow socket teardown.
- Pops the entry AND soft-releases the evicted agent's LLM client pool so the httpx connection
- (sockets + held buffers) is freed promptly rather than waiting on CPython GC — AIAgent holds
- reference cycles (callbacks, tool state) that delay refcount collection, so a manual release is
- required to keep gateway RSS flat across many /new, /model, undo and reset operations (#29298, same
- leak class as #25315).
- ``release_after_turn``: an agent still mid-turn is released when that turn's running slot
- clears (``_release_running_agent_state``) instead of being left to the garbage collector.
- """
- from gateway.run import _AGENT_PENDING_SENTINEL
- # Prompt-stability state rides the agent-cache lifecycle: a fresh agent must re-render its
- # session-context bytes (the pin) and re-see the current voice-channel state once.
- state = self._peek_session_state(session_key)
- if state is not None:
- state.conversation.ephemeral_pin = None
- state.conversation.vc_last = None
- # Tests build runners with ``_agent_cache_lock = None``; evict lock-free then. With the lock
- # present ``_agent_cache`` is read directly (an initialized runner always has it).
- _lock = getattr(self, "_agent_cache_lock", None)
- evicted = None
- if _lock:
- with _lock:
- evicted = self._agent_cache.pop(session_key, None)
- else:
- _cache = getattr(self, "_agent_cache", None)
- if _cache is not None:
- evicted = _cache.pop(session_key, None)
- agent = _first_agent(evicted)
- # Never tear down an agent that's mid-turn — its client, sandbox and child subagents are in use.
- if agent is None or agent is _AGENT_PENDING_SENTINEL:
- return
- if id(agent) in self._running_agent_ids():
- if release_after_turn and state is not None:
- state.turn.retired_agents.append(agent)
- return
- if commit_memory:
- target, args = self._commit_then_release_soft, (agent, session_key, True)
- else:
- target, args = self._release_evicted_agent_soft, (agent,)
- self._spawn_release_thread(target, args, f"agent-evict-{str(session_key)[:24]}", inline_fallback=True,
- session_key=session_key)
- def _evict_cached_agent_at_boundary(self, session_key: str) -> None:
- """Reset and compress end the conversation the provider was tracking, so the old agent's
- transcript is committed (``on_session_end``) before its clients go; a plain evict is not."""
- self._evict_cached_agent(session_key, commit_memory=True)
- def _spawn_release_thread(self, target, args: tuple, name: str, *, inline_fallback: bool,
- session_key: Optional[str] = None) -> None:
- """Run a release on a daemon thread. ``inline_fallback`` runs it inline (best-effort) when no
- thread can start (interpreter shutdown); otherwise a spawn failure propagates, as on main.
- The thread runs inside the owning profile's scope (see ``_run_release_in_profile_scope``)."""
- import contextvars
- ctx = contextvars.copy_context()
- try:
- threading.Thread(target=ctx.run, args=(self._run_release_in_profile_scope, target, args, session_key),
- daemon=True, name=name).start()
- except Exception:
- if not inline_fallback:
- raise
- with suppress(Exception):
- ctx.run(self._run_release_in_profile_scope, target, args, session_key)
- def _run_release_in_profile_scope(self, target, args: tuple, session_key: Optional[str]) -> None:
- """Call ``target(*args)`` under the profile that OWNS ``session_key``.
- Threads start with an EMPTY context, so a bare thread would commit end-of-session memory
- (provider ``on_session_end`` reads credentials/home at call time) under the launch profile.
- And the LRU-cap eviction runs inside the REQUESTING turn, whose agent may belong to another
- profile — so "some scope is present" is not enough either. The owner comes from the session
- key: a named profile's home, else the DEFAULT profile (``agent:main:`` keys), which is the
- root Hermes dir even when the gateway was launched under a named profile. Its scope is
- entered unless the current one already is the owner's."""
- from agent.secret_scope import current_secret_scope, is_multiplex_active
- scope = nullcontext()
- if is_multiplex_active():
- from gateway.run import _profile_runtime_scope
- from hermes_constants import get_default_hermes_root, get_hermes_home, hermes_home_key
- owner = None
- store = getattr(self, "session_store", None)
- if session_key and store is not None:
- try:
- owner = store._profile_home_for_key(session_key)
- except Exception:
- logger.warning("Could not resolve the owning profile for %s; releasing under the default profile",
- session_key, exc_info=True)
- owner_home = Path(owner) if owner else get_default_hermes_root()
- if current_secret_scope() is None or hermes_home_key(get_hermes_home()) != hermes_home_key(owner_home):
- scope = _profile_runtime_scope(owner_home)
- with scope:
- target(*args)
- def _commit_memory_before_soft_evict(self, agent: Any, key: str, boundary: bool = False) -> None:
- """Commit the live transcript to memory providers before resource-only eviction. A session
- boundary (reset, compress) also ends the context engine's session; that engine can run without a provider."""
- if agent is None or not hasattr(agent, "commit_memory_session"):
- return
- has_provider = getattr(agent, "_memory_manager", None) is not None
- has_engine = boundary and getattr(agent, "context_compressor", None) is not None
- if not has_provider and not has_engine:
- return
- try:
- messages = getattr(agent, "_session_messages", None)
- agent.commit_memory_session(messages if isinstance(messages, list) else None)
- logger.debug(
- "Committed on_session_end extraction before soft-evicting "
- "session=%s (resource eviction)", key,
- )
- except Exception as _e:
- logger.debug("Pre-evict memory commit failed for %s: %s", key, _e)
- def _commit_then_release_soft(self, agent: Any, key: str, boundary: bool = False) -> None:
- """Commit end-of-session memory (if warranted), then soft-release — on the daemon eviction
- thread. Order matters: commit needs the live memory manager before ``release_clients``."""
- self._commit_memory_before_soft_evict(agent, key, boundary)
- self._release_evicted_agent_soft(agent)
- def _release_evicted_agent_soft(self, agent: Any) -> None:
- """Soft cleanup for cache-evicted agents: unlike _cleanup_agent_resources, the session may
- resume, so terminal sandbox, browser daemon and bg processes outlive the AIAgent instance."""
- if agent is None:
- return
- with suppress(Exception):
- if hasattr(agent, "release_clients"):
- agent.release_clients()
- else:
- # Older agent instance (shouldn't happen in practice) — legacy full-close path.
- self._cleanup_agent_resources(agent)
- # Free conversation history — tens of MB of tool output on heavy 100+-tool-call sessions.
- # release_clients() preserves session tool state for resume, but the message list is rebuilt from
- # persisted session JSON on the next turn, so dropping it here is safe.
- if hasattr(agent, "_session_messages"):
- agent._session_messages = []
- # _db_flush_scan_prefix (run_agent.py, stamped on every successful flush) is a shallow copy
- # sharing every message dict of the flushed transcript, so leaving it pins the multi-MB strings
- # this eviction frees. Pressure-evictable agents have flushed by definition, so it's populated.
- if hasattr(agent, "_db_flush_scan_prefix"):
- agent._db_flush_scan_prefix = None
- def _agent_cache_bounds(self):
- """Operator-configured agent-cache bounds, resolved once per process (lazily, not in
- ``__init__``, so ``__new__``-constructed test / slash-command runners work too)."""
- from gateway.run import _load_gateway_config
- bounds = getattr(self, "_agent_cache_bounds_cache", None)
- if bounds is None:
- from gateway.agent_cache_pressure import resolve_agent_cache_bounds
- try:
- bounds = resolve_agent_cache_bounds(_load_gateway_config())
- except Exception as _e:
- logger.debug("Agent cache bounds config read failed: %s", _e)
- # Resolve from an empty config rather than bare AgentCacheBounds(): the dataclass default
- # has memory_high_mb=None (pressure pass OFF) but an *absent* section means "auto" — a
- # transient config read failure must not permanently disable the OOM valve.
- bounds = resolve_agent_cache_bounds({})
- self._agent_cache_bounds_cache = bounds
- return bounds
- def _agent_cache_cap(self) -> int:
- """Effective LRU cap — the configured override, else the default."""
- from gateway.run import _AGENT_CACHE_MAX_SIZE
- return self._agent_cache_bounds().max_size or _AGENT_CACHE_MAX_SIZE
- def _agent_cache_idle_ttl(self) -> float:
- """Effective idle TTL in seconds — configured override, else default."""
- from gateway.run import _AGENT_CACHE_IDLE_TTL_SECS
- return self._agent_cache_bounds().idle_ttl_secs or _AGENT_CACHE_IDLE_TTL_SECS
- def _sweep_agent_cache_under_pressure(self) -> int:
- """Shed cached transcripts once the gateway heap nears its budget; returns count evicted.
- The LRU cap counts entries and the idle sweep counts seconds; neither knows one cached agent
- pins a full ``_session_messages`` transcript (tens of MB), so RSS climbs until the cgroup
- throttles. Above the anonymous-RSS budget this soft-evicts LRU agents (transcript rebuilt
- from the persisted session next turn). Never touched: agents mid-turn, the most recently
- used sessions, and transcripts not yet on disk.
- A gateway serving many chats can hold every warm transcript for the TTL window.
- Pressure eviction bounds that heap before the cgroup throttles and SIGTERM can
- no longer flush inside systemd's stop timeout (#80764).
- """
- from gateway.run import _AGENT_PENDING_SENTINEL
- from gateway.agent_cache_pressure import (
- plan_pressure_evictions, read_anon_rss_mb, transcript_persistence_caught_up
- )
- bounds = self._agent_cache_bounds()
- _cache = getattr(self, "_agent_cache", None)
- _lock = getattr(self, "_agent_cache_lock", None)
- # Nothing cached — whatever is using the heap, it isn't us, and warning about it every tick
- # would point at the wrong subsystem.
- if not bounds.memory_high_mb or not _cache or _lock is None:
- return 0
- rss_mb = read_anon_rss_mb()
- if rss_mb is None or rss_mb < bounds.memory_high_mb:
- return 0
- running_ids = self._running_agent_ids()
- def _is_live(agent: Any) -> bool:
- return agent is not None and agent is not _AGENT_PENDING_SENTINEL and id(agent) not in running_ids
- def _is_evictable(key: str, agent: Any) -> bool:
- return _is_live(agent) and transcript_persistence_caught_up(agent)
- with _lock:
- ordered = [(key, _first_agent(entry)) for key, entry in _cache.items()]
- plan = plan_pressure_evictions(
- ordered, is_evictable=_is_evictable, max_evictions=bounds.max_evictions_per_pass,
- protect_recent=bounds.protect_recent,
- )
- for key, _ in plan:
- _cache.pop(key, None)
- if not plan:
- _mid_turn = sum(1 for _, a in ordered if a is not None and id(a) in running_ids)
- _unflushed = sum(1 for _, a in ordered if _is_live(a) and not transcript_persistence_caught_up(a))
- logger.warning(
- "Agent cache pressure: anon RSS %dMB over budget %dMB but no "
- "evictable session (%d cached, %d mid-turn, %d blocked on "
- "un-flushed persistence)%s",
- rss_mb, bounds.memory_high_mb, len(ordered), _mid_turn, _unflushed,
- (
- " — transcripts are not reaching the session DB "
- "(session persistence disabled or failing?); the memory "
- "valve cannot shed sessions until they persist."
- if _unflushed and not _mid_turn
- else " — memory will keep climbing until those turns finish."
- ),
- )
- return 0
- evicted_count = len(plan)
- logger.warning(
- "Agent cache pressure: anon RSS %dMB over budget %dMB — evicting %d LRU session(s): %s",
- rss_mb, bounds.memory_high_mb, evicted_count, ", ".join(key for key, _ in plan),
- )
- try:
- threading.Thread(target=self._release_pressure_batch, args=(plan,), daemon=True,
- name="agent-cache-pressure").start()
- except Exception:
- # Thread spawn failed (interpreter shutdown): release inline, unguarded (as on main).
- self._release_pressure_batch(plan)
- # _release_pressure_batch drains `plan` in place (so the trim runs with no lingering agent
- # refs) — len(plan) is 0 once the daemon thread finishes, hence the pre-captured count.
- return evicted_count
- def _release_pressure_batch(self, plan: list[tuple]) -> None:
- """Release a pressure-evicted batch sequentially on one daemon thread, then ``malloc_trim`` so
- RSS actually falls. The plan is drained (``pop`` + ``del``), not iterated, so no local
- reference pins evicted agents during ``gc.collect`` + trim (else the valve over-evicts)."""
- while plan:
- key, agent = plan.pop(0) # FIFO — evict LRU-first order preserved
- try:
- # Pressure sweeps run from the unscoped housekeeping watcher: enter each owner's scope.
- self._run_release_in_profile_scope(self._commit_then_release_soft, (agent, key), key)
- except Exception as _e:
- logger.debug("Pressure release failed for %s: %s", key, _e)
- del agent
- with suppress(Exception):
- from hermes_cli.mem_trim import trim_memory
- trim_memory(force=True, reason="agent_cache_pressure")
- def _enforce_agent_cache_cap(self) -> None:
- """Evict oldest cached agents past the LRU cap (requires _agent_cache_lock); cleanup on a
- daemon thread. Mid-turn agents are SKIPPED, so the cache may stay over cap until the next
- insert."""
- _cache = getattr(self, "_agent_cache", None)
- # OrderedDict.popitem(last=False) pops oldest; plain dict lacks the arg so skip enforcement
- # if a test fixture swapped the cache type.
- if _cache is None or not hasattr(_cache, "move_to_end"):
- return
- # Snapshot of agent instances mid-turn, keyed by id() so lookup is O(1) and independent of
- # AIAgent.__eq__ (which MagicMock overrides in tests).
- running_ids = self._running_agent_ids()
- # Walk LRU → MRU; only the first (size - cap) LRU positions are candidates. An active slot is
- # SKIPPED rather than evicting a newer entry — that would penalise a fresh session (no cache
- # history) to protect a long-running one. Cache may stay over cap until the next insert.
- cap = self._agent_cache_cap()
- candidates = [(key, _tuple_agent(_cache.get(key))) for key in list(_cache.keys())[:max(0, len(_cache) - cap)]]
- evict_plan = [(key, agent) for key, agent in candidates if agent is None or id(agent) not in running_ids]
- for key, _ in evict_plan:
- _cache.pop(key, None)
- remaining_over_cap = len(_cache) - cap
- if remaining_over_cap > 0:
- logger.warning(
- "Agent cache over cap (%d > %d); %d excess slot(s) held by "
- "mid-turn agents — will re-check on next insert.",
- len(_cache), cap, remaining_over_cap,
- )
- for key, agent in evict_plan:
- logger.info("Agent cache at cap; evicting LRU session=%s (cache_size=%d)", key, len(_cache))
- if agent is not None:
- # Commit end-of-session memory, then soft-release, both on the daemon thread so the
- # (possibly network-bound) provider call never blocks the held cache lock.
- self._spawn_release_thread(self._commit_then_release_soft, (agent, key), f"agent-cache-evict-{key[:24]}",
- inline_fallback=False, session_key=key)
- def _sweep_idle_cached_agents(self) -> int:
- """Evict cached agents idle past the idle TTL (lock acquired internally; cleanup on daemon
- threads; mid-turn agents SKIPPED); returns the number evicted."""
- _cache = getattr(self, "_agent_cache", None)
- _lock = getattr(self, "_agent_cache_lock", None)
- if _cache is None or _lock is None:
- return 0
- now = time.time()
- idle_ttl = self._agent_cache_idle_ttl()
- to_evict: list[tuple] = []
- running_ids = self._running_agent_ids()
- with _lock:
- for key, entry in list(_cache.items()):
- agent = _tuple_agent(entry)
- if agent is None or id(agent) in running_ids:
- continue # mid-turn — don't tear it down
- last_activity = getattr(agent, "_last_activity_ts", None)
- if last_activity is None or (now - last_activity) <= idle_ttl:
- continue
- to_evict.append((key, agent))
- for key, _ in to_evict:
- _cache.pop(key, None)
- for key, agent in to_evict:
- logger.info("Agent cache idle-TTL evict: session=%s (idle=%.0fs)", key, now - getattr(agent, "_last_activity_ts", now))
- self._spawn_release_thread(self._commit_then_release_soft, (agent, key), f"agent-cache-idle-{key[:24]}",
- inline_fallback=False, session_key=key)
- return len(to_evict)
|