memory_manager.py 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935
  1. """MemoryManager — fans the agent's memory hooks out to registered providers.
  2. The builtin provider is always allowed; only ONE external plugin provider may be
  3. registered at a time (tool-schema bloat, conflicting backends).
  4. """
  5. from __future__ import annotations
  6. import contextvars
  7. import inspect
  8. import json
  9. import logging
  10. import re
  11. import threading
  12. import time
  13. from concurrent.futures import Future, ThreadPoolExecutor, wait
  14. from functools import partial
  15. from typing import Any, Callable, Dict, List, Optional
  16. from agent.redact import redact_for_egress
  17. from agent.memory_caller_scope import bound_scope, caller_scope, claim_instance, in_owner_home, refused_tool_message
  18. # Moved to agent.memory_context / agent.memory_manager_tools; still importable here because agent.memory_manager is a public
  19. # module for plugins (hermes_cli/plugin_validate_private_imports.py).
  20. from agent.memory_context import StreamingContextScrubber, build_memory_context_block, sanitize_context
  21. from agent.memory_manager_tools import (inject_memory_provider_tools, memory_provider_tools_enabled,
  22. memory_provider_tools_exposed, normalize_tool_schema)
  23. from agent.memory_provider import MemoryProvider, PRE_COMPRESS_CHECKPOINT_API_VERSION, ctx_bound, spawn_context_thread
  24. from agent.skill_commands import extract_user_instruction_from_skill_message
  25. from hermes_constants import hermes_home_key
  26. from tools.hook_output_spill import get_spill_config, spill_if_oversized
  27. from tools.registry import tool_error
  28. logger = logging.getLogger(__name__)
  29. # Providers that predate the checkpoint-API attribute are on the best-effort v1 contract.
  30. _LEGACY_PRE_COMPRESS_API_VERSION = 1
  31. # shutdown_all() drain bound; workers are daemon threads so a wedged provider never
  32. # blocks interpreter exit.
  33. _SYNC_DRAIN_TIMEOUT_S = 5.0
  34. _EXTERNAL_PREFETCH_TIMEOUT_S = 8.0
  35. # A background provider call (sync_turn / queue_prefetch) running longer than this is treated as
  36. # stuck. Well above the shutdown drain bound and the prefetch timeout so a slow-but-healthy backend
  37. # is never counted.
  38. _BACKGROUND_CALL_STALL_S = 30.0
  39. # Admission for external provider calls per (home, provider); see _StuckCallGate. Python cannot stop a
  40. # thread, so every stuck call pins one until it returns. New calls stop once this many calls that started
  41. # after the provider's last successful return are overdue; older orphans belong to an outage it has
  42. # since recovered from.
  43. _MAX_STUCK_CALLS = 4
  44. # While the gate is closed, one call per interval goes through, so a recovered backend proves itself
  45. # without waiting for an orphan to return.
  46. _PROBE_INTERVAL_S = 60.0
  47. # Hard cap on calls in flight per (home, provider), orphans included: failed probes pin one more thread
  48. # each, so admission (probes too) stops here and a backend that stays wedged pins a bounded number.
  49. _MAX_INFLIGHT_CALLS = 16
  50. # -- Signature introspection (providers are duck-typed; call shapes vary) -----
  51. def _signature_params(fn: Callable[..., Any]):
  52. """``fn``'s parameter mapping, or None when uninspectable (C callables, exotic proxies)."""
  53. try:
  54. return inspect.signature(fn).parameters
  55. except (TypeError, ValueError):
  56. return None
  57. def _has_var_kwargs(params) -> bool:
  58. return any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
  59. def _accepts_require_checkpoint(fn: Callable[..., Any]) -> bool:
  60. """True if ``fn`` can receive the ``require_checkpoint`` keyword (unreadable signatures -> False).
  61. Bare-shape v2 providers (``on_pre_compress(self, messages)``) would raise TypeError on the
  62. keyword, which the host would re-raise as a checkpoint failure despite a successful write.
  63. """
  64. params = _signature_params(fn)
  65. if params is None:
  66. return False
  67. kind = getattr(params.get("require_checkpoint"), "kind", None)
  68. return _has_var_kwargs(params) or kind in (inspect.Parameter.KEYWORD_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD)
  69. # -- Provider egress scrubbing -----------------------------------------------
  70. def _scrub(text: str) -> str:
  71. """The shared egress scrub. Like chat-platform and cron delivery it holds even when
  72. ``security.redact_secrets`` is off (that setting governs local logs, not what leaves the agent)
  73. and fails closed."""
  74. return redact_for_egress(text)
  75. _TOKEN_RE = re.compile(r"[A-Za-z0-9_\-./+]{16,}")
  76. def _redact_for_provider(*values: Any) -> Any:
  77. """Copies of ``values`` with every string scrubbed before it reaches a memory provider.
  78. Providers archive whatever they are handed (#115104), so each manager fan-out (turn sync, recall
  79. queries, session-end and pre-compress transcripts, memory-tool mirrors, delegation results, provider
  80. tool args) goes through ``_scrub``. Walks dicts/lists/tuples so tool-call arguments and multimodal text
  81. parts are covered; ``data:`` URLs (inline media) pass through. A value the scrub masked anywhere in the
  82. hand-off is then masked everywhere in it, so a key detected in a tool result is also caught where the
  83. model repeats it bare in prose. Never mutates the caller's objects: unchanged values come back as-is,
  84. changed containers as shallow copies. One value in → one value out; several → a tuple.
  85. """
  86. learned: set = set()
  87. def walk(value: Any, fix) -> Any:
  88. if isinstance(value, str):
  89. return value if not value or value.startswith("data:") else fix(value)
  90. if isinstance(value, dict):
  91. out = {k: walk(v, fix) for k, v in value.items()}
  92. return value if all(out[k] is value[k] for k in value) else out
  93. if isinstance(value, (list, tuple)):
  94. items = [walk(v, fix) for v in value]
  95. if all(new is old for new, old in zip(items, value)):
  96. return value
  97. return tuple(items) if isinstance(value, tuple) else items
  98. return value
  99. def scrub(text: str) -> str:
  100. out = _scrub(text)
  101. if out != text:
  102. learned.update(set(_TOKEN_RE.findall(text)) - set(_TOKEN_RE.findall(out)))
  103. return out
  104. scrubbed = [walk(v, scrub) for v in values]
  105. if learned:
  106. pattern = re.compile("|".join(re.escape(t) for t in sorted(learned, key=len, reverse=True)))
  107. scrubbed = [walk(v, lambda t: pattern.sub("[redacted]", t) if pattern.search(t) else t) for v in scrubbed]
  108. return scrubbed[0] if len(scrubbed) == 1 else tuple(scrubbed)
  109. def _redact_messages_for_egress(messages: Optional[list[Any]]) -> Optional[list[Any]]:
  110. """Scrubbed copies of whole transcript rows for a provider; a ``pre_llm_call`` veto's typed text
  111. (display-only metadata) never leaves the host."""
  112. from agent.message_metadata import without_withheld_prompts
  113. return None if messages is None else list(_redact_for_provider(without_withheld_prompts(messages)))
  114. class _StuckCallGate:
  115. """Process-wide admission for one kind of external provider call, keyed by (Hermes home, provider).
  116. A new MemoryManager is built per agent, so a per-instance view would let every fresh manager start
  117. one more call against a hung backend and pin one more thread. Shared, the gate closes once
  118. _MAX_STUCK_CALLS calls that started after the key's last successful return are past their deadline,
  119. admits one half-open probe per _PROBE_INTERVAL_S while closed, and never lets more than
  120. _MAX_INFLIGHT_CALLS be in flight. Calls not yet overdue are not counted, so healthy concurrent
  121. sessions run side by side. Each call removes only its own entry.
  122. """
  123. def __init__(self) -> None:
  124. self.lock = threading.Lock()
  125. self.calls: dict[tuple[str, str], dict[object, tuple[float, float]]] = {} # token -> (start, deadline)
  126. self.last_ok: dict[tuple[str, str], float] = {}
  127. self.probe_at: dict[tuple[str, str], float] = {} # interval start: gate close, then each probe
  128. self.skips: dict[tuple[str, str], int] = {}
  129. def enter(self, key: tuple[str, str], token: object, timeout: float) -> Optional[tuple[int, int, int]]:
  130. """Register ``token`` and return None, or ``(stuck, in_flight, skips)`` when the call is skipped."""
  131. now = time.monotonic()
  132. with self.lock:
  133. calls = self.calls.setdefault(key, {})
  134. last_ok = self.last_ok.get(key, float("-inf"))
  135. stuck = sum(start > last_ok and now > deadline for start, deadline in calls.values())
  136. if len(calls) >= _MAX_INFLIGHT_CALLS:
  137. admit = False
  138. elif stuck < _MAX_STUCK_CALLS:
  139. admit = True
  140. self.probe_at.pop(key, None)
  141. self.skips.pop(key, None) # gate open: the next outage warns on its first skip
  142. else:
  143. admit = now - self.probe_at.setdefault(key, now) >= _PROBE_INTERVAL_S
  144. if admit:
  145. self.probe_at[key] = now
  146. if not admit:
  147. skips = self.skips[key] = self.skips.get(key, 0) + 1
  148. return stuck, len(calls), skips
  149. calls[token] = (now, now + timeout)
  150. return None
  151. def leave(self, key: tuple[str, str], token: object, *, ok: bool) -> None:
  152. with self.lock:
  153. calls = self.calls.get(key)
  154. if calls is None or calls.pop(token, None) is None:
  155. return
  156. if calls:
  157. if ok:
  158. self.last_ok[key] = time.monotonic()
  159. return
  160. for state in (self.calls, self.last_ok, self.probe_at, self.skips):
  161. state.pop(key, None)
  162. class MemoryManager:
  163. """Builtin provider (always first) plus at most one external provider.
  164. Failures in one provider never block the other: every fan-out hook logs and
  165. swallows per-provider exceptions.
  166. """
  167. # External prefetch threads (#134392) and background sync_turn / queue_prefetch calls. The home is
  168. # part of the key because one gateway process can serve several profiles, each with its own config
  169. # and credentials for a same-named provider: one profile's hung backend must not silence another's.
  170. _PREFETCH_GATE = _StuckCallGate()
  171. _BACKGROUND_GATE = _StuckCallGate()
  172. def __init__(self, *, external_prefetch_timeout: Optional[float] = None) -> None:
  173. self._providers: list[MemoryProvider] = []
  174. self._tool_to_provider: dict[str, MemoryProvider] = {}
  175. self._external_prefetch_spill_config: Optional[dict[str, Any]] = None
  176. self._has_external: bool = False
  177. timeout = external_prefetch_timeout
  178. timeout = _EXTERNAL_PREFETCH_TIMEOUT_S if timeout is None else float(timeout)
  179. if timeout <= 0:
  180. raise ValueError("external_prefetch_timeout must be positive")
  181. self._external_prefetch_timeout = timeout
  182. # Captured at build time, inside the profile scope the agent is built for.
  183. self._home_key = hermes_home_key()
  184. # Single-worker background executor for end-of-turn sync/prefetch, created lazily so
  185. # the builtin-only path spawns no threads; one worker serializes a provider's writes.
  186. self._sync_executor: Optional[ThreadPoolExecutor] = None
  187. self._sync_executor_lock = threading.Lock()
  188. # Futures by durability class ("write" / "prefetch") so shutdown can drain FIFO
  189. # within a bound, then report exactly what it abandoned.
  190. self._background_futures: dict[Future, str] = {}
  191. self._shutting_down = False
  192. self._teardown_claim: Optional[str] = None # see claim_teardown()
  193. self._shutdown_drain_state: dict[str, Any] = {
  194. "status": "not_started", "abandoned_writes": 0, "abandoned_prefetches": 0, "active_tasks": 0,
  195. }
  196. # The profile ``initialize_all`` bound the providers to; tool calls must come from it (#140058).
  197. self._bound_scope: Optional[dict[str, Any]] = None
  198. self._tool_call_keywords_cache: dict[int, Optional[frozenset[str]]] = {}
  199. def _each_provider(self, label: str, call: Callable[[MemoryProvider], Any], *, level: int = logging.DEBUG,
  200. providers: Optional[list[MemoryProvider]] = None, exc_info: bool = False) -> list[Any]:
  201. """Call ``call(provider)`` per provider, logging+swallowing failures; returns successes in order.
  202. ``label`` completes the log line ``Memory provider '<name>' <label>: <exc>``."""
  203. results: list[Any] = []
  204. for provider in self._providers if providers is None else providers:
  205. try:
  206. results.append(call(provider))
  207. except Exception as e:
  208. logger.log(level, "Memory provider '%s' %s: %s", provider.name, label, e, exc_info=exc_info)
  209. return results
  210. def _owner_allows(self, hook: str) -> bool:
  211. """Whether a lifecycle hook may reach the providers: only from the profile they were initialized
  212. for (#140058; see ``agent/memory_caller_scope.py``). Nothing to reach, nothing to refuse."""
  213. return not self._providers or caller_scope(self._bound_scope, f"hook {hook}") is not None
  214. def _tool_call_keywords(self, provider: MemoryProvider) -> Optional[frozenset[str]]:
  215. """The keywords ``provider.handle_tool_call`` declares; None when it takes any (``**kwargs``, or
  216. a signature that can't be read). Worked out once per provider, not on every tool call."""
  217. if id(provider) not in self._tool_call_keywords_cache:
  218. params = _signature_params(provider.handle_tool_call)
  219. self._tool_call_keywords_cache[id(provider)] = (
  220. None if params is None or _has_var_kwargs(params) else frozenset(params))
  221. return self._tool_call_keywords_cache[id(provider)]
  222. def _drop_provider(self, provider: MemoryProvider) -> None:
  223. self._tool_call_keywords_cache.pop(id(provider), None)
  224. self._providers = [p for p in self._providers if p is not provider]
  225. self._tool_to_provider = {t: p for t, p in self._tool_to_provider.items() if p is not provider}
  226. self._has_external = any(p.name != "builtin" for p in self._providers)
  227. def add_provider(self, provider: MemoryProvider) -> None:
  228. """Register a provider; builtin always accepted, only ONE external allowed."""
  229. if provider.name != "builtin" and self._has_external:
  230. existing = next((p.name for p in self._providers if p.name != "builtin"), "unknown")
  231. logger.warning(
  232. "Rejected memory provider '%s' — external provider '%s' is "
  233. "already registered. Only one external memory provider is "
  234. "allowed at a time. Configure which one via memory.provider "
  235. "in config.yaml.", provider.name, existing,
  236. )
  237. return
  238. # Load schemas BEFORE mutating any manager state: a provider whose schema
  239. # load raises must leave `_providers` / `_has_external` untouched, otherwise
  240. # it blocks every later external provider in this process (#9948).
  241. schemas = list(provider.get_tool_schemas())
  242. if provider.name != "builtin":
  243. from hermes_cli.config import load_config
  244. # Recall is provider-ranked: keep it whole unless this profile opts in.
  245. # Snapshot both gates at registration, never re-read them mid-conversation.
  246. memory = load_config().get("memory", {})
  247. spill_config = get_spill_config()
  248. if not (isinstance(memory, dict) and memory.get("prefetch_spill_enabled", False) is True):
  249. # Off by default, but keep a ceiling so a runaway provider can't overflow the context.
  250. spill_config["max_chars"] = max(spill_config["max_chars"] * 10, 100_000)
  251. self._external_prefetch_spill_config = spill_config
  252. self._has_external = True
  253. self._providers.append(provider)
  254. # Core tool names are reserved: built-ins always win at agent init, so a shadowing
  255. # provider tool would linger in ``_tool_to_provider`` and hijack dispatch.
  256. # ``clarify``, ``delegate_task``). Reject it here, at the door, so it never enters the routing table
  257. # at all — matching the built-ins-always-win invariant used by the TTS/browser/search provider
  258. # registries. See #40466.
  259. from toolsets import _HERMES_CORE_TOOLS
  260. for raw_schema in schemas:
  261. schema = normalize_tool_schema(raw_schema)
  262. if schema is None:
  263. continue
  264. tool_name = schema["name"]
  265. if tool_name in _HERMES_CORE_TOOLS:
  266. logger.warning(
  267. "Memory provider '%s' tool '%s' shadows a reserved core "
  268. "tool name; registration ignored. Core tools always win — "
  269. "rename the provider's tool to something unique.", provider.name, tool_name,
  270. )
  271. elif tool_name in self._tool_to_provider:
  272. logger.warning(
  273. "Memory tool name conflict: '%s' already registered by %s, "
  274. "ignoring from %s", tool_name, self._tool_to_provider[tool_name].name, provider.name,
  275. )
  276. else:
  277. self._tool_to_provider[tool_name] = provider
  278. logger.info("Memory provider '%s' registered (%d tools)", provider.name, len(schemas))
  279. @property
  280. def providers(self) -> list[MemoryProvider]:
  281. return list(self._providers)
  282. def get_provider(self, name: str) -> Optional[MemoryProvider]:
  283. return next((p for p in self._providers if p.name == name), None)
  284. def build_system_prompt(self) -> str:
  285. """Join every provider's non-empty ``system_prompt_block()`` with blank lines."""
  286. blocks = self._each_provider("system_prompt_block() failed", lambda p: p.system_prompt_block(),
  287. level=logging.WARNING)
  288. return "\n\n".join(b for b in blocks if b and b.strip())
  289. # A /skill or /bundle turn embeds the whole skill body in the model-facing message;
  290. # providers get just the user's instruction (None for a bare invocation).
  291. _strip_skill_scaffolding = staticmethod(extract_user_instruction_from_skill_message)
  292. def prefetch_all(self, query: str, *, session_id: str = "") -> str:
  293. """Merge non-empty prefetch context from all providers (failures are non-fatal)."""
  294. clean_query = self._strip_skill_scaffolding(query)
  295. if not clean_query or not self._owner_allows("prefetch"):
  296. return ""
  297. clean_query = _redact_for_provider(clean_query)
  298. parts = self._each_provider(
  299. "prefetch failed (non-fatal)", lambda p: self._prefetch_provider(p, clean_query, session_id=session_id),
  300. )
  301. return "\n\n".join(p for p in parts if p and p.strip())
  302. def _prefetch_provider(self, provider: MemoryProvider, query: str, *, session_id: str = "") -> str:
  303. """Run one provider's prefetch; external providers are bounded by a timeout. A stuck external
  304. call keeps running on its daemon thread; _PREFETCH_GATE decides whether later turns call again."""
  305. if provider.name == "builtin":
  306. return provider.prefetch(query, session_id=session_id)
  307. result_box: dict[str, Any] = {}
  308. key = self._registry_key(provider)
  309. def _run() -> None:
  310. try:
  311. result_box["value"] = provider.prefetch(query, session_id=session_id) or ""
  312. except Exception as exc: # pragma: no cover - re-raised by caller
  313. result_box["error"] = exc
  314. finally:
  315. self._PREFETCH_GATE.leave(key, thread, ok="error" not in result_box)
  316. thread = spawn_context_thread(_run, name=f"memory-prefetch-{provider.name}")
  317. skipped = self._PREFETCH_GATE.enter(key, thread, self._external_prefetch_timeout)
  318. if skipped:
  319. logger.debug("Memory provider '%s' prefetch skipped (%d stuck, %d in flight)",
  320. provider.name, *skipped[:2])
  321. return ""
  322. try:
  323. thread.start()
  324. except RuntimeError:
  325. # Never started, so its finally never runs: drop the entry or it would count forever.
  326. self._PREFETCH_GATE.leave(key, thread, ok=False)
  327. raise
  328. thread.join(self._external_prefetch_timeout)
  329. if thread.is_alive():
  330. logger.warning(
  331. "Memory provider '%s' prefetch timed out after %.1fs; the call keeps running in the "
  332. "background (later turns may skip it while the provider stays stuck)", provider.name,
  333. self._external_prefetch_timeout,
  334. )
  335. return ""
  336. if "error" in result_box:
  337. raise result_box["error"]
  338. result = result_box.get("value", "")
  339. if result and result.strip():
  340. # Opt-in spill limits recall replayed with the user turn's api_content;
  341. # the registration snapshot leaves provider-ranked results intact by default.
  342. result = spill_if_oversized(
  343. result, session_id=session_id, source=f"{provider.name} memory prefetch",
  344. config=self._external_prefetch_spill_config,
  345. )
  346. return result
  347. def describe_recall(self) -> str:
  348. """Deterministic recall indicator line (e.g. ``"🧠 Provider — recalled 3 memories"``); ``""`` if none.
  349. Call right after :meth:`prefetch_all` so the user SEES memory was used even if the model is silent."""
  350. segments: list[str] = []
  351. for status in self._each_provider("recall_status failed (non-fatal)", lambda p: p.recall_status()):
  352. if status is None:
  353. continue
  354. # count <= 0: content injected but no discrete count (reflect)
  355. detail = ("recalled 1 memory" if status.count == 1 else f"recalled {status.count} memories"
  356. if status.count > 1 else "recalled relevant memory")
  357. segments.append(f"{status.glyph} {status.provider_label} — {detail}")
  358. return " ".join(segments)
  359. def queue_prefetch_all(self, query: str, *, session_id: str = "") -> None:
  360. """Queue background prefetch on all providers for the next turn (see ``sync_all``)."""
  361. providers = list(self._providers)
  362. clean_query = self._strip_skill_scaffolding(query) if providers else None
  363. if not clean_query or not self._owner_allows("queue_prefetch"):
  364. return
  365. clean_query = _redact_for_provider(clean_query)
  366. self._submit_background(lambda: self._each_provider(
  367. "queue_prefetch failed (non-fatal)",
  368. lambda p: self._call_unless_stalled(p, lambda: p.queue_prefetch(clean_query, session_id=session_id)),
  369. providers=providers,
  370. ), kind="prefetch")
  371. @staticmethod
  372. def _provider_sync_accepts(provider: MemoryProvider, keyword: str) -> bool:
  373. """Whether ``sync_turn`` accepts ``keyword`` (uninspectable → assume yes)."""
  374. params = _signature_params(provider.sync_turn)
  375. return params is None or _has_var_kwargs(params) or keyword in params
  376. def sync_all(self, user_content: str, assistant_content: str, *, session_id: str = "",
  377. messages: Optional[list[dict[str, Any]]] = None,
  378. turn_author: Optional[dict[str, Any]] = None) -> None:
  379. """Sync a completed turn to all providers on the background worker.
  380. Never inline: a provider's ``sync_turn`` may block for minutes, which kept ``run_conversation``
  381. open after the user saw the response. The single worker also serializes writes (turn N before N+1).
  382. ``turn_author`` reaches only providers whose ``sync_turn`` accepts it.
  383. Everything forwarded is provider egress: turn strings AND the ``messages`` transcript slice
  384. (tool outputs included) go through ``_redact_for_provider`` so secrets are never archived
  385. verbatim in a provider's store (#115104).
  386. """
  387. providers = list(self._providers)
  388. clean_user_content = self._strip_skill_scaffolding(user_content) if providers else None
  389. if not clean_user_content or not self._owner_allows("sync_turn"):
  390. return
  391. clean_user_content, assistant_content, redacted_messages = _redact_for_provider(
  392. clean_user_content, assistant_content, messages)
  393. optional_kwargs = {"messages": redacted_messages, "turn_author": turn_author}
  394. def _sync(provider: MemoryProvider) -> None:
  395. kwargs: dict[str, Any] = {"session_id": session_id}
  396. for keyword, value in optional_kwargs.items():
  397. if value is not None and self._provider_sync_accepts(provider, keyword):
  398. kwargs[keyword] = value
  399. self._call_unless_stalled(
  400. provider, lambda: provider.sync_turn(clean_user_content, assistant_content, **kwargs))
  401. self._submit_background(
  402. lambda: self._each_provider("sync_turn failed", _sync, level=logging.WARNING, providers=providers)
  403. )
  404. def _submit_background(self, fn, *, kind: str = "write") -> None:
  405. """Queue ``fn`` on the serialized worker (created lazily; None once shutting down) and track its
  406. durability class. Runs under the caller's contextvars (``ctx_bound``). If the executor is
  407. unavailable outside shutdown, run inline — the historical fail-safe."""
  408. fn = ctx_bound(in_owner_home(self._bound_scope, fn))
  409. executor = None if self._shutting_down else self._sync_executor
  410. if executor is None and not self._shutting_down:
  411. with self._sync_executor_lock:
  412. if self._sync_executor is None and not self._shutting_down:
  413. try:
  414. # Daemon workers: a wedged provider must never block interpreter exit.
  415. from tools.daemon_pool import DaemonThreadPoolExecutor
  416. self._sync_executor = DaemonThreadPoolExecutor(max_workers=1, thread_name_prefix="mem-sync")
  417. except Exception as e: # pragma: no cover - resource exhaustion
  418. logger.warning("Failed to create memory sync executor: %s", e)
  419. executor = self._sync_executor
  420. future = None
  421. try:
  422. # Submit+track atomically with the shutdown snapshot. The callback is attached
  423. # outside the lock: an already-completed future invokes callbacks synchronously.
  424. with self._sync_executor_lock:
  425. if self._shutting_down:
  426. logger.warning("Memory manager is shutting down; rejecting late %s task", kind)
  427. return
  428. if executor is not None:
  429. future = executor.submit(fn)
  430. self._background_futures[future] = kind
  431. except RuntimeError:
  432. if self._shutting_down:
  433. logger.warning("Memory manager shut down during %s submission; task rejected", kind)
  434. return
  435. if future is not None:
  436. future.add_done_callback(self._forget_background_future)
  437. return
  438. try:
  439. fn()
  440. except Exception as e: # pragma: no cover - fn guards internally
  441. logger.debug("Inline memory background task failed: %s", e)
  442. def _call_unless_stalled(self, provider: MemoryProvider, call: Callable[[], Any]) -> None:
  443. """Run a background provider call unless _BACKGROUND_GATE skips it (the provider has stuck calls);
  444. skipping costs that turn's write/prefetch but stops a wedged backend from pinning one more
  445. thread per fresh manager. The builtin provider is never gated."""
  446. if provider.name == "builtin":
  447. call()
  448. return
  449. key = self._registry_key(provider)
  450. token = object()
  451. skipped = self._BACKGROUND_GATE.enter(key, token, _BACKGROUND_CALL_STALL_S)
  452. if skipped:
  453. stuck, in_flight, skips = skipped
  454. if skips == 1 or skips % 100 == 0:
  455. logger.warning(
  456. "Memory provider '%s' has %d background calls stuck for over %.0fs (%d in flight); "
  457. "skipping its sync/prefetch until it recovers (%d skipped)", provider.name, stuck,
  458. _BACKGROUND_CALL_STALL_S, in_flight, skips,
  459. )
  460. return
  461. ok = False
  462. try:
  463. call()
  464. ok = True
  465. finally:
  466. self._BACKGROUND_GATE.leave(key, token, ok=ok)
  467. def _registry_key(self, provider: MemoryProvider) -> tuple[str, str]:
  468. """Process-wide registry key: the home the providers were bound to (``initialize_all``, which may
  469. name another profile than the one this manager was built in), else the build-time home."""
  470. return ((self._bound_scope or {}).get("home_key") or self._home_key, provider.name)
  471. def _forget_background_future(self, future: Future) -> None:
  472. with self._sync_executor_lock:
  473. self._background_futures.pop(future, None)
  474. def flush_pending(self, timeout: Optional[float] = None) -> bool:
  475. """Block until queued sync/prefetch work has drained (False on timeout).
  476. With a single worker, a sentinel task completing proves every earlier task ran."""
  477. executor = self._sync_executor
  478. if executor is None:
  479. return True
  480. try:
  481. executor.submit(lambda: None).result(timeout=timeout)
  482. except Exception as e:
  483. return isinstance(e, RuntimeError) # executor already shut down — nothing pending
  484. return True
  485. def get_all_tool_schemas(self) -> list[dict[str, Any]]:
  486. """Collect deduplicated tool schemas from all providers; reserved core tool names are
  487. skipped because :meth:`add_provider` refuses to route them."""
  488. from toolsets import _HERMES_CORE_TOOLS
  489. schemas: list[dict[str, Any]] = []
  490. seen = set()
  491. def _collect(provider: MemoryProvider) -> None:
  492. for raw_schema in provider.get_tool_schemas():
  493. schema = normalize_tool_schema(raw_schema)
  494. if schema is None:
  495. logger.warning(
  496. "Memory provider '%s' returned a tool schema with "
  497. "no resolvable name; skipping (%r)", provider.name, raw_schema,
  498. )
  499. elif schema["name"] not in _HERMES_CORE_TOOLS and schema["name"] not in seen:
  500. schemas.append(schema)
  501. seen.add(schema["name"])
  502. self._each_provider("get_tool_schemas() failed", _collect, level=logging.WARNING)
  503. return schemas
  504. def get_all_tool_names(self) -> set:
  505. return set(self._tool_to_provider)
  506. def has_tool(self, tool_name: str) -> bool:
  507. return tool_name in self._tool_to_provider
  508. def handle_tool_call(self, tool_name: str, args: dict[str, Any], **kwargs) -> str:
  509. """Route a tool call to its provider; returns a JSON string (tool_error on failure)."""
  510. provider = self._tool_to_provider.get(tool_name)
  511. if provider is None:
  512. return tool_error(f"No memory provider handles tool '{tool_name}'")
  513. scope = caller_scope(self._bound_scope, f"tool {tool_name}")
  514. if scope is None:
  515. return tool_error(refused_tool_message(tool_name))
  516. kwargs = {**kwargs, **scope}
  517. # A provider written against ``handle_tool_call(tool_name, args)`` gets only what it declares.
  518. accepted = self._tool_call_keywords(provider)
  519. if accepted is not None:
  520. kwargs = {k: v for k, v in kwargs.items() if k in accepted}
  521. args = _redact_for_provider(args)
  522. from hermes_cli.observability.shared_metrics_loop import record_provider_memory_call
  523. try:
  524. result = provider.handle_tool_call(tool_name, args, **kwargs)
  525. except Exception as e:
  526. logger.error("Memory provider '%s' handle_tool_call(%s) failed: %s", provider.name, tool_name, e)
  527. record_provider_memory_call(provider.name, tool_name, args, raised=True)
  528. return tool_error(f"Memory tool '{tool_name}' failed: {e}")
  529. record_provider_memory_call(provider.name, tool_name, args, result)
  530. return result
  531. def on_turn_start(self, turn_number: int, message: str, **kwargs) -> None:
  532. if not self._owner_allows("on_turn_start"):
  533. return
  534. message = _redact_for_provider(message)
  535. def _tick(p: MemoryProvider) -> None:
  536. # A provider written before the author kwargs declares (turn_number, message) only; it still gets its tick.
  537. params = _signature_params(p.on_turn_start)
  538. accepted = kwargs if params is None or _has_var_kwargs(params) else {k: v for k, v in kwargs.items() if k in params}
  539. p.on_turn_start(turn_number, message, **accepted)
  540. self._each_provider("on_turn_start failed", _tick)
  541. def on_session_end(self, messages: list[dict[str, Any]]) -> None:
  542. if not self._owner_allows("on_session_end"):
  543. return
  544. messages = _redact_messages_for_egress(messages or [])
  545. self._each_provider("on_session_end failed", lambda p: p.on_session_end(messages), level=logging.WARNING,
  546. exc_info=True)
  547. def commit_session_boundary_async(self, messages: list[dict[str, Any]], *, new_session_id: str,
  548. parent_session_id: str = "", reason: str = "new_session") -> None:
  549. """Queue old-session extraction + provider rebinding as ONE serialized task.
  550. ``on_session_end`` (LLM-bound, seconds) must run strictly BEFORE ``on_session_switch`` rebinds
  551. provider state; an ad-hoc thread raced the inline switch and misattributed transcripts.
  552. Running extraction inline blocked the /new command for the whole LLM round-trip (#16454); running it
  553. on an ad-hoc thread raced the inline switch — providers key off internal state, so a late
  554. ``on_session_end`` ran against post-switch bindings (transcript misattributed to the new session id,
  555. double-ingest of the old turn buffer, new-session buffers cleared).
  556. Submitting BOTH hooks as one task on the manager's single background worker gives both properties at
  557. a single chokepoint: the caller returns immediately, and the worker's FIFO order serializes
  558. end→switch against every other provider write (per-turn ``sync_all``, prefetches), which already
  559. share the same worker. If the executor is unavailable, ``_submit_background`` degrades to inline
  560. execution — the pre-#16454 synchronous behavior, slow but correct.
  561. """
  562. if not self._providers or not self._owner_allows("session_boundary"):
  563. return
  564. snapshot = list(messages or [])
  565. def _run() -> None: # both hooks already guard per-provider
  566. try:
  567. self.on_session_end(snapshot)
  568. except Exception as e: # pragma: no cover
  569. logger.warning("Session-boundary extraction failed: %s", e)
  570. try:
  571. self.on_session_switch(new_session_id, parent_session_id=parent_session_id, reset=True, reason=reason)
  572. except Exception as e: # pragma: no cover
  573. logger.warning("Session-boundary switch failed: %s", e)
  574. self._submit_background(_run)
  575. def on_session_switch(self, new_session_id: str, *, parent_session_id: str = "", reset: bool = False,
  576. rewound: bool = False, **kwargs) -> None:
  577. """Notify providers that ``AIAgent.session_id`` rotated without teardown
  578. (``/resume``, ``/branch``, ``/reset``, ``/new``, compression). ``rewound=True``
  579. (``/undo``): same id, truncated transcript."""
  580. if not new_session_id or not self._owner_allows("on_session_switch"):
  581. return
  582. if rewound: # forward only when set so it never pollutes providers' **kwargs
  583. kwargs["rewound"] = True
  584. self._each_provider(
  585. "on_session_switch failed",
  586. lambda p: p.on_session_switch(new_session_id, parent_session_id=parent_session_id, reset=reset, **kwargs),
  587. )
  588. @staticmethod
  589. def _checkpoint_api_version(provider: MemoryProvider) -> Optional[int]:
  590. """Provider's advertised pre-compress checkpoint API version; None if unparseable."""
  591. try:
  592. return int(getattr(provider, "pre_compress_checkpoint_api_version", _LEGACY_PRE_COMPRESS_API_VERSION))
  593. except (TypeError, ValueError):
  594. return None
  595. def supports_pre_compress_checkpoint(self, api_version: int = PRE_COMPRESS_CHECKPOINT_API_VERSION) -> bool:
  596. """Return whether an active provider guarantees checkpoint API support."""
  597. versions = (self._checkpoint_api_version(p) for p in self._providers)
  598. return any(v is not None and v >= api_version for v in versions)
  599. def on_pre_compress(self, messages: list[dict[str, Any]], *,
  600. evidence_messages: Optional[list[dict[str, Any]]] = None, require_checkpoint: bool = False,
  601. checkpoint_api_version: int = PRE_COMPRESS_CHECKPOINT_API_VERSION) -> str:
  602. """Notify providers before compression; return their combined summary-prompt text.
  603. ``messages`` is the raw v1 transcript; ``evidence_messages`` is the host-normalized list handed
  604. only to checkpoint (v2+) providers. With ``require_checkpoint`` at least one checkpoint provider
  605. must succeed — its exception propagates so the caller keeps the uncompressed transcript.
  606. """
  607. if not self._owner_allows("on_pre_compress"):
  608. if require_checkpoint: # the caller keeps the uncompressed transcript
  609. raise RuntimeError("Memory pre-compress checkpoint refused; see `hermes logs errors`")
  610. return ""
  611. parts = []
  612. checkpoint_succeeded = False
  613. from agent.message_metadata import without_withheld_prompts
  614. messages, evidence_messages = _redact_for_provider(
  615. without_withheld_prompts(messages or []), without_withheld_prompts(evidence_messages))
  616. for provider in self._providers:
  617. version = self._checkpoint_api_version(provider)
  618. if version is None:
  619. version = _LEGACY_PRE_COMPRESS_API_VERSION
  620. is_checkpoint_provider = version >= checkpoint_api_version
  621. use_evidence = is_checkpoint_provider and evidence_messages is not None
  622. provider_messages = evidence_messages if use_evidence else messages
  623. kwargs: dict[str, Any] = {}
  624. # v1 providers and bare-shape v2 providers never see the signal.
  625. if is_checkpoint_provider and _accepts_require_checkpoint(provider.on_pre_compress):
  626. kwargs["require_checkpoint"] = require_checkpoint
  627. try:
  628. result = provider.on_pre_compress(provider_messages, **kwargs)
  629. if result and result.strip():
  630. parts.append(result)
  631. checkpoint_succeeded = checkpoint_succeeded or is_checkpoint_provider
  632. except Exception as e:
  633. logger.debug("Memory provider '%s' on_pre_compress failed: %s", provider.name, e)
  634. if require_checkpoint and is_checkpoint_provider:
  635. raise
  636. if require_checkpoint and not checkpoint_succeeded:
  637. raise RuntimeError(
  638. f"No active memory provider completed pre-compress checkpoint API v{checkpoint_api_version}"
  639. )
  640. return "\n\n".join(parts)
  641. @staticmethod
  642. def _provider_memory_write_metadata_mode(provider: MemoryProvider) -> str:
  643. """How to pass metadata to ``on_memory_write``: "keyword", "positional", or "legacy" (none)."""
  644. params = _signature_params(provider.on_memory_write)
  645. if params is None or _has_var_kwargs(params) or "metadata" in params:
  646. return "keyword"
  647. accepted = sum(p.kind is not inspect.Parameter.VAR_POSITIONAL for p in params.values())
  648. return "positional" if accepted >= 4 else "legacy"
  649. def on_memory_write(self, action: str, target: str, content: str,
  650. metadata: Optional[dict[str, Any]] = None) -> None:
  651. """Notify external providers when the built-in memory tool writes (skips builtin, the source)."""
  652. if not self._owner_allows("on_memory_write"):
  653. return
  654. content, metadata = _redact_for_provider(content, dict(metadata or {}))
  655. def _notify(provider: MemoryProvider) -> None:
  656. mode = self._provider_memory_write_metadata_mode(provider)
  657. if mode == "legacy":
  658. provider.on_memory_write(action, target, content)
  659. elif mode == "positional":
  660. provider.on_memory_write(action, target, content, dict(metadata or {}))
  661. else:
  662. provider.on_memory_write(action, target, content, metadata=dict(metadata or {}))
  663. external = [p for p in self._providers if p.name != "builtin"]
  664. self._each_provider("on_memory_write failed", _notify, providers=external)
  665. # Actions mirrored to external providers; non-mutating results (errors, staged) are
  666. # filtered by ``notify_memory_tool_write`` first.
  667. _MIRRORED_MEMORY_ACTIONS = {"add", "replace", "patch", "remove"}
  668. @staticmethod
  669. def _memory_tool_result_succeeded(result: Any) -> bool:
  670. """True only when the built-in memory tool actually committed a write. Fails closed (non-JSON,
  671. non-dict, missing ``success``, staged for approval) so providers never mirror a write that did not land."""
  672. if isinstance(result, str):
  673. try:
  674. result = json.loads(result)
  675. except Exception:
  676. return False
  677. return isinstance(result, dict) and result.get("success") is True and result.get("staged") is not True
  678. def notify_memory_tool_write(self, tool_result: Any, tool_args: dict[str, Any], *,
  679. build_metadata: Optional[Callable[[], dict[str, Any]]] = None) -> None:
  680. """Mirror a built-in memory tool call to external providers.
  681. Gates on a committed write, expands single-op and batched ``operations`` shapes, keeps only
  682. mutating actions, and forwards ``old_text`` plus provenance from ``build_metadata``.
  683. ``previous_content`` comes only from the committed store result, never the search
  684. argument: a partial provider registry cannot safely resolve that argument itself.
  685. """
  686. if not self._memory_tool_result_succeeded(tool_result):
  687. return
  688. result = json.loads(tool_result) if isinstance(tool_result, str) else tool_result
  689. target = str(tool_args.get("target") or "memory")
  690. operations = tool_args.get("operations")
  691. batched = isinstance(operations, list) and bool(operations)
  692. for index, op in enumerate(operations if batched else [tool_args], start=1):
  693. action = str(op.get("action") or "") if isinstance(op, dict) else ""
  694. if action not in self._MIRRORED_MEMORY_ACTIONS:
  695. continue
  696. try:
  697. metadata = dict(build_metadata() if build_metadata else {})
  698. metadata.pop("previous_content", None)
  699. old_text = op.get("old_text")
  700. if old_text:
  701. metadata["old_text"] = str(old_text)
  702. field = {"replace": "replaced", "patch": "patched", "remove": "removed"}.get(action)
  703. if field:
  704. if batched:
  705. entries = result.get(f"{field}_entries", {})
  706. previous = entries.get(str(index), entries.get(index)) if isinstance(entries, dict) else None
  707. else:
  708. previous = result.get(f"{field}_entry")
  709. if field == "patched" and previous is None:
  710. previous = result.get("patched_entry_before")
  711. if isinstance(previous, str) and previous:
  712. metadata["previous_content"] = previous
  713. mirrored_action, content = action, str(op.get("content") or op.get("new_text") or "")
  714. if action == "patch":
  715. # To a provider a patch replaced the whole entry with its edited form; mirror it as
  716. # that replace, with the entry the store reports, so providers that know
  717. # add/replace/remove handle it unchanged.
  718. if batched:
  719. after_entries = result.get("patched_entries_after") or {}
  720. after = after_entries.get(str(index), after_entries.get(index))
  721. else:
  722. after = result.get("patched_entry_after")
  723. mirrored_action, content = "replace", str(after or "")
  724. self.on_memory_write(mirrored_action, target, content, metadata=metadata)
  725. except Exception as e:
  726. logger.debug("notify_memory_tool_write failed for op %s: %s", action, e)
  727. def on_delegation(self, task: str, result: str, *, child_session_id: str = "", **kwargs) -> None:
  728. if not self._owner_allows("on_delegation"):
  729. return
  730. task, result = _redact_for_provider(task, result)
  731. self._each_provider(
  732. "on_delegation failed",
  733. lambda p: p.on_delegation(task, result, child_session_id=child_session_id, **kwargs),
  734. )
  735. def claim_teardown(self, *, session_end: bool) -> bool:
  736. """Claim extraction + ``shutdown_all()`` for end-of-session shutdown or soft release. The first
  737. claimer wins and the loser skips, so the two never overlap and providers (no idempotency
  738. contract) are never shut down twice. Session-end callers may re-claim (the gateway claims before
  739. its flush, then ``shutdown_memory_provider`` claims again); a soft release is once-only and also
  740. loses to a manager another owner is already draining."""
  741. with self._sync_executor_lock:
  742. if self._teardown_claim is None and (session_end or not self._shutting_down):
  743. self._teardown_claim = "session_end" if session_end else "soft_release"
  744. return True
  745. return session_end and self._teardown_claim == "session_end"
  746. def release_soft(self) -> None:
  747. """Soft release (cache eviction, CLI agent rebuild): stop provider writer threads, which otherwise leak
  748. one per evicted agent, WITHOUT on_session_end(): sweeps commit first and a pending session end owns
  749. both. Flush first, since shutdown_all()'s short drain would cancel writes the old worker finished."""
  750. if self.claim_teardown(session_end=False):
  751. self.flush_pending(timeout=10)
  752. self.shutdown_all()
  753. def shutdown_all(self) -> None:
  754. """Drain the background executor (bounded), then shut providers down in reverse order."""
  755. self._drain_sync_executor()
  756. self._each_provider("shutdown failed", lambda p: p.shutdown(), level=logging.WARNING,
  757. providers=self._providers[::-1])
  758. @property
  759. def shutdown_drain_state(self) -> dict[str, Any]:
  760. """Snapshot of the most recent bounded shutdown drain outcome."""
  761. with self._sync_executor_lock:
  762. return dict(self._shutdown_drain_state)
  763. def _drain_sync_executor(self) -> None:
  764. """Give queued FIFO work a bounded chance, then abandon explicitly."""
  765. with self._sync_executor_lock:
  766. self._shutting_down = True
  767. executor = self._sync_executor
  768. self._sync_executor = None
  769. tracked = dict(self._background_futures)
  770. self._shutdown_drain_state = {
  771. "status": "draining" if executor is not None else "drained",
  772. "abandoned_writes": 0, "abandoned_prefetches": 0,
  773. "active_tasks": sum(not future.done() for future in tracked),
  774. }
  775. if executor is None:
  776. return
  777. # shutdown(wait=False) closes submission without touching the FIFO; waiting on the
  778. # tracked futures lets the worker run every queued task in order up to the deadline.
  779. executor.shutdown(wait=False, cancel_futures=False)
  780. _, pending = wait(tuple(tracked), timeout=_SYNC_DRAIN_TIMEOUT_S)
  781. cancelled = [tracked[future] for future in pending if future.cancel()]
  782. active_tasks = len(pending) - len(cancelled)
  783. abandoned_prefetches = cancelled.count("prefetch")
  784. abandoned_writes = len(cancelled) - abandoned_prefetches
  785. with self._sync_executor_lock:
  786. self._shutdown_drain_state.update(
  787. status="timed_out" if pending else "drained", abandoned_writes=abandoned_writes,
  788. abandoned_prefetches=abandoned_prefetches, active_tasks=active_tasks,
  789. )
  790. if not pending:
  791. return
  792. logger.warning(
  793. "Memory shutdown drain timed out after %.2fs; abandoning %d queued "
  794. "memory write(s) and %d queued prefetch(es); %d active task(s) remain detached",
  795. _SYNC_DRAIN_TIMEOUT_S, abandoned_writes, abandoned_prefetches, active_tasks,
  796. )
  797. def initialize_for_agent(self, say: Optional[Callable[[str], None]] = None, **kwargs) -> bool:
  798. """``initialize_all`` for an agent build; True while an external provider is still served. One
  799. refused as another profile's shared instance is dropped, and ``say`` (the agent's startup
  800. warning sink) tells the user this profile now runs without it."""
  801. before = [p for p in self._providers if p.name != "builtin"]
  802. self.initialize_all(**kwargs)
  803. for provider in (p for p in before if not any(p is kept for kept in self._providers)):
  804. message = (f"⚠ Memory provider {provider.name!r} is disabled for this profile: it handed out the "
  805. "provider instance another profile already uses. Built-in memory still works; the "
  806. "operator can see why with `hermes logs errors`.")
  807. (say or logger.warning)(message)
  808. return self._has_external
  809. def initialize_all(self, session_id: str, **kwargs) -> None:
  810. """Initialize all providers, injecting ``hermes_home`` so they resolve profile-scoped paths."""
  811. if "hermes_home" not in kwargs:
  812. from hermes_constants import get_hermes_home
  813. kwargs["hermes_home"] = str(get_hermes_home())
  814. self._bound_scope = bound_scope(kwargs["hermes_home"], kwargs.get("agent_identity"))
  815. # One instance, one profile: never rebind an instance another profile's manager bound (#140058).
  816. for provider in [p for p in self._providers if p.name != "builtin"]:
  817. if not claim_instance(provider, kwargs["hermes_home"]):
  818. self._drop_provider(provider)
  819. self._each_provider("initialize failed", lambda p: p.initialize(session_id=session_id, **kwargs),
  820. level=logging.WARNING)