plugins_dispatch.py 38 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763
  1. """Plugin hook / middleware / event-bus / system-prompt-section dispatch.
  2. Mixed into :class:`hermes_cli.plugins.PluginManager`. ``_resolve_hook_callback_timeout`` stays on
  3. the origin (tests patch it there) and is looked up lazily.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import contextvars
  8. import copy
  9. import inspect
  10. import logging
  11. import queue
  12. import re
  13. import threading
  14. import time
  15. import types
  16. from dataclasses import dataclass
  17. from typing import Any, Callable, Dict, List, Mapping, Optional, Set, Union
  18. from hermes_cli.middleware import OBSERVER_SCHEMA_VERSION
  19. logger = logging.getLogger("hermes_cli.plugins")
  20. # Allowlist of agent-turn hot-path hooks bounded by plugins.hook_callback_timeout (fail-open:
  21. # abandon without join — joining reintroduced a shutdown hang). Unlisted hooks run synchronously.
  22. # Intentionally unbounded: on_session_finalize/reset (last-chance flush — abandon can lose state);
  23. # subagent_start (observer); pre_gateway_dispatch (policy gate — neither fail mode is acceptable);
  24. # pre/post_approval_* (approval UX has its own timeout); kanban_* (own heartbeat/stale reclaim).
  25. # The goal is to stop a hung Python plugin callback from wedging the conversation loop (#76821) without
  26. # joining the worker (avoids the #6622 ThreadPoolExecutor shutdown hang). Hooks not listed below run
  27. # synchronously to completion. (on_session_start/end stay bounded — they sit on the common session-boundary
  28. # path.) - subagent_start — observer only; blocking delegation belongs in pre_tool_call. Lower frequency
  29. # than tool/LLM hooks. Abandoning is unsafe either way (fail-open skips auth-like checks; fail-closed can
  30. # drop legitimate messages). Prefer finish-or-exception fallthrough. - pre_approval_request /
  31. # post_approval_response — observers only (cannot veto); the approval UX already has its own timeout; not on
  32. # the tool loop hot path. - kanban_task_* — fire after the board DB commit, observers only, in
  33. # dispatcher/worker processes; kanban has its own heartbeat/stale reclaim. Abandon-without-join also leaves
  34. # a daemon thread that may still mutate shared state — safer for value-returning observers than for
  35. # gates/flushes.
  36. _HOOK_TIMEOUT_BOUNDED_HOOKS: set[str] = {
  37. "post_tool_call", "transform_terminal_output", "transform_tool_result", "transform_llm_output",
  38. "pre_llm_call", "post_llm_call", "pre_api_request", "post_api_request", "api_request_error",
  39. "pre_auxiliary_call", "post_auxiliary_call", "pre_verify", "on_session_start", "on_session_end",
  40. # Fail-open consumer on every inbound gateway message: a hung plugin must not stall the profile.
  41. "post_gateway_admission",
  42. # Observer on every local session's commands, fired by the gateway (gateway/session_commands.py).
  43. "pre_command",
  44. # Fires inside the compression commit window (lease held): a hung plugin must not wedge the session.
  45. "post_compaction",
  46. }
  47. # Hooks whose calls are told apart by another payload field: one session's commands are one
  48. # sequence, so a still-running callback skips that session's next command, not another session's.
  49. _HOOK_CALL_IDENTITY_FIELDS: dict[str, tuple[str, ...]] = {
  50. "pre_command": ("session_key",), "post_compaction": ("session_id",),
  51. }
  52. # Policy hooks: timeout / still-running must fail closed (block the tool).
  53. _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS: set[str] = {"pre_tool_call"}
  54. # Documented parent-thread serialization contract — never run on a timeout worker (hooks.md).
  55. _HOOK_CALLER_THREAD_HOOKS: set[str] = {"subagent_stop"}
  56. # After a timeout, suppress the same callback this long so a hung hook cannot pile up threads.
  57. _HOOK_TIMEOUT_SUPPRESSION_SECONDS = 60.0
  58. # Live workers a hung callback may accumulate before it is skipped outright (#105223 / #98382).
  59. _HOOK_MAX_ABANDONED_WORKERS = 3
  60. _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE = "pre_tool_call plugin callback timed out or is still running"
  61. # The one hook whose callback rewrites the reply the user would otherwise see streaming.
  62. STREAM_HOLDING_HOOK = "transform_llm_output"
  63. def stream_held() -> bool:
  64. """True when a plugin loaded in the active profile registered ``transform_llm_output`` with
  65. ``hold_stream=True``: that profile's replies reach the user only as the rewritten final.
  66. Lazy-discovers first, like ``has_hook``."""
  67. from hermes_cli.plugins import _delivery_manager
  68. return _delivery_manager().holds_stream()
  69. def held_stream_transforms() -> tuple[Callable, ...]:
  70. """:meth:`PluginDispatchMixin.held_transforms` of the active profile's manager (lazy-discovers)."""
  71. from hermes_cli.plugins import _delivery_manager
  72. return _delivery_manager().held_transforms()
  73. def _policy_error_block_directive(hook_name: str, cb: Callable, exc: BaseException) -> dict[str, str]:
  74. """Block directive for a fail-closed hook whose callback raised: names the callback and the
  75. error (truncated — a hook that embeds tool args in its exception must not grow the tool
  76. result) so the operator can tell a crashing guard from a slow one."""
  77. callback_name = getattr(cb, "__name__", repr(cb))
  78. return {"action": "block",
  79. "message": f"{hook_name} plugin callback {callback_name} raised {type(exc).__name__}: {str(exc)[:200]}"}
  80. # System-prompt sections are tightly bounded: they become high-trust prompt bytes charged every turn.
  81. SYSTEM_PROMPT_SECTION_POSITIONS = frozenset({"after_memory"})
  82. DEFAULT_SYSTEM_PROMPT_SECTION_MAX_CHARS = 4_000
  83. MAX_SYSTEM_PROMPT_SECTION_CHARS = 4_000
  84. MAX_SYSTEM_PROMPT_SECTIONS = 32
  85. MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS = 8_000
  86. _SYSTEM_PROMPT_SECTION_ID_RE = re.compile(r"^[a-z0-9][a-z0-9._-]{0,127}$")
  87. _SYSTEM_PROMPT_SECTION_HEADING_PREFIX = "## Plugin Context: "
  88. PLUGIN_SECTIONS_START = "<!-- hermes-plugin-sections:start -->"
  89. PLUGIN_SECTIONS_END = "<!-- hermes-plugin-sections:end -->"
  90. def is_valid_system_prompt_section_id(value: Any) -> bool:
  91. """Return whether *value* is a stable, heading-safe section identifier."""
  92. return isinstance(value, str) and bool(_SYSTEM_PROMPT_SECTION_ID_RE.fullmatch(value))
  93. def format_system_prompt_section(section_id: str, content: str) -> str:
  94. """Render an auditable, length-framed block recoverable from the full prompt."""
  95. return (
  96. f"{_SYSTEM_PROMPT_SECTION_HEADING_PREFIX}{section_id}\n"
  97. f"<!-- hermes-plugin-section-chars:{len(content)} -->\n\n{content}")
  98. def format_system_prompt_sections(sections: list) -> str:
  99. """Render the canonical container used for persistence recovery."""
  100. if not sections:
  101. return ""
  102. blocks = [format_system_prompt_section(item.id, item.content) for item in sections]
  103. return f"{PLUGIN_SECTIONS_START}\n" + "\n\n".join(blocks) + f"\n{PLUGIN_SECTIONS_END}"
  104. # Reserved event namespace prefix — only core may publish ``hermes:<event>``.
  105. HERMES_EVENT_NAMESPACE = "hermes"
  106. # Event recursion depth cap (subscribers may emit); over-deep emits are dropped with a warning.
  107. _EVENT_EMIT_DEPTH_CAP = 8
  108. # Max queued + running events per manager generation; emit never waits — a full budget drops.
  109. _EVENT_PENDING_CAP = 64
  110. _EVENT_WORKER_STOP = object()
  111. @dataclass(frozen=True)
  112. class PluginSystemPromptSection:
  113. """A plugin-owned section rendered once for each new session."""
  114. id: str
  115. content: str | Callable[[Mapping[str, Any]], str]
  116. position: str
  117. max_chars: int
  118. plugin: str
  119. @dataclass(frozen=True)
  120. class RenderedPluginSystemPromptSection:
  121. """Validated prompt bytes frozen on the owning AIAgent."""
  122. id: str
  123. content: str
  124. position: str
  125. plugin: str
  126. @dataclass(frozen=True)
  127. class _EventSubscription:
  128. """Host-owned subscription ledger entry."""
  129. owner: str
  130. callback: Callable
  131. @dataclass(frozen=True)
  132. class _QueuedPluginEvent:
  133. """Immutable dispatch envelope consumed by the event worker."""
  134. event: str
  135. payload: dict[str, Any]
  136. subscriptions: tuple[_EventSubscription, ...]
  137. depth: int
  138. generation: int
  139. # The emitter's contextvars: the single worker thread serves every profile, so each delivery
  140. # runs under the profile scope the emit happened in (#118538).
  141. context: contextvars.Context
  142. # Hook callback timeout (non-blocking abandon). Default cap per Python hook callback; overridden by
  143. # ``plugins.hook_callback_timeout``. Shell hooks enforce their own subprocess timeout.
  144. _HOOK_CALLBACK_TIMEOUT_SECS = 30.0
  145. _MAX_HOOK_CALLBACK_TIMEOUT_SECS = 600.0
  146. _HOOK_SKIPPED = object() # returned by _run_hook_callback_bounded on skip/timeout
  147. def _hook_call_identity(kwargs: dict[str, Any], hook_name: str = "") -> Optional[str]:
  148. """Identity of the call this callback fires for, or ``None`` when the event has none.
  149. Concurrent invocations of the same tool in one session must not collapse into one
  150. gate key: they are different work, and treating the second as a duplicate drops the
  151. hook as if a callback had timed out (upstream #98382). The identity is already in the
  152. payload; nothing new is plumbed. Deliberately not ``api_request_id`` — one API request
  153. carries many tool calls, which would re-collapse the keys.
  154. """
  155. for field in _HOOK_CALL_IDENTITY_FIELDS.get(hook_name, ("tool_call_id", "turn_id")):
  156. value = kwargs.get(field)
  157. if isinstance(value, str) and value:
  158. return value
  159. return None
  160. def _hook_uses_callback_timeout(hook_name: str, timeout: float) -> bool:
  161. """Whether *hook_name* should run under the non-blocking timeout path."""
  162. if timeout <= 0 or hook_name in _HOOK_CALLER_THREAD_HOOKS:
  163. return False
  164. return hook_name in _HOOK_TIMEOUT_BOUNDED_HOOKS or hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS
  165. _PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS = 30.0
  166. def resolve_plugin_command_result(result: Any) -> Any:
  167. """Resolve a plugin command result, awaiting async handlers: ``asyncio.run`` when no loop is
  168. running, else a helper thread with its own loop (30s bound so a hung handler cannot wedge the
  169. terminal)."""
  170. if not inspect.isawaitable(result):
  171. return result
  172. try:
  173. asyncio.get_running_loop()
  174. except RuntimeError:
  175. return asyncio.run(result)
  176. outcome: dict[str, Any] = {}
  177. failure: dict[str, BaseException] = {}
  178. done = threading.Event()
  179. def _runner() -> None:
  180. try:
  181. outcome["value"] = asyncio.run(result)
  182. except BaseException as exc: # pragma: no cover - re-raised below
  183. failure["exc"] = exc
  184. finally:
  185. done.set()
  186. # copy_context: the helper thread must see the caller's profile/secret scope, else an
  187. # async hook under a running loop reads the default HERMES_HOME and get_secret raises.
  188. threading.Thread(target=contextvars.copy_context().run, args=(_runner,),
  189. name="hermes-plugin-command-await", daemon=True).start()
  190. if not done.wait(timeout=_PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS):
  191. raise TimeoutError("Plugin command async handler did not complete within "
  192. f"{_PLUGIN_COMMAND_AWAIT_TIMEOUT_SECS:.0f}s")
  193. if "exc" in failure:
  194. raise failure["exc"]
  195. return outcome.get("value")
  196. class PluginDispatchMixin:
  197. @staticmethod
  198. def _hook_callback_kwargs(callback: Callable, payload: dict[str, Any]) -> dict[str, Any]:
  199. """The slice of *payload* a callback accepts: everything for ``**kwargs`` (or
  200. un-introspectable) callbacks, only declared names for narrow legacy signatures."""
  201. try:
  202. parameters = inspect.signature(callback).parameters
  203. except (TypeError, ValueError):
  204. return dict(payload) # no introspectable signature
  205. if any(p.kind == inspect.Parameter.VAR_KEYWORD for p in parameters.values()):
  206. return dict(payload)
  207. keyword_kinds = {inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY}
  208. return {
  209. name: value for name, value in payload.items()
  210. if name in parameters and parameters[name].kind in keyword_kinds
  211. }
  212. @classmethod
  213. def _invoke_hook_callback(cls, callback: Callable, payload: dict[str, Any]) -> Any:
  214. """Invoke a hook while withholding additive fields from narrow legacy callbacks.
  215. An ``async def`` callback returns a coroutine; resolve it the way plugin slash commands
  216. are (loop-safe), otherwise the bare coroutine object is appended to the results and the
  217. plugin's body never runs (#12449).
  218. """
  219. return resolve_plugin_command_result(callback(**cls._hook_callback_kwargs(callback, payload)))
  220. def invoke_hook(self, hook_name: str, **kwargs: Any) -> list[Any]:
  221. """Call all callbacks for *hook_name*; return their non-``None`` results.
  222. Payloads evolve additively: ``**kwargs`` callbacks get everything, narrow signatures only
  223. what they declare. Each callback is isolated. Bounded hooks and ``pre_tool_call`` run under
  224. ``plugins.hook_callback_timeout`` (worker abandoned, never joined); ``pre_tool_call`` fails
  225. closed with a block directive, others skip. ``_HOOK_CALLER_THREAD_HOOKS`` always run on the
  226. caller thread. ``pre_llm_call`` may return ``{"context": "..."}`` (or a str) to inject, or
  227. ``{"action": "block", "message": "..."}`` to end the turn before any inference request.
  228. """
  229. from agent.safe_worker_policy import safe_worker_enabled
  230. if safe_worker_enabled():
  231. return []
  232. # Gateway platform events define event-local envelopes; a bus-wide version here would turn
  233. # unrelated adapter payloads into one monolithic compatibility contract.
  234. if hook_name != "gateway_platform_event":
  235. kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
  236. return self._run_hook_callbacks(hook_name, self._hooks.get(hook_name, []), kwargs)
  237. def _run_hook_callbacks(self, hook_name: str, callbacks: Any, kwargs: dict[str, Any],
  238. failed: Optional[list] = None) -> list[Any]:
  239. """:meth:`invoke_hook`'s loop over ``callbacks``; each one that raised or timed out is
  240. appended to ``failed`` when given."""
  241. from hermes_cli.plugins import _resolve_hook_callback_timeout
  242. results: list[Any] = []
  243. timeout = _resolve_hook_callback_timeout()
  244. use_timeout = _hook_uses_callback_timeout(hook_name, timeout)
  245. fail_closed = hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS
  246. for cb in callbacks:
  247. try:
  248. if use_timeout:
  249. ret = self._run_hook_callback_bounded(hook_name, cb, kwargs, timeout)
  250. if ret is _HOOK_SKIPPED:
  251. if failed is not None:
  252. failed.append(cb)
  253. if fail_closed: # policy hook: fail closed with a block directive
  254. results.append({"action": "block", "message": _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE})
  255. continue
  256. else:
  257. ret = self._invoke_hook_callback(cb, kwargs)
  258. if ret is not None:
  259. results.append(ret)
  260. except (Exception, SystemExit) as exc: # health: allow BLE001 -- moved from invoke_hook: plugin callbacks are untrusted; each failure is reported and isolated
  261. self._report_hook_failure(hook_name, cb, kwargs, exc)
  262. if failed is not None:
  263. failed.append(cb)
  264. if fail_closed: # a guard that raised made no decision: same veto as a timeout
  265. results.append(_policy_error_block_directive(hook_name, cb, exc))
  266. return results
  267. def _report_hook_failure(
  268. self, hook_name: str, cb: Callable, kwargs: dict[str, Any], exc: BaseException, *, surface: str = "Hook"
  269. ) -> None:
  270. """One WARNING per distinct (hook, callback, error); identical repeats at DEBUG.
  271. A callback whose signature names a parameter the hook never sends (``tool_data`` instead
  272. of ``tool_name``/``args``) fails identically on every tool call — ~1700 WARNING lines an
  273. hour that bury real signals (#111922). The first report names the fields the hook does
  274. provide so the plugin author can fix the signature. The key names the callback by
  275. module/qualname (not ``id()``, which CPython recycles across plugin reloads) and
  276. truncates the message so a hook that embeds tool args in its error cannot grow the set
  277. per call; the set is cleared on unload alongside the timeout-suppression map.
  278. """
  279. callback_name = getattr(cb, "__name__", repr(cb))
  280. key = (hook_name, getattr(cb, "__module__", ""), getattr(cb, "__qualname__", callback_name),
  281. type(exc).__name__, str(exc)[:200])
  282. if key in self._hook_failures_reported:
  283. logger.debug("%s '%s' callback %s raised again: %s", surface, hook_name, callback_name, exc)
  284. return
  285. self._hook_failures_reported.add(key)
  286. logger.warning(
  287. "%s '%s' callback %s raised: %s (%s provides: %s; identical failures are logged at DEBUG from now on)",
  288. surface, hook_name, callback_name, exc, surface.lower(), ", ".join(sorted(kwargs)) or "no fields")
  289. def _run_hook_callback_bounded(
  290. self, hook_name: str, cb: Callable, kwargs: dict[str, Any], timeout: float
  291. ) -> Any:
  292. """Run one callback on a daemon worker with a wall-clock cap; ``_HOOK_SKIPPED`` when
  293. suppressed, still running for this call id, over the abandoned-worker cap, timed out
  294. (worker abandoned, never joined), or the worker could not be started. Exceptions
  295. propagate."""
  296. callback_name = getattr(cb, "__name__", repr(cb))
  297. # Suppression is a fact about the CALLBACK — a hung one must keep its back-off —
  298. # so that key stays coarse. The gate must instead tell CONCURRENT CALLS apart.
  299. suppression_key = (hook_name, id(cb))
  300. gate_key = (*suppression_key, _hook_call_identity(kwargs, hook_name))
  301. token = object()
  302. with self._hook_timeout_lock:
  303. suppressed_until = self._hook_timeout_suppressed_until.get(suppression_key)
  304. if (gate_key in self._hook_running_callbacks
  305. or (suppressed_until is not None and suppressed_until > time.monotonic())):
  306. logger.warning(
  307. "Hook '%s' callback %s skipped after previous "
  308. "timeout or while still running", hook_name, callback_name)
  309. return _HOOK_SKIPPED
  310. # Workers abandoned on timeout still hold threads. Once the suppression window has
  311. # passed, a fresh call id may start a new worker (a hung guard must not fail every
  312. # later tool call closed until restart, #105223), but only up to a small cap per
  313. # callback — expiring the bookkeeping while the hung worker lives must not leak a
  314. # thread per call (#98382). At the cap the callback keeps being skipped (fail-closed
  315. # for pre_tool_call) until one of its workers finishes and releases its slot.
  316. abandoned = self._hook_abandoned.get(suppression_key)
  317. if abandoned and len(abandoned) >= _HOOK_MAX_ABANDONED_WORKERS:
  318. logger.warning(
  319. "Hook '%s' callback %s (%s) skipped: %d abandoned worker(s) still running — "
  320. "the plugin is hung; fix or disable it (retried when a worker finishes)",
  321. hook_name, callback_name, getattr(cb, "__module__", "unknown plugin"), len(abandoned))
  322. return _HOOK_SKIPPED
  323. if suppressed_until is not None:
  324. self._hook_timeout_suppressed_until.pop(suppression_key, None)
  325. self._hook_running_callbacks[gate_key] = token
  326. context = contextvars.copy_context()
  327. done = threading.Event()
  328. outcome: dict[str, Any] = {}
  329. failure: dict[str, BaseException] = {}
  330. def _release_token() -> None:
  331. with self._hook_timeout_lock:
  332. if self._hook_running_callbacks.get(gate_key) is token:
  333. self._hook_running_callbacks.pop(gate_key, None)
  334. abandoned = self._hook_abandoned.get(suppression_key)
  335. if abandoned is not None:
  336. abandoned.discard(gate_key)
  337. if not abandoned:
  338. self._hook_abandoned.pop(suppression_key, None)
  339. def _runner() -> None:
  340. try:
  341. outcome["value"] = context.run(self._invoke_hook_callback, cb, kwargs)
  342. except BaseException as exc:
  343. failure["exc"] = exc
  344. finally:
  345. _release_token()
  346. done.set()
  347. thread = threading.Thread(target=_runner, name=f"hermes-hook-{callback_name}"[:40], daemon=True)
  348. try:
  349. thread.start()
  350. except RuntimeError as exc:
  351. _release_token() # the runner's finally never runs when OS thread creation fails
  352. logger.warning(
  353. "Hook '%s' callback %s worker failed to start: %s — skipping",
  354. hook_name, callback_name, exc)
  355. return _HOOK_SKIPPED
  356. if not done.wait(timeout=timeout): # do not join — that would reintroduce the hang
  357. with self._hook_timeout_lock:
  358. # See #6622.
  359. self._hook_timeout_suppressed_until[suppression_key] = (
  360. time.monotonic() + self._hook_timeout_suppression_seconds)
  361. # The worker may have finished (and released its token) between the wait
  362. # expiring and this lock; recording it as abandoned then would block the
  363. # callback for that call id until reload with no thread behind it.
  364. if self._hook_running_callbacks.get(gate_key) is token:
  365. self._hook_abandoned.setdefault(suppression_key, set()).add(gate_key)
  366. logger.warning(
  367. "Hook '%s' callback %s timed out after %gs — skipping", hook_name, callback_name, timeout)
  368. return _HOOK_SKIPPED
  369. if "exc" in failure:
  370. raise failure["exc"]
  371. return outcome.get("value")
  372. def _subscribe_event(self, owner: str, event: str, callback: Callable) -> None:
  373. """Add an owner-tagged event subscription in registration order."""
  374. if not callable(callback):
  375. raise TypeError("Event subscriber callback must be callable")
  376. with self._event_lock:
  377. self._subscriptions.setdefault(event, []).append(_EventSubscription(owner, callback))
  378. def _remove_plugin_subscriptions(self, owner: str) -> int:
  379. """Remove every subscription owned by *owner*; return the count. Queued envelopes re-check
  380. membership per callback, so this also cancels already-snapshotted deliveries.
  381. TODO(#64229): when the central plugin ownership ledger / registration handles land, route this
  382. owner-tagged bookkeeping through that ledger so per-plugin unload cancels event subscriptions
  383. alongside every other registration surface. This method is the integration seam.
  384. """
  385. removed = 0
  386. with self._event_lock:
  387. for event in list(self._subscriptions):
  388. entries = self._subscriptions[event]
  389. retained = [entry for entry in entries if entry.owner != owner]
  390. removed += len(entries) - len(retained)
  391. if retained:
  392. self._subscriptions[event] = retained
  393. else:
  394. del self._subscriptions[event]
  395. return removed
  396. def _ensure_event_worker_locked(self) -> None:
  397. worker = self._event_worker
  398. if worker is not None and worker.is_alive():
  399. return
  400. worker = threading.Thread(
  401. target=self._event_worker_loop, args=(self._event_queue,), name="hermes-plugin-events",
  402. daemon=True,
  403. )
  404. self._event_worker = worker
  405. worker.start()
  406. def _event_worker_loop(self, dispatch_queue: queue.Queue[Any]) -> None:
  407. while True:
  408. item = dispatch_queue.get()
  409. try:
  410. if item is _EVENT_WORKER_STOP:
  411. return
  412. self._deliver_event(item)
  413. finally:
  414. if item is not _EVENT_WORKER_STOP:
  415. self._mark_event_done(item.generation)
  416. dispatch_queue.task_done()
  417. def _mark_event_done(self, generation: int) -> None:
  418. with self._event_idle:
  419. pending = self._event_pending_by_generation.get(generation, 0)
  420. if pending > 0:
  421. self._event_pending_by_generation[generation] = pending - 1
  422. self._event_idle.notify_all()
  423. def _deliver_event(self, item: _QueuedPluginEvent) -> None:
  424. """Deliver one queued event on the host-owned worker thread."""
  425. with self._event_lock:
  426. if item.generation != self._event_generation:
  427. return
  428. previous_depth = getattr(self._emit_depth, "value", 0)
  429. self._emit_depth.value = item.depth
  430. try:
  431. for subscription in item.subscriptions:
  432. with self._event_lock:
  433. if item.generation != self._event_generation:
  434. break
  435. # Owner unload may have removed this entry after the event was queued.
  436. if not any(cur is subscription for cur in self._subscriptions.get(item.event, [])):
  437. continue
  438. callback = subscription.callback
  439. try:
  440. # Fresh deep copy per subscriber: no callback can mutate what the next sees.
  441. resolve_plugin_command_result(
  442. item.context.copy().run(callback, **copy.deepcopy(item.payload)))
  443. except (Exception, SystemExit) as exc:
  444. # A subscriber that fails identically on every emit is reported once (#111922).
  445. self._report_hook_failure(item.event, callback, item.payload, exc, surface="Event")
  446. finally:
  447. self._emit_depth.value = previous_depth
  448. def _wait_for_event_dispatch(self, timeout: float = 2.0) -> bool:
  449. """Wait for the current event generation to become idle (test helper)."""
  450. with self._event_idle:
  451. generation = self._event_generation
  452. return self._event_idle.wait_for(
  453. lambda: self._event_pending_by_generation.get(generation, 0) == 0, timeout=timeout)
  454. def _dispatch_event(self, event: str, payload: dict[str, Any]) -> int:
  455. """Queue *event* without blocking; return the subscriber count scheduled. Pending work is
  456. bounded per generation so a blocking subscriber costs one worker and later emits drop."""
  457. depth = getattr(self._emit_depth, "value", 0)
  458. if depth >= _EVENT_EMIT_DEPTH_CAP:
  459. logger.warning(
  460. "Event bus recursion cap (%d) exceeded while dispatching '%s' "
  461. "— dropping this emit to prevent an infinite loop", _EVENT_EMIT_DEPTH_CAP, event)
  462. return 0
  463. budget_msg = "Event bus pending budget (%d) exhausted while dispatching '%s' — dropping this emit"
  464. with self._event_lock:
  465. subscriptions = tuple(self._subscriptions.get(event, []))
  466. if not subscriptions:
  467. return 0
  468. generation = self._event_generation
  469. pending = self._event_pending_by_generation.get(generation, 0)
  470. if pending >= _EVENT_PENDING_CAP:
  471. logger.warning(budget_msg, _EVENT_PENDING_CAP, event)
  472. return 0
  473. item = _QueuedPluginEvent(
  474. event=event, payload=dict(payload), subscriptions=subscriptions, depth=depth + 1,
  475. generation=generation, context=contextvars.copy_context())
  476. try:
  477. self._event_queue.put_nowait(item)
  478. except queue.Full:
  479. logger.warning(budget_msg, _EVENT_PENDING_CAP, event)
  480. return 0
  481. self._event_pending_by_generation[generation] = pending + 1
  482. self._ensure_event_worker_locked()
  483. return len(subscriptions)
  484. def has_hook(self, hook_name: str) -> bool:
  485. """Return True when at least one callback is registered for a hook."""
  486. from agent.safe_worker_policy import safe_worker_enabled
  487. if safe_worker_enabled():
  488. return False
  489. return bool(self._hooks.get(hook_name))
  490. def track_hook(self, context: Any, hook_name: str, callback: Callable, *, hold_stream: bool = False):
  491. """Register ``callback`` for ``hook_name`` on behalf of ``context`` (unknown names warn but are
  492. still stored). ``hold_stream=True`` on ``transform_llm_output`` declares the rewrite must reach
  493. the user first, so this profile's replies are not streamed (:meth:`holds_stream`)."""
  494. from hermes_cli.plugins import VALID_HOOKS
  495. handle = context._track_callback("hook", hook_name, callback, self._hooks, VALID_HOOKS)
  496. if hold_stream and hook_name != STREAM_HOLDING_HOOK:
  497. logger.warning("Plugin '%s' set hold_stream on '%s'; only %s rewrites the reply, ignored",
  498. context.manifest.name, hook_name, STREAM_HOLDING_HOOK)
  499. else:
  500. handle.hold_stream = hold_stream
  501. return handle
  502. def holds_stream(self) -> bool:
  503. """True while a loaded plugin's ``transform_llm_output`` registration declared ``hold_stream``.
  504. Read under the discovery lock: a reload empties the ledger before plugins re-register, and a
  505. turn deciding in that window would stream text the plugin must rewrite first."""
  506. with self._discovery_lock:
  507. return any(r.hold_stream for owned in self._ownership_ledger.values() for r in owned)
  508. def held_transforms(self) -> tuple[Callable, ...]:
  509. """The ``transform_llm_output`` callbacks a turn that holds its stream must run, captured with
  510. the hold decision under the discovery lock; ``()`` when nothing holds. A reload during the turn
  511. empties ``_hooks`` before plugins re-register, so the held turn keeps the set it started with
  512. (:meth:`run_held_transforms`)."""
  513. with self._discovery_lock:
  514. if not any(r.hold_stream for owned in self._ownership_ledger.values() for r in owned):
  515. return ()
  516. return tuple(self._hooks.get(STREAM_HOLDING_HOOK, []))
  517. def run_held_transforms(self, callbacks: tuple[Callable, ...], **kwargs: Any) -> tuple[Optional[str], bool]:
  518. """Run a held turn's snapshotted transforms: ``(rewrite, failed)``. ``rewrite`` is the first
  519. non-empty string; ``failed`` is True when none came back and a callback raised or timed out,
  520. so the caller must not deliver the original."""
  521. from agent.safe_worker_policy import safe_worker_enabled
  522. if safe_worker_enabled():
  523. return None, False
  524. kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
  525. failed: list = []
  526. for ret in self._run_hook_callbacks(STREAM_HOLDING_HOOK, callbacks, kwargs, failed):
  527. if isinstance(ret, str) and ret:
  528. return ret, False
  529. return None, bool(failed)
  530. async def ainvoke_hook(self, hook_name: str, **kwargs: Any) -> list[Any]:
  531. """:meth:`invoke_hook` for callers that are already on an event loop.
  532. Same payload narrowing, per-callback isolation and result contract. The difference is
  533. where an ``async def`` callback runs: here it is awaited on the caller's own loop, so a
  534. callback that awaits anything scheduled on that loop can make progress. Through the
  535. sync path it runs on a helper thread while the caller blocks in ``done.wait()`` — on the
  536. gateway that stalls the whole event loop for the callback's duration. Sync callbacks
  537. run inline. Bounded hooks keep ``plugins.hook_callback_timeout`` via ``asyncio.wait_for``
  538. (the coroutine is cancelled, not abandoned); a timed-out ``pre_tool_call`` fails closed.
  539. """
  540. from hermes_cli.plugins import _resolve_hook_callback_timeout
  541. if hook_name != "gateway_platform_event":
  542. kwargs.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
  543. results: list[Any] = []
  544. timeout = _resolve_hook_callback_timeout()
  545. use_timeout = _hook_uses_callback_timeout(hook_name, timeout)
  546. fail_closed = hook_name in _HOOK_TIMEOUT_FAIL_CLOSED_HOOKS
  547. for cb in self._hooks.get(hook_name, []):
  548. callback_name = getattr(cb, "__name__", repr(cb))
  549. try:
  550. ret = cb(**self._hook_callback_kwargs(cb, kwargs))
  551. if inspect.isawaitable(ret):
  552. ret = await (asyncio.wait_for(ret, timeout) if use_timeout else ret)
  553. if ret is not None:
  554. results.append(ret)
  555. except TimeoutError:
  556. logger.warning("Hook '%s' callback %s timed out after %.0fs", hook_name, callback_name, timeout)
  557. if fail_closed: # policy hook: fail closed with a block directive
  558. results.append({"action": "block", "message": _PRE_TOOL_CALL_TIMEOUT_BLOCK_MESSAGE})
  559. except (Exception, SystemExit) as exc:
  560. # Same isolation + failure contract as the sync path (#111922 warn-once, #109624
  561. # a raising policy guard fails closed).
  562. self._report_hook_failure(hook_name, cb, kwargs, exc)
  563. if fail_closed:
  564. results.append(_policy_error_block_directive(hook_name, cb, exc))
  565. return results
  566. def iter_hook_callbacks(self, hook_name: str) -> tuple[Callable, ...]:
  567. """Return a stable snapshot of callbacks registered for a hook."""
  568. from agent.safe_worker_policy import safe_worker_enabled
  569. if safe_worker_enabled():
  570. return ()
  571. return tuple(self._hooks.get(hook_name, ()))
  572. def render_system_prompt_sections(
  573. self, session_info: Mapping[str, Any]
  574. ) -> list[RenderedPluginSystemPromptSection]:
  575. """Render all registered sections deterministically and fail open."""
  576. from agent.safe_worker_policy import safe_worker_enabled
  577. if safe_worker_enabled():
  578. return []
  579. frozen_info = types.MappingProxyType(dict(session_info))
  580. rendered: list[RenderedPluginSystemPromptSection] = []
  581. total_chars = len(PLUGIN_SECTIONS_START) + len(PLUGIN_SECTIONS_END) + 2
  582. for _section_id, section in sorted(self._system_prompt_sections.items()):
  583. if len(rendered) >= MAX_SYSTEM_PROMPT_SECTIONS:
  584. logger.warning(
  585. "Plugin system prompt section %s exceeded the section-count "
  586. "budget (%d) and was skipped", section.id, MAX_SYSTEM_PROMPT_SECTIONS)
  587. continue
  588. text = self._render_prompt_section_text(section, frozen_info)
  589. if text is None:
  590. continue
  591. rendered_chars = len(format_system_prompt_section(section.id, text))
  592. if rendered:
  593. rendered_chars += 2 # canonical ``\n\n`` separator
  594. if total_chars + rendered_chars > MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS:
  595. logger.warning(
  596. "Plugin system prompt section %s (%s) exceeded the aggregate "
  597. "session budget (%d chars) and was skipped", section.id, section.plugin,
  598. MAX_SYSTEM_PROMPT_SECTIONS_TOTAL_CHARS)
  599. continue
  600. rendered.append(
  601. RenderedPluginSystemPromptSection(
  602. id=section.id, content=text, position=section.position, plugin=section.plugin))
  603. total_chars += rendered_chars
  604. logger.info(
  605. "Session plugin prompt section: id=%s plugin=%s position=%s chars=%d", section.id,
  606. section.plugin, section.position, len(text))
  607. return rendered
  608. @staticmethod
  609. def _render_prompt_section_text(
  610. section: PluginSystemPromptSection, frozen_info: Mapping[str, Any]
  611. ) -> Optional[str]:
  612. """Evaluate one section; return its stripped text or None (with a warning) when skipped."""
  613. def _skip(detail: str, *args: Any) -> None:
  614. logger.warning(
  615. "Plugin system prompt section %s (%s) " + detail, section.id, section.plugin, *args)
  616. try:
  617. value = section.content(frozen_info) if callable(section.content) else section.content
  618. except (Exception, SystemExit) as exc:
  619. _skip("raised and was skipped: %s", exc)
  620. return None
  621. if not isinstance(value, str):
  622. _skip("returned %s, not str; skipped", type(value).__name__)
  623. return None
  624. text = value.strip()
  625. if not text:
  626. return None
  627. if PLUGIN_SECTIONS_START in text or PLUGIN_SECTIONS_END in text:
  628. _skip("contained a reserved persistence marker and was skipped")
  629. return None
  630. if len(text) > section.max_chars:
  631. _skip("exceeded max_chars (%d > %d) and was skipped", len(text), section.max_chars)
  632. return None
  633. return text
  634. def has_middleware(self, kind: str) -> bool:
  635. """Return True when at least one callback is registered for middleware."""
  636. from agent.safe_worker_policy import safe_worker_enabled
  637. if safe_worker_enabled():
  638. return False
  639. return bool(self._middleware.get(kind))
  640. def invoke_middleware(
  641. self, kind: str, *, _payload_key: Optional[str] = None, **kwargs: Any
  642. ) -> list[Any]:
  643. """Call middleware callbacks for *kind* (each isolated); return non-``None`` results.
  644. Request middleware passes ``_payload_key``: a dict returned under it becomes the payload the
  645. next callback sees, so rewrites compose, and each callback gets its own copy of the payload and
  646. of ``original_<key>`` so an in-place edit cannot leak.
  647. """
  648. from agent.safe_worker_policy import safe_worker_enabled
  649. from hermes_cli.middleware import _safe_copy
  650. if safe_worker_enabled():
  651. return []
  652. results: list[Any] = []
  653. for cb in self._middleware.get(kind, []):
  654. call_kwargs = kwargs
  655. if _payload_key:
  656. original_key = "original_" + _payload_key
  657. call_kwargs = {**kwargs, _payload_key: _safe_copy(kwargs[_payload_key]),
  658. original_key: _safe_copy(kwargs[original_key])}
  659. try:
  660. ret = cb(**call_kwargs)
  661. if ret is not None:
  662. results.append(ret)
  663. if _payload_key and isinstance(ret, dict) and isinstance(ret.get(_payload_key), dict):
  664. kwargs[_payload_key] = ret[_payload_key]
  665. except (Exception, SystemExit) as exc:
  666. # Runs once per tool call like a hook, so a mis-declared callback floods identically.
  667. self._report_hook_failure(kind, cb, kwargs, exc, surface="Middleware")
  668. return results