plugin_stream_hooks.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177
  1. """Asynchronous per-consumer plugin observers for streaming LLM output.
  2. Each registered hook callback gets its own bounded queue + daemon worker thread
  3. so plugin code never runs inline on the token path. Queues drop the oldest
  4. pending event when full; dispatchers for callbacks that are no longer
  5. registered are stopped lazily on the next lookup.
  6. """
  7. from __future__ import annotations
  8. import contextvars
  9. import logging
  10. import queue
  11. import threading
  12. from dataclasses import dataclass
  13. from typing import Any, Callable
  14. from hermes_cli.middleware import OBSERVER_SCHEMA_VERSION
  15. logger = logging.getLogger(__name__)
  16. _QUEUE_SIZE = 1024
  17. _STOP = object()
  18. @dataclass
  19. class _ConsumerDispatcher:
  20. hook_name: str
  21. callback: Callable[..., Any]
  22. events: queue.Queue[tuple[contextvars.Context, dict[str, Any]] | object]
  23. thread: threading.Thread | None = None
  24. _dispatcher_lock = threading.Lock()
  25. _dispatchers: dict[tuple[str, int], _ConsumerDispatcher] = {}
  26. def _callback_name(callback: Callable[..., Any]) -> str:
  27. return getattr(callback, "__name__", repr(callback))
  28. def _put_drop_oldest(events: queue.Queue[Any], item: Any) -> bool:
  29. """put_nowait; on a full queue evict the oldest pending event and retry once."""
  30. try:
  31. events.put_nowait(item)
  32. return True
  33. except queue.Full:
  34. try:
  35. events.get_nowait()
  36. events.task_done()
  37. except queue.Empty:
  38. pass
  39. try:
  40. events.put_nowait(item)
  41. return True
  42. except queue.Full:
  43. return False
  44. def _worker(dispatcher: _ConsumerDispatcher) -> None:
  45. while True:
  46. item = dispatcher.events.get()
  47. try:
  48. if item is _STOP:
  49. return
  50. context, payload = item
  51. payload = dict(payload)
  52. payload.setdefault("telemetry_schema_version", OBSERVER_SCHEMA_VERSION)
  53. try:
  54. # The worker outlives every turn and serves every profile; run the callback in the
  55. # enqueuing turn's contextvars so it sees that turn's profile scope (home override,
  56. # secrets), not an unbound context that fail-closed plugin bindings refuse (#118538).
  57. # ``copy()``: one snapshot fans out to N consumer threads and a Context can only be
  58. # entered by one thread at a time.
  59. context.copy().run(dispatcher.callback, **payload)
  60. except Exception as exc:
  61. # Fires once per streaming delta: a mis-declared callback fails identically every
  62. # time, so it goes through the manager's warn-once reporter (#111922).
  63. from hermes_cli.plugins import get_plugin_manager
  64. get_plugin_manager()._report_hook_failure(dispatcher.hook_name, dispatcher.callback, payload, exc)
  65. finally:
  66. dispatcher.events.task_done()
  67. def _registered_callbacks(hook_name: str) -> tuple[Callable[..., Any], ...]:
  68. try:
  69. from hermes_cli import plugins
  70. return plugins.iter_hook_callbacks(hook_name)
  71. except Exception:
  72. logger.debug("plugin stream hook callback lookup failed: %s", hook_name, exc_info=True)
  73. return ()
  74. def _stop_dispatcher(dispatcher: _ConsumerDispatcher, timeout: float = 1.0) -> None:
  75. _put_drop_oldest(dispatcher.events, _STOP)
  76. if dispatcher.thread is not None:
  77. dispatcher.thread.join(timeout=timeout)
  78. def _start_dispatcher(hook_name: str, callback: Callable[..., Any]) -> _ConsumerDispatcher:
  79. dispatcher = _ConsumerDispatcher(hook_name=hook_name, callback=callback, events=queue.Queue(maxsize=_QUEUE_SIZE))
  80. dispatcher.thread = threading.Thread(
  81. target=_worker, args=(dispatcher,), daemon=True, name=f"plugin-stream-hook:{hook_name}"
  82. )
  83. dispatcher.thread.start()
  84. return dispatcher
  85. def _dispatchers_for(hook_name: str) -> list[_ConsumerDispatcher]:
  86. """Live dispatcher per registered callback (restarting dead workers); stale
  87. ones for unregistered callbacks are stopped outside the lock."""
  88. callbacks = _registered_callbacks(hook_name)
  89. if not callbacks:
  90. return []
  91. callback_ids = {id(callback) for callback in callbacks}
  92. ready: list[_ConsumerDispatcher] = []
  93. with _dispatcher_lock:
  94. stale = [_dispatchers.pop(key) for key in list(_dispatchers) if key[0] == hook_name and key[1] not in callback_ids]
  95. for callback in callbacks:
  96. key = (hook_name, id(callback))
  97. dispatcher = _dispatchers.get(key)
  98. if dispatcher is None or dispatcher.thread is None or not dispatcher.thread.is_alive():
  99. dispatcher = _dispatchers[key] = _start_dispatcher(hook_name, callback)
  100. ready.append(dispatcher)
  101. for dispatcher in stale:
  102. _stop_dispatcher(dispatcher, timeout=0.2)
  103. return ready
  104. def enqueue_plugin_stream_hook(hook_name: str, **payload: Any) -> bool:
  105. """Queue an observer hook for each consumer without running plugin code inline."""
  106. queued = False
  107. item = (contextvars.copy_context(), dict(payload))
  108. for dispatcher in _dispatchers_for(hook_name):
  109. if _put_drop_oldest(dispatcher.events, item):
  110. queued = True
  111. else:
  112. logger.debug(
  113. "plugin stream hook queue full after drop-oldest: %s callback=%s",
  114. hook_name, _callback_name(dispatcher.callback),
  115. )
  116. return queued
  117. def has_stream_observer_hooks() -> bool:
  118. return any(_registered_callbacks(name) for name in ("on_stream_start", "on_stream_delta", "on_stream_end"))
  119. def has_reasoning_stream_observer_hooks() -> bool:
  120. return stream_reasoning_deltas_enabled() and bool(_registered_callbacks("on_stream_delta"))
  121. def stream_reasoning_deltas_enabled() -> bool:
  122. """Return True only when the user opted plugins into reasoning deltas.
  123. Read-only scalar lookup: skips ``load_config()``'s deepcopy. Callers on the token path
  124. should still cache the result per stream (``_fire_reasoning_delta`` does)."""
  125. try:
  126. from hermes_cli import config as config_mod
  127. config = config_mod.load_config_readonly()
  128. return bool(config_mod.cfg_get(config, "plugins", "stream_reasoning_deltas", default=False))
  129. except Exception:
  130. logger.debug("failed to read plugins.stream_reasoning_deltas", exc_info=True)
  131. return False
  132. def shutdown_plugin_stream_hook_dispatcher(timeout: float = 1.0) -> None:
  133. """Stop background stream hook dispatchers; used by tests and clean shutdown paths."""
  134. global _dispatchers
  135. with _dispatcher_lock:
  136. dispatchers = list(_dispatchers.values())
  137. _dispatchers = {}
  138. for dispatcher in dispatchers:
  139. _stop_dispatcher(dispatcher, timeout=timeout)