emitter.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. """Monitoring emitter: fire-and-forget queue + background dispatcher — the single seam between
  2. producers (gateway status hooks, diagnostic log handler) and consumers (OTLP streamers).
  3. Hot-path invariant: ``emit()`` MUST return in O(microseconds), MUST NOT block on disk/network, and
  4. MUST NEVER raise into the caller — a monitoring failure is logged locally and dropped. On a full
  5. queue the *oldest* event is dropped. A daemon thread fans batches out to fail-isolated
  6. subscribers. Nothing is persisted: monitoring is an egress path, not a store.
  7. """
  8. from __future__ import annotations
  9. import logging
  10. import queue
  11. import threading
  12. import time
  13. from contextlib import suppress
  14. from typing import Any, Dict, Optional
  15. logger = logging.getLogger(__name__)
  16. _MAX_QUEUE = 10_000 # ring-buffer depth; oldest dropped when full
  17. _DRAIN_BATCH = 256
  18. class MonitoringEmitter:
  19. """Owns the queue, the dispatcher thread, and the subscriber list."""
  20. def __init__(self, *, enabled: bool = True) -> None:
  21. self._enabled = enabled
  22. self._q: queue.Queue[dict[str, Any]] = queue.Queue(maxsize=_MAX_QUEUE)
  23. self._dropped = 0
  24. self._dispatched = 0
  25. self._stop = threading.Event()
  26. self._started = False
  27. self._lock = threading.Lock()
  28. self._thread: Optional[threading.Thread] = None
  29. # Subscribers are callable(batch: list[dict]), invoked on the dispatcher thread.
  30. self._subscribers: list = []
  31. # ── public API (hot path) ───────────────────────────────────────────────
  32. def emit(self, event: Any) -> None:
  33. """Enqueue a dataclass with ``to_dict()`` or a plain dict. Never blocks, never raises."""
  34. if not self._enabled:
  35. return
  36. try:
  37. payload = event.to_dict() if hasattr(event, "to_dict") else dict(event)
  38. payload.setdefault("ts_ns", time.time_ns())
  39. self._ensure_started()
  40. try:
  41. self._q.put_nowait(payload)
  42. except queue.Full:
  43. # Drop oldest to make room — bounded memory, newest-wins.
  44. try:
  45. self._q.get_nowait()
  46. self._q.task_done()
  47. self._dropped += 1
  48. self._q.put_nowait(payload)
  49. except Exception:
  50. self._dropped += 1
  51. except Exception: # the hot-path invariant: never propagate
  52. logger.debug("monitoring emit failed", exc_info=True)
  53. # ── lifecycle ───────────────────────────────────────────────────────────
  54. def _ensure_started(self) -> None:
  55. if self._started:
  56. return
  57. with self._lock:
  58. if self._started:
  59. return
  60. self._thread = threading.Thread(target=self._run, name="hermes-monitoring-dispatch", daemon=True)
  61. self._thread.start()
  62. self._started = True
  63. def _run(self) -> None:
  64. while not self._stop.is_set():
  65. try:
  66. first = self._q.get(timeout=0.5)
  67. except queue.Empty:
  68. continue
  69. batch = [first]
  70. while len(batch) < _DRAIN_BATCH:
  71. try:
  72. batch.append(self._q.get_nowait())
  73. except queue.Empty:
  74. break
  75. try:
  76. self._dispatch(batch)
  77. finally:
  78. for _ in batch:
  79. self._q.task_done()
  80. def _dispatch(self, batch) -> None:
  81. for sub in list(self._subscribers):
  82. try:
  83. sub(batch)
  84. except Exception:
  85. logger.debug("monitoring subscriber failed", exc_info=True)
  86. self._dispatched += len(batch)
  87. def subscribe(self, callback) -> None:
  88. """Register a live batch subscriber; the first subscriber enables collection."""
  89. if callback not in self._subscribers:
  90. self._subscribers.append(callback)
  91. self._enabled = True
  92. def unsubscribe(self, callback) -> None:
  93. with suppress(ValueError):
  94. self._subscribers.remove(callback)
  95. if not self._subscribers:
  96. self._enabled = False
  97. # ── introspection / shutdown (tests, CLI) ───────────────────────────────
  98. def flush(self, timeout: float = 2.0) -> None:
  99. """Wait boundedly for queued and in-flight batches to finish dispatch."""
  100. if timeout <= 0:
  101. return
  102. finished = threading.Event()
  103. def _wait_for_completion() -> None:
  104. self._q.join()
  105. finished.set()
  106. threading.Thread(target=_wait_for_completion, name="hermes-monitoring-flush", daemon=True).start()
  107. finished.wait(timeout=timeout)
  108. def stats(self) -> dict[str, int]:
  109. return {"queued": self._q.qsize(), "dispatched": self._dispatched, "dropped": self._dropped, "subscribers": len(self._subscribers)}
  110. def close(self) -> None:
  111. self._stop.set()
  112. if self._thread is not None:
  113. self._thread.join(timeout=2.0)
  114. self._started = False
  115. # ── process-wide singleton ──────────────────────────────────────────────────
  116. _EMITTER: Optional[MonitoringEmitter] = None
  117. _EMITTER_LOCK = threading.Lock()
  118. def get_emitter() -> MonitoringEmitter:
  119. """Return the process-wide monitoring emitter."""
  120. global _EMITTER
  121. if _EMITTER is not None:
  122. return _EMITTER
  123. with _EMITTER_LOCK:
  124. if _EMITTER is None:
  125. # Collection is opt-in: disabled until a plane exporter attaches its first subscriber.
  126. _EMITTER = MonitoringEmitter(enabled=False)
  127. return _EMITTER
  128. def emit(event: Any) -> None:
  129. """Module-level convenience: emit via the singleton."""
  130. get_emitter().emit(event)
  131. def reset_emitter_for_tests(emitter: Optional[MonitoringEmitter] = None) -> None:
  132. """Swap the singleton (tests only)."""
  133. global _EMITTER
  134. with _EMITTER_LOCK:
  135. if _EMITTER is not None and emitter is not _EMITTER:
  136. with suppress(Exception):
  137. _EMITTER.close()
  138. _EMITTER = emitter
  139. __all__ = ["MonitoringEmitter", "emit", "get_emitter", "reset_emitter_for_tests"]