periodic_scheduler.py 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164
  1. """One process-wide timer thread for periodic maintenance callbacks.
  2. Replaces the per-child ``while not stop.wait(interval): body()`` daemon
  3. threads (delegate heartbeat, durable turn-lease refresher, turn-liveness
  4. watchdog). With ~130 in-process subagents those added 2-3 sleeping OS
  5. threads per child. This module keeps ONE daemon thread that only orders due
  6. times; every due body runs on its own short-lived daemon worker. A blocked
  7. callback therefore cannot delay unrelated lease/liveness timers, while
  8. steady-state thread use stays near zero (workers exist only while a body is
  9. actually running, never one per scheduled handle).
  10. Semantics match the loop they replace: the first call happens ``interval``
  11. seconds after :func:`schedule`, and each following call ``interval`` seconds
  12. after the previous body *returned* (drift-free wrt. body duration was never
  13. a property of the old loops either). A body that returns ``False`` stops
  14. itself; a body that raises is logged at debug and rescheduled — one bad
  15. callback must never kill the shared thread. A handle never overlaps itself:
  16. it is re-queued only once its in-flight run has returned. A worker-start
  17. failure never retires the handle: it is re-queued and logged at warning.
  18. """
  19. from __future__ import annotations
  20. from contextvars import Context, copy_context
  21. import heapq
  22. import itertools
  23. import logging
  24. import threading
  25. import time
  26. from typing import Callable, Optional
  27. logger = logging.getLogger(__name__)
  28. _THREAD_NAME = "hermes-periodic-scheduler"
  29. _CALLBACK_THREAD_PREFIX = "hermes-periodic-callback"
  30. class ScheduledHandle:
  31. """Cancel token for one scheduled periodic callback."""
  32. __slots__ = ("_cancelled", "_context", "_fn", "_interval", "_runner", "_scheduler")
  33. def __init__(self, scheduler: PeriodicScheduler, fn: Callable[[], object], interval: float):
  34. self._scheduler = scheduler
  35. self._fn = fn
  36. self._interval = interval
  37. self._cancelled = False
  38. self._runner: Optional[threading.Thread] = None
  39. # Safe to reuse because this scheduler never overlaps runs of one handle.
  40. self._context = copy_context()
  41. @property
  42. def cancelled(self) -> bool:
  43. return self._cancelled
  44. def cancel(self, wait: Optional[float] = None) -> None:
  45. """Stop future runs. ``wait`` (seconds) additionally blocks until an
  46. in-flight run of this callback finishes — the analogue of
  47. ``thread.join(timeout=wait)`` on the old per-child thread."""
  48. self._scheduler._cancel(self, wait)
  49. class PeriodicScheduler:
  50. def __init__(self) -> None:
  51. self._cond = threading.Condition()
  52. self._heap: list = [] # (due, seq, handle)
  53. self._seq = itertools.count()
  54. self._thread: Optional[threading.Thread] = None
  55. def schedule(self, fn: Callable[[], object], interval: float) -> ScheduledHandle:
  56. handle = ScheduledHandle(self, fn, float(interval))
  57. with self._cond:
  58. self._requeue(handle)
  59. if self._thread is None or not self._thread.is_alive():
  60. self._thread = threading.Thread(target=self._run, name=_THREAD_NAME, daemon=True)
  61. self._thread.start()
  62. self._cond.notify()
  63. return handle
  64. def _cancel(self, handle: ScheduledHandle, wait: Optional[float]) -> None:
  65. with self._cond:
  66. self._retire(handle)
  67. self._cond.notify()
  68. runner = handle._runner
  69. if wait and runner is not None and threading.current_thread() is not runner:
  70. runner.join(wait)
  71. def _dispatch(self, handle: ScheduledHandle) -> None:
  72. """Start ``handle``'s body on its own worker. Called with ``_cond`` held
  73. so ``cancel`` can never observe a half-set runner."""
  74. runner = threading.Thread(
  75. target=self._run_callback,
  76. args=(handle, handle._fn, handle._context),
  77. name=f"{_CALLBACK_THREAD_PREFIX}-{id(handle):x}",
  78. daemon=True,
  79. )
  80. handle._runner = runner
  81. try:
  82. runner.start()
  83. except Exception:
  84. handle._runner = None
  85. logger.warning(
  86. "failed to start periodic callback worker %r; retrying in %s s",
  87. handle._fn,
  88. handle._interval,
  89. exc_info=True,
  90. )
  91. if not handle._cancelled:
  92. self._requeue(handle)
  93. self._cond.notify()
  94. @staticmethod
  95. def _retire(handle: ScheduledHandle) -> None:
  96. """Mark cancelled and drop the callback (``_cond`` held). The callback is usually a bound method of
  97. the handle's owner, which keeps the handle: an owner -> handle -> method -> owner cycle that outlives
  98. the turn and, on a quiet gateway, pins the whole agent until a full GC."""
  99. handle._cancelled = True
  100. handle._fn = handle._context = None
  101. def _requeue(self, handle: ScheduledHandle) -> None:
  102. """Push ``handle``'s next due time (``_cond`` held)."""
  103. heapq.heappush(self._heap, (time.monotonic() + handle._interval, next(self._seq), handle))
  104. def _run_callback(self, handle: ScheduledHandle, fn: Callable[[], object], context: Context) -> None:
  105. stop = False
  106. try:
  107. stop = context.run(fn) is False
  108. except Exception:
  109. logger.debug("periodic callback %r raised", fn, exc_info=True)
  110. finally:
  111. with self._cond:
  112. handle._runner = None
  113. if stop:
  114. self._retire(handle)
  115. elif not handle._cancelled:
  116. self._requeue(handle)
  117. self._cond.notify()
  118. def _run(self) -> None:
  119. while True:
  120. with self._cond:
  121. while True:
  122. if not self._heap:
  123. self._cond.wait()
  124. continue
  125. due, _, handle = self._heap[0]
  126. if handle._cancelled:
  127. heapq.heappop(self._heap)
  128. continue
  129. delay = due - time.monotonic()
  130. if delay > 0:
  131. self._cond.wait(delay)
  132. continue
  133. heapq.heappop(self._heap)
  134. self._dispatch(handle)
  135. break
  136. _DEFAULT = PeriodicScheduler()
  137. def schedule(fn: Callable[[], object], interval: float) -> ScheduledHandle:
  138. """Run ``fn()`` every ``interval`` seconds via the shared scheduler."""
  139. return _DEFAULT.schedule(fn, interval)