turn_executor.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125
  1. """Unbounded executor for gateway turn bodies (see ``GatewayRunner._get_executor`` in gateway/run.py)."""
  2. from __future__ import annotations
  3. import concurrent.futures
  4. import contextvars
  5. import logging
  6. import queue
  7. import threading
  8. logger = logging.getLogger(__name__)
  9. # A parked worker exits after this long without work, so a burst's threads do not outlive it by much.
  10. _IDLE_TIMEOUT_S = 60.0
  11. class _Worker:
  12. __slots__ = ("inbox", "thread")
  13. def __init__(self) -> None:
  14. self.thread: threading.Thread | None = None
  15. self.inbox: queue.SimpleQueue = queue.SimpleQueue()
  16. class _UnboundedThreadExecutor(concurrent.futures.Executor):
  17. """Every submitted item starts at once on its own thread; no queue, no cap.
  18. ``ThreadPoolExecutor(max_workers=None)`` is NOT unbounded (it is ``min(32, cpu_count + 4)``),
  19. which is the same silent queue at a larger number. An item goes to a parked (idle) worker when
  20. one exists, otherwise to a new thread. Reuse matters because ``submit`` runs on the event loop
  21. and ``Thread.start()`` waits there until the new thread gets the GIL: with many turns running,
  22. every submit (the handoff watcher's 2 s idle gates included) froze the loop (#142061). Handing an
  23. item to a parked worker is a queue put. A worker parked for ``idle_timeout`` exits.
  24. A reused thread must look fresh to its next item: each item runs in an empty
  25. ``contextvars.Context`` (what a new thread starts with), and ``reset_reused_thread_state`` drops
  26. the thread-keyed state a turn leaves behind before the worker parks. A worker whose reset fails
  27. is retired, never reused.
  28. ``_threads`` holds the BUSY threads only, as before: ``_stop_pool`` / ``_shutdown_executor`` join
  29. and count those. A parked worker holds no work; shutdown wakes it to exit.
  30. """
  31. def __init__(self, thread_name_prefix: str = "", idle_timeout: float = _IDLE_TIMEOUT_S):
  32. self._prefix = thread_name_prefix
  33. self._threads: set = set()
  34. self._idle: list[_Worker] = []
  35. self._shutdown = False
  36. self._lock = threading.Lock()
  37. self._n = 0
  38. self._idle_timeout = idle_timeout
  39. def submit(self, fn, /, *args, **kwargs):
  40. fut: concurrent.futures.Future = concurrent.futures.Future()
  41. item = (fut, fn, args, kwargs)
  42. # One critical section for check + hand-off/start + register (as ThreadPoolExecutor.submit does), so a
  43. # concurrent shutdown() either refuses this item or sees its thread; never a live, uncounted one.
  44. with self._lock:
  45. if self._shutdown:
  46. raise RuntimeError("cannot schedule new futures after shutdown")
  47. if self._idle:
  48. worker = self._idle.pop() # LIFO: the warmest thread; the coldest time out
  49. self._threads.add(worker.thread)
  50. worker.inbox.put(item)
  51. return fut
  52. self._n += 1
  53. worker = _Worker()
  54. # health: allow HX012 -- a worker serves many items; each runs in a fresh empty Context, as a new thread would
  55. t = threading.Thread(target=self._work, args=(worker, item), name=f"{self._prefix}_{self._n}", daemon=True)
  56. worker.thread = t
  57. # Start BEFORE registering: at the OS thread limit start() raises, and an unstarted thread
  58. # left in _threads would make shutdown's join() raise and skip the quiesce decision.
  59. t.start()
  60. self._threads.add(t)
  61. return fut
  62. def _work(self, worker: _Worker, item) -> None:
  63. while item is not None:
  64. fut, fn, args, kwargs = item
  65. del item
  66. if fut.set_running_or_notify_cancel():
  67. try:
  68. fut.set_result(contextvars.Context().run(fn, *args, **kwargs))
  69. except BaseException as exc: # health: allow BLE001 -- delivered to the item's future, as ThreadPoolExecutor does
  70. fut.set_exception(exc)
  71. del fut, fn, args, kwargs # a parked worker must not pin the last item's arguments
  72. item = self._park(worker)
  73. def _park(self, worker: _Worker):
  74. """Reset this thread, then wait for its next item; None means exit."""
  75. from gateway.turn_thread_reset import reset_reused_thread_state
  76. try:
  77. reset_reused_thread_state()
  78. reusable = True
  79. except Exception:
  80. logger.warning("turn worker state reset failed; retiring the thread", exc_info=True)
  81. reusable = False
  82. # Taking the lock also waits for submit() to finish registering this thread.
  83. with self._lock:
  84. self._threads.discard(worker.thread)
  85. if self._shutdown or not reusable:
  86. return None
  87. self._idle.append(worker)
  88. try:
  89. return worker.inbox.get(timeout=self._idle_timeout)
  90. except queue.Empty:
  91. with self._lock:
  92. if worker in self._idle:
  93. self._idle.remove(worker)
  94. return None
  95. # submit() claimed this worker as the timeout fired: its item is already on the way.
  96. return worker.inbox.get()
  97. def shutdown(self, wait: bool = True, *, cancel_futures: bool = False):
  98. # cancel_futures is accepted for API parity only: there is no queue, so nothing is pending.
  99. with self._lock:
  100. self._shutdown = True
  101. threads = list(self._threads)
  102. idle, self._idle = self._idle, []
  103. for worker in idle:
  104. worker.inbox.put(None)
  105. if wait:
  106. for t in threads:
  107. t.join()