mcp_tool_loop.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. """Background-loop plumbing for tools.mcp_tool: cross-process discovery file lock, scheduling
  2. coroutines onto the MCP loop from caller threads (with profile HOME override and dashboard OAuth
  3. flow propagation) and the loop's exception handler. Origin state (``_lock``, ``_mcp_loop``) is
  4. read through ``_core`` so ``mock.patch("tools.mcp_tool.X")`` keeps working."""
  5. from __future__ import annotations
  6. import asyncio
  7. import concurrent.futures
  8. import contextlib
  9. import errno
  10. import logging
  11. import os
  12. import threading
  13. import time
  14. from typing import Any, Coroutine, Optional
  15. from tools.mcp_tool_common import _core
  16. from tools import mcp_tool_lifecycle as _lifecycle
  17. logger = logging.getLogger("tools.mcp_tool")
  18. class _LockCookie:
  19. """Holds a cross-process file lock; ``release()`` drops it. The file object MUST stay open while
  20. held: both fcntl and portalocker locks are tied to the descriptor."""
  21. def __init__(self, fh: Any) -> None:
  22. self._fh = fh
  23. def release(self) -> None:
  24. if self._fh is None:
  25. return
  26. # Best effort: an unlock/close failure must never propagate out of discovery.
  27. with contextlib.suppress(Exception):
  28. if os.name == "posix":
  29. import fcntl
  30. fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)
  31. else:
  32. import portalocker
  33. portalocker.unlock(self._fh)
  34. with contextlib.suppress(Exception):
  35. self._fh.close()
  36. self._fh = None
  37. def _acquire_lock_on_fh(fh: Any) -> bool:
  38. """Non-blocking exclusive lock (fcntl on POSIX, portalocker elsewhere). False when another process
  39. holds it; unexpected errors propagate so the caller can treat locking as unavailable."""
  40. if os.name == "posix":
  41. import fcntl
  42. try:
  43. fcntl.flock(fh.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  44. except OSError as e:
  45. if e.errno in (errno.EACCES, errno.EAGAIN, errno.EWOULDBLOCK):
  46. return False
  47. raise
  48. return True
  49. import portalocker
  50. try:
  51. portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB)
  52. except portalocker.LockException:
  53. return False
  54. return True
  55. def _try_acquire_mcp_discovery_lock() -> Any:
  56. """``_LockCookie`` (acquired), ``None`` (held by another process) or ``_LOCK_UNAVAILABLE``
  57. (locking broken: run discovery unguarded)."""
  58. # The cached path lives on the ORIGIN module (tests reset ``tools.mcp_tool._MCP_DISCOVERY_LOCK_PATH``).
  59. # A routed profile (multiplexed gateway) locks under ITS home: the launch profile's lock file
  60. # would serialize discovery across profiles and never coordinate with B's own single-profile processes.
  61. from tools import mcp_tool as _origin
  62. try:
  63. from hermes_constants import get_hermes_home, get_hermes_home_override
  64. if get_hermes_home_override() is not None:
  65. lock_path = str(get_hermes_home() / ".mcp-discovery.lock")
  66. else:
  67. if _origin._MCP_DISCOVERY_LOCK_PATH is None:
  68. _origin._MCP_DISCOVERY_LOCK_PATH = str(get_hermes_home() / ".mcp-discovery.lock")
  69. lock_path = _origin._MCP_DISCOVERY_LOCK_PATH
  70. fh = open(lock_path, "w", encoding="utf-8")
  71. except Exception:
  72. return _core._LOCK_UNAVAILABLE
  73. try:
  74. acquired = _acquire_lock_on_fh(fh)
  75. except Exception:
  76. fh.close()
  77. return _core._LOCK_UNAVAILABLE
  78. if acquired:
  79. return _LockCookie(fh)
  80. fh.close()
  81. return None
  82. def _mcp_loop_exception_handler(loop, context):
  83. """Suppress the benign 'Event loop is closed' RuntimeError httpx finalizers raise against the
  84. dead loop during shutdown; forward the rest."""
  85. exc = context.get("exception")
  86. if not (isinstance(exc, RuntimeError) and "Event loop is closed" in str(exc)):
  87. loop.default_exception_handler(context)
  88. def _wrap_with_home_override(coro: Coroutine) -> Coroutine:
  89. """Carry the caller's context-local HERMES_HOME override into ``coro`` (task-local on the MCP
  90. loop, so concurrent scopes don't interfere)."""
  91. try:
  92. from hermes_constants import get_hermes_home_override, reset_hermes_home_override, set_hermes_home_override
  93. home_override = get_hermes_home_override()
  94. except Exception:
  95. home_override = None
  96. if not home_override:
  97. return coro
  98. async def _scoped():
  99. token = set_hermes_home_override(home_override)
  100. try:
  101. return await coro
  102. finally:
  103. reset_hermes_home_override(token)
  104. return _scoped()
  105. def _wrap_with_dashboard_oauth_flow(coro):
  106. """Propagate a dashboard OAuth flow onto the dedicated MCP loop task."""
  107. try:
  108. from tools.mcp_dashboard_oauth import dashboard_oauth_flow, get_dashboard_oauth_flow
  109. flow = get_dashboard_oauth_flow()
  110. except Exception:
  111. flow = None
  112. if flow is None:
  113. return coro
  114. async def _scoped():
  115. with dashboard_oauth_flow(flow):
  116. return await coro
  117. return _scoped()
  118. def _running_loop() -> Optional[asyncio.AbstractEventLoop]:
  119. """The MCP loop when it is up, else None (read under ``_lock``)."""
  120. with _core._lock:
  121. loop = _core._mcp_loop
  122. return loop if loop is not None and loop.is_running() else None
  123. def _run_on_mcp_loop(coro_or_factory, timeout: float = 30):
  124. """Schedule a coroutine (or zero-arg factory — avoids leaking a never-awaited coroutine when the
  125. loop is down) on the MCP loop and block until done, polling so user interrupts are honored."""
  126. from tools.interrupt import is_interrupted
  127. from agent.async_utils import safe_schedule_threadsafe
  128. loop = _running_loop()
  129. if loop is None:
  130. if asyncio.iscoroutine(coro_or_factory):
  131. coro_or_factory.close()
  132. raise RuntimeError("MCP event loop is not running")
  133. # run_coroutine_threadsafe copies the LOOP thread's context, so a per-request profile scope
  134. # would vanish here; re-establish it inside the task's own context.
  135. coro = _wrap_with_dashboard_oauth_flow(_wrap_with_home_override(
  136. coro_or_factory() if callable(coro_or_factory) else coro_or_factory))
  137. future = safe_schedule_threadsafe(coro, loop, logger=logger, log_message="MCP scheduling failed")
  138. if future is None:
  139. raise RuntimeError("MCP event loop unavailable (failed to schedule)")
  140. start_time = time.monotonic()
  141. deadline = None if timeout is None else start_time + timeout
  142. while True:
  143. if is_interrupted():
  144. future.cancel()
  145. raise InterruptedError("User sent a new message")
  146. remaining = 0.1 if deadline is None else deadline - time.monotonic()
  147. if remaining <= 0:
  148. future.cancel()
  149. raise TimeoutError(f"MCP call timed out after {time.monotonic() - start_time:.1f}s "
  150. f"(configured timeout: {float(timeout):.1f}s)")
  151. try:
  152. return future.result(timeout=min(0.1, remaining))
  153. except concurrent.futures.TimeoutError:
  154. # Aliases builtin TimeoutError, so it also fires for the coroutine's own timeout: a done
  155. # future must yield its outcome.
  156. if future.done():
  157. return future.result()
  158. def _signal_reconnect(server: Any) -> bool:
  159. """Ask a server task to rebuild its transport, thread-safely: the event lives on the MCP loop,
  160. so set via ``call_soon_threadsafe`` when it runs (direct ``.set()`` otherwise). False when the
  161. server has no reconnect machinery."""
  162. event = getattr(server, "_reconnect_event", None)
  163. if event is None:
  164. return False
  165. loop = _core._mcp_loop
  166. if isinstance(event, asyncio.Event) and loop is not None and loop.is_running():
  167. loop.call_soon_threadsafe(event.set)
  168. else:
  169. event.set()
  170. return True
  171. def reconnect_mcp_server(server_name: str) -> bool:
  172. """Ask a currently-live MCP server to rebuild after external re-auth."""
  173. from tools.mcp_tool_scope import _resolve_server_key
  174. with _core._lock:
  175. server = _core._servers.get(_resolve_server_key(server_name))
  176. return server is not None and _signal_reconnect(server)
  177. def _wait_for_server_session_ready(srv: Any, *, old_session: Any = None, timeout: float = 15.0) -> bool:
  178. """Poll until the server exposes a usable, ready session (during a reconnect ``srv.session`` is
  179. briefly None or stale; retrying blindly burns breaker strikes). With ``old_session`` the observed
  180. session must differ. Iteration-bounded, not deadline-bounded: tests freeze ``time.monotonic``."""
  181. iterations = max(1, int(max(float(timeout), 0.0) / 0.25))
  182. for i in range(iterations):
  183. session = getattr(srv, "session", None)
  184. ready = getattr(srv, "_ready", None)
  185. try:
  186. is_ready = bool(ready.is_set()) if hasattr(ready, "is_set") else True
  187. except Exception:
  188. is_ready = True
  189. if session is not None and session is not old_session and is_ready:
  190. return True
  191. if i < iterations - 1:
  192. time.sleep(0.25)
  193. return False
  194. def _signal_reconnect_and_wait(server_name: str, srv: Any, *, op_description: str, timeout: float = 15.0) -> bool:
  195. """Request a transport rebuild and wait for the fresh session. ``_ready`` is cleared on the loop
  196. BEFORE ``_reconnect_event`` is set, else the readiness poll returns at once on the dead session."""
  197. loop = _core._mcp_loop
  198. if loop is None or not loop.is_running():
  199. return False
  200. def _request_reconnect() -> None:
  201. ready, reconnect_event = getattr(srv, "_ready", None), getattr(srv, "_reconnect_event", None)
  202. if hasattr(ready, "clear"):
  203. ready.clear()
  204. if hasattr(reconnect_event, "set"):
  205. reconnect_event.set()
  206. old_session = getattr(srv, "session", None)
  207. logger.info("MCP server '%s': %s requesting transport reconnect", server_name, op_description)
  208. loop.call_soon_threadsafe(_request_reconnect)
  209. return _wait_for_server_session_ready(srv, old_session=old_session, timeout=timeout)
  210. def _ensure_mcp_loop():
  211. """Start the background loop thread if not running. The loop/thread handles live on the ORIGIN
  212. module (tests read and reset ``tools.mcp_tool._mcp_loop``), so they are written there."""
  213. from tools import mcp_tool as _origin
  214. with _core._lock:
  215. if _origin._mcp_loop is not None and _origin._mcp_loop.is_running():
  216. return
  217. loop = _origin._mcp_loop = asyncio.new_event_loop()
  218. loop.set_exception_handler(_mcp_loop_exception_handler)
  219. _origin._mcp_thread = threading.Thread(target=loop.run_forever, name="mcp-event-loop", daemon=True)
  220. _origin._mcp_thread.start()
  221. def _stop_mcp_loop(*, only_if_idle: bool = False) -> bool:
  222. """Stop the background event loop and join its thread."""
  223. from tools import mcp_tool as _origin
  224. with _core._lock:
  225. if only_if_idle and (_core._servers or _core._server_connecting):
  226. logger.debug("Leaving MCP event loop running; active servers are registered or connecting")
  227. return False
  228. loop, thread = _origin._mcp_loop, _origin._mcp_thread
  229. _origin._mcp_loop = _origin._mcp_thread = None
  230. if loop is None:
  231. return True
  232. # Drain before stopping: tasks still suspended when the loop closes get resumed by the GC
  233. # against a closed loop. shutdown_mcp_servers only reaps _servers; everything else ends here.
  234. future = None
  235. # Drain before stopping: closing the loop with tasks still suspended leaves their coroutines for the GC,
  236. # whose finalizer then resumes them to run cleanup against a loop that is already closed -> "Event loop
  237. # is closed" (#60197).
  238. if loop.is_running():
  239. from agent.async_utils import safe_schedule_threadsafe
  240. future = safe_schedule_threadsafe(
  241. _lifecycle._drain_and_stop_mcp_loop(), loop, logger=logger,
  242. log_message="MCP loop drain: failed to schedule", log_level=logging.WARNING)
  243. if future is not None:
  244. try:
  245. future.result(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT + 1)
  246. except TimeoutError:
  247. logger.warning("Timed out waiting for MCP loop drain after %.1fs", _core._MCP_LOOP_DRAIN_TIMEOUT + 1)
  248. except BaseException as exc:
  249. logger.warning("Error draining MCP loop tasks: %s", exc)
  250. elif not loop.is_closed():
  251. try:
  252. loop.run_until_complete(_lifecycle._drain_mcp_loop_tasks(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT))
  253. except BaseException as exc:
  254. logger.warning("Error draining stopped MCP loop tasks: %s", exc)
  255. if future is None and loop.is_running(): # drain-and-stop wasn't scheduled: stop it ourselves
  256. loop.call_soon_threadsafe(loop.stop)
  257. if thread is not None:
  258. thread.join(timeout=5)
  259. if thread.is_alive():
  260. logger.warning("MCP event loop thread did not stop within 5.0s")
  261. try:
  262. loop.close()
  263. except Exception as exc:
  264. logger.warning("Unable to close MCP event loop cleanly: %s", exc)
  265. # The loop is gone, so no session can be in flight: reap active too.
  266. _lifecycle._kill_orphaned_mcp_children(include_active=True)
  267. return True