mcp_startup.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319
  1. """Shared CLI/TUI-safe helpers for background MCP discovery."""
  2. from __future__ import annotations
  3. import sys
  4. import threading
  5. from contextlib import contextmanager, nullcontext
  6. from contextvars import copy_context
  7. from typing import Dict, Iterator, Optional, Set
  8. from hermes_constants import hermes_home_key
  9. _mcp_discovery_lock = threading.Lock()
  10. # Discovery slot per profile home (``hermes_home_key()`` follows the context-local HERMES_HOME
  11. # override): a shared Desktop/dashboard backend serving several profiles runs one discovery per
  12. # profile instead of the first profile to build an agent claiming the slot for everybody (#67605).
  13. # A single-profile process has exactly one key, so behaviour is the old single-slot form.
  14. _mcp_discovery_started: set[str] = set()
  15. _mcp_discovery_thread: dict[str, threading.Thread] = {}
  16. _mcp_discovery_deferred: Optional[threading.Timer] = None
  17. # Process-wide MCP server-name allowlist derived from ``-t/--toolsets``.
  18. # ``None`` = no filter (spawn every configured server). Set once at CLI
  19. # startup by ``set_mcp_server_filter`` and honored by every discovery path
  20. # in this module (inline, background, deferred), so a ``-t terminal``
  21. # oneshot never cold-starts MCP subprocesses it cannot use.
  22. _mcp_server_filter: Optional[list[str]] = None
  23. def set_mcp_server_filter(toolsets: object) -> Optional[list[str]]:
  24. """Derive the MCP spawn allowlist from a ``-t/--toolsets`` value.
  25. Built-in toolset names in the list are harmless (they never match a
  26. configured ``mcp_servers`` key). ``all``/``*`` or an empty/absent value
  27. clears the filter. Returns the stored list for logging/tests.
  28. """
  29. global _mcp_server_filter
  30. names: list[str] = []
  31. if isinstance(toolsets, str):
  32. names = [t.strip() for t in toolsets.split(",") if t.strip()]
  33. elif isinstance(toolsets, (list, tuple, set)):
  34. for item in toolsets:
  35. names.extend(t.strip() for t in str(item).split(",") if t.strip())
  36. if not names or "all" in names or "*" in names:
  37. _mcp_server_filter = None
  38. else:
  39. _mcp_server_filter = names
  40. return _mcp_server_filter
  41. def get_mcp_server_filter() -> Optional[list[str]]:
  42. return _mcp_server_filter
  43. def _has_configured_mcp_servers() -> bool:
  44. """Cheap config probe so non-MCP users avoid importing the MCP stack."""
  45. try:
  46. from hermes_cli.config import read_raw_config
  47. raw_config = read_raw_config() or {}
  48. if isinstance(raw_config.get("mcp_servers"), dict) and raw_config["mcp_servers"]:
  49. return True
  50. from hermes_cli.agent_plugins import has_enabled_agent_plugin_mcp
  51. return has_enabled_agent_plugin_mcp(raw_config)
  52. except Exception:
  53. return True # conservative: still try discovery in the background; startup can't block
  54. def _discovery_registered_servers(status) -> bool:
  55. """True when a discovery run left servers usable: a live session OR a lazy registration.
  56. A ``lazy: true`` server never connects until first use, so an all-lazy config (the
  57. memory-saving setup the feature exists for) looked like a run that achieved nothing:
  58. the zero-connected warning fired on every startup and the retry path re-ran discovery
  59. on every later call (#111717).
  60. """
  61. return any(entry.get("connected") or entry.get("status") == "lazy" for entry in (status or []))
  62. def _any_mcp_connected() -> bool:
  63. from tools.mcp_tool_discovery import get_mcp_status
  64. return _discovery_registered_servers(get_mcp_status() or [])
  65. def _servers_awaiting_connect() -> list[str]:
  66. from tools.mcp_tool_discovery import mcp_servers_awaiting_connect
  67. pending = mcp_servers_awaiting_connect()
  68. return pending if _mcp_server_filter is None else [n for n in pending if n in _mcp_server_filter]
  69. def start_background_mcp_discovery(*, logger, thread_name: str) -> None:
  70. """Spawn one background MCP discovery thread per profile home.
  71. If the first run exits without connecting any server (e.g. startup cancellation / OOM restart),
  72. later calls may retry instead of pinning the profile in "already started" with zero MCP tools.
  73. Likewise a server added to ``mcp_servers`` after that run (``hermes mcp add`` against a running
  74. Desktop backend) is connected by the next call, which every agent build makes, so a new session
  75. gets its tools without a reload (#76954). Discovery is additive: live servers are untouched.
  76. """
  77. home_key = hermes_home_key()
  78. with _mcp_discovery_lock:
  79. if home_key in _mcp_discovery_started:
  80. thread = _mcp_discovery_thread.get(home_key)
  81. if thread is not None and thread.is_alive():
  82. return
  83. try:
  84. connected = _any_mcp_connected()
  85. pending = _servers_awaiting_connect() if connected else []
  86. except Exception:
  87. return
  88. if connected and not pending:
  89. return
  90. if connected:
  91. logger.info("MCP server(s) %s not connected yet; running discovery", ", ".join(pending))
  92. else:
  93. logger.warning(
  94. "Background MCP discovery previously exited with no connected "
  95. "servers; retrying discovery thread"
  96. )
  97. _mcp_discovery_started.discard(home_key)
  98. _mcp_discovery_thread.pop(home_key, None)
  99. _mcp_discovery_started.add(home_key)
  100. if not _has_configured_mcp_servers():
  101. return
  102. # Bare threads start from an empty context: run discovery under a copy of the caller's, so
  103. # the context-local HERMES_HOME override (multi-profile dashboard/desktop backends, #67605)
  104. # AND the profile's secret scope reach it. Without the scope a session switched to profile
  105. # X would discover the LAUNCH profile's mcp_servers, and ``${TOKEN}`` interpolation / the
  106. # stdio child env would fail closed (multiplex) or resolve the launch profile's value.
  107. # The config gate above already runs on the caller's thread, so it sees the same context.
  108. def _discover() -> None:
  109. try:
  110. _discover_mcp_tools_without_interactive_oauth()
  111. try:
  112. if not _any_mcp_connected():
  113. logger.warning("Background MCP discovery completed with zero connected servers")
  114. except Exception:
  115. logger.debug("Failed to inspect MCP status after background discovery", exc_info=True)
  116. except Exception:
  117. logger.debug("Background MCP tool discovery failed", exc_info=True)
  118. finally:
  119. with _mcp_discovery_lock:
  120. _mcp_discovery_thread.pop(home_key, None)
  121. thread = threading.Thread(target=copy_context().run, args=(_discover,), name=thread_name, daemon=True)
  122. _mcp_discovery_thread[home_key] = thread
  123. thread.start()
  124. def _resolve_discovery_timeout(explicit: float | None, *, single_query: bool = False) -> float:
  125. """Resolve the MCP discovery wait bound: explicit arg > config.yaml > ``DEFAULT_CONFIG``.
  126. Lazy and fail-safe: a missing/invalid value or broken config falls back to a short bound so
  127. startup can never hang or crash.
  128. """
  129. if explicit is not None:
  130. return explicit
  131. key = "mcp_single_query_discovery_timeout" if single_query else "mcp_discovery_timeout"
  132. fallback = 15.0 if single_query else 1.5
  133. try:
  134. from hermes_cli.config import load_config, DEFAULT_CONFIG
  135. default = float(DEFAULT_CONFIG.get(key, fallback))
  136. except Exception:
  137. return fallback
  138. try:
  139. val = float((load_config() or {}).get(key, default))
  140. return val if val > 0 else default
  141. except Exception:
  142. return default
  143. # GIL budget for discovery: the discovery thread does bursty CPU work (mcp/pydantic imports,
  144. # JSON-RPC schema parsing, tool registration) that, at the default 5 ms switch interval, rides
  145. # the GIL convoy effect and can starve concurrent threads — the agent-build thread and the main
  146. # event loop — for tens of seconds (#60371: "agent initialization timed out" after a serve
  147. # restart, _wait_agent(30s) error 5032). While discovery runs we drop the switch interval so its
  148. # CPU bursts are sliced finely enough that waiters stay responsive, then restore it.
  149. _DISCOVERY_SWITCH_INTERVAL_S = 0.0005
  150. @contextmanager
  151. def _discovery_gil_budget() -> Iterator[None]:
  152. """Temporarily lower the interpreter switch interval so discovery's CPU work can't
  153. monopolize the GIL against concurrent threads (see _DISCOVERY_SWITCH_INTERVAL_S)."""
  154. prev = sys.getswitchinterval()
  155. sys.setswitchinterval(_DISCOVERY_SWITCH_INTERVAL_S)
  156. try:
  157. yield
  158. finally:
  159. sys.setswitchinterval(prev)
  160. def _discover_mcp_tools_without_interactive_oauth() -> None:
  161. """Run MCP discovery without letting OAuth read from the user's stdin."""
  162. try:
  163. from tools.mcp_oauth import suppress_interactive_oauth
  164. except Exception:
  165. suppress_interactive_oauth = nullcontext
  166. with _discovery_gil_budget(), suppress_interactive_oauth():
  167. from tools.mcp_tool_discovery import discover_mcp_tools
  168. # Only pass the kwarg when a filter is set: many tests (and any
  169. # out-of-tree caller) stub discover_mcp_tools with a zero-arg
  170. # callable, and the unfiltered call shape is unchanged.
  171. if _mcp_server_filter is None:
  172. discover_mcp_tools()
  173. else:
  174. discover_mcp_tools(allowed_mcp_names=_mcp_server_filter)
  175. def defer_background_mcp_discovery(*, logger, thread_name: str, delay: float | None) -> None:
  176. """Arm ``start_background_mcp_discovery`` to run ``delay`` seconds from now.
  177. Used by the Desktop ``serve`` backend after its socket is announced: the thread's first act is
  178. the ~350ms ``mcp`` SDK import, which would hold the GIL against the renderer's connect + first
  179. hydration reads (or the web_server import) if started earlier.
  180. ``delay=None`` arms without a clock: the standalone dashboard fires it from the first ``/api/ws``
  181. client (``start_deferred_mcp_discovery_now``) or the first agent build (``wait_for_mcp_discovery``),
  182. so an idle, unvisited dashboard never spawns the configured stdio MCP servers (#58733).
  183. """
  184. global _mcp_discovery_deferred
  185. with _mcp_discovery_lock:
  186. if hermes_home_key() in _mcp_discovery_started or _mcp_discovery_deferred is not None:
  187. return
  188. def _fire() -> None:
  189. global _mcp_discovery_deferred
  190. with _mcp_discovery_lock:
  191. _mcp_discovery_deferred = None
  192. start_background_mcp_discovery(logger=logger, thread_name=thread_name)
  193. # ``None`` builds the Timer only as the holder of ``_fire``; it is never started and
  194. # ``start_deferred_mcp_discovery_now`` runs ``timer.function()`` directly.
  195. timer = threading.Timer(0 if delay is None else delay, _fire)
  196. timer.daemon = True
  197. timer.name = f"{thread_name}-deferred"
  198. _mcp_discovery_deferred = timer
  199. if delay is not None:
  200. timer.start()
  201. def start_deferred_mcp_discovery_now() -> None:
  202. """Run an armed deferred start immediately (idempotent, thread-safe)."""
  203. global _mcp_discovery_deferred
  204. with _mcp_discovery_lock: # take the slot atomically: two racing first clients fire once
  205. timer, _mcp_discovery_deferred = _mcp_discovery_deferred, None
  206. if timer is None:
  207. return
  208. timer.cancel()
  209. timer.function()
  210. def wait_for_mcp_discovery(timeout: float | None = None, *, single_query: bool = False) -> None:
  211. """Wait for background MCP discovery before the first tool snapshot.
  212. ``join`` returns the instant discovery completes, so this only blocks for a still-pending
  213. server's real connect time. ``single_query`` uses ``mcp_single_query_discovery_timeout``
  214. (15s vs 1.5s) because one-shot sessions have no second turn to recover.
  215. """
  216. start_deferred_mcp_discovery_now()
  217. thread = _current_home_thread()
  218. if thread is None or not thread.is_alive():
  219. return
  220. thread.join(timeout=_resolve_discovery_timeout(timeout, single_query=single_query))
  221. def _current_home_thread() -> Optional[threading.Thread]:
  222. """Discovery thread for the profile home the caller is scoped to, if any."""
  223. return _mcp_discovery_thread.get(hermes_home_key())
  224. def mcp_discovery_in_flight() -> bool:
  225. """True if THIS module's discovery thread (for the current profile home) is still running.
  226. Mirrors ``tui_gateway.entry.mcp_discovery_in_flight``; surfaces that start discovery here
  227. (desktop, dashboard sidecar) populate this thread, so the late-refresh scheduler consults both.
  228. Those processes populate THIS module's ``_mcp_discovery_thread``, not ``tui_gateway.entry``'s, so the
  229. late-refresh scheduler must consult both to decide whether a slow server's tools are still pending (see
  230. #51587).
  231. """
  232. thread = _current_home_thread()
  233. return thread is not None and thread.is_alive()
  234. def join_mcp_discovery(timeout: float | None = None) -> bool:
  235. """Block up to ``timeout`` for THIS module's discovery; True once complete, False if still
  236. running. For the off-critical-path late-refresh waiter (accepts a long wait, reports outcome)."""
  237. thread = _current_home_thread()
  238. if thread is None:
  239. return True
  240. thread.join(timeout=timeout)
  241. return not thread.is_alive()
  242. def ensure_mcp_discovery_before_agent_build(
  243. *,
  244. logger,
  245. timeout: float | None = None,
  246. single_query: bool = False,
  247. thread_name: str = "cli-mcp-discovery") -> None:
  248. """Give configured MCP tools a bounded chance to register before AIAgent.
  249. Non-interactive first turns (``chat -q``, ``hermes -z``) can construct ``AIAgent`` before any
  250. path started discovery, and ``wait_for_mcp_discovery()`` only joins an existing thread — so
  251. start discovery if needed, then wait up to the configured bound.
  252. """
  253. try:
  254. start_background_mcp_discovery(logger=logger, thread_name=thread_name)
  255. wait_for_mcp_discovery(timeout=timeout, single_query=single_query)
  256. except Exception:
  257. logger.debug("MCP discovery readiness check failed before agent build", exc_info=True)