mcp_tool_discovery.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950
  1. """Connecting and discovery for tools.mcp_tool: per-server connect cooldown, connect /
  2. lazy-start / recycled-stdio wake-up, ``register_mcp_servers`` / ``discover_mcp_tools`` and
  3. the status / probe public API. Origin state (``_servers``, ``_lock``, the loop, patchable
  4. helpers) is read through ``_core`` so ``mock.patch("tools.mcp_tool.X")`` keeps working."""
  5. from __future__ import annotations
  6. import asyncio
  7. import logging
  8. import threading
  9. import time
  10. from contextlib import contextmanager
  11. from pathlib import Path
  12. from typing import Dict, List, Optional, Set, Tuple
  13. from tools.mcp_tool_common import _core, _parse_boolish, mcp_server_enabled
  14. from tools import mcp_tool_config as _config
  15. from tools import mcp_tool_errors as _errors
  16. from tools import mcp_tool_lifecycle as _lifecycle
  17. from tools import mcp_tool_loop as _loop
  18. from tools import mcp_tool_registration as _registration
  19. from tools.mcp_tool_schema import MCP_TOOL_NAME_PREFIX
  20. from tools.mcp_tool_scope import _key_name, _key_scope, _key_visible_in_scope, _resolve_server_key, _server_key
  21. logger = logging.getLogger("tools.mcp_tool")
  22. # Default max concurrent MCP server connections per discovery pass (one unbounded
  23. # `asyncio.gather` spawned every server's subprocess tree simultaneously); config.yaml
  24. # ``mcp.discovery_concurrency`` overrides it, 0 = unlimited (#117373).
  25. _DISCOVERY_CONNECT_CONCURRENCY = 4
  26. def _discovery_connect_concurrency() -> int:
  27. """``mcp.discovery_concurrency`` from config (0 = unlimited); a non-integer or negative value
  28. warns and falls back to the default rather than silently running unbounded."""
  29. try:
  30. from hermes_cli.config import load_config
  31. raw = (load_config().get("mcp") or {}).get("discovery_concurrency", _DISCOVERY_CONNECT_CONCURRENCY)
  32. except Exception:
  33. return _DISCOVERY_CONNECT_CONCURRENCY
  34. if isinstance(raw, bool) or not isinstance(raw, int) or raw < 0:
  35. logger.warning("mcp.discovery_concurrency=%r is not a non-negative integer; using %d",
  36. raw, _DISCOVERY_CONNECT_CONCURRENCY)
  37. return _DISCOVERY_CONNECT_CONCURRENCY
  38. return raw
  39. def _record_connect_failure(server_name: str) -> None:
  40. """Stamp a geometric, capped retry cooldown after a failed connect (under ``_lock``)."""
  41. key = _server_key(server_name)
  42. n = _core._server_connect_failures.get(key, 0) + 1
  43. _core._server_connect_failures[key] = n
  44. backoff = min(_core._CONNECT_RETRY_BASE_BACKOFF_SEC * (2 ** (n - 1)), _core._CONNECT_RETRY_MAX_BACKOFF_SEC)
  45. _core._server_connect_retry_after[key] = time.monotonic() + backoff
  46. def _clear_connect_failure(server_name: str) -> None:
  47. """Clear the connect-cooldown state after a successful connection."""
  48. key = _server_key(server_name)
  49. _core._server_connect_failures.pop(key, None)
  50. _core._server_connect_retry_after.pop(key, None)
  51. def _connect_cooldown_active(server_name: str) -> bool:
  52. """True if ``server_name`` is still within its retry cooldown (this scope's connection: one
  53. profile's failing ``x`` must not shadow another profile's healthy ``x``)."""
  54. deadline = _core._server_connect_retry_after.get(_server_key(server_name))
  55. return deadline is not None and time.monotonic() < deadline
  56. def _owner_scope_home() -> Optional[Path]:
  57. """The profile home whose secret scope MCP credential reads must resolve under, or None for a
  58. single-profile process (scope key ``None``).
  59. The owner is the profile the connection is keyed under (``_mcp_registry_scope()``), never the
  60. ambient one, so a served profile is never handed another profile's token (#111151). A scope the
  61. caller already bound is rebuilt rather than trusted: a bound mapping is a snapshot, and the
  62. gateway's boot-time one is taken before the profile's external secret source may have answered
  63. (#119092) — the rebuild retries that hydration (cached once it succeeds)."""
  64. scope_key = _core._mcp_registry_scope()
  65. if scope_key is None:
  66. return None
  67. from agent.secret_scope import current_secret_scope_home
  68. return Path(current_secret_scope_home() or scope_key)
  69. async def _install_owner_secret_scope():
  70. """Bind the connection OWNER's profile secret scope when the caller has none; else None.
  71. ``MCPServerTask.start`` ensure_futures the run task, which copies THIS context, so one
  72. binding covers transport bring-up (``_build_safe_env`` stdio child env) and every later
  73. revival inside that task; unscoped, those ``get_secret`` reads fail closed under multiplexing
  74. and the server parks with zero tools (#113746). ``${VAR}`` refs are interpolated earlier, at
  75. config load, under :func:`_owner_secret_scope`.
  76. """
  77. from agent.secret_scope import set_secret_scope
  78. home = _owner_scope_home()
  79. if home is None:
  80. return None
  81. # Off-loop: an external source runs a helper subprocess (once per home, then cached).
  82. return set_secret_scope(await asyncio.to_thread(_owner_secret_mapping, home), profile_home=str(home))
  83. _HYDRATE_RETRY_INTERVAL_SEC = 30.0
  84. _hydrate_retry_after: dict[str, float] = {}
  85. def _owner_secret_mapping(home: Path) -> dict[str, str]:
  86. """The owner's FRESH secret mapping. The launch profile's is ``launch_secret_scope`` (its files
  87. over the frozen launch env): a credential injected only by systemd ``Environment=`` /
  88. ``op run`` / Compose has no file to rebuild from, so a files-only rebuild dropped it, left the
  89. header literal ``${VAR}`` and parked a server that worked. A served profile gets its own ``.env``
  90. + external sources only. A source that did not fully hydrate is retried at most once per
  91. interval per home: every retry is a helper subprocess, and connect/reconnect loops are tight."""
  92. from agent.secret_scope import _is_process_home, build_profile_secret_scope
  93. from hermes_cli import env_loader
  94. key = str(home.resolve())
  95. if time.monotonic() >= _hydrate_retry_after.get(key, 0.0):
  96. env_loader.hydrate_profile_secret_sources(home)
  97. if key not in env_loader._APPLIED_HOMES:
  98. _hydrate_retry_after[key] = time.monotonic() + _HYDRATE_RETRY_INTERVAL_SEC
  99. if _is_process_home(home):
  100. from tui_gateway.launch_profile_policy import launch_secret_scope
  101. return launch_secret_scope(home)
  102. return build_profile_secret_scope(home)
  103. @contextmanager
  104. def _owner_secret_scope():
  105. """Sync twin of :func:`_install_owner_secret_scope` for the config load in the caller's thread.
  106. ``_load_mcp_config`` interpolates ``${VAR}`` header/URL refs through ``get_secret`` and swallows
  107. the resulting ``UnscopedSecretError`` into ``{}``, so an unscoped discover / reconcile / status /
  108. probe for a routed profile saw ZERO servers — stdio siblings included — before the connect-site
  109. binding could ever run (#113746). Same owner rule; scope key ``None`` binds nothing."""
  110. from agent.secret_scope import reset_secret_scope, set_secret_scope
  111. home = _owner_scope_home()
  112. if home is None:
  113. yield
  114. return
  115. token = set_secret_scope(_owner_secret_mapping(home), profile_home=str(home))
  116. try:
  117. yield
  118. finally:
  119. reset_secret_scope(token)
  120. async def _connect_server(name: str, config: dict) -> _core.MCPServerTask:
  121. """Create an MCPServerTask, start it, return once ready (tear down with ``server.shutdown()``
  122. on the same loop). Raises on bad config, missing HTTP support or connect failure."""
  123. server = _core.MCPServerTask(name)
  124. claim = _core._connect_server_claim.get()
  125. if claim is not None:
  126. claim(server)
  127. # The run task copies this context: don't retain the discovery closure for its life.
  128. claim_token = _core._connect_server_claim.set(None) if claim is not None else None
  129. scope_token = None
  130. try:
  131. scope_token = await _install_owner_secret_scope()
  132. # The config was rendered at load time (a lazy server's at boot): re-render under the
  133. # owner's fresh scope so a ref left literal before its secret source answered resolves
  134. # now, and refuse to send one that still does not.
  135. config = _config._require_rendered_remote(name, _config._interpolate_env_vars(config))
  136. await server.start(config)
  137. except asyncio.CancelledError:
  138. raise # start() already reaps server._task; shutdown() here could swallow the cancel
  139. except BaseException:
  140. # Discovery owns claimed tasks (recoverable park); standalone probes must reap locally.
  141. if claim is None:
  142. try:
  143. await server.shutdown()
  144. except Exception as shutdown_exc:
  145. logger.debug("MCP server '%s' shutdown during orphan-reap failed: %s", name, shutdown_exc)
  146. raise
  147. finally:
  148. if scope_token is not None:
  149. from agent.secret_scope import reset_secret_scope
  150. reset_secret_scope(scope_token)
  151. if claim_token is not None:
  152. _core._connect_server_claim.reset(claim_token)
  153. return server
  154. def _request_lazy_reconnect(server_name: str, server: _core.MCPServerTask) -> bool:
  155. """Wake a recycled stdio server and wait briefly for a fresh session."""
  156. loop = _loop._running_loop() if server._is_recycled_stdio() else None
  157. if loop is None:
  158. return False
  159. def _wake() -> None:
  160. server._ready.clear()
  161. server._reconnect_event.set()
  162. loop.call_soon_threadsafe(_wake)
  163. async def _await_ready() -> bool:
  164. deadline = time.monotonic() + _core._RECYCLED_RECONNECT_TIMEOUT
  165. while time.monotonic() < deadline:
  166. if server.session is not None and server._ready.is_set():
  167. return True
  168. await asyncio.sleep(0.05)
  169. return False
  170. try:
  171. return bool(_loop._run_on_mcp_loop(_await_ready, timeout=_core._RECYCLED_RECONNECT_TIMEOUT))
  172. except Exception as exc:
  173. logger.warning("MCP server '%s': lazy reconnect after stdio recycle failed: %s", server_name, exc)
  174. return False
  175. def _resolve_server_lazy(name: str, config: dict) -> bool:
  176. """True when ``mcp_servers.<name>.lazy`` defers connect to first tool use (default off).
  177. Gated per-server by ``mcp_servers.<name>.lazy`` in config (default OFF), following the same per-server
  178. key pattern as ``idle_timeout_seconds``. Design from #56832 (Vansh5632).
  179. """
  180. return _parse_boolish(config.get("lazy", False), default=False)
  181. def _note_connect_failure(name: str, exc: BaseException) -> str:
  182. """Record a failed connect (under ``_lock``): error text for status, cooldown stamp."""
  183. message = _errors._format_connect_error(exc)
  184. with _core._lock:
  185. key = _server_key(name)
  186. _core._server_connecting.discard(key)
  187. _core._server_connect_errors[key] = message
  188. _record_connect_failure(name)
  189. return message
  190. def _stderr_tail_note(name: str) -> str:
  191. """Operator-facing stderr tail for a connect-failure log line (#125300).
  192. ``Connection closed`` alone hides why the child died — the reason sits in the shared
  193. mcp-stderr.log. The stored status message stays single-line; only the log line quotes
  194. the child's output."""
  195. from tools.mcp_tool_config import _tail_server_stderr
  196. tail = _tail_server_stderr(name)
  197. if not tail:
  198. return ""
  199. indented = "\n".join(f" {line}" for line in tail.splitlines())
  200. return f"\n child stderr tail (full log: mcp-stderr.log):\n{indented}"
  201. def _note_connect_success(name: str) -> None:
  202. """Clear connecting/error/cooldown state after a successful connect (under ``_lock``)."""
  203. with _core._lock:
  204. key = _server_key(name)
  205. _core._server_connecting.discard(key)
  206. _core._server_connect_errors.pop(key, None)
  207. _clear_connect_failure(name)
  208. def _adopt_server(name: str, server: _core.MCPServerTask) -> None:
  209. """Publish *server* into ``_servers`` under the connecting scope's key (under ``_lock``)."""
  210. with _core._lock:
  211. key = _server_key(name)
  212. _core._servers[key] = server
  213. _core._server_scope_keys[key] = _core._mcp_registry_scope()
  214. def _ensure_lazy_server_connected(server_name: str) -> bool:
  215. """Connect a lazily-registered server on demand (sync; blocks). Honours the cooldown and the
  216. ``_server_connecting`` dedup set; routes through ``_discover_and_register_server`` so
  217. park/recycle/cooldown bookkeeping stays in one place. True when a live session exists.
  218. See #50394.
  219. """
  220. with _core._lock:
  221. key = _resolve_server_key(server_name)
  222. server = _core._servers.get(key)
  223. if server is not None and server.session is not None:
  224. return True
  225. config = _core._lazy_server_configs.get(key)
  226. if (not config or _connect_cooldown_active(server_name)
  227. or key in _core._server_connecting):
  228. return False
  229. _core._server_connecting.add(key)
  230. _core._server_connect_errors.pop(key, None)
  231. logger.info("MCP server '%s': lazy start on first use", server_name)
  232. _loop._ensure_mcp_loop()
  233. connect_timeout = config.get("connect_timeout", _core._DEFAULT_CONNECT_TIMEOUT)
  234. try:
  235. _loop._run_on_mcp_loop(lambda: _discover_and_register_server(server_name, config),
  236. timeout=float(connect_timeout) + 30.0)
  237. except BaseException as exc:
  238. logger.warning("Lazy MCP connect failed for '%s': %s%s",
  239. server_name, _note_connect_failure(server_name, exc), _stderr_tail_note(server_name))
  240. return False
  241. _note_connect_success(server_name)
  242. with _core._lock:
  243. _core._lazy_server_configs.pop(key, None)
  244. stale_fingerprint = _core._lazy_server_fingerprints.pop(key, None)
  245. cached_names = _core._lazy_server_tool_names.pop(key, None) or []
  246. server = _core._servers.get(key)
  247. live_names = set(getattr(server, "_registered_tool_names", []) or [])
  248. # The cached manifest may advertise tools the live server no longer serves.
  249. phantom_names = [n for n in cached_names if n not in live_names]
  250. if phantom_names:
  251. for tool_name in phantom_names:
  252. _registration._deregister_mcp_tool_all_scopes(key, tool_name)
  253. logger.info("MCP server '%s': deregistered %d phantom cached tool(s) not served live (stale schema-cache "
  254. "fingerprint %s): %s", server_name, len(phantom_names), stale_fingerprint, ", ".join(phantom_names))
  255. return server is not None and server.session is not None
  256. def _get_connected_server_for_call(server_name: str) -> Optional[_core.MCPServerTask]:
  257. """Return a connected server; the single first-use connect point for lazy servers and
  258. the wake-up point for recycled stdio ones.
  259. Also the single first-use connect point for lazy (schema-cache registered) servers, so raw tool calls
  260. AND the resource/prompt utility handlers all trigger the deferred spawn (#56832).
  261. """
  262. with _core._lock:
  263. key = _resolve_server_key(server_name)
  264. server = _core._servers.get(key)
  265. is_lazy = key in _core._lazy_server_configs
  266. if is_lazy and (server is None or server.session is None):
  267. _ensure_lazy_server_connected(server_name)
  268. elif server is not None and server.session is None and server._is_recycled_stdio():
  269. _request_lazy_reconnect(server_name, server)
  270. else:
  271. return server
  272. with _core._lock:
  273. return _core._servers.get(key)
  274. async def _discover_and_register_server(name: str, config: dict) -> list[str]:
  275. """Connect one server, register its tools; return the registered names."""
  276. # The claim fires inside _connect_server while this frame is suspended (list, not nonlocal).
  277. claimed: list[_core.MCPServerTask] = []
  278. claim_token = _core._connect_server_claim.set(claimed.append)
  279. try:
  280. server = await asyncio.wait_for(_connect_server(name, config),
  281. timeout=config.get("connect_timeout", _core._DEFAULT_CONNECT_TIMEOUT))
  282. except BaseException:
  283. server = claimed[0] if claimed else None
  284. task = server._task if server is not None else None
  285. task_cancelling = task.cancelling() if task is not None and hasattr(task, "cancelling") else 0
  286. if (server is not None and server._error is not None and task is not None
  287. and not task.done() and not task_cancelling):
  288. # Recoverable park: the run task self-probes, so adopt it for shutdown/revival.
  289. _adopt_server(name, server)
  290. elif server is not None:
  291. await server.shutdown()
  292. raise
  293. finally:
  294. _core._connect_server_claim.reset(claim_token)
  295. with _core._lock:
  296. key = _server_key(name)
  297. _core._server_connecting.discard(key)
  298. _core._server_connect_errors.pop(key, None)
  299. _adopt_server(name, server)
  300. registered_names = _registration._register_server_tools(name, server, config)
  301. server._registered_tool_names = list(registered_names)
  302. logger.info("MCP server '%s' (%s): registered %d tool(s): %s", name,
  303. "HTTP" if "url" in config else "stdio", len(registered_names), ", ".join(registered_names))
  304. return registered_names
  305. def _select_new_servers(servers: dict[str, dict]) -> dict[str, dict]:
  306. """Pick connect candidates (enabled, not connected/connecting/lazy, not in backoff) and
  307. refresh per-server bookkeeping. Known servers without a live session are parked or
  308. mid-reconnect with tools deregistered, so nothing else can nudge them: signal a reconnect."""
  309. with _core._lock:
  310. current_scope = _core._mcp_registry_scope()
  311. # This scope's own connections OR shared ones it adopted (``register_connected_into_current_scope``
  312. # ran first): a same-named server owned by ANOTHER profile with other credentials is not
  313. # "connected" for us and must be a candidate, or this profile ends up silently tool-less.
  314. keys = {k: _resolve_server_key(k, current_scope, current=False) for k in servers}
  315. # Only attempt servers that aren't already connected (or currently connecting) and are enabled.
  316. # Checking ``_server_connecting`` prevents duplicate subprocess spawns when ``discover_mcp_tools()``
  317. # is called from multiple entry-points before the first batch finishes (#58862).
  318. new_servers = {
  319. k: v for k, v in servers.items()
  320. if keys[k] not in _core._servers and keys[k] not in _core._server_connecting
  321. and keys[k] not in _core._lazy_server_configs
  322. and mcp_server_enabled(v) and not _connect_cooldown_active(k)}
  323. stale_cached = [_core._servers[keys[k]] for k, v in servers.items()
  324. if keys[k] in _core._servers and mcp_server_enabled(v)
  325. and getattr(_core._servers[keys[k]], "session", None) is None]
  326. for srv_name in new_servers:
  327. _core._server_connecting.add(keys[srv_name])
  328. _core._server_scope_keys[keys[srv_name]] = current_scope
  329. _core._server_connect_errors.pop(keys[srv_name], None)
  330. # Track which servers opt-in to parallel tool calls (idempotent). Keyed by THIS profile's own
  331. # key: the opt-in is the calling profile's policy, so B's parallel-safe `x` never makes A's
  332. # same-named serial `x` (own connection or adopted) run two calls at once.
  333. for srv_name, srv_cfg in servers.items():
  334. own_key = _server_key(srv_name, current_scope, current=False)
  335. if _parse_boolish(srv_cfg.get("supports_parallel_tool_calls", False), default=False):
  336. _core._parallel_safe_servers.add(own_key)
  337. else:
  338. _core._parallel_safe_servers.discard(own_key)
  339. for srv in stale_cached:
  340. auth_park = getattr(srv, "_auth_park", None)
  341. # Parked on an OAuth failure and not re-authorized since: leave it to its timed self-probe.
  342. if auth_park is None or auth_park[0].credential_stamp() != auth_park[1]:
  343. _loop._signal_reconnect(srv)
  344. return new_servers
  345. def _register_lazy_from_cache(new_servers: dict[str, dict]) -> tuple[dict[str, dict], int, int]:
  346. """Register ``lazy: true`` servers from a valid schema-cache entry without connecting
  347. (missing/stale entry or failed registration -> eager). Returns (eager servers, lazy tool
  348. count, lazy server count)."""
  349. # A missing or stale cache entry falls back to the normal eager connect below (which write-through
  350. # refreshes the cache for next time). See #56832.
  351. eager_servers: dict[str, dict] = dict(new_servers)
  352. lazy_registered = 0
  353. lazy_server_count = 0
  354. try:
  355. from tools.mcp_schema_cache import config_fingerprint, get_cached_entry
  356. except Exception: # pragma: no cover - cache module missing
  357. return eager_servers, 0, 0
  358. for name, cfg in new_servers.items():
  359. if not _resolve_server_lazy(name, cfg):
  360. continue
  361. entry = get_cached_entry(name, config_fingerprint(cfg))
  362. if not entry:
  363. continue
  364. with _core._lock:
  365. _core._server_connecting.discard(_server_key(name))
  366. try:
  367. names = _registration._register_from_cache_sync(name, cfg, entry)
  368. except Exception as exc:
  369. logger.warning("Failed lazy MCP registration for '%s': %s", name, exc)
  370. with _core._lock:
  371. _core._server_connecting.add(_server_key(name))
  372. continue
  373. eager_servers.pop(name, None)
  374. lazy_registered += len(names)
  375. lazy_server_count += 1
  376. return eager_servers, lazy_registered, lazy_server_count
  377. async def _discover_all(new_servers: dict[str, dict]) -> None:
  378. """Connect every candidate concurrently; record per-server outcome.
  379. Concurrency is bounded: every stdio server spawns child processes, so an
  380. unbounded gather turns a config with many servers into a simultaneous
  381. N-process spawn burst (RAM/CPU spike, EMFILE risk) on every backend boot.
  382. """
  383. # Flat cap for all transports; 0 (unlimited) keeps the original unbounded gather.
  384. cap = _discovery_connect_concurrency()
  385. semaphore = asyncio.Semaphore(cap) if cap > 0 else None
  386. async def _connect_bounded(name: str, cfg: dict):
  387. if semaphore is None:
  388. return await _discover_and_register_server(name, cfg)
  389. async with semaphore:
  390. return await _discover_and_register_server(name, cfg)
  391. results = await asyncio.gather(
  392. *(_connect_bounded(name, cfg) for name, cfg in new_servers.items()),
  393. return_exceptions=True)
  394. for name, result in zip(new_servers, results):
  395. if isinstance(result, BaseException):
  396. command = new_servers.get(name, {}).get("command")
  397. message = _note_connect_failure(name, result)
  398. logger.warning("Failed to connect to MCP server '%s'%s: %s%s",
  399. name, f" (command={command})" if command else "", message,
  400. _stderr_tail_note(name))
  401. else:
  402. _note_connect_success(name)
  403. def _run_discovery_pass(new_servers: dict[str, dict]) -> None:
  404. """Run ``_discover_all`` on the MCP loop with the interrupt flag parked; clean up
  405. ``_server_connecting`` when the pass dies early."""
  406. # Executor threads are reused: a prior session's stale interrupt must not cancel this pass.
  407. from tools.interrupt import is_interrupted as _is_interrupted, set_interrupt as _set_interrupt
  408. _was_interrupted = _is_interrupted()
  409. if _was_interrupted:
  410. _set_interrupt(False)
  411. try:
  412. # Budget scales with the concurrency cap: a bounded gather finishes in
  413. # ceil(N/cap) waves, so N > cap multiplies the wall clock the base
  414. # (unbounded) gather never needed. 120s per wave keeps the original
  415. # per-wave ceiling; a slow fleet aborts later, not never. Capped by
  416. # _MCP_DISCOVERY_PASS_MAX_SEC so one stuck connect cannot pin the
  417. # calling thread (and the cross-process discovery lock) for tens of
  418. # minutes; the lock waiter's budget is derived from the same ceiling.
  419. cap = _discovery_connect_concurrency() or len(new_servers) or 1
  420. waves = max(1, -(-len(new_servers) // cap))
  421. timeout = min(120 * waves, _core._MCP_DISCOVERY_PASS_MAX_SEC)
  422. _loop._run_on_mcp_loop(lambda: _discover_all(new_servers), timeout=timeout)
  423. except (TimeoutError, InterruptedError) as _e:
  424. # Stranded _server_connecting entries would block future reconnects.
  425. how = "timed out" if isinstance(_e, TimeoutError) else "interrupted"
  426. with _core._lock:
  427. stale = [n for n in new_servers if _server_key(n) in _core._server_connecting]
  428. if stale:
  429. logger.warning("MCP discovery %s while %d server(s) were still connecting; clearing stale "
  430. "connecting set: %s", how, len(stale), ", ".join(stale))
  431. for _sn in stale:
  432. _core._server_connecting.discard(_server_key(_sn))
  433. _core._server_connect_errors.setdefault(
  434. _server_key(_sn), f"Connection attempt {how} during discovery")
  435. # Its attempt is still running on the MCP loop; without a cooldown the next
  436. # reconcile tick would spawn a second one beside it.
  437. _record_connect_failure(_sn)
  438. raise
  439. finally:
  440. if _was_interrupted:
  441. _set_interrupt(True)
  442. def _connected_summary(names, *, lazy_tools: int = 0,
  443. lazy_servers: int = 0) -> tuple[int, int, list[tuple[str, str]]]:
  444. """(tool count, connected count, ``[(failed name, reason)]``) for candidate names, plus lazy
  445. servers. The reason is the recorded connect error; a candidate this pass never attempted (still
  446. inside its retry cooldown from an earlier failure) has none."""
  447. with _core._lock:
  448. keys = {n: _server_key(n) for n in names}
  449. connected = [n for n in names
  450. if keys[n] in _core._servers and keys[n] not in _core._server_connect_errors]
  451. tool_count = sum(len(getattr(_core._servers[keys[n]], "_registered_tool_names", [])) for n in connected)
  452. failed = [(n, _core._server_connect_errors.get(keys[n]) or "not attempted (in retry cooldown)")
  453. for n in names if n not in connected]
  454. return tool_count + lazy_tools, len(connected) + lazy_servers, failed
  455. def _log_summary(prefix: str, names, **lazy) -> None:
  456. """Log ``<prefix> N tool(s) from M server(s) (K failed: name (reason), ...)`` when anything
  457. happened. The failures are named on the summary line itself (#114746): the count alone left
  458. the failing server identifiable only by elimination, and a candidate skipped for its retry
  459. cooldown never gets a per-server WARNING at all."""
  460. new_tool_count, connected_count, failed = _connected_summary(names, **lazy)
  461. if new_tool_count or failed or lazy.get("lazy_servers"):
  462. summary = f"{prefix} {new_tool_count} tool(s) from {connected_count} server(s)"
  463. if failed:
  464. summary += f" ({len(failed)} failed: " + "; ".join(
  465. f"{name} ({reason})" for name, reason in failed) + ")"
  466. if lazy.get("lazy_servers"):
  467. summary += f" ({lazy['lazy_servers']} lazy, not spawned yet)"
  468. logger.info(summary)
  469. def register_mcp_servers(servers: dict[str, dict]) -> list[str]:
  470. """Connect ``{name: config}`` servers and register their tools; idempotent for connected
  471. names, ``enabled: false`` skipped without disconnecting. Returns every MCP tool name."""
  472. from agent.safe_worker_policy import safe_worker_enabled
  473. if safe_worker_enabled():
  474. # A troubleshooting worker never spawns MCP processes; nor may it borrow tools
  475. # another caller already connected in this process.
  476. return []
  477. if not _core._ensure_mcp_sdk():
  478. logger.debug("MCP SDK not available -- skipping explicit MCP registration")
  479. return []
  480. servers = _config._filter_suspicious_mcp_servers(servers)
  481. try:
  482. return _register_mcp_servers(servers)
  483. finally:
  484. # An owner's scoped reload orphaned adopters of its shared connections: now that this
  485. # pass (its rediscovery) is done, give them their tools back under their own scope.
  486. _lifecycle._reregister_orphaned_adopters()
  487. def _register_mcp_servers(servers: dict[str, dict]) -> list[str]:
  488. scoped_healed = _registration.register_connected_into_current_scope(servers)
  489. if not servers:
  490. logger.debug("No explicit MCP servers provided")
  491. return _registration._existing_tool_names() if _core._mcp_registry_scope() is not None else []
  492. new_servers = _select_new_servers(servers)
  493. if scoped_healed:
  494. logger.info("MCP: registered %d already-connected server(s) into this profile scope", scoped_healed)
  495. if not new_servers:
  496. return _registration._existing_tool_names()
  497. new_servers, lazy_registered, lazy_server_count = _register_lazy_from_cache(new_servers)
  498. if not new_servers:
  499. if lazy_registered:
  500. logger.info("MCP: registered %d lazy tool(s) from schema cache (no processes spawned)",
  501. lazy_registered)
  502. return _registration._existing_tool_names()
  503. _loop._ensure_mcp_loop()
  504. _run_discovery_pass(new_servers)
  505. _log_summary("MCP: registered", new_servers, lazy_tools=lazy_registered, lazy_servers=lazy_server_count)
  506. return _registration._existing_tool_names()
  507. def _acquire_discovery_lock_with_retry():
  508. """Cross-process guard: a lock loser waits for the holder then discovers itself; unavailable
  509. locking or an expired wait runs unguarded (fail-soft). None / _LOCK_UNAVAILABLE = unguarded."""
  510. cookie = _loop._try_acquire_mcp_discovery_lock()
  511. if cookie is not None:
  512. return cookie
  513. logger.debug("Another process holds MCP discovery lock -- retrying with backoff")
  514. for _ in range(_core._MCP_DISCOVERY_LOCK_MAX_RETRIES):
  515. time.sleep(_core._MCP_DISCOVERY_LOCK_RETRY_DELAY_S)
  516. cookie = _loop._try_acquire_mcp_discovery_lock()
  517. if cookie is not None:
  518. break
  519. # Cross-process discovery guard (#62771). A lock loser waits for the holder, then performs its own
  520. # process-local discovery. If locking is unavailable or the bounded wait expires, preserve the previous
  521. # fail-soft behavior by running discovery unguarded.
  522. if cookie is None:
  523. logger.warning("MCP discovery lock still held after %d retries -- running discovery unguarded",
  524. _core._MCP_DISCOVERY_LOCK_MAX_RETRIES)
  525. elif cookie is not _core._LOCK_UNAVAILABLE:
  526. logger.debug("Retry succeeded -- acquired MCP discovery lock")
  527. return cookie
  528. def discover_mcp_tools(allowed_mcp_names: Optional[list[str]] = None) -> list[str]:
  529. """Entry point: load config, connect servers, register tools. [] without the ``mcp``
  530. package; idempotent (only servers missing from a previous call are retried).
  531. ``allowed_mcp_names``: spawn only the MCP servers named in it (built-in toolset names in the
  532. list simply don't match); ``None`` spawns every configured server. Used by
  533. ``hermes -z -t <toolsets>`` to skip cold-starting servers the caller doesn't need (10-60s
  534. each); it only affects which servers start, not which names ``-t`` validation can see."""
  535. from agent.safe_worker_policy import safe_worker_enabled
  536. if safe_worker_enabled():
  537. return []
  538. with _owner_secret_scope():
  539. servers = _config._load_mcp_config()
  540. if not servers:
  541. logger.debug("No MCP servers configured")
  542. return []
  543. if allowed_mcp_names is not None:
  544. allowed_set = {str(n) for n in allowed_mcp_names}
  545. filtered = {name: cfg for name, cfg in servers.items() if name in allowed_set}
  546. if len(filtered) != len(servers):
  547. logger.debug("MCP discovery filter: spawning %d/%d configured server(s) per --toolsets filter "
  548. "(skipped: %s)", len(filtered), len(servers), ",".join(sorted(set(servers) - set(filtered))))
  549. servers = filtered
  550. if not servers:
  551. logger.debug("No MCP servers in --toolsets filter; skipping MCP load entirely")
  552. return []
  553. # SDK import deferred to here so a config without servers — or a -t filter that keeps
  554. # none — never pays it.
  555. if not _core._ensure_mcp_sdk():
  556. logger.debug("MCP SDK not available -- skipping MCP tool discovery")
  557. return []
  558. cookie = _acquire_discovery_lock_with_retry()
  559. try:
  560. with _core._lock:
  561. keys = {name: _resolve_server_key(name) for name in servers}
  562. new_server_names = [name for name, cfg in servers.items()
  563. if keys[name] not in _core._servers and keys[name] not in _core._server_connecting
  564. and mcp_server_enabled(cfg)]
  565. prior_lazy = set(_core._lazy_server_configs)
  566. tool_names = register_mcp_servers(servers)
  567. if new_server_names:
  568. # A lazily registered server never connected, so it must not be counted as failed
  569. # (the old summary read "N failed" for a healthy all-lazy config, #111717). Reporting
  570. # it separately also keeps an already-lazy server from being re-announced on a
  571. # repeat discovery. Lazy state is keyed by resolved server key, not by name.
  572. with _core._lock:
  573. lazy_now = [n for n in new_server_names if keys[n] in _core._lazy_server_configs]
  574. newly_lazy = [n for n in lazy_now if keys[n] not in prior_lazy]
  575. lazy_tools = sum(len(_core._lazy_server_tool_names.get(keys[n], [])) for n in newly_lazy)
  576. _log_summary(" MCP:", [n for n in new_server_names if n not in lazy_now],
  577. lazy_tools=lazy_tools, lazy_servers=len(newly_lazy))
  578. return tool_names
  579. finally:
  580. if cookie not in (None, _core._LOCK_UNAVAILABLE):
  581. cookie.release()
  582. def reconcile_mcp_servers_with_config() -> dict[str, list[str]]:
  583. """Bring the live server set in step with ``mcp_servers`` as it is on disk NOW: tear down
  584. servers that were removed from config or set ``enabled: false`` (a parked server keeps
  585. self-probing forever otherwise — for hours after the user deleted its entry), then connect
  586. anything enabled that is not live via :func:`discover_mcp_tools` — newly configured, or one
  587. whose earlier connect failed and whose cooldown has lapsed. Scoped to the current registry
  588. scope (one multiplexed profile's config prunes only its own connections). A lazily registered
  589. (schema-cache) server loses its cached tools; one still mid-connect cannot be torn down yet and
  590. is reported under ``"pending"`` so the caller retries. Returns
  591. ``{"removed": [...], "added": [...], "pending": [...]}``; a no-op when nothing changed."""
  592. with _owner_secret_scope():
  593. servers = _config._load_mcp_config()
  594. wanted = {name for name, cfg in servers.items() if mcp_server_enabled(cfg)}
  595. scope = _core._mcp_registry_scope()
  596. with _core._lock:
  597. owned = [key for key, owner in _core._server_scope_keys.items() if owner == scope]
  598. live = {_key_name(key) for key in owned if key in _core._servers}
  599. connecting = {_key_name(key) for key in owned if key in _core._server_connecting}
  600. lazy = {key for key in _core._lazy_server_configs
  601. if _key_scope(key) == scope and _key_name(key) not in wanted}
  602. stale = sorted(live - wanted)
  603. if stale:
  604. logger.info("MCP server(s) %s no longer in config (or disabled); disconnecting", ", ".join(stale))
  605. _lifecycle.shutdown_mcp_servers(scope=scope, names=set(stale))
  606. for key in lazy:
  607. _forget_lazy_server(key)
  608. added = _awaiting_connect(wanted, scope)
  609. if added:
  610. discover_mcp_tools()
  611. return {"removed": stale + sorted(_key_name(k) for k in lazy), "added": added,
  612. "pending": sorted(connecting - wanted)}
  613. def _unregistered(wanted: set[str], scope) -> set[str]:
  614. with _core._lock:
  615. # Same resolution ``_select_new_servers`` applies: this scope's own connection OR a shared
  616. # one it adopted from another profile counts as live. Owner==scope alone misses the adopted
  617. # case, so a multiplexed profile would re-enter discovery (cross-process lock) and log
  618. # "added" every tick forever for a server that is already serving it.
  619. known = {name for name in wanted
  620. if (key := _resolve_server_key(name, scope, current=False)) in _core._servers
  621. or key in _core._server_connecting or key in _core._lazy_server_configs}
  622. return wanted - known
  623. def _awaiting_connect(wanted: set[str], scope) -> list[str]:
  624. # A configured server that is not live is retried — the only way one whose FIRST connect
  625. # failed comes back (#112445) — but only once its connect cooldown lapsed: ``discover_mcp_tools``
  626. # would skip it anyway, and entering it takes the cross-process discovery lock (up to 120 s of
  627. # waiting when another process holds it) and logs a failed pass, every call, for nothing.
  628. return sorted(name for name in _unregistered(wanted, scope) if not _connect_cooldown_active(name))
  629. def _enabled_config_names() -> set[str]:
  630. with _owner_secret_scope():
  631. servers = _config._load_mcp_config()
  632. return {name for name, cfg in servers.items() if mcp_server_enabled(cfg)}
  633. def _not_connected(wanted: set[str], scope) -> set[str]:
  634. with _core._lock:
  635. # A parked task stays registered (it self-probes) but serves nothing, as does one whose connect
  636. # recorded an error since its last success, and a lazy registration whose first-use connect failed
  637. # (a timed-out one leaves no task; one that never started has no error).
  638. serving_nothing = {name for name in wanted
  639. if ((key := _resolve_server_key(name, scope, current=False)) in _core._servers
  640. and (key in _core._server_connect_errors
  641. or getattr(_core._servers[key], "_park_reason", None) is not None))
  642. or (key in _core._lazy_server_configs and key in _core._server_connect_errors)}
  643. return _unregistered(wanted, scope) | serving_nothing
  644. def mcp_servers_not_connected() -> list[str]:
  645. """Enabled ``mcp_servers`` entries the current registry scope has no live, connecting or lazy
  646. registration for, in connect cooldown or not, or whose registration (lazy included) is parked or
  647. failed its connect: after a reload, the servers that did not connect."""
  648. return sorted(_not_connected(_enabled_config_names(), _core._mcp_registry_scope()))
  649. def mcp_servers_ready() -> list[str]:
  650. """Enabled ``mcp_servers`` entries the current registry scope serves now: live, or lazily registered
  651. and not failed on its first use."""
  652. wanted, scope = _enabled_config_names(), _core._mcp_registry_scope()
  653. with _core._lock:
  654. registered = {name for name in wanted
  655. if (key := _resolve_server_key(name, scope, current=False)) in _core._servers
  656. or key in _core._lazy_server_configs}
  657. return sorted(registered - _not_connected(wanted, scope))
  658. def mcp_servers_awaiting_connect() -> list[str]:
  659. """Enabled ``mcp_servers`` entries the current registry scope has no live, connecting or lazy
  660. registration for, whose connect cooldown has lapsed: added to config since the last discovery
  661. run, or one whose earlier connect failed. Read-only — what the next :func:`discover_mcp_tools`
  662. would connect."""
  663. return _awaiting_connect(_enabled_config_names(), _core._mcp_registry_scope())
  664. # One settings-driven reconnect at a time (see restart_mcp_servers_with_new_inputs).
  665. _RESTART_LOCK = threading.Lock()
  666. def restart_mcp_servers_with_new_inputs(names: set[str]) -> list[str]:
  667. """Reconnect each live server in *names* whose resolved launch inputs (stdio command/env/cwd,
  668. HTTP url/headers) no longer hash to what it connected with, e.g. after a plugin settings form
  669. saved a new ``${VAR}`` its ``env`` passes through. A server whose inputs did not change keeps
  670. its connection. Returns the names restarted.
  671. Serialized: a second save waits for the first reconnect, then compares against what THAT one
  672. connected with, so a newer key is never lost to an older in-flight start. A connection this
  673. profile only borrows (same inputs as another profile's) is left running for its owner; this
  674. profile just detaches and connects its own."""
  675. with _RESTART_LOCK:
  676. scope = _core._mcp_registry_scope()
  677. owned, borrowed = [], []
  678. with _owner_secret_scope():
  679. servers = _config._load_mcp_config()
  680. for name in sorted(names):
  681. config = servers.get(name)
  682. with _core._lock:
  683. key = _resolve_server_key(name, scope, current=False)
  684. server = _core._servers.get(key)
  685. recorded = getattr(server, "_resolved_identity", None)
  686. if config is None or recorded is None or not mcp_server_enabled(config):
  687. continue
  688. # Render as the connect path does (plugin servers join the config after load's render pass).
  689. if _registration._adopter_identity_digest(name, _config._interpolate_env_vars(config)) != recorded:
  690. (owned if key == _server_key(name, scope, current=False) else borrowed).append((name, key))
  691. if not owned and not borrowed:
  692. return []
  693. changed = sorted(name for name, _key in owned + borrowed)
  694. logger.info("MCP server(s) %s: launch inputs changed; reconnecting", ", ".join(changed))
  695. if owned:
  696. _lifecycle.shutdown_mcp_servers(scope=scope, names={name for name, _key in owned})
  697. for _name, key in borrowed:
  698. _registration._remove_server_scope(key, scope)
  699. for name in changed:
  700. _clear_connect_failure(name)
  701. discover_mcp_tools()
  702. return changed
  703. def _forget_lazy_server(key) -> None:
  704. """Drop a schema-cache (lazy) registration whose config entry is gone: its cached tools would
  705. otherwise stay callable and spawn the server on first use."""
  706. with _core._lock:
  707. _core._lazy_server_configs.pop(key, None)
  708. _core._lazy_server_fingerprints.pop(key, None)
  709. cached_names = _core._lazy_server_tool_names.pop(key, None) or []
  710. for tool_name in cached_names:
  711. _registration._deregister_mcp_tool_all_scopes(key, tool_name)
  712. def is_mcp_tool_parallel_safe(tool_name: str) -> bool:
  713. """True when the tool's server opted into ``supports_parallel_tool_calls`` (provenance
  714. captured at registration, never the ambiguous ``mcp__{server}__{tool}`` shape)."""
  715. if not tool_name.startswith(MCP_TOOL_NAME_PREFIX):
  716. return False
  717. with _core._lock:
  718. server_name = _core._mcp_tool_server_names.get(tool_name)
  719. return bool(server_name and _server_key(server_name) in _core._parallel_safe_servers)
  720. def get_mcp_status(configured: Optional[dict[str, dict]] = None, *, include_runtime: bool = True) -> list[dict]:
  721. """Per-server status dicts for banner/TUI: name, transport, tools, connected, disabled,
  722. status (connected / disabled / connecting / failed / lazy / configured) and error for failed.
  723. Reads cached runtime state only; never connects.
  724. ``lazy`` is a registered-but-not-spawned server (``lazy: true``, tools from the schema
  725. cache): its tools are callable and the process starts on first use. Reporting it as
  726. ``configured`` (never registered) misreads a working setup (#111717)."""
  727. if configured is None:
  728. with _owner_secret_scope():
  729. configured = _config._load_mcp_config()
  730. else:
  731. configured = dict(configured)
  732. if not configured:
  733. return []
  734. current_scope = _core._mcp_registry_scope()
  735. with _core._lock:
  736. def visible(key) -> bool:
  737. # Runtime state belongs to the profile that adopted it; under a multiplexer only that
  738. # profile's view may show it, and ``include_runtime=False`` hides the launch profile's
  739. # servers from a status read scoped to a different profile.
  740. return include_runtime and _core._server_visible_in_scope(key, current_scope)
  741. active_servers = {_key_name(k): s for k, s in _core._servers.items() if visible(k)}
  742. connecting = {_key_name(k) for k in _core._server_connecting if visible(k)}
  743. connect_errors = {_key_name(k): e for k, e in _core._server_connect_errors.items() if visible(k)}
  744. # A lazy registration is not a live connection: ``_server_visible_in_scope`` reads the
  745. # adoption/teardown maps a lazy server never populates, so use the registration-level
  746. # predicate ``_resolve_server_key`` already relies on for this state.
  747. lazy_tools = {_key_name(k): len(v) for k, v in _core._lazy_server_tool_names.items()
  748. if include_runtime and _key_visible_in_scope(k, current_scope)
  749. and k in _core._lazy_server_configs}
  750. result: list[dict] = []
  751. for name, cfg in configured.items():
  752. enabled = mcp_server_enabled(cfg) # evaluated unconditionally: malformed values warn even when connected
  753. server = active_servers.get(name)
  754. live = server is not None and server.session is not None
  755. # A server that connected and then parked (reconnects exhausted) records no connect error,
  756. # but it is down all the same; never-tried servers stay "configured".
  757. parked = server is not None and not live and server._park_reason is not None
  758. # An in-flight or failed first-use connect outranks "lazy": that server is no longer
  759. # merely waiting to be spawned, and the error is the actionable part.
  760. status = ("connected" if live else "disabled" if not enabled else "connecting" if name in connecting
  761. else "failed" if name in connect_errors or parked else "lazy" if name in lazy_tools
  762. else "configured")
  763. entry = {"name": name, "transport": cfg.get("transport", "http") if "url" in cfg else "stdio",
  764. "tools": 0, "connected": False, "disabled": status == "disabled", "status": status}
  765. if live:
  766. entry["connected"] = True
  767. entry["tools"] = (len(server._registered_tool_names) if hasattr(server, "_registered_tool_names")
  768. else len(server._tools))
  769. if server._sampling:
  770. entry["sampling"] = dict(server._sampling.metrics)
  771. elif status == "failed" and name in connect_errors:
  772. entry["error"] = connect_errors[name]
  773. elif status == "lazy":
  774. entry["tools"] = lazy_tools[name]
  775. # Parked on a rejected OAuth credential: the one failure only the user can fix (sign in again).
  776. entry["needs_auth"] = bool(server is not None and not live and server._auth_park is not None)
  777. result.append(entry)
  778. return result
  779. def mcp_server_reconnecting(name: str) -> bool:
  780. """True when this profile's connection to *name* connected once in this process and is now
  781. between sessions (degraded/parked) after a transient failure: the run task is alive and
  782. self-probing, so the outage is environmental and heals on its own. A server that never
  783. connected here (wrong URL, other profile's credentials) is not reconnecting, and neither is one
  784. parked on a PERMANENT error (revoked credentials, endpoint gone): its self-probe fails the same
  785. way every time, so callers must treat it as blocked rather than wait forever. Reads cached
  786. state; never connects."""
  787. with _core._lock:
  788. server = _core._servers.get(_resolve_server_key(name))
  789. if server is None or server.session is not None or not server._ever_connected:
  790. return False
  791. if server._park_reason and "permanent" in server._park_reason:
  792. return False
  793. return server._task is None or not server._task.done()
  794. def probe_mcp_server_tools() -> dict[str, list[tuple]]:
  795. """Connect each enabled server, list ``(tool_name, description)``, disconnect; nothing is
  796. registered and failed servers are omitted."""
  797. if not _core._ensure_mcp_sdk():
  798. return {}
  799. with _owner_secret_scope():
  800. enabled = {k: v for k, v in (_config._load_mcp_config() or {}).items() if mcp_server_enabled(v)}
  801. if not enabled:
  802. return {}
  803. _loop._ensure_mcp_loop()
  804. result: dict[str, list[tuple]] = {}
  805. probed_servers: list[_core.MCPServerTask] = []
  806. async def _probe_all():
  807. coros = [asyncio.wait_for(_connect_server(name, cfg),
  808. timeout=cfg.get("connect_timeout", _core._DEFAULT_CONNECT_TIMEOUT))
  809. for name, cfg in enabled.items()]
  810. outcomes = await asyncio.gather(*coros, return_exceptions=True)
  811. for name, outcome in zip(enabled, outcomes):
  812. if isinstance(outcome, Exception):
  813. logger.debug("Probe: failed to connect to '%s': %s", name, outcome)
  814. continue
  815. probed_servers.append(outcome)
  816. result[name] = [(t.name, getattr(t, "description", "") or "") for t in outcome._tools]
  817. await asyncio.gather(*(s.shutdown() for s in probed_servers), return_exceptions=True)
  818. try:
  819. _loop._run_on_mcp_loop(_probe_all, timeout=120)
  820. except Exception as exc:
  821. logger.debug("MCP probe failed: %s", exc)
  822. finally:
  823. _lifecycle._stop_mcp_loop_if_idle()
  824. return result
  825. def has_registered_mcp_tools() -> bool:
  826. """True if any MCP server has registered TOOLS (not merely connected), so the per-turn
  827. refresh hook stays idle for zero-tool servers."""
  828. with _core._lock:
  829. return bool(_core._mcp_tool_server_names)
  830. def get_registered_mcp_server_names() -> set:
  831. """Server names that registered at least one tool (live, filtered — not config.yaml)."""
  832. with _core._lock:
  833. return set(_core._mcp_tool_server_names.values())