mcp_tool.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620
  1. #!/usr/bin/env python3
  2. """MCP (Model Context Protocol) client: connects to the ``mcp_servers`` configured in
  3. ~/.hermes/config.yaml (stdio, Streamable HTTP or SSE), discovers their tools and registers them
  4. into the hermes tool registry. The ``mcp`` package is optional (no-op without it).
  5. One background event loop (``_mcp_loop``) in a daemon thread runs each server as a long-lived
  6. Task (``MCPServerTask``) so the transport's anyio cancel scopes enter and exit in one Task; every
  7. ``_servers``/loop mutation holds ``_lock``. This module keeps the SDK loader, ``MCPServerTask`` and
  8. all shared state; the ``mcp_tool_*`` siblings read that state back through ``tools.mcp_tool`` at
  9. call time (``_core``) and are imported directly by their callers."""
  10. import asyncio
  11. import contextvars
  12. import importlib
  13. import importlib.util
  14. import inspect
  15. import logging
  16. import os
  17. import threading
  18. import time
  19. from typing import Any, Callable, Dict, List, Optional, Set
  20. logger = logging.getLogger(__name__)
  21. from tools.mcp_tool_common import _DEFAULT_TOOL_TIMEOUT, mcp_field
  22. from tools.mcp_tool_config import _npx_cached_bin, _packaged_npx_entry
  23. from tools.mcp_tool_sampling import ElicitationHandler, SamplingHandler
  24. from tools.mcp_tool_transport import MCPServerTransportMixin
  25. from tools.mcp_tool_server_run import MCPServerRunMixin
  26. from tools.mcp_tool_health import MCPServerHealthMixin
  27. # Wall-clock bound on the fail-open OSV malware preflight before a stdio spawn; just ABOVE
  28. # osv_check._TIMEOUT (10s) so it only bites when a stalled SSL handshake defeats that.
  29. _OSV_MALWARE_CHECK_TIMEOUT_S = 12.0
  30. async def _preflight_stdio_command(server_name: str, command: str, args: list) -> tuple[str, list]:
  31. """OSV malware preflight (off-loop, wall-clock bound, fail-open on timeout), THEN the
  32. cached-npx swap. The preflight must see the REAL command/args: anything that rewrites argv to a
  33. wrapper or resolved binary has to happen after it, or the check silently inspects the wrapper
  34. and becomes a no-op (``_infer_ecosystem`` keys off the command basename being npx/uvx/pipx)."""
  35. from tools.osv_check import check_package_for_malware
  36. try:
  37. malware_error = await asyncio.wait_for(
  38. asyncio.to_thread(check_package_for_malware, command, args), timeout=_OSV_MALWARE_CHECK_TIMEOUT_S)
  39. except TimeoutError:
  40. logger.warning("MCP server '%s': OSV malware preflight timed out after %.0fs "
  41. "(network slow/unreachable) — proceeding without the check.",
  42. server_name, _OSV_MALWARE_CHECK_TIMEOUT_S)
  43. malware_error = None
  44. if malware_error:
  45. raise ValueError(f"MCP server '{server_name}': {malware_error}")
  46. # npx resolves the package and then FORKS, staying resident as the real server's parent for
  47. # nothing (~48 MB per server, measured). Hermes already supervises the child (shared death
  48. # supervisor), so a cached package is spawned directly; a cache miss leaves npx untouched.
  49. if os.path.basename(command).lower().startswith("npx"):
  50. # A sealed Windows payload's own npx cannot start its node through cmd.exe.
  51. packaged = await asyncio.to_thread(_packaged_npx_entry, command, args)
  52. if packaged:
  53. logger.debug("MCP server '%s': running %s on the packaged node", server_name, packaged[1][0])
  54. return packaged
  55. cached = _npx_cached_bin(args)
  56. if cached:
  57. direct_command, direct_args = cached
  58. logger.debug("MCP server '%s': using cached npx binary %s (skipping the "
  59. "resident `npm exec` parent)", server_name, direct_command)
  60. command, args = direct_command, direct_args
  61. return command, args
  62. # ---- Optional MCP SDK: availability probe now, symbol import on first use ----
  63. _MCP_AVAILABLE = _MCP_HTTP_AVAILABLE = _MCP_NEW_HTTP = _MCP_LEGACY_HTTP = False
  64. _MCP_SAMPLING_TYPES = _MCP_NOTIFICATION_TYPES = _MCP_ELICITATION_TYPES = False
  65. _MCP_MESSAGE_HANDLER_SUPPORTED = _MCP_LOGGING_CALLBACK_SUPPORTED = False
  66. sse_client = None
  67. # Fallback for SDKs without LATEST_PROTOCOL_VERSION (Streamable HTTP arrived with 2025-03-26).
  68. LATEST_PROTOCOL_VERSION = "2025-03-26"
  69. # Newest revision ``ClientSession.initialize()`` speaks; from 2026-07-28 the handshake is a
  70. # per-request envelope so this can be OLDER than LATEST_PROTOCOL_VERSION, and the
  71. # MCP-Protocol-Version header must be seeded from THIS one.
  72. LATEST_HANDSHAKE_VERSION = LATEST_PROTOCOL_VERSION
  73. # Importing ``mcp`` costs ~260ms, so it is deferred to first use (_ensure_mcp_sdk); availability
  74. # is decided now via find_spec so every ``if not _MCP_AVAILABLE`` gate / patch / skipif holds.
  75. try:
  76. _MCP_AVAILABLE = importlib.util.find_spec("mcp") is not None
  77. except Exception:
  78. _MCP_AVAILABLE = False
  79. if not _MCP_AVAILABLE:
  80. logger.debug("mcp package not installed -- MCP tool support disabled")
  81. ClientSession: Any = None
  82. _MCP_SDK_IMPORT_ATTEMPTED = False
  83. _MCP_SDK_IMPORT_LOCK = threading.Lock()
  84. # Optional SDK type families (module, names, debug message when absent), bound in this order to
  85. # _MCP_SAMPLING_TYPES / _MCP_ELICITATION_TYPES / _MCP_NOTIFICATION_TYPES; an older SDK only
  86. # loses that feature, not MCP.
  87. _OPTIONAL_TYPE_FAMILIES = (
  88. ("mcp.types", ("CreateMessageResult", "CreateMessageResultWithTools", "ErrorData", "SamplingCapability",
  89. "SamplingToolsCapability", "TextContent", "ToolUseContent"),
  90. "MCP sampling types not available -- sampling disabled"),
  91. ("mcp.types", ("ElicitRequestParams", "ElicitResult"),
  92. "MCP elicitation types not available -- elicitation disabled"),
  93. ("mcp.types", ("ServerNotification", "ToolListChangedNotification", "PromptListChangedNotification",
  94. "ResourceListChangedNotification"),
  95. "MCP notification types not available -- dynamic tool discovery disabled"),
  96. )
  97. # Bound by _ensure_mcp_sdk(); module __getattr__ (PEP 562) imports the SDK on first external
  98. # access so mock.patch("tools.mcp_tool.stdio_client") sees a real original, never clobbered.
  99. _MCP_SDK_LAZY_SYMBOLS = frozenset(
  100. {"StdioServerParameters", "stdio_client", "streamablehttp_client", "streamable_http_client"}
  101. | {n for _mod, names, _msg in _OPTIONAL_TYPE_FAMILIES for n in names})
  102. def __getattr__(name: str):
  103. if name in _MCP_SDK_LAZY_SYMBOLS:
  104. _ensure_mcp_sdk()
  105. try:
  106. return globals()[name]
  107. except KeyError:
  108. pass # SDK missing or symbol absent on this SDK build
  109. raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
  110. def _import_sdk_names(module: str, names: tuple, missing_msg: Optional[str] = None) -> bool:
  111. """Bind ``names`` from SDK ``module`` into this module's globals; False (nothing bound,
  112. optional debug line) when this SDK build lacks the module or any of the names."""
  113. try:
  114. mod = importlib.import_module(module)
  115. values = {n: getattr(mod, n) for n in names}
  116. except (ImportError, AttributeError):
  117. if missing_msg:
  118. logger.debug(missing_msg)
  119. return False
  120. globals().update(values)
  121. return True
  122. def _ensure_mcp_sdk() -> bool:
  123. """Import the optional ``mcp`` SDK on first use; return availability. Idempotent and
  124. thread-safe; honors a test-patched ``_MCP_AVAILABLE=False`` (no import) and pre-installed
  125. mocks (``ClientSession`` already set means no re-import)."""
  126. global _MCP_SDK_IMPORT_ATTEMPTED, _MCP_AVAILABLE, _MCP_HTTP_AVAILABLE, _MCP_NEW_HTTP, _MCP_LEGACY_HTTP
  127. global _MCP_SAMPLING_TYPES, _MCP_NOTIFICATION_TYPES, _MCP_ELICITATION_TYPES, sse_client
  128. global _MCP_MESSAGE_HANDLER_SUPPORTED, _MCP_LOGGING_CALLBACK_SUPPORTED, LATEST_HANDSHAKE_VERSION
  129. global _JSONRPC_METHOD_NOT_FOUND
  130. if not _MCP_AVAILABLE:
  131. return False
  132. if _MCP_SDK_IMPORT_ATTEMPTED or ClientSession is not None:
  133. return _MCP_AVAILABLE
  134. with _MCP_SDK_IMPORT_LOCK:
  135. if _MCP_SDK_IMPORT_ATTEMPTED or ClientSession is not None:
  136. return _MCP_AVAILABLE
  137. if (_import_sdk_names("mcp", ("ClientSession", "StdioServerParameters"))
  138. and _import_sdk_names("mcp.client.stdio", ("stdio_client",))):
  139. _MCP_AVAILABLE = True
  140. # mcp >= 1.24 ships streamable_http_client; 2.0 dropped the deprecated
  141. # streamablehttp_client alias. Either one gives HTTP.
  142. _MCP_NEW_HTTP = _import_sdk_names("mcp.client.streamable_http", ("streamable_http_client",))
  143. _MCP_LEGACY_HTTP = _import_sdk_names("mcp.client.streamable_http", ("streamablehttp_client",))
  144. _MCP_HTTP_AVAILABLE = _MCP_NEW_HTTP or _MCP_LEGACY_HTTP
  145. _import_sdk_names("mcp.types", ("LATEST_PROTOCOL_VERSION",),
  146. "mcp.types.LATEST_PROTOCOL_VERSION not available -- using fallback protocol version")
  147. if not _import_sdk_names("mcp.client.session", ("LATEST_HANDSHAKE_VERSION",)):
  148. # Pre-2.x SDKs: newest revision IS the handshake revision.
  149. LATEST_HANDSHAKE_VERSION = LATEST_PROTOCOL_VERSION
  150. if not _import_sdk_names("mcp.client.sse", ("sse_client",),
  151. "mcp.client.sse.sse_client not available -- SSE transport disabled"):
  152. sse_client = None
  153. _MCP_SAMPLING_TYPES, _MCP_ELICITATION_TYPES, _MCP_NOTIFICATION_TYPES = [
  154. _import_sdk_names(*family) for family in _OPTIONAL_TYPE_FAMILIES]
  155. else:
  156. logger.debug("mcp package not installed -- MCP tool support disabled")
  157. if _MCP_AVAILABLE:
  158. try:
  159. _JSONRPC_METHOD_NOT_FOUND = importlib.import_module("mcp.types").METHOD_NOT_FOUND
  160. except Exception: # pragma: no cover — SDK without the constant
  161. pass
  162. _MCP_MESSAGE_HANDLER_SUPPORTED = _client_session_accepts("message_handler")
  163. if _MCP_AVAILABLE and not _MCP_MESSAGE_HANDLER_SUPPORTED:
  164. logger.debug("MCP SDK does not support message_handler -- dynamic tool discovery disabled")
  165. _MCP_LOGGING_CALLBACK_SUPPORTED = _client_session_accepts("logging_callback")
  166. _MCP_SDK_IMPORT_ATTEMPTED = True
  167. return _MCP_AVAILABLE
  168. _SDK_HTTPX_MOD = None
  169. def sdk_httpx():
  170. """The httpx module the *installed* MCP SDK is built against (mcp 2.0 moved to ``httpx2``).
  171. Every object crossing the SDK boundary (AsyncClient, OAuth Request, exception classes) must
  172. come from the module the SDK itself imports or it fails at the transport layer. Resolved
  173. from the SDK's transport module, else the newest present; ``None`` if neither imports."""
  174. global _SDK_HTTPX_MOD
  175. if _SDK_HTTPX_MOD is not None:
  176. return _SDK_HTTPX_MOD
  177. try:
  178. from mcp.client import streamable_http as _transport
  179. _SDK_HTTPX_MOD = getattr(_transport, "httpx2", None) or getattr(_transport, "httpx", None)
  180. except ImportError:
  181. _SDK_HTTPX_MOD = None
  182. for fallback in ("httpx2", "httpx"):
  183. if _SDK_HTTPX_MOD is not None:
  184. break
  185. try:
  186. _SDK_HTTPX_MOD = importlib.import_module(fallback)
  187. except ImportError:
  188. pass
  189. return _SDK_HTTPX_MOD
  190. def _client_session_accepts(kwarg: str) -> bool:
  191. """Whether this SDK's ``ClientSession.__init__`` takes ``kwarg`` (older SDKs lack
  192. ``message_handler`` and ``logging_callback``)."""
  193. if not _MCP_AVAILABLE:
  194. return False
  195. try:
  196. return kwarg in inspect.signature(ClientSession).parameters
  197. except (TypeError, ValueError):
  198. return False
  199. # MCP logging levels (RFC 5424 syslog severities) -> Python logging levels.
  200. # Port of anomalyco/opencode#34529's serverLog mapping.
  201. _MCP_LOG_LEVEL_MAP = {
  202. "debug": logging.DEBUG, "info": logging.INFO, "notice": logging.INFO,
  203. "warning": logging.WARNING, "error": logging.ERROR, "critical": logging.ERROR,
  204. "alert": logging.ERROR, "emergency": logging.ERROR}
  205. # ---- Reconnect / keepalive tuning ----
  206. _DEFAULT_CONNECT_TIMEOUT = 60 # seconds for initial connection per server
  207. _MAX_RECONNECT_RETRIES = 5
  208. _MAX_INITIAL_CONNECT_RETRIES = 3 # retries for the very first connection attempt
  209. _MAX_BACKOFF_SECONDS = 60
  210. _RECYCLED_RECONNECT_TIMEOUT = 15.0
  211. # Parked servers (tools deregistered) self-probe on this cadence: nothing else can revive them.
  212. _PARKED_RETRY_INTERVAL = 300
  213. # Bounded wait for a respawned stdio child when a call finds it dead (gateway restarts kill
  214. # every MCP child); bounded so a broken server still parks via run()'s rapid-drop budget.
  215. _STDIO_RESPAWN_WAIT_SEC = 15.0
  216. # Remote clients MUST ping faster than the server's idle-session TTL (short-TTL servers need a
  217. # smaller configured ``keepalive_interval``); stdio only opts in explicitly because local pipes
  218. # have no remote session TTL. The floor stops a tiny interval busy-looping.
  219. _DEFAULT_KEEPALIVE_INTERVAL, _MIN_KEEPALIVE_INTERVAL = 180, 5
  220. # One bounded cancellation cycle at final shutdown so resistant tasks cannot hang exit.
  221. _MCP_LOOP_DRAIN_TIMEOUT = 3.0
  222. # JSON-RPC 2.0 "method not found" (server without optional ``ping``); _ensure_mcp_sdk()
  223. # overrides it from mcp.types once loaded.
  224. _JSONRPC_METHOD_NOT_FOUND = -32601
  225. # nextCursor pagination cap so a forever-cursor cannot spin discovery (50 pages = thousands).
  226. _MCP_LIST_MAX_PAGES = 50
  227. async def _paginate_full_list(list_method, items_attr: str, server_name: str,
  228. cache_meta_out: Optional[dict] = None):
  229. """Drain a paginated ``list_*`` call by following ``nextCursor``; ``cache_meta_out`` gets the
  230. first page's SEP-2549 hints. Callers must hold the server's ``_rpc_lock``."""
  231. items: list = []
  232. cursor = None
  233. for _ in range(_MCP_LIST_MAX_PAGES):
  234. if not cursor:
  235. result = await list_method()
  236. else:
  237. # mcp 2.0 takes params=PaginatedRequestParams, 1.x takes cursor=.
  238. # Inspect before awaiting: an internal TypeError is not a signature mismatch.
  239. import inspect
  240. try:
  241. signature = inspect.signature(list_method)
  242. except (TypeError, ValueError):
  243. accepts_params = True # Opaque callables use the current SDK convention.
  244. else:
  245. accepts_params = any(
  246. p.kind == inspect.Parameter.VAR_KEYWORD
  247. or (p.name == "params" and p.kind != inspect.Parameter.POSITIONAL_ONLY)
  248. for p in signature.parameters.values()
  249. )
  250. if accepts_params:
  251. import mcp.types as _types # late: keeps the SDK import lazy
  252. _params_cls = getattr(_types, "PaginatedRequestParams", None)
  253. if _params_cls is not None:
  254. result = await list_method(params=_params_cls(cursor=cursor))
  255. else:
  256. result = await list_method(cursor=cursor)
  257. else:
  258. result = await list_method(cursor=cursor)
  259. if cache_meta_out is not None and not items:
  260. for key, snake, camel in (("ttl_ms", "ttl_ms", "ttlMs"), ("cache_scope", "cache_scope", "cacheScope")):
  261. hint = mcp_field(result, snake, camel)
  262. if hint is not None:
  263. cache_meta_out[key] = hint
  264. items.extend(getattr(result, items_attr, None) or [])
  265. cursor = mcp_field(result, "next_cursor", "nextCursor")
  266. # Cursor is an opaque string; anything else (incl. mocks) = last page.
  267. if not isinstance(cursor, str) or not cursor:
  268. break
  269. else:
  270. logger.warning("MCP server '%s': %s pagination exceeded %d pages; truncating at %d items",
  271. server_name, items_attr, _MCP_LIST_MAX_PAGES, len(items))
  272. return items
  273. # ---- Server task -- each MCP server lives in one long-lived asyncio Task ----
  274. class MCPServerTask(MCPServerRunMixin, MCPServerTransportMixin, MCPServerHealthMixin):
  275. """One MCP server connection in one long-lived asyncio Task (the transport's anyio cancel
  276. scopes must enter/exit in the same Task). Run state machine, transport bring-up and
  277. keepalive/liveness live in the three mixins."""
  278. __slots__ = (
  279. "_auth_park",
  280. "_auth_type",
  281. "_config",
  282. "_elicitation",
  283. "_error",
  284. "_ever_connected",
  285. "_idle_timeout_seconds",
  286. "_inflight_tasks",
  287. "_last_park_line",
  288. "_last_tool_call_at",
  289. "_lifecycle_started_at",
  290. "_list_cache_meta",
  291. "_max_lifetime_seconds",
  292. "_park_reason",
  293. "_pending_call_context",
  294. "_pending_refresh_tasks",
  295. "_permanent_grace_used",
  296. "_ping_unsupported",
  297. "_ready",
  298. "_reconnect_event",
  299. "_reconnect_retries",
  300. "_reconnecting",
  301. "_recycled_reason",
  302. "_refresh_lock",
  303. "_registered_tool_names",
  304. "_resolved_identity",
  305. "_rpc_lock",
  306. "_sampling",
  307. "_session_proven",
  308. "_shutdown_event",
  309. "_sse_fallback",
  310. "_stdio_child_pids",
  311. "_suspect_reason",
  312. "_task",
  313. "_teardown_race",
  314. "_tools",
  315. "_was_parked",
  316. "initialize_result",
  317. "name",
  318. "session",
  319. "tool_timeout",
  320. )
  321. def __init__(self, name: str):
  322. self.name = name
  323. self.session: Optional[Any] = None
  324. self.tool_timeout: float = _DEFAULT_TOOL_TIMEOUT
  325. self._task: Optional[asyncio.Task] = None
  326. self._ready = asyncio.Event()
  327. self._shutdown_event = asyncio.Event()
  328. # Set -> _run_http/_run_stdio exit cleanly and run() re-enters the transport.
  329. self._reconnect_event = asyncio.Event()
  330. self._tools: list = []
  331. self._registered_tool_names: list[str] = []
  332. self._config: dict = {}
  333. self._error: Optional[Exception] = None
  334. self._sampling: Optional[SamplingHandler] = None
  335. self._elicitation: Optional[ElicitationHandler] = None
  336. self._reconnect_retries: int = 0
  337. # Rapid-drop budget: a session is UNPROVEN until it survives a keepalive interval or a
  338. # successful call; only a proven session clears the budget, so a post-handshake flapper
  339. # still parks.
  340. # Rapid-drop budget (#62212): a freshly (re)established session is UNPROVEN until it demonstrates
  341. # real health — it survived at least one full keepalive interval (keepalive success path) or served
  342. # at least one successful tool call. Only a proven session clears the reconnect budget; a transport
  343. # that flaps right after the handshake keeps getting charged and still reaches the park instead of
  344. # hot-cycling respawns forever.
  345. self._session_proven: bool = False
  346. # Never cleared (unlike _ready): separates first-connect from reconnect failures.
  347. self._ever_connected: bool = False
  348. # Latched when the Streamable HTTP -> SSE fallback connects: reconnects reuse SSE directly.
  349. self._sse_fallback: bool = False
  350. # Status/URL/body of the last HTTP rejection the Streamable HTTP client saw; names the real
  351. # cause when the SDK reports only ``Server returned an error response``.
  352. self._http_rejection: dict = {}
  353. # True from park until proven healthy again; logs the revival once.
  354. self._was_parked: bool = False
  355. # Why the server is parked (the revival_reason handed to _park), None once healthy again.
  356. # Lets cron preflight tell a network-blip park (recovering) from a permanent-error park
  357. # (revoked credentials, dead endpoint) that must not run the job tool-less forever.
  358. self._park_reason: Optional[str] = None
  359. self._last_park_line: Optional[str] = None # last park WARNING text; identical re-parks log at DEBUG
  360. # (token storage, credential stamp) while parked on an OAuth failure; see _arm_auth_park.
  361. self._auth_park: Optional[tuple] = None
  362. # Digest of the resolved inputs the transport last connected with; a cross-profile adopter
  363. # must resolve the same digest in its own scope. None until the transport publishes it.
  364. self._resolved_identity: Optional[str] = None
  365. # In-flight RPC tasks so a deliberate teardown fails them fast; _reconnecting is True
  366. # during that teardown so _track_inflight_rpc turns the cancel into a retryable error.
  367. # In-flight RPC bookkeeping (#48069 salvage): user-visible requests registered while running so a
  368. # reconnect/shutdown teardown can fail them fast instead of orphaning them on a dying transport.
  369. self._inflight_tasks: set = set()
  370. self._reconnecting: bool = False
  371. # Latched by races (teardown-vs-keepalive, auth-lock corruption); ensure_healthy()
  372. # verifies before the next call.
  373. # See #77765, #81051, #84132.
  374. self._suspect_reason: Optional[str] = None
  375. # Teardown that failed in-flight calls => next reconnect is RACE RECOVERY, not a
  376. # budget charge.
  377. self._teardown_race: bool = False
  378. # One-time grace: auth/permanent failure on a PROVEN session gets one suspect+reconnect
  379. # cycle before parking.
  380. self._permanent_grace_used: bool = False
  381. # Children of the current stdio transport: in-flight calls fail FAST when one dies.
  382. # PIDs of the stdio subprocess spawned for the current transport (captured in _run_stdio). Used to
  383. # fail in-flight calls FAST when the child dies instead of waiting out the full tool timeout
  384. # (#81995).
  385. self._stdio_child_pids: set[int] = set()
  386. self._auth_type: str = ""
  387. self._refresh_lock = asyncio.Lock()
  388. # A stdio session is one JSON-RPC stream (a concurrent list_tools can wedge a tool
  389. # call): serialize client-initiated RPCs per server (HTTP too, for ordering).
  390. self._rpc_lock = asyncio.Lock()
  391. self._pending_refresh_tasks: set[asyncio.Task] = set()
  392. # contextvars snapshot inside session.call_tool(): the SDK runs elicitation/create on a
  393. # task that does not inherit HERMES_SESSION_PLATFORM, so the callback replays this.
  394. self._pending_call_context: Optional[contextvars.Context] = None
  395. self._lifecycle_started_at = self._last_tool_call_at = time.monotonic()
  396. self._idle_timeout_seconds = self._max_lifetime_seconds = self._recycled_reason = None
  397. # Handshake InitializeResult: the server's REAL advertised capabilities.
  398. # Captures the ``InitializeResult`` returned by ``await session.initialize()`` so downstream code
  399. # can inspect the server's real advertised capabilities (``.capabilities.resources``,
  400. # ``.capabilities.prompts``) instead of assuming every ``ClientSession`` method attribute
  401. # corresponds to a supported server method. See #18051.
  402. self.initialize_result: Optional[Any] = None
  403. # SEP-2549 cache hints from the last tools/list (ttl_ms, cache_scope).
  404. self._list_cache_meta: dict = {}
  405. # Latched when ``ping`` returns -32601; keepalives then use list_tools. Reset per connect.
  406. self._ping_unsupported: bool = False
  407. # Content types a real Streamable-HTTP endpoint may return on the initial POST/GET;
  408. # anything else on a 2xx means the URL is not an MCP endpoint.
  409. _MCP_CONTENT_TYPES = ("application/json", "text/event-stream")
  410. # ---- Module-level state (every mutation under ``_lock``) ----
  411. #
  412. # Every ledger below is keyed by the CONNECTION KEY from ``tools.mcp_tool_scope``: the bare
  413. # server name outside a multiplexer, ``(owner_scope, name)`` under one. Two profiles naming the
  414. # same server with their own credentials are two connections; a name-keyed ledger let the first
  415. # profile's connection shadow the second's (never connected, silently tool-less — #106005).
  416. _servers: dict[Any, MCPServerTask] = {}
  417. # Profile registry scope per live connection (None outside multiplex) so a multiplexed
  418. # /reload-mcp tears down only its own profile's servers.
  419. _server_scope_keys: dict[Any, Optional[str]] = {}
  420. # Registry scopes that have adopted a live server connection. The owning scope above remains
  421. # authoritative for connection teardown; this set preserves visibility for shared connections.
  422. _server_tool_scopes: dict[Any, set] = {}
  423. _server_connecting: set = set()
  424. _server_connect_errors: dict[Any, str] = {}
  425. # adopter scope -> server names whose shared connection an owner's scoped shutdown tore down;
  426. # drained by the next discovery pass so the adopter is re-registered (see mcp_tool_lifecycle).
  427. _orphaned_adopters: dict[str, set] = {}
  428. # Lazy startup: servers registered from the schema cache without connecting; popped on
  429. # first real connection.
  430. # Keyed by connection key; entries are popped once a real connection is established on first use. See #56832.
  431. _lazy_server_configs: dict[Any, dict] = {}
  432. _lazy_server_fingerprints: dict[Any, str] = {}
  433. _lazy_server_tool_names: dict[Any, list[str]] = {}
  434. # Task-local claim around ``_connect_server``: discovery retains a recoverable parked task
  435. # while standalone probes never publish failed servers into module-global ownership.
  436. _connect_server_claim: contextvars.ContextVar[Optional[Callable[[MCPServerTask], None]]] = (
  437. contextvars.ContextVar("mcp_connect_server_claim", default=None))
  438. # Per-server connect cooldown: a server that fails to spawn never reaches ``_servers``, so
  439. # without it every ``discover_mcp_tools()`` (one per worker session) would respawn it — a
  440. # restart storm whose unreaped children destabilise healthy servers. Exponential-backoff
  441. # deadline honoured by ``register_mcp_servers``; cleared on success.
  442. # Connection-retry cooldown (per-server isolation against restart storms). A single stdio MCP server that
  443. # fails to spawn (bad PATH, ``exec: not found``, crash-on-start) is never recorded in ``_servers`` --
  444. # ``start()`` raises and ``_discover_and_register_server`` aborts before the ``_servers[name] = server``
  445. # line. Without a cooldown, EVERY subsequent ``discover_mcp_tools()`` (one per agent worker session, i.e.
  446. # every few seconds) sees the server as "not connected" and re-spawns it from scratch. That is the restart
  447. # storm in #50394: the failing server is re-attempted on the shared MCP event loop on every worker session,
  448. # the subprocesses pile up unreaped, and the churn destabilises the healthy co-located servers (their tools
  449. # intermittently surface as "Unknown tool"). Fix: after a failed connection attempt, stamp a monotonic
  450. # ``retry_after`` deadline with exponential backoff. ``register_mcp_servers`` skips a server whose cooldown
  451. # has not elapsed, so a chronically failing server is retried on a backoff schedule instead of on every
  452. # worker session -- isolating it from the rest of the bridge. A successful connection clears the state.
  453. _server_connect_retry_after: dict[Any, float] = {} # connection key -> monotonic deadline
  454. _server_connect_failures: dict[Any, int] = {} # connection key -> consecutive failures
  455. _CONNECT_RETRY_BASE_BACKOFF_SEC, _CONNECT_RETRY_MAX_BACKOFF_SEC = 30.0, 600.0
  456. # Per-server circuit breaker: closed -> open (calls short-circuit until the cooldown) ->
  457. # half-open (next call probes). Mutate only via _bump_server_error / _reset_server_error.
  458. # After _CIRCUIT_BREAKER_THRESHOLD consecutive failures, the handler returns a "server unreachable" message
  459. # that tells the model to stop retrying, preventing the 90-iteration burn loop described in #10447. State
  460. # machine: closed — error count below threshold; all calls go through. open — threshold reached;
  461. # calls short-circuit until the cooldown elapses. half-open — cooldown elapsed; the next call is a probe
  462. # that actually hits the session. Probe success → closed. Probe failure → reopens (cooldown re-armed).
  463. # ``_server_breaker_opened_at`` records the monotonic timestamp when the breaker most recently transitioned
  464. # into the open state. Use the ``_bump_server_error`` / ``_reset_server_error`` helpers to mutate this state
  465. # — they keep the count and timestamp in sync.
  466. _server_error_counts: dict[Any, int] = {}
  467. _server_breaker_opened_at: dict[Any, float] = {}
  468. # True while every strike in the current streak was the tool's own error payload (server reachable,
  469. # call rejected); picks the open-breaker wording, since "unreachable" was false for that case (#11113).
  470. _server_errors_all_application: dict[Any, bool] = {}
  471. _CIRCUIT_BREAKER_THRESHOLD, _CIRCUIT_BREAKER_COOLDOWN_SEC = 3, 60.0
  472. # Trust-tier gating (``trust: full | untrusted``): on an untrusted server every write-capable
  473. # call (discovery-time ``readOnlyHint`` not exactly True; malformed fails closed) needs approval
  474. # before the RPC fires. A lying readOnlyHint can only skip approval for calls the operator was
  475. # already warned about, never widen access. Missing trust = full; unrecognized = untrusted (a
  476. # typo must never disable the gate). Classified at CALL time from DISCOVERY data: no schema
  477. # mutation, prompt cache intact. ``_server_trust_levels`` is keyed by the CONSUMING profile's own
  478. # key (its policy for the name, even when it adopted another profile's connection);
  479. # ``_tool_read_only_hints`` by the connection key (the server's own tool annotations).
  480. _server_trust_levels: dict[Any, str] = {}
  481. _tool_read_only_hints: dict[Any, dict[str, bool]] = {}
  482. # Declared per-tool costs + per-chat spend limit (``tools.mcp_tool_costs.CostPolicy``), keyed like
  483. # ``_server_trust_levels``: the consuming profile's policy. Absent = no costs, behaviour unchanged.
  484. _server_cost_policies: dict[Any, Any] = {}
  485. _TRUST_FULL, _TRUST_UNTRUSTED = "full", "untrusted"
  486. def _bump_server_error(server_name: str, *, application: bool = False) -> None:
  487. """Count a failure; at the threshold (re)stamp the breaker-open time. Keyed by the calling
  488. scope's connection so one profile's failing server never opens another profile's breaker.
  489. *application*: the call completed and the payload was an error (transport is fine)."""
  490. from tools.mcp_tool_scope import _resolve_server_key
  491. key = _resolve_server_key(server_name)
  492. n = _server_error_counts.get(key, 0) + 1
  493. _server_error_counts[key] = n
  494. _server_errors_all_application[key] = application and (n == 1 or _server_errors_all_application.get(key, False))
  495. if n >= _CIRCUIT_BREAKER_THRESHOLD:
  496. _server_breaker_opened_at[key] = time.monotonic()
  497. def _reset_server_error(server_name: str) -> None:
  498. """Close the breaker on any unambiguous success signal."""
  499. from tools.mcp_tool_scope import _resolve_server_key
  500. key = _resolve_server_key(server_name)
  501. _server_error_counts[key] = 0
  502. _server_breaker_opened_at.pop(key, None)
  503. _server_errors_all_application.pop(key, None)
  504. # Servers opted into parallel tool calls, keyed by the consuming profile's own key (``foo-bar``/
  505. # ``foo_bar`` sanitize alike but must not share policy; neither do two profiles' same-named servers).
  506. _parallel_safe_servers: set = set()
  507. # registry tool name -> raw server name (the generated name is lossy; never re-parse it).
  508. _mcp_tool_server_names: dict[str, str] = {}
  509. # Dedicated event loop in a background daemon thread; _lock guards the loop handles, _servers,
  510. # the status maps and the PID ledgers.
  511. _mcp_loop: Optional[asyncio.AbstractEventLoop] = None
  512. _mcp_thread: Optional[threading.Thread] = None
  513. _lock = threading.Lock()
  514. def _mcp_registry_scope() -> Optional[str]:
  515. """Registry scope for MCP registrations: the session's own tool scope when one is bound, else a
  516. profile overlay when this process serves profiles, else None. Under ``gateway.multiplex_profiles``
  517. every turn runs scoped; a process that serves a routed profile through the HERMES_HOME override
  518. (dashboard/desktop backend, per-profile cron ticker) is a multiplexer too, even with the flag
  519. off — keying its connections by the bare name would hand one profile's credentialed connection
  520. to every other served profile (#111151). Single-profile processes (no override, or an override
  521. naming their own home) keep bare names."""
  522. from tools.registry import current_session_tool_scope
  523. session_scope = current_session_tool_scope()
  524. if session_scope is not None:
  525. return session_scope
  526. from agent.secret_scope import serves_routed_profile
  527. if not serves_routed_profile():
  528. return None
  529. from tools.registry import registry
  530. return registry.current_scope_key()
  531. def _server_registry_scope(key) -> Optional[str]:
  532. """Scope owning the connection under *key*'s tools: the one captured at adoption (teardown
  533. runs on the MCP loop without the discovering profile's context), else the current one."""
  534. if key in _server_scope_keys:
  535. return _server_scope_keys[key]
  536. from tools.mcp_tool_scope import _key_scope
  537. return _key_scope(key) or _mcp_registry_scope()
  538. def _server_visible_in_scope(key, scope: Optional[str]) -> bool:
  539. """Whether the live connection under *key* is visible from ``scope`` without changing its
  540. teardown owner."""
  541. if scope is None:
  542. return True
  543. return (_server_scope_keys.get(key) == scope
  544. or scope in _server_tool_scopes.get(key, ()))
  545. # Cross-process discovery guard: advisory file lock so gateway + CLI + TUI don't all discover.
  546. # See issue #62771.
  547. _LOCK_UNAVAILABLE: Any = object() # sentinel: locking broken/unavailable
  548. _MCP_DISCOVERY_LOCK_PATH: Optional[str] = None # resolved lazily
  549. # A discovery pass (bounded gathers, one 120 s budget per wave) may legitimately
  550. # run past the 120 s a scatter completes in. The waiter must outlast the worst
  551. # legitimate holder (pass ceiling + slack), or it fails over at 120 s, discovers
  552. # unguarded beside a still-connecting holder, and spawns duplicate stdio trees.
  553. # See #117373: the concurrency cap made the old 120 s waiter budget stale.
  554. _MCP_DISCOVERY_PASS_MAX_SEC = 300 # overall pass ceiling
  555. _MCP_DISCOVERY_LOCK_RETRY_DELAY_S = 0.5
  556. # Waiter budget (max_retries * delay) must outlast the pass ceiling: 320 s > 300 s.
  557. _MCP_DISCOVERY_LOCK_MAX_RETRIES = int(
  558. _MCP_DISCOVERY_PASS_MAX_SEC / _MCP_DISCOVERY_LOCK_RETRY_DELAY_S) + 20