mcp_tool_transport.py 46 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710
  1. """Transport bring-up for MCPServerTask: stdio spawn (OSV preflight, cached-npx swap, child PID
  2. ledger + death-supervisor registration), Streamable HTTP / SSE connect (preflight, identity header, client certs, OAuth),
  3. protocol negotiation and initial tool discovery. Split from tools/mcp_tool.py."""
  4. import logging
  5. import asyncio
  6. import os
  7. import urllib.parse
  8. import urllib.request
  9. from contextlib import asynccontextmanager
  10. from typing import Dict, Optional, Set
  11. from utils import normalize_proxy_url
  12. from agent.proxy_bypass import is_loopback_host, should_bypass_proxy
  13. from agent import runtime_cwd as _runtime_cwd
  14. from tools.mcp_tool_errors import _MCP_HTTP_MAX_BODY_BYTES, NonMcpEndpointError, _apply_identity_header, _describe_http_failure, _handshake_answered_with_unsupported_version, _handshake_rejected_as_modern, _is_streamable_http_rejection, _make_http_rejection_recorder, _make_mcp_body_cap_transport, _make_redirect_header_stripper, _resolve_client_cert, _unwrap_exception_group
  15. from tools.mcp_tool_lifecycle import _filter_mcp_children, _leader_start_time, _orphan_stdio_pid_servers, _orphan_stdio_pids, _stdio_pgids, _stdio_pids, _stdio_starttimes
  16. from hermes_cli.process_identity import own_process_groups, register_child, release_process_groups
  17. from tools.mcp_tool_common import _core
  18. from tools.mcp_tool_node_abi import node_abi_error
  19. from tools import mcp_tool_config as _config
  20. from tools import mcp_tool_events as _events
  21. from tools import mcp_tool_lifecycle as _lifecycle
  22. from tools import mcp_tool_registration as _registration
  23. logger = logging.getLogger("tools.mcp_tool")
  24. _PROBE_INITIALIZE_BODY = ( # JSON-RPC ``initialize`` body for the content-type preflight POST
  25. '{"jsonrpc":"2.0","id":"_probe","method":"initialize","params":{"protocolVersion":"2025-03-26",'
  26. '"capabilities":{},"clientInfo":{"name":"hermes-probe","version":"0.1"}}}')
  27. def _content_type_base(resp) -> str:
  28. """``content-type`` header of *resp* without parameters, lowercased."""
  29. return resp.headers.get("content-type", "").split(";")[0].strip().lower()
  30. def _is_2xx(resp) -> bool:
  31. return 200 <= resp.status_code < 300
  32. def _present(**kwargs) -> dict:
  33. """*kwargs* minus the ``None`` values (optional httpx client arguments)."""
  34. return {k: v for k, v in kwargs.items() if v is not None}
  35. def _mcp_proxy_mounts(httpx_mod, url: str, ssl_verify, client_cert, server_name: str = "") -> Optional[dict]:
  36. """Proxy transports for the caller-owned MCP HTTP client, or ``None`` for a direct connect.
  37. httpx auto-detects proxies only when ``transport is None``
  38. (``allow_env_proxies = trust_env and transport is None``). The wire-body cap is exactly that
  39. custom transport, so HTTP_PROXY / HTTPS_PROXY and the OS (Windows-registry / macOS) proxy were
  40. silently ignored for every HTTP/SSE MCP server: on a network that reaches the MCP host only
  41. through a proxy, the connect failed with ``All connection attempts failed`` and the server was
  42. parked. Rebuild httpx's own behaviour as explicit ``mounts`` — same source order (environment
  43. first, then the OS proxy), ``NO_PROXY`` / platform bypass list respected, ``socks://``
  44. normalized, and TLS settings identical to the transport they accompany.
  45. NO_PROXY goes through ``agent.proxy_bypass.should_bypass_proxy`` — the one matcher the LLM
  46. transport and the gateway adapters use (CIDR ranges and ``*.host`` forms the stdlib check
  47. does not understand) — plus ``urllib.request.proxy_bypass`` for the OS bypass list
  48. (Windows ``ProxyOverride`` / macOS exceptions). Loopback is never dialed through a proxy
  49. (``agent.proxy_bypass.is_loopback_host``), NO_PROXY or not.
  50. A mount wins over ``transport=`` for the URLs it matches, so each proxy transport is wrapped in
  51. the same wire-body cap as the direct one. A proxy the installed httpx cannot build (e.g.
  52. ``socks://`` without socksio) raises here and surfaces as this server's connect error.
  53. """
  54. host = urllib.parse.urlsplit(url).hostname or ""
  55. if not host or is_loopback_host(host) or should_bypass_proxy(url) or urllib.request.proxy_bypass(host):
  56. return None
  57. proxies = urllib.request.getproxies()
  58. mounts: dict = {}
  59. for scheme in ("http", "https"):
  60. proxy_url = normalize_proxy_url(proxies.get(scheme) or proxies.get("all"))
  61. if not proxy_url:
  62. continue
  63. # verify/cert apply to the CONNECT+TLS leg, so the proxy transport needs its own copy.
  64. mounts[f"{scheme}://"] = _make_mcp_body_cap_transport(httpx_mod, httpx_mod.AsyncHTTPTransport(
  65. proxy=proxy_url, verify=ssl_verify, **_present(cert=client_cert)))
  66. return mounts or None
  67. def _pgroup_alive(pgid: Optional[int]) -> bool:
  68. """Signal 0 to the group succeeds iff any member is alive (POSIX only)."""
  69. try:
  70. os.killpg(pgid, 0) # windows-footgun: ok — guarded by AttributeError below
  71. return True
  72. except (AttributeError, TypeError, OSError): # non-POSIX / pgid None / gone
  73. return False
  74. class LiveEndpointUnavailable(ConnectionError):
  75. """A declared runtime file did not provide a usable live endpoint."""
  76. def _live_endpoint(server_name: str) -> Optional[tuple[str, dict]]:
  77. from agent.redact import register_vault_redaction_value
  78. from hermes_platform import declaration
  79. from hermes_platform.host import facts
  80. from hermes_platform.resolver.app import AppResolver
  81. from tools.mcp_liveness import liveness_for
  82. live = liveness_for(server_name)
  83. if live.kind != "server_json":
  84. return None
  85. decl = declaration.lookup(server_name)
  86. definition = decl.app_for(facts.os_family()) if decl is not None else None
  87. endpoint = AppResolver(live.app_definition(definition)).endpoint() if definition is not None else None
  88. if endpoint is None:
  89. raise LiveEndpointUnavailable(f"MCP server '{server_name}' has no usable live endpoint")
  90. if endpoint.token:
  91. register_vault_redaction_value(endpoint.token)
  92. headers = {"Authorization": f"Bearer {endpoint.token}"} if endpoint.token else {}
  93. return endpoint.url, headers
  94. def _http_endpoint(server_name: str, config: dict) -> tuple[str, dict]:
  95. """URL + configured headers, overlaid by a ``server_json`` live endpoint's URL and bearer."""
  96. url, headers = config["url"], dict(config.get("headers") or {})
  97. live = _live_endpoint(server_name)
  98. if live is not None:
  99. url, live_headers = live
  100. headers.update(live_headers)
  101. return url, headers
  102. def _stdio_launch(config: dict) -> tuple:
  103. """(command, env, cwd) a stdio child is spawned with, resolved in the current profile's scope."""
  104. command, env = _config._resolve_stdio_command(config["command"], _config._build_safe_env(config.get("env")))
  105. # A stdio child inherits this process's cwd when none is configured. Hosted sessions (ACP,
  106. # gateway) pin a logical cwd via agent.runtime_cwd; without it the child resolves relative
  107. # paths against the daemon's launch dir, not the session workspace. Explicit config always
  108. # wins; an existing session/TERMINAL_CWD anchor becomes the default; else native (None).
  109. # The resolved default is fixed for the connection's lifetime.
  110. cwd = config.get("cwd")
  111. if cwd is None:
  112. cwd = _runtime_cwd.resolve_context_cwd() or None
  113. return command, env, cwd
  114. def _connect_inputs(server_name: str, config: dict) -> tuple[list, set]:
  115. """What a connection is opened with, resolved in the current profile's scope: stdio
  116. ``[command, env, cwd]``; HTTP ``[url, headers]`` after the live endpoint and the identity
  117. header. The owner hashes this very list and an adopter recomputes it, so both digests are
  118. built from one code path. Also returns the configured header names (HTTP only), captured
  119. before the identity header is merged in, for the strict-redirect boundary."""
  120. if "url" in config:
  121. url, headers = _http_endpoint(server_name, config)
  122. configured_header_names = {key.lower() for key in headers}
  123. return [url, _apply_identity_header(server_name, config, headers)], configured_header_names
  124. return list(_stdio_launch(config)), set()
  125. class MCPServerTransportMixin:
  126. """Methods of :class:`tools.mcp_tool.MCPServerTask` (mixed in; relies on its attributes)."""
  127. __slots__ = ()
  128. def _advertises_tools(self) -> bool:
  129. """False only when captured capabilities omit ``tools`` (prompt-/resource-only servers,
  130. where ``tools/list`` raises -32601); True without capability info (legacy fallback).
  131. Per the MCP spec, ``InitializeResult.capabilities.tools`` is non-None iff the server implements the
  132. ``tools/*`` request family. Prompt-only or resource-only servers omit it, and calling ``tools/list``
  133. against them raises ``MCPError(-32601 Method not found)`` — which previously killed the connection
  134. during discovery and made every keepalive fail. (Ported from anomalyco/opencode#31271.)
  135. """
  136. caps = getattr(self.initialize_result, "capabilities", None)
  137. return caps is None or getattr(caps, "tools", None) is not None
  138. def _session_kwargs(self) -> dict:
  139. """ClientSession kwargs: client identity, sampling, elicitation, notification + logging callbacks."""
  140. kwargs = {}
  141. for handler in (self._sampling, self._elicitation):
  142. if handler:
  143. kwargs.update(handler.session_kwargs())
  144. if _core._MCP_NOTIFICATION_TYPES and _core._MCP_MESSAGE_HANDLER_SUPPORTED:
  145. kwargs["message_handler"] = self._make_message_handler()
  146. if _core._MCP_LOGGING_CALLBACK_SUPPORTED:
  147. kwargs["logging_callback"] = self._make_logging_callback()
  148. if _core._client_session_accepts("client_info"):
  149. # Servers key logs/behaviour on clientInfo; the SDK default is "mcp"/"0.1.0".
  150. from mcp.types import Implementation
  151. from hermes_cli.version_info import get_version_info
  152. kwargs["client_info"] = Implementation(name="hermes-agent", version=get_version_info().base_version)
  153. return kwargs
  154. async def _negotiate_session(self, session, connect_timeout: float):
  155. """Negotiate the protocol era (``initialize`` vs ``server/discover``; both expose
  156. ``.capabilities``). ``auto`` tries the legacy handshake FIRST, falling back to discover only
  157. on a modern-only signal (-32022 / initialize -32601) — the reverse of the SDK's discover-first
  158. mode, so handshake-era servers pay zero extra round-trips. ``stateless`` probes discover first
  159. (one legacy retry on any error); ``legacy`` is handshake only. A TIMEOUT never falls back."""
  160. def call(method: str):
  161. return asyncio.wait_for(getattr(session, method)(), timeout=connect_timeout)
  162. async def attempt(primary, fallback, should_fallback, log_fmt, *log_extra):
  163. try:
  164. return await call(primary)
  165. except Exception as exc:
  166. if isinstance(exc, asyncio.TimeoutError) or not should_fallback(exc):
  167. raise
  168. logger.info(log_fmt, self.name, exc, *log_extra)
  169. try:
  170. return await call(fallback)
  171. except Exception as fallback_exc:
  172. # #113359: the server ANSWERED ``initialize`` (200, valid result) but named a version the
  173. # SDK's handshake refuses (e.g. 2026-07-28 echoed to a 2025-11-25 offer), and it has no
  174. # ``server/discover`` either. The wire handshake succeeded, so complete it ourselves.
  175. if (isinstance(fallback_exc, asyncio.TimeoutError) or primary != "initialize"
  176. or not _handshake_answered_with_unsupported_version(exc)):
  177. raise
  178. logger.info("MCP server '%s': server/discover also failed (%s) — completing the handshake "
  179. "at %s, the version this client offered", self.name, fallback_exc,
  180. _core.LATEST_HANDSHAKE_VERSION)
  181. return await asyncio.wait_for(self._complete_handshake_at_offered_version(session),
  182. timeout=connect_timeout)
  183. mode = str((self._config or {}).get("protocol", "auto")).lower().strip()
  184. if mode in ("stateless", "modern", "2026-07-28"):
  185. return await attempt("discover", "initialize", lambda exc: True,
  186. "MCP server '%s': server/discover rejected (%s) despite "
  187. "protocol=%s — falling back to the legacy handshake", mode)
  188. if mode in ("legacy", "handshake"):
  189. return await call("initialize")
  190. if mode != "auto":
  191. logger.warning("MCP server '%s': unknown protocol=%r — treating as 'auto' "
  192. "(valid: auto, stateless, legacy)", self.name, mode)
  193. # mcp 1.x has no server/discover client — nothing to fall back to.
  194. return await attempt(
  195. "initialize", "discover", lambda exc: _handshake_rejected_as_modern(exc) and hasattr(session, "discover"),
  196. "MCP server '%s': legacy handshake rejected (%s) — retrying via server/discover (2026-07-28 stateless server)")
  197. async def _complete_handshake_at_offered_version(self, session):
  198. """Re-run the legacy ``initialize`` exchange the SDK already proved works against this server and
  199. adopt its result pinned to the version WE offered (#113359). ``ClientSession.initialize()`` raises
  200. on a ``protocolVersion`` outside its handshake set even though the server answered 200, and a
  201. stateless server that echoes 2026-07-28 to every offer has no ``server/discover`` — so this is the
  202. only way to reach ``notifications/initialized`` and ``tools/list``. Pinning to the offered version
  203. keeps later requests legacy-shaped (envelope and MCP-Protocol-Version header), the form the
  204. handshake itself just proved the server accepts. The returned result keeps the server's own
  205. version for logging/diagnostics."""
  206. from mcp import types # late: keeps the SDK import lazy
  207. offered = _core.LATEST_HANDSHAKE_VERSION
  208. build_caps = getattr(session, "_build_capabilities", None)
  209. capabilities = build_caps(offered) if callable(build_caps) else types.ClientCapabilities()
  210. client_info = getattr(session, "_client_info", None) or types.Implementation(name="hermes-agent", version="0")
  211. result = await session.send_request(
  212. types.InitializeRequest(params=types.InitializeRequestParams(
  213. protocolVersion=offered, capabilities=capabilities, clientInfo=client_info)),
  214. types.InitializeResult)
  215. session.adopt(result.model_copy(update={"protocol_version": offered}))
  216. await session.send_notification(types.InitializedNotification())
  217. return result
  218. async def _serve_session(self, session, connect_timeout: float,
  219. label: str = "", mark_lifecycle: bool = False) -> str:
  220. """Handshake, discover, publish readiness, then serve until a lifecycle event. Clears stale
  221. breaker state but leaves the session UNPROVEN: flapping transports handshake fine and drop
  222. moments later, so only keepalive/tool-call success clears the reconnect budget."""
  223. self.initialize_result = await self._negotiate_session(session, connect_timeout)
  224. self.session = session
  225. if mark_lifecycle:
  226. self._mark_lifecycle_started()
  227. await self._discover_tools()
  228. self._ready.set()
  229. self._ever_connected = True
  230. _core._reset_server_error(self.name)
  231. # Session is live again: clear any breaker state from a prior outage so the first call after
  232. # recovery isn't gated on a stale consecutive-failure count (#16788).
  233. # A completed handshake alone is NOT proof of health: a flapping transport can handshake fine and
  234. # drop moments later, forever (#62212). The session must prove itself (keepalive success, a
  235. # successful tool call, or — stdio without a keepalive — surviving a full default interval
  236. # idle with the child alive) before the reconnect budget is cleared — see _mark_session_proven.
  237. # Session is live again: clear any breaker state from a prior outage so the first call after
  238. # recovery isn't gated on a stale consecutive-failure count (#16788).
  239. # Unproven until keepalive/tool-call success (#62212).
  240. # Session is live again: clear any breaker state from a prior outage so the first call after
  241. # recovery isn't gated on a stale failure count (#16788).
  242. # Unproven until keepalive/tool-call success (#62212).
  243. # Session is live again: clear any breaker state from a prior outage so the first call after
  244. # recovery isn't gated on a stale consecutive-failure count (#16788).
  245. # Unproven until keepalive/tool-call success (#62212).
  246. self._session_proven = False
  247. events_task = _events.start_refresh_task(self)
  248. try:
  249. reason = await self._wait_for_lifecycle_event()
  250. finally:
  251. if events_task is not None:
  252. events_task.cancel()
  253. if label and reason == "reconnect":
  254. logger.info("MCP server '%s': reconnect requested — tearing down %s session", self.name, label)
  255. return reason
  256. async def _serve_transport(self, transport_cm, label: str, connect_timeout: float) -> str:
  257. """Open *transport_cm*, wrap its streams in a ClientSession and serve it. Streams are indexed,
  258. not unpacked (mcp 1.x yields a 3-tuple, 2.x a pair); a TaskGroup drop maps to ``"reconnect"``."""
  259. try:
  260. async with transport_cm as _streams:
  261. async with _core.ClientSession(_streams[0], _streams[1], **self._session_kwargs()) as session:
  262. return await self._serve_session(session, connect_timeout, label)
  263. except BaseExceptionGroup as _eg:
  264. return self._reconnect_or_reraise_group(_eg)
  265. # ------------------------------------------------------------------ stdio
  266. def _track_spawned_children(self, new_pids: set[int]) -> None:
  267. """Ledger the freshly spawned stdio children (pids, pgids, machine spawn ledger). pgids are
  268. captured while alive (getpgid fails after exit; the sweep needs them for reparented descendants)."""
  269. new_pgids: dict[int, int] = {}
  270. new_starts: dict[int, int] = {}
  271. for pid in new_pids:
  272. try:
  273. new_pgids[pid] = os.getpgid(pid)
  274. except ProcessLookupError:
  275. # Raced and already exited. The SDK spawns with start_new_session=True, so the
  276. # child was its own group leader (pgid == pid): keep that group covered — any
  277. # descendant it left behind still has to be reaped; the prune forgets the group
  278. # once nothing in it is alive.
  279. new_pgids[pid] = pid
  280. except (AttributeError, OSError): # Windows (os.getpgid is POSIX-only)
  281. pass
  282. # Record the leader's start time so a later sweep can detect PID/PGID
  283. # recycling before signalling (see _stdio_starttimes, #43044).
  284. start = _leader_start_time(pid)
  285. if start is not None:
  286. new_starts[pid] = start
  287. with _core._lock:
  288. _stdio_pids.update(dict.fromkeys(new_pids, self.name))
  289. _stdio_pgids.update(new_pgids)
  290. _stdio_starttimes.update(new_starts)
  291. # Machine spawn ledger (startup sweeps reap orphans after an unclean exit); best-effort.
  292. for _pid in new_pids:
  293. try:
  294. register_child(_pid, "mcp-helper")
  295. except Exception:
  296. logger.debug("spawn-ledger register_child failed for MCP helper pid %s", _pid, exc_info=True)
  297. # Hand the pgroups to the shared parent-death supervisor so an ungraceful exit of this
  298. # process (kill -9, crash, force-quit) can't leave this server — or its descendants, e.g.
  299. # mcp-remote's spawned `node` — running forever. The graceful paths (shutdown,
  300. # _kill_orphaned_mcp_children) still reap as before; this only covers when they never run.
  301. own_process_groups(new_pgids.values())
  302. def _release_spawned_children(self, new_pids: set[int]) -> None:
  303. """Drop the ledger entries; a child (or its pgroup) still alive means SDK teardown failed
  304. (common on mid-way cancel on Linux: setsid() children escape) — mark it orphaned for the sweep."""
  305. from gateway.status import _pid_exists
  306. # Groups with nothing left alive; the supervisor forgets them after the lock is released.
  307. # Groups still alive stay registered on purpose, so the supervisor still reaps them if this
  308. # process dies before the orphan sweep runs.
  309. released_pgids: list = []
  310. with _core._lock:
  311. for pid in new_pids:
  312. _stdio_pids.pop(pid, None)
  313. # Windows-safe pid probe; the child may be gone while descendants remain in its pgroup.
  314. if _pid_exists(pid) or _pgroup_alive(_stdio_pgids.get(pid)):
  315. _orphan_stdio_pids.add(pid)
  316. _orphan_stdio_pid_servers[pid] = self.name
  317. else: # nothing to reap — drop the pgid so PID reuse can't surface stale pgroup state
  318. dropped = _stdio_pgids.pop(pid, None)
  319. if dropped is not None:
  320. released_pgids.append(dropped)
  321. _stdio_starttimes.pop(pid, None)
  322. release_process_groups(released_pgids)
  323. async def _run_stdio(self, config: dict):
  324. """Run the server using stdio transport."""
  325. if config.get("identity_header") is not None: # copy-pasted HTTP block: warn, don't mislead
  326. logger.warning("MCP server '%s': identity_header is only supported on "
  327. "HTTP/SSE transports — ignored for stdio servers", self.name)
  328. if not _core._ensure_mcp_sdk():
  329. raise ImportError(f"MCP server '{self.name}' requires the 'mcp' Python SDK, but "
  330. "it is not installed. Run `hermes setup` to install MCP support, then retry.")
  331. command = config.get("command")
  332. if not command:
  333. raise ValueError(f"MCP server '{self.name}' has no 'command' in config")
  334. inputs, _ = _connect_inputs(self.name, config)
  335. # Hash the inputs this attempt spawns with, never a second resolution of them.
  336. self._resolved_identity = _registration._identity_digest(inputs)
  337. command, safe_env, stdio_cwd = inputs
  338. # OSV malware preflight, then the cached-npx swap (ordering enforced there).
  339. command, args = await _core._preflight_stdio_command(self.name, command, config.get("args", []))
  340. server_params = _core.StdioServerParameters(
  341. command=command, args=args, env=safe_env or None, cwd=stdio_cwd,
  342. # Windows pipes can split non-UTF-8 bytes at chunk boundaries; substitute, don't raise.
  343. encoding_error_handler="replace")
  344. # Windows has no POSIX parent-death supervisor / killpg safety net (#61059): when this
  345. # process dies ungracefully (crash, force-quit), the stdio child trees — npx.cmd →
  346. # node.exe — survive as orphans with ParentId=null and pile up across restarts. Attaching
  347. # THIS process to a KILL_ON_JOB_CLOSE job before the spawn makes every child (and
  348. # grandchild) created after it die with the parent at the kernel level, so no orphan can
  349. # outlive us. Idempotent; BREAKAWAY_OK keeps deliberate breakaway children escaping.
  350. # Self-guards: a cheap no-op returning False on non-Windows.
  351. try:
  352. from hermes_cli.process_identity import attach_self_to_kill_on_close_job
  353. attach_self_to_kill_on_close_job()
  354. except Exception:
  355. logger.debug("job-object self-attach failed before stdio spawn", exc_info=True)
  356. # Reap orphans of prior attempts first (else retries pile up zombie pairs); unscoped on purpose;
  357. # off-loop because the reaper blocks up to 2s.
  358. await asyncio.to_thread(_lifecycle._kill_orphaned_mcp_children)
  359. pids_before = _lifecycle._snapshot_child_pids() # so the new child can be identified after spawn
  360. # Reap any orphaned subprocesses from prior failed connection attempts before spawning a new one.
  361. # Without this, each retry in the run() reconnect loop spawns a fresh process pair while the
  362. # previous failed pair lingers — leading to rapid zombie accumulation (see #57355, #57228). The
  363. # unscoped sweep also opportunistically reaps orphans left by *other* servers that never reconnect;
  364. # per-server filtering via ``server_name`` remains available for scoped call sites. Run in a worker
  365. # thread: the reaper blocks up to 2s (SIGTERM → wait → SIGKILL) when orphans exist, which would
  366. # otherwise stall the shared MCP event loop.
  367. new_pids: set = set()
  368. # Subprocess stderr goes to ~/.hermes/logs/mcp-stderr.log so banners can't corrupt the TUI.
  369. _config._write_stderr_log_header(self.name)
  370. stderr = _config._StderrTee(_config._get_mcp_stderr_log())
  371. try:
  372. async with _core.stdio_client(server_params, errlog=stderr.sink) as (read_stream, write_stream):
  373. # New PIDs for force-kill cleanup, minus non-MCP children (slash_worker, LSP) racing
  374. # into the window: they share the TUI's pgid — leaking them would killpg() the TUI.
  375. new_pids = _filter_mcp_children(_lifecycle._snapshot_child_pids() - pids_before)
  376. if new_pids:
  377. self._track_spawned_children(new_pids)
  378. self._stdio_child_pids = set(new_pids) # so in-flight calls fail fast when the child dies
  379. async with _core.ClientSession(read_stream, write_stream, **self._session_kwargs()) as session:
  380. # Bound the handshake here (``connect_timeout`` only bounds the caller's ``.result()``):
  381. # a server that never answers ``initialize`` would leak child + pipes per retry until EMFILE.
  382. connect_timeout = float(config.get("connect_timeout", _core._DEFAULT_CONNECT_TIMEOUT))
  383. return await self._serve_session(session, connect_timeout, mark_lifecycle=True)
  384. except Exception as exc:
  385. # The SDK only sees "Connection closed"; the child's stderr says why (#124264).
  386. abi_error = node_abi_error(self.name, await asyncio.to_thread(stderr.close))
  387. if abi_error is not None:
  388. raise abi_error from exc
  389. raise
  390. finally: # clean exit, exceptions AND cancellation
  391. stderr.close(timeout=0)
  392. if new_pids:
  393. self._release_spawned_children(new_pids)
  394. # ------------------------------------------------------------------- HTTP
  395. async def _preflight_content_type(self, url: str, *, headers: Optional[dict] = None,
  396. ssl_verify: bool = True, client_cert=None, timeout: float = 5.0,
  397. strict_redirect_headers: bool = False) -> None:
  398. """Probe *url* before the SDK connects: a plain web page would make the SDK sit out the full
  399. ``connect_timeout`` before an opaque ``CancelledError``; this raises NonMcpEndpointError within
  400. ``timeout``. Allow-list based: only a 2xx with a definite non-MCP content type is rejected, and
  401. only after a JSON-RPC ``initialize`` POST also fails to look like MCP (some servers serve a UI
  402. on GET but speak MCP via POST). Anything else passes — the handshake stays the source of truth.
  403. Own httpx client, OUTSIDE the SDK's anyio task group, so the error isn't group-wrapped."""
  404. try:
  405. import httpx as _httpx
  406. except ImportError:
  407. return # No httpx → skip probe; SDK import would have failed first.
  408. def _non_mcp_2xx(resp) -> bool:
  409. # Only judge 2xx (4xx/5xx may be an auth challenge); no content type advertised → trust the SDK.
  410. ct = _content_type_base(resp)
  411. return _is_2xx(resp) and bool(ct) and ct not in self._MCP_CONTENT_TYPES
  412. probe_headers = dict(headers) if headers else {}
  413. # Same route as the SDK client: TLS on an explicit transport (which also turns off httpx's own
  414. # env proxy auto-detection) plus the repo's proxy mounts, so the probe and the handshake agree.
  415. # Same redirect boundary as the transport client too: httpx strips Authorization on a
  416. # cross-origin hop natively, but forwards every other configured header verbatim — under
  417. # strict_redirect_headers those must not leave the configured origin on the probe either.
  418. probe_transport = _httpx.AsyncHTTPTransport(verify=ssl_verify, **_present(cert=client_cert))
  419. _build_client = _make_redirect_header_stripper(
  420. _httpx, _httpx.URL(url), strict=strict_redirect_headers,
  421. configured_header_names={key.lower() for key in probe_headers})
  422. try:
  423. async with _build_client(
  424. follow_redirects=True, timeout=_httpx.Timeout(timeout), transport=probe_transport,
  425. **_present(mounts=_mcp_proxy_mounts(_httpx, url, ssl_verify, client_cert, self.name))) as client:
  426. resp = await client.head(url, headers=probe_headers) # cheapest; GET on 405/501
  427. if resp.status_code in (405, 501):
  428. resp = await client.get(url, headers=probe_headers)
  429. # Non-MCP content type on HEAD/GET: try a JSON-RPC POST so POST-only servers pass.
  430. if _non_mcp_2xx(resp):
  431. post_resp = await client.post(
  432. url, content=_PROBE_INITIALIZE_BODY,
  433. headers={**probe_headers, "Content-Type": "application/json",
  434. "Accept": "application/json, text/event-stream"})
  435. if _is_2xx(post_resp) and _content_type_base(post_resp) in self._MCP_CONTENT_TYPES:
  436. resp = post_resp
  437. except _httpx.HTTPError:
  438. return # DNS/connect/timeout/transport error — let the SDK try.
  439. if not _non_mcp_2xx(resp):
  440. return
  441. ct_base = _content_type_base(resp)
  442. raise NonMcpEndpointError(f"MCP server '{self.name}' at {url} returned Content-Type '{ct_base}', not an MCP "
  443. f"response (expected one of: {', '.join(self._MCP_CONTENT_TYPES)}). The URL most likely "
  444. "points at a web page rather than an MCP endpoint — check it resolves to a Streamable "
  445. "HTTP / SSE endpoint (e.g. https://host/mcp, not https://host/).")
  446. def _reconnect_or_reraise_group(self, eg: BaseExceptionGroup) -> str:
  447. """Map an SDK transport TaskGroup failure to a clean ``"reconnect"``: HTTP/SSE stream pumps run in an anyio
  448. TaskGroup, so a transient drop escapes as a ``BaseExceptionGroup`` that would otherwise park the server for
  449. 300s over a sub-second glitch. Re-raise when it is not one: shutdown in progress (``_shutdown_event`` is
  450. set before cancel), KeyboardInterrupt/SystemExit or a real CancelledError in the group, or no live session
  451. this attempt (``_ready`` unset — connect failures must back off, not hot-loop).
  452. Streamable-HTTP / SSE transports run their stream pump inside an anyio TaskGroup. A transient stream
  453. drop (idle timeout, brief backend blip, server-side TCP close) surfaces as a ``BaseExceptionGroup``
  454. escaping the transport context manager. Left unwrapped it reaches ``run()``'s error path, which
  455. applies exponential backoff and eventually *parks* the server for 300s and deregisters its tools — a
  456. multi-minute tool outage for what is usually a sub-second glitch while the POST path stays healthy
  457. (issue #66092).
  458. - the group carries a ``KeyboardInterrupt`` / ``SystemExit`` — fatal signals must propagate to the
  459. interpreter, never be converted into a reconnect; - the group carries a real ``CancelledError``
  460. (task cancellation must propagate to asyncio, mirroring the ``run()`` guard for #9930); - we never
  461. reached a live session this attempt (``_ready`` unset) — a connect/handshake failure SHOULD fall
  462. through to ``run()``'s backoff rather than hot-loop reconnects against a broken endpoint.
  463. """
  464. if (self._shutdown_event.is_set()
  465. or eg.split((KeyboardInterrupt, SystemExit))[0] is not None
  466. or eg.split(asyncio.CancelledError)[0] is not None
  467. or not self._ready.is_set()):
  468. raise eg
  469. logger.debug("MCP server '%s': transport TaskGroup exited after a live session "
  470. "(%r) — reconnecting immediately instead of backing off", self.name, eg)
  471. return "reconnect"
  472. def _build_oauth_auth(self, url: str, config: dict):
  473. """OAuth 2.1 PKCE via the central MCPOAuthManager (one provider reused across reconnects and
  474. CLI paths). Setup failures re-raise (after a warning) so only this server is reported failed."""
  475. if self._auth_type != "oauth":
  476. return None
  477. try:
  478. from tools.mcp_oauth_manager import get_manager
  479. return get_manager().get_or_build_provider(self.name, url, config.get("oauth"))
  480. except Exception as exc:
  481. logger.warning("MCP OAuth setup failed for '%s': %s", self.name, exc)
  482. raise
  483. def _sse_transport(self, url: str, headers: dict, connect_timeout: float,
  484. ssl_verify, client_cert, oauth_auth, strict_cfg_headers: bool):
  485. """``sse_client`` context manager for ``transport: sse`` entries."""
  486. if strict_cfg_headers: # fail closed: SSE cannot enforce the redirect boundary
  487. raise ValueError(f"MCP server '{self.name}': strict_redirect_headers is "
  488. "not supported on the SSE transport.")
  489. if _core.sse_client is None:
  490. raise ImportError(f"MCP server '{self.name}' requires SSE transport but "
  491. "mcp.client.sse.sse_client is not available. "
  492. "Upgrade the mcp package to get SSE support.")
  493. # sse_read_timeout bounds the gap between events: SSE servers idle for minutes, so 300s (the
  494. # Streamable HTTP read timeout), not tool_timeout. ``auth`` must be forwarded or OAuth SSE 401s silently.
  495. sse_kwargs: dict = {"url": url, "headers": headers or None, "timeout": float(connect_timeout),
  496. "sse_read_timeout": 300.0, **_present(auth=oauth_auth)}
  497. # Always own the client: the httpx_client_factory forwards the SDK's (headers, auth, timeout),
  498. # installs the wire-body cap, layers TLS on the inner transport (client-level verify/cert are
  499. # inert once a custom transport= is passed) and re-adds the proxy mounts that custom transport
  500. # would otherwise suppress. Client MUST come from the SDK's httpx (httpx2 on mcp >= 2.0).
  501. _httpx_mod = _core.sdk_httpx()
  502. def _sse_client_factory(headers=None, timeout=None, auth=None):
  503. inner_transport = _httpx_mod.AsyncHTTPTransport(verify=ssl_verify, **_present(cert=client_cert))
  504. return _httpx_mod.AsyncClient(
  505. follow_redirects=True,
  506. timeout=timeout if timeout is not None else _httpx_mod.Timeout(30.0, read=300.0),
  507. transport=_make_mcp_body_cap_transport(_httpx_mod, inner_transport),
  508. **_present(mounts=_mcp_proxy_mounts(_httpx_mod, url, ssl_verify, client_cert, self.name),
  509. headers=headers, auth=auth))
  510. sse_kwargs["httpx_client_factory"] = _sse_client_factory
  511. return _core.sse_client(**sse_kwargs)
  512. def _streamable_http_transport(self, url: str, headers: dict, connect_timeout: float,
  513. ssl_verify, client_cert, oauth_auth,
  514. strict_cfg_headers: bool, configured_header_names: set):
  515. """Streamable HTTP context manager: mcp >= 1.24.0 gets a caller-owned httpx client; on the
  516. deprecated API (mcp < 1.24.0) the SDK owns the client."""
  517. if not _core._MCP_NEW_HTTP:
  518. if strict_cfg_headers: # fail closed: without an owned client redirects can't be hooked
  519. raise ImportError(f"MCP server '{self.name}' requires mcp >= 1.24.0 to "
  520. "enforce the portable redirect-header boundary "
  521. "(strict_redirect_headers). Upgrade the mcp package.")
  522. return _core.streamablehttp_client(url, headers=headers, timeout=float(connect_timeout), verify=ssl_verify,
  523. **_present(auth=oauth_auth))
  524. # Explicit AsyncClient matching the SDK's create_mcp_http_client defaults; MUST come from the
  525. # SDK's httpx (httpx2 on mcp >= 2.0) since the SDK sends its own Requests through it.
  526. httpx = _core.sdk_httpx()
  527. _build_client = _make_redirect_header_stripper(
  528. httpx, httpx.URL(url), strict=strict_cfg_headers, configured_header_names=configured_header_names)
  529. # verify/cert live on the inner transport: a custom transport= makes client-level TLS kwargs
  530. # inert — and suppresses httpx's own proxy auto-detection, hence the explicit mounts=.
  531. inner_transport = httpx.AsyncHTTPTransport(verify=ssl_verify, **_present(cert=client_cert))
  532. client_kwargs: dict = {"follow_redirects": True, "timeout": httpx.Timeout(float(connect_timeout), read=300.0),
  533. **({"headers": headers} if headers else {}),
  534. "event_hooks": {"response": [_make_http_rejection_recorder(self._http_rejection)]},
  535. "transport": _make_mcp_body_cap_transport(httpx, inner_transport),
  536. **_present(mounts=_mcp_proxy_mounts(httpx, url, ssl_verify, client_cert, self.name),
  537. auth=oauth_auth)}
  538. @asynccontextmanager
  539. async def _owned_client_streams(): # the SDK skips cleanup when http_client is provided
  540. async with _build_client(**client_kwargs) as http_client:
  541. # Without max_sse_event_size the SDK parses each event against httpx2's 1 MiB default,
  542. # a tighter second cap under our wire cap: a large tools/list or tool result (an
  543. # editor screenshot) failed outright.
  544. async with _core.streamable_http_client(url, http_client=http_client,
  545. max_sse_event_size=_MCP_HTTP_MAX_BODY_BYTES) as streams:
  546. yield streams
  547. return _owned_client_streams()
  548. async def _run_http(self, config: dict):
  549. """Run the server using HTTP/StreamableHTTP (or SSE) transport."""
  550. _core._ensure_mcp_sdk()
  551. if not _core._MCP_HTTP_AVAILABLE:
  552. raise ImportError(f"MCP server '{self.name}' requires HTTP transport but "
  553. "mcp.client.streamable_http is not available. "
  554. "Upgrade the mcp package to get HTTP support.")
  555. # Agent Plugins v1 strict_redirect_headers: configured headers MUST NOT follow a cross-origin
  556. # redirect — their names are captured BEFORE client-generated headers are merged in.
  557. inputs, configured_header_names = _connect_inputs(self.name, config)
  558. # Hash the endpoint this attempt connects to: a second read of the runtime file could
  559. # publish another endpoint's identity alongside this session.
  560. self._resolved_identity = _registration._identity_digest(inputs)
  561. url, headers = inputs
  562. logger.debug("MCP server '%s': connecting to %s", self.name, url)
  563. self._http_rejection = {} # last 4xx/5xx the owned client saw this attempt (recorder hook)
  564. # Seed MCP-Protocol-Version (user override wins) from the HANDSHAKE version, not the latest: a
  565. # 2026-07-28 header routes the handshake-era ``initialize()`` onto the envelope ladder, which rejects it.
  566. if not any(key.lower() == "mcp-protocol-version" for key in headers):
  567. headers["mcp-protocol-version"] = _core.LATEST_HANDSHAKE_VERSION
  568. connect_timeout = config.get("connect_timeout", _core._DEFAULT_CONNECT_TIMEOUT)
  569. common = (url, headers, connect_timeout, config.get("ssl_verify", True), _resolve_client_cert(self.name, config),
  570. self._build_oauth_auth(url, config), bool(config.get("strict_redirect_headers")))
  571. if config.get("transport") == "sse":
  572. return await self._serve_transport(self._sse_transport(*common), "SSE", float(connect_timeout))
  573. if self._sse_fallback:
  574. # A prior connect already proved this server SSE-only: skip the doomed Streamable
  575. # HTTP attempt on reconnects instead of flapping into the retry budget.
  576. logger.info("MCP server '%s': using latched SSE fallback transport", self.name)
  577. return await self._serve_transport(self._sse_transport(*common), "SSE", float(connect_timeout))
  578. transport = self._streamable_http_transport(*common, configured_header_names)
  579. label = "HTTP" if _core._MCP_NEW_HTTP else "legacy HTTP"
  580. try:
  581. return await self._serve_transport(transport, label, float(connect_timeout))
  582. except Exception as exc:
  583. # The SDK folds a non-2xx it cannot parse into ``-32603 Server returned an error response``;
  584. # the recorder hook kept the status/URL/body the server actually sent (#114350, #113359).
  585. http_detail = _describe_http_failure(exc, self._http_rejection)
  586. # SSE-only servers (or their load balancers) reject the Streamable HTTP chunked
  587. # ``initialize`` POST — with a 400-family status or an opaque SDK INTERNAL_ERROR —
  588. # previously a permanent failure with 0 active tools unless the user set
  589. # ``transport: sse`` (#53676, #104343). Retry over SSE on the initial connect, as
  590. # the MCP spec's transport-fallback behavior describes. Never on reconnect after a
  591. # proven session (``_ever_connected``: a genuine rejection on an established
  592. # transport must not silently switch transports), never on a timeout (not a
  593. # transport mismatch — ``_is_streamable_http_rejection`` matches neither), and never
  594. # with ``strict_redirect_headers`` (SSE cannot enforce that boundary).
  595. if (self._ever_connected or common[-1] or not _is_streamable_http_rejection(exc)):
  596. if http_detail != str(_unwrap_exception_group(exc)): # opaque SDK error + a recorded rejection
  597. raise ConnectionError(f"MCP server '{self.name}': Streamable HTTP connect failed "
  598. f"({http_detail})") from exc
  599. raise
  600. logger.warning(
  601. "MCP server '%s': Streamable HTTP rejected the initial connect (%s) — retrying "
  602. "over SSE. If this connects, set `transport: sse` for this server in config.yaml "
  603. "to skip the failed attempt on future startups.",
  604. self.name, http_detail)
  605. try:
  606. self._sse_fallback = True
  607. return await self._serve_transport(self._sse_transport(*common), "SSE", float(connect_timeout))
  608. except Exception as sse_exc:
  609. if self._ever_connected: # SSE session was live and dropped: transient, keep the latch
  610. raise
  611. self._sse_fallback = False
  612. raise ConnectionError(
  613. f"MCP server '{self.name}': both Streamable HTTP and SSE transports failed "
  614. f"(Streamable HTTP: {http_detail}; SSE: "
  615. f"{_unwrap_exception_group(sse_exc)}). Check the URL points at an MCP "
  616. "endpoint, or pin `transport: sse` if the server is SSE-only.") from sse_exc
  617. # -------------------------------------------------------------- discovery
  618. # Legacy Streamable-HTTP transport TaskGroup dropped: reconnect immediately instead of backoff/park
  619. # (#66092).
  620. async def _discover_tools(self):
  621. """Discover tools from the connected session. Capability-gated: prompt-/resource-only
  622. servers raise ``MCPError(-32601)`` on ``tools/list``, which would abort the connection.
  623. Skip the call when the server doesn't advertise the ``tools`` capability. (Ported from
  624. anomalyco/opencode#31271.)
  625. """
  626. self._ping_unsupported = False # fresh transport: re-probe ``ping`` across the reconnect
  627. if self.session is None:
  628. return
  629. if not self._advertises_tools():
  630. logger.info("MCP server '%s': does not advertise 'tools' capability — "
  631. "skipping tools/list (prompts/resources remain available)", self.name)
  632. self._tools = []
  633. else:
  634. async with self._rpc_lock:
  635. self._list_cache_meta = {}
  636. self._tools = await _core._paginate_full_list(
  637. self.session.list_tools, "tools", self.name, cache_meta_out=self._list_cache_meta)
  638. self._register_discovered_tools_if_needed()
  639. def _register_discovered_tools_if_needed(self) -> None:
  640. """Publish freshly discovered tools when none are registered (initial registration normally happens in
  641. ``_discover_and_register_server``). Outage handling may clear ``_ready`` and deregister stale tools;
  642. ownership via ``_servers`` authorizes publishing before readiness is restored so a revival (or a server
  643. retained after a recoverable initial failure) never comes back with zero tools."""
  644. if self._registered_tool_names:
  645. return
  646. with _core._lock:
  647. owned = [key for key, live in _core._servers.items() if live is self]
  648. if not owned and not self._ready.is_set():
  649. return
  650. self._registered_tool_names = _registration._register_server_tools(self.name, self, self._config)
  651. with _core._lock: # a retained initial-failure server that just published tools has recovered
  652. for key in owned:
  653. if _core._servers.get(key) is self:
  654. _core._server_connect_errors.pop(key, None)