mcp_tool_health.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297
  1. """Session health for MCPServerTask: dynamic tool refresh on list_changed notifications, server
  2. log forwarding, keepalive probes, suspect-mark / lazy-verify, in-flight call fail-fast, stdio
  3. child liveness and stdio idle/lifetime recycling."""
  4. import asyncio
  5. import json
  6. import logging
  7. import time
  8. from typing import Iterable, Optional
  9. from tools.mcp_tool_errors import _is_method_not_found_error, _unwrap_exception_group
  10. from tools.mcp_tool_schema import mcp_prefixed_tool_name
  11. from tools.mcp_tool_common import _core
  12. from tools import mcp_tool_registration as _registration
  13. logger = logging.getLogger("tools.mcp_tool")
  14. _KEEPALIVE_RPC_TIMEOUT = 30.0
  15. class MCPServerHealthMixin:
  16. """Methods of :class:`tools.mcp_tool.MCPServerTask` (mixed in; relies on its attributes)."""
  17. __slots__ = ()
  18. def _is_http(self) -> bool:
  19. return "url" in self._config
  20. def _is_recycled_stdio(self) -> bool:
  21. """True when a stdio server was intentionally recycled."""
  22. return not self._is_http() and self._recycled_reason is not None
  23. def mark_tool_call(self) -> None:
  24. """Record that a user-visible MCP operation is starting."""
  25. self._last_tool_call_at = time.monotonic()
  26. def _mark_lifecycle_started(self) -> None:
  27. self._lifecycle_started_at = self._last_tool_call_at = time.monotonic()
  28. self._recycled_reason = None
  29. def _stdio_recycle_deadlines(self):
  30. """``[(deadline, reason), ...]`` for the lifetime/idle limits; empty for HTTP or while an RPC holds the lock."""
  31. if self._is_http() or self._rpc_lock.locked():
  32. return []
  33. limits = ((self._lifecycle_started_at, self._max_lifetime_seconds, "max_lifetime_seconds"),
  34. (self._last_tool_call_at, self._idle_timeout_seconds, "idle_timeout_seconds"))
  35. return [(start + limit, reason) for start, limit, reason in limits if limit is not None]
  36. def _stdio_recycle_reason(self, now: Optional[float] = None) -> Optional[str]:
  37. """The stdio recycle reason if idle/age limits have elapsed (lifetime wins), else None."""
  38. now = time.monotonic() if now is None else now
  39. return next((reason for deadline, reason in self._stdio_recycle_deadlines() if now >= deadline), None)
  40. def _next_stdio_recycle_deadline(self) -> Optional[float]:
  41. return min((d for d, _ in self._stdio_recycle_deadlines()), default=None)
  42. def _mark_stdio_recycled(self, reason: str) -> None:
  43. """Mark a stdio session dormant before its transport finishes closing."""
  44. self._recycled_reason = reason
  45. self.session = None
  46. def _schedule_tools_refresh(self) -> asyncio.Task:
  47. """Schedule a background tool refresh (failures logged) and keep it strongly referenced."""
  48. async def _run():
  49. try:
  50. await self._refresh_tools()
  51. except Exception:
  52. logger.exception("MCP server '%s': dynamic tool refresh failed", self.name)
  53. task = asyncio.create_task(_run())
  54. self._pending_refresh_tasks.add(task)
  55. task.add_done_callback(self._pending_refresh_tasks.discard)
  56. return task
  57. def _make_logging_callback(self):
  58. """``logging_callback`` forwarding server ``notifications/message`` into Hermes logging (SDK default drops them).
  59. Routes MCP ``notifications/message`` log notifications from the server into Hermes' logging
  60. (agent.log via hermes_logging), tagged with the server name. Without this, the SDK's default
  61. callback silently discards them, so server-side warnings/errors during a tool call were invisible.
  62. Port of anomalyco/opencode#34529.
  63. """
  64. async def _on_log(params):
  65. try:
  66. level = _core._MCP_LOG_LEVEL_MAP.get(str(getattr(params, "level", "info")).lower(), logging.INFO)
  67. data = getattr(params, "data", None)
  68. if not isinstance(data, str):
  69. try:
  70. data = json.dumps(data, ensure_ascii=False, default=str)
  71. except (TypeError, ValueError):
  72. data = str(data)
  73. if len(data) > 2000: # cap payloads so a chatty server can't flood agent.log
  74. data = data[:2000] + "... [truncated]"
  75. logger_name = getattr(params, "logger", None)
  76. origin = f"{self.name}/{logger_name}" if logger_name else self.name
  77. logger.log(level, "MCP server log [%s]: %s", origin, data)
  78. except Exception:
  79. logger.debug("Failed to handle MCP log notification from '%s'", self.name, exc_info=True)
  80. return _on_log
  81. def _make_message_handler(self):
  82. """``message_handler``: only ``ToolListChangedNotification`` triggers a refresh; prompt/resource changes log."""
  83. async def _handler(message):
  84. try:
  85. if isinstance(message, Exception):
  86. logger.debug("MCP message handler (%s): exception: %s", self.name, message)
  87. return
  88. if not (_core._MCP_NOTIFICATION_TYPES and isinstance(message, _core.ServerNotification)):
  89. return
  90. # mcp 2.0 made ServerNotification a plain union (payload IS the message) instead of
  91. # a RootModel (payload under ``.root``); without this unwrap refreshes silently stop.
  92. payload = getattr(message, "root", message)
  93. if isinstance(payload, _core.ToolListChangedNotification):
  94. logger.info("MCP server '%s': received tools/list_changed notification", self.name)
  95. # Separate task: refreshing synchronously inside the handler can wedge the stdio
  96. # JSON-RPC stream when list_changed arrives while another request is in flight.
  97. self._schedule_tools_refresh()
  98. await asyncio.sleep(0) # one tick so short-lived contexts (and tests) observe it
  99. elif isinstance(payload, _core.PromptListChangedNotification):
  100. logger.debug("MCP server '%s': prompts/list_changed (ignored)", self.name)
  101. elif isinstance(payload, _core.ResourceListChangedNotification):
  102. logger.debug("MCP server '%s': resources/list_changed (ignored)", self.name)
  103. except Exception:
  104. logger.exception("Error in MCP message handler for '%s'", self.name)
  105. return _handler
  106. def _deregister_owned(self, tool_names: Iterable[str]) -> None:
  107. """Deregister *tool_names* this server's toolset still owns (never a colliding name owned by another server)."""
  108. from tools.registry import registry
  109. for tool_name in tool_names:
  110. if registry.get_toolset_for_tool(tool_name) == f"mcp-{self.name}":
  111. _registration._deregister_mcp_tool_all_scopes(self, tool_name)
  112. async def _refresh_tools(self):
  113. """Re-fetch tools on ``tools/list_changed`` and update the registry. The lock serializes rapid-fire
  114. notifications; after the list_tools ``await`` all mutations are synchronous — atomic on the event loop."""
  115. if getattr(self, '_editor_frozen_manifest', None) is not None:
  116. return # Editor tool policy changes apply only to a new canonical session.
  117. if not self._advertises_tools():
  118. return # tools/list would raise MCPError(-32601)
  119. async with self._refresh_lock:
  120. old_tool_names = set(self._registered_tool_names)
  121. async with self._rpc_lock:
  122. # Snapshot the session only once the RPC lock is held: run() resets self.session
  123. # to None on every transport teardown (reconnect, backoff, park, cancel) outside
  124. # both locks, so a refresh queued behind the lock can wake up mid-restart
  125. # (#109824). Skipping is correct — the reconnect's own discovery re-lists tools
  126. # and the next tools/list_changed re-arms this refresh against the live session.
  127. session = self.session
  128. if session is None:
  129. logger.debug("MCP server '%s': skipping dynamic tool refresh; session not connected", self.name)
  130. return
  131. new_mcp_tools = await _core._paginate_full_list(session.list_tools, "tools", self.name)
  132. # Remove only stale names first — no nuke-and-repave: live turns may hold tool-call
  133. # IDs pointing at existing handlers; in-place replacement avoids "not connected" races.
  134. self._deregister_owned(old_tool_names - {mcp_prefixed_tool_name(self.name, tool.name) for tool in new_mcp_tools})
  135. # Re-register; a raw name can become ambiguous after normalization without changing
  136. # its normalized name, so also drop old entries the final registration no longer owns.
  137. self._tools = new_mcp_tools
  138. registered_names = _registration._register_server_tools(self.name, self, self._config)
  139. self._deregister_owned(old_tool_names - set(registered_names))
  140. self._registered_tool_names = registered_names
  141. new_tool_names = set(registered_names)
  142. changes = [f"{label}: {', '.join(sorted(names))}" for label, names in
  143. (("added", new_tool_names - old_tool_names), ("removed", old_tool_names - new_tool_names)) if names]
  144. if changes:
  145. logger.warning("MCP server '%s': tools changed dynamically — %s. "
  146. "Verify these changes are expected.", self.name, "; ".join(changes))
  147. else:
  148. logger.info("MCP server '%s': dynamically refreshed %d tool(s) (no changes)",
  149. self.name, len(self._registered_tool_names))
  150. async def _keepalive_probe(self) -> None:
  151. """Exercise the session; raise on a genuine connection failure. ``ping`` first (cheap,
  152. OPTIONAL); on -32601 latch ``_ping_unsupported`` (reset per transport connection) and fall
  153. back to ``list_tools`` when the server advertises tools, else the -32601 propagates."""
  154. async def list_tools():
  155. await asyncio.wait_for(self.session.list_tools(), timeout=_KEEPALIVE_RPC_TIMEOUT)
  156. if not self._ping_unsupported:
  157. try:
  158. await asyncio.wait_for(self.session.send_ping(), timeout=_KEEPALIVE_RPC_TIMEOUT)
  159. return
  160. except Exception as exc:
  161. if _is_method_not_found_error(exc):
  162. if not self._advertises_tools(): # ping definitively unsupported, nothing to fall back to
  163. raise
  164. self._ping_unsupported = True
  165. logger.info("MCP server '%s': does not implement the optional 'ping' utility (-32601); "
  166. "using 'list_tools' for keepalive on this connection.", self.name)
  167. elif isinstance(exc, (TimeoutError, asyncio.TimeoutError)) and self._advertises_tools():
  168. # A server that silently drops ping looks like a dead transport: confirm with
  169. # list_tools before declaring it dead, else propagate the original failure.
  170. try:
  171. await list_tools()
  172. except Exception:
  173. raise exc from None
  174. self._ping_unsupported = True # latch so later keepalives skip the 30s wait
  175. logger.info("MCP server '%s': ping timed out but list_tools succeeded — server "
  176. "silently drops ping; using 'list_tools' for keepalive on this connection.", self.name)
  177. return
  178. else:
  179. raise # closed transport, expired session, etc. — real failure
  180. await list_tools()
  181. def _mark_session_proven(self) -> None:
  182. """Record that the session demonstrated real health (keepalive or tool-call success).
  183. Only then is the reconnect budget cleared: a handshake that drops moments later must keep
  184. consuming ``_reconnect_retries`` so a flapping transport still reaches the park.
  185. Called from the keepalive success path (session survived a full keepalive interval — for
  186. stdio without a keepalive, a full default interval idle with the child alive) and the
  187. tool-call success path. See #62212.
  188. """
  189. if self._session_proven:
  190. return
  191. self._session_proven = True
  192. self._reconnect_retries = 0
  193. self._park_reason = None
  194. if self._was_parked:
  195. self._was_parked = False
  196. logger.warning("MCP server '%s': revived — session healthy again after "
  197. "parking (state: parked → connected)", self.name)
  198. # A proven fresh transport clears the one-time permanent-failure grace and any race bookkeeping.
  199. self._permanent_grace_used = self._teardown_race = False
  200. def mark_suspect(self, reason: str) -> None:
  201. """Latch a suspicion (no I/O); the NEXT call verifies via :meth:`ensure_healthy` and recycles on failure.
  202. The NEXT call verifies via :meth:`ensure_healthy` and recycles the transport if the probe fails,
  203. instead of the connection silently staying poisoned until process restart (#81051/#77765/#84132).
  204. """
  205. if self._suspect_reason is None and reason:
  206. logger.warning("MCP server '%s': connection marked suspect (%s); next call will health-check it",
  207. self.name, reason)
  208. self._suspect_reason = reason or None
  209. async def ensure_healthy(self, timeout: float = 5.0) -> bool:
  210. """Verify a suspect connection before reuse; recycle if dead. True when healthy (suspicion
  211. cleared). On failure requests a reconnect, drops the stale session so the caller's
  212. no-session path takes over, and returns False. Never raises."""
  213. reason = self._suspect_reason
  214. if not reason:
  215. return True
  216. if self.session is None: # nothing to verify — the reconnect path owns recovery now
  217. self._suspect_reason = None
  218. self._reconnect_event.set()
  219. return False
  220. try:
  221. await asyncio.wait_for(self._keepalive_probe(), timeout=timeout)
  222. except Exception as exc:
  223. root = _unwrap_exception_group(exc)
  224. logger.warning("MCP server '%s': suspect connection (%s) failed health check (%s: %s) — "
  225. "requesting reconnect (state: suspect → degraded)",
  226. self.name, reason, type(root).__name__, root)
  227. self._suspect_reason = None
  228. self.mark_suspect(f"health check failed after {reason}")
  229. self.session = None
  230. self._ready.clear()
  231. self._reconnect_event.set()
  232. return False
  233. logger.info("MCP server '%s': suspect connection passed health check (%s) — clearing suspicion",
  234. self.name, reason)
  235. self._suspect_reason = None
  236. self._mark_session_proven()
  237. return True
  238. def _fail_inflight_calls(self, reason: str) -> None:
  239. """Cancel every in-flight RPC BEFORE the transport unwinds: the SDK does not always fail
  240. pending requests when streams close, so a call would otherwise wait out the full tool
  241. timeout. Cancelling anything flags ``_teardown_race`` so run() treats the next reconnect
  242. as recovery rather than charging the rapid-drop budget."""
  243. victims = [t for t in self._inflight_tasks if not t.done()]
  244. if not victims:
  245. return
  246. self._reconnecting = self._teardown_race = True
  247. self.mark_suspect(f"{reason} tore down {len(victims)} in-flight call(s)")
  248. for task in victims:
  249. task.cancel()
  250. def _stdio_children_dead(self) -> bool:
  251. """True when every stdio child we spawned has exited. Best-effort: False (unknown → don't
  252. fail fast) for HTTP, no captured PIDs, missing psutil, or a failed probe."""
  253. pids = getattr(self, "_stdio_child_pids", None)
  254. if not pids or self._is_http():
  255. return False
  256. try:
  257. import psutil
  258. return not any(psutil.pid_exists(pid) for pid in pids) # Windows-safe, no signal noise
  259. except Exception: # missing psutil or failed probe → unknown → don't fail fast
  260. return False
  261. async def _watch_stdio_children(self) -> None:
  262. """Poll child liveness during a stdio RPC; resolves when a tracked child dies so the caller cancels the RPC.
  263. See #81995.
  264. """
  265. while not self._stdio_children_dead():
  266. # Async context — never block the loop (#36163).
  267. await asyncio.sleep(0.25)