mcp_tool_handlers.py 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879
  1. """Registry-facing sync handlers for MCP tools and utility tools (resources/prompts), plus the per-call recovery
  2. ladder: trust gating, circuit breaker, auth (401) refresh, session-expired reconnect and dead-stdio respawn retry."""
  3. import logging
  4. import asyncio
  5. import contextvars
  6. import inspect
  7. import json
  8. import time
  9. from contextlib import asynccontextmanager
  10. from functools import partial
  11. from types import SimpleNamespace
  12. from typing import Any, Callable, Dict, List, Optional, Tuple
  13. from hermes_platform import declaration
  14. from tools.registry import invalidate_check_fn_cache, tool_error
  15. from tools.ansi_strip import strip_unicode_tags
  16. from tools.mcp_tool_common import _exc_str, _sanitize_error, mcp_field, _core
  17. from tools import mcp_tool_loop as _loop
  18. from tools.mcp_tool_content import (
  19. _MCP_HARD_RESULT_CAP_CHARS, _cache_mcp_audio_block, _cache_mcp_image_block, _mcp_result_with_native_images,
  20. _render_mcp_dropped_block_notice, _render_mcp_resource_block, _strip_reserved_meta_keys,
  21. _truncate_mcp_text_result)
  22. from tools.mcp_tool_errors import (
  23. _auth_error_detail, _is_auth_error, _is_session_expired_error, _mcp_call_failed_message)
  24. logger = logging.getLogger("tools.mcp_tool")
  25. _MISSING = object()
  26. declaration.on_change = invalidate_check_fn_cache
  27. _NEEDS_REAUTH_MSG = (
  28. "MCP server '{s}' requires re-authentication.{detail} Run `hermes mcp login {s}` (or delete the tokens file under "
  29. "~/.hermes/mcp-tokens/ and restart). Do NOT retry this tool — ask the user to re-authenticate.")
  30. _STDIO_NO_RESPAWN_MSG = (
  31. "MCP server '{s}' stdio subprocess had exited (this is not a timeout — the call never reached the server). A "
  32. "respawn was requested but no fresh session came back within {t:.0f}s. Wait a few seconds before retrying; if it "
  33. "keeps failing the server is not starting and needs the user.")
  34. _STDIO_DIED_AGAIN_MSG = (
  35. "MCP server '{s}' respawned its stdio subprocess and it exited again immediately. The server is not starting "
  36. "cleanly — do NOT retry this tool; ask the user to check the server's command and its stderr log.")
  37. _STDIO_OUTCOME_UNCERTAIN_MSG = (
  38. "MCP server '{s}' lost its stdio subprocess after the tool call began. The operation may have completed, so "
  39. "Hermes did not replay it. Do NOT retry automatically; inspect the external state first.")
  40. _SESSION_OUTCOME_UNCERTAIN_MSG = (
  41. "The MCP transport session to '{s}' expired while this write-capable call was in flight, so the outcome is "
  42. "UNKNOWN — the operation may or may not have taken effect server-side. It was NOT automatically retried to "
  43. "avoid a duplicate side effect. The connection has {state}. Verify whether the operation took effect (e.g. "
  44. "with a read-only tool) before re-invoking it.")
  45. def _tool_is_read_only(server_name: str, tool_name: str) -> bool:
  46. """True only when discovery captured ``readOnlyHint=True`` for the tool. Missing or malformed
  47. metadata fails safe to False (treated as write-capable). readOnlyHint is a property of the
  48. connection's tools, so it lives under the connection key."""
  49. from tools.mcp_tool_scope import _resolve_server_key
  50. return _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True
  51. def _trust_approval_key(server_name: str, tool_name: str) -> str:
  52. """Approval key a session/always answer to the trust gate is kept under: one (server, tool) pair,
  53. so allowing one write tool never covers the server's others. Session entries are per chat;
  54. always entries land in the active profile's ``command_allowlist``."""
  55. return f"mcp:{server_name}/{tool_name}"
  56. def _trust_granted(server_name: str, tool_name: str) -> bool:
  57. """A remembered Session/Always answer, or YOLO / ``approvals.mode: off`` for the calling session.
  58. Covers only the trust prompt: the spend limit is a budget, not an approval, and still asks."""
  59. from tools.approval import is_approval_bypass_active, is_approved
  60. from tools.approval_context import get_current_session_key
  61. return is_approval_bypass_active() or is_approved(get_current_session_key(),
  62. _trust_approval_key(server_name, tool_name))
  63. # Keeps the whole trust-gate card inside Discord's 2,000-character message cap (the tightest button
  64. # surface), so the truncation marker is never cut off by the adapter's own fit.
  65. _TRUST_GATE_ARGUMENTS_MAX_CHARS = 1_200
  66. _TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER = "\n... [arguments truncated]"
  67. def _is_secret_argument_name(key: Any) -> bool:
  68. from agent.redact import SECRET_HEADER_NAME_LIST, is_secret_field_name
  69. return is_secret_field_name(key) or (isinstance(key, str) and key.casefold() in SECRET_HEADER_NAME_LIST)
  70. def _mask_secret_fields(value: Any) -> Any:
  71. """Mask every value under a credential-named field (``db_password``, ``client_secret``) or API-key
  72. header name (``x-api-key``), whatever its type: the text redactor below only catches the
  73. ``key: value`` and token shapes it knows."""
  74. if isinstance(value, dict):
  75. return {k: "***" if v not in (None, "") and _is_secret_argument_name(k) else _mask_secret_fields(v)
  76. for k, v in value.items()}
  77. if isinstance(value, list):
  78. return [_mask_secret_fields(v) for v in value]
  79. return value
  80. def _format_trust_gate_arguments(args: dict) -> str:
  81. """Redacted, bounded JSON of the call's arguments for the approval card. Plain text, no
  82. fence: every surface already renders the prompt inside a code block."""
  83. import unicodedata
  84. from agent.redact import redact_sensitive_text
  85. rendered = json.dumps(_mask_secret_fields(args), ensure_ascii=False, indent=2, default=str)
  86. # Scan and redact only a window well past the cap: a multi-megabyte payload (a file write) would
  87. # otherwise hold the prompt back for seconds.
  88. window = 4 * _TRUST_GATE_ARGUMENTS_MAX_CHARS
  89. cut, rendered = len(rendered) > window, rendered[:window]
  90. # Bidi overrides, invisible format characters and the C1/DEL controls JSON leaves raw could make the
  91. # card show something other than what the server receives; render them as escapes.
  92. rendered = "".join(ch.encode("unicode_escape").decode()
  93. if ch != "\n" and unicodedata.category(ch) in ("Cc", "Cf") else ch for ch in rendered)
  94. # force: the approval prompt is a display boundary even with security.redact_secrets off. The CLI
  95. # prints it as-is. Redact before truncating so a cut cannot split a secret past its pattern.
  96. rendered = redact_sensitive_text(rendered, force=True)
  97. if cut or len(rendered) > _TRUST_GATE_ARGUMENTS_MAX_CHARS:
  98. keep = _TRUST_GATE_ARGUMENTS_MAX_CHARS - len(_TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER)
  99. rendered = rendered[:keep].rstrip() + _TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER
  100. return rendered
  101. def _trust_gate_check(server_name: str, tool_name: str, booked: Optional[list] = None,
  102. args: Optional[dict] = None) -> Optional[str]:
  103. """Approval gate for write-capable tools on ``trust: untrusted`` servers, plus the declared spend
  104. limit on any server. None to proceed, else a ``tool_error``. Fail-closed: approval-system errors
  105. block. An admitted costed call is charged to the chat's tally and appended to *booked* so the
  106. caller can refund it when the call never reaches the server."""
  107. from tools.mcp_tool_scope import _server_key
  108. from tools import mcp_tool_costs as _costs
  109. # Trust and costs are the calling profile's own policy (an adopter of a shared connection keeps its own).
  110. key = _server_key(server_name)
  111. trust = _core._server_trust_levels.get(key, _core._TRUST_FULL)
  112. gated = (trust == _core._TRUST_UNTRUSTED and not _tool_is_read_only(server_name, tool_name)
  113. and not _trust_granted(server_name, tool_name))
  114. charge = None
  115. if key in _core._server_cost_policies: # no table: not even a session lookup, main's path byte for byte
  116. from tools.approval_context import get_current_session_key
  117. charge = _costs.pending_charge(key, server_name, tool_name, get_current_session_key())
  118. if charge is None:
  119. return _ask_for_call(server_name, tool_name, None, booked, args=args) if gated else None
  120. if not gated and charge.reserve_within_limit()[0]:
  121. if booked is not None:
  122. booked.append(charge)
  123. return None
  124. with charge.asking(): # re-read under the slot: a call that asked first may have booked meanwhile
  125. return _ask_for_call(server_name, tool_name, charge, booked, gated=gated, args=args)
  126. def _ask_for_call(server_name: str, tool_name: str, charge, booked: Optional[list], *,
  127. gated: bool = True, args: Optional[dict] = None) -> Optional[str]:
  128. """Ask the user about one gated and/or costed call; book the charge only on an explicit yes."""
  129. spent, over = 0.0, False
  130. if charge is not None:
  131. if gated:
  132. spent = charge.spent()
  133. over = charge.over_limit(spent)
  134. else:
  135. reserved, spent = charge.reserve_within_limit()
  136. if reserved:
  137. if booked is not None:
  138. booked.append(charge)
  139. return None
  140. over = True
  141. extra = charge.lines(spent) if charge is not None else []
  142. if gated:
  143. head = f"MCP tool '{tool_name}' on UNTRUSTED server '{server_name}' wants to run."
  144. tail = "This tool is write-capable (no readOnlyHint=true annotation) and may modify external state."
  145. # The limit notice leads, and the arguments trail: a prompt cut to a platform's cap keeps the notice.
  146. message = " ".join([head, *(["This call goes past the chat's spend limit."] if over else []), *extra, tail])
  147. if args:
  148. message += f"\n\nArguments:\n{_format_trust_gate_arguments(args)}"
  149. # Over the limit the answer covers this call only: a remembered trust grant must not
  150. # pre-approve the next over-limit call.
  151. remember_key = None if over else _trust_approval_key(server_name, tool_name)
  152. scope = "once" if over else "once, for this session, or always"
  153. description = (f"Server '{server_name}' is configured 'trust: untrusted'. "
  154. f"Approve to run '{tool_name}' {scope}, or deny to block it.")
  155. title = f"MCP server '{server_name}' is asking"
  156. else:
  157. message = " ".join([f"MCP tool '{tool_name}' on server '{server_name}' would take this chat past its "
  158. f"spend limit.", *extra])
  159. description = (f"Server '{server_name}' declares a spend limit per chat. Approve to run '{tool_name}' "
  160. f"once (every further paid call in this chat asks again), or deny to block it.")
  161. title = f"MCP server '{server_name}' spend limit"
  162. remember_key = None
  163. try: # lazy: tools.approval routes the prompt to whichever surface owns the session
  164. from tools.approval_prompt import request_elicitation_consent
  165. answer = request_elicitation_consent(message, description, surface=f"mcp-trust/{server_name}", title=title,
  166. remember_key=remember_key)
  167. except Exception as exc:
  168. logger.error("MCP trust gate: approval check failed for %s.%s: %s", server_name, tool_name, exc, exc_info=True)
  169. if over and charge is not None:
  170. return tool_error(charge.refusal(spent))
  171. return tool_error(f"MCP tool '{tool_name}' on untrusted server '{server_name}' was blocked: the approval "
  172. f"system was unavailable (fail-closed).")
  173. if answer == "accept":
  174. if charge is not None:
  175. charge.book()
  176. if booked is not None:
  177. booked.append(charge)
  178. return None
  179. logger.info("MCP trust gate: user %s '%s' on %s server '%s'", "cancelled" if answer == "cancel" else "denied",
  180. tool_name, "untrusted" if gated else "spend-limited", server_name)
  181. if over and charge is not None:
  182. return tool_error(charge.refusal(spent))
  183. return tool_error(f"The user did not approve running write-capable MCP tool '{tool_name}' on untrusted server "
  184. f"'{server_name}'. The command was NOT run. Do not retry without explicit user direction.")
  185. def _check_circuit_breaker(server_name: str) -> Optional[str]:
  186. """Open-breaker error, or None when calls may proceed. After the cooldown the breaker is
  187. half-open: the next call probes; success resets, failure re-bumps and re-arms the cooldown."""
  188. from tools.mcp_tool_scope import _resolve_server_key
  189. key = _resolve_server_key(server_name)
  190. failures = _core._server_error_counts.get(key, 0)
  191. age = time.monotonic() - _core._server_breaker_opened_at.get(key, 0.0)
  192. if failures < _core._CIRCUIT_BREAKER_THRESHOLD or age >= _core._CIRCUIT_BREAKER_COOLDOWN_SEC:
  193. return None
  194. retry_in = max(1, int(_core._CIRCUIT_BREAKER_COOLDOWN_SEC - age))
  195. if _core._server_errors_all_application.get(key):
  196. # The server answered every time; the calls were rejected. Calling it "unreachable" sent the
  197. # model to the user instead of to its own arguments (#11113).
  198. return tool_error(f"MCP server '{server_name}' rejected the last {failures} calls (it is reachable; see the "
  199. f"error text those calls returned). Paused for ~{retry_in}s. Do NOT repeat the same call — "
  200. f"fix the arguments/URL/target or use a different approach.")
  201. return tool_error(f"MCP server '{server_name}' is unreachable after {failures} consecutive failures. "
  202. f"Auto-retry available in ~{retry_in}s. Do NOT retry "
  203. f"this tool yet — use alternative approaches or ask the user to check the MCP server.")
  204. def _acquire_call_server(server_name: str, tool_timeout: float):
  205. """``(server, None)`` when a call may be dispatched, else ``(None, error)``. No session: a
  206. reconnect may be completing, so wait briefly before a breaker strike; still down -> ask the
  207. server task to rebuild (probing a dead transport would re-arm the breaker forever)."""
  208. from tools import mcp_tool_discovery as _discovery # lazy: discovery -> registration -> handlers cycle
  209. not_connected = tool_error(f"MCP server '{server_name}' is not connected")
  210. from tools.mcp_liveness import unavailable_details
  211. details = unavailable_details(server_name)
  212. if details is not None:
  213. decl, current, sentence = details
  214. not_connected = tool_error(
  215. sentence,
  216. server=server_name,
  217. state=current.state,
  218. app={
  219. "name": decl.name,
  220. "version": current.availability.version,
  221. "path": current.availability.path,
  222. },
  223. user_action=current.user_action,
  224. retry=current.retry,
  225. )
  226. server = _discovery._get_connected_server_for_call(server_name)
  227. wait = min(5.0, float(tool_timeout or 5.0))
  228. if server and (server.session or _loop._wait_for_server_session_ready(server, timeout=wait)):
  229. return server, None
  230. _core._bump_server_error(server_name)
  231. if server and _loop._signal_reconnect(server):
  232. return None, tool_error(f"MCP server '{server_name}' transport is down; reconnect requested. Do NOT retry this "
  233. f"tool immediately — give it a few seconds to come back.")
  234. return None, not_connected
  235. def _result_is_error(result) -> bool:
  236. """True only for a JSON payload carrying an ``error`` key (non-JSON = success)."""
  237. try:
  238. return "error" in json.loads(result)
  239. except (json.JSONDecodeError, TypeError):
  240. return False
  241. def _record_call_outcome(server_name: str, result) -> Any:
  242. """Breaker bookkeeping: an error payload from the tool itself still counts as a strike (#10447),
  243. flagged as an application error so the open-breaker message stays truthful."""
  244. if _result_is_error(result):
  245. _core._bump_server_error(server_name, application=True)
  246. else:
  247. _core._reset_server_error(server_name)
  248. return result
  249. def _strike(server_name: str, message: str, **extra) -> str:
  250. """Breaker strike + the ``tool_error`` payload for *message*."""
  251. _core._bump_server_error(server_name)
  252. return tool_error(message, **extra)
  253. def _mcp_loop_running() -> bool:
  254. return _core._mcp_loop is not None and _core._mcp_loop.is_running()
  255. def _lookup_reconnectable_server(server_name: str, require_loop: bool = False):
  256. """The registered server object when it can be signalled to reconnect, else None.
  257. With *require_loop*, also None unless the MCP loop is running (nothing to wait on)."""
  258. from tools.mcp_tool_scope import _resolve_server_key
  259. with _core._lock:
  260. srv = _core._servers.get(_resolve_server_key(server_name))
  261. ok = srv is not None and hasattr(srv, "_reconnect_event") and (_mcp_loop_running() or not require_loop)
  262. return srv if ok else None
  263. def _retry_once(server_name: str, retry_call, op_description: str, what: str):
  264. """Re-run ``retry_call`` after a recovery step. Returns the result when the RPC completed
  265. (an application error is still the tool's real answer, and still a breaker strike per #10447);
  266. None when the retry raised (caller falls through)."""
  267. try:
  268. result = retry_call()
  269. except Exception as retry_exc:
  270. logger.warning("MCP %s/%s retry after %s failed: %s", server_name, op_description, what, retry_exc)
  271. return None
  272. return _record_call_outcome(server_name, result)
  273. def _handle_auth_error_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str):
  274. """OAuth recovery + one retry; None when *exc* is not an auth error. ``handle_401`` decides
  275. viability; if viable, signal a reconnect (fresh credentials), wait ready, retry once. Any
  276. failure returns the structured ``needs_reauth`` error so the model stops refreshing."""
  277. if not _is_auth_error(exc):
  278. return None
  279. from tools.mcp_oauth_manager import get_manager
  280. try:
  281. recovered = _loop._run_on_mcp_loop(lambda: get_manager().handle_401(server_name, None), timeout=10)
  282. except Exception as rec_exc:
  283. logger.warning("MCP OAuth '%s': recovery attempt failed: %s", server_name, rec_exc)
  284. recovered = False
  285. if recovered:
  286. srv = _lookup_reconnectable_server(server_name)
  287. # Recovery + reconnect is independent evidence of viability: close the breaker here, not only on
  288. # retry success (else a failing retry pins it open forever).
  289. if srv is not None and _loop._signal_reconnect_and_wait(
  290. server_name, srv, op_description=f"{op_description} after OAuth recovery", timeout=15):
  291. _core._reset_server_error(server_name)
  292. result = _retry_once(server_name, retry_call, op_description, "auth recovery")
  293. if result is not None:
  294. return result
  295. return _strike(server_name, _NEEDS_REAUTH_MSG.format(s=server_name, detail=_auth_error_detail(exc)),
  296. needs_reauth=True, server=server_name)
  297. def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str,
  298. *, call_may_have_side_effects: bool = False):
  299. """Transport reconnect + one retry on session expiry; None to fall through. Skips
  300. ``handle_401``: the token is valid, only the server-side session is stale.
  301. Unlike :func:`_handle_auth_error_and_retry`, this does **not** call the OAuth manager's ``handle_401`` —
  302. the access token is still valid, only the server-side session state is stale. Setting
  303. ``_reconnect_event`` causes the server task's lifecycle loop to tear down the current
  304. ``streamablehttp_client`` + ``ClientSession`` and rebuild them, reusing the existing OAuth provider
  305. instance. See #13383.
  306. At-most-once for writes: a session-expired shape can be synthesized by a proxy AFTER the upstream
  307. executed the request, and the transport errors this classifier also matches (``ClosedResourceError``,
  308. broken pipe) routinely fire mid-response. With ``call_may_have_side_effects`` the transport is still
  309. healed but the call is never re-run; the model gets an ``outcome_uncertain`` error instead (same
  310. contract as the mid-call stdio death path). Callers pass True unless the tool is positively read-only.
  311. """
  312. if not _is_session_expired_error(exc):
  313. return None
  314. srv = _lookup_reconnectable_server(server_name, require_loop=True)
  315. if call_may_have_side_effects:
  316. # Even without a signallable server the outcome is still uncertain: a generic "call failed"
  317. # would invite the model to re-invoke a write that may already have landed.
  318. reconnected = srv is not None and _loop._signal_reconnect_and_wait(
  319. server_name, srv, op_description=f"{op_description} (write, no auto-retry)", timeout=15)
  320. if reconnected: # session state failed, not server health: no breaker strike
  321. _core._reset_server_error(server_name)
  322. else:
  323. _core._bump_server_error(server_name)
  324. logger.warning("MCP server '%s': %s failed with a session-expired/transport error after the request may "
  325. "have been dispatched; NOT auto-retrying a write-capable tool (reconnect %s).",
  326. server_name, op_description, "succeeded" if reconnected else "failed")
  327. return tool_error(_SESSION_OUTCOME_UNCERTAIN_MSG.format(
  328. s=server_name, state="been re-established" if reconnected else "not recovered yet"),
  329. outcome_uncertain=True, server=server_name)
  330. if srv is None:
  331. return None
  332. logger.info("MCP server '%s': %s failed with session-expired error (%s); signalling transport reconnect "
  333. "and retrying once.", server_name, op_description, exc)
  334. if not _loop._signal_reconnect_and_wait(server_name, srv, op_description=op_description, timeout=15):
  335. logger.warning("MCP server '%s': reconnect did not ready within 15s after session-expired error; "
  336. "falling through to error response.", server_name)
  337. return None
  338. return _retry_once(server_name, retry_call, op_description, "session reconnect")
  339. class _StdioChildExited(RuntimeError):
  340. """Stdio subprocess gone when (or while) a call ran. Deliberately NOT a TimeoutError."""
  341. def __init__(self, message: str, *, in_flight: bool):
  342. super().__init__(message)
  343. self.in_flight = in_flight
  344. def _handle_stdio_child_exited_and_retry(server_name: str, exc: Exception, retry_call, op_description: str):
  345. """Respawn a dead stdio child; retry once only when it was dead before dispatch.
  346. A mid-call exit is ambiguous: the server may have applied a side effect before its
  347. response pipe disappeared. Reconnect for future calls but never replay that operation.
  348. None means this is not our error. This function never spawns itself: it sets
  349. ``_reconnect_event`` and waits, so spawn frequency stays governed by ``run()``'s rapid-drop
  350. budget. A pre-dispatch retry whose child dies again reports and stops.
  351. Why retrying here cannot hot-cycle respawns: this function never spawns anything. It sets
  352. ``_reconnect_event`` (one signal, same as before) and waits for the server task to publish a fresh
  353. session. Spawn frequency stays governed entirely by ``run()``'s rapid-drop budget, which parks a
  354. transport that keeps dropping without proving healthy (#62212).
  355. """
  356. if not isinstance(exc, _StdioChildExited):
  357. return None
  358. reconnected = False
  359. srv = _lookup_reconnectable_server(server_name)
  360. if srv is not None:
  361. action = "reconnecting without replay" if exc.in_flight else "respawning and retrying once"
  362. logger.info("MCP server '%s': %s found the stdio subprocess dead (%s); %s.",
  363. server_name, op_description, exc, action)
  364. if _mcp_loop_running():
  365. reconnected = _loop._signal_reconnect_and_wait(
  366. server_name, srv, op_description=op_description, timeout=_core._STDIO_RESPAWN_WAIT_SEC)
  367. else: # No MCP loop to wait on (non-async adapters, tests): still request the respawn.
  368. _loop._signal_reconnect(srv)
  369. if exc.in_flight:
  370. return _strike(
  371. server_name,
  372. _STDIO_OUTCOME_UNCERTAIN_MSG.format(s=server_name),
  373. outcome_uncertain=True,
  374. )
  375. if not reconnected:
  376. return _strike(server_name, _STDIO_NO_RESPAWN_MSG.format(s=server_name, t=_core._STDIO_RESPAWN_WAIT_SEC))
  377. try:
  378. return _record_call_outcome(server_name, retry_call())
  379. except _StdioChildExited as retry_exc:
  380. # Died again right after respawn: broken server; run()'s budget takes it to the park.
  381. logger.warning("MCP server '%s': %s stdio subprocess exited again right after respawn (%s); not retrying "
  382. "further.", server_name, op_description, retry_exc)
  383. if retry_exc.in_flight:
  384. return _strike(
  385. server_name,
  386. _STDIO_OUTCOME_UNCERTAIN_MSG.format(s=server_name),
  387. outcome_uncertain=True,
  388. )
  389. return _strike(server_name, _STDIO_DIED_AGAIN_MSG.format(s=server_name))
  390. except Exception as retry_exc:
  391. logger.warning("MCP %s/%s retry after stdio respawn failed: %s", server_name, op_description, retry_exc)
  392. return _strike(server_name, _sanitize_error(
  393. f"MCP call failed after respawning the stdio subprocess for '{server_name}': "
  394. f"{type(retry_exc).__name__}: {_exc_str(retry_exc)}"))
  395. def _dispatch(server_name: str, server: Any, op: str, call, tool_timeout: float, recoverers,
  396. on_final_failure: Callable[[BaseException], None], record_outcome: bool = False) -> str:
  397. """Mark the call started on *server* (doubles may lack ``mark_tool_call``), run coroutine function *call*
  398. on the MCP loop and, on failure, walk ``recoverers`` (``(server_name, exc, retry_call, op) -> Optional[str]``,
  399. None = not its kind; order matters). Unrecovered exceptions go through ``on_final_failure`` and become the
  400. generic call-failed error. ``record_outcome`` applies breaker bookkeeping to the FIRST attempt only."""
  401. if callable(getattr(server, "mark_tool_call", None)):
  402. server.mark_tool_call()
  403. def call_once():
  404. return _loop._run_on_mcp_loop(call, timeout=tool_timeout)
  405. try:
  406. result = call_once()
  407. return _record_call_outcome(server_name, result) if record_outcome else result
  408. except InterruptedError:
  409. return tool_error("MCP call interrupted: user sent a new message")
  410. except Exception as exc:
  411. for recover in recoverers:
  412. recovered = recover(server_name, exc, call_once, op)
  413. if recovered is not None:
  414. return recovered
  415. on_final_failure(exc)
  416. return tool_error(_mcp_call_failed_message(exc))
  417. @asynccontextmanager
  418. async def _track_inflight_rpc(server: Any, server_name: str, op: str, *, retry_safe: bool = True):
  419. """Register the running RPC so teardown can fail it fast. A deliberate teardown
  420. (``_reconnecting`` set first) turns the cancel into a retryable RuntimeError; external
  421. cancels propagate unchanged. Doubles without ``_inflight_tasks`` skip tracking.
  422. Every user-visible request family wraps its RPC in this context (#48069 salvage). If a deliberate
  423. reconnect/shutdown teardown cancels the task (``_fail_inflight_calls`` sets ``_reconnecting`` first),
  424. the cancel is converted into a clean retryable RuntimeError instead of a raw CancelledError; external
  425. cancels (caller timeout, user interrupt) propagate unchanged. ``retry_safe=False`` (a write-capable
  426. ``tools/call``) words the error as outcome-uncertain instead of inviting a replay.
  427. """
  428. inflight, task = getattr(server, "_inflight_tasks", None), asyncio.current_task()
  429. tracked = task is not None and inflight is not None
  430. if tracked:
  431. inflight.add(task)
  432. try:
  433. yield
  434. except asyncio.CancelledError:
  435. if getattr(server, "_reconnecting", False):
  436. advice = ("retry the request on the rebuilt session" if retry_safe else
  437. "the request may already have been dispatched, so verify its effect before re-invoking")
  438. raise RuntimeError(f"MCP {op} on '{server_name}' was aborted by a reconnect teardown; {advice}") from None
  439. raise
  440. finally:
  441. if tracked:
  442. inflight.discard(task)
  443. async def _call_tool_racing_stdio_death(server, server_name: str, tool_name: str, args: dict, progress_callback=None):
  444. """``session.call_tool`` that fails fast when the stdio child is/gets dead: pre-call (a dead
  445. child must not hold the slot for the full timeout) and mid-call (race against
  446. ``_watch_stdio_children``). Both raise :class:`_StdioChildExited` for the respawn path, which
  447. owns the reconnect signal; only pre-call failure is safe to replay. callable()/``is True``
  448. because MagicMock attributes are truthy."""
  449. # Fast-fail (#81995): a stdio subprocess that is already dead must not own this call slot — fail
  450. # immediately instead of waiting out the full tool timeout on a transport nobody will ever answer.
  451. _stdio_dead = getattr(server, "_stdio_children_dead", None)
  452. if callable(_stdio_dead) and _stdio_dead() is True:
  453. raise _StdioChildExited(
  454. f"MCP stdio subprocess for '{server_name}' had already exited when the call was dispatched",
  455. in_flight=False,
  456. )
  457. # A progress callback makes the SDK send ``_meta.progressToken``, the spec's opt-in for
  458. # ``notifications/progress``; without one the server must stay silent.
  459. extra = {"progress_callback": progress_callback} if progress_callback is not None else {}
  460. _call_coro = server.session.call_tool(tool_name, arguments=args, **extra)
  461. _watch_children = getattr(server, "_watch_stdio_children", None)
  462. if not (inspect.iscoroutinefunction(_watch_children) and asyncio.iscoroutine(_call_coro)):
  463. # Stubbed sessions return a non-awaitable, or there is no child-watcher to race: plain await.
  464. return await _call_coro if asyncio.iscoroutine(_call_coro) else _call_coro
  465. # Fast-fail machinery (#81995): the RPC races a stdio-children watcher so a dead subprocess fails the
  466. # call immediately instead of riding out the full tool timeout.
  467. rpc_task = asyncio.ensure_future(_call_coro)
  468. watch_task = asyncio.ensure_future(_watch_children())
  469. try:
  470. done, _pending = await asyncio.wait({rpc_task, watch_task}, return_when=asyncio.FIRST_COMPLETED)
  471. if watch_task in done and not rpc_task.done():
  472. rpc_task.cancel()
  473. raise _StdioChildExited(
  474. f"MCP stdio subprocess for '{server_name}' exited mid-call",
  475. in_flight=True,
  476. )
  477. try:
  478. return await rpc_task
  479. except Exception as exc:
  480. # The SDK usually sees the closed pipe before the 250 ms watcher poll does. On a stdio
  481. # server a transport-closure error after dispatch is the same ambiguous mid-call death;
  482. # it must not fall through to the session-expired recoverer, which replays the call.
  483. _is_http = getattr(server, "_is_http", None)
  484. if callable(_is_http) and _is_http() is False and _is_session_expired_error(exc):
  485. raise _StdioChildExited(
  486. f"MCP stdio subprocess for '{server_name}' closed its transport mid-call",
  487. in_flight=True,
  488. ) from exc
  489. raise
  490. finally:
  491. watch_task.cancel()
  492. if not rpc_task.done():
  493. rpc_task.cancel()
  494. await asyncio.gather(rpc_task, watch_task, return_exceptions=True)
  495. # ---------------------------------------------------------- result rendering
  496. def _error_result_text(result) -> str:
  497. """Concatenated text of an ``isError`` result's blocks (EmbeddedResource error payloads
  498. carry text under ``.resource.text``)."""
  499. texts = (getattr(b, "text", None) or getattr(getattr(b, "resource", None), "text", None) for b in (result.content or []))
  500. return "".join(str(t) for t in texts if t)
  501. def _render_content_blocks(result, server_name: str) -> tuple[str, int, list[str]]:
  502. """Text passes through; image/audio blocks are cached (MEDIA: tags); resource blocks are
  503. materialized rather than silently dropped; unsupported blocks become an inline drop notice
  504. (kimi-code#3227). Returns ``(text, usable_parts, image_paths)``: ``usable_parts`` counts REAL
  505. rendered blocks (whitespace-only text and drop notices excluded) for the structuredContent
  506. arbitration; ``image_paths`` are the files THIS call's image blocks were cached to — never
  507. parsed back out of the text, where a server-written ``MEDIA:`` line could name any local file."""
  508. parts: list[str] = []
  509. usable_parts = 0
  510. image_paths: list[str] = []
  511. # MCP tool results can also include ImageContent blocks (screenshot / Blockbench / Playwright etc.);
  512. # cache those via the gateway's image-cache helper so they flow through Hermes' MEDIA: tag convention
  513. # and out to messaging adapters that render images natively. Without this, image blocks were silently
  514. # dropped and the agent got an empty response. Distilled from #17915 (c3115644151) and #10848
  515. # (gnanirahulnutakki), both too stale to cherry-pick. #10848's approach (integrate with Hermes' MEDIA
  516. # tag + cache_image_from_bytes) was the cleaner of the two — plugs into existing infrastructure.
  517. for block in (result.content or []):
  518. if getattr(block, "text", None):
  519. parts.append(strip_unicode_tags(block.text))
  520. if block.text.strip():
  521. usable_parts += 1
  522. continue
  523. image = _cache_mcp_image_block(block)
  524. if image.startswith("MEDIA:"):
  525. image_paths.append(image[len("MEDIA:"):])
  526. rendered = image or _cache_mcp_audio_block(block) or _render_mcp_resource_block(block, server_name)
  527. if rendered:
  528. parts.append(rendered)
  529. usable_parts += 1
  530. continue
  531. block_type = getattr(block, "type", None) or type(block).__name__
  532. if block_type in {"text", "resource", "audio", "image"}: # benign empty render
  533. logger.debug("MCP %s: content block type %r rendered empty", server_name, block_type)
  534. else:
  535. logger.warning("MCP %s: dropping unsupported content block type %r", server_name, block_type)
  536. # Surface the drop to the MODEL, not just the log: a silent drop leaves the agent
  537. # believing the tool returned less than it did, with no way to recover.
  538. parts.append(_render_mcp_dropped_block_notice(block, block_type))
  539. # Hard-cap pathological payloads; ordinary large results pass to spillover.
  540. return _truncate_mcp_text_result("\n".join(parts)), usable_parts, image_paths
  541. def _capped_structured_content(result):
  542. """``structuredContent`` (or None); over the hard cap it degrades to the head+tail
  543. truncated JSON string (multi-MB JSON flood guard)."""
  544. # Hard-cap pathological payloads before they propagate (#56059); ordinary large results pass untouched
  545. # to the spillover layer. Arbitration against ``content`` lives in _render_call_tool_result.
  546. # Server-level `_meta` is also surfaced (ported from
  547. # MoonshotAI/kimi-code#2596): servers return namespaced metadata there (validated contracts,
  548. # browser-handoff payloads, ...) that was previously invisible to the agent. Protocol-reserved keys are
  549. # dropped first (kimi-code#2600) — per the MCP spec's key-name rules a prefix is reserved when a
  550. # `modelcontextprotocol` or `mcp` label is followed by at least one more label (e.g.
  551. # `modelcontextprotocol.io/...`, `tools.mcp.com/...`); those carry host/protocol plumbing, not
  552. # model-facing data. Unprefixed and vendor-namespaced keys (`com.example.mcp/...`) pass through — their
  553. # semantics belong to the server.
  554. structured = mcp_field(result, "structured_content", "structuredContent")
  555. try:
  556. as_json = json.dumps(structured, ensure_ascii=False, default=str) if structured is not None else ""
  557. except (TypeError, ValueError):
  558. return structured
  559. return _truncate_mcp_text_result(as_json) if len(as_json) > _MCP_HARD_RESULT_CAP_CHARS else structured
  560. def _content_dual_emits_structured(result, structured) -> bool:
  561. """True when some text block is ``structuredContent`` serialized as JSON — the spec's
  562. backwards-compat dual-emit ("a tool that returns structured content SHOULD also return the
  563. serialized JSON in a TextContent block"). Compared as parsed JSON so whitespace, indent, key
  564. order and ``ensure_ascii`` escaping do not matter; checked per block because the spec puts the
  565. copy in *a* block and a server may add a status line next to it. Deterministic equality, not a
  566. richness heuristic: a prose summary or a reorganised rendering fails it and keeps its
  567. ``structuredContent`` (#115430)."""
  568. for block in (result.content or []):
  569. text = getattr(block, "text", None)
  570. if not text:
  571. continue
  572. try:
  573. if json.loads(text) == structured:
  574. return True
  575. except (TypeError, ValueError):
  576. continue
  577. return False
  578. def _render_call_tool_result(result, server_name: str, image_paths: Optional[list[str]] = None) -> str:
  579. """Pure: ``CallToolResult`` -> handler JSON (``image_paths``, when given, receives the files this
  580. call's image blocks were cached to). ``content`` and ``structuredContent`` are both
  581. forwarded, except that a ``structuredContent`` whose JSON also sits verbatim in a text block
  582. (the spec's backwards-compat dual-emit; compared as parsed JSON) is dropped, because that copy
  583. would reach the model twice (kimi-code#3234). Any other usable text — a status line, a prose
  584. summary, a reorganised rendering — keeps ``structuredContent`` alongside it (#115430): the
  585. earlier "content wins whenever it rendered anything" rule irreversibly lost servers whose
  586. data lived only in ``structuredContent``, while the residual duplicate for a faithful
  587. reorganisation costs only tokens, so data-preservation wins. No richness or size heuristic is
  588. used. ``structuredContent`` fills ``result`` when the blocks rendered effectively empty
  589. (structuredContent-only servers); ``_meta`` minus reserved keys is always surfaced."""
  590. if mcp_field(result, "is_error", "isError", False):
  591. return tool_error(_sanitize_error(_truncate_mcp_text_result(_error_result_text(result) or "MCP tool returned an error")))
  592. text_result, usable_parts, cached_images = _render_content_blocks(result, server_name)
  593. if image_paths is not None:
  594. image_paths.extend(cached_images)
  595. structured = _capped_structured_content(result)
  596. meta = _strip_reserved_meta_keys(mcp_field(result, "meta", "meta"))
  597. # A str here is the over-cap truncation stand-in (wire structuredContent is always an object): next to
  598. # usable text it would be a second multi-MB copy — the flood #56059 caps — so it only fills an empty result.
  599. if structured is not None and usable_parts > 0 and (isinstance(structured, str) or _content_dual_emits_structured(result, structured)):
  600. structured = None
  601. if structured is None and meta is None:
  602. return json.dumps({"result": text_result}, ensure_ascii=False)
  603. # Key order is part of the output: "result" leads when there is text, otherwise "_meta" precedes it.
  604. payload: dict[str, Any] = {"result": text_result} if text_result else {}
  605. # Cap structuredContent too — a malicious server could flood context via a multi-MB JSON payload
  606. # (#56059). When the serialized form exceeds the hard cap, replace it with the truncated string (head +
  607. # tail preserved) so it degrades gracefully instead of flooding downstream.
  608. if structured is not None:
  609. payload["structuredContent" if text_result else "result"] = structured
  610. if meta is not None:
  611. payload["_meta"] = meta
  612. payload.setdefault("result", text_result)
  613. try:
  614. return json.dumps(payload, ensure_ascii=False)
  615. except (TypeError, ValueError): # Non-serializable metadata: drop the extras, keep the call.
  616. return json.dumps({"result": text_result}, ensure_ascii=False)
  617. def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float, *, native_images: bool = False):
  618. """Sync handler (``handler(args_dict, **kwargs)``) calling an MCP tool via the background loop. Always a JSON
  619. string, except with ``native_images`` (the registry handler the agent loop dispatches): a result carrying
  620. images then becomes the ``_multimodal`` envelope when the active route takes images in tool results.
  621. Direct callers (plugins' ``ctx.call_mcp``) keep the string contract and the ``MEDIA:`` paths."""
  622. op = f"tools/call {tool_name}"
  623. def _handler(args: dict, **kwargs) -> Any:
  624. # Security boundary: untrusted-server write tools need approval before ANY transport work (incl. lazy spawn).
  625. booked: list = [] # a costed call's charge; refunded when the call never reaches the server
  626. error = _trust_gate_check(server_name, tool_name, booked, args)
  627. if error is not None:
  628. return error
  629. server, error = None, _check_circuit_breaker(server_name)
  630. if error is None:
  631. server, error = _acquire_call_server(server_name, tool_timeout)
  632. if server is None:
  633. for charge in booked:
  634. charge.refund()
  635. return error
  636. # Only a tool annotated readOnlyHint=True is replayed after session expiry; a 401 is always
  637. # pre-dispatch so the auth recoverer keeps its retry for every tool.
  638. read_only = _tool_is_read_only(server_name, tool_name)
  639. image_paths: list[str] = [] # the LAST attempt's cached images (a recoverer may retry _call)
  640. # Read on the agent's thread: the coroutine below runs on the MCP loop, outside this context.
  641. from agent.tool_progress_relay import current_tool_progress
  642. report = current_tool_progress()
  643. async def _on_progress(progress, total, message):
  644. if report is not None:
  645. report(progress, total, message)
  646. async def _call():
  647. async with server._rpc_lock, _track_inflight_rpc(server, server_name, op, retry_safe=read_only):
  648. server._pending_call_context = contextvars.copy_context() # for the elicitation callback
  649. try:
  650. result = await _call_tool_racing_stdio_death(
  651. server, server_name, tool_name, args, _on_progress if report is not None else None)
  652. finally:
  653. server._pending_call_context = None
  654. if getattr(server, "_mark_session_proven", None) is not None: # round-trip done: transport healthy
  655. server._mark_session_proven()
  656. image_paths.clear()
  657. return _render_call_tool_result(result, server_name, image_paths)
  658. def _on_failure(exc):
  659. _core._bump_server_error(server_name)
  660. logger.error("MCP tool %s/%s call failed: %s", server_name, tool_name, exc)
  661. session_expired = partial(_handle_session_expired_and_retry, call_may_have_side_effects=not read_only)
  662. text = _dispatch(
  663. server_name, server, op, _call, tool_timeout,
  664. (_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, session_expired),
  665. _on_failure, record_outcome=True)
  666. # Off the MCP loop on purpose: image prep is CPU work (bounded vision pool) that must not stall other servers' I/O.
  667. return _mcp_result_with_native_images(text, image_paths) if native_images and not _result_is_error(text) else text
  668. return _handler
  669. def _make_utility_handler(op: str, log_label: str, rpc, render, required: Optional[str] = None):
  670. """``(server_name, tool_timeout) -> sync handler`` for one utility tool: ``rpc(session, args,
  671. server_name)`` awaited under ``_rpc_lock``, ``render(result, server_name)`` -> JSON-able
  672. payload, ``required`` validated before any transport work."""
  673. def _factory(server_name: str, tool_timeout: float):
  674. def _handler(args: dict, **kwargs) -> str:
  675. from tools import mcp_tool_discovery as _discovery # lazy: import cycle
  676. server = _discovery._get_connected_server_for_call(server_name)
  677. if not server or not server.session:
  678. return tool_error(f"MCP server '{server_name}' is not connected")
  679. if required and not args.get(required):
  680. return tool_error(f"Missing required parameter '{required}'")
  681. async def _call():
  682. async with server._rpc_lock:
  683. result = await rpc(server.session, args, server_name)
  684. return json.dumps(render(result, server_name), ensure_ascii=False)
  685. return _dispatch(
  686. server_name, server, op, _call, tool_timeout,
  687. (_handle_auth_error_and_retry, _handle_session_expired_and_retry),
  688. lambda exc: logger.error("MCP %s/%s failed: %s", server_name, log_label, exc))
  689. return _handler
  690. return _factory
  691. def _pick(obj, *specs) -> dict:
  692. """``{out_key: value}`` for each ``(out_key, attr[, truthy])`` present on *obj* (presence check so SDK models
  693. and stubs behave alike; ``truthy`` also skips falsy). Key order = spec order."""
  694. entry = {}
  695. for out_key, attr, *truthy in specs:
  696. value = getattr(obj, attr, _MISSING)
  697. if value is not _MISSING and (value or not (truthy and truthy[0])):
  698. entry[out_key] = value
  699. return entry
  700. def _render_resource_list(all_resources, server_name: str) -> dict:
  701. resources = []
  702. for r in all_resources:
  703. entry = _pick(r, ("uri", "uri"), ("name", "name"), ("description", "description", True))
  704. if "uri" in entry:
  705. entry["uri"] = str(entry["uri"])
  706. mime = mcp_field(r, "mime_type", "mimeType")
  707. if mime:
  708. entry["mimeType"] = mime # camelCase: this is the tool's own JSON output shape
  709. resources.append(entry)
  710. return {"resources": resources}
  711. def _render_read_resource(result, server_name: str) -> dict:
  712. parts: list[str] = []
  713. for block in getattr(result, "contents", []):
  714. if getattr(block, "text", None) is not None:
  715. parts.append(strip_unicode_tags(block.text))
  716. elif getattr(block, "blob", None) is not None: # binary -> document cache, like EmbeddedResource blocks
  717. rendered = _render_mcp_resource_block(SimpleNamespace(type="resource", resource=block), server_name)
  718. parts.append(rendered or f"[binary data, {len(block.blob)} bytes]")
  719. return {"result": "\n".join(parts)}
  720. def _render_prompt_list(all_prompts, server_name: str) -> dict:
  721. prompts = []
  722. for p in all_prompts:
  723. entry = _pick(p, ("name", "name"), ("description", "description", True))
  724. if getattr(p, "arguments", None):
  725. entry["arguments"] = [{"name": a.name, **_pick(a, ("description", "description", True), ("required", "required"))}
  726. for a in p.arguments]
  727. prompts.append(entry)
  728. return {"prompts": prompts}
  729. def _render_get_prompt(result, server_name: str) -> dict:
  730. messages = []
  731. for msg in getattr(result, "messages", []):
  732. entry = _pick(msg, ("role", "role"))
  733. if hasattr(msg, "content"):
  734. entry["content"] = strip_unicode_tags(msg.content.text if hasattr(msg.content, "text") else str(msg.content))
  735. messages.append(entry)
  736. return {"messages": messages, **_pick(result, ("description", "description", True))}
  737. _make_list_resources_handler = _make_utility_handler(
  738. "resources/list", "list_resources",
  739. lambda session, args, sn: _core._paginate_full_list(session.list_resources, "resources", sn), _render_resource_list)
  740. _make_read_resource_handler = _make_utility_handler(
  741. "resources/read", "read_resource",
  742. lambda session, args, sn: session.read_resource(args["uri"]), _render_read_resource, required="uri")
  743. _make_list_prompts_handler = _make_utility_handler(
  744. "prompts/list", "list_prompts",
  745. lambda session, args, sn: _core._paginate_full_list(session.list_prompts, "prompts", sn), _render_prompt_list)
  746. _make_get_prompt_handler = _make_utility_handler(
  747. "prompts/get", "get_prompt",
  748. lambda session, args, sn: session.get_prompt(args["name"], arguments=args.get("arguments", {})),
  749. _render_get_prompt, required="name")
  750. def _make_check_fn(server_name: str):
  751. """Connection-alive check; lazy (schema-cache registered) servers count as available.
  752. When the server's owner registered an application declaration (`requires.app`), the
  753. application must also be present on this host, or the tools are not offered even while a
  754. stale connection lingers. With no declaration registered the check is the connection check
  755. alone. Returns a plain bool: the registry caches ``bool(fn())``.
  756. """
  757. from tools.mcp_tool_scope import _resolve_server_key
  758. def _connected() -> bool:
  759. with _core._lock:
  760. key = _resolve_server_key(server_name)
  761. server = _core._servers.get(key)
  762. return ((server is not None and (server.session is not None or server._is_recycled_stdio()))
  763. or key in _core._lazy_server_configs)
  764. def _check() -> bool:
  765. if not _connected():
  766. return False
  767. return _declared_app_offerable(server_name)
  768. return _check
  769. def _declared_app_offerable(server_name: str) -> bool:
  770. """True unless the registered declaration is unavailable on this host. Called only for a
  771. connected server, so a reachable loopback port outranks the interactive-session rule."""
  772. from hermes_platform import declaration
  773. from hermes_platform.resolver.availability import availability
  774. decl = declaration.lookup(server_name)
  775. if decl is None:
  776. return True
  777. return bool(availability(decl).offerable)