| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879 |
- """Registry-facing sync handlers for MCP tools and utility tools (resources/prompts), plus the per-call recovery
- ladder: trust gating, circuit breaker, auth (401) refresh, session-expired reconnect and dead-stdio respawn retry."""
- import logging
- import asyncio
- import contextvars
- import inspect
- import json
- import time
- from contextlib import asynccontextmanager
- from functools import partial
- from types import SimpleNamespace
- from typing import Any, Callable, Dict, List, Optional, Tuple
- from hermes_platform import declaration
- from tools.registry import invalidate_check_fn_cache, tool_error
- from tools.ansi_strip import strip_unicode_tags
- from tools.mcp_tool_common import _exc_str, _sanitize_error, mcp_field, _core
- from tools import mcp_tool_loop as _loop
- from tools.mcp_tool_content import (
- _MCP_HARD_RESULT_CAP_CHARS, _cache_mcp_audio_block, _cache_mcp_image_block, _mcp_result_with_native_images,
- _render_mcp_dropped_block_notice, _render_mcp_resource_block, _strip_reserved_meta_keys,
- _truncate_mcp_text_result)
- from tools.mcp_tool_errors import (
- _auth_error_detail, _is_auth_error, _is_session_expired_error, _mcp_call_failed_message)
- logger = logging.getLogger("tools.mcp_tool")
- _MISSING = object()
- declaration.on_change = invalidate_check_fn_cache
- _NEEDS_REAUTH_MSG = (
- "MCP server '{s}' requires re-authentication.{detail} Run `hermes mcp login {s}` (or delete the tokens file under "
- "~/.hermes/mcp-tokens/ and restart). Do NOT retry this tool — ask the user to re-authenticate.")
- _STDIO_NO_RESPAWN_MSG = (
- "MCP server '{s}' stdio subprocess had exited (this is not a timeout — the call never reached the server). A "
- "respawn was requested but no fresh session came back within {t:.0f}s. Wait a few seconds before retrying; if it "
- "keeps failing the server is not starting and needs the user.")
- _STDIO_DIED_AGAIN_MSG = (
- "MCP server '{s}' respawned its stdio subprocess and it exited again immediately. The server is not starting "
- "cleanly — do NOT retry this tool; ask the user to check the server's command and its stderr log.")
- _STDIO_OUTCOME_UNCERTAIN_MSG = (
- "MCP server '{s}' lost its stdio subprocess after the tool call began. The operation may have completed, so "
- "Hermes did not replay it. Do NOT retry automatically; inspect the external state first.")
- _SESSION_OUTCOME_UNCERTAIN_MSG = (
- "The MCP transport session to '{s}' expired while this write-capable call was in flight, so the outcome is "
- "UNKNOWN — the operation may or may not have taken effect server-side. It was NOT automatically retried to "
- "avoid a duplicate side effect. The connection has {state}. Verify whether the operation took effect (e.g. "
- "with a read-only tool) before re-invoking it.")
- def _tool_is_read_only(server_name: str, tool_name: str) -> bool:
- """True only when discovery captured ``readOnlyHint=True`` for the tool. Missing or malformed
- metadata fails safe to False (treated as write-capable). readOnlyHint is a property of the
- connection's tools, so it lives under the connection key."""
- from tools.mcp_tool_scope import _resolve_server_key
- return _core._tool_read_only_hints.get(_resolve_server_key(server_name), {}).get(tool_name) is True
- def _trust_approval_key(server_name: str, tool_name: str) -> str:
- """Approval key a session/always answer to the trust gate is kept under: one (server, tool) pair,
- so allowing one write tool never covers the server's others. Session entries are per chat;
- always entries land in the active profile's ``command_allowlist``."""
- return f"mcp:{server_name}/{tool_name}"
- def _trust_granted(server_name: str, tool_name: str) -> bool:
- """A remembered Session/Always answer, or YOLO / ``approvals.mode: off`` for the calling session.
- Covers only the trust prompt: the spend limit is a budget, not an approval, and still asks."""
- from tools.approval import is_approval_bypass_active, is_approved
- from tools.approval_context import get_current_session_key
- return is_approval_bypass_active() or is_approved(get_current_session_key(),
- _trust_approval_key(server_name, tool_name))
- # Keeps the whole trust-gate card inside Discord's 2,000-character message cap (the tightest button
- # surface), so the truncation marker is never cut off by the adapter's own fit.
- _TRUST_GATE_ARGUMENTS_MAX_CHARS = 1_200
- _TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER = "\n... [arguments truncated]"
- def _is_secret_argument_name(key: Any) -> bool:
- from agent.redact import SECRET_HEADER_NAME_LIST, is_secret_field_name
- return is_secret_field_name(key) or (isinstance(key, str) and key.casefold() in SECRET_HEADER_NAME_LIST)
- def _mask_secret_fields(value: Any) -> Any:
- """Mask every value under a credential-named field (``db_password``, ``client_secret``) or API-key
- header name (``x-api-key``), whatever its type: the text redactor below only catches the
- ``key: value`` and token shapes it knows."""
- if isinstance(value, dict):
- return {k: "***" if v not in (None, "") and _is_secret_argument_name(k) else _mask_secret_fields(v)
- for k, v in value.items()}
- if isinstance(value, list):
- return [_mask_secret_fields(v) for v in value]
- return value
- def _format_trust_gate_arguments(args: dict) -> str:
- """Redacted, bounded JSON of the call's arguments for the approval card. Plain text, no
- fence: every surface already renders the prompt inside a code block."""
- import unicodedata
- from agent.redact import redact_sensitive_text
- rendered = json.dumps(_mask_secret_fields(args), ensure_ascii=False, indent=2, default=str)
- # Scan and redact only a window well past the cap: a multi-megabyte payload (a file write) would
- # otherwise hold the prompt back for seconds.
- window = 4 * _TRUST_GATE_ARGUMENTS_MAX_CHARS
- cut, rendered = len(rendered) > window, rendered[:window]
- # Bidi overrides, invisible format characters and the C1/DEL controls JSON leaves raw could make the
- # card show something other than what the server receives; render them as escapes.
- rendered = "".join(ch.encode("unicode_escape").decode()
- if ch != "\n" and unicodedata.category(ch) in ("Cc", "Cf") else ch for ch in rendered)
- # force: the approval prompt is a display boundary even with security.redact_secrets off. The CLI
- # prints it as-is. Redact before truncating so a cut cannot split a secret past its pattern.
- rendered = redact_sensitive_text(rendered, force=True)
- if cut or len(rendered) > _TRUST_GATE_ARGUMENTS_MAX_CHARS:
- keep = _TRUST_GATE_ARGUMENTS_MAX_CHARS - len(_TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER)
- rendered = rendered[:keep].rstrip() + _TRUST_GATE_ARGUMENTS_TRUNCATION_MARKER
- return rendered
- def _trust_gate_check(server_name: str, tool_name: str, booked: Optional[list] = None,
- args: Optional[dict] = None) -> Optional[str]:
- """Approval gate for write-capable tools on ``trust: untrusted`` servers, plus the declared spend
- limit on any server. None to proceed, else a ``tool_error``. Fail-closed: approval-system errors
- block. An admitted costed call is charged to the chat's tally and appended to *booked* so the
- caller can refund it when the call never reaches the server."""
- from tools.mcp_tool_scope import _server_key
- from tools import mcp_tool_costs as _costs
- # Trust and costs are the calling profile's own policy (an adopter of a shared connection keeps its own).
- key = _server_key(server_name)
- trust = _core._server_trust_levels.get(key, _core._TRUST_FULL)
- gated = (trust == _core._TRUST_UNTRUSTED and not _tool_is_read_only(server_name, tool_name)
- and not _trust_granted(server_name, tool_name))
- charge = None
- if key in _core._server_cost_policies: # no table: not even a session lookup, main's path byte for byte
- from tools.approval_context import get_current_session_key
- charge = _costs.pending_charge(key, server_name, tool_name, get_current_session_key())
- if charge is None:
- return _ask_for_call(server_name, tool_name, None, booked, args=args) if gated else None
- if not gated and charge.reserve_within_limit()[0]:
- if booked is not None:
- booked.append(charge)
- return None
- with charge.asking(): # re-read under the slot: a call that asked first may have booked meanwhile
- return _ask_for_call(server_name, tool_name, charge, booked, gated=gated, args=args)
- def _ask_for_call(server_name: str, tool_name: str, charge, booked: Optional[list], *,
- gated: bool = True, args: Optional[dict] = None) -> Optional[str]:
- """Ask the user about one gated and/or costed call; book the charge only on an explicit yes."""
- spent, over = 0.0, False
- if charge is not None:
- if gated:
- spent = charge.spent()
- over = charge.over_limit(spent)
- else:
- reserved, spent = charge.reserve_within_limit()
- if reserved:
- if booked is not None:
- booked.append(charge)
- return None
- over = True
- extra = charge.lines(spent) if charge is not None else []
- if gated:
- head = f"MCP tool '{tool_name}' on UNTRUSTED server '{server_name}' wants to run."
- tail = "This tool is write-capable (no readOnlyHint=true annotation) and may modify external state."
- # The limit notice leads, and the arguments trail: a prompt cut to a platform's cap keeps the notice.
- message = " ".join([head, *(["This call goes past the chat's spend limit."] if over else []), *extra, tail])
- if args:
- message += f"\n\nArguments:\n{_format_trust_gate_arguments(args)}"
- # Over the limit the answer covers this call only: a remembered trust grant must not
- # pre-approve the next over-limit call.
- remember_key = None if over else _trust_approval_key(server_name, tool_name)
- scope = "once" if over else "once, for this session, or always"
- description = (f"Server '{server_name}' is configured 'trust: untrusted'. "
- f"Approve to run '{tool_name}' {scope}, or deny to block it.")
- title = f"MCP server '{server_name}' is asking"
- else:
- message = " ".join([f"MCP tool '{tool_name}' on server '{server_name}' would take this chat past its "
- f"spend limit.", *extra])
- description = (f"Server '{server_name}' declares a spend limit per chat. Approve to run '{tool_name}' "
- f"once (every further paid call in this chat asks again), or deny to block it.")
- title = f"MCP server '{server_name}' spend limit"
- remember_key = None
- try: # lazy: tools.approval routes the prompt to whichever surface owns the session
- from tools.approval_prompt import request_elicitation_consent
- answer = request_elicitation_consent(message, description, surface=f"mcp-trust/{server_name}", title=title,
- remember_key=remember_key)
- except Exception as exc:
- logger.error("MCP trust gate: approval check failed for %s.%s: %s", server_name, tool_name, exc, exc_info=True)
- if over and charge is not None:
- return tool_error(charge.refusal(spent))
- return tool_error(f"MCP tool '{tool_name}' on untrusted server '{server_name}' was blocked: the approval "
- f"system was unavailable (fail-closed).")
- if answer == "accept":
- if charge is not None:
- charge.book()
- if booked is not None:
- booked.append(charge)
- return None
- logger.info("MCP trust gate: user %s '%s' on %s server '%s'", "cancelled" if answer == "cancel" else "denied",
- tool_name, "untrusted" if gated else "spend-limited", server_name)
- if over and charge is not None:
- return tool_error(charge.refusal(spent))
- return tool_error(f"The user did not approve running write-capable MCP tool '{tool_name}' on untrusted server "
- f"'{server_name}'. The command was NOT run. Do not retry without explicit user direction.")
- def _check_circuit_breaker(server_name: str) -> Optional[str]:
- """Open-breaker error, or None when calls may proceed. After the cooldown the breaker is
- half-open: the next call probes; success resets, failure re-bumps and re-arms the cooldown."""
- from tools.mcp_tool_scope import _resolve_server_key
- key = _resolve_server_key(server_name)
- failures = _core._server_error_counts.get(key, 0)
- age = time.monotonic() - _core._server_breaker_opened_at.get(key, 0.0)
- if failures < _core._CIRCUIT_BREAKER_THRESHOLD or age >= _core._CIRCUIT_BREAKER_COOLDOWN_SEC:
- return None
- retry_in = max(1, int(_core._CIRCUIT_BREAKER_COOLDOWN_SEC - age))
- if _core._server_errors_all_application.get(key):
- # The server answered every time; the calls were rejected. Calling it "unreachable" sent the
- # model to the user instead of to its own arguments (#11113).
- return tool_error(f"MCP server '{server_name}' rejected the last {failures} calls (it is reachable; see the "
- f"error text those calls returned). Paused for ~{retry_in}s. Do NOT repeat the same call — "
- f"fix the arguments/URL/target or use a different approach.")
- return tool_error(f"MCP server '{server_name}' is unreachable after {failures} consecutive failures. "
- f"Auto-retry available in ~{retry_in}s. Do NOT retry "
- f"this tool yet — use alternative approaches or ask the user to check the MCP server.")
- def _acquire_call_server(server_name: str, tool_timeout: float):
- """``(server, None)`` when a call may be dispatched, else ``(None, error)``. No session: a
- reconnect may be completing, so wait briefly before a breaker strike; still down -> ask the
- server task to rebuild (probing a dead transport would re-arm the breaker forever)."""
- from tools import mcp_tool_discovery as _discovery # lazy: discovery -> registration -> handlers cycle
- not_connected = tool_error(f"MCP server '{server_name}' is not connected")
- from tools.mcp_liveness import unavailable_details
- details = unavailable_details(server_name)
- if details is not None:
- decl, current, sentence = details
- not_connected = tool_error(
- sentence,
- server=server_name,
- state=current.state,
- app={
- "name": decl.name,
- "version": current.availability.version,
- "path": current.availability.path,
- },
- user_action=current.user_action,
- retry=current.retry,
- )
- server = _discovery._get_connected_server_for_call(server_name)
- wait = min(5.0, float(tool_timeout or 5.0))
- if server and (server.session or _loop._wait_for_server_session_ready(server, timeout=wait)):
- return server, None
- _core._bump_server_error(server_name)
- if server and _loop._signal_reconnect(server):
- return None, tool_error(f"MCP server '{server_name}' transport is down; reconnect requested. Do NOT retry this "
- f"tool immediately — give it a few seconds to come back.")
- return None, not_connected
- def _result_is_error(result) -> bool:
- """True only for a JSON payload carrying an ``error`` key (non-JSON = success)."""
- try:
- return "error" in json.loads(result)
- except (json.JSONDecodeError, TypeError):
- return False
- def _record_call_outcome(server_name: str, result) -> Any:
- """Breaker bookkeeping: an error payload from the tool itself still counts as a strike (#10447),
- flagged as an application error so the open-breaker message stays truthful."""
- if _result_is_error(result):
- _core._bump_server_error(server_name, application=True)
- else:
- _core._reset_server_error(server_name)
- return result
- def _strike(server_name: str, message: str, **extra) -> str:
- """Breaker strike + the ``tool_error`` payload for *message*."""
- _core._bump_server_error(server_name)
- return tool_error(message, **extra)
- def _mcp_loop_running() -> bool:
- return _core._mcp_loop is not None and _core._mcp_loop.is_running()
- def _lookup_reconnectable_server(server_name: str, require_loop: bool = False):
- """The registered server object when it can be signalled to reconnect, else None.
- With *require_loop*, also None unless the MCP loop is running (nothing to wait on)."""
- from tools.mcp_tool_scope import _resolve_server_key
- with _core._lock:
- srv = _core._servers.get(_resolve_server_key(server_name))
- ok = srv is not None and hasattr(srv, "_reconnect_event") and (_mcp_loop_running() or not require_loop)
- return srv if ok else None
- def _retry_once(server_name: str, retry_call, op_description: str, what: str):
- """Re-run ``retry_call`` after a recovery step. Returns the result when the RPC completed
- (an application error is still the tool's real answer, and still a breaker strike per #10447);
- None when the retry raised (caller falls through)."""
- try:
- result = retry_call()
- except Exception as retry_exc:
- logger.warning("MCP %s/%s retry after %s failed: %s", server_name, op_description, what, retry_exc)
- return None
- return _record_call_outcome(server_name, result)
- def _handle_auth_error_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str):
- """OAuth recovery + one retry; None when *exc* is not an auth error. ``handle_401`` decides
- viability; if viable, signal a reconnect (fresh credentials), wait ready, retry once. Any
- failure returns the structured ``needs_reauth`` error so the model stops refreshing."""
- if not _is_auth_error(exc):
- return None
- from tools.mcp_oauth_manager import get_manager
- try:
- recovered = _loop._run_on_mcp_loop(lambda: get_manager().handle_401(server_name, None), timeout=10)
- except Exception as rec_exc:
- logger.warning("MCP OAuth '%s': recovery attempt failed: %s", server_name, rec_exc)
- recovered = False
- if recovered:
- srv = _lookup_reconnectable_server(server_name)
- # Recovery + reconnect is independent evidence of viability: close the breaker here, not only on
- # retry success (else a failing retry pins it open forever).
- if srv is not None and _loop._signal_reconnect_and_wait(
- server_name, srv, op_description=f"{op_description} after OAuth recovery", timeout=15):
- _core._reset_server_error(server_name)
- result = _retry_once(server_name, retry_call, op_description, "auth recovery")
- if result is not None:
- return result
- return _strike(server_name, _NEEDS_REAUTH_MSG.format(s=server_name, detail=_auth_error_detail(exc)),
- needs_reauth=True, server=server_name)
- def _handle_session_expired_and_retry(server_name: str, exc: BaseException, retry_call, op_description: str,
- *, call_may_have_side_effects: bool = False):
- """Transport reconnect + one retry on session expiry; None to fall through. Skips
- ``handle_401``: the token is valid, only the server-side session is stale.
- Unlike :func:`_handle_auth_error_and_retry`, this does **not** call the OAuth manager's ``handle_401`` —
- the access token is still valid, only the server-side session state is stale. Setting
- ``_reconnect_event`` causes the server task's lifecycle loop to tear down the current
- ``streamablehttp_client`` + ``ClientSession`` and rebuild them, reusing the existing OAuth provider
- instance. See #13383.
- At-most-once for writes: a session-expired shape can be synthesized by a proxy AFTER the upstream
- executed the request, and the transport errors this classifier also matches (``ClosedResourceError``,
- broken pipe) routinely fire mid-response. With ``call_may_have_side_effects`` the transport is still
- healed but the call is never re-run; the model gets an ``outcome_uncertain`` error instead (same
- contract as the mid-call stdio death path). Callers pass True unless the tool is positively read-only.
- """
- if not _is_session_expired_error(exc):
- return None
- srv = _lookup_reconnectable_server(server_name, require_loop=True)
- if call_may_have_side_effects:
- # Even without a signallable server the outcome is still uncertain: a generic "call failed"
- # would invite the model to re-invoke a write that may already have landed.
- reconnected = srv is not None and _loop._signal_reconnect_and_wait(
- server_name, srv, op_description=f"{op_description} (write, no auto-retry)", timeout=15)
- if reconnected: # session state failed, not server health: no breaker strike
- _core._reset_server_error(server_name)
- else:
- _core._bump_server_error(server_name)
- logger.warning("MCP server '%s': %s failed with a session-expired/transport error after the request may "
- "have been dispatched; NOT auto-retrying a write-capable tool (reconnect %s).",
- server_name, op_description, "succeeded" if reconnected else "failed")
- return tool_error(_SESSION_OUTCOME_UNCERTAIN_MSG.format(
- s=server_name, state="been re-established" if reconnected else "not recovered yet"),
- outcome_uncertain=True, server=server_name)
- if srv is None:
- return None
- logger.info("MCP server '%s': %s failed with session-expired error (%s); signalling transport reconnect "
- "and retrying once.", server_name, op_description, exc)
- if not _loop._signal_reconnect_and_wait(server_name, srv, op_description=op_description, timeout=15):
- logger.warning("MCP server '%s': reconnect did not ready within 15s after session-expired error; "
- "falling through to error response.", server_name)
- return None
- return _retry_once(server_name, retry_call, op_description, "session reconnect")
- class _StdioChildExited(RuntimeError):
- """Stdio subprocess gone when (or while) a call ran. Deliberately NOT a TimeoutError."""
- def __init__(self, message: str, *, in_flight: bool):
- super().__init__(message)
- self.in_flight = in_flight
- def _handle_stdio_child_exited_and_retry(server_name: str, exc: Exception, retry_call, op_description: str):
- """Respawn a dead stdio child; retry once only when it was dead before dispatch.
- A mid-call exit is ambiguous: the server may have applied a side effect before its
- response pipe disappeared. Reconnect for future calls but never replay that operation.
- None means this is not our error. This function never spawns itself: it sets
- ``_reconnect_event`` and waits, so spawn frequency stays governed by ``run()``'s rapid-drop
- budget. A pre-dispatch retry whose child dies again reports and stops.
- Why retrying here cannot hot-cycle respawns: this function never spawns anything. It sets
- ``_reconnect_event`` (one signal, same as before) and waits for the server task to publish a fresh
- session. Spawn frequency stays governed entirely by ``run()``'s rapid-drop budget, which parks a
- transport that keeps dropping without proving healthy (#62212).
- """
- if not isinstance(exc, _StdioChildExited):
- return None
- reconnected = False
- srv = _lookup_reconnectable_server(server_name)
- if srv is not None:
- action = "reconnecting without replay" if exc.in_flight else "respawning and retrying once"
- logger.info("MCP server '%s': %s found the stdio subprocess dead (%s); %s.",
- server_name, op_description, exc, action)
- if _mcp_loop_running():
- reconnected = _loop._signal_reconnect_and_wait(
- server_name, srv, op_description=op_description, timeout=_core._STDIO_RESPAWN_WAIT_SEC)
- else: # No MCP loop to wait on (non-async adapters, tests): still request the respawn.
- _loop._signal_reconnect(srv)
- if exc.in_flight:
- return _strike(
- server_name,
- _STDIO_OUTCOME_UNCERTAIN_MSG.format(s=server_name),
- outcome_uncertain=True,
- )
- if not reconnected:
- return _strike(server_name, _STDIO_NO_RESPAWN_MSG.format(s=server_name, t=_core._STDIO_RESPAWN_WAIT_SEC))
- try:
- return _record_call_outcome(server_name, retry_call())
- except _StdioChildExited as retry_exc:
- # Died again right after respawn: broken server; run()'s budget takes it to the park.
- logger.warning("MCP server '%s': %s stdio subprocess exited again right after respawn (%s); not retrying "
- "further.", server_name, op_description, retry_exc)
- if retry_exc.in_flight:
- return _strike(
- server_name,
- _STDIO_OUTCOME_UNCERTAIN_MSG.format(s=server_name),
- outcome_uncertain=True,
- )
- return _strike(server_name, _STDIO_DIED_AGAIN_MSG.format(s=server_name))
- except Exception as retry_exc:
- logger.warning("MCP %s/%s retry after stdio respawn failed: %s", server_name, op_description, retry_exc)
- return _strike(server_name, _sanitize_error(
- f"MCP call failed after respawning the stdio subprocess for '{server_name}': "
- f"{type(retry_exc).__name__}: {_exc_str(retry_exc)}"))
- def _dispatch(server_name: str, server: Any, op: str, call, tool_timeout: float, recoverers,
- on_final_failure: Callable[[BaseException], None], record_outcome: bool = False) -> str:
- """Mark the call started on *server* (doubles may lack ``mark_tool_call``), run coroutine function *call*
- on the MCP loop and, on failure, walk ``recoverers`` (``(server_name, exc, retry_call, op) -> Optional[str]``,
- None = not its kind; order matters). Unrecovered exceptions go through ``on_final_failure`` and become the
- generic call-failed error. ``record_outcome`` applies breaker bookkeeping to the FIRST attempt only."""
- if callable(getattr(server, "mark_tool_call", None)):
- server.mark_tool_call()
- def call_once():
- return _loop._run_on_mcp_loop(call, timeout=tool_timeout)
- try:
- result = call_once()
- return _record_call_outcome(server_name, result) if record_outcome else result
- except InterruptedError:
- return tool_error("MCP call interrupted: user sent a new message")
- except Exception as exc:
- for recover in recoverers:
- recovered = recover(server_name, exc, call_once, op)
- if recovered is not None:
- return recovered
- on_final_failure(exc)
- return tool_error(_mcp_call_failed_message(exc))
- @asynccontextmanager
- async def _track_inflight_rpc(server: Any, server_name: str, op: str, *, retry_safe: bool = True):
- """Register the running RPC so teardown can fail it fast. A deliberate teardown
- (``_reconnecting`` set first) turns the cancel into a retryable RuntimeError; external
- cancels propagate unchanged. Doubles without ``_inflight_tasks`` skip tracking.
- Every user-visible request family wraps its RPC in this context (#48069 salvage). If a deliberate
- reconnect/shutdown teardown cancels the task (``_fail_inflight_calls`` sets ``_reconnecting`` first),
- the cancel is converted into a clean retryable RuntimeError instead of a raw CancelledError; external
- cancels (caller timeout, user interrupt) propagate unchanged. ``retry_safe=False`` (a write-capable
- ``tools/call``) words the error as outcome-uncertain instead of inviting a replay.
- """
- inflight, task = getattr(server, "_inflight_tasks", None), asyncio.current_task()
- tracked = task is not None and inflight is not None
- if tracked:
- inflight.add(task)
- try:
- yield
- except asyncio.CancelledError:
- if getattr(server, "_reconnecting", False):
- advice = ("retry the request on the rebuilt session" if retry_safe else
- "the request may already have been dispatched, so verify its effect before re-invoking")
- raise RuntimeError(f"MCP {op} on '{server_name}' was aborted by a reconnect teardown; {advice}") from None
- raise
- finally:
- if tracked:
- inflight.discard(task)
- async def _call_tool_racing_stdio_death(server, server_name: str, tool_name: str, args: dict, progress_callback=None):
- """``session.call_tool`` that fails fast when the stdio child is/gets dead: pre-call (a dead
- child must not hold the slot for the full timeout) and mid-call (race against
- ``_watch_stdio_children``). Both raise :class:`_StdioChildExited` for the respawn path, which
- owns the reconnect signal; only pre-call failure is safe to replay. callable()/``is True``
- because MagicMock attributes are truthy."""
- # Fast-fail (#81995): a stdio subprocess that is already dead must not own this call slot — fail
- # immediately instead of waiting out the full tool timeout on a transport nobody will ever answer.
- _stdio_dead = getattr(server, "_stdio_children_dead", None)
- if callable(_stdio_dead) and _stdio_dead() is True:
- raise _StdioChildExited(
- f"MCP stdio subprocess for '{server_name}' had already exited when the call was dispatched",
- in_flight=False,
- )
- # A progress callback makes the SDK send ``_meta.progressToken``, the spec's opt-in for
- # ``notifications/progress``; without one the server must stay silent.
- extra = {"progress_callback": progress_callback} if progress_callback is not None else {}
- _call_coro = server.session.call_tool(tool_name, arguments=args, **extra)
- _watch_children = getattr(server, "_watch_stdio_children", None)
- if not (inspect.iscoroutinefunction(_watch_children) and asyncio.iscoroutine(_call_coro)):
- # Stubbed sessions return a non-awaitable, or there is no child-watcher to race: plain await.
- return await _call_coro if asyncio.iscoroutine(_call_coro) else _call_coro
- # Fast-fail machinery (#81995): the RPC races a stdio-children watcher so a dead subprocess fails the
- # call immediately instead of riding out the full tool timeout.
- rpc_task = asyncio.ensure_future(_call_coro)
- watch_task = asyncio.ensure_future(_watch_children())
- try:
- done, _pending = await asyncio.wait({rpc_task, watch_task}, return_when=asyncio.FIRST_COMPLETED)
- if watch_task in done and not rpc_task.done():
- rpc_task.cancel()
- raise _StdioChildExited(
- f"MCP stdio subprocess for '{server_name}' exited mid-call",
- in_flight=True,
- )
- try:
- return await rpc_task
- except Exception as exc:
- # The SDK usually sees the closed pipe before the 250 ms watcher poll does. On a stdio
- # server a transport-closure error after dispatch is the same ambiguous mid-call death;
- # it must not fall through to the session-expired recoverer, which replays the call.
- _is_http = getattr(server, "_is_http", None)
- if callable(_is_http) and _is_http() is False and _is_session_expired_error(exc):
- raise _StdioChildExited(
- f"MCP stdio subprocess for '{server_name}' closed its transport mid-call",
- in_flight=True,
- ) from exc
- raise
- finally:
- watch_task.cancel()
- if not rpc_task.done():
- rpc_task.cancel()
- await asyncio.gather(rpc_task, watch_task, return_exceptions=True)
- # ---------------------------------------------------------- result rendering
- def _error_result_text(result) -> str:
- """Concatenated text of an ``isError`` result's blocks (EmbeddedResource error payloads
- carry text under ``.resource.text``)."""
- texts = (getattr(b, "text", None) or getattr(getattr(b, "resource", None), "text", None) for b in (result.content or []))
- return "".join(str(t) for t in texts if t)
- def _render_content_blocks(result, server_name: str) -> tuple[str, int, list[str]]:
- """Text passes through; image/audio blocks are cached (MEDIA: tags); resource blocks are
- materialized rather than silently dropped; unsupported blocks become an inline drop notice
- (kimi-code#3227). Returns ``(text, usable_parts, image_paths)``: ``usable_parts`` counts REAL
- rendered blocks (whitespace-only text and drop notices excluded) for the structuredContent
- arbitration; ``image_paths`` are the files THIS call's image blocks were cached to — never
- parsed back out of the text, where a server-written ``MEDIA:`` line could name any local file."""
- parts: list[str] = []
- usable_parts = 0
- image_paths: list[str] = []
- # MCP tool results can also include ImageContent blocks (screenshot / Blockbench / Playwright etc.);
- # cache those via the gateway's image-cache helper so they flow through Hermes' MEDIA: tag convention
- # and out to messaging adapters that render images natively. Without this, image blocks were silently
- # dropped and the agent got an empty response. Distilled from #17915 (c3115644151) and #10848
- # (gnanirahulnutakki), both too stale to cherry-pick. #10848's approach (integrate with Hermes' MEDIA
- # tag + cache_image_from_bytes) was the cleaner of the two — plugs into existing infrastructure.
- for block in (result.content or []):
- if getattr(block, "text", None):
- parts.append(strip_unicode_tags(block.text))
- if block.text.strip():
- usable_parts += 1
- continue
- image = _cache_mcp_image_block(block)
- if image.startswith("MEDIA:"):
- image_paths.append(image[len("MEDIA:"):])
- rendered = image or _cache_mcp_audio_block(block) or _render_mcp_resource_block(block, server_name)
- if rendered:
- parts.append(rendered)
- usable_parts += 1
- continue
- block_type = getattr(block, "type", None) or type(block).__name__
- if block_type in {"text", "resource", "audio", "image"}: # benign empty render
- logger.debug("MCP %s: content block type %r rendered empty", server_name, block_type)
- else:
- logger.warning("MCP %s: dropping unsupported content block type %r", server_name, block_type)
- # Surface the drop to the MODEL, not just the log: a silent drop leaves the agent
- # believing the tool returned less than it did, with no way to recover.
- parts.append(_render_mcp_dropped_block_notice(block, block_type))
- # Hard-cap pathological payloads; ordinary large results pass to spillover.
- return _truncate_mcp_text_result("\n".join(parts)), usable_parts, image_paths
- def _capped_structured_content(result):
- """``structuredContent`` (or None); over the hard cap it degrades to the head+tail
- truncated JSON string (multi-MB JSON flood guard)."""
- # Hard-cap pathological payloads before they propagate (#56059); ordinary large results pass untouched
- # to the spillover layer. Arbitration against ``content`` lives in _render_call_tool_result.
- # Server-level `_meta` is also surfaced (ported from
- # MoonshotAI/kimi-code#2596): servers return namespaced metadata there (validated contracts,
- # browser-handoff payloads, ...) that was previously invisible to the agent. Protocol-reserved keys are
- # dropped first (kimi-code#2600) — per the MCP spec's key-name rules a prefix is reserved when a
- # `modelcontextprotocol` or `mcp` label is followed by at least one more label (e.g.
- # `modelcontextprotocol.io/...`, `tools.mcp.com/...`); those carry host/protocol plumbing, not
- # model-facing data. Unprefixed and vendor-namespaced keys (`com.example.mcp/...`) pass through — their
- # semantics belong to the server.
- structured = mcp_field(result, "structured_content", "structuredContent")
- try:
- as_json = json.dumps(structured, ensure_ascii=False, default=str) if structured is not None else ""
- except (TypeError, ValueError):
- return structured
- return _truncate_mcp_text_result(as_json) if len(as_json) > _MCP_HARD_RESULT_CAP_CHARS else structured
- def _content_dual_emits_structured(result, structured) -> bool:
- """True when some text block is ``structuredContent`` serialized as JSON — the spec's
- backwards-compat dual-emit ("a tool that returns structured content SHOULD also return the
- serialized JSON in a TextContent block"). Compared as parsed JSON so whitespace, indent, key
- order and ``ensure_ascii`` escaping do not matter; checked per block because the spec puts the
- copy in *a* block and a server may add a status line next to it. Deterministic equality, not a
- richness heuristic: a prose summary or a reorganised rendering fails it and keeps its
- ``structuredContent`` (#115430)."""
- for block in (result.content or []):
- text = getattr(block, "text", None)
- if not text:
- continue
- try:
- if json.loads(text) == structured:
- return True
- except (TypeError, ValueError):
- continue
- return False
- def _render_call_tool_result(result, server_name: str, image_paths: Optional[list[str]] = None) -> str:
- """Pure: ``CallToolResult`` -> handler JSON (``image_paths``, when given, receives the files this
- call's image blocks were cached to). ``content`` and ``structuredContent`` are both
- forwarded, except that a ``structuredContent`` whose JSON also sits verbatim in a text block
- (the spec's backwards-compat dual-emit; compared as parsed JSON) is dropped, because that copy
- would reach the model twice (kimi-code#3234). Any other usable text — a status line, a prose
- summary, a reorganised rendering — keeps ``structuredContent`` alongside it (#115430): the
- earlier "content wins whenever it rendered anything" rule irreversibly lost servers whose
- data lived only in ``structuredContent``, while the residual duplicate for a faithful
- reorganisation costs only tokens, so data-preservation wins. No richness or size heuristic is
- used. ``structuredContent`` fills ``result`` when the blocks rendered effectively empty
- (structuredContent-only servers); ``_meta`` minus reserved keys is always surfaced."""
- if mcp_field(result, "is_error", "isError", False):
- return tool_error(_sanitize_error(_truncate_mcp_text_result(_error_result_text(result) or "MCP tool returned an error")))
- text_result, usable_parts, cached_images = _render_content_blocks(result, server_name)
- if image_paths is not None:
- image_paths.extend(cached_images)
- structured = _capped_structured_content(result)
- meta = _strip_reserved_meta_keys(mcp_field(result, "meta", "meta"))
- # A str here is the over-cap truncation stand-in (wire structuredContent is always an object): next to
- # usable text it would be a second multi-MB copy — the flood #56059 caps — so it only fills an empty result.
- if structured is not None and usable_parts > 0 and (isinstance(structured, str) or _content_dual_emits_structured(result, structured)):
- structured = None
- if structured is None and meta is None:
- return json.dumps({"result": text_result}, ensure_ascii=False)
- # Key order is part of the output: "result" leads when there is text, otherwise "_meta" precedes it.
- payload: dict[str, Any] = {"result": text_result} if text_result else {}
- # Cap structuredContent too — a malicious server could flood context via a multi-MB JSON payload
- # (#56059). When the serialized form exceeds the hard cap, replace it with the truncated string (head +
- # tail preserved) so it degrades gracefully instead of flooding downstream.
- if structured is not None:
- payload["structuredContent" if text_result else "result"] = structured
- if meta is not None:
- payload["_meta"] = meta
- payload.setdefault("result", text_result)
- try:
- return json.dumps(payload, ensure_ascii=False)
- except (TypeError, ValueError): # Non-serializable metadata: drop the extras, keep the call.
- return json.dumps({"result": text_result}, ensure_ascii=False)
- def _make_tool_handler(server_name: str, tool_name: str, tool_timeout: float, *, native_images: bool = False):
- """Sync handler (``handler(args_dict, **kwargs)``) calling an MCP tool via the background loop. Always a JSON
- string, except with ``native_images`` (the registry handler the agent loop dispatches): a result carrying
- images then becomes the ``_multimodal`` envelope when the active route takes images in tool results.
- Direct callers (plugins' ``ctx.call_mcp``) keep the string contract and the ``MEDIA:`` paths."""
- op = f"tools/call {tool_name}"
- def _handler(args: dict, **kwargs) -> Any:
- # Security boundary: untrusted-server write tools need approval before ANY transport work (incl. lazy spawn).
- booked: list = [] # a costed call's charge; refunded when the call never reaches the server
- error = _trust_gate_check(server_name, tool_name, booked, args)
- if error is not None:
- return error
- server, error = None, _check_circuit_breaker(server_name)
- if error is None:
- server, error = _acquire_call_server(server_name, tool_timeout)
- if server is None:
- for charge in booked:
- charge.refund()
- return error
- # Only a tool annotated readOnlyHint=True is replayed after session expiry; a 401 is always
- # pre-dispatch so the auth recoverer keeps its retry for every tool.
- read_only = _tool_is_read_only(server_name, tool_name)
- image_paths: list[str] = [] # the LAST attempt's cached images (a recoverer may retry _call)
- # Read on the agent's thread: the coroutine below runs on the MCP loop, outside this context.
- from agent.tool_progress_relay import current_tool_progress
- report = current_tool_progress()
- async def _on_progress(progress, total, message):
- if report is not None:
- report(progress, total, message)
- async def _call():
- async with server._rpc_lock, _track_inflight_rpc(server, server_name, op, retry_safe=read_only):
- server._pending_call_context = contextvars.copy_context() # for the elicitation callback
- try:
- result = await _call_tool_racing_stdio_death(
- server, server_name, tool_name, args, _on_progress if report is not None else None)
- finally:
- server._pending_call_context = None
- if getattr(server, "_mark_session_proven", None) is not None: # round-trip done: transport healthy
- server._mark_session_proven()
- image_paths.clear()
- return _render_call_tool_result(result, server_name, image_paths)
- def _on_failure(exc):
- _core._bump_server_error(server_name)
- logger.error("MCP tool %s/%s call failed: %s", server_name, tool_name, exc)
- session_expired = partial(_handle_session_expired_and_retry, call_may_have_side_effects=not read_only)
- text = _dispatch(
- server_name, server, op, _call, tool_timeout,
- (_handle_stdio_child_exited_and_retry, _handle_auth_error_and_retry, session_expired),
- _on_failure, record_outcome=True)
- # Off the MCP loop on purpose: image prep is CPU work (bounded vision pool) that must not stall other servers' I/O.
- return _mcp_result_with_native_images(text, image_paths) if native_images and not _result_is_error(text) else text
- return _handler
- def _make_utility_handler(op: str, log_label: str, rpc, render, required: Optional[str] = None):
- """``(server_name, tool_timeout) -> sync handler`` for one utility tool: ``rpc(session, args,
- server_name)`` awaited under ``_rpc_lock``, ``render(result, server_name)`` -> JSON-able
- payload, ``required`` validated before any transport work."""
- def _factory(server_name: str, tool_timeout: float):
- def _handler(args: dict, **kwargs) -> str:
- from tools import mcp_tool_discovery as _discovery # lazy: import cycle
- server = _discovery._get_connected_server_for_call(server_name)
- if not server or not server.session:
- return tool_error(f"MCP server '{server_name}' is not connected")
- if required and not args.get(required):
- return tool_error(f"Missing required parameter '{required}'")
- async def _call():
- async with server._rpc_lock:
- result = await rpc(server.session, args, server_name)
- return json.dumps(render(result, server_name), ensure_ascii=False)
- return _dispatch(
- server_name, server, op, _call, tool_timeout,
- (_handle_auth_error_and_retry, _handle_session_expired_and_retry),
- lambda exc: logger.error("MCP %s/%s failed: %s", server_name, log_label, exc))
- return _handler
- return _factory
- def _pick(obj, *specs) -> dict:
- """``{out_key: value}`` for each ``(out_key, attr[, truthy])`` present on *obj* (presence check so SDK models
- and stubs behave alike; ``truthy`` also skips falsy). Key order = spec order."""
- entry = {}
- for out_key, attr, *truthy in specs:
- value = getattr(obj, attr, _MISSING)
- if value is not _MISSING and (value or not (truthy and truthy[0])):
- entry[out_key] = value
- return entry
- def _render_resource_list(all_resources, server_name: str) -> dict:
- resources = []
- for r in all_resources:
- entry = _pick(r, ("uri", "uri"), ("name", "name"), ("description", "description", True))
- if "uri" in entry:
- entry["uri"] = str(entry["uri"])
- mime = mcp_field(r, "mime_type", "mimeType")
- if mime:
- entry["mimeType"] = mime # camelCase: this is the tool's own JSON output shape
- resources.append(entry)
- return {"resources": resources}
- def _render_read_resource(result, server_name: str) -> dict:
- parts: list[str] = []
- for block in getattr(result, "contents", []):
- if getattr(block, "text", None) is not None:
- parts.append(strip_unicode_tags(block.text))
- elif getattr(block, "blob", None) is not None: # binary -> document cache, like EmbeddedResource blocks
- rendered = _render_mcp_resource_block(SimpleNamespace(type="resource", resource=block), server_name)
- parts.append(rendered or f"[binary data, {len(block.blob)} bytes]")
- return {"result": "\n".join(parts)}
- def _render_prompt_list(all_prompts, server_name: str) -> dict:
- prompts = []
- for p in all_prompts:
- entry = _pick(p, ("name", "name"), ("description", "description", True))
- if getattr(p, "arguments", None):
- entry["arguments"] = [{"name": a.name, **_pick(a, ("description", "description", True), ("required", "required"))}
- for a in p.arguments]
- prompts.append(entry)
- return {"prompts": prompts}
- def _render_get_prompt(result, server_name: str) -> dict:
- messages = []
- for msg in getattr(result, "messages", []):
- entry = _pick(msg, ("role", "role"))
- if hasattr(msg, "content"):
- entry["content"] = strip_unicode_tags(msg.content.text if hasattr(msg.content, "text") else str(msg.content))
- messages.append(entry)
- return {"messages": messages, **_pick(result, ("description", "description", True))}
- _make_list_resources_handler = _make_utility_handler(
- "resources/list", "list_resources",
- lambda session, args, sn: _core._paginate_full_list(session.list_resources, "resources", sn), _render_resource_list)
- _make_read_resource_handler = _make_utility_handler(
- "resources/read", "read_resource",
- lambda session, args, sn: session.read_resource(args["uri"]), _render_read_resource, required="uri")
- _make_list_prompts_handler = _make_utility_handler(
- "prompts/list", "list_prompts",
- lambda session, args, sn: _core._paginate_full_list(session.list_prompts, "prompts", sn), _render_prompt_list)
- _make_get_prompt_handler = _make_utility_handler(
- "prompts/get", "get_prompt",
- lambda session, args, sn: session.get_prompt(args["name"], arguments=args.get("arguments", {})),
- _render_get_prompt, required="name")
- def _make_check_fn(server_name: str):
- """Connection-alive check; lazy (schema-cache registered) servers count as available.
- When the server's owner registered an application declaration (`requires.app`), the
- application must also be present on this host, or the tools are not offered even while a
- stale connection lingers. With no declaration registered the check is the connection check
- alone. Returns a plain bool: the registry caches ``bool(fn())``.
- """
- from tools.mcp_tool_scope import _resolve_server_key
- def _connected() -> bool:
- with _core._lock:
- key = _resolve_server_key(server_name)
- server = _core._servers.get(key)
- return ((server is not None and (server.session is not None or server._is_recycled_stdio()))
- or key in _core._lazy_server_configs)
- def _check() -> bool:
- if not _connected():
- return False
- return _declared_app_offerable(server_name)
- return _check
- def _declared_app_offerable(server_name: str) -> bool:
- """True unless the registered declaration is unavailable on this host. Called only for a
- connected server, so a reachable loopback port outranks the interactive-session rule."""
- from hermes_platform import declaration
- from hermes_platform.resolver.availability import availability
- decl = declaration.lookup(server_name)
- if decl is None:
- return True
- return bool(availability(decl).offerable)
|