mcp_tool_content.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. """Rendering of MCP tool-result content blocks into model-facing text: size
  2. capping, _meta filtering, image/audio caching to MEDIA tags, resource links and
  3. embedded resources."""
  4. import base64
  5. import json
  6. import logging
  7. import mimetypes
  8. from typing import Any, Dict, List, Optional, Tuple
  9. from tools.ansi_strip import strip_unicode_tags
  10. from tools.mcp_tool_common import mcp_field
  11. from tools.mcp_tool_schema import mcp_prefixed_tool_name
  12. from tools.tool_output_truncate import truncate_head_tail
  13. logger = logging.getLogger("tools.mcp_tool")
  14. # Hard ceiling for one MCP text payload (chars), deliberately far ABOVE the budget layer's 50K
  15. # spillover threshold so ordinary large results reach spillover intact; only floods are lossy.
  16. # This is the FIRST line of defense against a buggy or malicious MCP server returning multi-megabyte text:
  17. # without it the full payload is allocated, JSON-encoded and handed downstream before the budget/spillover
  18. # layer ever sees it (#56059). Distilled from #56060 (Stoltemberg), #56072 (AlexFucuson9) and #56511
  19. # (Tranquil-Flow), which capped at get_max_bytes() (50K) — correct protection, but at that level it would
  20. # truncate before spillover could preserve the data. The 40% head / 60% tail split is #56511's shape.
  21. _MCP_HARD_RESULT_CAP_CHARS = 2_000_000
  22. # Cap on decoded resource bytes per block (a misbehaving server can't fill the cache disk).
  23. # Base64 expands ~4/3; oversized payloads are rejected BEFORE decoding (never doubled in memory).
  24. _MCP_RESOURCE_MAX_BYTES = 50 * 1024 * 1024
  25. _MCP_RESOURCE_MAX_B64_CHARS = _MCP_RESOURCE_MAX_BYTES * 4 // 3 + 4
  26. def _truncate_mcp_text_result(text: str, max_chars: int = _MCP_HARD_RESULT_CAP_CHARS) -> str:
  27. """Bound pathological MCP text before it propagates (#56059)."""
  28. return truncate_head_tail(text, max_chars, label="MCP RESULT")
  29. def _is_reserved_mcp_meta_key(key: str) -> bool:
  30. """True if an MCP ``_meta`` key uses a protocol-reserved prefix: a ``modelcontextprotocol``
  31. or ``mcp`` label followed by at least one more label. A trailing one
  32. (``com.example.mcp/...``) is a vendor namespace.
  33. Ported from MoonshotAI/kimi-code#2600.
  34. """
  35. slash = key.find("/")
  36. if slash <= 0:
  37. return False
  38. labels = key[:slash].split(".")
  39. return any(label in ("modelcontextprotocol", "mcp") and i < len(labels) - 1 for i, label in enumerate(labels))
  40. def _strip_reserved_meta_keys(meta) -> Optional[dict[str, Any]]:
  41. """Drop protocol-reserved keys from ``_meta``; None if nothing model-facing remains or the
  42. input wasn't a mapping."""
  43. if not isinstance(meta, dict):
  44. return None
  45. out = {k: v for k, v in meta.items() if isinstance(k, str) and (not _is_reserved_mcp_meta_key(k))}
  46. return out or None
  47. def _base_mime(mime_type) -> str:
  48. """``type/subtype`` of a MIME string, lower-cased, parameters dropped."""
  49. return str(mime_type or "").split(";", 1)[0].strip().lower()
  50. def _mcp_image_extension_for_mime_type(mime_type: str) -> str:
  51. """File extension for an MCP image MIME type (``.png`` fallback)."""
  52. normalized = _base_mime(mime_type)
  53. if normalized in {"image/jpeg", "image/jpg"}:
  54. return ".jpg"
  55. return mimetypes.guess_extension(normalized) or ".png"
  56. def _decode_block_b64(data, what: str, label: str, *, cap_what: Optional[str] = None,
  57. cap_suffix: str = "", decode_fail: str = "") -> tuple[Optional[bytes], str]:
  58. """Base64-decode one block payload: ``(bytes, "")`` or ``(None, inline_marker)``. With
  59. ``cap_what`` the payload is rejected on b64 length BEFORE decoding and on decoded size
  60. after. Decode failures warn and return ``decode_fail`` ("" = drop the block)."""
  61. if cap_what and len(data) > _MCP_RESOURCE_MAX_B64_CHARS:
  62. return None, f"[MCP {cap_what} too large to cache: ~{len(data) * 3 // 4} bytes{cap_suffix}]"
  63. try:
  64. raw_bytes = base64.b64decode(data)
  65. except (TypeError, ValueError) as exc:
  66. logger.warning("MCP %s decode failed (%s): %s", what, label, exc)
  67. return None, decode_fail
  68. if cap_what and len(raw_bytes) > _MCP_RESOURCE_MAX_BYTES:
  69. return None, f"[MCP {cap_what} too large to cache: {len(raw_bytes)} bytes{cap_suffix}]"
  70. return raw_bytes, ""
  71. def _write_block_cache(writer: str, what: str, skip_label: str, *args,
  72. unavailable: str = "", failed: str = "", **kwargs) -> tuple[Optional[str], str]:
  73. """Call ``gateway.platforms.base.<writer>(*args, **kwargs)``: ``(path, "")`` or ``(None,
  74. marker)``. Fail-open so one bad block never kills the tool result: gateway deps missing
  75. (cron without gateway) → ``unavailable``; any other cache error → warning + ``failed``."""
  76. try:
  77. import gateway.platforms.base as _base
  78. return getattr(_base, writer)(*args, **kwargs), ""
  79. except ImportError:
  80. logger.debug("MCP %s caching skipped — gateway.platforms.base unavailable", skip_label)
  81. return None, unavailable
  82. except Exception as exc:
  83. logger.warning("MCP %s cache failed: %s", what, exc)
  84. return None, failed
  85. _WAV_MIME_EXT = {"audio/wav": ".wav", "audio/x-wav": ".wav", "audio/wave": ".wav"}
  86. def _cache_mcp_media_block(block, kind: str, writer: str, ext_for, *, cap_what: Optional[str] = None) -> str:
  87. """Cache an image/audio block and return a ``MEDIA:<path>`` tag. "" (logging, not raising)
  88. when the block isn't ``kind`` media, the base64 is malformed, or the cache rejects the
  89. bytes: the caller falls through to any text blocks."""
  90. data = getattr(block, "data", None)
  91. mime = _base_mime(mcp_field(block, "mime_type", "mimeType"))
  92. if data is None or not mime.startswith(f"{kind}/"):
  93. return ""
  94. raw_bytes, err = _decode_block_b64(data, f"{kind} block", mime, cap_what=cap_what)
  95. if raw_bytes is None:
  96. return err
  97. path, err = _write_block_cache(writer, f"{kind} block", kind, raw_bytes, ext=ext_for(mime))
  98. return err if path is None else f"MEDIA:{path}"
  99. def _cache_mcp_image_block(block) -> str:
  100. """Cache an ``ImageContent`` block and return a ``MEDIA:<path>`` tag ("" on any failure)."""
  101. return _cache_mcp_media_block(block, "image", "cache_image_from_bytes", _mcp_image_extension_for_mime_type)
  102. _MCP_NATIVE_IMAGE_MAX = 4 # images attached natively from one tool result; the rest stay MEDIA: paths
  103. _MCP_NATIVE_IMAGE_CANDIDATES = 16 # images prepared at most per result: skips refill the 4 slots, within a bound
  104. def _mcp_native_image_part(path: str) -> Optional[tuple[dict[str, Any], Optional[str]]]:
  105. """``(image_url part, scale note)`` for one cached MCP image, sized like every other native embed (the
  106. result is re-sent each later turn: ``vision.embed_target_bytes``, 1568 px long edge, JPEG) and normalized
  107. to a provider-accepted format (BMP and friends → PNG). None when the file cannot be embedded safely. The
  108. note maps embedded coordinates back to the original image (a screenshot's pixels are the screen's)."""
  109. from pathlib import Path
  110. from PIL import Image
  111. from tools.vision_tools import (_EMBED_MAX_DIMENSION, _MAX_BASE64_BYTES, _build_scale_note,
  112. _resize_image_for_vision)
  113. from tools.vision_tools_history_budget import resolve_embed_target_bytes
  114. from tools.vision_tools_image_prep import (_detect_image_mime_type_from_bytes, _normalize_to_supported_image,
  115. _validate_raster_image_decodable)
  116. src = Path(path)
  117. mime = _detect_image_mime_type_from_bytes(src.read_bytes())
  118. if not mime:
  119. return None
  120. normalized, mime, err = _normalize_to_supported_image(src, mime)
  121. if err or normalized is None:
  122. return None
  123. scale: dict[str, int] = {}
  124. target = min(resolve_embed_target_bytes(), _MAX_BASE64_BYTES)
  125. try:
  126. # A valid header over a truncated pixel stream passes the sniff and the cache; one undecodable
  127. # part makes the provider reject the whole request, so decode every frame first (as vision_analyze).
  128. if _validate_raster_image_decodable(normalized):
  129. return None
  130. with Image.open(normalized) as image:
  131. dims = image.size
  132. url = _resize_image_for_vision(normalized, mime_type=mime, max_base64_bytes=target,
  133. max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True, scale_out=scale)
  134. finally:
  135. if normalized != src:
  136. normalized.unlink(missing_ok=True)
  137. # The resizer is best-effort on both caps (a 64 px short-edge floor keeps a 60000x64 strip at 60000 px; the
  138. # quality ladder can bottom out above a low vision.embed_target_bytes). The part is re-sent every later turn,
  139. # so attach only what fits both; the MEDIA: path still carries the rest (vision_analyze can read it).
  140. if max(scale.get("new_width", dims[0]), scale.get("new_height", dims[1])) > _EMBED_MAX_DIMENSION:
  141. return None
  142. if len(url) > target:
  143. return None
  144. return {"type": "image_url", "image_url": {"url": url}}, _build_scale_note(scale or None, None)
  145. def _mcp_result_with_native_images(text: str, image_paths: list[str]) -> Any:
  146. """*text* as-is, or the ``_multimodal`` envelope carrying the call's cached images when the active route
  147. takes images inside tool results — the same gate as ``vision_analyze`` and ``computer_use`` captures
  148. (``agent.image_input_mode``, an explicit ``auxiliary.vision`` backend, catalog vision, provider support).
  149. The text half keeps the ``MEDIA:`` paths so sharing and full-resolution reads still work. Any failure
  150. keeps the text: the paths already carry the images."""
  151. if not image_paths:
  152. return text
  153. try:
  154. import contextvars
  155. from tools.vision_tools import _should_use_native_vision_fast_path, _vision_cpu_executor
  156. from tools.vision_tools_history_budget import repeat_refusal
  157. if not _should_use_native_vision_fast_path():
  158. return text
  159. except Exception: # deliberate boundary: the MEDIA: paths already carry the images, so any failure keeps the text
  160. logger.debug("MCP native image gate failed, keeping MEDIA: paths", exc_info=True)
  161. return text
  162. attached, notes = [], ""
  163. candidates = image_paths[:_MCP_NATIVE_IMAGE_CANDIDATES]
  164. # Prepare in waves sized to the free slots, so a damaged or already-in-context image hands its slot to the
  165. # next one instead of hiding it. Decode/resize runs on the bounded vision pool (a parallel tool batch of
  166. # image-heavy calls must not decode dozens of large images at once on tool threads), each job in a copy of
  167. # the caller's context: the active runtime (a managed local model: no WebP) and the profile's vision
  168. # settings are ContextVars a bare pool thread would not see.
  169. while candidates and len(attached) < _MCP_NATIVE_IMAGE_MAX:
  170. wave, candidates = candidates[:_MCP_NATIVE_IMAGE_MAX - len(attached)], candidates[_MCP_NATIVE_IMAGE_MAX - len(attached):]
  171. jobs = [(p, _vision_cpu_executor.submit(contextvars.copy_context().run, _mcp_native_image_part, p)) for p in wave]
  172. for path, job in jobs:
  173. try:
  174. ready = job.result()
  175. except Exception: # one bad file keeps its MEDIA: path; the others still attach
  176. logger.debug("MCP native image prep failed for %s", path, exc_info=True)
  177. continue
  178. if not ready:
  179. continue
  180. part, note = ready
  181. # vision.max_calls_per_image: a polled screenshot tool re-sends the same pixels under a fresh cache
  182. # path each call, so the reservation keys on the resized data URL (identical pixels, identical key).
  183. if repeat_refusal(part["image_url"]["url"]):
  184. notes += f"\n- MEDIA:{path}: not attached; this image is already in context (vision.max_calls_per_image)."
  185. continue
  186. attached.append(part)
  187. if note:
  188. notes += f"\n- MEDIA:{path}: {note}"
  189. if not attached:
  190. if not notes:
  191. return text
  192. # Keep the result valid JSON: the refusal lines ride inside the envelope, not after its closing brace.
  193. try:
  194. payload = json.loads(text)
  195. payload["result"] = f"{payload.get('result') or ''}{notes}"
  196. return json.dumps(payload, ensure_ascii=False)
  197. except (TypeError, ValueError, AttributeError):
  198. return text + notes
  199. # The header and the scale notes are their own short part: an oversized tool text gets spilled to a file and
  200. # replaced by its head, and the coordinate map must survive that next to the resized screenshot it describes.
  201. header = "The image(s) from this call are attached — inspect them with your native vision."
  202. return {"_multimodal": True,
  203. "content": [{"type": "text", "text": text}, {"type": "text", "text": header + notes}, *attached],
  204. "text_summary": text}
  205. def _cache_mcp_audio_block(block) -> str:
  206. """Cache an ``AudioContent`` block and return a ``MEDIA:<path>`` tag ("" on any failure)."""
  207. return _cache_mcp_media_block(
  208. block, "audio", "cache_audio_from_bytes",
  209. lambda mime: _WAV_MIME_EXT.get(mime) or mimetypes.guess_extension(mime) or ".ogg",
  210. cap_what="audio resource")
  211. def _mcp_resource_filename(uri: str, mime_type: str) -> str:
  212. """Safe display filename from the URI's last path segment, used only as a name hint:
  213. ``cache_document_from_bytes`` re-sanitizes and prefixes it, so remote path components
  214. can't steer the cache location."""
  215. import re as _re
  216. from pathlib import Path
  217. from urllib.parse import urlparse, unquote
  218. name = ""
  219. if uri:
  220. try:
  221. name = Path(unquote(urlparse(str(uri)).path or "")).name
  222. except (ValueError, TypeError):
  223. pass
  224. # Strip control chars (hostile URIs could inject newlines/ANSI into the filename and
  225. # transcript marker) and cap length, preserving the extension.
  226. name = _re.sub(r"[\x00-\x1f\x7f]", "", name).strip()
  227. if len(name) > 150:
  228. stem, dot, ext = name.rpartition(".")
  229. name = stem[: 150 - len(ext) - 1] + "." + ext if dot and 0 < len(ext) <= 12 else name[:150]
  230. if not name or name in {".", ".."}:
  231. ext = mimetypes.guess_extension(_base_mime(mime_type)) or ".bin"
  232. name = f"resource{ext}"
  233. return name
  234. def _render_mcp_dropped_block_notice(block, block_type: str) -> str:
  235. """Inline notice for an unsupported MCP content block (kimi-code#3227): silently dropping it
  236. leaves the model unaware content went missing. Carries whatever handles the block exposes —
  237. mime type, uri, size, name — so the agent can fetch or reason about the missing content."""
  238. details = [f"type={block_type}"]
  239. mime = mcp_field(block, "mime_type", "mimeType", None)
  240. if mime:
  241. details.append(f"mimeType={mime}")
  242. uri = getattr(block, "uri", None) or getattr(getattr(block, "resource", None), "uri", None)
  243. if uri:
  244. details.append(f"uri={uri}")
  245. for size_attr in ("size", "sizeInBytes"):
  246. size = getattr(block, size_attr, None)
  247. if isinstance(size, int):
  248. details.append(f"size={size}")
  249. break
  250. name = getattr(block, "name", None)
  251. if name and isinstance(name, str):
  252. details.append(f"name={name}")
  253. return f"[MCP content dropped: unsupported block ({', '.join(details)})]"
  254. def _render_mcp_resource_block(block, server_name: str = "") -> str:
  255. """Render a ``ResourceLink`` or ``EmbeddedResource`` block as text: embedded text → the
  256. text; embedded blob → decoded (size-capped) into the document cache with a path marker;
  257. link → the URI plus a pointer at the server's read_resource tool (no fetch here — links
  258. are only readable via the originating session). "" for non-resource blocks; failures are
  259. reported inline rather than silently dropped."""
  260. block_type = getattr(block, "type", "")
  261. if block_type == "resource_link" or (hasattr(block, "uri") and not hasattr(block, "resource") and block_type != "text"):
  262. uri = getattr(block, "uri", None)
  263. if not uri:
  264. return ""
  265. name = getattr(block, "name", "") or ""
  266. mime = mcp_field(block, "mime_type", "mimeType", "") or ""
  267. details = f"uri={uri}" + (f", name={name}" if name else "") + (f", mimeType={mime}" if mime else "")
  268. reader = mcp_prefixed_tool_name(server_name, "read_resource") if server_name else "the MCP server's read_resource tool"
  269. return f"[MCP resource link: {details} — fetch it with {reader}]"
  270. resource = getattr(block, "resource", None)
  271. if resource is None:
  272. return ""
  273. text = getattr(resource, "text", None)
  274. if text is not None:
  275. return strip_unicode_tags(str(text))
  276. blob = getattr(resource, "blob", None)
  277. if blob is None:
  278. return ""
  279. uri = str(getattr(resource, "uri", "") or "")
  280. mime = str(mcp_field(resource, "mime_type", "mimeType", "") or "")
  281. raw_bytes, err = _decode_block_b64(
  282. blob, "embedded resource", mime or uri, cap_what="embedded resource", cap_suffix=f", uri={uri}",
  283. decode_fail=f"[MCP embedded resource could not be decoded: {mime or uri}]")
  284. if raw_bytes is None:
  285. return err
  286. kind = mime or "unknown type"
  287. path, err = _write_block_cache(
  288. "cache_document_from_bytes", "embedded resource", "resource", raw_bytes, _mcp_resource_filename(uri, mime),
  289. unavailable=f"[MCP embedded resource received ({len(raw_bytes)} bytes, {kind}) but document cache unavailable in this process]",
  290. failed=f"[MCP embedded resource could not be cached: {mime or uri}]")
  291. if path is None:
  292. return err
  293. return f"[MCP resource saved to {path} ({kind}, {len(raw_bytes)} bytes) — read it with read_file or terminal tools]"