vision_message_prep.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359
  1. """Image-part handling for ``AIAgent`` API messages.
  2. Vision capability probes, non-vision text fallbacks (cached ``vision_analyze`` descriptions), tool-result
  3. image stripping, and provider quirks (Anthropic dot preservation, Qwen portal message shaping).
  4. """
  5. import logging
  6. import asyncio
  7. import base64
  8. import copy
  9. import hashlib
  10. import json
  11. import os
  12. import tempfile
  13. from pathlib import Path
  14. from typing import Any, List, Optional
  15. from agent.lazy_forward import forward_static as _forward_static
  16. from agent.tool_dispatch_helpers import _is_multimodal_tool_result, _multimodal_text_summary
  17. from utils import base_url_host_matches, base_url_hostname
  18. # Same logger name as the origin module so log records / caplog filters are unchanged.
  19. logger = logging.getLogger("run_agent")
  20. _IMAGE_PART_TYPES = {"image_url", "input_image"}
  21. _TEXT_PART_TYPES = {"text", "input_text"}
  22. _DATA_URL_SUFFIXES = {
  23. "image/png": ".png", "image/gif": ".gif", "image/webp": ".webp", "image/jpeg": ".jpg", "image/jpg": ".jpg"
  24. }
  25. def _is_image_part(part: Any) -> bool:
  26. return isinstance(part, dict) and part.get("type") in _IMAGE_PART_TYPES
  27. def _salvage_text_parts(content: list, *, any_dict_text: bool) -> list[str]:
  28. """Stripped, non-empty text from string parts and text-typed dict parts (or any dict's
  29. ``text`` when ``any_dict_text``), in order."""
  30. texts: list[str] = []
  31. for part in content:
  32. if isinstance(part, str):
  33. text = part.strip()
  34. elif isinstance(part, dict) and (any_dict_text or part.get("type") in _TEXT_PART_TYPES):
  35. text = str(part.get("text", "") or "").strip()
  36. else:
  37. continue
  38. if text:
  39. texts.append(text)
  40. return texts
  41. def _provider_model_key(agent: Any) -> tuple[str, str]:
  42. """``(provider.lower(), model)`` as recorded in ``_no_list_tool_content_models``.
  43. Module-level so ``MagicMock(spec=AIAgent)`` agents in tests don't swallow it."""
  44. return (
  45. (getattr(agent, "provider", "") or "").strip().lower(),
  46. (getattr(agent, "model", "") or "").strip(),
  47. )
  48. class VisionMessagePrepMixin:
  49. """Vision probes + image-part fallbacks for outgoing messages (see module docstring)."""
  50. @staticmethod
  51. def _content_has_image_parts(content: Any) -> bool:
  52. return isinstance(content, list) and any(_is_image_part(part) for part in content)
  53. # 20 MB base64 ≈ 15 MB decoded — prevents OOM from an oversized data: URL in a shared gateway process.
  54. _MAX_DATA_URL_BASE64_BYTES = 20 * 1024 * 1024
  55. @staticmethod
  56. def _materialize_data_url_for_vision(image_url: str) -> tuple[str, Optional[Path]]:
  57. header, _, data = str(image_url or "").partition(",")
  58. if len(data) > VisionMessagePrepMixin._MAX_DATA_URL_BASE64_BYTES:
  59. logger.warning("data-URL payload too large (%d bytes), skipping", len(data))
  60. return "", None
  61. mime = header[len("data:"):].split(";", 1)[0].strip() if header.startswith("data:") else ""
  62. suffix = _DATA_URL_SUFFIXES.get(mime if mime.startswith("image/") else "image/jpeg", ".jpg")
  63. tmp = tempfile.NamedTemporaryFile(prefix="anthropic_image_", suffix=suffix, delete=False)
  64. try:
  65. with tmp:
  66. tmp.write(base64.b64decode(data))
  67. except Exception:
  68. # delete=False means a corrupt/unsupported data URL would otherwise
  69. # leak a zero-byte temp file on every failed materialization.
  70. try:
  71. os.unlink(tmp.name)
  72. except OSError:
  73. pass
  74. raise
  75. return tmp.name, Path(tmp.name)
  76. def _describe_image_for_anthropic_fallback(self, image_url: str, role: str) -> str:
  77. cache_key = hashlib.sha256(str(image_url or "").encode("utf-8")).hexdigest()
  78. cached = self._anthropic_image_fallback_cache.get(cache_key)
  79. if cached:
  80. return cached
  81. role_label = {"assistant": "assistant", "tool": "tool result"}.get(role, "user")
  82. analysis_prompt = (
  83. "Describe everything visible in this image in thorough detail. "
  84. "Include any text, code, UI, data, objects, people, layout, colors, "
  85. "and any other notable visual information."
  86. )
  87. vision_source = str(image_url or "")
  88. is_data_url = vision_source.startswith("data:")
  89. cleanup_path: Optional[Path] = None
  90. if is_data_url:
  91. vision_source, cleanup_path = self._materialize_data_url_for_vision(vision_source)
  92. try:
  93. from tools.vision_tools import vision_analyze_tool
  94. result_json = asyncio.run(vision_analyze_tool(image_url=vision_source, user_prompt=analysis_prompt))
  95. result = json.loads(result_json) if isinstance(result_json, str) else {}
  96. description = (result.get("analysis") or "").strip()
  97. except Exception as e:
  98. description = f"Image analysis failed: {e}"
  99. finally:
  100. if cleanup_path and cleanup_path.exists():
  101. try:
  102. cleanup_path.unlink()
  103. except OSError:
  104. pass
  105. note = f"[The {role_label} attached an image. Here's what it contains:\n{description or 'Image analysis failed.'}]"
  106. if vision_source and not is_data_url:
  107. note += f"\n[If you need a closer look, use vision_analyze with image_url: {vision_source}]"
  108. self._anthropic_image_fallback_cache[cache_key] = note
  109. return note
  110. def _model_supports_vision(self) -> bool:
  111. """True if the active provider+model reports native vision (config override
  112. > models.dev; see ``image_routing._supports_vision_override``)."""
  113. try:
  114. from hermes_cli.config import load_config
  115. from agent.image_routing import _lookup_supports_vision
  116. provider = (getattr(self, "provider", "") or "").strip()
  117. model = (getattr(self, "model", "") or "").strip()
  118. return _lookup_supports_vision(provider, model, load_config()) is True
  119. except Exception:
  120. return False
  121. def _provider_supports_vision_tool_messages(self) -> bool:
  122. """True if the active provider accepts list-type tool content (some, e.g. Xiaomi MiMo, take
  123. multimodal user messages but 400 on list-type tool content; profile ``supports_vision_tool_messages``)."""
  124. try:
  125. from providers import routed_model_rejects_vision_tool_messages
  126. return not routed_model_rejects_vision_tool_messages(
  127. (getattr(self, "provider", "") or "").strip(),
  128. (getattr(self, "model", "") or "").strip(),
  129. )
  130. except Exception:
  131. pass
  132. return True # default: assume compatible
  133. def _preprocess_anthropic_content(self, content: Any, role: str) -> Any:
  134. if not self._content_has_image_parts(content):
  135. return content
  136. image_notes: list[str] = []
  137. for part in filter(_is_image_part, content):
  138. image_data = part.get("image_url", {})
  139. image_url = image_data.get("url", "") if isinstance(image_data, dict) else str(image_data or "")
  140. image_notes.append(
  141. self._describe_image_for_anthropic_fallback(image_url, role) if image_url
  142. else "[An image was attached but no image source was available.]"
  143. )
  144. # Text parts and unknown dict types both contribute their ``text``.
  145. prefix = "\n\n".join(note for note in image_notes if note).strip()
  146. suffix = "\n".join(_salvage_text_parts(content, any_dict_text=True)).strip()
  147. if prefix and suffix:
  148. return f"{prefix}\n\n{suffix}"
  149. return prefix or suffix or "[A multimodal message was converted to text for Anthropic compatibility.]"
  150. def _get_transport(self, api_mode: str | None = None):
  151. """Return the cached transport for the given (or current) api_mode (lazy; None if unregistered)."""
  152. mode = api_mode or self.api_mode
  153. cache = getattr(self, "_transport_cache", None)
  154. if cache is None:
  155. cache = self._transport_cache = {}
  156. if cache.get(mode) is None:
  157. from agent.transports import get_transport
  158. cache[mode] = get_transport(mode)
  159. return cache[mode]
  160. def _prepare_messages_for_non_vision_model(self, api_messages: list) -> list:
  161. """Replace native image parts with cached vision_analyze text when the active model lacks vision;
  162. vision-capable models pass through unchanged (the provider adapter handles image parts natively)."""
  163. if not any(
  164. isinstance(msg, dict) and self._content_has_image_parts(msg.get("content")) for msg in api_messages
  165. ) or self._model_supports_vision():
  166. return api_messages
  167. transformed = copy.deepcopy(api_messages)
  168. for msg in transformed:
  169. if isinstance(msg, dict):
  170. msg["content"] = self._preprocess_anthropic_content(
  171. msg.get("content"), str(msg.get("role", "user") or "user")
  172. )
  173. return transformed
  174. # Same transform for the Anthropic route (callers/tests patch this name independently).
  175. _prepare_anthropic_messages_for_api = _prepare_messages_for_non_vision_model
  176. def _tool_result_content_for_active_model(self, tool_name: str, result: Any) -> Any:
  177. """Tool message content that is safe for the active model. Text-only providers must not receive
  178. image parts: a rejected tool result becomes canonical history and can break the next user turn."""
  179. if not _is_multimodal_tool_result(result):
  180. return result
  181. content = result.get("content") or []
  182. if not self._content_has_image_parts(content):
  183. return content
  184. if self._model_supports_vision():
  185. # Vision on paper, but the provider rejects list-type tool content (or we already learned that
  186. # in-session): short-circuit to a text summary.
  187. if not self._provider_supports_vision_tool_messages():
  188. logger.debug(
  189. "Tool %s: provider %s does not accept list-type tool "
  190. "content — sending text summary",
  191. tool_name, getattr(self, "provider", ""),
  192. )
  193. return _multimodal_text_summary(result)
  194. key = _provider_model_key(self)
  195. if key in (getattr(self, "_no_list_tool_content_models", None) or ()):
  196. logger.debug(
  197. "Tool %s: model %s/%s known to reject list-type tool "
  198. "content this session — sending text summary",
  199. tool_name, key[0], key[1],
  200. )
  201. return _multimodal_text_summary(result)
  202. return content
  203. summary = _multimodal_text_summary(result)
  204. if tool_name == "computer_use":
  205. return json.dumps({
  206. "error": (
  207. "computer_use returned screenshot/image content, but the active "
  208. "model/provider does not support image input. Switch to a "
  209. "vision-capable model for desktop computer use, or use browser "
  210. "tools for browser tasks."
  211. ),
  212. "text_summary": summary,
  213. })
  214. logger.warning(
  215. "Tool %s returned image content for non-vision model %s/%s; "
  216. "falling back to text summary",
  217. tool_name, self.provider, self.model,
  218. )
  219. return summary
  220. _try_shrink_image_parts_in_messages = _forward_static("agent.conversation_compression", "try_shrink_image_parts_in_messages")
  221. def _try_strip_image_parts_from_tool_messages(
  222. self, api_messages: list, *, remember_model: bool = True
  223. ) -> bool:
  224. """Downgrade list-type tool messages to text in place; True if any were downgraded.
  225. Recovery for providers that 400 on list-type tool content (e.g. MiMo "text is not set"). By default
  226. records (provider, model) in ``_no_list_tool_content_models`` so later results downgrade without a
  227. round-trip; 413 recovery passes ``remember_model=False`` (body too large ≠ provider rejects lists).
  228. """
  229. if not isinstance(api_messages, list):
  230. return False
  231. if remember_model:
  232. # Record (provider, model) so we don't relearn this lesson.
  233. key = _provider_model_key(self)
  234. if not hasattr(self, "_no_list_tool_content_models"):
  235. self._no_list_tool_content_models = set()
  236. if key[1]: # only record when we actually have a model id
  237. self._no_list_tool_content_models.add(key)
  238. changed = False
  239. for msg in api_messages:
  240. if not isinstance(msg, dict) or msg.get("role") != "tool":
  241. continue
  242. content = msg.get("content")
  243. # List content without image parts is left alone; stripping wouldn't reduce ambiguity.
  244. if not self._content_has_image_parts(content):
  245. continue
  246. # Salvage any text parts so the model still sees some signal.
  247. msg["content"] = "\n\n".join(_salvage_text_parts(content, any_dict_text=False)) or (
  248. "[image content removed — provider does not accept "
  249. "list-type tool message content]"
  250. )
  251. changed = True
  252. return changed
  253. def _anthropic_preserve_dots(self) -> bool:
  254. """True for anthropic-compatible endpoints that keep dots in model names (DashScope, MiniMax, Xiaomi
  255. MiMo, OpenCode Go/Zen, ZAI/Zhipu; Bedrock's dotted inference-profile IDs 400 on the hyphenated form).
  256. Alibaba/DashScope keeps dots (e.g. qwen3.5-plus). OpenCode Go/Zen keeps dots for non-Claude models
  257. (e.g. minimax-m2.5-free). ``global.anthropic.claude-opus-4-7``,
  258. ``us.anthropic.claude-sonnet-4-5-20250929-v1:0``) and rejects the hyphenated form with ``HTTP 400
  259. The provided model identifier is invalid``. Regression for #11976; mirrors the opencode-go fix for
  260. #5211
  261. """
  262. if (getattr(self, "provider", "") or "").lower() in {
  263. "alibaba", "minimax", "minimax-cn", "opencode-go", "opencode-zen", "zai", "bedrock", "xiaomi", "vertex",
  264. }:
  265. return True
  266. base = (getattr(self, "base_url", "") or "").lower()
  267. host = base_url_hostname(base)
  268. return (
  269. "dashscope" in host
  270. or base_url_host_matches(base, "aliyuncs.com")
  271. or "minimax" in host
  272. or (base_url_host_matches(base, "opencode.ai") and "/zen/" in base)
  273. or base_url_host_matches(base, "bigmodel.cn")
  274. or base_url_host_matches(base, "xiaomimimo.com")
  275. # Vertex AI OpenAI-compat endpoint — Gemini model ids keep dots
  276. # (e.g. google/gemini-3.5-flash); the hyphenated form is wrong.
  277. or base_url_host_matches(base, "aiplatform.googleapis.com")
  278. # AWS Bedrock runtime endpoints — defense-in-depth when
  279. # ``provider`` is unset but ``base_url`` still names Bedrock.
  280. or host.startswith("bedrock-runtime.")
  281. )
  282. def _is_qwen_portal(self) -> bool:
  283. """Return True when the base URL targets Qwen Portal."""
  284. return base_url_host_matches(self._base_url_lower, "portal.qwen.ai")
  285. def _qwen_prepare_chat_messages(self, api_messages: list) -> list:
  286. """Deep-copy ``api_messages`` and shape them for Qwen Portal (see the in-place variant)."""
  287. prepared = copy.deepcopy(api_messages)
  288. self._qwen_prepare_chat_messages_inplace(prepared)
  289. return prepared
  290. def _qwen_prepare_chat_messages_inplace(self, messages: list) -> None:
  291. """Qwen Portal shaping, in place: every content becomes a list of parts (bare strings → text
  292. dicts, dicts kept), then ``cache_control`` is injected on the last part of the system message."""
  293. for msg in messages:
  294. if not isinstance(msg, dict):
  295. continue
  296. content = msg.get("content")
  297. if isinstance(content, str):
  298. msg["content"] = [{"type": "text", "text": content}]
  299. elif isinstance(content, list):
  300. normalized_parts = [
  301. {"type": "text", "text": part} if isinstance(part, str) else part
  302. for part in content if isinstance(part, (str, dict))
  303. ]
  304. if normalized_parts:
  305. msg["content"] = normalized_parts
  306. for msg in messages:
  307. if isinstance(msg, dict) and msg.get("role") == "system":
  308. content = msg.get("content")
  309. if isinstance(content, list) and content and isinstance(content[-1], dict):
  310. content[-1]["cache_control"] = {"type": "ephemeral"}
  311. break