image_routing.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566
  1. """Routing helpers for inbound user-attached images.
  2. ``native`` attaches images as OpenAI-style ``image_url`` parts; ``text`` runs
  3. ``vision_analyze`` up-front and prepends the lossy description (right for
  4. non-vision models). :func:`decide_image_input_mode` picks once per turn from
  5. ``agent.image_input_mode`` (``auto`` | ``native`` | ``text``): in ``auto`` an
  6. explicit ``auxiliary.vision`` backend forces ``text`` even for vision-capable
  7. main models (``native`` is the absolute override); else ``supports_vision``
  8. (config override or catalog) decides. ``vision_analyze`` stays a tool regardless.
  9. """
  10. from __future__ import annotations
  11. import base64
  12. import logging
  13. import mimetypes
  14. import os
  15. import re
  16. from contextlib import suppress
  17. from io import BytesIO
  18. from pathlib import Path
  19. from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple
  20. logger = logging.getLogger(__name__)
  21. _VALID_MODES = frozenset({"auto", "native", "text"})
  22. # Extensions extract_image_refs() auto-attaches. Documents/archives are excluded:
  23. # the gateway routes them via send_document and a PDF must never become a vision part.
  24. _IMAGE_EXTS = (".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".heic")
  25. _IMAGE_EXT_PATTERN = "|".join(e.lstrip(".") for e in _IMAGE_EXTS)
  26. # Absolute / home-relative local image path. Matches the same shape gateway's
  27. # extract_local_files() uses: anchors to ``~/``, ``/``, or a Windows drive
  28. # (``C:\``), ignores matches inside URLs (the ``(?<![/:\w.])`` lookbehind), and
  29. # case-insensitive on the extension.
  30. _LOCAL_IMAGE_PATH_RE = re.compile(
  31. r"(?<![/:\w.])(?:~/|/|[A-Za-z]:[\\/])(?:[\w.\-]+[\\/])*[\w.\-]+\.(?:"
  32. + _IMAGE_EXT_PATTERN
  33. + r")\b",
  34. re.IGNORECASE,
  35. )
  36. _IMAGE_URL_RE = re.compile(
  37. r"https?://[^\s<>\"']+?\.(?:" + _IMAGE_EXT_PATTERN + r")(?:\?[^\s<>\"']*)?", re.IGNORECASE,
  38. )
  39. _CODE_SPAN_RES = (re.compile(r"```[^\n]*\n.*?```", re.DOTALL), re.compile(r"`[^`\n]+`"))
  40. def _matches_outside_code(pattern: re.Pattern, text: str) -> Iterable[str]:
  41. """Yield ``pattern`` matches whose start is not inside a fenced block or inline backticks."""
  42. spans = [(m.start(), m.end()) for p in _CODE_SPAN_RES for m in p.finditer(text)]
  43. return (m.group(0) for m in pattern.finditer(text) if not any(s <= m.start() < e for s, e in spans))
  44. def _existing_file(candidate: str) -> Optional[str]:
  45. """Normalized path when it is a regular file; None otherwise (incl. OSError on pathological input).
  46. The return value is the OS-canonical spelling of the real file (same norm class as
  47. ``str(Path(...))``), so callers can compare it against actual paths — ``~`` expansion
  48. alone would leave the textual ``/`` separators of the source text in place on Windows."""
  49. expanded = os.path.expanduser(candidate)
  50. try:
  51. return os.path.normpath(expanded) if os.path.isfile(expanded) else None
  52. except OSError:
  53. return None
  54. def extract_image_refs(text: str) -> tuple[list[str], list[str]]:
  55. """Scan free-form text for image references → ``(local_paths, urls)``, each
  56. ordered and deduplicated. Local paths must exist as files; URLs are not
  57. validated (the provider fetches them). Code spans are skipped so pasted
  58. snippets aren't live attachments (mirrors ``BaseAdapter.extract_local_files``)."""
  59. if not isinstance(text, str) or not text:
  60. return [], []
  61. local_paths = dict.fromkeys(
  62. p for p in map(_existing_file, _matches_outside_code(_LOCAL_IMAGE_PATH_RE, text)) if p
  63. )
  64. # Trailing punctuation is almost certainly prose ("see https://x/a.png.").
  65. urls = dict.fromkeys(u.rstrip(".,;:!?)]>") for u in _matches_outside_code(_IMAGE_URL_RE, text))
  66. return list(local_paths), list(urls)
  67. _BOOL_TOKENS = {
  68. **dict.fromkeys(("true", "yes", "on", "1"), True),
  69. **dict.fromkeys(("false", "no", "off", "0"), False),
  70. }
  71. def _coerce_capability_bool(raw: Any) -> Optional[bool]:
  72. """Strict boolean coercion for capability overrides: real bools, 0/1 and YAML
  73. boolean tokens only; anything else is None so the caller falls through to
  74. models.dev. ``bool("false")`` is True, so a quoted ``supports_vision: "false"``
  75. would otherwise silently enable native routing on a model that can't handle it."""
  76. if isinstance(raw, bool):
  77. return raw
  78. if isinstance(raw, int):
  79. return bool(raw) if raw in (0, 1) else None
  80. return _BOOL_TOKENS.get(raw.strip().lower()) if isinstance(raw, str) else None
  81. def _dict_or_empty(raw: Any) -> dict[str, Any]:
  82. return raw if isinstance(raw, dict) else {}
  83. def _clean_str(raw: Any) -> str:
  84. return str(raw or "").strip()
  85. def _runtime_main(key: str) -> Any:
  86. """Context-local credential source or stripped runtime text; "" when unavailable."""
  87. try:
  88. from agent.auxiliary_client import _runtime_main_value
  89. value = _runtime_main_value(key)
  90. return value if key == "api_key" else _clean_str(value)
  91. except Exception:
  92. return ""
  93. def _model_supports_vision_override(models_cfg: Any, model: str) -> Optional[bool]:
  94. """Per-model ``supports_vision`` (or ``vision`` alias) from a ``models`` mapping."""
  95. per_model = _dict_or_empty(_dict_or_empty(models_cfg).get(model))
  96. return _coerce_capability_bool(per_model.get("supports_vision", per_model.get("vision")))
  97. def _custom_provider_entries(cfg: dict[str, Any], names: Iterable[str]) -> Iterable[dict[str, Any]]:
  98. """Yield legacy ``custom_providers`` entries matching ``names`` (case-insensitive);
  99. ``names`` is the outer loop so list order cannot let a persisted default shadow the live route."""
  100. entries = _custom_provider_list(cfg)
  101. for wanted in (n.strip().lower() for n in names):
  102. yield from (e for e in entries if _clean_str(e.get("name")).lower() == wanted)
  103. def _custom_provider_list(cfg: dict[str, Any]) -> list[dict[str, Any]]:
  104. """Dict entries of the legacy ``custom_providers`` list (empty when absent/malformed)."""
  105. raw = cfg.get("custom_providers")
  106. return [e for e in raw if isinstance(e, dict)] if isinstance(raw, list) else []
  107. def _supports_vision_override(
  108. cfg: Optional[dict[str, Any]],
  109. provider: str,
  110. model: str,
  111. *,
  112. requested_provider: str = "",
  113. ) -> Optional[bool]:
  114. """Resolve user-declared vision capability from config.yaml; None when unset.
  115. First hit wins: ``model.supports_vision`` → ``providers.<p>.models.<model>``
  116. → legacy ``custom_providers[].models.<model>``. Named custom providers are
  117. rewritten to ``provider="custom"`` at runtime while config keeps the user's
  118. name under ``model.provider``, so the requested, runtime and config
  119. identities are all tried, plus the bare ``<name>`` of any ``custom:<name>``.
  120. """
  121. if not isinstance(cfg, dict):
  122. return None
  123. model_cfg = _dict_or_empty(cfg.get("model"))
  124. top = _coerce_capability_bool(model_cfg.get("supports_vision"))
  125. if top is not None:
  126. return top
  127. candidates: list[str] = []
  128. for candidate in filter(None, (requested_provider, provider, _clean_str(model_cfg.get("provider")))):
  129. candidates.append(candidate)
  130. if candidate.startswith("custom:") and candidate[len("custom:"):]:
  131. candidates.append(candidate[len("custom:"):])
  132. candidates = list(dict.fromkeys(candidates))
  133. providers_cfg = _dict_or_empty(cfg.get("providers"))
  134. model_maps = [_dict_or_empty(providers_cfg.get(p)).get("models") for p in candidates]
  135. model_maps += [entry.get("models") for entry in _custom_provider_entries(cfg, candidates)]
  136. return next((v for v in (_model_supports_vision_override(m, model) for m in model_maps) if v is not None), None)
  137. def _resolve_inference_value(
  138. cfg: Optional[dict[str, Any]],
  139. provider: str,
  140. key: str,
  141. *,
  142. runtime_ok: Callable[[str], bool],
  143. ) -> str:
  144. """``base_url`` / ``api_key`` of the active inference provider. Order: runtime
  145. value (when ``runtime_ok`` accepts it) → ``model.<key>`` → ``providers.<name>.<key>``
  146. → ``custom_providers[].<key>``, ``<name>`` covering the provider and
  147. ``model.provider`` in both bare and ``custom:``-prefixed forms."""
  148. runtime = _runtime_main(key)
  149. # A declared source owns authentication even when its mint fails.
  150. if key == "api_key":
  151. from agent.command_token_source import materialize_probe_api_key
  152. if callable(runtime):
  153. return materialize_probe_api_key(runtime)
  154. runtime = materialize_probe_api_key(runtime)
  155. if runtime and runtime_ok(runtime):
  156. return runtime
  157. if not isinstance(cfg, dict):
  158. return ""
  159. model_cfg = _dict_or_empty(cfg.get("model"))
  160. value = _clean_str(model_cfg.get(key))
  161. if value:
  162. return value
  163. names: set[str] = set()
  164. for p in filter(None, (provider, _clean_str(model_cfg.get("provider")))):
  165. names.add(p)
  166. names.add(p.split(":", 1)[1] if p.lower().startswith("custom:") else f"custom:{p}")
  167. lowered = {n.lower() for n in names}
  168. providers_cfg = _dict_or_empty(cfg.get("providers"))
  169. entries = [e for e in map(providers_cfg.get, names) if isinstance(e, dict)]
  170. entries += [
  171. e for e in _custom_provider_list(cfg)
  172. if _clean_str(e.get("name")) in names or _clean_str(e.get("name")).lower() in lowered
  173. ]
  174. return next((v for v in (_clean_str(e.get(key)) for e in entries) if v), "")
  175. def _resolve_inference_base_url(cfg: Optional[dict[str, Any]], provider: str) -> str:
  176. """Best-effort base URL for the active inference provider; the runtime value is
  177. only trusted when it belongs to the requested provider (or none was requested)."""
  178. requested = _clean_str(provider).lower()
  179. return _resolve_inference_value(
  180. cfg, provider, "base_url",
  181. runtime_ok=lambda _: not requested or requested == _runtime_main("provider").lower(),
  182. )
  183. def _resolve_inference_api_key(cfg: Optional[dict[str, Any]], provider: str) -> str:
  184. """Best-effort API key, resolved like :func:`_resolve_inference_base_url` so it
  185. matches the base URL actually probed; otherwise the local server-type probe hits
  186. a keyed remote endpoint without Authorization and sprays 401s on every image turn.
  187. Mirrors :func:`_resolve_inference_base_url`'s resolution order (runtime value, then ``model.api_key``,
  188. then the providers blocks) so the key matches the base URL actually being probed. See #89863.
  189. """
  190. return _resolve_inference_value(cfg, provider, "api_key", runtime_ok=lambda _: True)
  191. def _should_probe_ollama_vision(provider: str, base_url: str, api_key: str = "") -> bool:
  192. """True when the active provider likely fronts a local Ollama server. Fingerprint
  193. probing is only valid for LOCAL endpoints: remote OpenAI-compatible APIs (sglang,
  194. vLLM) expose Ollama-compat routes that can misidentify, and probing them without
  195. an api_key returns 401 on every leg."""
  196. if _clean_str(provider).lower() == "ollama":
  197. return True
  198. if not base_url:
  199. return False
  200. try:
  201. from agent.model_metadata import detect_local_server_type, is_local_endpoint
  202. # Forward the key: an unauthorized probe can never produce a positive verdict.
  203. return bool(is_local_endpoint(base_url)) and detect_local_server_type(base_url, api_key=api_key) == "ollama"
  204. except Exception:
  205. return False
  206. def _coerce_mode(raw: Any) -> str:
  207. """Normalize a config value into one of the valid modes (default ``auto``)."""
  208. mode = raw.strip().lower() if isinstance(raw, str) else ""
  209. return mode if mode in _VALID_MODES else "auto"
  210. def _explicit_aux_vision_override(cfg: Optional[dict[str, Any]]) -> bool:
  211. """True when the user configured a specific ``auxiliary.vision`` backend — the
  212. de-facto image route in ``auto`` mode even when the main model has native vision.
  213. ``auto``/empty provider with no model and no base_url is not explicit."""
  214. vision = _dict_or_empty(_dict_or_empty(_dict_or_empty(cfg).get("auxiliary")).get("vision"))
  215. return bool(vision) and not (
  216. _clean_str(vision.get("provider")).lower() in {"", "auto"}
  217. and not _clean_str(vision.get("model"))
  218. and not _clean_str(vision.get("base_url"))
  219. )
  220. def _probe_managed_runtime(provider: str, model: str, cfg: Optional[dict[str, Any]]) -> Optional[bool]:
  221. """Managed local runtime verdict: the server receiving the image is the authority
  222. on whether it can see (its /props reports modalities). Cloud catalogs have never
  223. heard of a local GGUF, so without this every local model reads as text-only and
  224. screenshots detour to a cloud auxiliary."""
  225. from hermes_cli.local_runtime.capabilities import is_managed_provider, managed_model_supports_vision
  226. managed = is_managed_provider(provider, _resolve_inference_base_url(cfg, provider) or "")
  227. return managed_model_supports_vision(model) if managed else None
  228. def _probe_models_dev(provider: str, model: str, cfg: Optional[dict[str, Any]]) -> Optional[bool]:
  229. """models.dev catalog verdict. ``allow_network=True`` on purpose: this runs only
  230. when an image needs routing, and the text-only-main guard depends on catalog
  231. data — a cold cache returning "unknown" would reintroduce attempting the call.
  232. The fetch is cached (4h TTL) and backoff-limited."""
  233. from agent.models_dev import get_model_capabilities
  234. # allow_network=True on purpose: vision-capability lookup runs when an image actually needs routing (not
  235. # per turn), and the #31179 text-only-main guard depends on catalog data — a cold cache returning
  236. # "unknown" would fall back to attempting the call and reintroduce the bug. This preserves the
  237. # historical network-on-cold-cache behavior for this one path; the fetch is cached (4h TTL) and
  238. # backoff-limited after failures.
  239. if (provider or "").strip().lower() == "openai-codex":
  240. # A VALID Codex ``-900k`` picker variant is a Hermes-side alias of its base slug; the catalog
  241. # only knows the base, so look that up. The runtime model id stays untouched (the transport
  242. # owns wire normalization) and ineligible ``-900k`` strings pass through unchanged (#102189).
  243. from agent.model_metadata import strip_codex_context_variant_suffix
  244. model = strip_codex_context_variant_suffix(model)
  245. caps = get_model_capabilities(provider, model, allow_network=True)
  246. return None if caps is None else caps.supports_vision
  247. def _probe_ollama(provider: str, model: str, cfg: Optional[dict[str, Any]]) -> Optional[bool]:
  248. """Ollama ``/api/show`` verdict for local endpoints (see :func:`_should_probe_ollama_vision`)."""
  249. base_url = _resolve_inference_base_url(cfg, provider)
  250. if not base_url and _clean_str(provider).lower() == "ollama":
  251. base_url = "http://localhost:11434/v1"
  252. api_key = _resolve_inference_api_key(cfg, provider)
  253. if not _should_probe_ollama_vision(provider, base_url, api_key=api_key):
  254. return None
  255. from agent.model_metadata import query_ollama_supports_vision
  256. return query_ollama_supports_vision(model, base_url, api_key=api_key)
  257. # Capability probes after the config override, in priority order; each returns
  258. # True/False or None (unknown → next probe). Exceptions are logged and treated as None.
  259. _VISION_PROBES: tuple[tuple[str, Callable[..., Optional[bool]]], ...] = (
  260. ("managed-runtime caps lookup", _probe_managed_runtime),
  261. ("caps lookup", _probe_models_dev),
  262. ("ollama vision probe", _probe_ollama),
  263. )
  264. def _lookup_supports_vision(
  265. provider: str,
  266. model: str,
  267. cfg: Optional[dict[str, Any]] = None,
  268. *,
  269. requested_provider: str = "",
  270. ) -> Optional[bool]:
  271. """Return True/False if vision capability can be resolved, None if unknown.
  272. Order: config ``supports_vision`` override → :data:`_VISION_PROBES`
  273. (managed local runtime → models.dev catalog → Ollama probe). A plugin's
  274. per-model ``ProviderProfile.model_capabilities`` reaches this through the
  275. catalog probe. The provider-wide ``ProviderProfile.supports_vision`` flag is
  276. deliberately NOT consulted: it declares that tool-result messages may carry
  277. images (``tools.vision_tools``), not that every model on the provider can see
  278. user attachments — relays such as ``router`` set it for mixed catalogs.
  279. """
  280. # Named custom providers are canonicalized to ``provider="custom"``; the
  281. # original name lives in the context-local main runtime. Borrow it only on an
  282. # exact provider+model match so background/auxiliary lookups never take
  283. # another turn's identity.
  284. if (
  285. not requested_provider
  286. and _runtime_main("provider").lower() == _clean_str(provider).lower()
  287. and _runtime_main("model") == _clean_str(model)
  288. ):
  289. requested_provider = _runtime_main("requested_provider")
  290. override = _supports_vision_override(cfg, provider, model, requested_provider=requested_provider)
  291. if override is not None:
  292. return override
  293. if not provider or not model:
  294. return None
  295. for label, probe in _VISION_PROBES:
  296. try:
  297. verdict = probe(provider, model, cfg)
  298. except Exception as exc: # pragma: no cover - defensive
  299. logger.debug("image_routing: %s failed for %s:%s — %s", label, provider, model, exc)
  300. continue
  301. if verdict is not None:
  302. return verdict
  303. return None
  304. def decide_image_input_mode(
  305. provider: str,
  306. model: str,
  307. cfg: Optional[dict[str, Any]],
  308. *,
  309. requested_provider: str = "",
  310. ) -> str:
  311. """Return ``"native"`` or ``"text"`` for the given turn (``cfg`` None behaves as
  312. auto; ``requested_provider`` is the identity before runtime canonicalization)."""
  313. mode_cfg = _coerce_mode(_dict_or_empty(_dict_or_empty(cfg).get("agent")).get("image_input_mode"))
  314. if mode_cfg != "auto":
  315. return mode_cfg
  316. if _explicit_aux_vision_override(cfg): # auto: an explicit auxiliary.vision backend wins
  317. return "text"
  318. # Keep the three-argument call contract for callers/tests that replace the lookup hook.
  319. extra = {"requested_provider": requested_provider} if requested_provider else {}
  320. return "native" if _lookup_supports_vision(provider, model, cfg, **extra) is True else "text"
  321. # Image size handling is REACTIVE: attach at full size and let
  322. # ``run_agent._try_shrink_image_parts_in_messages`` shrink + retry on rejection
  323. # (e.g. Anthropic's 5 MB ceiling as HTTP 400). Provider ceilings are partial and
  324. # evolving; a proactive table would go stale and silently degrade quality.
  325. # Magic-byte signatures as ((offset, bytes), ...) conjunctions, checked in order.
  326. # Platforms lie about content-type (Discord serves PNG as ``image/webp`` for
  327. # proxied stickers) and Anthropic rejects a mismatched media_type with HTTP 400.
  328. # ISO-BMFF family (HEIC/HEIF/AVIF): 'ftyp' at 4..8, major brand at 8..12.
  329. _FTYP_BRANDS = {
  330. **dict.fromkeys((b"avif", b"avis"), "image/avif"),
  331. **dict.fromkeys((b"heic", b"heix", b"hevc", b"hevx", b"mif1", b"msf1", b"heim", b"heis"), "image/heic"),
  332. }
  333. _MAGIC: tuple[tuple[tuple[tuple[int, bytes], ...], str], ...] = (
  334. (((0, b"\x89PNG\r\n\x1a\n"),), "image/png"),
  335. (((0, b"\xff\xd8\xff"),), "image/jpeg"),
  336. (((0, b"GIF87a"),), "image/gif"), (((0, b"GIF89a"),), "image/gif"),
  337. (((0, b"RIFF"), (8, b"WEBP")), "image/webp"),
  338. (((0, b"BM"),), "image/bmp"),
  339. *((((4, b"ftyp"), (8, brand)), mime) for brand, mime in _FTYP_BRANDS.items()),
  340. (((0, b"II*\x00"),), "image/tiff"), (((0, b"MM\x00*"),), "image/tiff"),
  341. (((0, b"\x00\x00\x01\x00"),), "image/x-icon"),
  342. )
  343. def _sniff_mime_from_bytes(raw: bytes) -> Optional[str]:
  344. """Detect image MIME from magic bytes; None if unrecognised."""
  345. if not raw:
  346. return None
  347. for conditions, mime in _MAGIC:
  348. if all(raw[off:off + len(sig)] == sig for off, sig in conditions):
  349. return mime
  350. # SVG is text: look for an <svg tag near the start (skip BOM/whitespace).
  351. head = raw[:512].lstrip().lower()
  352. return "image/svg+xml" if head.startswith((b"<?xml", b"<svg")) and b"<svg" in head else None
  353. # Formats every major vision provider accepts natively. Anything else is transcoded
  354. # to PNG before declaring media_type or the provider returns HTTP 400 and the turn
  355. # fails; chat platforms freely accept AVIF (Chromium screenshots), HEIC (iPhone),
  356. # TIFF, BMP and ICO. SVG is vector — Pillow cannot rasterize it — so it is skipped.
  357. _UNIVERSALLY_SUPPORTED_MIMES = frozenset({"image/png", "image/jpeg", "image/gif", "image/webp"})
  358. def _transcode_to_png(raw: bytes) -> Optional[bytes]:
  359. """Decode with Pillow and re-encode as PNG; None when impossible. HEIC/HEIF and
  360. AVIF need optional Pillow plugins, registered on demand; a missing plugin just
  361. looks like "can't decode" so the caller skips the image and the turn proceeds."""
  362. try:
  363. from PIL import Image
  364. except ImportError:
  365. logger.info(
  366. "image_routing: Pillow not installed; cannot transcode "
  367. "non-standard image format to PNG. Run `hermes pm repair` to restore Pillow, "
  368. "or convert the image to PNG before sending it."
  369. )
  370. return None
  371. with suppress(Exception):
  372. import pillow_heif # type: ignore
  373. pillow_heif.register_heif_opener()
  374. with suppress(Exception):
  375. import pillow_avif # type: ignore
  376. try:
  377. with Image.open(BytesIO(raw)) as im:
  378. # Normalise exotic modes to RGBA so PNG can serialise and transparency survives.
  379. if im.mode not in {"RGB", "RGBA", "L", "LA", "P"}:
  380. im = im.convert("RGBA")
  381. buf = BytesIO()
  382. im.save(buf, format="PNG", optimize=False)
  383. return buf.getvalue()
  384. except Exception as exc:
  385. logger.info("image_routing: Pillow could not transcode image to PNG -- %s", exc)
  386. return None
  387. _SUFFIX_MIMES = {
  388. ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png",
  389. ".gif": "image/gif", ".webp": "image/webp", ".bmp": "image/bmp",
  390. }
  391. def _guess_mime(path: Path, raw: Optional[bytes] = None) -> str:
  392. """Image MIME for *path*: magic bytes (authoritative) → ``mimetypes`` → suffix → jpeg."""
  393. sniffed = _sniff_mime_from_bytes(raw) if raw is not None else None
  394. mime = sniffed or mimetypes.guess_type(str(path))[0]
  395. if mime and mime.startswith("image/"):
  396. return mime
  397. # mimetypes on some Linux distros mis-maps .jpg; default to jpeg.
  398. return _SUFFIX_MIMES.get(path.suffix.lower(), "image/jpeg")
  399. def _accepted_mimes() -> frozenset:
  400. """Provider-accepted MIME set for the current main runtime. The managed local
  401. server decodes fewer formats (no WebP — and a WebP part fails SILENTLY: the model
  402. confabulates a description), so its narrower set transcodes those here."""
  403. try:
  404. from agent.auxiliary_client import _runtime_main_value
  405. from hermes_cli.local_runtime.capabilities import ACCEPTED_IMAGE_MIMES, is_managed_provider
  406. if is_managed_provider(str(_runtime_main_value("provider") or ""), str(_runtime_main_value("base_url") or "")):
  407. return ACCEPTED_IMAGE_MIMES
  408. except Exception:
  409. pass
  410. return _UNIVERSALLY_SUPPORTED_MIMES
  411. def _file_to_data_url(path: Path) -> Optional[str]:
  412. """Encode a local image as a base64 data URL at native size (the agent retry
  413. loop shrinks on rejection, so lenient providers pay no silent quality tax);
  414. MIMEs outside the accepted set are transcoded to PNG. None when unreadable,
  415. blocked by the read guard, or untranscodable — the caller reports ``skipped``."""
  416. try:
  417. from agent.file_safety import raise_if_read_blocked
  418. raise_if_read_blocked(str(path))
  419. except ValueError as exc:
  420. logger.warning("image_routing: blocked local image attachment %s -- %s", path, exc)
  421. return None
  422. except Exception:
  423. pass # Keep attachment routing best-effort if the guard itself is unavailable.
  424. try:
  425. raw = path.read_bytes()
  426. except Exception as exc:
  427. logger.warning("image_routing: failed to read %s — %s", path, exc)
  428. return None
  429. mime = _guess_mime(path, raw=raw)
  430. if mime not in _accepted_mimes():
  431. if (transcoded := _transcode_to_png(raw)) is None:
  432. logger.warning(
  433. "image_routing: %s is %s which is not accepted by the active provider "
  434. "and could not be transcoded to PNG; skipping this attachment.", path, mime,
  435. )
  436. return None
  437. logger.info("image_routing: transcoded %s (%s) -> image/png for provider compatibility", path.name, mime)
  438. raw, mime = transcoded, "image/png"
  439. return f"data:{mime};base64,{base64.b64encode(raw).decode('ascii')}"
  440. def build_native_content_parts(
  441. user_text: str,
  442. image_paths: list[str],
  443. image_urls: Optional[list[str]] = None,
  444. ) -> tuple[list[dict[str, Any]], list[str]]:
  445. """Build an OpenAI-style ``content`` list for a user turn.
  446. Local paths become base64 ``data:`` URLs; remote URLs pass through verbatim.
  447. When any image attaches, one text part combines the caption (or a neutral
  448. default) with a ``[Image attached at: <path>]`` / ``[Image attached: <url>]``
  449. hint per image — a string handle for tools taking an image path/URL, mirroring
  450. ``Runner._enrich_message_with_vision``. Returns ``(content_parts, skipped)``;
  451. ``skipped`` holds unreadable local paths (URLs are never skipped).
  452. """
  453. skipped: list[str] = []
  454. attached: list[tuple[str, str]] = [] # (url, hint)
  455. for raw_path in image_paths:
  456. p = Path(raw_path)
  457. data_url = _file_to_data_url(p) if p.exists() and p.is_file() else None
  458. if data_url:
  459. attached.append((data_url, f"[Image attached at: {raw_path}]"))
  460. else:
  461. skipped.append(str(raw_path))
  462. attached += [(u, f"[Image attached: {u}]") for u in ((u or "").strip() for u in image_urls or []) if u]
  463. text = (user_text or "").strip()
  464. if not attached:
  465. return ([{"type": "text", "text": text}] if text else []), skipped
  466. combined_text = f"{text or 'What do you see in this image?'}\n\n" + "\n".join(h for _, h in attached)
  467. image_parts = [{"type": "image_url", "image_url": {"url": u}} for u, _ in attached]
  468. return [{"type": "text", "text": combined_text}, *image_parts], skipped
  469. __all__ = ["build_native_content_parts", "decide_image_input_mode", "extract_image_refs"]