vision_tools.py 58 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192
  1. #!/usr/bin/env python3
  2. """Vision tools: ``vision_analyze`` (image) and ``video_analyze``.
  3. Images resolve through :mod:`tools.image_source`, are normalized to a provider-supported format
  4. (:mod:`tools.vision_tools_image_prep`), then either attach natively to a vision-capable main
  5. model (multimodal tool-result envelope) or are described by the auxiliary vision LLM router.
  6. """
  7. import base64
  8. import asyncio
  9. import json
  10. from concurrent.futures import ThreadPoolExecutor
  11. from io import BytesIO
  12. import logging
  13. import os
  14. import uuid
  15. from pathlib import Path
  16. from typing import Any, Awaitable, Callable, Dict, NamedTuple, Optional
  17. from urllib.parse import urlparse
  18. import httpx
  19. from hermes_dir_acl import make_user_dir
  20. # ``agent.auxiliary_client`` costs ~50 ms cold; only the handlers need it. Both names stay
  21. # module attributes so tests can patch ``tools.vision_tools.async_call_llm`` (truthy-skip: mocks win).
  22. async_call_llm: Any = None
  23. extract_content_or_reasoning: Any = None
  24. def _load_auxiliary_client() -> None:
  25. global async_call_llm, extract_content_or_reasoning
  26. if async_call_llm is None or extract_content_or_reasoning is None:
  27. from agent import auxiliary_client as _aux
  28. async_call_llm = async_call_llm or _aux.async_call_llm
  29. extract_content_or_reasoning = extract_content_or_reasoning or _aux.extract_content_or_reasoning
  30. from hermes_constants import get_hermes_dir
  31. from tools.debug_helpers import DebugSession
  32. from tools.website_policy import check_website_access
  33. from tools.vision_tools_history_budget import (
  34. native_turn_duplicate as _native_turn_duplicate,
  35. record_embed as _record_embed,
  36. release_embed as _release_embed,
  37. repeat_refusal as _repeat_refusal,
  38. resolve_repeat_cap as _resolve_repeat_cap,
  39. resolve_embed_target_bytes as _resolve_embed_target_bytes,
  40. )
  41. from tools.vision_tools_image_prep import (
  42. _VISION_MAX_VALIDATED_AGGREGATE_PIXELS,
  43. _VISION_MAX_VALIDATED_FRAME_COUNT,
  44. _crop_image_region,
  45. _determine_mime_type,
  46. _image_exceeds_dimension,
  47. _normalize_to_supported_image,
  48. _validate_raster_image_decodable)
  49. logger = logging.getLogger(__name__)
  50. _debug = DebugSession("vision_tools", env_var="VISION_TOOLS_DEBUG")
  51. def _cfg_auxiliary(*keys: str, default=None):
  52. """``auxiliary.<keys...>`` from config.yaml; ``default`` when config is unavailable."""
  53. try:
  54. from hermes_cli.config import cfg_get, load_config
  55. return cfg_get(load_config(), "auxiliary", *keys, default=default)
  56. except Exception:
  57. return default
  58. def _read_vision_setting(env_var: str, key: str, cast, minimum=None):
  59. """Env var → ``auxiliary.vision.<key>`` → None. Values that fail ``cast`` or fall below
  60. ``minimum`` are skipped in favor of the next source (a cap can never be disabled by a bad value)."""
  61. def _accept(raw):
  62. try:
  63. val = cast(raw)
  64. except (TypeError, ValueError):
  65. return None
  66. return val if minimum is None or val >= minimum else None
  67. val = _accept(os.getenv(env_var, "").strip() or None)
  68. return val if val is not None else _accept(_cfg_auxiliary("vision", key))
  69. # HTTP download timeout (separate from ``auxiliary.vision.timeout``, which governs the LLM call).
  70. _VISION_DOWNLOAD_TIMEOUT = _read_vision_setting("HERMES_VISION_DOWNLOAD_TIMEOUT", "download_timeout", float)
  71. if _VISION_DOWNLOAD_TIMEOUT is None:
  72. _VISION_DOWNLOAD_TIMEOUT = 30.0
  73. # Hard cap on downloaded media (50 MB): bounds memory/disk against attacker-hosted files.
  74. _VISION_MAX_DOWNLOAD_BYTES = 50 * 1024 * 1024
  75. # CPU-burst cap: a turn can fan out dozens of vision_analyze calls (base64 encode + Pillow
  76. # resize each) that would saturate every core and starve the shared event loop. Only the CPU
  77. # burst is capped (LLM calls stay concurrent), on a dedicated executor sized to usable cores.
  78. # Must be a threading primitive — each call runs via model_tools._run_async on a PER-THREAD
  79. # loop, so an asyncio semaphore cannot coordinate; the default executor is shared with the
  80. # gateway/web server and is deliberately NOT used.
  81. def _detect_host_cpus() -> int:
  82. """Usable CPU count (``sched_getaffinity`` honors cpuset pinning), at least 1."""
  83. try:
  84. return max(1, len(os.sched_getaffinity(0))) # type: ignore[attr-defined]
  85. except (AttributeError, OSError):
  86. return max(1, os.cpu_count() or 1)
  87. def _resolve_vision_cpu_workers() -> int:
  88. """HERMES_VISION_MAX_CONCURRENCY → ``auxiliary.vision.max_concurrency`` → host cores (< 1 ignored)."""
  89. val = _read_vision_setting("HERMES_VISION_MAX_CONCURRENCY", "max_concurrency", int, minimum=1)
  90. return val or _detect_host_cpus()
  91. _VISION_CPU_WORKERS = _resolve_vision_cpu_workers()
  92. _vision_cpu_executor = ThreadPoolExecutor(
  93. max_workers=_VISION_CPU_WORKERS, thread_name_prefix="vision-encode",
  94. )
  95. async def _run_encode_on_cpu_executor(fn, *args, **kwargs):
  96. """Run a sync encode/resize callable on the bounded vision CPU executor (never the LLM call)."""
  97. import functools
  98. loop = asyncio.get_running_loop()
  99. return await loop.run_in_executor(_vision_cpu_executor, functools.partial(fn, *args, **kwargs))
  100. async def _validate_image_url_async(url: str) -> bool:
  101. """HTTP(S) shape check (scheme, netloc; extension-less CDN URLs pass) + SSRF guard with DNS
  102. off the event loop."""
  103. if not (isinstance(url, str) and url.startswith(("http://", "https://")) and urlparse(url).netloc):
  104. return False
  105. from tools.url_safety import async_is_safe_url
  106. return await async_is_safe_url(url)
  107. def _is_retryable_download_error(error: Exception) -> bool:
  108. """Transient failures only: 429, 5xx, transport errors and anything unclassified. Fail-fast on
  109. other 4xx, PermissionError (policy/SSRF block) and ValueError (too large / blocked redirect)."""
  110. if isinstance(error, (PermissionError, ValueError)):
  111. return False
  112. if isinstance(error, httpx.HTTPStatusError):
  113. status = error.response.status_code
  114. return status == 429 or not 400 <= status < 500
  115. return True
  116. _DOWNLOAD_USER_AGENT = (
  117. "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
  118. "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
  119. )
  120. def _managed_install() -> bool:
  121. """True when a package manager (NixOS) owns this install's modes.
  122. Mirrors the check ``hermes_cli.config._secure_dir`` makes internally, read
  123. here so the *creation* mode honours the same carve-out as reconciliation.
  124. Import is local and failure means "not managed": an unimportable config
  125. module is the single-user source-install case, where 0700 is correct.
  126. """
  127. try:
  128. from hermes_cli.config import is_managed
  129. return bool(is_managed())
  130. except Exception: # pragma: no cover - defensive
  131. return False
  132. def _secure_cache_dir(new_subpath: str, old_name: str) -> Path:
  133. """Resolve a Hermes media-cache dir, creating it owner-only (0700).
  134. A downloaded image or video is as sensitive as whatever the user pointed
  135. the agent at — a private attachment, an internal screenshot, a document
  136. scan. Created with a bare ``mkdir(parents=True, exist_ok=True)`` these
  137. inherited the umask and landed 0755, readable by every other local
  138. account. ``HERMES_HOME`` is 0700 by default so default-config exposure is
  139. narrow; the concrete scenario is the documented ``HERMES_HOME_MODE=0701``
  140. hatch (letting nginx/caddy traverse to a served subdirectory), where a
  141. 0755 child really is world-readable.
  142. The mode is passed to ``mkdir`` so it is set *at creation*, leaving no
  143. window where the directory sits world-readable before a follow-up chmod.
  144. Policy is then reconciled through ``hermes_cli.config._secure_dir`` — the
  145. house helper — rather than a hand-rolled chmod, which is what keeps this
  146. correct off a single-user desktop: managed/NixOS installs are skipped,
  147. ``HERMES_HOME_MODE`` stays honoured, and ``HERMES_UID``/``HERMES_GID``
  148. ownership is applied so a root-created dir does not lock out uid-mapped
  149. Docker workers (#34107).
  150. The managed/NixOS carve-out applies to **creation as well as
  151. reconciliation**. ``cache/vision`` and ``cache/video`` are not among the
  152. directories the NixOS module's ``systemd.tmpfiles`` rules pre-create, so
  153. they are made lazily at runtime; a hardcoded 0700 here would be the only
  154. thing setting their mode and would silently override a design that pins
  155. ``stateDir/.hermes`` to ``2770`` and runs the gateway with ``UMask =
  156. "0007"`` so "interactive users in the hermes group can read/write"
  157. gateway-created state. On such a host the gateway and a hostUsers CLI
  158. share one ``$HERMES_HOME``, so a 0700 cache created by whichever ran
  159. first makes vision fail with EACCES for the other. Skipping the explicit
  160. mode there lets the inherited setgid + umask land 2770, matching
  161. ``ensure_hermes_home``'s managed branch and its ``logs/curator``
  162. lazy-mkdir precedent.
  163. Running it unconditionally also heals a directory an older Hermes left at
  164. 0755. That retroactive tighten is safe *here* because this is
  165. Hermes-private scratch that the same user re-reads in the same call —
  166. there is no user-shared content to strand. Note ``parents=True`` applies
  167. the mode to the leaf only, so an intermediate ``cache/`` keeps its default
  168. mode; it is shared with other subsystems and holds only directory names.
  169. """
  170. cache_dir = get_hermes_dir(new_subpath, old_name)
  171. if _managed_install():
  172. # Managed mode: the NixOS-configured umask/setgid owns the mode, and
  173. # _secure_dir would no-op here anyway.
  174. cache_dir.mkdir(parents=True, exist_ok=True)
  175. return cache_dir
  176. # mode= is honored only for the leaf and is not further masked by umask
  177. # for the bits we care about; _secure_dir reconciles anything unusual.
  178. make_user_dir(cache_dir, parents=True, exist_ok=True, mode=0o700)
  179. try:
  180. from hermes_cli.config import _secure_dir
  181. _secure_dir(cache_dir)
  182. except Exception as exc: # pragma: no cover - defensive
  183. logger.debug("vision: cache dir chmod skipped: %s", exc)
  184. return cache_dir
  185. def _write_private_bytes(path: Path, data: bytes) -> None:
  186. """Write ``data`` to ``path`` with owner-only (0600) permissions.
  187. ``Path.write_bytes`` lands 0644 under a default umask. The enclosing cache
  188. dir is 0700, so this matters only under ``HERMES_HOME_MODE=0701`` — but
  189. ``tools.computer_use.tool`` already writes private captures into this
  190. very directory, and one directory with two file-mode conventions is the
  191. kind of inconsistency that rots. POSIX mode bits are advisory on Windows
  192. (``os.chmod`` there only toggles the read-only flag), so on an ``os.open``
  193. failure this degrades to a plain write rather than failing the download.
  194. """
  195. flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
  196. try:
  197. fd = os.open(str(path), flags, 0o600)
  198. except OSError:
  199. path.write_bytes(data)
  200. return
  201. with os.fdopen(fd, "wb") as handle:
  202. handle.write(data)
  203. async def _ssrf_redirect_guard(response):
  204. """Re-validate each redirect target (a public URL 302ing to 169.254.169.254 would otherwise
  205. bypass the pre-flight check). Async because httpx.AsyncClient awaits hooks."""
  206. from tools.url_safety import async_is_safe_url, redirect_target_from_response
  207. redirect_url = redirect_target_from_response(response)
  208. if redirect_url and not await async_is_safe_url(redirect_url):
  209. raise ValueError(f"Blocked redirect to private/internal address: {redirect_url}")
  210. async def _download_media(
  211. url: str, destination: Path, max_retries: int, *,
  212. media_label: str, accept: str, max_bytes: int, timeout: float, retry_all: bool) -> Path:
  213. """SSRF-safe streaming download with exponential backoff (2s/4s/8s). ``retry_all=False`` (images)
  214. retries only :func:`_is_retryable_download_error` errors — a 404/403 never succeeds on retry."""
  215. from utils import atomic_replace
  216. from tools.url_safety import create_ssrf_safe_async_client
  217. destination.parent.mkdir(parents=True, exist_ok=True)
  218. last_error = None
  219. for attempt in range(max_retries):
  220. try:
  221. blocked = check_website_access(url)
  222. if blocked:
  223. raise PermissionError(blocked["message"])
  224. # follow_redirects for CDNs; the client validates DNS at connect
  225. # time and the hook re-validates each redirect target.
  226. async with create_ssrf_safe_async_client(
  227. timeout=timeout, follow_redirects=True, event_hooks={"response": [_ssrf_redirect_guard]},
  228. ) as client, client.stream(
  229. "GET", url, headers={"User-Agent": _DOWNLOAD_USER_AGENT, "Accept": accept},
  230. ) as response:
  231. response.raise_for_status()
  232. # Content-Length gives an early reject but servers can omit or lie,
  233. # so the streaming cap below is authoritative.
  234. cl = response.headers.get("content-length")
  235. try:
  236. declared_size = int(cl) if cl else None
  237. except ValueError:
  238. declared_size = None
  239. if declared_size is not None and declared_size > max_bytes:
  240. raise ValueError(f"{media_label} too large ({declared_size} bytes, max {max_bytes})")
  241. blocked = check_website_access(str(response.url))
  242. if blocked:
  243. raise PermissionError(blocked["message"])
  244. # Stream to a temp file, atomically moved onto destination on success.
  245. tmp_destination = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
  246. bytes_written = 0
  247. try:
  248. with tmp_destination.open("wb") as f:
  249. async for chunk in response.aiter_bytes():
  250. if not chunk:
  251. continue
  252. bytes_written += len(chunk)
  253. if bytes_written > max_bytes:
  254. raise ValueError(
  255. f"{media_label} too large ({bytes_written} bytes, max {max_bytes})"
  256. )
  257. f.write(chunk)
  258. atomic_replace(tmp_destination, destination)
  259. except Exception:
  260. try:
  261. tmp_destination.unlink(missing_ok=True)
  262. except OSError:
  263. logger.debug("Could not delete partial download: %s", tmp_destination, exc_info=True)
  264. raise
  265. return destination
  266. except Exception as e:
  267. last_error = e
  268. if attempt >= max_retries - 1 or (not retry_all and not _is_retryable_download_error(e)):
  269. logger.error("%s download failed after %s attempt(s): %s",
  270. media_label, attempt + 1, str(e)[:100], exc_info=True)
  271. if not retry_all:
  272. raise
  273. break
  274. wait_time = 2 ** (attempt + 1)
  275. logger.warning("%s download failed (attempt %s/%s): %s",
  276. media_label, attempt + 1, max_retries, str(e)[:50])
  277. if not retry_all:
  278. logger.warning("Retrying in %ss...", wait_time)
  279. await asyncio.sleep(wait_time)
  280. # Reaching here means max_retries was non-positive (or video exhausted retries).
  281. if last_error is not None:
  282. raise last_error
  283. raise RuntimeError(
  284. f"_download_{media_label.lower()} exited retry loop without attempting (max_retries={max_retries})")
  285. async def _download_image(image_url: str, destination: Path, max_retries: int = 3) -> Path:
  286. """Download an image with SSRF protection and error-class-aware retry."""
  287. return await _download_media(
  288. image_url, destination, max_retries,
  289. media_label="Image", accept="image/*,*/*;q=0.8",
  290. max_bytes=_VISION_MAX_DOWNLOAD_BYTES, timeout=_VISION_DOWNLOAD_TIMEOUT, retry_all=False)
  291. def _image_to_base64_data_url(image_path: Path, mime_type: Optional[str] = None) -> str:
  292. """``data:<mime>;base64,...`` for a file (MIME from extension when not given)."""
  293. mime = mime_type or _determine_mime_type(image_path)
  294. return f"data:{mime};base64,{base64.b64encode(image_path.read_bytes()).decode('ascii')}"
  295. # Absolute hard ceiling for vision payloads (20 MB): no major provider accepts more.
  296. _MAX_BASE64_BYTES = 20 * 1024 * 1024
  297. # Proactive embed caps for history reuse: the native path bakes the data URL into the tool
  298. # result, re-sent every later turn (a 4 MB embed cost ~100-260K billed tokens). Anthropic
  299. # downsamples to a 1568px long edge anyway, so pixels past that cost wire bytes for no fidelity.
  300. # The 20 MB hard ceiling / Anthropic 5 MB reject-cap still apply as safety nets; those are one-shot viewing
  301. # limits, not history-reuse sizes. A 4 MB / 7900px embed was observed at ~400K chars and ~100–260K billed
  302. # tokens per image (#92699), so we size for model reading instead: the byte budget is
  303. # ``vision.embed_target_bytes`` (default 256 KB, see vision_tools_history_budget) — it keeps a 1568px
  304. # screenshot cheap enough to ride the session (PNGs that exceed it are downscaled further by the
  305. # byte-budget ladder), well under every provider's per-image limit.
  306. _EMBED_MAX_DIMENSION = 1568
  307. # Target when auto-resizing after a provider size rejection (retry once).
  308. _RESIZE_TARGET_BYTES = 5 * 1024 * 1024
  309. _SIZE_ERROR_HINTS = (
  310. "too large", "payload", "413", "content_too_large",
  311. "request_too_large", "exceeds", "size limit",
  312. )
  313. def _is_image_size_error(error: Exception) -> bool:
  314. """Detect if an API error is related to image or payload size."""
  315. err_str = str(error).lower()
  316. return any(hint in err_str for hint in _SIZE_ERROR_HINTS + ("image_url", "invalid_request"))
  317. def _build_scale_note(scale_info: Optional[dict], crop_offset: Optional[dict]) -> Optional[str]:
  318. """Coordinate-mapping disclosure for downscale and/or region crop; ``None`` when neither applied."""
  319. parts = []
  320. if scale_info:
  321. ow, oh = scale_info["orig_width"], scale_info["orig_height"]
  322. nw, nh = scale_info["new_width"], scale_info["new_height"]
  323. fx, fy = (ow / nw if nw else 1.0), (oh / nh if nh else 1.0)
  324. axes = (f"any coordinates you report by {fx:.2f}" if f"{fx:.2f}" == f"{fy:.2f}"
  325. else f"any x coordinates you report by {fx:.2f} and any y coordinates by {fy:.2f}")
  326. parts.append(f"Image downscaled from {ow}x{oh} to {nw}x{nh} for vision; "
  327. f"multiply {axes} to map back to the original image.")
  328. if crop_offset:
  329. parts.append(
  330. f"Analysis was performed on a cropped region of the original "
  331. f"image starting at offset ({crop_offset['x']}, "
  332. f"{crop_offset['y']}); coordinates are relative to that crop "
  333. f"origin — add the offset to map back to the full image.")
  334. return " ".join(parts) if parts else None
  335. def _import_pillow_for_resize():
  336. """Return ``PIL.Image`` or None (Pillow is a lazy-installable soft dependency). ``prompt=False``:
  337. a blocking input() deadlocks the CLI where prompt_toolkit owns stdin; the install is already
  338. gated by security.allow_lazy_installs."""
  339. try:
  340. from PIL import Image
  341. except ImportError:
  342. try:
  343. # pm-era wiring: never raise a blocking input() prompt mid-session. Under the
  344. # interactive CLI prompt_toolkit owns stdin, so a bare input() deadlocks the
  345. # terminal (#40490). The install is already gated by security.allow_lazy_installs,
  346. # so reaching here is opt-in.
  347. from pm import ensure_import
  348. ensure_import("vision")
  349. from PIL import Image
  350. except Exception:
  351. return None
  352. return Image
  353. def _resize_image_for_vision(image_path: Path, mime_type: Optional[str] = None,
  354. max_base64_bytes: int = _RESIZE_TARGET_BYTES,
  355. max_dimension: Optional[int] = None,
  356. scale_out: Optional[dict] = None,
  357. force_jpeg: bool = False) -> str:
  358. """Base64 data URL, progressively downscaled with Pillow while over budget.
  359. Halves dimensions (aspect-preserving, 64px floor) up to 4 times; JPEG also walks a
  360. quality ladder (85/70/50) per step. Without Pillow, or if it still doesn't fit, returns
  361. the best attempt (or raw bytes) and lets the caller apply the size check.
  362. ``max_dimension``: force a downscale above this long edge even when bytes fit
  363. (Anthropic's 8000px cap is independent of bytes). ``force_jpeg``: re-encode PNG as JPEG
  364. when resizing — halving PNG dimensions destroys text legibility on dense screenshots.
  365. Args: max_dimension: If set, images whose longest side exceeds this pixel count are forcibly downscaled
  366. even if they're under the byte budget. Anthropic enforces an 8000 px per-side cap independently of the 5
  367. MB byte cap. force_jpeg: Re-encode as JPEG even for PNG input when a resize is needed. History-reuse
  368. embeds (#92699) opt in so a text-heavy screenshot keeps its readable resolution and shrinks via JPEG
  369. quality instead. Images already under both caps are returned unchanged (still PNG).
  370. """
  371. file_size = image_path.stat().st_size
  372. estimated_b64 = (file_size * 4) // 3 + 100 # base64 ~4/3 + data URL header
  373. data_url = None
  374. if estimated_b64 <= max_base64_bytes and not (
  375. max_dimension is not None and _image_exceeds_dimension(image_path, max_dimension)
  376. ):
  377. data_url = _image_to_base64_data_url(image_path, mime_type=mime_type)
  378. if len(data_url) <= max_base64_bytes:
  379. return data_url
  380. def _raw() -> str:
  381. return data_url or _image_to_base64_data_url(image_path, mime_type=mime_type)
  382. Image = _import_pillow_for_resize()
  383. if Image is None:
  384. logger.info("Pillow not installed — cannot auto-resize oversized image")
  385. return _raw() # caller will raise the size error
  386. logger.info("Image file is %.1f MB (estimated base64 %.1f MB, limit %.1f MB, max_dimension=%s), auto-resizing...",
  387. file_size / (1024 * 1024), estimated_b64 / (1024 * 1024),
  388. max_base64_bytes / (1024 * 1024), max_dimension)
  389. # JPEG for photos (smaller), PNG for transparency — unless force_jpeg.
  390. is_png = (mime_type or _determine_mime_type(image_path)) == "image/png" and not force_jpeg
  391. pil_format, out_mime = ("PNG", "image/png") if is_png else ("JPEG", "image/jpeg")
  392. try:
  393. img = Image.open(image_path)
  394. except Exception as exc:
  395. logger.info("Pillow cannot open image for resizing: %s", exc)
  396. return _raw()
  397. # JPEG cannot encode alpha/palette modes (force_jpeg routes PNGs here).
  398. if not is_png and img.mode not in {"RGB", "L"}:
  399. img = img.convert("RGB")
  400. quality_steps = (None,) if is_png else (85, 70, 50)
  401. orig_dims = prev_dims = (img.width, img.height)
  402. candidate = None
  403. def _record_scale(w: int, h: int) -> None:
  404. if scale_out is not None and (w, h) != orig_dims:
  405. scale_out.update(orig_width=orig_dims[0], orig_height=orig_dims[1], new_width=w, new_height=h)
  406. for attempt in range(5):
  407. if attempt > 0:
  408. # Halve, then re-derive from whichever axis hit the 64px floor so both shrink equally.
  409. new_w, new_h = max(int(img.width * 0.5), 64), max(int(img.height * 0.5), 64)
  410. if new_w == 64 and img.width > 0:
  411. new_h = max(int(img.height * (64 / img.width)), 64)
  412. elif new_h == 64 and img.height > 0:
  413. new_w = max(int(img.width * (64 / img.height)), 64)
  414. if (new_w, new_h) == prev_dims:
  415. break
  416. img = img.resize((new_w, new_h), Image.LANCZOS)
  417. prev_dims = (new_w, new_h)
  418. logger.info("Resized to %dx%d (attempt %d)", new_w, new_h, attempt)
  419. dims_ok = max_dimension is None or max(img.width, img.height) <= max_dimension
  420. for q in quality_steps:
  421. buf = BytesIO()
  422. img.save(buf, format=pil_format, **({} if q is None else {"quality": q}))
  423. candidate = f"data:{out_mime};base64,{base64.b64encode(buf.getvalue()).decode('ascii')}"
  424. if len(candidate) <= max_base64_bytes and dims_ok:
  425. logger.info("Auto-resized image fits: %.1f MB (quality=%s, %dx%d)",
  426. len(candidate) / (1024 * 1024), q, img.width, img.height)
  427. _record_scale(img.width, img.height)
  428. return candidate
  429. if candidate is None:
  430. return _raw()
  431. logger.warning("Auto-resize could not fit image under %.1f MB (best: %.1f MB)",
  432. max_base64_bytes / (1024 * 1024), len(candidate) / (1024 * 1024))
  433. _record_scale(img.width, img.height)
  434. return candidate
  435. # Native fast path: a vision-capable main model gets the image bytes as a multimodal
  436. # tool-result envelope (the agent loop unwraps it into an OpenAI-style content list on the
  437. # `tool` role); no aux LLM, no information loss. Providers whose tool results accept image
  438. # content: Anthropic Messages (and aggregators proxying Claude — assume support), OpenAI
  439. # Chat/Responses. Gemini is gated on model: only 3.x supports multimodal functionResponse.
  440. _TOOL_RESULT_MEDIA_PROVIDERS = frozenset({
  441. "openrouter", "nous", "vertex", "bedrock", "anthropic-vertex", "google-vertex",
  442. "anthropic", "claude", "anthropic-direct",
  443. "openai", "openai-chat", "openai-codex", "azure-openai",
  444. })
  445. _GEMINI_PROVIDERS = frozenset({"google", "gemini", "google-gemini", "google-vertex-gemini"})
  446. def _profile_rejects_tool_media(provider: str, model: str = "") -> bool:
  447. """Hard veto: the provider's ``ProviderProfile`` declares
  448. ``supports_vision_tool_messages=False`` — images are accepted in user
  449. messages but list-type tool-result content is rejected with 400
  450. (xiaomi/MiMo "text is not set"). ``supports_vision`` alone must not
  451. override this, or the multimodal tool-result envelope 400s every turn
  452. and the image never enters context (#89981).
  453. """
  454. try:
  455. from providers import routed_model_rejects_vision_tool_messages
  456. return routed_model_rejects_vision_tool_messages(provider, model)
  457. except Exception:
  458. return False
  459. def _supports_media_in_tool_results(provider: str, model: str) -> bool:
  460. """Whether provider+model accepts image content inside a tool-result message. Unknown
  461. providers are False (caller falls back to aux-LLM text) unless their ``ProviderProfile``
  462. declares ``supports_vision``; ``supports_vision_tool_messages=False`` is a hard veto."""
  463. p = provider.strip().lower() if isinstance(provider, str) else ""
  464. if not p or _profile_rejects_tool_media(p, model):
  465. return False
  466. if p in _TOOL_RESULT_MEDIA_PROVIDERS:
  467. return True
  468. if p in _GEMINI_PROVIDERS:
  469. m = model.strip().lower() if isinstance(model, str) else ""
  470. return any(tag in m for tag in ("gemini-3", "gemini-pro-3", "gemini-flash-3"))
  471. try:
  472. from providers import get_provider_profile
  473. profile = get_provider_profile(p)
  474. return profile is not None and bool(profile.supports_vision)
  475. except Exception:
  476. return False
  477. def _accepts_tool_result_images(provider: str, model: str, cfg: Optional[dict[str, Any]]) -> bool:
  478. """One gate for both native lanes — the ``vision_analyze`` fast path and the ``computer_use`` capture route
  479. (#115248): the profile's ``supports_vision_tool_messages=False`` veto first, then either the provider's tool
  480. results are known to carry media or the per-model capability lookup (config override → catalog incl. the
  481. profile's ``model_capabilities`` → local probes) attests the model as vision-capable."""
  482. if _profile_rejects_tool_media(provider, model):
  483. return False
  484. if _supports_media_in_tool_results(provider, model):
  485. return True
  486. from agent.image_routing import _lookup_supports_vision
  487. return _lookup_supports_vision(provider, model, cfg) is True
  488. def _native_tool_result_images(provider: str, model: str, cfg: Optional[dict[str, Any]]) -> bool:
  489. """THE gate for every tool that can hand the main model pixels (``vision_analyze``, browser and
  490. ``computer_use`` screenshots, MCP ``ImageContent``): image routing resolves to ``native``
  491. (``agent.image_input_mode``, an explicit ``auxiliary.vision`` backend, the catalog) AND the
  492. route accepts images inside tool results. One predicate, so the lane never depends on which
  493. tool produced the image."""
  494. from agent.image_routing import decide_image_input_mode
  495. return decide_image_input_mode(provider, model, cfg) == "native" and _accepts_tool_result_images(provider, model, cfg)
  496. def _should_use_native_vision_fast_path() -> bool:
  497. """:func:`_native_tool_result_images` for the active main model; any failure → False."""
  498. try:
  499. from agent.auxiliary_client import _read_main_provider, _read_main_model
  500. from hermes_cli.config import load_config
  501. return _native_tool_result_images(_read_main_provider(), _read_main_model(), load_config())
  502. except Exception as exc:
  503. logger.debug("Native vision fast-path check failed: %s", exc)
  504. return False
  505. def _build_native_vision_tool_result(
  506. image_url: str, question: str, image_data_url: str, image_size_bytes: int,
  507. scale_note: Optional[str] = None,
  508. ) -> dict[str, Any]:
  509. """Multimodal tool-result envelope. The text part is intentionally minimal (the model already
  510. has the question); ``text_summary`` is the fallback for providers without multimodal tool results."""
  511. text_part = (
  512. "Image loaded into your context — you can see it natively now. "
  513. "Use your built-in vision to answer the user.")
  514. if isinstance(question, str) and question.strip():
  515. text_part += f"\n\nQuestion: {question.strip()}"
  516. if scale_note:
  517. text_part += f"\n\nNote: {scale_note}"
  518. return {
  519. "_multimodal": True,
  520. "content": [
  521. {"type": "text", "text": text_part},
  522. {"type": "image_url", "image_url": {"url": image_data_url}}],
  523. "text_summary": (
  524. f"Image attached natively for the main model ({image_size_bytes / 1024:.1f} KB). "
  525. "Answer using built-in vision."),
  526. "meta": {"image_url": image_url[:200], "size_bytes": image_size_bytes, "native_vision": True}}
  527. def _unlink_quietly(path: Optional[Path]) -> None:
  528. if path is not None:
  529. try:
  530. path.unlink(missing_ok=True)
  531. except Exception:
  532. pass
  533. class _ImagePrepError(ValueError):
  534. """Raised by :func:`_prepare_image`; the message is user-facing."""
  535. class _PreparedImage(NamedTuple):
  536. """Temp image ready to encode; ``path`` is owned by the caller (delete it)."""
  537. path: Path
  538. mime: Optional[str]
  539. size_bytes: int
  540. crop_offset: dict
  541. async def _prepare_image(
  542. image_url: str, task_id: Optional[str], region: Optional[list], *, validate_decode: bool,
  543. ) -> _PreparedImage:
  544. """Resolve → materialize → normalize → (validate) → (crop). Raises ``_ImagePrepError``.
  545. Unsupported formats (SVG, BMP) become PNG BEFORE encoding — an unsupported media_type baked
  546. into immutable history would 400 on every resume. The crop runs BEFORE any downscale so the
  547. region keeps the full resolution budget. On error no temp file is left."""
  548. from tools.image_source import ImageResolutionError, ResolveContext, resolve_image_source
  549. try:
  550. resolved = await resolve_image_source(image_url, ResolveContext(task_id=task_id))
  551. except ImageResolutionError as exc:
  552. raise _ImagePrepError(str(exc)) from exc
  553. temp_dir = _secure_cache_dir("cache/vision", "temp_vision_images")
  554. path = temp_dir / f"temp_image_{uuid.uuid4()}.img"
  555. await asyncio.to_thread(_write_private_bytes, path, resolved.data)
  556. mime, size_bytes, crop_offset = resolved.mime, len(resolved.data), {}
  557. try:
  558. normalized_path, mime, norm_err = await asyncio.to_thread(_normalize_to_supported_image, path, mime)
  559. if norm_err or normalized_path is None:
  560. raise _ImagePrepError(norm_err or "Image normalization failed.")
  561. if normalized_path != path:
  562. _unlink_quietly(path)
  563. path = normalized_path
  564. size_bytes = path.stat().st_size
  565. if validate_decode:
  566. decode_error = await _run_encode_on_cpu_executor(
  567. _validate_raster_image_decodable, path,
  568. _VISION_MAX_VALIDATED_FRAME_COUNT, _VISION_MAX_VALIDATED_AGGREGATE_PIXELS)
  569. if decode_error:
  570. raise _ImagePrepError(decode_error)
  571. if region is not None:
  572. cropped_path, cropped_mime, crop_err = await asyncio.to_thread(
  573. _crop_image_region, path, region, offset_out=crop_offset)
  574. if crop_err or cropped_path is None:
  575. raise _ImagePrepError(crop_err or "Region crop failed.")
  576. _unlink_quietly(path)
  577. path, mime, size_bytes = cropped_path, cropped_mime, cropped_path.stat().st_size
  578. except BaseException:
  579. _unlink_quietly(path)
  580. raise
  581. return _PreparedImage(path, mime, size_bytes, crop_offset)
  582. def _too_large_message(image_data_url: str) -> str:
  583. return (
  584. f"Image too large for vision API: base64 payload is {len(image_data_url) / (1024 * 1024):.1f} MB "
  585. f"(limit {_MAX_BASE64_BYTES / (1024 * 1024):.0f} MB) even after resizing. Run `hermes pm repair` "
  586. f"to restore Pillow for auto-resize, or compress the image manually.")
  587. async def _resize_prepared(prepared: _PreparedImage, scale_info: dict, **kwargs) -> str:
  588. """Run :func:`_resize_image_for_vision` on the CPU executor for a prepared image."""
  589. return await _run_encode_on_cpu_executor(
  590. _resize_image_for_vision, prepared.path, mime_type=prepared.mime, scale_out=scale_info, **kwargs,
  591. )
  592. async def _vision_analyze_native(
  593. image_url: str, question: str, task_id: Optional[str] = None, region: Optional[list] = None,
  594. ) -> Any:
  595. """Fast path for vision-capable main models: a ``_multimodal`` envelope dict on success,
  596. or a JSON error string (the normal tool-result contract) on failure."""
  597. if not isinstance(image_url, str) or not image_url.strip():
  598. return tool_error("image_url is required", success=False)
  599. already_native = _native_turn_duplicate(image_url, region)
  600. if already_native is not None:
  601. return already_native
  602. # A cap > 0 RESERVES the slot here (atomic check-and-count); released below if no embed happens.
  603. refusal = _repeat_refusal(image_url)
  604. if refusal is not None:
  605. return refusal
  606. reserved = _resolve_repeat_cap() > 0
  607. prepared: Optional[_PreparedImage] = None
  608. embedded = False
  609. try:
  610. from tools.interrupt import is_interrupted
  611. if is_interrupted():
  612. return tool_error("Interrupted", success=False)
  613. try:
  614. prepared = await _prepare_image(image_url, task_id, region, validate_decode=True)
  615. except _ImagePrepError as exc:
  616. return tool_error(str(exc), success=False)
  617. image_data_url = await _run_encode_on_cpu_executor(
  618. _image_to_base64_data_url, prepared.path, mime_type=prepared.mime)
  619. # Proactive embed cap: this image is re-sent on every later turn, so resize DOWN to the
  620. # history-reuse target whenever the byte or long-edge cap is exceeded, not just at 20 MB.
  621. _scale_info: dict = {}
  622. # Anthropic still rejects >5 MB / >8000px with a non-retryable 400, but those are one-shot viewing
  623. # limits — history embeds are sized smaller so repeated vision_analyze turns don't blow the context
  624. # (#92699).
  625. embed_target_bytes = _resolve_embed_target_bytes()
  626. _over_dims = await _run_encode_on_cpu_executor(
  627. _image_exceeds_dimension, prepared.path, _EMBED_MAX_DIMENSION)
  628. if len(image_data_url) > embed_target_bytes or _over_dims:
  629. image_data_url = await _resize_prepared(
  630. prepared, _scale_info,
  631. max_base64_bytes=embed_target_bytes, max_dimension=_EMBED_MAX_DIMENSION, force_jpeg=True)
  632. # Reject rather than embed a session-wedging payload.
  633. if len(image_data_url) > _MAX_BASE64_BYTES:
  634. return tool_error(_too_large_message(image_data_url), success=False)
  635. embedded = True
  636. if not reserved:
  637. _record_embed(image_url)
  638. return _build_native_vision_tool_result(
  639. image_url=image_url, question=question, image_data_url=image_data_url,
  640. image_size_bytes=prepared.size_bytes,
  641. scale_note=_build_scale_note(_scale_info or None, prepared.crop_offset or None))
  642. except Exception as exc:
  643. logger.warning("Native vision fast path failed: %s", exc)
  644. return tool_error(f"Native vision failed: {exc}", success=False)
  645. finally:
  646. if reserved and not embedded:
  647. _release_embed(image_url)
  648. # Only delete temp files we created — never user-provided paths.
  649. if prepared is not None:
  650. _unlink_quietly(prepared.path)
  651. def _aux_call_kwargs(messages: list, model: Optional[str], default_timeout: float, *,
  652. min_timeout: Optional[float] = None) -> dict:
  653. """``async_call_llm`` kwargs with ``auxiliary.vision.timeout`` / ``.temperature`` from config.
  654. Local vision models (llama.cpp, ollama) can take well over 30s, hence generous defaults
  655. (temperature 0.1); ``min_timeout`` lets video enforce a floor."""
  656. timeout, temperature = default_timeout, 0.1
  657. try:
  658. _vision_cfg = _cfg_auxiliary("vision", default={}) or {}
  659. if _vision_cfg.get("timeout") is not None:
  660. timeout = max(float(_vision_cfg["timeout"]), float("-inf") if min_timeout is None else min_timeout)
  661. if _vision_cfg.get("temperature") is not None:
  662. temperature = float(_vision_cfg["temperature"])
  663. except Exception:
  664. pass
  665. return {"task": "vision", "messages": messages, "temperature": temperature, "timeout": timeout,
  666. **({"model": model} if model else {})}
  667. def _media_messages(user_prompt: str, part_type: str, data_url: str) -> list:
  668. """Single user message: text + one ``image_url``/``video_url`` data-URL part."""
  669. return [{"role": "user", "content": [
  670. {"type": "text", "text": user_prompt}, {"type": part_type, part_type: {"url": data_url}}]}]
  671. # Aux-LLM error classification: first matching hint set wins (billing → capability → size/format).
  672. _BILLING_HINTS = ("402", "insufficient", "payment required", "credits", "billing")
  673. _IMAGE_ERROR_RULES = (
  674. (_BILLING_HINTS,
  675. "Insufficient credits or payment required. Please top up your "
  676. "API provider account and try again. Error: {e}"),
  677. (("does not support", "not support image", "content_policy", "multimodal",
  678. "unrecognized request argument", "image input"),
  679. "{model} does not support vision or our request was not "
  680. "accepted by the server. Error: {e}"),
  681. (("invalid_request", "image_url"),
  682. "The vision API rejected the image. This can happen when the "
  683. "image is in an unsupported format, corrupted, or still too "
  684. "large after auto-resize. Try a smaller JPEG/PNG and retry. "
  685. "Error: {e}"),
  686. )
  687. _VIDEO_ERROR_RULES = (
  688. (_BILLING_HINTS, _IMAGE_ERROR_RULES[0][1]),
  689. (("does not support", "not support video", "content_policy", "multimodal",
  690. "unrecognized request argument", "video input", "video_url"),
  691. "The model does not support video analysis or the request was "
  692. "rejected. Ensure you're using a video-capable model "
  693. "(e.g. google/gemini-2.5-flash). Error: {e}"),
  694. (_SIZE_ERROR_HINTS,
  695. "The video is too large for the API. Try compressing or trimming "
  696. "the video (max ~50 MB). Error: {e}"),
  697. )
  698. # kind -> (debug tool name, error rules)
  699. _ANALYSIS_KINDS = {
  700. "image": ("vision_analyze_tool", _IMAGE_ERROR_RULES),
  701. "video": ("video_analyze_tool", _VIDEO_ERROR_RULES)}
  702. def _debug_call_data(kind: str, source: str, user_prompt: str, model) -> dict:
  703. prompt = user_prompt[:200] + "..." if len(user_prompt) > 200 else user_prompt
  704. return {
  705. "parameters": {f"{kind}_url": source, "user_prompt": prompt, "model": model},
  706. "error": None, "success": False, "analysis_length": 0, "model_used": model, f"{kind}_size_bytes": 0}
  707. async def _call_vision_llm(call_kwargs: dict, empty_log: str, response=None):
  708. """Aux vision LLM analysis text, retrying once on empty content (reasoning-only response).
  709. ``response``: an already-made first call (the image path handles size-error retries itself)."""
  710. _load_auxiliary_client()
  711. if response is None:
  712. response = await async_call_llm(**call_kwargs)
  713. analysis = extract_content_or_reasoning(response)
  714. if not analysis:
  715. logger.warning(empty_log)
  716. analysis = extract_content_or_reasoning(await async_call_llm(**call_kwargs))
  717. return analysis
  718. async def _run_analysis(
  719. kind: str, source: str, user_prompt: str, model: Optional[str],
  720. stage: Callable[[str, dict, list], Awaitable[tuple]]) -> str:
  721. """Aux-LLM analysis skeleton for image/video: interrupt check → ``stage`` → JSON result.
  722. ``stage(user_prompt, debug_call_data, temp_paths)`` returns ``(analysis, scale_note)``; temp
  723. files it appends to ``temp_paths`` are deleted in ``finally``. Returns JSON
  724. ``{"success": bool, "analysis": str}`` (``analysis`` carries the error explanation on failure).
  725. """
  726. tool_name, rules = _ANALYSIS_KINDS[kind]
  727. if not isinstance(user_prompt, str):
  728. user_prompt = str(user_prompt) if user_prompt is not None else ""
  729. debug_call_data = _debug_call_data(kind, source, user_prompt, model)
  730. temp_paths: list = []
  731. def finish(result: dict) -> str:
  732. _debug.log_call(tool_name, debug_call_data)
  733. _debug.save()
  734. return json.dumps(result, indent=2, ensure_ascii=False)
  735. try:
  736. from tools.interrupt import is_interrupted
  737. if is_interrupted():
  738. return tool_error("Interrupted", success=False)
  739. logger.info("Analyzing %s: %s", kind, source[:60])
  740. logger.info("User prompt: %s", user_prompt[:100])
  741. analysis, scale_note = await stage(user_prompt, debug_call_data, temp_paths)
  742. analysis_length = len(analysis) if analysis else 0
  743. logger.info("%s analysis completed (%s characters)", kind.capitalize(), analysis_length)
  744. analysis = analysis or f"There was a problem with the request and the {kind} could not be analyzed."
  745. result = {"success": True, "analysis": f"[{scale_note}] {analysis}" if scale_note else analysis}
  746. if scale_note:
  747. result["scale_note"] = scale_note
  748. debug_call_data.update(success=True, analysis_length=analysis_length)
  749. return finish(result)
  750. except Exception as e:
  751. error_msg = f"Error analyzing {kind}: {e!s}"
  752. logger.error("%s", error_msg, exc_info=True)
  753. err_str = str(e).lower()
  754. template = next(
  755. (tpl for hints, tpl in rules if any(hint in err_str for hint in hints)),
  756. f"There was a problem with the request and the {kind} could not be analyzed. Error: {{e}}")
  757. analysis = template.format(e=e, model=model)
  758. debug_call_data["error"] = error_msg
  759. return finish({"success": False, "error": error_msg, "analysis": analysis})
  760. finally:
  761. for path in temp_paths:
  762. if path and path.exists():
  763. try:
  764. path.unlink()
  765. logger.debug("Cleaned up temporary %s file", kind)
  766. except Exception as cleanup_error:
  767. logger.warning("Could not delete temporary file: %s", cleanup_error, exc_info=True)
  768. async def vision_analyze_tool(
  769. image_url: str, user_prompt: str, model: str | None = None,
  770. task_id: Optional[str] = None, region: Optional[list] = None) -> str:
  771. """Describe an image (URL, local path, data: URL) with the auxiliary vision LLM. ``user_prompt``
  772. is pre-formatted by the caller. Temp images live under $HERMES_HOME/cache/vision/."""
  773. async def stage(prompt: str, debug_call_data: dict, temp_paths: list) -> tuple:
  774. prepared = await _prepare_image(image_url, task_id, region, validate_decode=False)
  775. temp_paths.append(prepared.path)
  776. logger.info("Image ready (%.1f KB)", prepared.size_bytes / 1024)
  777. # Send at full resolution first; on a size rejection, downscale and retry.
  778. logger.info("Converting image to base64...")
  779. image_data_url = await _run_encode_on_cpu_executor(
  780. _image_to_base64_data_url, prepared.path, mime_type=prepared.mime)
  781. logger.info("Image converted to base64 (%.1f KB)", len(image_data_url) / 1024)
  782. _scale_info: dict = {}
  783. if len(image_data_url) > _MAX_BASE64_BYTES:
  784. image_data_url = await _resize_prepared(prepared, _scale_info)
  785. if len(image_data_url) > _MAX_BASE64_BYTES:
  786. raise ValueError(_too_large_message(image_data_url))
  787. debug_call_data["image_size_bytes"] = prepared.size_bytes
  788. messages = _media_messages(prompt, "image_url", image_data_url)
  789. logger.info("Processing image with vision model...")
  790. call_kwargs = _aux_call_kwargs(messages, model, 120.0)
  791. _load_auxiliary_client()
  792. try:
  793. response = await async_call_llm(**call_kwargs)
  794. except Exception as _api_err:
  795. if not (_is_image_size_error(_api_err) and len(image_data_url) > _RESIZE_TARGET_BYTES):
  796. raise
  797. logger.info(
  798. "API rejected image (%.1f MB, likely too large); auto-resizing to ~%.0f MB and retrying...",
  799. len(image_data_url) / (1024 * 1024), _RESIZE_TARGET_BYTES / (1024 * 1024))
  800. image_data_url = await _resize_prepared(prepared, _scale_info)
  801. messages[0]["content"][1]["image_url"]["url"] = image_data_url
  802. response = await async_call_llm(**call_kwargs)
  803. analysis = await _call_vision_llm(
  804. call_kwargs, "Vision LLM returned empty content, retrying once", response)
  805. return analysis, _build_scale_note(_scale_info or None, prepared.crop_offset or None)
  806. return await _run_analysis("image", image_url, user_prompt, model, stage)
  807. def check_video_requirements() -> bool:
  808. """True when ``call_llm(task="vision")`` could resolve a client.
  809. Mirrors its fallback chain: explicit ``auxiliary.vision.provider``, then auto (main
  810. provider → openrouter → nous) — without the auto step the tool would vanish whenever
  811. the explicit name was unresolvable. Probe mode skips real SDK client construction.
  812. See #31179.
  813. """
  814. from agent.auxiliary_client import aux_probe_mode, resolve_vision_provider_client
  815. # No blanket except: a resolver crash must reach the registry, which logs it with a
  816. # traceback; a swallowed exception reads as "no vision backend configured" (#87950).
  817. with aux_probe_mode():
  818. return any(
  819. resolve_vision_provider_client(**kw)[1] is not None for kw in ({}, {"provider": "auto"})
  820. )
  821. def check_vision_requirements() -> bool:
  822. """Image gate (``vision_analyze``): an aux vision client OR the native fast
  823. path. The handler attaches pixels straight to a vision-capable main model, so a main model on a
  824. provider the aux resolver does not know (OAuth, local vLLM) must not hide a working tool (#47149).
  825. ``video_analyze`` keeps the aux-only gate — its handler has no native path.
  826. """
  827. return _should_use_native_vision_fast_path() or check_video_requirements()
  828. from tools.registry import registry, tool_error
  829. VISION_ANALYZE_SCHEMA = {
  830. "name": "vision_analyze",
  831. # Routing mechanics are deliberately absent (the route is automatic and the
  832. # native result says so itself); region keeps its pre-effect guidance — a
  833. # model that doesn't know crops keep full resolution never zooms.
  834. "description": (
  835. # Dieted (#95681): routing mechanics (native attach vs aux-model text fallback) removed — the route
  836. # is automatic and the native path's own tool result says "you can see it natively now"; the schema
  837. # doesn't need to predict plumbing.
  838. "Load an image into the conversation so you can see it. Call it "
  839. "any time the user references an image — then answer from what "
  840. "you see."
  841. ),
  842. "parameters": {
  843. "type": "object",
  844. "properties": {
  845. "image_url": {
  846. "type": "string",
  847. "description": "Image URL (http/https), local file path, or data: URL to load."
  848. },
  849. "question": {
  850. "type": "string",
  851. "description": "Your question or request about the image."
  852. },
  853. "region": {
  854. "type": "array",
  855. "items": {"type": "integer"},
  856. "minItems": 4,
  857. "maxItems": 4,
  858. "description": (
  859. "Optional [x1, y1, x2, y2] crop in ORIGINAL-image pixel "
  860. "coordinates, applied before any downscaling — the crop "
  861. "keeps full resolution. Load the full image first, then "
  862. "re-call with a region to zoom into small text or fine "
  863. "detail."
  864. )
  865. }
  866. },
  867. "required": ["image_url", "question"]
  868. }
  869. }
  870. def _configured_aux_model(sections: tuple, env_vars: tuple) -> Optional[str]:
  871. """First non-empty ``auxiliary.<section>.model`` from config.yaml, else the first non-empty
  872. env var (legacy override), else None."""
  873. for section in sections:
  874. _vmodel = _cfg_auxiliary(section, "model")
  875. if _vmodel:
  876. if str(_vmodel).strip():
  877. return str(_vmodel).strip()
  878. break
  879. return next((v for v in (os.getenv(e, "").strip() for e in env_vars) if v), None)
  880. async def _handle_vision_analyze(args: dict[str, Any], **kw: Any) -> str:
  881. image_url, question, region = args.get("image_url", ""), args.get("question", ""), args.get("region")
  882. task_id = kw.get("task_id")
  883. # No concurrency gate around the whole analysis — the CPU burst is bounded inside the
  884. # encode/resize step, so multi-image fan-out keeps full request concurrency.
  885. if _should_use_native_vision_fast_path():
  886. logger.info("vision_analyze: native fast path")
  887. return await _vision_analyze_native(image_url, question, task_id=task_id, region=region)
  888. # Legacy path: aux LLM describes the image and we return its text.
  889. full_prompt = (
  890. "Fully describe and explain everything about this image, then answer the "
  891. f"following question:\n\n{question}")
  892. model = _configured_aux_model(("vision",), ("AUXILIARY_VISION_MODEL",))
  893. return await vision_analyze_tool(image_url, full_prompt, model, task_id=task_id, region=region)
  894. registry.register(
  895. name="vision_analyze",
  896. toolset="vision",
  897. schema=VISION_ANALYZE_SCHEMA,
  898. handler=_handle_vision_analyze,
  899. check_fn=check_vision_requirements,
  900. is_async=True,
  901. emoji="👁️")
  902. # --- video_analyze --------------------------------------------------------
  903. # Extension → MIME. avi/mkv fall back to mp4.
  904. _VIDEO_MIME_TYPES = {
  905. ".mp4": "video/mp4",
  906. ".webm": "video/webm",
  907. ".mov": "video/mov",
  908. ".avi": "video/mp4",
  909. ".mkv": "video/mp4",
  910. ".mpeg": "video/mpeg",
  911. ".mpg": "video/mpeg",
  912. }
  913. _MAX_VIDEO_BASE64_BYTES = 50 * 1024 * 1024 # 50 MB hard cap
  914. _VIDEO_SIZE_WARN_BYTES = 20 * 1024 * 1024
  915. def _detect_video_mime_type(video_path: Path) -> Optional[str]:
  916. """Video MIME type from extension, or None if unsupported."""
  917. return _VIDEO_MIME_TYPES.get(video_path.suffix.lower())
  918. def _unsupported_video_format(suffix: str) -> str:
  919. return f"Unsupported video format: '{suffix}'. Supported: {', '.join(sorted(_VIDEO_MIME_TYPES.keys()))}"
  920. def _video_to_base64_data_url(video_path: Path, mime_type: Optional[str] = None) -> str:
  921. mime = mime_type or _detect_video_mime_type(video_path) or "video/mp4"
  922. return f"data:{mime};base64,{base64.b64encode(video_path.read_bytes()).decode('ascii')}"
  923. async def _materialize_video(video_url: str, task_id: Optional[str], temp_paths: list) -> Path:
  924. """Local video path for a terminal-backend path, local file, or HTTP(S) URL. Only files created
  925. here are appended to ``temp_paths`` — never user-provided paths. Terminal-backend reads use the
  926. shared media resolver with ``permitted=("video",)`` — the exact pipeline vision_analyze uses
  927. (media-cache host reads, bounded in-sandbox exec-read, credential-read guard, ingest cap)."""
  928. from tools.image_source import (
  929. ImageResolutionError, ResolveContext, _is_local_terminal_backend, resolve_image_source,
  930. )
  931. source = video_url.removeprefix("file://")
  932. local_path = Path(os.path.expanduser(source))
  933. lowered = (video_url or "").strip().lower()
  934. path_like = bool(lowered) and not lowered.startswith(("http://", "https://", "data:"))
  935. if not _is_local_terminal_backend() and path_like:
  936. logger.info("Reading video source via terminal backend: %s", video_url)
  937. suffix = Path(source).suffix.lower()
  938. if suffix not in _VIDEO_MIME_TYPES:
  939. raise ValueError(_unsupported_video_format(suffix))
  940. try:
  941. resolved = await resolve_image_source(
  942. video_url, ResolveContext(task_id=task_id), permitted=("video",))
  943. except ImageResolutionError as exc:
  944. raise ValueError(f"Could not read video from terminal backend: {exc}") from exc
  945. temp_dir = _secure_cache_dir("cache/video", "temp_video_files")
  946. path = temp_dir / f"terminal_video_{uuid.uuid4()}{suffix}"
  947. _write_private_bytes(path, resolved.data)
  948. temp_paths.append(path)
  949. return path
  950. if local_path.is_file():
  951. from agent.file_safety import raise_if_read_blocked
  952. raise_if_read_blocked(str(local_path))
  953. logger.info("Using local video file: %s", video_url)
  954. return local_path
  955. if await _validate_image_url_async(video_url):
  956. blocked = check_website_access(video_url)
  957. if blocked:
  958. raise PermissionError(blocked["message"])
  959. # Hardened here rather than inside _download_media: that helper takes
  960. # a caller-supplied destination (tools/image_source.py hands its
  961. # sibling a /tmp NamedTemporaryFile it owns the mode of), and
  962. # chmod-ing an arbitrary caller's parent to 0700 would be a
  963. # destructive side effect on a path Hermes does not own.
  964. temp_dir = _secure_cache_dir("cache/video", "temp_video_files")
  965. path = temp_dir / f"temp_video_{uuid.uuid4()}.mp4"
  966. temp_paths.append(path)
  967. # Video downloads retry every failure class (legacy behavior).
  968. await _download_media(
  969. video_url, path, 3, media_label="Video", accept="video/*,*/*;q=0.8",
  970. max_bytes=_MAX_VIDEO_BASE64_BYTES, timeout=60.0, retry_all=True)
  971. return path
  972. raise ValueError("Invalid video source. Provide an HTTP/HTTPS URL or a valid local file path.")
  973. async def video_analyze_tool(
  974. video_url: str, user_prompt: str, model: str | None = None, task_id: Optional[str] = None) -> str:
  975. """Analyze a video via multimodal LLM. Returns JSON {success, analysis}."""
  976. async def stage(prompt: str, debug_call_data: dict, temp_paths: list) -> tuple:
  977. temp_video_path = await _materialize_video(video_url, task_id, temp_paths)
  978. video_size_bytes = temp_video_path.stat().st_size
  979. video_size_mb = video_size_bytes / (1024 * 1024)
  980. logger.info("Video ready (%.1f MB)", video_size_mb)
  981. detected_mime = _detect_video_mime_type(temp_video_path)
  982. if not detected_mime:
  983. raise ValueError(_unsupported_video_format(temp_video_path.suffix))
  984. if video_size_bytes > _VIDEO_SIZE_WARN_BYTES:
  985. logger.warning("Video is %.1f MB — may be slow or rejected", video_size_mb)
  986. video_data_url = _video_to_base64_data_url(temp_video_path, mime_type=detected_mime)
  987. if len(video_data_url) > _MAX_VIDEO_BASE64_BYTES:
  988. raise ValueError(
  989. f"Video too large for API: base64 payload is {len(video_data_url) / (1024 * 1024):.1f} MB "
  990. f"(limit {_MAX_VIDEO_BASE64_BYTES / (1024 * 1024):.0f} MB). "
  991. f"Compress or trim the video and retry.")
  992. debug_call_data["video_size_bytes"] = video_size_bytes
  993. messages = _media_messages(prompt, "video_url", video_data_url)
  994. call_kwargs = _aux_call_kwargs(messages, model, 180.0, min_timeout=180.0)
  995. analysis = await _call_vision_llm(call_kwargs, "Empty video response, retrying once")
  996. return analysis, None
  997. return await _run_analysis("video", video_url, user_prompt, model, stage)
  998. VIDEO_ANALYZE_SCHEMA = {
  999. "name": "video_analyze",
  1000. "description": (
  1001. "Analyze a video from a URL or local file path using a multimodal AI model. "
  1002. "Sends the video to a video-capable model (e.g. Gemini) for understanding. "
  1003. "Use this for video files — for images, use vision_analyze instead. "
  1004. "Supports mp4, webm, mov, avi, mkv, mpeg formats. "
  1005. "Note: large videos (>20 MB) may be slow; max ~50 MB."
  1006. ),
  1007. "parameters": {
  1008. "type": "object",
  1009. "properties": {
  1010. "video_url": {
  1011. "type": "string",
  1012. "description": "Video URL (http/https) or local file path to analyze.",
  1013. },
  1014. "question": {
  1015. "type": "string",
  1016. "description": "Your specific question about the video. The AI will describe what happens in the video and answer your question.",
  1017. },
  1018. },
  1019. "required": ["video_url", "question"],
  1020. },
  1021. }
  1022. def _handle_video_analyze(args: dict[str, Any], **kw: Any) -> Awaitable[str]:
  1023. video_url, question = args.get("video_url", ""), args.get("question", "")
  1024. full_prompt = (
  1025. "Fully describe and explain everything happening in this video, "
  1026. "including visual content, motion, audio cues, text overlays, and scene "
  1027. f"transitions. Then answer the following question:\n\n{question}")
  1028. model = _configured_aux_model(("video", "vision"), ("AUXILIARY_VIDEO_MODEL", "AUXILIARY_VISION_MODEL"))
  1029. return video_analyze_tool(video_url, full_prompt, model, task_id=kw.get("task_id"))
  1030. registry.register(
  1031. name="video_analyze",
  1032. toolset="video",
  1033. schema=VIDEO_ANALYZE_SCHEMA,
  1034. handler=_handle_video_analyze,
  1035. check_fn=check_video_requirements,
  1036. is_async=True,
  1037. emoji="🎬")