image_source.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354
  1. """Single resolver for every media source (data:/http(s)/file/local/container) -> bytes + mime.
  2. Everything funnels through :func:`resolve_image_source` so size and magic-byte checks run
  3. exactly once. Images are the default; video callers opt in via ``permitted=("video",)``.
  4. Security (GHSA-gpxw-6wxv-w3qq): under a non-local backend vision is confined like the file
  5. tools — host-read only inside a media cache (bind-mounted into the sandbox), anything else is
  6. exec-read *inside the sandbox*, so ``vision_analyze('/etc/passwd')`` never reads the host.
  7. """
  8. from __future__ import annotations
  9. import asyncio
  10. import base64
  11. import os
  12. import re
  13. from dataclasses import dataclass
  14. from pathlib import Path
  15. from typing import Optional
  16. # Raw-bytes INGEST budget for sandbox and data: sources (50 MiB). The sandbox exec-read returns
  17. # the file as base64 on stdout, so captured text + decoded bytes + copies peak near 4x the file
  18. # in this process; it stays at the 50MB download cap. HTTP downloads keep their own stream cap
  19. # (vision_tools._VISION_MAX_DOWNLOAD_BYTES).
  20. _MAX_INGEST_BYTES = 50 * 1024 * 1024
  21. # Host-file reads (250 MiB): bounded during the read, big enough for full-res camera/scan
  22. # images, which call sites downscale before the 20MB provider payload cap.
  23. _MAX_HOST_READ_BYTES = 250 * 1024 * 1024
  24. # Chunk size for host reads whose st_size under-reports (see _read_host_bytes_bounded).
  25. _HOST_READ_CHUNK = 1024 * 1024
  26. class ImageResolutionError(Exception):
  27. def __init__(self, message: str, *, src: str = "", origin: str = ""):
  28. super().__init__(message)
  29. self.src, self.origin = src, origin
  30. class UnsupportedScheme(ImageResolutionError): ...
  31. class SourceUnsafe(ImageResolutionError): ... # SSRF / path-allowlist
  32. class SourceTooLarge(ImageResolutionError): ...
  33. class SourceNotFound(ImageResolutionError): ...
  34. class NotAnImage(ImageResolutionError): ...
  35. @dataclass
  36. class ResolveContext:
  37. task_id: Optional[str] = None
  38. @dataclass
  39. class ResolvedImage:
  40. data: bytes
  41. mime: str
  42. origin: str # one of: data | http | file | local | container
  43. # Explicit URL scheme ("ftp://", "s3://"). Bare Windows drive paths lack the "//".
  44. _SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://")
  45. async def resolve_image_source(
  46. src: str, ctx: ResolveContext, *, permitted: tuple = ("image",)) -> ResolvedImage:
  47. if not isinstance(src, str) or not src.strip():
  48. raise SourceNotFound("image_url is required", src=str(src))
  49. s = src.strip()
  50. if s.startswith("data:"):
  51. data, mime = _resolve_data_url(s)
  52. return _finalize(data, mime, "data", s, permitted)
  53. if s.startswith(("http://", "https://")):
  54. reason = _http_block_reason(s)
  55. if reason:
  56. raise SourceUnsafe(reason, src=s)
  57. return _finalize(await _download_to_bytes(s), "", "http", s, permitted)
  58. if _SCHEME_RE.match(s) and not s.lower().startswith("file://"):
  59. raise UnsupportedScheme(
  60. "Unrecognized image source scheme. Use an http(s) URL, a local "
  61. "file path, a file:// URI, or a data: URL.",
  62. src=s)
  63. # Everything else is a filesystem path — including bare relative names like "pic.png"
  64. # (a path-shape gate here regressed them once).
  65. candidate = s[len("file://"):] if s.lower().startswith("file://") else s
  66. p = Path(os.path.expanduser(candidate))
  67. host_target = _permitted_host_read_target(p, ctx)
  68. if host_target is not None and host_target.is_file():
  69. _guard_credential_read(host_target, s)
  70. # Videos are never downscaled and video_analyze refuses over 50 MB of base64, so the
  71. # larger host budget only helps images; videos keep the ingest cap and fail at the stat.
  72. cap = _MAX_HOST_READ_BYTES if "image" in permitted else _MAX_INGEST_BYTES
  73. data = await asyncio.to_thread(_read_host_bytes_bounded, host_target, s, cap)
  74. return _finalize(data, "", "file", s, permitted)
  75. if _is_local_terminal_backend():
  76. # Any path was host-readable, so a miss means the file doesn't exist.
  77. raise SourceNotFound(f"media file not found: '{p}'", src=s, origin="file")
  78. return await _resolve_container_fallback(p, ctx, s, permitted)
  79. def _read_host_bytes_bounded(path: Path, src: str, cap: int) -> bytes:
  80. """Read a host file with ``cap`` enforced DURING the read. ``read_bytes()`` buffered the whole
  81. file before ``_finalize`` checked its size. ``read(n)`` preallocates n bytes, so the first read
  82. asks for the stat'd size (not the cap); when ``st_size`` under-reports (procfs-style files) the
  83. rest streams in chunks and stops past ``cap``. The honest-size check reads nothing at all."""
  84. size = path.stat().st_size
  85. if size > cap:
  86. raise SourceTooLarge("media exceeds size limit", src=src, origin="file")
  87. with path.open("rb") as fh:
  88. data = fh.read(size + 1)
  89. if len(data) > size:
  90. parts, total = [data], len(data)
  91. while total <= cap and (chunk := fh.read(_HOST_READ_CHUNK)):
  92. parts.append(chunk)
  93. total += len(chunk)
  94. if total > cap:
  95. raise SourceTooLarge("media exceeds size limit", src=src, origin="file")
  96. data = b"".join(parts)
  97. return data
  98. def _guard_credential_read(host_target: Path, src: str) -> None:
  99. """Shared credential-read guard: refuse secret-bearing files (.env, auth.json) with a specific
  100. error. Guard import is best-effort; a real block always propagates."""
  101. try:
  102. # Shared credential-read guard (agent.file_safety, #57698): refuse secret-bearing files (.env,
  103. # auth.json, ...) with an intentional, specific error instead of relying on the magic-byte sniff to
  104. # reject them incidentally. Same chokepoint the image-gen/video-gen provider plugins enforce on
  105. # model-supplied local paths. Import is best-effort (guard unavailability must not break image
  106. # loading); a real block always propagates.
  107. from agent.file_safety import raise_if_read_blocked
  108. except Exception:
  109. return
  110. try:
  111. raise_if_read_blocked(str(host_target))
  112. except ValueError as exc:
  113. raise SourceUnsafe(str(exc), src=src, origin="file")
  114. def _resolve_data_url(s: str) -> tuple[bytes, str]:
  115. header, _, payload = s.partition(",")
  116. if ";base64" not in header:
  117. raise NotAnImage("data: URL must be base64-encoded", src=s[:64])
  118. declared = header[len("data:"):].split(";", 1)[0].strip() or "application/octet-stream"
  119. # Cheap pre-decode size gate on the encoded length (~4/3 expansion).
  120. if (len(payload) * 3) // 4 > _MAX_INGEST_BYTES:
  121. raise SourceTooLarge("data: URL exceeds size limit", src=s[:64])
  122. try:
  123. data = base64.b64decode(payload, validate=True)
  124. except Exception as exc:
  125. raise NotAnImage(f"invalid base64 in data: URL: {exc}", src=s[:64])
  126. return data, declared # real mime verified in _finalize via magic bytes
  127. def _http_block_reason(url: str) -> Optional[str]:
  128. """Block reason, or None when allowed. Refuses policy-blocked URLs BEFORE any network I/O;
  129. ``_download_image`` re-checks per attempt and against the final redirect target (intentional)."""
  130. from tools.url_safety import is_safe_url
  131. from tools.website_policy import check_website_access
  132. if not is_safe_url(url):
  133. return "blocked: unsafe or private URL"
  134. if blocked := check_website_access(url):
  135. return blocked.get("message") or "blocked by website policy"
  136. return None
  137. async def _download_to_bytes(url: str) -> bytes:
  138. import tempfile
  139. from tools.vision_tools import _download_image
  140. with tempfile.NamedTemporaryFile(suffix=".img", delete=False) as tf:
  141. tmp = Path(tf.name)
  142. try:
  143. # Enforces the 50MB stream cap, redirect SSRF guard, and website policy.
  144. await _download_image(url, tmp)
  145. return await asyncio.to_thread(tmp.read_bytes)
  146. except PermissionError as exc: # website policy block
  147. raise SourceUnsafe(str(exc), src=url, origin="http")
  148. finally:
  149. tmp.unlink(missing_ok=True)
  150. def _is_local_terminal_backend() -> bool:
  151. """True when the terminal backend runs directly on the host (keys off ``TERMINAL_ENV``, read
  152. through the per-turn terminal scope so a routed multiplex profile sees ITS backend)."""
  153. from tools.terminal_scope import terminal_env
  154. return terminal_env("TERMINAL_ENV", "local").strip().lower() in ("local", "")
  155. # Host-side media caches: the only host paths vision may read under a non-local backend
  156. # (gateway inbound media + the tools' own download temp dirs).
  157. _MEDIA_CACHE_SUBDIRS = (
  158. "cache", # cache/images, cache/vision, cache/video(s), cache/audio
  159. "images", # desktop/clipboard/PDF uploads (tui_gateway)
  160. "image_cache", "audio_cache", "video_cache", "temp_vision_images", "temp_video_files")
  161. def _media_cache_roots() -> list:
  162. from hermes_constants import get_hermes_home
  163. home = get_hermes_home()
  164. return [home / sub for sub in _MEDIA_CACHE_SUBDIRS]
  165. def _permitted_host_read_target(p: Path, ctx: ResolveContext) -> Optional[Path]:
  166. """Host path to read, or ``None`` (caller exec-reads inside the sandbox instead).
  167. Local backend: any path. Non-local: only paths inside a media cache root (a
  168. container-visible cache path is first translated back to its host mount).
  169. """
  170. if _is_local_terminal_backend():
  171. try:
  172. return p.resolve()
  173. except Exception:
  174. return p
  175. from tools.credential_files import from_agent_visible_cache_path
  176. try:
  177. real = Path(from_agent_visible_cache_path(str(p))).resolve()
  178. except Exception:
  179. return None
  180. if any(real.is_relative_to(root.resolve()) for root in _media_cache_roots()):
  181. return real
  182. return None
  183. def _get_active_env(task_id: Optional[str]):
  184. if not task_id:
  185. return None
  186. try:
  187. from tools.terminal_tool_lifecycle import get_active_env
  188. return get_active_env(task_id)
  189. except Exception:
  190. return None
  191. def _ensure_container_env(task_id: Optional[str]) -> None:
  192. """Lazily bring up the sandbox before an in-sandbox read (vision may be a session's first
  193. action). Best-effort: failure leaves the env absent and the caller hits the fail-closed error.
  194. Unlike the terminal tool, vision never triggered environment creation, so a session whose first action
  195. is ``vision_analyze`` on a container-only path under a non-local backend found no active env and failed
  196. — until a terminal command happened to create one (issue #62825).
  197. """
  198. if not task_id:
  199. return
  200. try:
  201. from tools.terminal_tool import ensure_task_env
  202. ensure_task_env(task_id)
  203. except Exception:
  204. pass
  205. async def _resolve_container_fallback(
  206. p: Path, ctx: ResolveContext, src: str, permitted: tuple = ("image",)) -> ResolvedImage:
  207. """Read the bytes inside the sandbox; fail-closed when no env exists (a non-cache host
  208. path under a sandbox must never leak via a host fallback).
  209. Cold-start retry: under Docker the first exec against a fresh container can fail (empty
  210. pipe) while a second succeeds. On final failure the container's output is folded into the
  211. error so "no such file" / "permission denied" / "never came up" are distinguishable.
  212. We retry once with a short delay before giving up, so callers don't see "could not read inside the
  213. sandbox" on a file that is verifiably readable on the immediate retry. See #76566.
  214. """
  215. import shlex
  216. # Bring the sandbox up on demand: without this, the first vision_analyze of a session (before any
  217. # terminal command) has no active env to read from under a non-local backend (issue #62825).
  218. _ensure_container_env(ctx.task_id)
  219. env = _get_active_env(ctx.task_id)
  220. if env is None:
  221. raise SourceNotFound(
  222. f"'{p}' is not reachable inside the sandbox and no active sandbox "
  223. f"session is available to read it",
  224. src=src, origin="container")
  225. from tools.terminal_tool_config import translate_mounted_host_path
  226. translated = translate_mounted_host_path(
  227. str(p),
  228. getattr(env, "host_cwd", None) or "",
  229. getattr(env, "host_cwd_mount", None) or "/workspace",
  230. )
  231. if translated:
  232. p = Path(translated)
  233. # Bound the read INSIDE the sandbox: head -c caps at ingest-limit+1 (+1 distinguishes "at the
  234. # cap" from "over") so /dev/zero can't stream unbounded base64 into host memory. The input
  235. # redirect avoids argv (leading-dash paths); tr -d instead of GNU-only base64 -w0 (BusyBox).
  236. cmd = f"head -c {_MAX_INGEST_BYTES + 1} < {shlex.quote(str(p))} | base64 | tr -d '\\n'"
  237. last_res: dict = {"returncode": 1, "output": ""}
  238. for attempt in range(2):
  239. last_res = await asyncio.to_thread(env.execute, cmd)
  240. if last_res.get("returncode", 1) == 0:
  241. break
  242. if attempt == 0:
  243. await asyncio.sleep(0.15) # covers Docker exec warm-up in practice
  244. if last_res.get("returncode", 1) != 0:
  245. diag = (last_res.get("output") or "").strip().splitlines()
  246. first = next((ln.strip() for ln in diag if ln.strip()), "")
  247. suffix = f" ({first[:200]})" if first else ""
  248. raise SourceNotFound(f"could not read '{p}' inside the sandbox{suffix}", src=src, origin="container")
  249. try:
  250. data = base64.b64decode(last_res.get("output", ""), validate=True)
  251. except Exception as exc:
  252. raise NotAnImage(f"sandbox returned non-image data for '{p}': {exc}", src=src)
  253. if len(data) > _MAX_INGEST_BYTES:
  254. raise SourceTooLarge("media exceeds size limit", src=src, origin="container")
  255. return _finalize(data, "", "container", src, permitted)
  256. def _finalize(
  257. data: bytes, declared_mime: str, origin: str, src: str, permitted: tuple = ("image",)
  258. ) -> ResolvedImage:
  259. """Chokepoint: size ceiling + type check. Images by magic bytes; video (opt-in) by
  260. extension + mp4 sniff — enough because every downstream consumer re-validates. Each
  261. reader enforces its own tighter cap first; this is the ceiling for all of them."""
  262. from tools.vision_tools_image_prep import _detect_image_mime_type_from_bytes
  263. if len(data) > _MAX_HOST_READ_BYTES:
  264. raise SourceTooLarge("media exceeds size limit", src=src, origin=origin)
  265. sniffed = _detect_image_mime_type_from_bytes(data)
  266. if sniffed is not None:
  267. if "image" not in permitted:
  268. raise NotAnImage("source is an image, but this argument takes a video", src=src, origin=origin)
  269. return ResolvedImage(data=data, mime=sniffed, origin=origin)
  270. if "image" in permitted and b"<svg" in data[:4096].lower():
  271. # Pass SVG through — call sites rasterize it to PNG before embedding.
  272. return ResolvedImage(data=data, mime="image/svg+xml", origin=origin)
  273. if "video" in permitted:
  274. video_mime = _detect_video_mime(data, src)
  275. if video_mime is not None:
  276. return ResolvedImage(data=data, mime=video_mime, origin=origin)
  277. raise NotAnImage("source is not a recognized video (mp4 expected)", src=src, origin=origin)
  278. raise NotAnImage("source is not a recognized image", src=src, origin=origin)
  279. def _detect_video_mime(data: bytes, src: str) -> Optional[str]:
  280. """Video MIME from the extension table, else the ISO base-media ``ftyp`` magic at
  281. offset 4 (covers extensionless data: URLs / query-string URLs)."""
  282. from urllib.parse import urlsplit
  283. from tools.vision_tools import _detect_video_mime_type
  284. path_part = urlsplit(src).path if _SCHEME_RE.match(src) else src
  285. by_extension = _detect_video_mime_type(Path(path_part))
  286. if by_extension is not None:
  287. return by_extension
  288. if len(data) > 12 and data[4:8] == b"ftyp":
  289. return "video/mp4"
  290. return None
  291. async def resolve_local_source_to_data_url(
  292. src: str, task_id: Optional[str], *, permitted: tuple = ("image",)) -> str:
  293. """Convert a path-like media source into a ``data:`` URL via the resolver.
  294. Dispatch-layer chokepoint for generation tools so providers never read model-supplied paths
  295. off the HOST under a sandbox. URL-shaped sources pass through; callers apply this only under
  296. a non-local backend.
  297. """
  298. s = (src or "").strip()
  299. if not s or s.lower().startswith(("http://", "https://", "data:")):
  300. return src
  301. resolved = await resolve_image_source(s, ResolveContext(task_id=task_id), permitted=permitted)
  302. encoded = base64.b64encode(resolved.data).decode("ascii")
  303. mime = resolved.mime or "application/octet-stream"
  304. return f"data:{mime};base64,{encoded}"