| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354 |
- """Single resolver for every media source (data:/http(s)/file/local/container) -> bytes + mime.
- Everything funnels through :func:`resolve_image_source` so size and magic-byte checks run
- exactly once. Images are the default; video callers opt in via ``permitted=("video",)``.
- Security (GHSA-gpxw-6wxv-w3qq): under a non-local backend vision is confined like the file
- tools — host-read only inside a media cache (bind-mounted into the sandbox), anything else is
- exec-read *inside the sandbox*, so ``vision_analyze('/etc/passwd')`` never reads the host.
- """
- from __future__ import annotations
- import asyncio
- import base64
- import os
- import re
- from dataclasses import dataclass
- from pathlib import Path
- from typing import Optional
- # Raw-bytes INGEST budget for sandbox and data: sources (50 MiB). The sandbox exec-read returns
- # the file as base64 on stdout, so captured text + decoded bytes + copies peak near 4x the file
- # in this process; it stays at the 50MB download cap. HTTP downloads keep their own stream cap
- # (vision_tools._VISION_MAX_DOWNLOAD_BYTES).
- _MAX_INGEST_BYTES = 50 * 1024 * 1024
- # Host-file reads (250 MiB): bounded during the read, big enough for full-res camera/scan
- # images, which call sites downscale before the 20MB provider payload cap.
- _MAX_HOST_READ_BYTES = 250 * 1024 * 1024
- # Chunk size for host reads whose st_size under-reports (see _read_host_bytes_bounded).
- _HOST_READ_CHUNK = 1024 * 1024
- class ImageResolutionError(Exception):
- def __init__(self, message: str, *, src: str = "", origin: str = ""):
- super().__init__(message)
- self.src, self.origin = src, origin
- class UnsupportedScheme(ImageResolutionError): ...
- class SourceUnsafe(ImageResolutionError): ... # SSRF / path-allowlist
- class SourceTooLarge(ImageResolutionError): ...
- class SourceNotFound(ImageResolutionError): ...
- class NotAnImage(ImageResolutionError): ...
- @dataclass
- class ResolveContext:
- task_id: Optional[str] = None
- @dataclass
- class ResolvedImage:
- data: bytes
- mime: str
- origin: str # one of: data | http | file | local | container
- # Explicit URL scheme ("ftp://", "s3://"). Bare Windows drive paths lack the "//".
- _SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*://")
- async def resolve_image_source(
- src: str, ctx: ResolveContext, *, permitted: tuple = ("image",)) -> ResolvedImage:
- if not isinstance(src, str) or not src.strip():
- raise SourceNotFound("image_url is required", src=str(src))
- s = src.strip()
- if s.startswith("data:"):
- data, mime = _resolve_data_url(s)
- return _finalize(data, mime, "data", s, permitted)
- if s.startswith(("http://", "https://")):
- reason = _http_block_reason(s)
- if reason:
- raise SourceUnsafe(reason, src=s)
- return _finalize(await _download_to_bytes(s), "", "http", s, permitted)
- if _SCHEME_RE.match(s) and not s.lower().startswith("file://"):
- raise UnsupportedScheme(
- "Unrecognized image source scheme. Use an http(s) URL, a local "
- "file path, a file:// URI, or a data: URL.",
- src=s)
- # Everything else is a filesystem path — including bare relative names like "pic.png"
- # (a path-shape gate here regressed them once).
- candidate = s[len("file://"):] if s.lower().startswith("file://") else s
- p = Path(os.path.expanduser(candidate))
- host_target = _permitted_host_read_target(p, ctx)
- if host_target is not None and host_target.is_file():
- _guard_credential_read(host_target, s)
- # Videos are never downscaled and video_analyze refuses over 50 MB of base64, so the
- # larger host budget only helps images; videos keep the ingest cap and fail at the stat.
- cap = _MAX_HOST_READ_BYTES if "image" in permitted else _MAX_INGEST_BYTES
- data = await asyncio.to_thread(_read_host_bytes_bounded, host_target, s, cap)
- return _finalize(data, "", "file", s, permitted)
- if _is_local_terminal_backend():
- # Any path was host-readable, so a miss means the file doesn't exist.
- raise SourceNotFound(f"media file not found: '{p}'", src=s, origin="file")
- return await _resolve_container_fallback(p, ctx, s, permitted)
- def _read_host_bytes_bounded(path: Path, src: str, cap: int) -> bytes:
- """Read a host file with ``cap`` enforced DURING the read. ``read_bytes()`` buffered the whole
- file before ``_finalize`` checked its size. ``read(n)`` preallocates n bytes, so the first read
- asks for the stat'd size (not the cap); when ``st_size`` under-reports (procfs-style files) the
- rest streams in chunks and stops past ``cap``. The honest-size check reads nothing at all."""
- size = path.stat().st_size
- if size > cap:
- raise SourceTooLarge("media exceeds size limit", src=src, origin="file")
- with path.open("rb") as fh:
- data = fh.read(size + 1)
- if len(data) > size:
- parts, total = [data], len(data)
- while total <= cap and (chunk := fh.read(_HOST_READ_CHUNK)):
- parts.append(chunk)
- total += len(chunk)
- if total > cap:
- raise SourceTooLarge("media exceeds size limit", src=src, origin="file")
- data = b"".join(parts)
- return data
- def _guard_credential_read(host_target: Path, src: str) -> None:
- """Shared credential-read guard: refuse secret-bearing files (.env, auth.json) with a specific
- error. Guard import is best-effort; a real block always propagates."""
- try:
- # Shared credential-read guard (agent.file_safety, #57698): refuse secret-bearing files (.env,
- # auth.json, ...) with an intentional, specific error instead of relying on the magic-byte sniff to
- # reject them incidentally. Same chokepoint the image-gen/video-gen provider plugins enforce on
- # model-supplied local paths. Import is best-effort (guard unavailability must not break image
- # loading); a real block always propagates.
- from agent.file_safety import raise_if_read_blocked
- except Exception:
- return
- try:
- raise_if_read_blocked(str(host_target))
- except ValueError as exc:
- raise SourceUnsafe(str(exc), src=src, origin="file")
- def _resolve_data_url(s: str) -> tuple[bytes, str]:
- header, _, payload = s.partition(",")
- if ";base64" not in header:
- raise NotAnImage("data: URL must be base64-encoded", src=s[:64])
- declared = header[len("data:"):].split(";", 1)[0].strip() or "application/octet-stream"
- # Cheap pre-decode size gate on the encoded length (~4/3 expansion).
- if (len(payload) * 3) // 4 > _MAX_INGEST_BYTES:
- raise SourceTooLarge("data: URL exceeds size limit", src=s[:64])
- try:
- data = base64.b64decode(payload, validate=True)
- except Exception as exc:
- raise NotAnImage(f"invalid base64 in data: URL: {exc}", src=s[:64])
- return data, declared # real mime verified in _finalize via magic bytes
- def _http_block_reason(url: str) -> Optional[str]:
- """Block reason, or None when allowed. Refuses policy-blocked URLs BEFORE any network I/O;
- ``_download_image`` re-checks per attempt and against the final redirect target (intentional)."""
- from tools.url_safety import is_safe_url
- from tools.website_policy import check_website_access
- if not is_safe_url(url):
- return "blocked: unsafe or private URL"
- if blocked := check_website_access(url):
- return blocked.get("message") or "blocked by website policy"
- return None
- async def _download_to_bytes(url: str) -> bytes:
- import tempfile
- from tools.vision_tools import _download_image
- with tempfile.NamedTemporaryFile(suffix=".img", delete=False) as tf:
- tmp = Path(tf.name)
- try:
- # Enforces the 50MB stream cap, redirect SSRF guard, and website policy.
- await _download_image(url, tmp)
- return await asyncio.to_thread(tmp.read_bytes)
- except PermissionError as exc: # website policy block
- raise SourceUnsafe(str(exc), src=url, origin="http")
- finally:
- tmp.unlink(missing_ok=True)
- def _is_local_terminal_backend() -> bool:
- """True when the terminal backend runs directly on the host (keys off ``TERMINAL_ENV``, read
- through the per-turn terminal scope so a routed multiplex profile sees ITS backend)."""
- from tools.terminal_scope import terminal_env
- return terminal_env("TERMINAL_ENV", "local").strip().lower() in ("local", "")
- # Host-side media caches: the only host paths vision may read under a non-local backend
- # (gateway inbound media + the tools' own download temp dirs).
- _MEDIA_CACHE_SUBDIRS = (
- "cache", # cache/images, cache/vision, cache/video(s), cache/audio
- "images", # desktop/clipboard/PDF uploads (tui_gateway)
- "image_cache", "audio_cache", "video_cache", "temp_vision_images", "temp_video_files")
- def _media_cache_roots() -> list:
- from hermes_constants import get_hermes_home
- home = get_hermes_home()
- return [home / sub for sub in _MEDIA_CACHE_SUBDIRS]
- def _permitted_host_read_target(p: Path, ctx: ResolveContext) -> Optional[Path]:
- """Host path to read, or ``None`` (caller exec-reads inside the sandbox instead).
- Local backend: any path. Non-local: only paths inside a media cache root (a
- container-visible cache path is first translated back to its host mount).
- """
- if _is_local_terminal_backend():
- try:
- return p.resolve()
- except Exception:
- return p
- from tools.credential_files import from_agent_visible_cache_path
- try:
- real = Path(from_agent_visible_cache_path(str(p))).resolve()
- except Exception:
- return None
- if any(real.is_relative_to(root.resolve()) for root in _media_cache_roots()):
- return real
- return None
- def _get_active_env(task_id: Optional[str]):
- if not task_id:
- return None
- try:
- from tools.terminal_tool_lifecycle import get_active_env
- return get_active_env(task_id)
- except Exception:
- return None
- def _ensure_container_env(task_id: Optional[str]) -> None:
- """Lazily bring up the sandbox before an in-sandbox read (vision may be a session's first
- action). Best-effort: failure leaves the env absent and the caller hits the fail-closed error.
- Unlike the terminal tool, vision never triggered environment creation, so a session whose first action
- is ``vision_analyze`` on a container-only path under a non-local backend found no active env and failed
- — until a terminal command happened to create one (issue #62825).
- """
- if not task_id:
- return
- try:
- from tools.terminal_tool import ensure_task_env
- ensure_task_env(task_id)
- except Exception:
- pass
- async def _resolve_container_fallback(
- p: Path, ctx: ResolveContext, src: str, permitted: tuple = ("image",)) -> ResolvedImage:
- """Read the bytes inside the sandbox; fail-closed when no env exists (a non-cache host
- path under a sandbox must never leak via a host fallback).
- Cold-start retry: under Docker the first exec against a fresh container can fail (empty
- pipe) while a second succeeds. On final failure the container's output is folded into the
- error so "no such file" / "permission denied" / "never came up" are distinguishable.
- We retry once with a short delay before giving up, so callers don't see "could not read inside the
- sandbox" on a file that is verifiably readable on the immediate retry. See #76566.
- """
- import shlex
- # Bring the sandbox up on demand: without this, the first vision_analyze of a session (before any
- # terminal command) has no active env to read from under a non-local backend (issue #62825).
- _ensure_container_env(ctx.task_id)
- env = _get_active_env(ctx.task_id)
- if env is None:
- raise SourceNotFound(
- f"'{p}' is not reachable inside the sandbox and no active sandbox "
- f"session is available to read it",
- src=src, origin="container")
- from tools.terminal_tool_config import translate_mounted_host_path
- translated = translate_mounted_host_path(
- str(p),
- getattr(env, "host_cwd", None) or "",
- getattr(env, "host_cwd_mount", None) or "/workspace",
- )
- if translated:
- p = Path(translated)
- # Bound the read INSIDE the sandbox: head -c caps at ingest-limit+1 (+1 distinguishes "at the
- # cap" from "over") so /dev/zero can't stream unbounded base64 into host memory. The input
- # redirect avoids argv (leading-dash paths); tr -d instead of GNU-only base64 -w0 (BusyBox).
- cmd = f"head -c {_MAX_INGEST_BYTES + 1} < {shlex.quote(str(p))} | base64 | tr -d '\\n'"
- last_res: dict = {"returncode": 1, "output": ""}
- for attempt in range(2):
- last_res = await asyncio.to_thread(env.execute, cmd)
- if last_res.get("returncode", 1) == 0:
- break
- if attempt == 0:
- await asyncio.sleep(0.15) # covers Docker exec warm-up in practice
- if last_res.get("returncode", 1) != 0:
- diag = (last_res.get("output") or "").strip().splitlines()
- first = next((ln.strip() for ln in diag if ln.strip()), "")
- suffix = f" ({first[:200]})" if first else ""
- raise SourceNotFound(f"could not read '{p}' inside the sandbox{suffix}", src=src, origin="container")
- try:
- data = base64.b64decode(last_res.get("output", ""), validate=True)
- except Exception as exc:
- raise NotAnImage(f"sandbox returned non-image data for '{p}': {exc}", src=src)
- if len(data) > _MAX_INGEST_BYTES:
- raise SourceTooLarge("media exceeds size limit", src=src, origin="container")
- return _finalize(data, "", "container", src, permitted)
- def _finalize(
- data: bytes, declared_mime: str, origin: str, src: str, permitted: tuple = ("image",)
- ) -> ResolvedImage:
- """Chokepoint: size ceiling + type check. Images by magic bytes; video (opt-in) by
- extension + mp4 sniff — enough because every downstream consumer re-validates. Each
- reader enforces its own tighter cap first; this is the ceiling for all of them."""
- from tools.vision_tools_image_prep import _detect_image_mime_type_from_bytes
- if len(data) > _MAX_HOST_READ_BYTES:
- raise SourceTooLarge("media exceeds size limit", src=src, origin=origin)
- sniffed = _detect_image_mime_type_from_bytes(data)
- if sniffed is not None:
- if "image" not in permitted:
- raise NotAnImage("source is an image, but this argument takes a video", src=src, origin=origin)
- return ResolvedImage(data=data, mime=sniffed, origin=origin)
- if "image" in permitted and b"<svg" in data[:4096].lower():
- # Pass SVG through — call sites rasterize it to PNG before embedding.
- return ResolvedImage(data=data, mime="image/svg+xml", origin=origin)
- if "video" in permitted:
- video_mime = _detect_video_mime(data, src)
- if video_mime is not None:
- return ResolvedImage(data=data, mime=video_mime, origin=origin)
- raise NotAnImage("source is not a recognized video (mp4 expected)", src=src, origin=origin)
- raise NotAnImage("source is not a recognized image", src=src, origin=origin)
- def _detect_video_mime(data: bytes, src: str) -> Optional[str]:
- """Video MIME from the extension table, else the ISO base-media ``ftyp`` magic at
- offset 4 (covers extensionless data: URLs / query-string URLs)."""
- from urllib.parse import urlsplit
- from tools.vision_tools import _detect_video_mime_type
- path_part = urlsplit(src).path if _SCHEME_RE.match(src) else src
- by_extension = _detect_video_mime_type(Path(path_part))
- if by_extension is not None:
- return by_extension
- if len(data) > 12 and data[4:8] == b"ftyp":
- return "video/mp4"
- return None
- async def resolve_local_source_to_data_url(
- src: str, task_id: Optional[str], *, permitted: tuple = ("image",)) -> str:
- """Convert a path-like media source into a ``data:`` URL via the resolver.
- Dispatch-layer chokepoint for generation tools so providers never read model-supplied paths
- off the HOST under a sandbox. URL-shaped sources pass through; callers apply this only under
- a non-local backend.
- """
- s = (src or "").strip()
- if not s or s.lower().startswith(("http://", "https://", "data:")):
- return src
- resolved = await resolve_image_source(s, ResolveContext(task_id=task_id), permitted=permitted)
- encoded = base64.b64encode(resolved.data).decode("ascii")
- mime = resolved.mime or "application/octet-stream"
- return f"data:{mime};base64,{encoded}"
|