terminal_image.py 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. """Terminal graphics: capability detection and image encoders for the CLI and the TUI.
  2. One home for the escape-level work so the pet mascot and inline images share it. Protocols
  3. in fidelity order: ``kitty`` (kitty, Ghostty, WezTerm), ``iterm`` (iTerm2, WezTerm),
  4. ``sixel`` (xterm -ti vt340, foot, mlterm, …), ``unicode`` (24-bit half-blocks).
  5. Inline images (:func:`render_inline_image`) use kitty Unicode placeholders and half-blocks, whose
  6. cells a line-based renderer can measure, plus iTerm2 images for a painter that reserves blank rows
  7. and draws over them (the classic CLI; Ink can't). Sixel terminals get half-blocks. PIL is
  8. imported lazily; a missing Pillow or an unreadable file yields ``None``.
  9. """
  10. from __future__ import annotations
  11. import base64
  12. import io
  13. import logging
  14. import os
  15. import re
  16. import warnings
  17. import zlib
  18. from dataclasses import dataclass, field
  19. from itertools import groupby
  20. from pathlib import Path
  21. from typing import Any
  22. logger = logging.getLogger(__name__)
  23. _DA1_REPLY = re.compile(rb"\x1b\[\?[0-9;]*c")
  24. def _is_wezterm() -> bool:
  25. return os.environ.get("TERM_PROGRAM", "").lower() == "wezterm" or bool(os.environ.get("WEZTERM_PANE"))
  26. def detect_terminal_graphics() -> str:
  27. """Richest protocol (``kitty``/``iterm``/``sixel``/``unicode``) from env vars only — never a DA1 query that could hang a pipe."""
  28. term = os.environ.get("TERM", "").lower()
  29. term_program = os.environ.get("TERM_PROGRAM", "").lower()
  30. # VS Code/Cursor set TERM_PROGRAM=vscode but don't scrub inherited
  31. # ITERM_SESSION_ID/KITTY_WINDOW_ID; trusting those emits a protocol xterm.js
  32. # can't show (blank frame). Inline images there are opt-in, so default to
  33. # half-blocks; users who enabled them can pin the render mode in config.
  34. if term_program == "vscode":
  35. return "unicode"
  36. if os.environ.get("KITTY_WINDOW_ID") or "kitty" in term or "ghostty" in term or term_program == "ghostty" or _is_wezterm():
  37. return "kitty" # WezTerm speaks kitty and iterm; kitty has richer placement
  38. if term_program == "iterm.app" or os.environ.get("ITERM_SESSION_ID"):
  39. return "iterm"
  40. if term_program == "mintty" or "foot" in term or "mlterm" in term or "sixel" in term:
  41. return "sixel"
  42. return "unicode"
  43. def supports_iterm_images() -> bool:
  44. """True when the terminal paints iTerm2 inline images (OSC 1337): iTerm2, WezTerm, Konsole,
  45. and VS Code once :func:`probe_vscode_images` saw its image support switched on."""
  46. if os.environ.get("TERM_PROGRAM", "").lower() == "vscode":
  47. return _VSCODE_IMAGES is True
  48. return detect_terminal_graphics() == "iterm" or _is_wezterm() or "KONSOLE_VERSION" in os.environ
  49. # VS Code draws images only with ``terminal.integrated.enableImages`` on, and nothing in the
  50. # environment says so. Its image addon is the only part that answers XTSMGRAPHICS, so one query
  51. # fenced by DA1 tells: measured live, the reply lands in ~7 ms and every run at a 10 ms deadline
  52. # was right; 8 ms missed some and let a late reply leak into the next read. 50 ms leaves headroom
  53. # without being felt, since the read ends as soon as the DA1 fence arrives.
  54. _VSCODE_IMAGES: bool | None = None
  55. _VSCODE_PROBE_TIMEOUT_S = 0.05
  56. _VSCODE_PROBE = "\x1b[?1;1;0S\x1b[c"
  57. _SIXEL_REPLY = b"\x1b[?1;0;"
  58. def probe_vscode_images() -> bool | None:
  59. """Whether VS Code's terminal will draw images; ``None`` outside a local VS Code terminal.
  60. Run once before prompt_toolkit owns the tty (the reply is read from stdin). Skipped over SSH,
  61. where the reply often misses the deadline (the prompt's parser then drops it, see
  62. ``hermes_cli.pt_input_extras``). Cached.
  63. """
  64. global _VSCODE_IMAGES
  65. if _VSCODE_IMAGES is not None or os.environ.get("TERM_PROGRAM", "").lower() != "vscode":
  66. return _VSCODE_IMAGES
  67. if any(os.environ.get(v) for v in ("SSH_CONNECTION", "SSH_CLIENT", "SSH_TTY")):
  68. return None
  69. try:
  70. import select
  71. import sys
  72. import termios
  73. import time
  74. import tty
  75. except ImportError: # Windows: no termios
  76. return None
  77. if not (sys.stdin.isatty() and sys.stdout.isatty()):
  78. return None
  79. fd = sys.stdin.fileno()
  80. old = termios.tcgetattr(fd)
  81. buf = b""
  82. try:
  83. tty.setcbreak(fd)
  84. sys.stdout.write(_VSCODE_PROBE)
  85. sys.stdout.flush()
  86. deadline = time.monotonic() + _VSCODE_PROBE_TIMEOUT_S
  87. while (left := deadline - time.monotonic()) > 0 and select.select([fd], [], [], left)[0]:
  88. chunk = os.read(fd, 256)
  89. buf += chunk
  90. if not chunk or _DA1_REPLY.search(buf):
  91. break
  92. finally:
  93. # A reply past the deadline: drain it while still in cbreak (in canonical mode a reply
  94. # without a newline is invisible to select and would reach the prompt as typed text).
  95. # Skipped once the DA1 fence arrived, so a normal start pays no extra wait.
  96. drain = time.monotonic() + (0 if _DA1_REPLY.search(buf) else 0.05)
  97. while (left := drain - time.monotonic()) > 0 and select.select([fd], [], [], left)[0]:
  98. chunk = os.read(fd, 256)
  99. buf += chunk
  100. if not chunk or _DA1_REPLY.search(buf):
  101. break
  102. termios.tcsetattr(fd, termios.TCSAFLUSH, old)
  103. _VSCODE_IMAGES = _SIXEL_REPLY in buf
  104. return _VSCODE_IMAGES
  105. def supports_kitty_placeholders() -> bool:
  106. """True when the terminal paints kitty Unicode placeholders; WezTerm speaks kitty APC but renders placeholders as tofu."""
  107. return detect_terminal_graphics() == "kitty" and not _is_wezterm()
  108. def _png_b64(image) -> str:
  109. buf = io.BytesIO()
  110. image.save(buf, format="PNG")
  111. return base64.standard_b64encode(buf.getvalue()).decode("ascii")
  112. # Nominal terminal cell size in pixels. kitty fits an image to its cell rectangle
  113. # preserving aspect, so a frame that isn't a whole cell multiple rounds up, clipping
  114. # the bottom row ("clipped feet"); snapping to an exact multiple avoids that.
  115. CELL_W, CELL_H = 8, 16
  116. def cell_box(image) -> tuple[int, int]:
  117. """Terminal cell box (~8×16 px per cell) for an image at its current pixel size.
  118. kitty stretches the image to fill ``c``×``r`` cells, so track the scaled pixel size, not a native-aspect column count (that upscales small pets).
  119. """
  120. return max(1, image.width // CELL_W), max(1, image.height // CELL_H)
  121. def _kitty_apc(ctrl: str, data: str) -> str:
  122. """kitty APC escape for *data*, chunked into ≤4096-byte ``m`` pieces (``m=1`` = more chunks follow)."""
  123. pieces = [data[i : i + 4096] for i in range(0, len(data), 4096)] or [""]
  124. last = len(pieces) - 1
  125. return "".join(f"\x1b_G{ctrl + ',' if i == 0 else ''}m={int(i != last)};{piece}\x1b\\" for i, piece in enumerate(pieces))
  126. def encode_kitty(image) -> str:
  127. """kitty transmit+display at the cursor; ``c``/``r`` pin the cell box so frames overwrite each other."""
  128. cols, rows = cell_box(image)
  129. return _kitty_apc(f"f=100,a=T,q=2,c={cols},r={rows}", _png_b64(image))
  130. # kitty Unicode placeholders: Ink owns the screen and measures every cell, so it
  131. # can't host raw kitty image escapes. Transmit once as a virtual placement (U=1),
  132. # then print ordinary-width placeholder cells (U+10EEEE + row diacritic) whose
  133. # foreground color encodes the image id; the terminal paints the image underneath.
  134. # https://sw.kovidgoyal.net/kitty/graphics-protocol/#unicode-placeholders
  135. _KITTY_PLACEHOLDER = "\U0010eeee"
  136. # Row diacritics by index, verbatim from kitty's gen/rowcolumn-diacritics.txt.
  137. _ROWCOL_DIACRITICS: tuple[int, ...] = (
  138. 0x0305, 0x030D, 0x030E, 0x0310, 0x0312, 0x033D, 0x033E, 0x033F, 0x0346, 0x034A, 0x034B, 0x034C, 0x0350, 0x0351, 0x0352, 0x0357, 0x035B, 0x0363, 0x0364, 0x0365,
  139. 0x0366, 0x0367, 0x0368, 0x0369, 0x036A, 0x036B, 0x036C, 0x036D, 0x036E, 0x036F, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, 0x0592, 0x0593, 0x0594, 0x0595, 0x0597,
  140. 0x0598, 0x0599, 0x059C, 0x059D, 0x059E, 0x059F, 0x05A0, 0x05A1, 0x05A8, 0x05A9, 0x05AB, 0x05AC, 0x05AF, 0x05C4, 0x0610, 0x0611, 0x0612, 0x0613, 0x0614, 0x0615,
  141. 0x0616, 0x0617, 0x0657, 0x0658, 0x0659, 0x065A, 0x065B, 0x065D, 0x065E, 0x06D6, 0x06D7, 0x06D8, 0x06D9, 0x06DA, 0x06DB, 0x06DC, 0x06DF, 0x06E0, 0x06E1, 0x06E2,
  142. 0x06E4, 0x06E7, 0x06E8, 0x06EB, 0x06EC, 0x0730, 0x0732, 0x0733, 0x0735, 0x0736, 0x073A, 0x073D, 0x073F, 0x0740, 0x0741, 0x0743, 0x0745, 0x0747, 0x0749, 0x074A,
  143. 0x07EB, 0x07EC, 0x07ED, 0x07EE, 0x07EF, 0x07F0, 0x07F1, 0x07F3, 0x0816, 0x0817, 0x0818, 0x0819, 0x081B, 0x081C, 0x081D, 0x081E, 0x081F, 0x0820, 0x0821, 0x0822,
  144. 0x0823, 0x0825, 0x0826, 0x0827, 0x0829, 0x082A, 0x082B, 0x082C, 0x082D, 0x0951, 0x0953, 0x0954, 0x0F82, 0x0F83, 0x0F86, 0x0F87, 0x135D, 0x135E, 0x135F, 0x17DD,
  145. 0x193A, 0x1A17, 0x1A75, 0x1A76, 0x1A77, 0x1A78, 0x1A79, 0x1A7A, 0x1A7B, 0x1A7C, 0x1B6B, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73, 0x1CD0, 0x1CD1,
  146. 0x1CD2, 0x1CDA, 0x1CDB, 0x1CE0, 0x1DC0, 0x1DC1, 0x1DC3, 0x1DC4, 0x1DC5, 0x1DC6, 0x1DC7, 0x1DC8, 0x1DC9, 0x1DCB, 0x1DCC, 0x1DD1, 0x1DD2, 0x1DD3, 0x1DD4, 0x1DD5,
  147. 0x1DD6, 0x1DD7, 0x1DD8, 0x1DD9, 0x1DDA, 0x1DDB, 0x1DDC, 0x1DDD, 0x1DDE, 0x1DDF, 0x1DE0, 0x1DE1, 0x1DE2, 0x1DE3, 0x1DE4, 0x1DE5, 0x1DE6, 0x1DFE, 0x20D0, 0x20D1,
  148. 0x20D4, 0x20D5, 0x20D6, 0x20D7, 0x20DB, 0x20DC, 0x20E1, 0x20E7, 0x20E9, 0x20F0, 0x2CEF, 0x2CF0, 0x2CF1, 0x2DE0, 0x2DE1, 0x2DE2, 0x2DE3, 0x2DE4, 0x2DE5, 0x2DE6,
  149. 0x2DE7, 0x2DE8, 0x2DE9, 0x2DEA, 0x2DEB, 0x2DEC, 0x2DED, 0x2DEE, 0x2DEF, 0x2DF0, 0x2DF1, 0x2DF2, 0x2DF3, 0x2DF4, 0x2DF5, 0x2DF6, 0x2DF7, 0x2DF8, 0x2DF9, 0x2DFA,
  150. 0x2DFB, 0x2DFC, 0x2DFD, 0x2DFE, 0x2DFF, 0xA66F, 0xA67C, 0xA67D, 0xA6F0, 0xA6F1, 0xA8E0, 0xA8E1, 0xA8E2, 0xA8E3, 0xA8E4, 0xA8E5, 0xA8E6, 0xA8E7, 0xA8E8, 0xA8E9,
  151. 0xA8EA, 0xA8EB, 0xA8EC, 0xA8ED, 0xA8EE, 0xA8EF, 0xA8F0, 0xA8F1, 0xAAB0, 0xAAB2, 0xAAB3, 0xAAB7, 0xAAB8, 0xAABE, 0xAABF, 0xAAC1, 0xFE20, 0xFE21, 0xFE22, 0xFE23,
  152. 0xFE24, 0xFE25, 0xFE26, 0x10A0F, 0x10A38, 0x1D185, 0x1D186, 0x1D187, 0x1D188, 0x1D189, 0x1D1AA, 0x1D1AB, 0x1D1AC, 0x1D1AD, 0x1D242, 0x1D243, 0x1D244,
  153. )
  154. def kitty_color_hex(image_id: int) -> str:
  155. """Hex foreground color (``#rrggbb``) that encodes *image_id* for kitty."""
  156. return "#%06x" % (image_id & 0xFFFFFF)
  157. def kitty_placeholder_rows(cols: int, rows: int) -> list[str]:
  158. """Placeholder text grid: first cell carries the row diacritic, the rest auto-increment the column (fg color applied by the painter)."""
  159. cols = max(1, cols)
  160. return [
  161. _KITTY_PLACEHOLDER + chr(_ROWCOL_DIACRITICS[min(r, len(_ROWCOL_DIACRITICS) - 1)]) + _KITTY_PLACEHOLDER * (cols - 1)
  162. for r in range(max(1, rows))
  163. ]
  164. def encode_kitty_virtual(image, *, image_id: int, cols: int, rows: int) -> str:
  165. """kitty virtual placement (``U=1``; ``q=2`` mutes replies that would corrupt the screen). Re-sending the same ``i`` replaces the image in place."""
  166. return _kitty_apc(f"a=T,U=1,i={image_id},c={cols},r={rows},f=100,q=2", _png_b64(image))
  167. def encode_iterm(image, *, cols: int | None = None, rows: int | None = None) -> str:
  168. """iTerm2 inline image (OSC 1337 File) pinned to a cell box (default: the image's own).
  169. ``size`` is the decoded byte count; xterm.js (VS Code) drops an image whose ``size`` is the
  170. base64 length.
  171. """
  172. buf = io.BytesIO()
  173. image.save(buf, format="PNG")
  174. payload = base64.standard_b64encode(buf.getvalue()).decode("ascii")
  175. if cols is None or rows is None:
  176. cols, rows = cell_box(image)
  177. return f"\x1b]1337;File=inline=1;size={buf.tell()};preserveAspectRatio=1;width={cols};height={rows}:{payload}\x07"
  178. def encode_sixel(image) -> str:
  179. """DEC sixel via a compact hand-rolled encoder (Pillow has none); ≤255 adaptive colors, transparent pixels skipped."""
  180. from PIL import Image
  181. pal = image.convert("RGB").quantize(colors=255, method=Image.MEDIANCUT)
  182. palette = pal.getpalette() or []
  183. px = pal.load()
  184. alpha = image.getchannel("A").load()
  185. w, h = pal.size
  186. out = ["\x1bP0;1;0q", f'"1;1;{w:d};{h:d}']
  187. used = sorted({px[x, y] for y in range(h) for x in range(w)})
  188. for idx in used: # color registers on a 0..100 scale
  189. r, g, b = (palette[idx * 3 + c] if idx * 3 + c < len(palette) else 0 for c in range(3))
  190. out.append(f'#{idx:d};2;{r * 100 // 255:d};{g * 100 // 255:d};{b * 100 // 255:d}')
  191. for band in range(0, h, 6):
  192. ys = range(band, min(band + 6, h))
  193. for color_idx in used:
  194. chars = [chr(63 + sum(1 << (y - band) for y in ys if alpha[x, y] > 32 and px[x, y] == color_idx)) for x in range(w)]
  195. runs = ((ch, len(list(group))) for ch, group in groupby(chars)) # run-length: ``!<n><ch>`` for runs longer than 3
  196. out.append(f'#{color_idx:d}' + "".join(f"!{n:d}{ch}" if n > 3 else ch * n for ch, n in runs) + "$") # ``$`` = band CR
  197. out.append("-") # next band
  198. return "".join(out) + "\x1b\\"
  199. # A single half-block cell: top pixel + bottom pixel as (r, g, b, a) tuples.
  200. Cell = tuple[tuple[int, int, int, int], tuple[int, int, int, int]]
  201. def downscale_cells(image, *, target_cols: int, rows: int | None = None) -> list[list[Cell]]:
  202. """Downscale to rows of half-block cells (one terminal row = two pixel rows); shared by the ANSI encoder and Ink.
  203. Without *rows* the grid keeps the image's aspect at a 4-column floor; with it the grid is exactly ``target_cols``×``rows``.
  204. """
  205. from PIL import Image
  206. if rows is None:
  207. target_cols = max(4, target_cols)
  208. target_rows = max(2, round(target_cols * (image.height / max(1, image.width)) * 0.5) * 2)
  209. else:
  210. target_cols, target_rows = max(1, target_cols), max(1, rows) * 2
  211. px = image.resize((target_cols, target_rows), Image.LANCZOS).convert("RGBA").load()
  212. return [
  213. [(px[x, y], px[x, y + 1] if y + 1 < target_rows else (0, 0, 0, 0)) for x in range(target_cols)]
  214. for y in range(0, target_rows, 2)
  215. ]
  216. def half_block(top, bottom) -> str:
  217. """One truecolor ANSI half-block cell; a transparent half shows the terminal background."""
  218. (tr, tg, tb, ta), (br, bg, bb, ba) = top, bottom
  219. if ta < 32 and ba < 32:
  220. return "\x1b[0m "
  221. if ba < 32:
  222. return f"\x1b[0m\x1b[38;2;{tr};{tg};{tb}m▀"
  223. if ta < 32:
  224. return f"\x1b[0m\x1b[38;2;{br};{bg};{bb}m▄"
  225. return f"\x1b[38;2;{tr};{tg};{tb}m\x1b[48;2;{br};{bg};{bb}m▀"
  226. def encode_unicode(image, *, target_cols: int) -> str:
  227. """Truecolor ANSI half-blocks (one char = 2 vertical pixels)."""
  228. return "\n".join("".join(half_block(t, b) for t, b in row) + "\x1b[0m"
  229. for row in downscale_cells(image, target_cols=target_cols))
  230. # ── Inline images ────────────────────────────────────────────────────────────
  231. # Names accepted by ``display.inline_images``.
  232. _INLINE_IMAGE_MODES = ("auto", "kitty", "iterm", "unicode", "off")
  233. # The image files Pillow decodes: what the CLI and the TUI both draw (attachments and tool images).
  234. _RENDERABLE_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".ico"})
  235. # Pet ids live in [1, 0x7FFF] (``agent.pet.render.kitty_image_id``); images take the rest of
  236. # the 24-bit space the placeholder foreground color can carry, so neither replaces the other.
  237. _IMAGE_ID_BASE, _IMAGE_ID_SPAN = 0x8000, 0xFFFFFF - 0x8000
  238. # Multiplexers that swallow image transmits (and inherit the outer terminal's KITTY_WINDOW_ID).
  239. _MULTIPLEXER_ENV = ("TMUX", "STY")
  240. # Transmit at up to 2 px per nominal cell pixel: sharp on HiDPI cells without shipping
  241. # a full-resolution photo through the tty.
  242. _PIXELS_PER_CELL_PIXEL = 2
  243. # Refuse to decode a preview past this many pixels (after JPEG draft scaling): a 169 MP PNG is
  244. # under Pillow's bomb error and would fully decode (seconds, >1 GB) on the CLI's agent thread.
  245. _MAX_DECODE_PIXELS = 40_000_000
  246. def resolve_inline_mode(
  247. configured: Any, *, graphics: bool = True, overlays: bool = False, detected: str | None = None
  248. ) -> str:
  249. """Effective ``kitty``/``iterm``/``unicode``/``off`` for ``display.inline_images``.
  250. ``graphics`` says the painter owns a real terminal it can send escapes to; ``overlays`` that
  251. it can reserve blank rows and paint a cell-pinned image over them (the classic CLI's
  252. scrollback can, Ink's re-laid-out frame can't). ``auto`` picks kitty placeholders where they
  253. paint (kitty, Ghostty), then iTerm2 images over reserved rows where the painter allows it
  254. (iTerm2, WezTerm, Konsole), else half-blocks. tmux and GNU screen swallow both transmits,
  255. so ``auto`` stays on half-blocks there. Pinning a protocol opts in: the classic CLI wraps its
  256. transmits for tmux's ``allow-passthrough on``; the TUI and GNU screen send them unwrapped.
  257. ``detected`` is a remote painter's own ``auto`` pick (``kitty``/``unicode``), made from its
  258. terminal's env: a shared gateway serves many clients, and its own env names none of them.
  259. """
  260. if configured is False:
  261. return "off"
  262. mode = "auto" if configured in (None, True) else str(configured).strip().lower()
  263. mode = mode if mode in _INLINE_IMAGE_MODES else "auto"
  264. if mode == "iterm" and not overlays:
  265. mode = "auto"
  266. if mode == "off":
  267. return "off"
  268. if mode == "unicode" or not graphics:
  269. return "unicode"
  270. if mode in ("kitty", "iterm"):
  271. return mode
  272. if detected is not None:
  273. return "kitty" if detected == "kitty" else "unicode"
  274. if any(os.environ.get(name) for name in _MULTIPLEXER_ENV):
  275. return "unicode"
  276. if supports_kitty_placeholders():
  277. return "kitty"
  278. return "iterm" if overlays and supports_iterm_images() else "unicode"
  279. def local_image_path(candidate: Any) -> str | None:
  280. """*candidate* as an existing local image file path, else ``None`` (URLs and data URLs are not fetched)."""
  281. if not isinstance(candidate, str) or not candidate.strip() or "://" in candidate or candidate.startswith("data:"):
  282. return None
  283. path = Path(os.path.expanduser(candidate.strip()))
  284. try:
  285. return str(path) if path.suffix.lower() in _RENDERABLE_EXTENSIONS and path.is_file() else None
  286. except (OSError, ValueError):
  287. return None
  288. # Result fields that name an image the call produced (browser_exec, computer_use), read from the
  289. # result or from a multimodal envelope's ``meta``.
  290. _RESULT_IMAGE_KEYS = ("image_path", "screenshot_path")
  291. # Tools whose image is what the call showed or made: (``args``/``result``, key). Nothing else's
  292. # args count: a ``path`` is just as often a write, a refused patch or a read that can't show it.
  293. _TOOL_IMAGE_FIELDS: dict[str, tuple[tuple[str, str], ...]] = {
  294. "vision_analyze": (("args", "image_url"),),
  295. "image_generate": (("result", "image"),),
  296. }
  297. def tool_image_path(tool_name: str, args: Any, result: Any, *, is_error: bool) -> str | None:
  298. """The local image a successful tool call produced or looked at, else ``None``.
  299. Named fields only, so a tool that lists image paths in free text never floods the transcript,
  300. and nothing for a failed call: a refused write or an unreadable file must not look like success.
  301. """
  302. if is_error:
  303. return None
  304. if isinstance(result, str):
  305. from utils import safe_json_loads
  306. result = safe_json_loads(result)
  307. result = result if isinstance(result, dict) else {}
  308. meta = result.get("meta") if isinstance(result.get("meta"), dict) else {}
  309. sources = {"args": args if isinstance(args, dict) else {}, "result": result}
  310. for source, key in (*_TOOL_IMAGE_FIELDS.get(tool_name, ()), *(("result", key) for key in _RESULT_IMAGE_KEYS)):
  311. for candidate in (sources[source].get(key), meta.get(key) if source == "result" else None):
  312. if path := local_image_path(candidate):
  313. return path
  314. return None
  315. @dataclass(frozen=True)
  316. class InlineImage:
  317. """One image laid out on a ``cols``×``rows`` cell grid.
  318. kitty mode: send ``transmit`` once, then paint ``placeholder`` rows in ``color``.
  319. unicode mode: paint ``cells`` as half-blocks.
  320. """
  321. mode: str
  322. cols: int
  323. rows: int
  324. color: str = ""
  325. transmit: str = ""
  326. placeholder: list[str] = field(default_factory=list)
  327. cells: list[list[Cell]] = field(default_factory=list)
  328. def _image_id(path: Path, cols: int, rows: int) -> int:
  329. """Stable kitty image id per file revision and cell grid: re-renders reuse the terminal copy, an
  330. edited file gets a new one, and two sizes on screen at once each keep their own placement (kitty
  331. holds one virtual placement per id)."""
  332. st = path.stat()
  333. key = f"{path.resolve()}:{st.st_mtime_ns}:{st.st_size}:{cols}x{rows}"
  334. return _IMAGE_ID_BASE + zlib.crc32(key.encode()) % _IMAGE_ID_SPAN
  335. def _grid_for(width: int, height: int, *, max_cols: int, max_rows: int) -> tuple[int, int]:
  336. """Cell grid for a *width*×*height* image: never wider than its natural size, never past the caps."""
  337. cols = max(1, min(max_cols, -(-width // CELL_W)))
  338. rows = max(1, round(cols * height / width * CELL_W / CELL_H))
  339. if rows > max_rows:
  340. rows = max_rows
  341. cols = max(1, min(cols, round(rows * width / height * CELL_H / CELL_W)))
  342. return cols, rows
  343. def _decode_preview(source: Path, *, max_cols: int, max_rows: int):
  344. """``(image, cols, rows)``: *source* decoded no larger than its preview needs, upright, RGBA.
  345. The grid comes from the header size; JPEG decodes straight at the target scale (``draft``);
  346. a pixel count past ``_MAX_DECODE_PIXELS`` or a Pillow bomb warning refuses (raises)
  347. before any pixel is decoded.
  348. """
  349. from PIL import Image, ImageOps
  350. with warnings.catch_warnings():
  351. warnings.simplefilter("error", Image.DecompressionBombWarning)
  352. with Image.open(source) as opened:
  353. width, height = opened.size
  354. # Only ``draft`` (JPEG) can shrink the decode, and EXIF on a PNG is read by decoding it.
  355. if width * height > _MAX_DECODE_PIXELS and type(opened).draft is Image.Image.draft:
  356. raise ValueError(f"{width}x{height} is past the preview decode cap")
  357. swapped = opened.getexif().get(0x0112) in (5, 6, 7, 8) # EXIF orientations that turn 90°
  358. if swapped:
  359. width, height = height, width
  360. cols, rows = _grid_for(width, height, max_cols=max_cols, max_rows=max_rows)
  361. box = (cols * CELL_W * _PIXELS_PER_CELL_PIXEL, rows * CELL_H * _PIXELS_PER_CELL_PIXEL)
  362. box = box[::-1] if swapped else box
  363. opened.draft("RGB", box)
  364. if opened.width * opened.height > _MAX_DECODE_PIXELS:
  365. raise ValueError(f"{opened.width}x{opened.height} is past the preview decode cap")
  366. opened.thumbnail(box, Image.LANCZOS)
  367. image = ImageOps.exif_transpose(opened).convert("RGBA")
  368. return image, cols, rows
  369. def render_inline_image(path: str | Path, *, mode: str, max_cols: int, max_rows: int) -> InlineImage | None:
  370. """Lay out the image at *path* for *mode* (``kitty``/``iterm``/``unicode``); ``None`` when it can't be shown.
  371. ``iterm`` returns a ``transmit`` pinned to exactly ``cols``×``rows`` cells for a painter that
  372. reserved those rows; it draws at the cursor and owns no text cells.
  373. """
  374. if mode not in ("kitty", "iterm", "unicode") or max_cols < 1 or max_rows < 1:
  375. return None
  376. try:
  377. source = Path(path)
  378. image, cols, rows = _decode_preview(source, max_cols=max_cols, max_rows=max_rows)
  379. if mode == "unicode":
  380. return InlineImage(mode=mode, cols=cols, rows=rows, cells=downscale_cells(image, target_cols=cols, rows=rows))
  381. if mode == "iterm":
  382. return InlineImage(mode=mode, cols=cols, rows=rows, transmit=encode_iterm(image, cols=cols, rows=rows))
  383. image_id = _image_id(source, cols, rows)
  384. return InlineImage(
  385. mode=mode, cols=cols, rows=rows, color=kitty_color_hex(image_id),
  386. transmit=encode_kitty_virtual(image, image_id=image_id, cols=cols, rows=rows),
  387. placeholder=kitty_placeholder_rows(cols, rows))
  388. except Exception as exc: # health: allow BLE001 -- PIL raises arbitrary types decoding an untrusted file; display-only
  389. logger.debug("inline image render failed (%s): %s", path, exc)
  390. return None