image_eviction_policy.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288
  1. """Send-path image eviction policy shared by both stateless outbound passes.
  2. Two passes retire old tool-result images from the per-request copy of the conversation:
  3. ``agent.context_compressor.evict_stale_outbound_tool_images`` on the OpenAI-shaped list and
  4. ``agent.anthropic_message_convert._evict_old_screenshots`` on the Anthropic wire list. Both run
  5. from scratch on a fresh clone every request, so they must agree on one policy or the second
  6. pass re-evicts on a different frontier than the first (#113517). This module is stdlib-only so
  7. the wire converter, a leaf, can import it without dragging in the compaction stack.
  8. Why the trigger is a provider limit, not a keep-newest count: retiring an image edits a message
  9. the provider has already cached, and Anthropic matches its prompt cache on an exact byte prefix.
  10. A keep-newest-N window retires one more message on every new image, so every turn is a
  11. full-prefix miss. Holding images until the request would cross a real API limit and then
  12. retiring a batch costs one slower turn per batch and nothing below the limit.
  13. 20 is the documented threshold at which Anthropic applies a stricter per-image dimension cap
  14. (2000 px) to EVERY image in the request, counting images nested in tool_result content. Crossing
  15. it costs nothing by itself: it only matters when some image actually exceeds 2000 px on a side.
  16. Hermes already shrinks its own tool images far below that before they enter history
  17. (``vision_analyze`` and browser screenshots embed at 1568 px, ``computer_use`` at 1456 px by
  18. default), so the 21st screenshot of a run trips no provider constraint and a batch retire there
  19. buys a full prefix rewrite for nothing — and, on the preserved-thinking models, invalidates every
  20. later thinking block. Anthropic's own computer-use guidance now says to keep screenshots at or
  21. under 2000 px and NOT to prune them client-side for exactly that reason.
  22. So the block ceiling is CONDITIONAL on the request's actual dimensions: the strict 20 applies
  23. only when some image exceeds ``MANY_IMAGE_DIMENSION_LIMIT``, when the stricter per-side cap is
  24. live and the retire genuinely protects the request; otherwise the ceiling is the hard count
  25. ``OUTBOUND_IMAGE_HARD_LIMIT``. The hard ceilings above 20 are real but distant (100 images per
  26. request on 200K-context models, 600 otherwise) and the 32 MB request-size limit usually binds
  27. first, which the byte budget guards with headroom for text. Classification reads image HEADERS
  28. only — never a full decode of every base64 payload per request — and treats unreadable or unknown
  29. dimensions as unsafe, so a payload this module cannot measure keeps today's strict trigger.
  30. """
  31. from __future__ import annotations
  32. import base64
  33. import struct
  34. from typing import Any, Iterable, Optional, Sequence
  35. OUTBOUND_IMAGE_LIMIT = 20
  36. # Hard per-request image count on 200K-context Claude models; the ceiling once the many-image
  37. # stricter cap is provably not live (see module docstring). Deliberately not 600 — the byte
  38. # budget below binds first for a 200K-context transcript, and the byte ceiling needs no context
  39. # knowledge to be safe.
  40. OUTBOUND_IMAGE_HARD_LIMIT = 100
  41. OUTBOUND_IMAGE_BUDGET_BYTES = 24_000_000
  42. IMAGE_EVICTION_BATCH = 8
  43. # Satisfiability floor — see outbound_image_retire_count.
  44. OUTBOUND_IMAGE_FLOOR = 3
  45. # Anthropic's stricter many-image per-side cap (px): above it, every image in the request is held
  46. # to this limit, so an image over it keeps the strict trigger live.
  47. MANY_IMAGE_DIMENSION_LIMIT = 2000
  48. # Bytes of an embedded image to decode when reading its header. PNG IHDR ends at byte 24; a JPEG
  49. # SOF marker lives within the first few KB. ~32 KB of base64 is generous headroom, and only the
  50. # prefix is ever decoded (base64 expands 3 bytes per 4 chars).
  51. _DIMENSION_PROBE_CHARS = 32 * 1024
  52. _JPEG_SOF_MARKERS = frozenset({0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF})
  53. def image_dimensions_from_bytes(raw: bytes) -> Optional[tuple[int, int]]:
  54. """(width, height) for PNG / JPEG bytes, or None when unreadable. PNG: IHDR. JPEG: walk
  55. segments (skipping 0xFF fill bytes) to the first SOF marker; stop at SOS. Lives here so both
  56. the send-path passes and the tool layer share one header-only reader."""
  57. if raw.startswith(b"\x89PNG\r\n\x1a\n") and len(raw) >= 24:
  58. width, height = struct.unpack(">II", raw[16:24]) # cannot fail: 8 bytes are guaranteed present
  59. return int(width), int(height)
  60. if raw.startswith(b"\xff\xd8") and len(raw) > 4:
  61. i = 2
  62. while i + 9 < len(raw):
  63. if raw[i] != 0xFF:
  64. i += 1
  65. continue
  66. marker, i = raw[i + 1], i + 2
  67. while marker == 0xFF and i < len(raw):
  68. marker, i = raw[i], i + 1
  69. if marker in {0xD8, 0xD9}:
  70. continue
  71. if marker == 0xDA or i + 2 > len(raw):
  72. break
  73. segment_len = int.from_bytes(raw[i:i + 2], "big")
  74. if segment_len < 2 or i + segment_len > len(raw):
  75. break
  76. if marker in _JPEG_SOF_MARKERS and segment_len >= 7:
  77. return int.from_bytes(raw[i + 5:i + 7], "big"), int.from_bytes(raw[i + 3:i + 5], "big")
  78. i += segment_len
  79. return None
  80. def image_max_side_from_data(data: Optional[str]) -> Optional[int]:
  81. """Longest side (px) of a data-URL / bare-base64 image, or None when it cannot be read.
  82. Decodes only the leading base64 prefix (``_DIMENSION_PROBE_CHARS``), so classifying a request
  83. with dozens of images costs a few header reads, not a full decode of every payload. Accepts
  84. both ``data:image/...;base64,....`` and a bare base64 string; anything else (a remote URL, an
  85. unrecognized container) yields None.
  86. """
  87. if not data:
  88. return None
  89. payload = data
  90. if payload.startswith("data:"):
  91. _, separator, payload = payload.partition(",")
  92. if not separator or ";base64" not in data[:data.index(",")]:
  93. return None
  94. payload = payload.strip()
  95. prefix = payload[:_DIMENSION_PROBE_CHARS]
  96. prefix = prefix[: len(prefix) - (len(prefix) % 4)] # decode needs a multiple-of-four length
  97. if not prefix:
  98. return None
  99. try:
  100. raw = base64.b64decode(prefix, validate=False)
  101. except (ValueError, TypeError):
  102. return None
  103. dims = image_dimensions_from_bytes(raw)
  104. return max(dims) if dims is not None else None
  105. def image_within_many_image_limit(data: Optional[str]) -> bool:
  106. """True only when the image's longest side is known and within ``MANY_IMAGE_DIMENSION_LIMIT``.
  107. Unknown dimensions are NOT safe: the stricter per-request cap would still be live, so the
  108. caller keeps the strict trigger. See :func:`images_all_within_many_image_limit`.
  109. """
  110. longest = image_max_side_from_data(data)
  111. return longest is not None and longest <= MANY_IMAGE_DIMENSION_LIMIT
  112. def images_all_within_many_image_limit(datas: Iterable[Optional[str]]) -> bool:
  113. """True when EVERY image in the request is a readable image within the many-image limit.
  114. The Anthropic rule is per request: one oversized image holds every image in the request to the
  115. stricter cap, so a single unreadable or over-limit image must keep the strict block ceiling.
  116. Short-circuits on the first offender, which is what keeps this cheap on image-heavy turns.
  117. """
  118. for data in datas:
  119. if not image_within_many_image_limit(data):
  120. return False
  121. return True
  122. def outbound_image_block_limit(many_image_safe: bool) -> int:
  123. """Block ceiling for a request: the strict 20 unless every image is within the many-image cap."""
  124. return OUTBOUND_IMAGE_HARD_LIMIT if many_image_safe else OUTBOUND_IMAGE_LIMIT
  125. def outbound_image_retire_count(
  126. carrier_blocks_newest_first: Sequence[int],
  127. reserved_blocks: int,
  128. *,
  129. carrier_bytes_newest_first: Optional[Sequence[int]] = None,
  130. reserved_bytes: int = 0,
  131. many_image_safe: bool = False,
  132. limit: Optional[int] = None,
  133. budget: int = OUTBOUND_IMAGE_BUDGET_BYTES,
  134. batch: int = IMAGE_EVICTION_BATCH,
  135. floor: int = OUTBOUND_IMAGE_FLOOR,
  136. ) -> int:
  137. """How many of the OLDEST image-bearing tool results to retire.
  138. ``carrier_blocks_newest_first`` is the image-block count per image-bearing tool result,
  139. newest first; ``reserved_blocks`` counts images the pass must never rewrite (user
  140. uploads). The byte dimension is active only when ``carrier_bytes_newest_first`` is
  141. given. ``limit`` defaults to :func:`outbound_image_block_limit` of ``many_image_safe``,
  142. so both passes pick the same ceiling from the same per-request classification.
  143. The retire count must be a STEP FUNCTION of the overshoot, because the pass is recomputed
  144. on every request: an exact ``count - limit`` target moves the frontier on every new image,
  145. and a fixed one-batch retire stops enforcing the limit after the first batch. So the count
  146. advances in quanta until the request fits.
  147. The quantum is ``batch`` capped at ``window - floor``, where ``window`` is how many newest
  148. carriers fit under the ceiling. A quantum wider than that would step past the newest frames
  149. on every advance; cutting each step back to exactly ``total - floor`` instead makes the
  150. retire count track ``total`` again — the per-image frontier this policy exists to avoid,
  151. visible whenever a tool result carries several images or uploads fill most of the ceiling.
  152. Holding for ``window - floor`` turns per advance is the most the floor allows.
  153. The floor is a SATISFIABILITY floor: it shelters the newest frames only when reserved
  154. uploads alone breach the block ceiling (no retirement can fix that), and never under byte
  155. pressure — the request-size limit is hard and the provider answers 413. Even then the
  156. count stays a step function: it retires whole quanta down toward the floor (keeping
  157. ``floor`` to ``floor + batch - 1`` carriers, fewer when the byte budget binds), so the
  158. cached prefix is rewritten once per quantum rather than on every new image.
  159. """
  160. if limit is None:
  161. limit = outbound_image_block_limit(many_image_safe)
  162. total = len(carrier_blocks_newest_first)
  163. sizes = carrier_bytes_newest_first
  164. return max(
  165. (
  166. _retire_count_for_request(
  167. carrier_blocks_newest_first[total - m:],
  168. reserved_blocks,
  169. None if sizes is None else sizes[total - m:],
  170. reserved_bytes,
  171. limit,
  172. budget,
  173. batch,
  174. floor,
  175. )
  176. for m in range(1, total + 1)
  177. ),
  178. default=0,
  179. )
  180. def outbound_retired_image_ids(agent: Any) -> set[str]:
  181. """The session's latched set of tool-call ids whose images the send path has retired.
  182. Keyed on ``agent.session_id`` so a new session (or a compaction rotation, which rewrites the
  183. prefix anyway) starts clean and a reused provider call id cannot retire a fresh screenshot.
  184. """
  185. session_id = getattr(agent, "session_id", None)
  186. held = getattr(agent, "_outbound_retired_image_ids", None)
  187. if held is None or held[0] != session_id:
  188. held = (session_id, set())
  189. agent._outbound_retired_image_ids = held
  190. return held[1]
  191. def latched_retire_count(
  192. retire: int, carrier_ids_newest_first: Sequence[Optional[str]], retired_ids: Optional[set[str]]
  193. ) -> int:
  194. """Widen ``retire`` so every carrier up to the newest previously retired one stays retired,
  195. then record the carriers being retired into ``retired_ids``.
  196. The per-request decision cannot see history: a request that drops an upload (once it leaves
  197. the replay window) relaxes the ceiling, and re-deciding from scratch would restore
  198. screenshots an earlier request already retired, rewriting the cached prefix back.
  199. """
  200. if retired_ids is None:
  201. return retire
  202. total = len(carrier_ids_newest_first)
  203. latched = [n for n, cid in enumerate(carrier_ids_newest_first) if cid and cid in retired_ids]
  204. retire = max(retire, total - latched[0]) if latched else retire
  205. retired_ids.update(cid for cid in carrier_ids_newest_first[total - retire:] if cid)
  206. return retire
  207. def _retire_count_for_request(
  208. carrier_blocks_newest_first: Sequence[int],
  209. reserved_blocks: int,
  210. sizes: Optional[Sequence[int]],
  211. reserved_bytes: int,
  212. limit: int,
  213. budget: int,
  214. batch: int,
  215. floor: int,
  216. ) -> int:
  217. """Retire count for one request in isolation; see :func:`outbound_image_retire_count`."""
  218. total = len(carrier_blocks_newest_first)
  219. blocks_kept = [0] * (total + 1)
  220. bytes_kept = [0] * (total + 1)
  221. for i in range(total):
  222. blocks_kept[i + 1] = blocks_kept[i] + carrier_blocks_newest_first[i]
  223. bytes_kept[i + 1] = bytes_kept[i] + (sizes[i] if sizes is not None else 0)
  224. def _bytes_fit(kept: int) -> bool:
  225. return sizes is None or reserved_bytes + bytes_kept[kept] <= budget
  226. def _fits(kept: int) -> bool:
  227. return reserved_blocks + blocks_kept[kept] <= limit and _bytes_fit(kept)
  228. if _fits(total):
  229. return 0
  230. floor = min(max(floor, 0), total)
  231. if not _fits(0) and _bytes_fit(floor):
  232. # Reserved uploads alone breach the block ceiling: no retirement can make the request
  233. # fit it, so shelter the floor but keep advancing in whole quanta. Retiring down to
  234. # exactly ``total - floor`` here moves the frontier on every new image (#133999). The
  235. # quantum is capped by how many carriers the byte budget lets us keep, so byte
  236. # pressure narrows the step instead of collapsing it back to the floor.
  237. byte_window = max(k for k in range(floor, total + 1) if _bytes_fit(k))
  238. quantum = max(1, min(batch, byte_window - floor + 1))
  239. excess = total - floor
  240. return excess - excess % quantum
  241. window = max(k for k in range(total + 1) if _fits(k)) if _fits(0) else 0
  242. quantum = max(1, min(batch, window - floor))
  243. retire = 0
  244. while retire < total:
  245. retire = min(retire + quantum, total)
  246. if _fits(total - retire):
  247. break
  248. return retire