image_replay.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172
  1. """Re-attach user images on turns whose history was rebuilt from the session DB.
  2. The DB stores user rows as text (``session_persistence._durable_content``): an image part becomes a
  3. ``[screenshot]`` line at its position (``[screenshot detail=low]`` when the caller chose a detail).
  4. A turn rebuilt from the DB (every API-server request, ``--resume``, a gateway restart) therefore
  5. showed the model that placeholder instead of the photo the user asked about one message earlier.
  6. The bytes are not gone: the session-authority attach paths (CLI, the shared-gateway TUI, messaging
  7. gateway, API server) commit them under the profile's content-addressed ``native-inputs/<sha256>/<name>``
  8. and name that path in the same row (``[Image attached at: <path>]``,
  9. written by ``image_routing.build_native_content_parts`` / ``session_api_media.restore_api_images``).
  10. On the WIRE COPY only, when this turn sends images natively, the ``vision.replay_recent_images``
  11. newest images (default 3, 0 = off) go back as ``image_url`` parts; older ones drop in whole
  12. batches of that size (``replay_frontier``), so the cached prefix is rewritten at most once per
  13. batch rather than on every new photo. The data URL comes from the same
  14. ``_file_to_data_url`` the live turn used, so the rebuilt row is byte-identical to the request that
  15. first sent it and the prompt-cache prefix holds. Only retained bytes under this profile's
  16. ``native-inputs`` whose digest still matches are replayed; remote URLs (they can expire and 400
  17. every later turn), missing files (the gateway's settled-media sweep collects them after
  18. ``MEDIA_CACHE_MAX_AGE_HOURS``) and anything outside the window keep their ``[screenshot]`` text.
  19. """
  20. from __future__ import annotations
  21. import hashlib
  22. import logging
  23. import re
  24. from pathlib import Path
  25. from typing import Any, Optional
  26. logger = logging.getLogger(__name__)
  27. DEFAULT_REPLAY_RECENT_IMAGES = 3
  28. _HINT_RE = re.compile(r"^\[Image attached(?P<local> at)?: (?P<ref>.+)\]$")
  29. # ``session_persistence._image_marker``: the live part's ``detail`` rides on its marker.
  30. _MARKER_RE = re.compile(r"^\[screenshot(?: detail=(?P<detail>low|high|auto))?\]$")
  31. _IMAGE_PART_TYPES = {"image", "image_url", "input_image"}
  32. def resolve_replay_recent_images(cfg: dict) -> int:
  33. """``vision.replay_recent_images`` as an int >= 0; an unusable value warns and disables replay."""
  34. vision = (cfg or {}).get("vision")
  35. raw = (vision if isinstance(vision, dict) else {}).get("replay_recent_images", DEFAULT_REPLAY_RECENT_IMAGES)
  36. if raw is None:
  37. return DEFAULT_REPLAY_RECENT_IMAGES
  38. if isinstance(raw, int) and not isinstance(raw, bool) and raw >= 0:
  39. return raw
  40. logger.warning("vision.replay_recent_images=%r is not a non-negative integer (0 = off, default %d); "
  41. "not replaying earlier images", raw, DEFAULT_REPLAY_RECENT_IMAGES)
  42. return 0
  43. def _split_rebuilt_row(content: str) -> Optional[list[tuple[str, Any]]]:
  44. """A DB-rebuilt user row as its original parts, in order: ``("text", str)`` for each text run and
  45. ``("image", (ref, detail, marker_line))`` per ``[screenshot]`` line, the markers pairing 1:1 in order
  46. with the row's ``[Image attached…]`` hints wherever the image sat (API callers put it first;
  47. turn-time memory/hook context follows it). ``None`` when the counts disagree (a typed marker)."""
  48. lines = content.split("\n")
  49. markers = [(i, m.group("detail")) for i, line in enumerate(lines) if (m := _MARKER_RE.match(line))]
  50. hints = [m for line in lines if (m := _HINT_RE.match(line))]
  51. if not markers or len(hints) != len(markers):
  52. return None
  53. parts: list[tuple[str, Any]] = []
  54. start = 0
  55. for (i, detail), hint in zip(markers, hints):
  56. if i > start:
  57. parts.append(("text", "\n".join(lines[start:i])))
  58. parts.append(("image", (hint.group("ref") if hint.group("local") else "", detail, lines[i])))
  59. start = i + 1
  60. if start < len(lines):
  61. parts.append(("text", "\n".join(lines[start:])))
  62. return parts
  63. def _retained_data_url(ref: str) -> Optional[str]:
  64. """Data URL for a retained local image this profile still holds unchanged, else ``None``."""
  65. if not ref:
  66. return None
  67. from agent.image_routing import _file_to_data_url
  68. from gateway.session_ingress_media import native_inputs_root
  69. path = Path(ref)
  70. try:
  71. if path.parent.parent != native_inputs_root() or path.is_symlink() or not path.is_file():
  72. return None
  73. if hashlib.sha256(path.read_bytes()).hexdigest() != path.parent.name:
  74. return None
  75. except OSError:
  76. return None
  77. return _file_to_data_url(path)
  78. def replay_frontier(total: int, keep: int) -> int:
  79. """Oldest image slot still replayed when the request holds ``total`` images.
  80. The frontier moves forward only in whole batches of ``keep``: the newest ``keep`` to
  81. ``2 * keep - 1`` images replay, and an image that drops out stays dropped as the session
  82. grows. A strict newest-``keep`` window would turn one early row back into ``[screenshot]``
  83. text on every new photo, rewriting the cached prefix each time; this rewrites it at most once
  84. per ``keep`` new images. Counted by slot position (missing or changed files hold their slot),
  85. so a swept file never moves the frontier."""
  86. return (total - keep) // keep * keep if total > keep else 0
  87. def replay_recent_images(api_messages: list[dict[str, Any]], keep: int) -> int:
  88. """Rebuild retained images at or after ``replay_frontier`` in place on ``api_messages`` (wire
  89. copies); images still live in the conversation hold slots too. Returns the number re-attached."""
  90. slots = 0
  91. rows: list[tuple[dict[str, Any], list[tuple[str, Any]], int]] = []
  92. for msg in api_messages:
  93. if msg.get("role") != "user":
  94. continue
  95. content = msg.get("content")
  96. if isinstance(content, list):
  97. slots += sum(1 for p in content if isinstance(p, dict) and p.get("type") in _IMAGE_PART_TYPES)
  98. continue
  99. parts = _split_rebuilt_row(content) if isinstance(content, str) else None
  100. if parts is None:
  101. continue
  102. rows.append((msg, parts, slots))
  103. slots += sum(1 for kind, _ in parts if kind == "image")
  104. frontier = replay_frontier(slots, keep)
  105. attached = 0
  106. for msg, parts, slot in rows:
  107. rebuilt: list[dict[str, Any]] = []
  108. urls = 0
  109. for kind, value in parts:
  110. if kind == "text":
  111. rebuilt.append({"type": "text", "text": value})
  112. continue
  113. url = _retained_data_url(value[0]) if slot >= frontier else None
  114. slot += 1
  115. if url:
  116. rebuilt.append({"type": "image_url", "image_url": {"url": url, **({"detail": value[1]} if value[1] else {})}})
  117. urls += 1
  118. else:
  119. rebuilt.append({"type": "text", "text": value[2]})
  120. if urls:
  121. msg["content"] = rebuilt
  122. attached += urls
  123. return attached
  124. def _turn_replay_window(agent: Any) -> int:
  125. """Images to replay this turn: 0 unless the model takes images natively. Resolved once per turn
  126. (keyed on the turn's frozen admission clock and the model), not per tool-loop request."""
  127. provider = (getattr(agent, "provider", "") or "").strip()
  128. model = (getattr(agent, "model", "") or "").strip()
  129. key = (getattr(agent, "_current_turn_timestamp", None), provider, model)
  130. cached = getattr(agent, "_image_replay_window", None)
  131. if cached is not None and cached[0] == key:
  132. return cached[1]
  133. from agent.image_routing import decide_image_input_mode
  134. from hermes_cli.config import load_config_readonly
  135. cfg = load_config_readonly()
  136. keep = resolve_replay_recent_images(cfg)
  137. if keep > 0 and decide_image_input_mode(provider, model, cfg) != "native":
  138. keep = 0
  139. agent._image_replay_window = (key, keep)
  140. return keep
  141. def replay_for_agent(agent: Any, api_messages: list[dict[str, Any]]) -> None:
  142. """``build_api_messages`` hook: replay only when this turn's model takes images natively."""
  143. if not any(m.get("role") == "user" and isinstance(m.get("content"), str)
  144. and "[screenshot" in m["content"] for m in api_messages):
  145. return
  146. if (keep := _turn_replay_window(agent)) <= 0:
  147. return
  148. if attached := replay_recent_images(api_messages, keep):
  149. logger.info("image_replay: re-attached %d earlier image(s) to the request", attached)