vision_tools_history_budget.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180
  1. """History-reuse budgets for native vision embeds (config section ``vision``).
  2. A native ``vision_analyze`` result bakes the image into conversation history, where it is
  3. re-sent on every later API call. Two knobs bound that cost: ``vision.embed_target_bytes``
  4. (how large one embed may be) and ``vision.max_calls_per_image`` (how often the same image
  5. may be embedded per session). See #112095: a delegated subagent re-loaded five screenshots
  6. 158 times in 15 minutes because nothing refused the repeat.
  7. """
  8. from __future__ import annotations
  9. import contextlib
  10. import hashlib
  11. import json
  12. import os
  13. import re
  14. import threading
  15. from contextvars import ContextVar
  16. from pathlib import Path
  17. from typing import Any, Iterator, Optional
  18. from tools.registry import tool_error
  19. # 256 KB keeps a 1568px screenshot cheap enough to ride the session (#92699); the clamp keeps one
  20. # setting from turning every later request into a multi-megabyte resend or a useless thumbnail.
  21. _DEFAULT_EMBED_TARGET_BYTES = 256 * 1024
  22. _MIN_EMBED_TARGET_BYTES = 64 * 1024
  23. _MAX_EMBED_TARGET_BYTES = 4 * 1024 * 1024
  24. # Delegated subagents run unattended and cannot be steered mid-loop from the CLI, so they get a
  25. # cap by default; the main agent stays unlimited unless the user sets ``vision.max_calls_per_image``.
  26. _SUBAGENT_REPEAT_CAP = 3
  27. _REPEAT_COUNTS_MAX_KEYS = 4096
  28. _repeat_counts: dict[tuple[str, str], int] = {}
  29. _repeat_lock = threading.Lock()
  30. def _cfg_vision(key: str, default=None):
  31. """``vision.<key>`` from config.yaml; ``default`` when config is unavailable."""
  32. try:
  33. from hermes_cli.config import cfg_get, load_config
  34. return cfg_get(load_config(), "vision", key, default=default)
  35. except Exception:
  36. return default
  37. def resolve_embed_target_bytes() -> int:
  38. """``vision.embed_target_bytes`` clamped to 64 KiB..4 MiB; the 256 KB default on a bad value."""
  39. raw = _cfg_vision("embed_target_bytes", default=_DEFAULT_EMBED_TARGET_BYTES)
  40. try:
  41. if isinstance(raw, bool):
  42. raise ValueError("boolean is not a byte budget")
  43. target = int(raw)
  44. except (TypeError, ValueError, OverflowError):
  45. return _DEFAULT_EMBED_TARGET_BYTES
  46. return min(max(target, _MIN_EMBED_TARGET_BYTES), _MAX_EMBED_TARGET_BYTES)
  47. def resolve_repeat_cap() -> int:
  48. """Per-image embed cap for this session; 0 = unlimited.
  49. ``vision.max_calls_per_image`` unset (or unparseable) → ``_SUBAGENT_REPEAT_CAP`` inside a
  50. delegated subagent, unlimited for the main agent. An explicit value applies everywhere.
  51. """
  52. raw = _cfg_vision("max_calls_per_image")
  53. try:
  54. if raw is not None and raw != "" and not isinstance(raw, bool):
  55. return max(int(raw), 0)
  56. except (TypeError, ValueError):
  57. pass
  58. from agent.delegation_context import is_delegated_child_process_context
  59. return _SUBAGENT_REPEAT_CAP if is_delegated_child_process_context() else 0
  60. def _image_key(image_url: str) -> str:
  61. """Stable identity for an image source: local paths resolve (symlinks, ``~``, ``file://``),
  62. URLs drop their fragment, data URLs hash. Region crops are NOT part of the key — the incident
  63. loop alternated full loads and crops of the same files, and both re-embed the image."""
  64. if image_url.startswith("data:"):
  65. return "data:" + hashlib.sha256(image_url.encode("utf-8", "ignore")).hexdigest()[:32]
  66. stripped = image_url.split("#", 1)[0].removeprefix("file://")
  67. if "://" in stripped:
  68. return "url:" + stripped
  69. return "file:" + str(Path(os.path.expanduser(stripped)).resolve())
  70. def _count_key(image_url: str) -> tuple[str, str]:
  71. from gateway.session_context import get_session_env
  72. return get_session_env("HERMES_SESSION_ID", ""), _image_key(image_url)
  73. def repeat_refusal(image_url: str) -> Optional[str]:
  74. """Reserve one native embed of ``image_url`` for the current session; tool-error JSON when the
  75. per-session cap is already spent, else ``None``. Check and count are ONE lock section: a
  76. parallel tool batch on the same image (the incident's 4 concurrent calls) must not all pass a
  77. check taken before any of them recorded. Callers ``release_embed`` when the embed then fails."""
  78. cap = resolve_repeat_cap()
  79. if cap <= 0:
  80. return None
  81. key = _count_key(image_url)
  82. with _repeat_lock:
  83. count = _repeat_counts.get(key, 0)
  84. if count < cap:
  85. _record_embed_locked(key)
  86. return None
  87. return tool_error(
  88. f"vision_analyze refused: this image has already been loaded into context {count} time(s) "
  89. "in this session (region crops of the same file count too), and every native load re-sends "
  90. "the full image on each later API call. Answer from what you can already see, or ask the "
  91. f"user. (vision.max_calls_per_image = {cap}; 0 = unlimited)",
  92. success=False,
  93. )
  94. def _record_embed_locked(key: tuple[str, str]) -> None:
  95. _repeat_counts[key] = _repeat_counts.get(key, 0) + 1
  96. # Bound long-lived gateway memory: evict the oldest (session, image) entries.
  97. while len(_repeat_counts) > _REPEAT_COUNTS_MAX_KEYS:
  98. _repeat_counts.pop(next(iter(_repeat_counts)))
  99. def record_embed(image_url: str) -> None:
  100. """Count one successful native embed of ``image_url`` for the current session (uncapped
  101. sessions only — capped ones are counted by the reservation in :func:`repeat_refusal`)."""
  102. with _repeat_lock:
  103. _record_embed_locked(_count_key(image_url))
  104. def release_embed(image_url: str) -> None:
  105. """Give back a slot reserved by :func:`repeat_refusal` when the embed did not happen."""
  106. key = _count_key(image_url)
  107. with _repeat_lock:
  108. count = _repeat_counts.get(key, 0)
  109. if count > 1:
  110. _repeat_counts[key] = count - 1
  111. else:
  112. _repeat_counts.pop(key, None)
  113. # Images already riding the ACTIVE user turn as native content parts (#76411). Every surface that
  114. # attaches natively (gateway, CLI, TUI, delegated children) goes through
  115. # ``image_routing.build_native_content_parts``, which writes one ``[Image attached at: <path>]`` /
  116. # ``[Image attached: <url>]`` handle per image into the text part; ``conversation_loop.run_conversation``
  117. # scopes those handles here for the turn and the tool inherits them through contextvars.
  118. _NATIVE_HANDLE_RE = re.compile(r"^\[Image attached(?: at)?: (.+?)\]\s*$", re.MULTILINE)
  119. _native_turn_images: ContextVar[frozenset[str]] = ContextVar("vision_native_turn_images", default=frozenset())
  120. def _native_turn_keys(user_message: Any) -> frozenset[str]:
  121. if not isinstance(user_message, list) or not any(
  122. isinstance(p, dict) and p.get("type") == "image_url" for p in user_message
  123. ):
  124. return frozenset()
  125. text = "\n".join(p.get("text", "") for p in user_message if isinstance(p, dict) and p.get("type") == "text")
  126. return frozenset(_image_key(m.strip()) for m in _NATIVE_HANDLE_RE.findall(text) if m.strip())
  127. @contextlib.contextmanager
  128. def native_turn_images(user_message: Any) -> Iterator[None]:
  129. """Scope the images natively attached to ``user_message`` to the running turn."""
  130. token = _native_turn_images.set(_native_turn_keys(user_message))
  131. try:
  132. yield
  133. finally:
  134. _native_turn_images.reset(token)
  135. def native_turn_duplicate(image_url: str, region: Optional[list]) -> Optional[str]:
  136. """Text tool result when ``image_url`` already rides the active user turn natively, else
  137. ``None``. A native re-embed would put the identical pixels into the same request twice
  138. (the Telegram case in #76411); a ``region`` crop still embeds because it returns new detail."""
  139. if region is not None or _image_key(image_url) not in _native_turn_images.get():
  140. return None
  141. return json.dumps({
  142. "success": True,
  143. "already_in_context": True,
  144. "message": (
  145. "This image is already attached natively to the current user message — you can see it "
  146. "now. Answer with your built-in vision; pass a `region` to zoom into part of it."),
  147. })