web_tools_truncate.py 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198
  1. """Truncate-and-store pipeline for web_extract (no LLM).
  2. Pages at or under the char budget are returned whole; larger pages become a head+tail window plus a
  3. footer that says how much is shown, where the full text is stored (cache/web) and the exact read_file
  4. call that pages the omitted middle. Inline base64 images become ``[IMAGE: alt]`` placeholders. Logs under the
  5. origin (tools.web_tools) logger.
  6. """
  7. import logging
  8. import re
  9. from typing import Any, List, Optional
  10. logger = logging.getLogger("tools.web_tools")
  11. # Per-page char budget sent to the model (override: web.extract_char_limit); larger pages are head+tail
  12. # truncated, full text stored on disk.
  13. # Per-vendor client construction, request helpers and response normalizers live in
  14. # plugins.web.<vendor>.provider (parallel / tavily / firecrawl) since PR #25182.
  15. DEFAULT_EXTRACT_CHAR_LIMIT = 15000
  16. # Ceiling on the full-text file written to cache/web so a multi-MB page can't write unbounded bytes on
  17. # every extract; the model only ever sees char_limit.
  18. MAX_STORED_TEXT_CHARS = 2_000_000
  19. _CHAR_LIMIT_FLOOR, _CHAR_LIMIT_CEILING = 2000, 500_000
  20. def _clamp_char_limit(value: Any) -> int:
  21. """Clamp to [2k, 500k] (below 2k the truncation footer dominates; a config typo must not blow up context);
  22. raises TypeError/ValueError for non-numeric input."""
  23. return max(_CHAR_LIMIT_FLOOR, min(int(value), _CHAR_LIMIT_CEILING))
  24. def _clamp_or_default(value: Any) -> int:
  25. """``_clamp_char_limit(value)``; ``None`` or non-numeric input falls back to the default."""
  26. try:
  27. return DEFAULT_EXTRACT_CHAR_LIMIT if value is None else _clamp_char_limit(value)
  28. except (TypeError, ValueError):
  29. return DEFAULT_EXTRACT_CHAR_LIMIT
  30. def _get_extract_char_limit() -> int:
  31. """``web.extract_char_limit`` clamped to a sane range, else the default."""
  32. from tools.web_tools import _load_web_config # lazy: tests patch tools.web_tools._load_web_config
  33. return _clamp_or_default(_load_web_config().get("extract_char_limit"))
  34. def convert_base64_images_to_links(text: str) -> str:
  35. """Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders: markdown images
  36. (alt kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads. Real http(s) markdown
  37. image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them."""
  38. def _md_repl(m: re.Match[str]) -> str:
  39. return f"[IMAGE: {alt}]" if (alt := (m.group("alt") or "").strip()) else "[IMAGE]"
  40. out = re.sub(r"!\[(?P<alt>[^\]]*)\]\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)", _md_repl, text)
  41. out = re.sub(r"\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)", "[IMAGE]", out)
  42. return re.sub(r"data:image/[^;]+;base64,[A-Za-z0-9+/=]+", "[IMAGE]", out)
  43. def _store_full_text(url: str, content: str) -> Optional[str]:
  44. """Write the full page to cache/web; absolute path or None (best-effort: the truncated content is still
  45. returned). cache/web is mounted read-only into remote backends (credential_files _CACHE_DIRS) so
  46. read_file can page the complete text on any backend."""
  47. try:
  48. import hashlib
  49. from hermes_constants import get_hermes_dir
  50. from tools.web_result_cache import _host_slug
  51. cache_dir = get_hermes_dir("cache/web", "web_cache")
  52. cache_dir.mkdir(parents=True, exist_ok=True)
  53. path = cache_dir / f"{_host_slug(url)}-{hashlib.sha256(url.encode('utf-8')).hexdigest()[:10]}.md"
  54. if len(content) > MAX_STORED_TEXT_CHARS:
  55. content = content[:MAX_STORED_TEXT_CHARS] + (
  56. f"\n\n[... stored copy truncated at {MAX_STORED_TEXT_CHARS:,} chars "
  57. f"of {len(content):,}; re-extract a more specific URL for the rest ...]"
  58. )
  59. from tools.spill_safety import write_text_exclusive
  60. # Deterministic name in a well-known dir: refuse symlinks (lstat-unlink + exclusive create);
  61. # same-URL re-extraction legitimately overwrites. Not private: cache/web is bind-mounted
  62. # into remote backends' container UID.
  63. write_text_exclusive(path, content, private=False, overwrite=True)
  64. return str(path)
  65. except Exception as exc:
  66. logger.debug("Failed to store full web_extract text for %s: %s", url, exc)
  67. return None
  68. def _truncate_with_footer(content: str, url: str, char_limit: int) -> tuple[str, bool]:
  69. """Return (model_text, was_truncated). Pages over ``char_limit`` become a ~75% head / ~25% tail window cut
  70. on line boundaries, plus a footer saying how much is shown, where the full text is stored, and the
  71. read_file call that pages the omitted middle. Deterministic."""
  72. if len(content) <= char_limit:
  73. return content, False
  74. head_budget = int(char_limit * 0.75)
  75. tail_budget = char_limit - head_budget
  76. head, tail = content[:head_budget], content[-tail_budget:]
  77. # Snap both cuts to line boundaries (head back, tail forward) so we never slice mid-line.
  78. if (nl := head.rfind("\n")) > head_budget * 0.5:
  79. head = head[:nl]
  80. if 0 <= (nl := tail.find("\n")) < tail_budget * 0.5:
  81. tail = tail[nl + 1:]
  82. stored_path = _store_full_text(url, content)
  83. if stored_path:
  84. # The footer is read by the AGENT, whose read_file runs inside the active backend: render the
  85. # path where docker/modal/ssh/... see the mounted cache, not the host path (#72389, #81984).
  86. from tools.credential_files import to_agent_visible_cache_path
  87. stored_path = to_agent_visible_cache_path(stored_path)
  88. footer_lines = [
  89. "", "─" * 8 + " [TRUNCATED] " + "─" * 8,
  90. f"Showing {len(head):,} chars (head) + {len(tail):,} chars (tail) "
  91. f"of {len(content):,} total clean characters.",
  92. ]
  93. if stored_path:
  94. # read_file is 1-indexed; +2 lands on the first line after the shown head.
  95. middle_start_line = head.count("\n") + 2
  96. footer_lines += [
  97. f"Full text saved to: {stored_path}",
  98. f'To read the omitted middle: read_file path="{stored_path}" '
  99. f"offset={middle_start_line} limit=200 (the file is the complete page; "
  100. f"raise/lower offset to page through it).",
  101. ]
  102. else:
  103. footer_lines.append(
  104. "Full text could not be stored; re-run web_extract on a more "
  105. "specific URL or open it in the browser for the complete page."
  106. )
  107. footer_lines.append("─" * 29)
  108. model_text = head + "\n\n[... middle omitted — see footer ...]\n\n" + tail
  109. return model_text + "\n" + "\n".join(footer_lines), True
  110. def _effective_char_limit(char_limit: Optional[int]) -> int:
  111. """Caller's ``char_limit`` (else config) clamped; non-numeric input falls back to the default."""
  112. return _clamp_or_default(char_limit) if char_limit is not None else _get_extract_char_limit()
  113. _UNAMBIGUOUS_BINARY_KINDS = ("SQLite", "ZIP", "gzip", "bzip2", "xz", "7-Zip", "ELF", "Mach-O", "PNG", "JPEG", "GIF", "TIFF", "FLAC", "Ogg")
  114. def _binary_payload_kind(text: str) -> str:
  115. """Magic-byte type name when a fetched body is a raw binary file, else ``""``. Backends return
  116. the body as text with NUL bytes dropped, so signatures are compared NUL-stripped on both sides.
  117. Only the file tools' own signature table; HTML/markdown/JSON never start with one."""
  118. from tools.file_operations import _MAGIC_SIGNATURES
  119. head = text[:32].encode("latin-1", "ignore").replace(b"\x00", b"")
  120. for prefix, name in _MAGIC_SIGNATURES:
  121. sig = prefix.replace(b"\x00", b"")
  122. if sig and name.startswith(_UNAMBIGUOUS_BINARY_KINDS) and head.startswith(sig):
  123. return name
  124. return ""
  125. def _truncate_results(results: list[dict], char_limit: int, debug_call_data: dict) -> None:
  126. """In place: replace each successful entry's content with its base64-cleaned, budgeted text;
  127. per-page truncation metrics go into ``debug_call_data``."""
  128. for result in results:
  129. url = result.get("url", "")
  130. raw_content = result.get("raw_content", "") or result.get("content", "")
  131. if result.get("error"):
  132. # An origin error page keeps its body (a 404 page can help) but stays in the budget.
  133. if body := result.get("content"):
  134. result["content"], _ = _truncate_with_footer(convert_base64_images_to_links(body), url, char_limit)
  135. continue
  136. if not raw_content:
  137. continue
  138. binary_kind = _binary_payload_kind(raw_content)
  139. if binary_kind:
  140. # A backend that fetched a raw file (SQLite, archive, executable) hands back its bytes as
  141. # "text"; 800K chars of that would enter context. Name the type and point at the tool that reads it.
  142. result["content"] = ""
  143. result["error"] = (
  144. f"URL returned binary content ({binary_kind}), not a page. Download it with the terminal "
  145. "(curl -L -o) and use read_file (SQLite/Office/PDF auto-extract) or terminal utilities on the file.")
  146. logger.info("%s (binary payload: %s, %d chars dropped)", url, binary_kind, len(raw_content))
  147. continue
  148. clean = convert_base64_images_to_links(raw_content)
  149. model_text, truncated = _truncate_with_footer(clean, url, char_limit)
  150. result["content"] = model_text
  151. if truncated:
  152. debug_call_data["pages_truncated"] += 1
  153. debug_call_data["truncation_metrics"].append(
  154. {"url": url, "original_size": len(clean), "sent_size": len(model_text)}
  155. )
  156. logger.info("%s (truncated %d -> %d chars)", url, len(clean), len(model_text))
  157. else:
  158. logger.info("%s (%d chars, whole)", url, len(clean))
  159. def _trim_results(results: list[dict]) -> list[dict]:
  160. """Keep only url/title/content/error per entry (+ blocked_by_policy when present)."""
  161. return [
  162. {
  163. "url": r.get("url", ""), "title": r.get("title", ""), "content": r.get("content", ""),
  164. "error": r.get("error"),
  165. **({"blocked_by_policy": r["blocked_by_policy"]} if "blocked_by_policy" in r else {}),
  166. }
  167. for r in results
  168. ]