web_result_cache.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334
  1. """Result caching for web_search / web_extract; both caches TTL-bounded (default 20 min,
  2. ``web.cache_ttl_minutes``; disable with ``web.cache_enabled: false``), only successful responses cache.
  3. * **Search memo** — in-memory, per-process, single-flighted: concurrent identical queries share one
  4. paid request. Limits bucket to 10/20/50/100 so near-identical requests share an entry.
  5. * **Extract cache** — disk-backed under ``cache/web`` (cross-process) with a JSON sidecar index:
  6. URL digest → (file, fetched_at, title). Hits re-run the normal truncate pipeline.
  7. Lives here, not in tool dispatch, so hits sit *after* every safety check and skip only the vendor call.
  8. """
  9. import hashlib
  10. import json
  11. import logging
  12. import re
  13. import threading
  14. import time
  15. from contextlib import suppress
  16. from pathlib import Path
  17. from typing import Dict, Optional, Tuple
  18. from urllib.parse import urlparse
  19. from utils import atomic_json_write
  20. logger = logging.getLogger(__name__)
  21. # Requested limits round UP to a bucket so cache keys collide on purpose.
  22. _LIMIT_BUCKETS = (10, 20, 50, 100)
  23. DEFAULT_TTL_MINUTES = 20
  24. _INDEX_FILENAME = "extract-index.json"
  25. _INDEX_MAX_ENTRIES = 500 # oldest entries evicted past this
  26. def _web_config() -> dict:
  27. try:
  28. from tools.web_tools import _load_web_config
  29. return _load_web_config()
  30. except Exception:
  31. return {}
  32. def cache_enabled() -> bool:
  33. """Both caches honor ``web.cache_enabled`` (default: on)."""
  34. return True if (val := _web_config().get("cache_enabled")) is None else bool(val)
  35. def ttl_seconds() -> float:
  36. """TTL from ``web.cache_ttl_minutes`` (default 20, clamped 1–1440)."""
  37. raw = _web_config().get("cache_ttl_minutes")
  38. try:
  39. minutes = float(raw) if raw is not None else DEFAULT_TTL_MINUTES
  40. except (TypeError, ValueError):
  41. minutes = DEFAULT_TTL_MINUTES
  42. return max(1.0, min(minutes, 1440.0)) * 60.0
  43. def bucket_limit(limit: int) -> int:
  44. """Round a requested result count up to the nearest bucket."""
  45. return next((b for b in _LIMIT_BUCKETS if limit <= b), _LIMIT_BUCKETS[-1])
  46. def normalize_query(query: str) -> str:
  47. """Case-fold and collapse whitespace so trivial variants share an entry."""
  48. return re.sub(r"\s+", " ", (query or "").strip().lower())
  49. def _host_slug(url: str) -> str:
  50. """Filesystem-safe hostname slug for cache filenames (``"page"`` when hostless). Shared with
  51. tools.web_tools_truncate."""
  52. host = (urlparse(url).hostname or "page").replace(":", "_")
  53. return re.sub(r"[^A-Za-z0-9._-]", "-", host)[:60].strip("-") or "page"
  54. def _deep_copy(response: dict) -> dict:
  55. """Defensive copy so callers mutating a hit never corrupt the cached entry."""
  56. return json.loads(json.dumps(response))
  57. # ─── Search memo (in-memory, single-flight) ───────────────────────────────────
  58. class SearchMemo:
  59. """TTL memo + single-flight coalescer for search responses. Thread-safe: the parallel tool-dispatch pool
  60. and subagents share this process, so identical queries genuinely race; per-key locks make the losers
  61. wait for (and share) the winner's response."""
  62. def __init__(self) -> None:
  63. self._store: dict[tuple, tuple[float, dict]] = {} # key -> (expires_at, response)
  64. self._store_lock = threading.Lock()
  65. self._key_locks: dict[tuple, threading.Lock] = {}
  66. @staticmethod
  67. def _key(provider: str, query: str, limit: int) -> tuple:
  68. return (provider, normalize_query(query), bucket_limit(limit))
  69. def lookup(self, provider: str, query: str, limit: int) -> Optional[dict]:
  70. if not cache_enabled():
  71. return None
  72. key = self._key(provider, query, limit)
  73. with self._store_lock:
  74. hit = self._store.get(key)
  75. if hit is None or time.monotonic() >= hit[0]:
  76. self._store.pop(key, None)
  77. return None
  78. logger.info("web_search cache hit: %r via %s", query, provider)
  79. return _deep_copy(hit[1])
  80. def store(self, provider: str, query: str, limit: int, response: dict) -> None:
  81. """Cache a SUCCESSFUL response for the bucketed key."""
  82. if not cache_enabled() or not isinstance(response, dict) or not response.get("success"):
  83. return
  84. key = self._key(provider, query, limit)
  85. with self._store_lock:
  86. now = time.monotonic() # opportunistic expiry sweep bounds memory
  87. for k in [k for k, (exp, _) in self._store.items() if now >= exp]:
  88. del self._store[k]
  89. self._store[key] = (now + ttl_seconds(), _deep_copy(response))
  90. def flight_lock(self, provider: str, query: str, limit: int) -> threading.Lock:
  91. """Per-key lock held around lookup-miss → paid request → store."""
  92. key = self._key(provider, query, limit)
  93. with self._store_lock:
  94. lock = self._key_locks.get(key)
  95. if lock is None:
  96. # Bound the lock table, but never evict a HELD lock: dropping one lets a concurrent
  97. # identical request mint a fresh lock and issue a duplicate paid call. locked() is a
  98. # safe snapshot under _store_lock because holders already have their reference.
  99. # See #94618.
  100. if len(self._key_locks) > 256:
  101. self._key_locks = {k: v for k, v in self._key_locks.items() if v.locked()}
  102. lock = self._key_locks[key] = threading.Lock()
  103. return lock
  104. def clear(self) -> None:
  105. """Drop all cached entries (tests; config changes)."""
  106. with self._store_lock:
  107. self._store.clear()
  108. self._key_locks.clear()
  109. search_memo = SearchMemo()
  110. def slice_search_response(response: dict, limit: int) -> dict:
  111. """Trim a bucketed response's result list down to the caller's limit."""
  112. try:
  113. web = response.get("data", {}).get("web")
  114. if isinstance(web, list) and len(web) > limit:
  115. out = _deep_copy(response)
  116. out["data"]["web"] = out["data"]["web"][:limit]
  117. return out
  118. except Exception:
  119. pass
  120. return response
  121. # ─── Extract cache (disk-backed, reuses cache/web) ────────────────────────────
  122. _index_lock = threading.Lock()
  123. def _cache_dir() -> Optional[Path]:
  124. try:
  125. from hermes_constants import get_hermes_dir
  126. d = get_hermes_dir("cache/web", "web_cache")
  127. d.mkdir(parents=True, exist_ok=True)
  128. return d
  129. except Exception:
  130. return None
  131. def _load_index() -> dict:
  132. try:
  133. data = json.loads((_cache_dir() / _INDEX_FILENAME).read_text(encoding="utf-8-sig"))
  134. return data if isinstance(data, dict) else {}
  135. except Exception:
  136. return {}
  137. def _save_index(index: dict) -> None:
  138. if (d := _cache_dir()) is None:
  139. return
  140. path = d / _INDEX_FILENAME
  141. try:
  142. if len(index) > _INDEX_MAX_ENTRIES:
  143. newest = sorted(index.items(), key=lambda kv: kv[1].get("fetched_at", 0), reverse=True)
  144. index = dict(newest[:_INDEX_MAX_ENTRIES])
  145. # CLI, gateway, cron, and subagents all write this index; the replace is atomic, so the worst case
  146. # under concurrent writers is a lost insert, never a truncated index.
  147. atomic_json_write(path, index, indent=None)
  148. except Exception as exc:
  149. logger.debug("Failed to save web extract cache index: %s", exc)
  150. def _url_digest(url: str, format: Optional[str], provider: str = "") -> str:
  151. # format AND provider are part of the key: html != markdown, and one backend's rendering is not another's.
  152. raw = f"{url}\n{format or 'markdown'}\n{provider or ''}"
  153. return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16]
  154. def _entry_file_path(url: str, format: Optional[str], provider: str) -> Optional[Path]:
  155. """Dedicated cache file per (url, format, provider) — deliberately NOT the truncate-store file
  156. (keyed on URL alone), which html/markdown or two providers' copies of one URL would overwrite.
  157. The truncate-store file keeps its role for read_file paging; these files exist only for cache reuse and
  158. carry the full key in their name. See #94618.
  159. """
  160. if (d := _cache_dir()) is None:
  161. return None
  162. slug = "page"
  163. with suppress(Exception):
  164. slug = _host_slug(url)
  165. return d / f"{slug}-{_url_digest(url, format, provider)}.cache.md"
  166. def _host_matches_pattern(host: str, pattern: str) -> bool:
  167. """Case-insensitive: exact, ``*.wildcard``, or bare-domain suffix
  168. (``mysite.dev`` also matches ``preview.mysite.dev``)."""
  169. host = host.lower().strip(".")
  170. pattern = (pattern or "").lower().strip().strip(".").removeprefix("*.")
  171. return bool(pattern) and (host == pattern or host.endswith("." + pattern))
  172. def _is_cache_exempt_host(url: str) -> bool:
  173. """True when the host matches ``web.cache_exempt_hosts`` — sites the user develops over public DNS
  174. (staging, tunnels, previews) that must fetch live."""
  175. try:
  176. patterns = _web_config().get("cache_exempt_hosts") or []
  177. host = (urlparse(url).hostname or "").strip("[]")
  178. if not isinstance(patterns, (list, tuple)) or not host:
  179. return False
  180. return any(_host_matches_pattern(host, str(p)) for p in patterns)
  181. except Exception:
  182. return False
  183. def _is_local_dev_url(url: str) -> bool:
  184. """True for loopback/private/LAN URLs — never cached: they are the user's own fast-changing dev servers.
  185. Hostname heuristics only, no DNS: this is a freshness decision, not a security boundary (SSRF enforcement
  186. lives in tools/url_safety.py, which blocks these by default anyway)."""
  187. try:
  188. host = (urlparse(url).hostname or "").strip("[]").lower()
  189. # Unparseable → don't cache; single-label (no "." / ":") == LAN name, not public DNS.
  190. if not host or host == "localhost" or host.endswith((".localhost", ".local")):
  191. return True
  192. if "." not in host and ":" not in host:
  193. return True
  194. import ipaddress
  195. try:
  196. ip = ipaddress.ip_address(host)
  197. except ValueError:
  198. return False # public DNS name
  199. return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_unspecified
  200. except Exception:
  201. return True
  202. def _cacheable(url: str) -> bool:
  203. """Extract-cache gate: enabled, not a local-dev host, not user-exempted."""
  204. return cache_enabled() and not (_is_local_dev_url(url) or _is_cache_exempt_host(url))
  205. def extract_cache_get(url: str, format: Optional[str] = None, provider: str = "") -> Optional[dict]:
  206. """Return {'url','title','content'} for a fresh cached page, else None."""
  207. if not _cacheable(url):
  208. return None
  209. with _index_lock:
  210. entry = _load_index().get(_url_digest(url, format, provider))
  211. if not entry or (time.time() - float(entry.get("fetched_at", 0))) >= ttl_seconds():
  212. return None
  213. try:
  214. file_path, cache_root = Path(entry["file"]), _cache_dir()
  215. # The index is plain JSON on disk; never let a tampered entry read outside cache/web.
  216. if cache_root.resolve() not in file_path.resolve().parents:
  217. return None
  218. content = file_path.read_text(encoding="utf-8-sig")
  219. except Exception:
  220. return None
  221. # The body file and its index entry are separate writes; serve only the body the entry (and its
  222. # ``final_url``) was recorded for. Entries from before the digest carry no ``final_url`` either.
  223. if ("content_sha256" in entry or "final_url" in entry) and entry.get("content_sha256") != _content_sha(content):
  224. return None
  225. logger.info("web_extract cache hit: %s", url)
  226. return {"url": url, "title": entry.get("title", ""), "content": content, "error": None, "cached": True,
  227. "final_url": entry.get("final_url") or url}
  228. def _content_sha(content: str) -> str:
  229. # Hash the text as ``extract_cache_get`` reads it back: text-mode reads translate CRLF/CR to LF
  230. # and ``utf-8-sig`` drops a leading BOM, so a raw-text digest would never match those pages.
  231. normalized = content.removeprefix("\ufeff").replace("\r\n", "\n").replace("\r", "\n")
  232. return hashlib.sha256(normalized.encode("utf-8")).hexdigest()
  233. def extract_cache_record_final_url(
  234. url: str, final_url: str, content: str, format: Optional[str] = None, provider: str = ""
  235. ) -> None:
  236. """Remember the post-redirect URL behind a cached page, so a hit can re-run the website policy and
  237. SSRF gates against the page it actually serves (a blocklist added after caching must still bite).
  238. Compare-and-set on *content*: when another fetch of the same key wrote its body in between (or this
  239. write was skipped), the entry is left alone rather than pairing that body with this final URL."""
  240. with _index_lock:
  241. index = _load_index()
  242. entry = index.get(_url_digest(url, format, provider))
  243. if entry is None or entry.get("content_sha256") != _content_sha(content):
  244. return
  245. entry["final_url"] = final_url
  246. _save_index(index)
  247. def extract_cache_put(
  248. url: str, content: str, title: str = "", format: Optional[str] = None, provider: str = ""
  249. ) -> bool:
  250. """Store one successful extraction's full clean text for TTL reuse; pages over the truncate-store
  251. ceiling are not cached (serving a capped copy back as if whole would silently lose the tail).
  252. True only when this call wrote the entry."""
  253. if not content or not _cacheable(url):
  254. return False
  255. try:
  256. from tools.web_tools_truncate import MAX_STORED_TEXT_CHARS
  257. file_path = _entry_file_path(url, format, provider)
  258. if len(content) > MAX_STORED_TEXT_CHARS or file_path is None:
  259. return False
  260. from tools.spill_safety import write_text_exclusive
  261. write_text_exclusive(file_path, content, private=False, overwrite=True)
  262. with _index_lock:
  263. index = _load_index()
  264. index[_url_digest(url, format, provider)] = {
  265. "url": url, "file": str(file_path), "title": title or "", "fetched_at": time.time(),
  266. "content_sha256": _content_sha(content),
  267. }
  268. _save_index(index)
  269. return True
  270. except Exception as exc:
  271. logger.debug("Failed to cache web extract for %s: %s", url, exc)
  272. return False