web_tools_extract.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. """web_extract helpers: URL validation, provider resolution, cache-aware dispatch.
  2. Order of controls (each is a gate, never skipped by a cache hit): secret-URL
  3. refusal -> SSRF filter (in web_tools.web_extract_tool) -> provider resolution
  4. (strict selection) -> per-URL website policy -> disk cache -> vendor call with
  5. one-shot keyless rescue -> website policy + SSRF on every page reported under a
  6. URL that was not requested (redirect targets). Logs under the origin
  7. (tools.web_tools) logger.
  8. """
  9. import asyncio
  10. import json
  11. import logging
  12. from typing import Any, Dict, List, Optional
  13. from tools.tool_backend_helpers import selection_error, selection_exists
  14. from tools.url_safety import normalize_url_for_request
  15. from tools.web_tools_rescue import _rescue_eligible, _rescue_extract
  16. logger = logging.getLogger("tools.web_tools")
  17. _NO_RESULT_ERROR = "Extract backend returned no result for this URL"
  18. _DEFAULT_EXTRACT_TIMEOUT_S = 120.0
  19. _EXTRACT_BACKENDS_HINT = "firecrawl, tavily, keenable, exa, or parallel."
  20. _INVALID_ITEM_ERROR = (
  21. "Invalid URL item at index {}: expected a URL string or an object with a string 'url' or 'href' field"
  22. )
  23. def _web_extract_url(value: Any) -> Optional[str]:
  24. """URL from a model-supplied extract item (str, or dict with ``url``/``href``); None if unusable.
  25. Models sometimes forward a whole search result instead of its URL, hence the dict form. Never
  26. stringify arbitrary objects into misleading fetch targets.
  27. """
  28. if isinstance(value, dict):
  29. value = value.get("url") or value.get("href")
  30. return (value.strip() or None) if isinstance(value, str) else None
  31. def _disabled_plugin_error(capability: str, disabled_key: str) -> str:
  32. """Error text when the configured backend's bundled plugin is disabled in config."""
  33. vendor = disabled_key.split("/", 1)[-1]
  34. return (
  35. f"web.{capability}_backend is set to '{vendor}', but its plugin ('{disabled_key}') is disabled "
  36. f"in config. Re-enable it with `hermes plugins enable {disabled_key}` "
  37. "(or remove it from plugins.disabled)."
  38. )
  39. def _no_provider_error(capability: str, fallback: str) -> str:
  40. """Error when no provider resolved: point at a disabled bundled plugin if that is the real cause."""
  41. from agent.web_search_registry import _disabled_web_plugin_for
  42. disabled_key = _disabled_web_plugin_for(capability=capability)
  43. return _disabled_plugin_error(capability, disabled_key) if disabled_key else fallback
  44. def _strict_selection_error(capability: str, backend: str) -> str:
  45. """Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection.
  46. Strict selection never silently switches to whatever the availability walk finds."""
  47. failure = f"no registered web {capability} provider has that name"
  48. return _no_provider_error(capability, selection_error("web", f"'{backend}'", failure))
  49. def _result_entry(url: str, error: Optional[str]) -> dict[str, Any]:
  50. return {"url": url, "title": "", "content": "", "error": error}
  51. def _extract_error_json(error: str) -> str:
  52. return json.dumps({"success": False, "error": error}, ensure_ascii=False)
  53. def _refuse_all(error: str):
  54. """Whole-call refusal tuple for ``_validate_extract_urls`` (exfiltration prevention)."""
  55. return None, None, None, json.dumps({"success": False, "error": error})
  56. def _merge_in_order(
  57. total: int, fixed: dict[int, dict], fetch_positions: list[int], fetch_urls: list[str], results: list[dict]
  58. ) -> list[dict]:
  59. """Rebuild a ``total``-long result list: *fixed* entries by position, fetched *results* at
  60. *fetch_positions* (a short provider list yields ``_NO_RESULT_ERROR`` entries for the rest)."""
  61. merged = dict(fixed)
  62. for pos, position in enumerate(fetch_positions):
  63. missing = _result_entry(fetch_urls[pos], _NO_RESULT_ERROR)
  64. merged[position] = results[pos] if pos < len(results) else missing
  65. return [merged[i] for i in range(total)]
  66. def _validate_extract_urls(urls: list[Any]):
  67. """Normalize model-supplied items and block URLs carrying secrets (percent-encoded forms are unquoted
  68. and checked too). Returns ``(normalized_urls, normalized_indices, invalid_urls, blocked_json)``;
  69. ``blocked_json`` is a whole-call refusal (exfiltration prevention) or None."""
  70. from agent.redact import _PREFIX_RE
  71. from urllib.parse import unquote
  72. normalized_urls, normalized_indices, invalid_urls = [], [], {}
  73. for index, item in enumerate(urls):
  74. _url = _web_extract_url(item)
  75. if _url is None:
  76. invalid_urls[index] = _result_entry("", _INVALID_ITEM_ERROR.format(index))
  77. continue
  78. normalized_url = normalize_url_for_request(_url)
  79. if any(_PREFIX_RE.search(c) for c in (_url, unquote(_url), normalized_url, unquote(normalized_url))):
  80. return _refuse_all(
  81. "Blocked: URL contains what appears to be an API key or token. "
  82. "Secrets must not be sent in URLs."
  83. )
  84. normalized_urls.append(normalized_url)
  85. normalized_indices.append(index)
  86. return normalized_urls, normalized_indices, invalid_urls, None
  87. def _resolve_extract_provider(backend: str):
  88. """Resolve the extract provider for *backend*; returns ``(provider, error_json)``.
  89. A registered search-only backend is a typed error (never a silent switch). An unregistered name with
  90. a stored web selection is a strict-selection error; with no selection, fall through to the walk.
  91. """
  92. from agent.web_search_registry import get_active_extract_provider, get_provider as _wsp_get_provider
  93. provider = _wsp_get_provider(backend) if backend else None
  94. if provider is not None and provider.supports_extract():
  95. return provider, None
  96. if provider is not None:
  97. return None, _extract_error_json(
  98. f"{provider.display_name} is a search-only backend and cannot extract URL content. "
  99. "Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT
  100. )
  101. if backend and selection_exists("web"):
  102. return None, _extract_error_json(_strict_selection_error("extract", backend))
  103. provider = get_active_extract_provider()
  104. if provider is None:
  105. fallback = "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT
  106. return None, _extract_error_json(_no_provider_error("extract", fallback))
  107. return provider, None
  108. def _extract_timeout_seconds() -> float:
  109. """Wall-clock cap for one provider ``extract()`` dispatch (``web.extract_timeout``, default 120s).
  110. A hanging backend (server keeps the response open without finishing) otherwise stalls the
  111. tool call indefinitely. 0 or a negative value disables the cap.
  112. """
  113. from tools.web_tools import _load_web_config
  114. try:
  115. return float(_load_web_config().get("extract_timeout", _DEFAULT_EXTRACT_TIMEOUT_S))
  116. except (TypeError, ValueError):
  117. return _DEFAULT_EXTRACT_TIMEOUT_S
  118. def _backend_failed_whole_batch(results: list[dict]) -> bool:
  119. from plugins.web._common import is_origin_http_error
  120. errors = [r.get("error") for r in results]
  121. return bool(errors) and all(errors) and not any(is_origin_http_error(e) for e in errors)
  122. async def _fetch_with_rescue(provider, fetch_urls: list[str], format: Optional[str]) -> tuple[list[dict], bool]:
  123. """Call ``provider.extract`` (async or sync-in-thread), with one-shot keyless rescue.
  124. Rescue fires on a raised exception — including a dispatch timeout — or when the WHOLE batch
  125. failed (backend outage, not per-page problems): an origin HTTP 4xx/5xx is the page's own answer,
  126. so a batch with one never triggers it. Returns ``(results, rescued)``.
  127. """
  128. import inspect
  129. timeout = _extract_timeout_seconds()
  130. try:
  131. if inspect.iscoroutinefunction(provider.extract):
  132. coro = provider.extract(fetch_urls, format=format)
  133. else: # sync extract() runs in a thread so network I/O never blocks the loop
  134. coro = asyncio.to_thread(provider.extract, fetch_urls, format=format)
  135. if timeout > 0:
  136. results = await asyncio.wait_for(coro, timeout=timeout)
  137. else:
  138. results = await coro
  139. except TimeoutError: # hanging backend — bounded, never a stalled tool call
  140. logger.warning("web_extract provider '%s' timed out after %.0fs for %d URL(s)",
  141. provider.name, timeout, len(fetch_urls))
  142. failed = [_result_entry(u, f"Extract timed out after {timeout:.0f}s via {provider.name}")
  143. for u in fetch_urls]
  144. if not _rescue_eligible(provider):
  145. return failed, False
  146. return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, failed), True
  147. except Exception as exc:
  148. if not _rescue_eligible(provider):
  149. raise
  150. failed = [_result_entry(u, str(exc)) for u in fetch_urls]
  151. return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, failed), True
  152. if _backend_failed_whole_batch(results) and _rescue_eligible(provider):
  153. return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, results), True
  154. return results, False
  155. async def _final_url_refusal(fetched: dict) -> Optional[dict]:
  156. """Refusal entry when a page's reported (post-redirect) URL fails the website policy or SSRF gate,
  157. in the shape Firecrawl's own redirect re-check returns; None when it passes. A policy check that
  158. raises refuses (fail closed): the requested URL passed, but nothing vouches for this one."""
  159. from tools.url_safety import async_is_safe_url
  160. from tools.website_policy import check_website_access
  161. final_url, title = fetched["url"], fetched.get("title", "")
  162. refusal = {"url": final_url, "title": title, "content": "", "raw_content": ""}
  163. try:
  164. blocked = check_website_access(final_url)
  165. except Exception as exc:
  166. logger.warning("Website policy check failed for redirect target %s", final_url, exc_info=True)
  167. return {**refusal, "error": f"Website policy check failed for redirect target: {exc}"}
  168. if blocked:
  169. logger.info("Blocked redirected web_extract for %s by rule %s", blocked["host"], blocked["rule"])
  170. policy = {k: blocked[k] for k in ("host", "rule", "source")}
  171. return {**refusal, "error": blocked["message"], "blocked_by_policy": policy}
  172. if not await async_is_safe_url(final_url):
  173. logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
  174. return {**refusal, "error": "Blocked: URL targets a private or internal network address"}
  175. return None
  176. async def _gate_final_urls(results: list[dict], requested: set[str]) -> list[dict]:
  177. """Re-gate every page a provider reports under a URL other than one we asked for (a redirect
  178. target, a canonicalised host): the SSRF and website-policy checks only ever saw the requested URL.
  179. Covers every provider shape — Keenable reports the target in ``url`` with ``sourceURL`` = requested,
  180. Exa/Parallel/Tavily report the vendor's URL in both — and rescued batches alike."""
  181. gated = []
  182. for fetched in results:
  183. url = fetched.get("url")
  184. if url and url not in requested and not fetched.get("error"):
  185. fetched = await _final_url_refusal(fetched) or fetched
  186. gated.append(fetched)
  187. return gated
  188. async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[str]) -> list[dict]:
  189. """Fetch *fetch_urls* (with keyless rescue), re-gate redirect targets, then cache the passes.
  190. Rescued batches are never cached."""
  191. from tools.web_result_cache import extract_cache_put, extract_cache_record_final_url
  192. results, rescued = await _fetch_with_rescue(provider, fetch_urls, format)
  193. requested = set(fetch_urls)
  194. results = await _gate_final_urls(results, requested)
  195. if rescued:
  196. return results
  197. # Cache each successful fetch under the REQUESTED url it reports as its own — never by list
  198. # position: providers omit failed URLs or return successes out of request order, and a positional
  199. # write filed one page's text under another URL's key for the whole TTL. ``metadata.sourceURL``
  200. # counts because Keenable/Firecrawl put the requested URL there when ``url`` is the redirect target.
  201. # An entry naming no requested URL is served but not cached (a miss re-fetches; a mis-key poisons).
  202. for fetched in results:
  203. meta = fetched.get("metadata")
  204. source = meta.get("sourceURL") if isinstance(meta, dict) else None
  205. url = next((u for u in (fetched.get("url"), source) if u in requested), None)
  206. _content = fetched.get("raw_content", "") or fetched.get("content", "")
  207. if url and _content and not fetched.get("error"):
  208. stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
  209. if stored and fetched.get("url") and fetched["url"] != url:
  210. extract_cache_record_final_url(url, fetched["url"], _content, format=format, provider=provider.name)
  211. return results
  212. async def _cached_final_url_allowed(hit: dict, requested: str) -> bool:
  213. """Gate a cache hit on the URL its content came from: a redirector cached before its target was
  214. blocklisted (or turned private) must not keep serving that target. Consumes ``final_url``."""
  215. final_url = hit.pop("final_url", requested)
  216. if final_url == requested: # the requested URL already passed both gates
  217. return True
  218. from tools.url_safety import async_is_safe_url
  219. from tools.website_policy import check_website_access
  220. try:
  221. if check_website_access(final_url) is not None:
  222. return False
  223. except Exception: # fail closed: a miss, and the live fetch re-gates the redirect
  224. logger.warning("Website policy check failed for cached final URL %s", final_url, exc_info=True)
  225. return False
  226. return await async_is_safe_url(final_url)
  227. async def _extract_safe_urls(provider, safe_urls: list[str], format: Optional[str]) -> list[dict]:
  228. """Serve cache hits, fetch the rest, and merge back in ``safe_urls`` order.
  229. The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate, SSRF gate, and provider
  230. resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a
  231. control; policy-blocked URLs are cache misses, judged on both the requested and the cached final URL. Keys include provider and format, so switching either
  232. within the TTL never serves the other's content."""
  233. from tools.web_result_cache import extract_cache_get
  234. from tools.website_policy import check_website_access as _check_site
  235. cached_results, fetch_urls, fetch_positions = {}, [], []
  236. for position, url in enumerate(safe_urls):
  237. try:
  238. _policy_block = _check_site(url)
  239. except Exception:
  240. _policy_block = None
  241. hit = extract_cache_get(url, format=format, provider=provider.name) if _policy_block is None else None
  242. if hit is not None and not await _cached_final_url_allowed(hit, url):
  243. hit = None # a miss: the live fetch re-checks the redirect and refuses with blocked_by_policy
  244. if hit is not None:
  245. cached_results[position] = hit
  246. else:
  247. fetch_urls.append(url)
  248. fetch_positions.append(position)
  249. if not fetch_urls:
  250. return [cached_results[i] for i in range(len(safe_urls))]
  251. logger.info("Web extract via %s: %d URL(s)", provider.name, len(fetch_urls))
  252. results = await _dispatch_extract(provider, fetch_urls, format)
  253. if not cached_results:
  254. return results
  255. return _merge_in_order(len(safe_urls), cached_results, fetch_positions, fetch_urls, results)