| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304 |
- """web_extract helpers: URL validation, provider resolution, cache-aware dispatch.
- Order of controls (each is a gate, never skipped by a cache hit): secret-URL
- refusal -> SSRF filter (in web_tools.web_extract_tool) -> provider resolution
- (strict selection) -> per-URL website policy -> disk cache -> vendor call with
- one-shot keyless rescue -> website policy + SSRF on every page reported under a
- URL that was not requested (redirect targets). Logs under the origin
- (tools.web_tools) logger.
- """
- import asyncio
- import json
- import logging
- from typing import Any, Dict, List, Optional
- from tools.tool_backend_helpers import selection_error, selection_exists
- from tools.url_safety import normalize_url_for_request
- from tools.web_tools_rescue import _rescue_eligible, _rescue_extract
- logger = logging.getLogger("tools.web_tools")
- _NO_RESULT_ERROR = "Extract backend returned no result for this URL"
- _DEFAULT_EXTRACT_TIMEOUT_S = 120.0
- _EXTRACT_BACKENDS_HINT = "firecrawl, tavily, keenable, exa, or parallel."
- _INVALID_ITEM_ERROR = (
- "Invalid URL item at index {}: expected a URL string or an object with a string 'url' or 'href' field"
- )
- def _web_extract_url(value: Any) -> Optional[str]:
- """URL from a model-supplied extract item (str, or dict with ``url``/``href``); None if unusable.
- Models sometimes forward a whole search result instead of its URL, hence the dict form. Never
- stringify arbitrary objects into misleading fetch targets.
- """
- if isinstance(value, dict):
- value = value.get("url") or value.get("href")
- return (value.strip() or None) if isinstance(value, str) else None
- def _disabled_plugin_error(capability: str, disabled_key: str) -> str:
- """Error text when the configured backend's bundled plugin is disabled in config."""
- vendor = disabled_key.split("/", 1)[-1]
- return (
- f"web.{capability}_backend is set to '{vendor}', but its plugin ('{disabled_key}') is disabled "
- f"in config. Re-enable it with `hermes plugins enable {disabled_key}` "
- "(or remove it from plugins.disabled)."
- )
- def _no_provider_error(capability: str, fallback: str) -> str:
- """Error when no provider resolved: point at a disabled bundled plugin if that is the real cause."""
- from agent.web_search_registry import _disabled_web_plugin_for
- disabled_key = _disabled_web_plugin_for(capability=capability)
- return _disabled_plugin_error(capability, disabled_key) if disabled_key else fallback
- def _strict_selection_error(capability: str, backend: str) -> str:
- """Error for a stored-but-unregistered backend: name the disabled plugin, else the bad selection.
- Strict selection never silently switches to whatever the availability walk finds."""
- failure = f"no registered web {capability} provider has that name"
- return _no_provider_error(capability, selection_error("web", f"'{backend}'", failure))
- def _result_entry(url: str, error: Optional[str]) -> dict[str, Any]:
- return {"url": url, "title": "", "content": "", "error": error}
- def _extract_error_json(error: str) -> str:
- return json.dumps({"success": False, "error": error}, ensure_ascii=False)
- def _refuse_all(error: str):
- """Whole-call refusal tuple for ``_validate_extract_urls`` (exfiltration prevention)."""
- return None, None, None, json.dumps({"success": False, "error": error})
- def _merge_in_order(
- total: int, fixed: dict[int, dict], fetch_positions: list[int], fetch_urls: list[str], results: list[dict]
- ) -> list[dict]:
- """Rebuild a ``total``-long result list: *fixed* entries by position, fetched *results* at
- *fetch_positions* (a short provider list yields ``_NO_RESULT_ERROR`` entries for the rest)."""
- merged = dict(fixed)
- for pos, position in enumerate(fetch_positions):
- missing = _result_entry(fetch_urls[pos], _NO_RESULT_ERROR)
- merged[position] = results[pos] if pos < len(results) else missing
- return [merged[i] for i in range(total)]
- def _validate_extract_urls(urls: list[Any]):
- """Normalize model-supplied items and block URLs carrying secrets (percent-encoded forms are unquoted
- and checked too). Returns ``(normalized_urls, normalized_indices, invalid_urls, blocked_json)``;
- ``blocked_json`` is a whole-call refusal (exfiltration prevention) or None."""
- from agent.redact import _PREFIX_RE
- from urllib.parse import unquote
- normalized_urls, normalized_indices, invalid_urls = [], [], {}
- for index, item in enumerate(urls):
- _url = _web_extract_url(item)
- if _url is None:
- invalid_urls[index] = _result_entry("", _INVALID_ITEM_ERROR.format(index))
- continue
- normalized_url = normalize_url_for_request(_url)
- if any(_PREFIX_RE.search(c) for c in (_url, unquote(_url), normalized_url, unquote(normalized_url))):
- return _refuse_all(
- "Blocked: URL contains what appears to be an API key or token. "
- "Secrets must not be sent in URLs."
- )
- normalized_urls.append(normalized_url)
- normalized_indices.append(index)
- return normalized_urls, normalized_indices, invalid_urls, None
- def _resolve_extract_provider(backend: str):
- """Resolve the extract provider for *backend*; returns ``(provider, error_json)``.
- A registered search-only backend is a typed error (never a silent switch). An unregistered name with
- a stored web selection is a strict-selection error; with no selection, fall through to the walk.
- """
- from agent.web_search_registry import get_active_extract_provider, get_provider as _wsp_get_provider
- provider = _wsp_get_provider(backend) if backend else None
- if provider is not None and provider.supports_extract():
- return provider, None
- if provider is not None:
- return None, _extract_error_json(
- f"{provider.display_name} is a search-only backend and cannot extract URL content. "
- "Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT
- )
- if backend and selection_exists("web"):
- return None, _extract_error_json(_strict_selection_error("extract", backend))
- provider = get_active_extract_provider()
- if provider is None:
- fallback = "No web extract provider configured. Set web.extract_backend to " + _EXTRACT_BACKENDS_HINT
- return None, _extract_error_json(_no_provider_error("extract", fallback))
- return provider, None
- def _extract_timeout_seconds() -> float:
- """Wall-clock cap for one provider ``extract()`` dispatch (``web.extract_timeout``, default 120s).
- A hanging backend (server keeps the response open without finishing) otherwise stalls the
- tool call indefinitely. 0 or a negative value disables the cap.
- """
- from tools.web_tools import _load_web_config
- try:
- return float(_load_web_config().get("extract_timeout", _DEFAULT_EXTRACT_TIMEOUT_S))
- except (TypeError, ValueError):
- return _DEFAULT_EXTRACT_TIMEOUT_S
- def _backend_failed_whole_batch(results: list[dict]) -> bool:
- from plugins.web._common import is_origin_http_error
- errors = [r.get("error") for r in results]
- return bool(errors) and all(errors) and not any(is_origin_http_error(e) for e in errors)
- async def _fetch_with_rescue(provider, fetch_urls: list[str], format: Optional[str]) -> tuple[list[dict], bool]:
- """Call ``provider.extract`` (async or sync-in-thread), with one-shot keyless rescue.
- Rescue fires on a raised exception — including a dispatch timeout — or when the WHOLE batch
- failed (backend outage, not per-page problems): an origin HTTP 4xx/5xx is the page's own answer,
- so a batch with one never triggers it. Returns ``(results, rescued)``.
- """
- import inspect
- timeout = _extract_timeout_seconds()
- try:
- if inspect.iscoroutinefunction(provider.extract):
- coro = provider.extract(fetch_urls, format=format)
- else: # sync extract() runs in a thread so network I/O never blocks the loop
- coro = asyncio.to_thread(provider.extract, fetch_urls, format=format)
- if timeout > 0:
- results = await asyncio.wait_for(coro, timeout=timeout)
- else:
- results = await coro
- except TimeoutError: # hanging backend — bounded, never a stalled tool call
- logger.warning("web_extract provider '%s' timed out after %.0fs for %d URL(s)",
- provider.name, timeout, len(fetch_urls))
- failed = [_result_entry(u, f"Extract timed out after {timeout:.0f}s via {provider.name}")
- for u in fetch_urls]
- if not _rescue_eligible(provider):
- return failed, False
- return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, failed), True
- except Exception as exc:
- if not _rescue_eligible(provider):
- raise
- failed = [_result_entry(u, str(exc)) for u in fetch_urls]
- return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, failed), True
- if _backend_failed_whole_batch(results) and _rescue_eligible(provider):
- return await asyncio.to_thread(_rescue_extract, provider.name, fetch_urls, results), True
- return results, False
- async def _final_url_refusal(fetched: dict) -> Optional[dict]:
- """Refusal entry when a page's reported (post-redirect) URL fails the website policy or SSRF gate,
- in the shape Firecrawl's own redirect re-check returns; None when it passes. A policy check that
- raises refuses (fail closed): the requested URL passed, but nothing vouches for this one."""
- from tools.url_safety import async_is_safe_url
- from tools.website_policy import check_website_access
- final_url, title = fetched["url"], fetched.get("title", "")
- refusal = {"url": final_url, "title": title, "content": "", "raw_content": ""}
- try:
- blocked = check_website_access(final_url)
- except Exception as exc:
- logger.warning("Website policy check failed for redirect target %s", final_url, exc_info=True)
- return {**refusal, "error": f"Website policy check failed for redirect target: {exc}"}
- if blocked:
- logger.info("Blocked redirected web_extract for %s by rule %s", blocked["host"], blocked["rule"])
- policy = {k: blocked[k] for k in ("host", "rule", "source")}
- return {**refusal, "error": blocked["message"], "blocked_by_policy": policy}
- if not await async_is_safe_url(final_url):
- logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
- return {**refusal, "error": "Blocked: URL targets a private or internal network address"}
- return None
- async def _gate_final_urls(results: list[dict], requested: set[str]) -> list[dict]:
- """Re-gate every page a provider reports under a URL other than one we asked for (a redirect
- target, a canonicalised host): the SSRF and website-policy checks only ever saw the requested URL.
- Covers every provider shape — Keenable reports the target in ``url`` with ``sourceURL`` = requested,
- Exa/Parallel/Tavily report the vendor's URL in both — and rescued batches alike."""
- gated = []
- for fetched in results:
- url = fetched.get("url")
- if url and url not in requested and not fetched.get("error"):
- fetched = await _final_url_refusal(fetched) or fetched
- gated.append(fetched)
- return gated
- async def _dispatch_extract(provider, fetch_urls: list[str], format: Optional[str]) -> list[dict]:
- """Fetch *fetch_urls* (with keyless rescue), re-gate redirect targets, then cache the passes.
- Rescued batches are never cached."""
- from tools.web_result_cache import extract_cache_put, extract_cache_record_final_url
- results, rescued = await _fetch_with_rescue(provider, fetch_urls, format)
- requested = set(fetch_urls)
- results = await _gate_final_urls(results, requested)
- if rescued:
- return results
- # Cache each successful fetch under the REQUESTED url it reports as its own — never by list
- # position: providers omit failed URLs or return successes out of request order, and a positional
- # write filed one page's text under another URL's key for the whole TTL. ``metadata.sourceURL``
- # counts because Keenable/Firecrawl put the requested URL there when ``url`` is the redirect target.
- # An entry naming no requested URL is served but not cached (a miss re-fetches; a mis-key poisons).
- for fetched in results:
- meta = fetched.get("metadata")
- source = meta.get("sourceURL") if isinstance(meta, dict) else None
- url = next((u for u in (fetched.get("url"), source) if u in requested), None)
- _content = fetched.get("raw_content", "") or fetched.get("content", "")
- if url and _content and not fetched.get("error"):
- stored = extract_cache_put(url, _content, fetched.get("title", ""), format=format, provider=provider.name)
- if stored and fetched.get("url") and fetched["url"] != url:
- extract_cache_record_final_url(url, fetched["url"], _content, format=format, provider=provider.name)
- return results
- async def _cached_final_url_allowed(hit: dict, requested: str) -> bool:
- """Gate a cache hit on the URL its content came from: a redirector cached before its target was
- blocklisted (or turned private) must not keep serving that target. Consumes ``final_url``."""
- final_url = hit.pop("final_url", requested)
- if final_url == requested: # the requested URL already passed both gates
- return True
- from tools.url_safety import async_is_safe_url
- from tools.website_policy import check_website_access
- try:
- if check_website_access(final_url) is not None:
- return False
- except Exception: # fail closed: a miss, and the live fetch re-gates the redirect
- logger.warning("Website policy check failed for cached final URL %s", final_url, exc_info=True)
- return False
- return await async_is_safe_url(final_url)
- async def _extract_safe_urls(provider, safe_urls: list[str], format: Optional[str]) -> list[dict]:
- """Serve cache hits, fetch the rest, and merge back in ``safe_urls`` order.
- The disk cache (tools/web_result_cache.py) sits AFTER the secret-URL gate, SSRF gate, and provider
- resolution, and is gated per-URL on the website policy — a hit skips only the vendor call, never a
- control; policy-blocked URLs are cache misses, judged on both the requested and the cached final URL. Keys include provider and format, so switching either
- within the TTL never serves the other's content."""
- from tools.web_result_cache import extract_cache_get
- from tools.website_policy import check_website_access as _check_site
- cached_results, fetch_urls, fetch_positions = {}, [], []
- for position, url in enumerate(safe_urls):
- try:
- _policy_block = _check_site(url)
- except Exception:
- _policy_block = None
- hit = extract_cache_get(url, format=format, provider=provider.name) if _policy_block is None else None
- if hit is not None and not await _cached_final_url_allowed(hit, url):
- hit = None # a miss: the live fetch re-checks the redirect and refuses with blocked_by_policy
- if hit is not None:
- cached_results[position] = hit
- else:
- fetch_urls.append(url)
- fetch_positions.append(position)
- if not fetch_urls:
- return [cached_results[i] for i in range(len(safe_urls))]
- logger.info("Web extract via %s: %d URL(s)", provider.name, len(fetch_urls))
- results = await _dispatch_extract(provider, fetch_urls, format)
- if not cached_results:
- return results
- return _merge_in_order(len(safe_urls), cached_results, fetch_positions, fetch_urls, results)
|