web_tools.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573
  1. #!/usr/bin/env python3
  2. """Generic web_search / web_extract tools over pluggable backends.
  3. Backend is selected during ``hermes tools`` (``web.backend`` in config.yaml; per
  4. capability via ``web.search_backend`` / ``web.extract_backend``). Every vendor
  5. implementation lives in ``plugins/web/<vendor>/provider.py`` and registers with
  6. ``agent.web_search_registry``; this module owns selection, safety gates,
  7. caching, keyless rescue, and the truncate-and-store result pipeline.
  8. Debug: ``WEB_TOOLS_DEBUG=true`` writes ``logs/web_tools_debug_<UUID>.json``.
  9. """
  10. import json
  11. import logging
  12. import os
  13. from typing import List, Any, Optional
  14. # Per-vendor client cache slots; plugins read/write these via tools.web_tools (tests reset them to None).
  15. _firecrawl_client = _firecrawl_client_config = _parallel_client = _async_parallel_client = _exa_client = None
  16. from plugins.web.firecrawl.provider import _is_tool_gateway_ready, check_firecrawl_api_key
  17. from tools.debug_helpers import DebugSession
  18. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_selection, selection_exists
  19. from tools.url_safety import async_is_safe_url
  20. from tools.web_tools_rescue import _managed_search_fallback, _rescue_eligible, _rescue_search
  21. from tools.web_tools_truncate import _effective_char_limit, _trim_results, _truncate_results, convert_base64_images_to_links
  22. from tools.web_tools_extract import (
  23. _extract_safe_urls, _merge_in_order, _no_provider_error, _resolve_extract_provider, _result_entry,
  24. _strict_selection_error, _validate_extract_urls,
  25. )
  26. logger = logging.getLogger(__name__)
  27. # ─── Backend Selection ────────────────────────────────────────────────────────
  28. def _env_value(name: str) -> str:
  29. """Resolve ``name`` via the config-aware env layer (``hermes config set`` values), then process env.
  30. Mirrors the SearXNG provider's ``_searxng_url()`` so that values set through Hermes' config/.env layer
  31. (``hermes config set``, ``hermes tools``) are honored here too — not just raw process-env exports.
  32. Without this, a config-only ``SEARXNG_URL`` (or any provider key) leaves the backend auto-detect cascade
  33. and ``check_web_api_key()`` blind to it. See #34290.
  34. """
  35. try:
  36. from hermes_cli.config import get_env_value
  37. val = get_env_value(name)
  38. except Exception:
  39. val = None
  40. return ((os.getenv(name, "") if val is None else val) or "").strip()
  41. def _has_env(name: str) -> bool:
  42. return bool(_env_value(name))
  43. def _load_web_config() -> dict:
  44. """Load the ``web:`` section from config.yaml; always a dict (a null section yields ``{}``)."""
  45. try:
  46. from hermes_cli.config import load_config
  47. return load_config().get("web") or {}
  48. except Exception:
  49. return {}
  50. def _configured_backend(key: str = "backend") -> str:
  51. """Lower-cased, stripped ``web.<key>`` value ("" when unset/null)."""
  52. return (_load_web_config().get(key) or "").lower().strip()
  53. def _registry_call(func_name: str, default, *args):
  54. """``agent.web_search_registry.<func_name>(*args)``, or *default* if it raised (registry never fatal)."""
  55. try:
  56. import agent.web_search_registry as registry_mod
  57. return getattr(registry_mod, func_name)(*args)
  58. except Exception as exc:
  59. logger.debug("web provider registry %s%r failed: %s", func_name, args, exc)
  60. return default
  61. def _registered_web_provider(backend: str):
  62. """Plugin-registered web provider by name, or ``None``."""
  63. return _registry_call("get_provider", None, backend) if backend else None
  64. def _list_registered_web_providers():
  65. """All plugin-registered web providers (empty list on failure)."""
  66. return _registry_call("list_providers", [])
  67. def _probe(provider, method: str, context: str = "") -> Optional[bool]:
  68. """``bool(provider.<method>())``, or ``None`` if it raised (a broken provider is unavailable; *context* is
  69. appended to the debug log line, e.g. " during readiness check")."""
  70. try:
  71. return bool(getattr(provider, method)())
  72. except Exception as exc:
  73. name = getattr(provider, "name", provider)
  74. logger.debug("web provider %r.%s() raised%s: %s", name, method, context, exc)
  75. return None
  76. def _get_backend() -> str:
  77. """Shared web backend name. A stored ``web.backend`` is returned as-is — no availability probe, no
  78. fallback — so a broken selection surfaces the vendor's honest error rather than silently rerouting.
  79. The managed ``use_gateway`` selection also resolves to firecrawl with no ladder. Autodetect runs
  80. whenever no SHARED web selection was ever stored: per-capability keys (``web.search_backend``,
  81. ``web.extract_backend``) name only their own capability and never reroute the other (#113017)."""
  82. configured = _configured_backend()
  83. if configured:
  84. # "nous" (managed subscription) is serviced by firecrawl, routed through the managed Tool Gateway.
  85. return "firecrawl" if configured == NOUS_MANAGED_PROVIDER else configured
  86. if read_selection("web") is not None:
  87. # Shared selection exists (use_gateway) but no shared name: firecrawl, no ladder.
  88. return "firecrawl"
  89. # Never-configured install.
  90. return _autodetect_backend() or _keyless_backend() or "firecrawl" # default (backward compat)
  91. def _autodetect_backend() -> Optional[str]:
  92. """Autodetect rungs above the keyless tier, or None. Explicit user credentials beat the managed-gateway
  93. probe (a Nous OAuth token's tier may not grant web access; the gateway then fails at runtime with no
  94. fallback). Free tiers trail paid."""
  95. backend_candidates = (
  96. ("tavily", _has_env("TAVILY_API_KEY")), ("perplexity", _has_env("PERPLEXITY_API_KEY")),
  97. ("exa", _has_env("EXA_API_KEY")),
  98. ("parallel", _has_env("PARALLEL_API_KEY")), ("keenable", _has_env("KEENABLE_API_KEY")),
  99. ("firecrawl", _has_env("FIRECRAWL_API_KEY") or _has_env("FIRECRAWL_API_URL")),
  100. ("firecrawl", _is_tool_gateway_ready()), ("searxng", _has_env("SEARXNG_URL")),
  101. ("brave-free", _has_env("BRAVE_SEARCH_API_KEY")), ("ddgs", _ddgs_package_importable()),
  102. )
  103. for backend, available in backend_candidates:
  104. if available:
  105. return backend
  106. # Plugin-contributed providers (built-ins are covered above); probe the held object directly.
  107. for provider in _list_registered_web_providers():
  108. if provider.name not in _LEGACY_WEB_BACKENDS and _probe(provider, "is_available"):
  109. return provider.name
  110. return None
  111. def _keyless_backend() -> Optional[str]:
  112. """Keyless free-tier backend name, or None. Strictly the last autodetect rung so it never
  113. pre-empts a keyed backend. Discovery must run first: reachable from contexts that haven't
  114. loaded plugins (subprocess runs, delegate children)."""
  115. try:
  116. _ensure_web_plugins_loaded()
  117. from agent.web_search_registry import _keyless_preference, _keyless_tier_enabled
  118. if _keyless_tier_enabled():
  119. for name in _keyless_preference():
  120. provider = _registered_web_provider(name)
  121. if provider is not None and _probe(provider, "is_keyless_available"):
  122. return name
  123. except Exception as exc:
  124. logger.debug("keyless fallback walk failed: %s", exc)
  125. return None
  126. def _managed_web_search() -> bool:
  127. """True when web_search is on the managed Nous route: the stored ``nous`` selection (a
  128. ``web.search_backend: nous`` pin, else the shared one), or a never-configured install whose
  129. autodetect lands on the gateway — the entitled Firecrawl gateway, or free Perplexity fast search
  130. for any Nous identity when nothing else is configured (search-only: the extract ladder is
  131. untouched). A stored vendor selection never is."""
  132. search_pin = _configured_backend("search_backend")
  133. if search_pin:
  134. return search_pin == NOUS_MANAGED_PROVIDER
  135. selected = read_selection("web")
  136. if selected is not None:
  137. return selected == NOUS_MANAGED_PROVIDER
  138. backend = _autodetect_backend()
  139. if backend == "firecrawl":
  140. return not (_has_env("FIRECRAWL_API_KEY") or _has_env("FIRECRAWL_API_URL")) and _is_tool_gateway_ready()
  141. from tools.managed_tool_gateway import peek_nous_access_token, resolve_free_search_gateway
  142. return backend is None and resolve_free_search_gateway(token_reader=peek_nous_access_token) is not None
  143. def _get_search_backend() -> str:
  144. """Backend for web_search: ``web.search_backend`` (strict, no probe) > ``web.backend`` > autodetect.
  145. The managed Nous route (a ``nous`` pin or shared selection) serves search from Perplexity (extract
  146. stays on Firecrawl); managed Firecrawl is the per-call fallback, see ``_memoized_search``."""
  147. pin = _configured_backend("search_backend")
  148. if pin and pin != NOUS_MANAGED_PROVIDER:
  149. return pin
  150. return "perplexity" if _managed_web_search() else _get_backend()
  151. def _get_extract_backend() -> str:
  152. """Backend for web_extract: ``web.extract_backend`` (strict, no probe) > ``web.backend`` > autodetect.
  153. A ``nous`` pin is managed Firecrawl; the client picks the gateway route for that capability."""
  154. pin = _configured_backend("extract_backend")
  155. return "firecrawl" if pin == NOUS_MANAGED_PROVIDER else pin or _get_backend()
  156. def _ddgs_package_importable() -> bool:
  157. """ddgs is the only backend gated on package presence; single symbol so tests can patch it."""
  158. try:
  159. import ddgs
  160. return True
  161. except ImportError:
  162. return False
  163. def _xai_available() -> bool:
  164. # Cheap probe only (env var OR auth.json OAuth): resolve_xai_http_credentials() may hit the network.
  165. try:
  166. from tools.xai_http import has_xai_credentials
  167. return has_xai_credentials()
  168. except Exception:
  169. return False
  170. # Built-in backends -> cheap availability probes; any other name is a plugin provider resolved via the
  171. # registry's ``is_available()``. Lambdas so test patches of module-level helpers (_ddgs_package_importable,
  172. # check_firecrawl_api_key) are honored at call time. ``xai`` is probed via has_xai_credentials(), not a
  173. # registered provider, though the registry's _LEGACY_PREFERENCE omits it — drop it if xai ever registers.
  174. _BUILTIN_AVAILABILITY = {
  175. "exa": lambda: _has_env("EXA_API_KEY"),
  176. "parallel": lambda: _has_env("PARALLEL_API_KEY"),
  177. "keenable": lambda: _has_env("KEENABLE_API_KEY"),
  178. "firecrawl": lambda: check_firecrawl_api_key(),
  179. "tavily": lambda: _has_env("TAVILY_API_KEY")
  180. or any(_configured_backend(k) == "tavily" for k in ("backend", "search_backend", "extract_backend")),
  181. "perplexity": lambda: _has_env("PERPLEXITY_API_KEY") or _managed_web_search(),
  182. "searxng": lambda: _has_env("SEARXNG_URL"),
  183. "brave-free": lambda: _has_env("BRAVE_SEARCH_API_KEY"),
  184. "ddgs": lambda: _ddgs_package_importable(),
  185. "xai": _xai_available,
  186. }
  187. _LEGACY_WEB_BACKENDS = frozenset(_BUILTIN_AVAILABILITY)
  188. def _is_backend_available(backend: str) -> bool:
  189. """True when *backend* is usable — the single availability chokepoint. Non-legacy names delegate to the
  190. registered provider's ``is_available()`` (unregistered names fall through); built-ins use cheap probes.
  191. For plugin-registered backends (any name outside :data:`_LEGACY_WEB_BACKENDS`), availability is
  192. delegated to the provider's ``is_available()`` via the web_search_registry. This is the single
  193. chokepoint through which ``_get_backend``, ``_get_capability_backend``, and ``check_web_api_key`` all
  194. resolve availability — fixing custom-provider discovery for every caller at once (issues #28651, #31873,
  195. #32698). Built-in backends keep their cheap hardcoded probes below.
  196. """
  197. backend = (backend or "").lower().strip()
  198. provider = None if backend in _LEGACY_WEB_BACKENDS else _registered_web_provider(backend)
  199. if provider is not None:
  200. return _probe(provider, "is_available") or False
  201. probe = _BUILTIN_AVAILABILITY.get(backend)
  202. return probe() if probe else False
  203. # ─── Firecrawl Client ──────────────────────────────────────────────────────── After PR #25182, the
  204. # firecrawl client, lazy SDK proxy, dual-auth config resolution, response normalizers, and
  205. # check_firecrawl_api_key() all live in plugins.web.firecrawl.provider.
  206. def _web_requires_env() -> list[str]:
  207. """Tool-registry metadata env vars for the web backends. Gateway vars are always listed: gating them
  208. on ``managed_nous_tools_enabled()`` cost a synchronous portal HTTP refresh at every CLI startup.
  209. Contract: set var -> tool sees it; extras are harmless for the not-logged-in."""
  210. return [
  211. "EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "PERPLEXITY_API_KEY", "KEENABLE_API_KEY", "FIRECRAWL_API_KEY",
  212. "FIRECRAWL_API_URL", "FIRECRAWL_GATEWAY_URL", "PERPLEXITY_GATEWAY_URL", "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_SCHEME",
  213. "TOOL_GATEWAY_USER_TOKEN",
  214. ]
  215. _debug = DebugSession("web_tools", env_var="WEB_TOOLS_DEBUG")
  216. # ─── Dispatch ─────────────────────────────────────────────────────────────────
  217. # ─── Exa / Parallel inline helpers — moved into plugins ────────────────────── After PR #25182, the exa
  218. # client + search/extract and parallel client + search/extract helpers all live in their respective plugins:
  219. # - plugins/web/exa/provider.py - plugins/web/parallel/provider.py Both plugins register through
  220. # agent.web_search_registry and the dispatchers in this file resolve them via get_active_*_provider().
  221. def _ensure_web_plugins_loaded() -> None:
  222. """Idempotently run plugin discovery so the web registry is populated. Dispatch is reachable from contexts
  223. that never triggered discovery (subprocess agent runs, delegate children, scripts); without it a
  224. configured backend yields a misleading "No web ... provider" error.
  225. Every bundled web provider (brave-free, ddgs, searxng, exa, parallel, tavily, firecrawl, keenable)
  226. registers itself via ``plugins/web/<vendor>/__init__.py`` during plugin discovery. Tool dispatch can be
  227. reached from contexts that haven't already triggered discovery — subprocess agent runs, delegate
  228. children, standalone scripts, certain test paths — and without it the registry is empty and
  229. ``get_provider('firecrawl')`` returns ``None`` even when the user has ``web.extract_backend: firecrawl``
  230. configured and ``FIRECRAWL_API_KEY`` set. See #27580.
  231. """
  232. try:
  233. from hermes_cli.plugins import _ensure_plugins_discovered
  234. _ensure_plugins_discovered()
  235. except Exception as exc:
  236. # Warning, not debug: a broken plugin import is otherwise invisible.
  237. logger.warning("Web plugin discovery failed (non-fatal): %s", exc)
  238. def _finish_debug(call_name: str, debug_call_data: dict, error_msg: Optional[str] = None) -> Optional[str]:
  239. """Log the call into the debug session; with *error_msg*, record it and return its ``tool_error`` envelope."""
  240. if error_msg is not None:
  241. logger.debug("%s", error_msg)
  242. debug_call_data["error"] = error_msg
  243. _debug.log_call(call_name, debug_call_data)
  244. _debug.save()
  245. return None if error_msg is None else tool_error(error_msg)
  246. def web_search_tool(query: str, limit: int = 5) -> str:
  247. """Search the web via the configured backend.
  248. Returns a JSON string ``{"success": bool, "data": {"web": [{"title", "url", "description", "position"},
  249. ...]}}`` (metadata only — use web_extract_tool for page content) or ``{"success": false, "error": ...}``.
  250. """
  251. try:
  252. limit = min(max(int(limit), 1), 100)
  253. except (TypeError, ValueError):
  254. limit = 5
  255. debug_call_data = {
  256. "parameters": {"query": query, "limit": limit}, "error": None, "results_count": 0,
  257. "original_response_size": 0, "final_response_size": 0,
  258. }
  259. try:
  260. from tools.interrupt import is_interrupted
  261. if is_interrupted():
  262. return tool_error("Interrupted", success=False)
  263. # Sync only — every provider's search() is sync.
  264. _ensure_web_plugins_loaded()
  265. from agent.web_search_registry import get_active_search_provider, get_provider as _wsp_get_provider
  266. backend = _get_search_backend()
  267. provider = _wsp_get_provider(backend) if backend else None
  268. if provider is None or not provider.supports_search():
  269. if provider is None and backend and selection_exists("web"):
  270. error_text = debug_call_data["error"] = _strict_selection_error("search", backend)
  271. _finish_debug("web_search_tool", debug_call_data)
  272. return json.dumps({"success": False, "error": error_text}, indent=2, ensure_ascii=False)
  273. # Never-configured install: legacy availability-walked autodetect.
  274. provider = get_active_search_provider()
  275. if provider is None:
  276. fallback = "No web search provider configured. Run `hermes tools` to set one up."
  277. response_data = {"success": False, "error": _no_provider_error("search", fallback)}
  278. else:
  279. logger.info("Web search via %s: '%s' (limit: %d)", provider.name, query, limit)
  280. response_data = _memoized_search(provider, query, limit)
  281. debug_call_data["results_count"] = len(response_data.get("data", {}).get("web", []))
  282. result_json = json.dumps(response_data, indent=2, ensure_ascii=False)
  283. debug_call_data["final_response_size"] = len(result_json)
  284. _finish_debug("web_search_tool", debug_call_data)
  285. return result_json
  286. except Exception as e:
  287. return _finish_debug("web_search_tool", debug_call_data, f"Error searching web: {e!s}")
  288. def _memoized_search(provider, query: str, limit: int) -> dict:
  289. """TTL memo + single-flight around the paid vendor call (tools/web_result_cache.py); sits after every
  290. safety/config check. The provider is asked for the BUCKETED count so near-identical limits share an entry;
  291. the caller's count is sliced out. Only successful, non-rescued responses are cached — caching a rescue
  292. would make the one-shot ring fallback sticky for a whole TTL."""
  293. from tools.web_result_cache import bucket_limit, search_memo, slice_search_response
  294. def _paid_search() -> tuple[dict, bool]:
  295. fetch_limit = bucket_limit(limit)
  296. try:
  297. resp = provider.search(query, fetch_limit)
  298. except Exception as exc:
  299. served = _served_after_failure(str(exc), fetch_limit)
  300. if served is None:
  301. raise
  302. return served, True
  303. if not resp.get("success"):
  304. served = _served_after_failure(str(resp.get("error", "")), fetch_limit)
  305. if served is not None:
  306. return served, True
  307. return resp, False
  308. def _served_after_failure(error: str, fetch_limit: int) -> Optional[dict]:
  309. """Managed Firecrawl for a failed managed Perplexity call, else the one-shot keyless rescue when
  310. eligible; None means the vendor's own failure stands."""
  311. fallback = _managed_search_fallback(provider, error, query, fetch_limit)
  312. if fallback is not None:
  313. return fallback
  314. return _rescue_search(provider.name, error, query, fetch_limit) if _rescue_eligible(provider) else None
  315. response_data = search_memo.lookup(provider.name, query, limit)
  316. if response_data is None:
  317. with search_memo.flight_lock(provider.name, query, limit):
  318. # Re-check inside the lock: a concurrent identical call may have stored.
  319. response_data = search_memo.lookup(provider.name, query, limit)
  320. if response_data is None:
  321. response_data, was_rescued = _paid_search()
  322. if not was_rescued:
  323. search_memo.store(provider.name, query, limit, response_data)
  324. return slice_search_response(response_data, limit)
  325. async def web_extract_tool(urls: list[Any], format: str | None = None, char_limit: Optional[int] = None) -> str:
  326. """Extract clean page content (no LLM) from URLs via the configured backend.
  327. Pages over ``char_limit`` (default web.extract_char_limit or 15000) are head+tail truncated with a footer
  328. pointing at the stored full text; inline base64 images become ``[IMAGE: alt]``. URLs carrying secrets are
  329. refused before any fetch; private-network URLs are blocked per entry. Returns JSON ``{"results": [...]}``.
  330. """
  331. normalized_urls, normalized_indices, invalid_urls, blocked = _validate_extract_urls(urls)
  332. if blocked is not None:
  333. return blocked
  334. debug_call_data = {
  335. "parameters": {"urls": normalized_urls, "format": format, "char_limit": char_limit}, "error": None,
  336. "pages_extracted": 0, "pages_truncated": 0, "original_response_size": 0, "final_response_size": 0,
  337. "truncation_metrics": [], "processing_applied": [],
  338. }
  339. try:
  340. logger.info("Extracting content from %d URL(s)", len(normalized_urls))
  341. # SSRF protection — filter private/internal URLs before any backend.
  342. safe_urls, safe_indices, ssrf_blocked = [], [], {}
  343. for index, url in zip(normalized_indices, normalized_urls):
  344. if await async_is_safe_url(url):
  345. safe_urls.append(url)
  346. safe_indices.append(index)
  347. else:
  348. ssrf_blocked[index] = _result_entry(
  349. url, "Blocked: URL targets a private or internal network address"
  350. )
  351. results = []
  352. if safe_urls:
  353. backend = _get_extract_backend()
  354. _ensure_web_plugins_loaded()
  355. provider, error_json = _resolve_extract_provider(backend)
  356. if error_json is not None:
  357. return error_json
  358. results = await _extract_safe_urls(provider, safe_urls, format)
  359. # Reconstruct input order across invalid, blocked, and provider entries (providers preserve
  360. # the order of the safe URL list they receive).
  361. if invalid_urls or ssrf_blocked:
  362. fixed = {**ssrf_blocked, **invalid_urls}
  363. results = _merge_in_order(len(urls), fixed, safe_indices, safe_urls, results)
  364. logger.info("Extracted content from %d pages", len(results))
  365. debug_call_data["pages_extracted"] = len(results)
  366. debug_call_data["original_response_size"] = len(json.dumps({"results": results}))
  367. debug_call_data["processing_applied"].append("truncate_and_store")
  368. _truncate_results(results, _effective_char_limit(char_limit), debug_call_data)
  369. trimmed = _trim_results(results)
  370. result_json = (
  371. json.dumps({"results": trimmed}, indent=2, ensure_ascii=False) if trimmed
  372. else tool_error("Content was inaccessible or not found")
  373. )
  374. # Belt-and-suspenders sweep of the serialized JSON: a provider may tuck a base64 blob in metadata.
  375. cleaned_result = convert_base64_images_to_links(result_json)
  376. debug_call_data["final_response_size"] = len(cleaned_result)
  377. debug_call_data["processing_applied"].append("base64_image_conversion")
  378. _finish_debug("web_extract_tool", debug_call_data)
  379. return cleaned_result
  380. except Exception as e:
  381. return _finish_debug("web_extract_tool", debug_call_data, f"Error extracting content: {e!s}")
  382. def _provider_is_ready(provider) -> bool:
  383. """True when *provider* is keyed-available OR keyless-capable, without raising.
  384. ``get_active_*_provider()`` returns an explicitly configured backend even when ``is_available()`` is
  385. False (so dispatch can emit a precise error), so readiness gates (tool check_fn, ``hermes doctor``)
  386. must probe for real. Keyless mode (Exa/Parallel free tier) is a working state, not a misconfig.
  387. See #78412.
  388. """
  389. if provider is None:
  390. return False
  391. ready = _probe(provider, "is_available", " during readiness check")
  392. if ready is None: # broken provider == not ready; don't try the keyless probe
  393. return False
  394. return bool(ready or _probe(provider, "is_keyless_available", " during readiness check"))
  395. # Credential probes that back other tools but serve no registered web backend: ``xai`` is
  396. # probed via has_xai_credentials() for TTS/media only, so it must not light this gate. A
  397. # stored ``web.backend: xai`` still counts since _get_backend returns a configured
  398. # selection as-is and dispatch surfaces the honest "unknown provider" error.
  399. _WEB_CHECK_SKIP = frozenset({"xai"})
  400. def check_web_api_key() -> bool:
  401. """``check_fn`` gate for web_search / web_extract: is any web backend available?
  402. A plugin-registered provider reporting ``is_available()`` must light the tools up even with no
  403. built-in credentials; resolution funnels through :func:`_is_backend_available`.
  404. See #28651, #31873.
  405. """
  406. # Boolean OR over configured + built-ins — probe order is irrelevant here.
  407. candidates = ([c for c in (_configured_backend(),) if c]
  408. + [b for b in _LEGACY_WEB_BACKENDS if b not in _WEB_CHECK_SKIP])
  409. if any(_is_backend_available(backend) for backend in candidates):
  410. return True
  411. # Plugin path. Discovery must run first: check_fn fires at tool-registration time, before any dispatch.
  412. try:
  413. _ensure_web_plugins_loaded()
  414. from agent.web_search_registry import get_active_search_provider, get_active_extract_provider
  415. for provider in (get_active_search_provider(), get_active_extract_provider()):
  416. if provider is not None and getattr(provider, "name", None) in _WEB_CHECK_SKIP:
  417. # The registry's single-eligible / legacy walk picked a built-in that _get_backend
  418. # never autodetects (the explicit-config case was handled above): the dispatcher
  419. # would route to the keyless tier instead, so gate on exactly that.
  420. if _keyless_backend() is not None:
  421. return True
  422. continue
  423. if _provider_is_ready(provider):
  424. return True
  425. return False
  426. except Exception as exc:
  427. logger.debug("web provider registry availability check failed: %s", exc)
  428. return False
  429. # ─── Registry ─────────────────────────────────────────────────────────────────
  430. from tools.registry import registry, tool_error
  431. WEB_SEARCH_SCHEMA = {
  432. "name": "web_search",
  433. "description": "Search the web: returns titles, URLs and snippets (5 by default). Read pages with web_extract. The query is passed through to the configured backend, so operators such as site:domain, filetype:pdf, intitle:word, -term, and \"exact phrase\" may work when the backend supports them.",
  434. "parameters": {
  435. "type": "object",
  436. "properties": {
  437. "query": {
  438. "type": "string",
  439. "description": "The search query to look up on the web. You may include backend-supported operators such as site:example.com, filetype:pdf, intitle:word, -term, or \"exact phrase\"."
  440. },
  441. "limit": {
  442. "type": "integer",
  443. "description": "Maximum number of results to return. Defaults to 5.",
  444. "minimum": 1,
  445. "maximum": 100,
  446. "default": 5
  447. }
  448. },
  449. "required": ["query"]
  450. }
  451. }
  452. WEB_EXTRACT_SCHEMA = {
  453. "name": "web_extract",
  454. "description": "Extract content from web page URLs. Returns clean page content in markdown/text (no LLM summarization — fast). Also works with PDF URLs (arxiv papers, documents) — pass the PDF link directly. Pages within the char budget (default 15000) return whole; larger pages return a head+tail window with a footer telling you the full text's saved file path and the read_file call to page through the omitted middle. Inline images appear as [IMAGE: alt] placeholders; real image URLs are kept as links. If a URL fails, times out, or needs JS, login or clicks, use a browser tool instead.",
  455. "parameters": {
  456. "type": "object",
  457. "properties": {
  458. "urls": {
  459. "type": "array",
  460. "items": {"type": "string"},
  461. "description": "List of URLs to extract content from (max 5 URLs per call)",
  462. "maxItems": 5
  463. },
  464. "char_limit": {
  465. "type": "integer",
  466. "description": "Optional per-page character budget sent back (default 15000). Pages larger than this are head+tail truncated with the full text stored to disk. Raise it when you need more of a long page inline.",
  467. "minimum": 2000
  468. }
  469. },
  470. "required": ["urls"]
  471. }
  472. }
  473. registry.register(
  474. name="web_search", toolset="web", schema=WEB_SEARCH_SCHEMA,
  475. handler=lambda args, **kw: web_search_tool(args.get("query", ""), limit=args.get("limit", 5)),
  476. check_fn=check_web_api_key, requires_env=_web_requires_env(), emoji="🔍",
  477. max_result_size_chars=100_000,
  478. )
  479. registry.register(
  480. name="web_extract", toolset="web", schema=WEB_EXTRACT_SCHEMA,
  481. handler=lambda args, **kw: web_extract_tool(
  482. args.get("urls", [])[:5] if isinstance(args.get("urls"), list) else [], "markdown",
  483. char_limit=args.get("char_limit"),
  484. ),
  485. check_fn=check_web_api_key, requires_env=_web_requires_env(), is_async=True, emoji="📄",
  486. max_result_size_chars=100_000,
  487. )