provider.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389
  1. """Firecrawl web search + extract provider (direct SDK, keyless cloud, or Nous tool-gateway).
  2. Config: ``web.backend`` / ``web.search_backend`` / ``web.extract_backend: firecrawl``.
  3. Env: FIRECRAWL_API_KEY, FIRECRAWL_API_URL (self-hosted), FIRECRAWL_GATEWAY_URL / TOOL_GATEWAY_*.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import logging
  8. from typing import Any, Dict, List, Optional
  9. import httpx
  10. from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema, with_origin_status
  11. from tools import managed_tool_gateway as _gateway
  12. from tools import tool_backend_helpers as _backend_helpers
  13. from tools.url_safety import is_safe_url
  14. # Module-level (cheap import) so tests can monkeypatch the policy gate on this module.
  15. from tools.website_policy import check_website_access
  16. logger = logging.getLogger(__name__)
  17. _FIRECRAWL_CLOUD_API_URL = "https://api.firecrawl.dev"
  18. # The SDK costs ~200ms of imports on a cold CLI; defer to first use (tests patch ``Firecrawl`` here).
  19. _FIRECRAWL_CLS_CACHE: Optional[type] = None
  20. def _load_firecrawl_cls() -> type:
  21. """Import and cache ``firecrawl.Firecrawl`` (lazy_deps install hint → ImportError)."""
  22. global _FIRECRAWL_CLS_CACHE
  23. if _FIRECRAWL_CLS_CACHE is None:
  24. try:
  25. from pm import ensure_import as _lazy_ensure
  26. _lazy_ensure("firecrawl")
  27. except ImportError:
  28. pass
  29. except Exception as exc:
  30. raise ImportError(str(exc))
  31. from firecrawl import Firecrawl as _cls # noqa: WPS433 — deliberately lazy
  32. _FIRECRAWL_CLS_CACHE = _cls
  33. return _FIRECRAWL_CLS_CACHE
  34. class _FirecrawlProxy:
  35. """Callable proxy that looks like ``firecrawl.Firecrawl`` but imports lazily."""
  36. __slots__ = ()
  37. def __call__(self, *args: Any, **kwargs: Any) -> Any:
  38. return _load_firecrawl_cls()(*args, **kwargs)
  39. def __instancecheck__(self, obj: Any) -> bool:
  40. return isinstance(obj, _load_firecrawl_cls())
  41. def __repr__(self) -> str:
  42. return "<lazy firecrawl.Firecrawl proxy>"
  43. Firecrawl = _FirecrawlProxy()
  44. # --- Client construction (direct vs managed-gateway) ---------------------------
  45. def _wt():
  46. """Client cache slots live on tools.web_tools so tests that reset ``_firecrawl_client`` there see it."""
  47. import tools.web_tools as _mod
  48. return _mod
  49. def _env(name: str) -> str:
  50. from hermes_cli.config import get_env_value
  51. return (get_env_value(name) or "").strip()
  52. def _get_direct_firecrawl_config() -> Optional[tuple]:
  53. """Direct Firecrawl ``(mode, kwargs, cache_key)`` or None. ``mode`` is ``"sdk"`` (keyed / self-hosted) or
  54. ``"keyless"`` (explicit selection + no credentials → anonymous public cloud; the explicit selection is
  55. required so an unconfigured install never silently routes to it)."""
  56. api_key = _env("FIRECRAWL_API_KEY")
  57. api_url = _env("FIRECRAWL_API_URL").rstrip("/")
  58. if api_key or api_url:
  59. return "sdk", {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v}, ("direct", api_url or None, api_key or None)
  60. if _is_explicit_firecrawl_selection():
  61. return "keyless", {"api_url": _FIRECRAWL_CLOUD_API_URL}, ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None)
  62. return None
  63. def _is_explicit_firecrawl_selection() -> bool:
  64. from plugins.web.keyless_mcp import _web_config_selects
  65. return _web_config_selects("firecrawl")
  66. def _use_keyless_ring(capability: Optional[str] = None) -> bool:
  67. """Route via the keyless ring only with no direct credentials, when the managed Nous
  68. gateway isn't the selected path for *capability*, and the keyless tier isn't disabled or pinned paid."""
  69. if _env("FIRECRAWL_API_KEY") or _env("FIRECRAWL_API_URL"):
  70. return False
  71. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
  72. from plugins.web.keyless_mcp import use_keyless
  73. # Both probes are optional layers: a failing probe never blocks the ring.
  74. for probe in (lambda: read_web_capability_selection(capability) == NOUS_MANAGED_PROVIDER,
  75. lambda: _is_tool_gateway_ready() and not _is_explicit_firecrawl_selection()):
  76. try:
  77. if probe():
  78. return False
  79. except Exception:
  80. pass
  81. return use_keyless("firecrawl", "")
  82. class _KeylessFirecrawlClient:
  83. """Minimal REST client for Firecrawl's keyless cloud mode; duck-types the SDK's
  84. ``search`` / ``scrape`` and never sends an Authorization header."""
  85. def __init__(self, api_url: str = _FIRECRAWL_CLOUD_API_URL):
  86. self.api_url = api_url.rstrip("/")
  87. def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
  88. response = httpx.post(f"{self.api_url}{path}", json=payload, headers={"Content-Type": "application/json"}, timeout=60.0)
  89. try:
  90. response.raise_for_status()
  91. except httpx.HTTPStatusError as exc:
  92. # Keep the vendor's reason (e.g. "your IP address looks suspicious") and the ring's ``HTTP <code>`` shape.
  93. raise httpx.HTTPStatusError(f"HTTP {response.status_code}: {response.text.strip()[:300]}", request=exc.request, response=response) from exc
  94. return response.json()
  95. search = lambda self, *, query, limit=5: self._post("/v2/search", {"query": query, "limit": limit})
  96. def scrape(self, *, url, formats, timeout=None):
  97. # _scrape_one passes the SDK's server-side ``timeout`` (ms); the v2 REST payload takes the same field.
  98. payload = {"url": url, "formats": formats}
  99. if timeout is not None:
  100. payload["timeout"] = timeout
  101. return self._post("/v2/scrape", payload)
  102. def _get_firecrawl_gateway_url() -> str:
  103. return _gateway.build_vendor_gateway_url("firecrawl")
  104. def _is_tool_gateway_ready() -> bool:
  105. """True when gateway URL + Nous Subscriber token are available."""
  106. return _gateway.resolve_managed_tool_gateway("firecrawl", token_reader=_gateway.peek_nous_access_token) is not None
  107. def check_firecrawl_api_key() -> bool:
  108. """True when the route selected via ``hermes tools`` (or, on a never-configured
  109. install, either route) is usable."""
  110. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
  111. def _usable(selected) -> bool:
  112. if selected == NOUS_MANAGED_PROVIDER:
  113. return _is_tool_gateway_ready()
  114. return _get_direct_firecrawl_config() is not None or (selected is None and _is_tool_gateway_ready())
  115. # Search and extract can each pick their route (own key vs gateway); usable if either one is.
  116. return any(_usable(read_web_capability_selection(cap)) for cap in ("search", "extract"))
  117. def is_managed_route(capability: Optional[str] = None) -> bool:
  118. """True when Firecrawl calls for *capability* go through the Nous Tool Gateway rather than the user's
  119. own key / instance — the same decision :func:`_get_firecrawl_client` makes, without building a client."""
  120. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
  121. selected = read_web_capability_selection(capability)
  122. if selected is not None or _is_explicit_firecrawl_selection():
  123. return selected == NOUS_MANAGED_PROVIDER
  124. return _get_direct_firecrawl_config() is None and _is_tool_gateway_ready()
  125. def _firecrawl_backend_help_suffix() -> str:
  126. return ", or use the Nous Tool Gateway via your subscription (FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)" if _backend_helpers.managed_nous_tools_enabled() else ""
  127. def _get_firecrawl_client(capability: Optional[str] = None) -> Any:
  128. """Get or create the cached Firecrawl client. Strict selection semantics on the stored selection for
  129. *capability* (``web.<capability>_backend``, else the shared ``web`` selection): ``"nous"`` → managed Tool
  130. Gateway ONLY; any other stored backend → direct Firecrawl ONLY (never a silent managed fallback billed to
  131. Nous); never-configured → direct when present, else managed. Raises ValueError when the resolved path is
  132. unusable."""
  133. wt = _wt()
  134. from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection, selection_error
  135. selected = read_web_capability_selection(capability)
  136. direct_config = _get_direct_firecrawl_config()
  137. def _managed():
  138. gw = _gateway.resolve_managed_tool_gateway("firecrawl", token_reader=_gateway.read_nous_access_token)
  139. if gw is None:
  140. return None
  141. return "sdk", {"api_key": gw.nous_user_token, "api_url": gw.gateway_origin}, ("tool-gateway", gw.gateway_origin, gw.nous_user_token)
  142. def _unconfigured_message() -> str:
  143. message = "Web tools are not configured. Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL for a self-hosted Firecrawl instance."
  144. if _backend_helpers.managed_nous_tools_enabled():
  145. return message + " With your Nous subscription you can also use the Tool Gateway. run `hermes tools` and select Nous Subscription as the web provider."
  146. return message + " " + _backend_helpers.nous_tool_gateway_unavailable_message("managed Firecrawl web tools")
  147. # (resolved config, log detail, error message) per selection state; the message is built lazily.
  148. if selected == NOUS_MANAGED_PROVIDER:
  149. resolved, log, message = _managed(), "the Nous Subscription web selection is stored but the tool gateway is unavailable.", lambda: selection_error(
  150. "web", NOUS_MANAGED_PROVIDER, "the Nous Tool Gateway is not available (not entitled or unreachable)")
  151. elif selected is not None or _is_explicit_firecrawl_selection():
  152. # Stored vendor selection (shared name, or a per-capability key naming firecrawl): direct only (no
  153. # credentials → explicit selection unlocks keyless cloud mode). A per-capability key naming ANOTHER
  154. # vendor is not a firecrawl selection: the other capability's ladder reached firecrawl on its own,
  155. # so it resolves exactly like a never-configured install (#113017).
  156. resolved, log, message = direct_config, "direct Firecrawl selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set.", lambda: selection_error(
  157. "web", selected or "firecrawl", "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set")
  158. elif direct_config is not None:
  159. resolved = direct_config
  160. else: # never-configured web section: legacy managed fallback
  161. resolved, log, message = _managed(), "missing direct config and tool-gateway auth.", _unconfigured_message
  162. if resolved is None:
  163. logger.error("Firecrawl client initialization failed: %s", log)
  164. raise ValueError(message())
  165. client_mode, kwargs, client_config = resolved
  166. cached = getattr(wt, "_firecrawl_client", None)
  167. if cached is not None and getattr(wt, "_firecrawl_client_config", None) == client_config:
  168. return cached
  169. wt._firecrawl_client = _KeylessFirecrawlClient(api_url=kwargs["api_url"]) if client_mode == "keyless" else Firecrawl(**kwargs)
  170. wt._firecrawl_client_config = client_config
  171. return wt._firecrawl_client
  172. # --- Response shape normalization (SDK / direct / gateway differ) --------------
  173. def _to_plain_object(value: Any) -> Any:
  174. """SDK objects (pydantic ``model_dump`` / ``__dict__``) → plain data when possible."""
  175. if value is None or isinstance(value, (dict, list, str, int, float, bool)):
  176. return value
  177. for attr, convert in (("model_dump", lambda v: v.model_dump()), ("__dict__", lambda v: {k: x for k, x in v.__dict__.items() if not k.startswith("_")})):
  178. if hasattr(value, attr):
  179. try:
  180. return convert(value)
  181. except Exception:
  182. pass
  183. return value
  184. def _normalize_result_list(values: Any) -> list[dict[str, Any]]:
  185. return [p for p in map(_to_plain_object, values) if isinstance(p, dict)] if isinstance(values, list) else []
  186. def _extract_web_search_results(response: Any) -> list[dict[str, Any]]:
  187. """Search results across SDK/direct/gateway response shapes."""
  188. plain = _to_plain_object(response)
  189. if isinstance(plain, dict):
  190. data = plain.get("data")
  191. if isinstance(data, list):
  192. return _normalize_result_list(data)
  193. candidates = [data.get("web"), data.get("results")] if isinstance(data, dict) else []
  194. for candidate in candidates + [plain.get("web"), plain.get("results")]:
  195. normalized = _normalize_result_list(candidate)
  196. if normalized:
  197. return normalized
  198. if hasattr(response, "web"):
  199. return _normalize_result_list(getattr(response, "web", []))
  200. return []
  201. def _extract_scrape_payload(scrape_result: Any) -> dict[str, Any]:
  202. plain = _to_plain_object(scrape_result)
  203. if not isinstance(plain, dict):
  204. return {}
  205. return plain["data"] if isinstance(plain.get("data"), dict) else plain
  206. def _with_scrape_status(entry: dict[str, Any], metadata: Any) -> dict[str, Any]:
  207. """Flag a scrape whose target answered >= 400. The REST API reports ``statusCode``; the v4 SDK's
  208. ``model_dump`` (direct key and Tool Gateway) reports ``status_code``."""
  209. if not isinstance(metadata, dict):
  210. return entry
  211. return with_origin_status(entry, metadata.get("statusCode", metadata.get("status_code")), metadata.get("error"))
  212. def _error_entry(url: str, error: str, *, title: str = "", raw: bool = False, blocked: Optional[dict[str, Any]] = None) -> dict[str, Any]:
  213. """Per-URL extract failure. ``raw`` adds ``raw_content`` (post-scrape failures carry
  214. it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``."""
  215. policy = {"blocked_by_policy": {k: blocked[k] for k in ("host", "rule", "source")}} if blocked else {}
  216. return {"url": url, "title": title, "content": "", **({"raw_content": ""} if raw else {}), "error": error, **policy}
  217. _SCRAPE_TIMEOUT_MSG = "Scrape timed out after 60s — page may be too large or unresponsive. Try browser_exec instead."
  218. _UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address"
  219. def _final_url(metadata: dict[str, Any], requested: str) -> str:
  220. """The URL Firecrawl actually fetched. ``url`` is the scrape engine's post-redirect URL on every shape
  221. (REST, v4 SDK ``model_dump``, Tool Gateway); ``sourceURL`` / ``source_url`` echo the REQUESTED url."""
  222. return metadata.get("url") or requested
  223. async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> dict[str, Any]:
  224. """Scrape one URL (60s timeout) and re-check SSRF + website policy against the
  225. post-redirect URL. Never raises for scrape errors; returns an error entry instead."""
  226. if blocked := check_website_access(url):
  227. logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"])
  228. return _error_entry(url, blocked["message"], blocked=blocked)
  229. try:
  230. logger.info("Firecrawl scraping: %s", url)
  231. try:
  232. # Pass timeout (ms) to Firecrawl so the server-side deadline
  233. # matches our asyncio deadline. Without this the API uses its
  234. # 30 s default, causing SCRAPE_TIMEOUT before our 60 s
  235. # client-side wait expires. See #43272.
  236. scrape_result = await asyncio.wait_for(
  237. asyncio.to_thread(
  238. _get_firecrawl_client("extract").scrape,
  239. url=url,
  240. formats=formats,
  241. timeout=60_000,
  242. ),
  243. timeout=60,
  244. )
  245. except TimeoutError:
  246. logger.warning("Firecrawl scrape timed out for %s", url)
  247. return _error_entry(url, _SCRAPE_TIMEOUT_MSG)
  248. payload = _extract_scrape_payload(scrape_result)
  249. metadata = payload.get("metadata", {})
  250. # SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object).
  251. if not isinstance(metadata, dict):
  252. metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else getattr(metadata, "__dict__", {})
  253. # The v4 SDK names it ``source_url``; the extract cache keys a redirected page by ``sourceURL``.
  254. metadata.setdefault("sourceURL", metadata.get("source_url") or url)
  255. title, final_url = metadata.get("title", ""), _final_url(metadata, url)
  256. if not is_safe_url(final_url):
  257. logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
  258. return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True)
  259. if final_blocked := check_website_access(final_url):
  260. logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"])
  261. return _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked)
  262. markdown, html = payload.get("markdown"), payload.get("html")
  263. content = markdown if format == "markdown" or (format is None and markdown) else html or markdown or ""
  264. entry = {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}
  265. return _with_scrape_status(entry, metadata)
  266. except Exception as scrape_err:
  267. logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
  268. return _error_entry(url, str(scrape_err), raw=True)
  269. class FirecrawlWebSearchProvider(BaseWebSearchProvider):
  270. """Firecrawl search + extract provider with dual auth paths."""
  271. NAME = "firecrawl"
  272. DISPLAY_NAME = "Firecrawl"
  273. EXTRACT = True
  274. KEYLESS = True # default-on ring member unless pinned ``paid``
  275. def is_available(self) -> bool:
  276. return check_firecrawl_api_key()
  277. def search(self, query: str, limit: int = 5) -> dict[str, Any]:
  278. """Pre-flight errors (ValueError / ImportError) propagate so the dispatcher emits
  279. the legacy ``tool_error`` envelope; in-flight errors become failure dicts."""
  280. from tools.interrupt import is_interrupted
  281. if is_interrupted():
  282. return search_fail("Interrupted")
  283. if _use_keyless_ring("search"):
  284. return keyless_search("Firecrawl", "firecrawl", query, limit, logger)
  285. logger.info("Firecrawl search: '%s' (limit=%d)", query, limit)
  286. client = _get_firecrawl_client("search")
  287. try:
  288. web_results = _extract_web_search_results(client.search(query=query, limit=limit))
  289. logger.info("Firecrawl: found %d search results", len(web_results))
  290. return search_ok(web_results)
  291. except Exception as exc:
  292. logger.warning("Firecrawl search error: %s", exc)
  293. return search_fail(f"Firecrawl search failed: {exc}")
  294. async def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
  295. """Per-URL scrape; failures become items with an ``error`` field.
  296. ``format``: "markdown" | "html" | both (markdown preferred)."""
  297. from tools.interrupt import is_interrupted as _is_interrupted
  298. if _is_interrupted():
  299. return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
  300. if _use_keyless_ring("extract"):
  301. return await asyncio.to_thread(keyless_extract, "Firecrawl", "firecrawl", urls, logger)
  302. format = kwargs.get("format")
  303. formats = [format] if format in ("markdown", "html") else ["markdown", "html"]
  304. return [
  305. {"url": url, "error": "Interrupted", "title": ""} if _is_interrupted() else await _scrape_one(url, formats, format)
  306. for url in urls
  307. ]
  308. def get_setup_schema(self) -> dict[str, Any]:
  309. return setup_schema(
  310. "Firecrawl", "keyless/paid · optional gateway",
  311. "Full search + extract; supports keyless cloud, direct API, and Nous tool-gateway routing.",
  312. "FIRECRAWL_API_KEY", "Firecrawl API key (optional; blank = keyless cloud or self-hosted)", "https://docs.firecrawl.dev/introduction",
  313. )