| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389 |
- """Firecrawl web search + extract provider (direct SDK, keyless cloud, or Nous tool-gateway).
- Config: ``web.backend`` / ``web.search_backend`` / ``web.extract_backend: firecrawl``.
- Env: FIRECRAWL_API_KEY, FIRECRAWL_API_URL (self-hosted), FIRECRAWL_GATEWAY_URL / TOOL_GATEWAY_*.
- """
- from __future__ import annotations
- import asyncio
- import logging
- from typing import Any, Dict, List, Optional
- import httpx
- from plugins.web._common import BaseWebSearchProvider, keyless_extract, keyless_search, search_fail, search_ok, setup_schema, with_origin_status
- from tools import managed_tool_gateway as _gateway
- from tools import tool_backend_helpers as _backend_helpers
- from tools.url_safety import is_safe_url
- # Module-level (cheap import) so tests can monkeypatch the policy gate on this module.
- from tools.website_policy import check_website_access
- logger = logging.getLogger(__name__)
- _FIRECRAWL_CLOUD_API_URL = "https://api.firecrawl.dev"
- # The SDK costs ~200ms of imports on a cold CLI; defer to first use (tests patch ``Firecrawl`` here).
- _FIRECRAWL_CLS_CACHE: Optional[type] = None
- def _load_firecrawl_cls() -> type:
- """Import and cache ``firecrawl.Firecrawl`` (lazy_deps install hint → ImportError)."""
- global _FIRECRAWL_CLS_CACHE
- if _FIRECRAWL_CLS_CACHE is None:
- try:
- from pm import ensure_import as _lazy_ensure
- _lazy_ensure("firecrawl")
- except ImportError:
- pass
- except Exception as exc:
- raise ImportError(str(exc))
- from firecrawl import Firecrawl as _cls # noqa: WPS433 — deliberately lazy
- _FIRECRAWL_CLS_CACHE = _cls
- return _FIRECRAWL_CLS_CACHE
- class _FirecrawlProxy:
- """Callable proxy that looks like ``firecrawl.Firecrawl`` but imports lazily."""
- __slots__ = ()
- def __call__(self, *args: Any, **kwargs: Any) -> Any:
- return _load_firecrawl_cls()(*args, **kwargs)
- def __instancecheck__(self, obj: Any) -> bool:
- return isinstance(obj, _load_firecrawl_cls())
- def __repr__(self) -> str:
- return "<lazy firecrawl.Firecrawl proxy>"
- Firecrawl = _FirecrawlProxy()
- # --- Client construction (direct vs managed-gateway) ---------------------------
- def _wt():
- """Client cache slots live on tools.web_tools so tests that reset ``_firecrawl_client`` there see it."""
- import tools.web_tools as _mod
- return _mod
- def _env(name: str) -> str:
- from hermes_cli.config import get_env_value
- return (get_env_value(name) or "").strip()
- def _get_direct_firecrawl_config() -> Optional[tuple]:
- """Direct Firecrawl ``(mode, kwargs, cache_key)`` or None. ``mode`` is ``"sdk"`` (keyed / self-hosted) or
- ``"keyless"`` (explicit selection + no credentials → anonymous public cloud; the explicit selection is
- required so an unconfigured install never silently routes to it)."""
- api_key = _env("FIRECRAWL_API_KEY")
- api_url = _env("FIRECRAWL_API_URL").rstrip("/")
- if api_key or api_url:
- return "sdk", {k: v for k, v in (("api_key", api_key), ("api_url", api_url)) if v}, ("direct", api_url or None, api_key or None)
- if _is_explicit_firecrawl_selection():
- return "keyless", {"api_url": _FIRECRAWL_CLOUD_API_URL}, ("direct-keyless", _FIRECRAWL_CLOUD_API_URL, None)
- return None
- def _is_explicit_firecrawl_selection() -> bool:
- from plugins.web.keyless_mcp import _web_config_selects
- return _web_config_selects("firecrawl")
- def _use_keyless_ring(capability: Optional[str] = None) -> bool:
- """Route via the keyless ring only with no direct credentials, when the managed Nous
- gateway isn't the selected path for *capability*, and the keyless tier isn't disabled or pinned paid."""
- if _env("FIRECRAWL_API_KEY") or _env("FIRECRAWL_API_URL"):
- return False
- from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
- from plugins.web.keyless_mcp import use_keyless
- # Both probes are optional layers: a failing probe never blocks the ring.
- for probe in (lambda: read_web_capability_selection(capability) == NOUS_MANAGED_PROVIDER,
- lambda: _is_tool_gateway_ready() and not _is_explicit_firecrawl_selection()):
- try:
- if probe():
- return False
- except Exception:
- pass
- return use_keyless("firecrawl", "")
- class _KeylessFirecrawlClient:
- """Minimal REST client for Firecrawl's keyless cloud mode; duck-types the SDK's
- ``search`` / ``scrape`` and never sends an Authorization header."""
- def __init__(self, api_url: str = _FIRECRAWL_CLOUD_API_URL):
- self.api_url = api_url.rstrip("/")
- def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
- response = httpx.post(f"{self.api_url}{path}", json=payload, headers={"Content-Type": "application/json"}, timeout=60.0)
- try:
- response.raise_for_status()
- except httpx.HTTPStatusError as exc:
- # Keep the vendor's reason (e.g. "your IP address looks suspicious") and the ring's ``HTTP <code>`` shape.
- raise httpx.HTTPStatusError(f"HTTP {response.status_code}: {response.text.strip()[:300]}", request=exc.request, response=response) from exc
- return response.json()
- search = lambda self, *, query, limit=5: self._post("/v2/search", {"query": query, "limit": limit})
- def scrape(self, *, url, formats, timeout=None):
- # _scrape_one passes the SDK's server-side ``timeout`` (ms); the v2 REST payload takes the same field.
- payload = {"url": url, "formats": formats}
- if timeout is not None:
- payload["timeout"] = timeout
- return self._post("/v2/scrape", payload)
- def _get_firecrawl_gateway_url() -> str:
- return _gateway.build_vendor_gateway_url("firecrawl")
- def _is_tool_gateway_ready() -> bool:
- """True when gateway URL + Nous Subscriber token are available."""
- return _gateway.resolve_managed_tool_gateway("firecrawl", token_reader=_gateway.peek_nous_access_token) is not None
- def check_firecrawl_api_key() -> bool:
- """True when the route selected via ``hermes tools`` (or, on a never-configured
- install, either route) is usable."""
- from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
- def _usable(selected) -> bool:
- if selected == NOUS_MANAGED_PROVIDER:
- return _is_tool_gateway_ready()
- return _get_direct_firecrawl_config() is not None or (selected is None and _is_tool_gateway_ready())
- # Search and extract can each pick their route (own key vs gateway); usable if either one is.
- return any(_usable(read_web_capability_selection(cap)) for cap in ("search", "extract"))
- def is_managed_route(capability: Optional[str] = None) -> bool:
- """True when Firecrawl calls for *capability* go through the Nous Tool Gateway rather than the user's
- own key / instance — the same decision :func:`_get_firecrawl_client` makes, without building a client."""
- from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection
- selected = read_web_capability_selection(capability)
- if selected is not None or _is_explicit_firecrawl_selection():
- return selected == NOUS_MANAGED_PROVIDER
- return _get_direct_firecrawl_config() is None and _is_tool_gateway_ready()
- def _firecrawl_backend_help_suffix() -> str:
- return ", or use the Nous Tool Gateway via your subscription (FIRECRAWL_GATEWAY_URL or TOOL_GATEWAY_DOMAIN)" if _backend_helpers.managed_nous_tools_enabled() else ""
- def _get_firecrawl_client(capability: Optional[str] = None) -> Any:
- """Get or create the cached Firecrawl client. Strict selection semantics on the stored selection for
- *capability* (``web.<capability>_backend``, else the shared ``web`` selection): ``"nous"`` → managed Tool
- Gateway ONLY; any other stored backend → direct Firecrawl ONLY (never a silent managed fallback billed to
- Nous); never-configured → direct when present, else managed. Raises ValueError when the resolved path is
- unusable."""
- wt = _wt()
- from tools.tool_backend_helpers import NOUS_MANAGED_PROVIDER, read_web_capability_selection, selection_error
- selected = read_web_capability_selection(capability)
- direct_config = _get_direct_firecrawl_config()
- def _managed():
- gw = _gateway.resolve_managed_tool_gateway("firecrawl", token_reader=_gateway.read_nous_access_token)
- if gw is None:
- return None
- return "sdk", {"api_key": gw.nous_user_token, "api_url": gw.gateway_origin}, ("tool-gateway", gw.gateway_origin, gw.nous_user_token)
- def _unconfigured_message() -> str:
- message = "Web tools are not configured. Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL for a self-hosted Firecrawl instance."
- if _backend_helpers.managed_nous_tools_enabled():
- return message + " With your Nous subscription you can also use the Tool Gateway. run `hermes tools` and select Nous Subscription as the web provider."
- return message + " " + _backend_helpers.nous_tool_gateway_unavailable_message("managed Firecrawl web tools")
- # (resolved config, log detail, error message) per selection state; the message is built lazily.
- if selected == NOUS_MANAGED_PROVIDER:
- resolved, log, message = _managed(), "the Nous Subscription web selection is stored but the tool gateway is unavailable.", lambda: selection_error(
- "web", NOUS_MANAGED_PROVIDER, "the Nous Tool Gateway is not available (not entitled or unreachable)")
- elif selected is not None or _is_explicit_firecrawl_selection():
- # Stored vendor selection (shared name, or a per-capability key naming firecrawl): direct only (no
- # credentials → explicit selection unlocks keyless cloud mode). A per-capability key naming ANOTHER
- # vendor is not a firecrawl selection: the other capability's ladder reached firecrawl on its own,
- # so it resolves exactly like a never-configured install (#113017).
- resolved, log, message = direct_config, "direct Firecrawl selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set.", lambda: selection_error(
- "web", selected or "firecrawl", "neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is set")
- elif direct_config is not None:
- resolved = direct_config
- else: # never-configured web section: legacy managed fallback
- resolved, log, message = _managed(), "missing direct config and tool-gateway auth.", _unconfigured_message
- if resolved is None:
- logger.error("Firecrawl client initialization failed: %s", log)
- raise ValueError(message())
- client_mode, kwargs, client_config = resolved
- cached = getattr(wt, "_firecrawl_client", None)
- if cached is not None and getattr(wt, "_firecrawl_client_config", None) == client_config:
- return cached
- wt._firecrawl_client = _KeylessFirecrawlClient(api_url=kwargs["api_url"]) if client_mode == "keyless" else Firecrawl(**kwargs)
- wt._firecrawl_client_config = client_config
- return wt._firecrawl_client
- # --- Response shape normalization (SDK / direct / gateway differ) --------------
- def _to_plain_object(value: Any) -> Any:
- """SDK objects (pydantic ``model_dump`` / ``__dict__``) → plain data when possible."""
- if value is None or isinstance(value, (dict, list, str, int, float, bool)):
- return value
- for attr, convert in (("model_dump", lambda v: v.model_dump()), ("__dict__", lambda v: {k: x for k, x in v.__dict__.items() if not k.startswith("_")})):
- if hasattr(value, attr):
- try:
- return convert(value)
- except Exception:
- pass
- return value
- def _normalize_result_list(values: Any) -> list[dict[str, Any]]:
- return [p for p in map(_to_plain_object, values) if isinstance(p, dict)] if isinstance(values, list) else []
- def _extract_web_search_results(response: Any) -> list[dict[str, Any]]:
- """Search results across SDK/direct/gateway response shapes."""
- plain = _to_plain_object(response)
- if isinstance(plain, dict):
- data = plain.get("data")
- if isinstance(data, list):
- return _normalize_result_list(data)
- candidates = [data.get("web"), data.get("results")] if isinstance(data, dict) else []
- for candidate in candidates + [plain.get("web"), plain.get("results")]:
- normalized = _normalize_result_list(candidate)
- if normalized:
- return normalized
- if hasattr(response, "web"):
- return _normalize_result_list(getattr(response, "web", []))
- return []
- def _extract_scrape_payload(scrape_result: Any) -> dict[str, Any]:
- plain = _to_plain_object(scrape_result)
- if not isinstance(plain, dict):
- return {}
- return plain["data"] if isinstance(plain.get("data"), dict) else plain
- def _with_scrape_status(entry: dict[str, Any], metadata: Any) -> dict[str, Any]:
- """Flag a scrape whose target answered >= 400. The REST API reports ``statusCode``; the v4 SDK's
- ``model_dump`` (direct key and Tool Gateway) reports ``status_code``."""
- if not isinstance(metadata, dict):
- return entry
- return with_origin_status(entry, metadata.get("statusCode", metadata.get("status_code")), metadata.get("error"))
- def _error_entry(url: str, error: str, *, title: str = "", raw: bool = False, blocked: Optional[dict[str, Any]] = None) -> dict[str, Any]:
- """Per-URL extract failure. ``raw`` adds ``raw_content`` (post-scrape failures carry
- it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``."""
- policy = {"blocked_by_policy": {k: blocked[k] for k in ("host", "rule", "source")}} if blocked else {}
- return {"url": url, "title": title, "content": "", **({"raw_content": ""} if raw else {}), "error": error, **policy}
- _SCRAPE_TIMEOUT_MSG = "Scrape timed out after 60s — page may be too large or unresponsive. Try browser_exec instead."
- _UNSAFE_REDIRECT_MSG = "Blocked: URL targets a private or internal network address"
- def _final_url(metadata: dict[str, Any], requested: str) -> str:
- """The URL Firecrawl actually fetched. ``url`` is the scrape engine's post-redirect URL on every shape
- (REST, v4 SDK ``model_dump``, Tool Gateway); ``sourceURL`` / ``source_url`` echo the REQUESTED url."""
- return metadata.get("url") or requested
- async def _scrape_one(url: str, formats: list[str], format: Optional[str]) -> dict[str, Any]:
- """Scrape one URL (60s timeout) and re-check SSRF + website policy against the
- post-redirect URL. Never raises for scrape errors; returns an error entry instead."""
- if blocked := check_website_access(url):
- logger.info("Blocked web_extract for %s by rule %s", blocked["host"], blocked["rule"])
- return _error_entry(url, blocked["message"], blocked=blocked)
- try:
- logger.info("Firecrawl scraping: %s", url)
- try:
- # Pass timeout (ms) to Firecrawl so the server-side deadline
- # matches our asyncio deadline. Without this the API uses its
- # 30 s default, causing SCRAPE_TIMEOUT before our 60 s
- # client-side wait expires. See #43272.
- scrape_result = await asyncio.wait_for(
- asyncio.to_thread(
- _get_firecrawl_client("extract").scrape,
- url=url,
- formats=formats,
- timeout=60_000,
- ),
- timeout=60,
- )
- except TimeoutError:
- logger.warning("Firecrawl scrape timed out for %s", url)
- return _error_entry(url, _SCRAPE_TIMEOUT_MSG)
- payload = _extract_scrape_payload(scrape_result)
- metadata = payload.get("metadata", {})
- # SDK may return a typed object for metadata (raw __dict__ here, unlike _to_plain_object).
- if not isinstance(metadata, dict):
- metadata = metadata.model_dump() if hasattr(metadata, "model_dump") else getattr(metadata, "__dict__", {})
- # The v4 SDK names it ``source_url``; the extract cache keys a redirected page by ``sourceURL``.
- metadata.setdefault("sourceURL", metadata.get("source_url") or url)
- title, final_url = metadata.get("title", ""), _final_url(metadata, url)
- if not is_safe_url(final_url):
- logger.info("Blocked redirected web_extract for unsafe final URL: %s", final_url)
- return _error_entry(final_url, _UNSAFE_REDIRECT_MSG, title=title, raw=True)
- if final_blocked := check_website_access(final_url):
- logger.info("Blocked redirected web_extract for %s by rule %s", final_blocked["host"], final_blocked["rule"])
- return _error_entry(final_url, final_blocked["message"], title=title, raw=True, blocked=final_blocked)
- markdown, html = payload.get("markdown"), payload.get("html")
- content = markdown if format == "markdown" or (format is None and markdown) else html or markdown or ""
- entry = {"url": final_url, "title": title, "content": content, "raw_content": content, "metadata": metadata}
- return _with_scrape_status(entry, metadata)
- except Exception as scrape_err:
- logger.debug("Firecrawl scrape failed for %s: %s", url, scrape_err)
- return _error_entry(url, str(scrape_err), raw=True)
- class FirecrawlWebSearchProvider(BaseWebSearchProvider):
- """Firecrawl search + extract provider with dual auth paths."""
- NAME = "firecrawl"
- DISPLAY_NAME = "Firecrawl"
- EXTRACT = True
- KEYLESS = True # default-on ring member unless pinned ``paid``
- def is_available(self) -> bool:
- return check_firecrawl_api_key()
- def search(self, query: str, limit: int = 5) -> dict[str, Any]:
- """Pre-flight errors (ValueError / ImportError) propagate so the dispatcher emits
- the legacy ``tool_error`` envelope; in-flight errors become failure dicts."""
- from tools.interrupt import is_interrupted
- if is_interrupted():
- return search_fail("Interrupted")
- if _use_keyless_ring("search"):
- return keyless_search("Firecrawl", "firecrawl", query, limit, logger)
- logger.info("Firecrawl search: '%s' (limit=%d)", query, limit)
- client = _get_firecrawl_client("search")
- try:
- web_results = _extract_web_search_results(client.search(query=query, limit=limit))
- logger.info("Firecrawl: found %d search results", len(web_results))
- return search_ok(web_results)
- except Exception as exc:
- logger.warning("Firecrawl search error: %s", exc)
- return search_fail(f"Firecrawl search failed: {exc}")
- async def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
- """Per-URL scrape; failures become items with an ``error`` field.
- ``format``: "markdown" | "html" | both (markdown preferred)."""
- from tools.interrupt import is_interrupted as _is_interrupted
- if _is_interrupted():
- return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
- if _use_keyless_ring("extract"):
- return await asyncio.to_thread(keyless_extract, "Firecrawl", "firecrawl", urls, logger)
- format = kwargs.get("format")
- formats = [format] if format in ("markdown", "html") else ["markdown", "html"]
- return [
- {"url": url, "error": "Interrupted", "title": ""} if _is_interrupted() else await _scrape_one(url, formats, format)
- for url in urls
- ]
- def get_setup_schema(self) -> dict[str, Any]:
- return setup_schema(
- "Firecrawl", "keyless/paid · optional gateway",
- "Full search + extract; supports keyless cloud, direct API, and Nous tool-gateway routing.",
- "FIRECRAWL_API_KEY", "Firecrawl API key (optional; blank = keyless cloud or self-hosted)", "https://docs.firecrawl.dev/introduction",
- )
|