provider.py 3.4 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879
  1. """Exa web search + content extraction via the ``exa-py`` SDK (lazy-installed).
  2. Env: ``EXA_API_KEY`` (https://exa.ai). Both methods are sync — Exa's SDK is
  3. sync-only; the dispatcher threads extract when the caller is async.
  4. """
  5. from __future__ import annotations
  6. import logging
  7. from typing import Any, Dict, List
  8. from plugins.web._common import (
  9. BaseWebSearchProvider, cached_sdk_client, document, exa_crawl_status, keyless_extract, keyless_search, keyless_variant_schema,
  10. provider_env, run_extract, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,
  11. )
  12. logger = logging.getLogger(__name__)
  13. _MISSING_KEY = "EXA_API_KEY environment variable not set. Get your API key at https://exa.ai"
  14. def _get_exa_client() -> Any:
  15. def _factory(api_key: str) -> Any:
  16. from exa_py import Exa # deliberately lazy
  17. client = Exa(api_key=api_key)
  18. client.headers["x-exa-integration"] = "hermes-agent"
  19. return client
  20. return cached_sdk_client("_exa_client", "EXA_API_KEY", _MISSING_KEY, "exa", _factory)
  21. def _crawl_status(error: Any) -> Any:
  22. """The origin status of a failed ``statuses[]`` entry: ``http_status_code``, else the one its tag names."""
  23. return getattr(error, "http_status_code", None) or exa_crawl_status(getattr(error, "tag", None))
  24. class ExaWebSearchProvider(BaseWebSearchProvider):
  25. """Exa search + extract provider."""
  26. NAME = "exa"
  27. DISPLAY_NAME = "Exa"
  28. KEY_ENV = "EXA_API_KEY"
  29. EXTRACT = True
  30. KEYLESS = True
  31. def search(self, query: str, limit: int = 5) -> dict[str, Any]:
  32. def _body() -> dict[str, Any]:
  33. if use_keyless("exa", provider_env("EXA_API_KEY")):
  34. return keyless_search("Exa", "exa", query, limit, logger)
  35. logger.info("Exa search: '%s' (limit=%d)", query, limit)
  36. response = _get_exa_client().search(query, num_results=limit, contents={"highlights": True})
  37. return search_ok([
  38. web_hit(r.url or "", r.title or "", " ".join(r.highlights or []), i + 1)
  39. for i, r in enumerate(response.results or [])
  40. ])
  41. return run_search("Exa", logger, _body, sdk=True)
  42. def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
  43. def _body() -> list[dict[str, Any]]:
  44. if use_keyless("exa", provider_env("EXA_API_KEY")):
  45. return keyless_extract("Exa", "exa", urls, logger)
  46. logger.info("Exa extract: %d URL(s)", len(urls))
  47. response = _get_exa_client().get_contents(urls, text=True)
  48. results = [document(r.url or "", r.title or "", r.text or "") for r in response.results or []]
  49. # A URL Exa could not fetch is absent from ``results``; its reason is in ``statuses``.
  50. return results + [
  51. vendor_error_entry(s.id or "", status=_crawl_status(s.error), reason=getattr(s.error, "tag", None))
  52. for s in getattr(response, "statuses", None) or [] if s.status == "error"
  53. ]
  54. return run_extract("Exa", logger, urls, _body, sdk=True)
  55. def get_setup_schema(self) -> dict[str, Any]:
  56. return keyless_variant_schema(
  57. "Exa", "EXA_API_KEY", "https://exa.ai",
  58. free_tag="Semantic + neural web search with content extraction on Exa's anonymous free tier. Rate-limited under burst load.",
  59. paid_tag="Semantic + neural web search with content extraction via the Exa SDK. Unthrottled, guaranteed service.",
  60. )