provider.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124
  1. """Tavily web search + content extraction (``/search``, ``/extract``; sync httpx).
  2. Env: ``TAVILY_API_KEY`` (https://app.tavily.com/home, optional), ``TAVILY_BASE_URL``.
  3. Keyed requests use ``Authorization: Bearer``; without a key the request is
  4. keyless (``X-Tavily-Access-Mode: keyless``). Tavily is NOT in the zero-config
  5. keyless ring — keyless access is opt-in by selecting Tavily in ``hermes tools``.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. from typing import Any, Dict, List, Optional
  10. import httpx
  11. from plugins.web._common import (
  12. SEARCH_LIMIT_CAP, BaseWebSearchProvider, document, extract_fail, http_status_detail, provider_env, run_extract,
  13. run_search, search_fail, search_ok, setup_schema, title_hit, use_keyless,
  14. )
  15. logger = logging.getLogger(__name__)
  16. _CLIENT_NAME = "hermes-agent"
  17. _SEARCH_PAYLOAD = {"include_raw_content": False, "include_images": False}
  18. def _tavily_headers(api_key: str) -> dict[str, str]:
  19. headers = {"X-Client-Name": _CLIENT_NAME}
  20. if api_key:
  21. headers["Authorization"] = f"Bearer {api_key}"
  22. else:
  23. headers["X-Tavily-Access-Mode"] = "keyless"
  24. return headers
  25. def _tavily_request(endpoint: str, payload: dict[str, Any], *, api_key: Optional[str] = None) -> dict[str, Any]:
  26. """POST to Tavily and return parsed JSON. ``api_key=None`` reads ``TAVILY_API_KEY``;
  27. pass ``""`` to force the keyless header even when a key exists
  28. (``web.provider_tier.tavily: free``). Non-2xx raises ValueError with the body so
  29. Tavily's rate-limit/upgrade text reaches the model."""
  30. if api_key is None:
  31. api_key = provider_env("TAVILY_API_KEY")
  32. base_url = provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
  33. url = f"{base_url}/{endpoint.lstrip('/')}"
  34. logger.info("Tavily %s request to %s", endpoint, url)
  35. response = httpx.post(url, json=payload, timeout=60, headers=_tavily_headers(api_key))
  36. if response.status_code >= 400:
  37. raise ValueError(http_status_detail(response))
  38. return response.json()
  39. def _normalize_tavily_search_results(response: dict[str, Any]) -> dict[str, Any]:
  40. return search_ok([
  41. title_hit(r.get("title", ""), r.get("url", ""), r.get("content", ""), i + 1)
  42. for i, r in enumerate(response.get("results", []))
  43. ])
  44. def _normalize_tavily_documents(response: dict[str, Any]) -> list[dict[str, Any]]:
  45. """Map ``/extract`` to documents without attributing missing URLs to a request."""
  46. documents = [
  47. document(r.get("url", ""), r.get("title", ""), r.get("raw_content", "") or r.get("content", ""))
  48. for r in response.get("results", [])
  49. ]
  50. documents += [_failed_document(f.get("url", ""), f.get("error", "extraction failed")) for f in response.get("failed_results", [])]
  51. documents += [_failed_document(str(u), "extraction failed") for u in response.get("failed_urls", [])]
  52. return documents
  53. def _failed_document(url: str, error: str) -> dict[str, Any]:
  54. return {"url": url, "title": "", "content": "", "raw_content": "", "error": error, "metadata": {"sourceURL": url}}
  55. def _missing_key_error(action: str) -> str:
  56. return f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home or select Tavily in `hermes tools` for opt-in keyless {action}."
  57. def _auth(action: str) -> tuple[Optional[str], Optional[str], str]:
  58. """``(request_key, missing_key_error, log_prefix)``: request key is ``""`` when forcing
  59. keyless, ``None`` when neither key nor keyless applies (``missing_key_error`` set)."""
  60. api_key = provider_env("TAVILY_API_KEY")
  61. force_keyless = use_keyless("tavily", api_key)
  62. if not force_keyless and not api_key:
  63. return None, _missing_key_error(action), ""
  64. return "" if force_keyless else api_key, None, "keyless " if force_keyless else ""
  65. class TavilyWebSearchProvider(BaseWebSearchProvider):
  66. """Tavily search + extract provider (keyed, or opt-in keyless)."""
  67. NAME = "tavily"
  68. DISPLAY_NAME = "Tavily"
  69. KEY_ENV = "TAVILY_API_KEY"
  70. EXTRACT = True
  71. KEYLESS = True
  72. def search(self, query: str, limit: int = 5) -> dict[str, Any]:
  73. def _body() -> dict[str, Any]:
  74. key, missing, prefix = _auth("search")
  75. if missing:
  76. return search_fail(missing)
  77. logger.info("Tavily %ssearch: '%s' (limit=%d)", prefix, query, limit)
  78. payload = {"query": query, "max_results": min(limit, SEARCH_LIMIT_CAP), **_SEARCH_PAYLOAD}
  79. return _normalize_tavily_search_results(_tavily_request("search", payload, api_key=key))
  80. return run_search("Tavily", logger, _body)
  81. def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
  82. def _body() -> list[dict[str, Any]]:
  83. key, missing, prefix = _auth("extract")
  84. if missing:
  85. return extract_fail(urls, missing)
  86. logger.info("Tavily %sextract: %d URL(s)", prefix, len(urls))
  87. raw = _tavily_request("extract", {"urls": urls, "include_images": False}, api_key=key)
  88. return _normalize_tavily_documents(raw)
  89. return run_extract("Tavily", logger, urls, _body)
  90. def get_setup_schema(self) -> dict[str, Any]:
  91. return setup_schema(
  92. "Tavily", "free · key optional", "Search + extract. Opt-in keyless; set TAVILY_API_KEY for higher limits.",
  93. "TAVILY_API_KEY", "Tavily API key (optional — keyless works when Tavily is selected)", "https://app.tavily.com/home",
  94. )