| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124 |
- """Tavily web search + content extraction (``/search``, ``/extract``; sync httpx).
- Env: ``TAVILY_API_KEY`` (https://app.tavily.com/home, optional), ``TAVILY_BASE_URL``.
- Keyed requests use ``Authorization: Bearer``; without a key the request is
- keyless (``X-Tavily-Access-Mode: keyless``). Tavily is NOT in the zero-config
- keyless ring — keyless access is opt-in by selecting Tavily in ``hermes tools``.
- """
- from __future__ import annotations
- import logging
- from typing import Any, Dict, List, Optional
- import httpx
- from plugins.web._common import (
- SEARCH_LIMIT_CAP, BaseWebSearchProvider, document, extract_fail, http_status_detail, provider_env, run_extract,
- run_search, search_fail, search_ok, setup_schema, title_hit, use_keyless,
- )
- logger = logging.getLogger(__name__)
- _CLIENT_NAME = "hermes-agent"
- _SEARCH_PAYLOAD = {"include_raw_content": False, "include_images": False}
- def _tavily_headers(api_key: str) -> dict[str, str]:
- headers = {"X-Client-Name": _CLIENT_NAME}
- if api_key:
- headers["Authorization"] = f"Bearer {api_key}"
- else:
- headers["X-Tavily-Access-Mode"] = "keyless"
- return headers
- def _tavily_request(endpoint: str, payload: dict[str, Any], *, api_key: Optional[str] = None) -> dict[str, Any]:
- """POST to Tavily and return parsed JSON. ``api_key=None`` reads ``TAVILY_API_KEY``;
- pass ``""`` to force the keyless header even when a key exists
- (``web.provider_tier.tavily: free``). Non-2xx raises ValueError with the body so
- Tavily's rate-limit/upgrade text reaches the model."""
- if api_key is None:
- api_key = provider_env("TAVILY_API_KEY")
- base_url = provider_env("TAVILY_BASE_URL") or "https://api.tavily.com"
- url = f"{base_url}/{endpoint.lstrip('/')}"
- logger.info("Tavily %s request to %s", endpoint, url)
- response = httpx.post(url, json=payload, timeout=60, headers=_tavily_headers(api_key))
- if response.status_code >= 400:
- raise ValueError(http_status_detail(response))
- return response.json()
- def _normalize_tavily_search_results(response: dict[str, Any]) -> dict[str, Any]:
- return search_ok([
- title_hit(r.get("title", ""), r.get("url", ""), r.get("content", ""), i + 1)
- for i, r in enumerate(response.get("results", []))
- ])
- def _normalize_tavily_documents(response: dict[str, Any]) -> list[dict[str, Any]]:
- """Map ``/extract`` to documents without attributing missing URLs to a request."""
- documents = [
- document(r.get("url", ""), r.get("title", ""), r.get("raw_content", "") or r.get("content", ""))
- for r in response.get("results", [])
- ]
- documents += [_failed_document(f.get("url", ""), f.get("error", "extraction failed")) for f in response.get("failed_results", [])]
- documents += [_failed_document(str(u), "extraction failed") for u in response.get("failed_urls", [])]
- return documents
- def _failed_document(url: str, error: str) -> dict[str, Any]:
- return {"url": url, "title": "", "content": "", "raw_content": "", "error": error, "metadata": {"sourceURL": url}}
- def _missing_key_error(action: str) -> str:
- return f"TAVILY_API_KEY is not set. Get a key at https://app.tavily.com/home or select Tavily in `hermes tools` for opt-in keyless {action}."
- def _auth(action: str) -> tuple[Optional[str], Optional[str], str]:
- """``(request_key, missing_key_error, log_prefix)``: request key is ``""`` when forcing
- keyless, ``None`` when neither key nor keyless applies (``missing_key_error`` set)."""
- api_key = provider_env("TAVILY_API_KEY")
- force_keyless = use_keyless("tavily", api_key)
- if not force_keyless and not api_key:
- return None, _missing_key_error(action), ""
- return "" if force_keyless else api_key, None, "keyless " if force_keyless else ""
- class TavilyWebSearchProvider(BaseWebSearchProvider):
- """Tavily search + extract provider (keyed, or opt-in keyless)."""
- NAME = "tavily"
- DISPLAY_NAME = "Tavily"
- KEY_ENV = "TAVILY_API_KEY"
- EXTRACT = True
- KEYLESS = True
- def search(self, query: str, limit: int = 5) -> dict[str, Any]:
- def _body() -> dict[str, Any]:
- key, missing, prefix = _auth("search")
- if missing:
- return search_fail(missing)
- logger.info("Tavily %ssearch: '%s' (limit=%d)", prefix, query, limit)
- payload = {"query": query, "max_results": min(limit, SEARCH_LIMIT_CAP), **_SEARCH_PAYLOAD}
- return _normalize_tavily_search_results(_tavily_request("search", payload, api_key=key))
- return run_search("Tavily", logger, _body)
- def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
- def _body() -> list[dict[str, Any]]:
- key, missing, prefix = _auth("extract")
- if missing:
- return extract_fail(urls, missing)
- logger.info("Tavily %sextract: %d URL(s)", prefix, len(urls))
- raw = _tavily_request("extract", {"urls": urls, "include_images": False}, api_key=key)
- return _normalize_tavily_documents(raw)
- return run_extract("Tavily", logger, urls, _body)
- def get_setup_schema(self) -> dict[str, Any]:
- return setup_schema(
- "Tavily", "free · key optional", "Search + extract. Opt-in keyless; set TAVILY_API_KEY for higher limits.",
- "TAVILY_API_KEY", "Tavily API key (optional — keyless works when Tavily is selected)", "https://app.tavily.com/home",
- )
|