keyless_mcp.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452
  1. """Keyless web search/extract via public free-tier endpoints (Exa, Parallel, Firecrawl, Keenable).
  2. Resolved strictly LAST — after every keyed backend, the managed gateway, ddgs and custom plugin
  3. providers — so it never pre-empts a deliberate setup. Privacy: no user identifiers are sent;
  4. Parallel gets a random per-process ``session_id`` (rate limiting only) and its optional
  5. ``model_name`` analytics field is deliberately omitted. Disable with ``web.keyless_fallback: false``.
  6. """
  7. from __future__ import annotations
  8. import json
  9. import logging
  10. import re
  11. import threading
  12. import uuid
  13. from typing import Any, Callable, Dict, List, Optional
  14. from plugins.web._common import (
  15. document as _page, exa_crawl_status, is_origin_http_error, keenable_origin_error, origin_http_error, page_error as _page_error, search_fail,
  16. search_ok, vendor_error_entry, web_hit as _row,
  17. )
  18. logger = logging.getLogger(__name__)
  19. EXA_MCP_URL = "https://mcp.exa.ai/mcp"
  20. PARALLEL_MCP_URL = "https://search.parallel.ai/mcp"
  21. KEENABLE_API_URL = "https://api.keenable.ai"
  22. _KEENABLE_TITLE = "hermes-agent"
  23. # Parallel free-tier rate-limit correlation id — random per process, never persisted.
  24. _SESSION_ID = uuid.uuid4().hex
  25. _TIMEOUT_SECONDS = 30
  26. class KeylessMCPError(RuntimeError):
  27. """A keyless MCP call failed (transport, rate limit, or tool error)."""
  28. _RATE_LIMIT_MARKERS = ("rate limit", "rate-limit", "ratelimit", "too many requests", "429", "quota exceeded", "slow down")
  29. # vendor -> (display label, env key, signup URL) for the standard failure hint.
  30. _VENDOR_HINTS = {
  31. "exa": ("Exa", "EXA_API_KEY", "https://exa.ai"), "parallel": ("Parallel", "PARALLEL_API_KEY", "https://parallel.ai"),
  32. "firecrawl": ("Firecrawl", "FIRECRAWL_API_KEY", "https://firecrawl.dev"), "keenable": ("Keenable", "KEENABLE_API_KEY", "https://keenable.ai"),
  33. }
  34. def _is_rate_limitish(message: str) -> bool:
  35. """Heuristic: does an error message look like free-tier throttling?"""
  36. return any(marker in (message or "").lower() for marker in _RATE_LIMIT_MARKERS)
  37. # A free tier that refuses this client (401/402/403: anonymous access revoked, IP reputation gate)
  38. # or errors server-side (5xx) fails every query from here, while another vendor may still serve it.
  39. # Anchored to the status the error starts with (after the ``Keyless <Vendor> search failed:`` prefix),
  40. # so a terminal error that echoes the query ("HTTP 400: invalid query 'http 503'") does not match.
  41. _VENDOR_REFUSAL_RE = re.compile(
  42. r"\s*(?:keyless\s+\w+\s+search\s+failed:\s*)?"
  43. r"(?:http(?:\s+status)?|status(?:\s+code)?|client\s+error|error\s+code)"
  44. r"\s*[:=']*\s*(?:40[123]|5\d\d)\b",
  45. re.IGNORECASE,
  46. )
  47. def _is_search_failover_eligible(message: str) -> bool:
  48. """Return whether another anonymous vendor may serve the search.
  49. Rate limits and structured HTTP 401/402/403/5xx vendor refusals are local to
  50. one free search endpoint. Free-text markers are deliberately ignored:
  51. vendors may echo the query in an otherwise terminal error.
  52. """
  53. return _is_rate_limitish(message) or bool(_VENDOR_REFUSAL_RE.match(message or ""))
  54. def _fail_msg(vendor: str, kind: str, exc: Any, *, other_backends: bool = True) -> str:
  55. label, env_key, site = _VENDOR_HINTS[vendor]
  56. alt = " or another web backend via `hermes tools`" if other_backends else ""
  57. return f"Keyless {label} {kind} failed: {exc}. Set {env_key} ({site}){alt} for reliable service."
  58. def _search(vendor: str, rows: Callable[[], list[dict[str, Any]]], catch: Any = (), fmt: Optional[Callable[[Exception], str]] = None) -> dict[str, Any]:
  59. """``search_ok(rows())``; :class:`KeylessMCPError` → standard vendor hint, exception
  60. types in ``catch`` → ``fmt(exc)``; anything else propagates."""
  61. try:
  62. return search_ok(rows())
  63. except KeylessMCPError as exc:
  64. return search_fail(_fail_msg(vendor, "search", exc))
  65. except catch as exc:
  66. return search_fail(fmt(exc))
  67. def _per_url(urls: list[str], fetch: Callable[[str], dict[str, Any]], vendor: str, catch: Any = Exception, hint: bool = False) -> list[dict[str, Any]]:
  68. """Per-URL extract loop: a ``catch`` failure becomes an error entry (``hint`` adds the ``hermes tools`` hint)."""
  69. def _one(url: str) -> dict[str, Any]:
  70. try:
  71. return fetch(url)
  72. except catch as exc:
  73. return _page_error(url, _fail_msg(vendor, "extract", exc, other_backends=hint))
  74. return [_one(u) for u in urls]
  75. # --- Tier / config ------------------------------------------------------------
  76. def keyless_enabled() -> bool:
  77. """Delegates to the registry so the ``web.keyless_fallback`` (default on) chokepoint lives with backend resolution."""
  78. try:
  79. from agent.web_search_registry import _keyless_tier_enabled
  80. return _keyless_tier_enabled()
  81. except Exception as exc:
  82. logger.debug("keyless_enabled(): registry helper unavailable: %s", exc)
  83. return True
  84. _BACKEND_KEYS = ("backend", "search_backend", "extract_backend")
  85. def _web_config_selects(name: str) -> bool:
  86. """True when any ``web.backend`` / ``search_backend`` / ``extract_backend`` names *name*."""
  87. import tools.web_tools as _wt
  88. web_cfg = _wt._load_web_config()
  89. return any((web_cfg.get(key) or "").lower().strip() == name for key in _BACKEND_KEYS)
  90. def provider_tier(name: str) -> str:
  91. """``web.provider_tier.<name>`` (``hermes tools`` Free/Paid rows): ``free``, ``paid``, or ``auto`` (anything else/unset)."""
  92. try:
  93. from hermes_cli.config import load_config
  94. tiers = (load_config().get("web") or {}).get("provider_tier") or {}
  95. value = str(tiers.get(name, "") or "").lower().strip()
  96. return value if value in ("free", "paid") else "auto"
  97. except Exception as exc:
  98. logger.debug("provider_tier(%r) config read failed: %s", name, exc)
  99. return "auto"
  100. def use_keyless(name: str, api_key: str) -> bool:
  101. """Single chokepoint for search + extract: ``free`` → keyless even with a key; ``paid`` → keyed even
  102. without one (the keyed path raises its usual missing-key error); ``auto`` → keyless only when no key + tier enabled."""
  103. tier = provider_tier(name)
  104. if tier in ("free", "paid"):
  105. return tier == "free"
  106. return not api_key and keyless_enabled()
  107. # --- MCP transport ------------------------------------------------------------
  108. def _parse_mcp_body(body: str) -> str:
  109. """First text content item from an MCP tools/call response — plain-JSON bodies
  110. (Parallel) or SSE ``data: {...}`` lines (Exa). Raises :class:`KeylessMCPError` for
  111. JSON-RPC errors and ``isError`` tool results (e.g. Exa's free-tier rate limit).
  112. SSE frames are split on the SSE line terminators (CRLF, CR, LF) only:
  113. ``str.splitlines()`` also breaks on U+0085 / U+2028 / U+2029, which legitimately
  114. occur inside CJK page text and would cut a ``data:`` line in two. A parsed envelope with no text is reported as such — it is
  115. the vendor's answer, not an unrecognized shape."""
  116. def _from_payload(payload: str) -> Optional[str]:
  117. payload = payload.strip()
  118. if not payload.startswith("{"):
  119. return None
  120. data = json.loads(payload)
  121. err = data.get("error")
  122. if err:
  123. raise KeylessMCPError(str(err.get("message") or err))
  124. result = data.get("result") or {}
  125. texts = [c.get("text", "") for c in result.get("content") or [] if isinstance(c, dict)]
  126. if result.get("isError"):
  127. raise KeylessMCPError(" ".join(t for t in texts if t) or "MCP tool call failed")
  128. return next((str(t) for t in texts if t), "")
  129. stripped = body.strip()
  130. candidates = [stripped] if stripped.startswith("{") else []
  131. candidates += [line[len("data: "):] for line in re.split(r"\r\n|\r|\n", body) if line.startswith("data: ")]
  132. envelope_seen = False
  133. for candidate in candidates:
  134. try:
  135. text = _from_payload(candidate)
  136. except json.JSONDecodeError:
  137. continue
  138. if text:
  139. return text
  140. envelope_seen = envelope_seen or text == ""
  141. raise KeylessMCPError("MCP response contained no text content" if envelope_seen else "Unrecognized MCP response shape")
  142. def _response_text(response: Any) -> str:
  143. """Body decoded with the declared charset, else UTF-8. JSON-RPC and SSE bodies are UTF-8 by
  144. spec, but ``text/event-stream`` carries no charset and ``requests`` then decodes ``.text`` as
  145. ISO-8859-1 — mojibake for every non-ASCII result and, for CJK, stray U+0085 line breaks that
  146. made the envelope unparseable."""
  147. from requests.utils import get_encoding_from_headers
  148. content_type = response.headers.get("Content-Type", "")
  149. declared = get_encoding_from_headers({"content-type": content_type}) if "charset=" in content_type.lower() else None
  150. return response.content.decode(declared or "utf-8", errors="replace")
  151. def mcp_call(url: str, tool: str, arguments: dict[str, Any], timeout: int = _TIMEOUT_SECONDS) -> str:
  152. """POST a JSON-RPC ``tools/call`` and return the text payload. Raises
  153. :class:`KeylessMCPError` on transport failures, non-2xx, JSON-RPC and tool errors."""
  154. import requests
  155. payload = {"jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": arguments}}
  156. headers = {"Content-Type": "application/json", "Accept": "application/json, text/event-stream", "User-Agent": "hermes-agent"}
  157. try:
  158. response = requests.post(url, json=payload, headers=headers, timeout=timeout)
  159. except requests.RequestException as exc:
  160. raise KeylessMCPError(f"request failed: {exc}") from exc
  161. if response.status_code >= 400:
  162. raise KeylessMCPError(f"HTTP {response.status_code}: {_response_text(response)[:300]}")
  163. return _parse_mcp_body(_response_text(response))
  164. # --- Parallel (search.parallel.ai) — JSON text payloads -----------------------
  165. def parallel_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
  166. def _rows() -> list[dict[str, Any]]:
  167. text = mcp_call(PARALLEL_MCP_URL, "web_search", {"objective": query, "search_queries": [query], "session_id": _SESSION_ID})
  168. results = json.loads(text).get("results") or []
  169. return [
  170. _row(r.get("url") or "", r.get("title") or "", " ".join(r.get("excerpts") or []), i + 1)
  171. for i, r in enumerate(results[:max(limit, 0)] if limit else results)
  172. ]
  173. return _search("parallel", _rows, (json.JSONDecodeError, TypeError, KeyError), lambda exc: f"Keyless Parallel search returned an unexpected payload: {exc}")
  174. def parallel_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
  175. try:
  176. data = json.loads(mcp_call(PARALLEL_MCP_URL, "web_fetch", {"urls": list(urls), "objective": "Full page content", "session_id": _SESSION_ID}))
  177. except (KeylessMCPError, json.JSONDecodeError, TypeError) as exc:
  178. message = _fail_msg("parallel", "extract", exc)
  179. return [_page_error(u, message) for u in urls]
  180. results = [
  181. _page(r.get("url") or "", r.get("title") or "", r.get("full_content") or r.get("content") or "\n\n".join(r.get("excerpts") or []))
  182. for r in data.get("results") or []
  183. ]
  184. results += [
  185. vendor_error_entry(e.get("url") or "", status=e.get("http_status_code"), reason=e.get("error_type"), body=e.get("content"))
  186. for e in data.get("errors") or []
  187. ]
  188. # URLs the endpoint silently dropped still get an error entry (per-URL contract).
  189. seen = {r["url"] for r in results}
  190. results.extend(_page_error(u, "no content returned") for u in urls if u not in seen)
  191. return results
  192. # --- Exa (mcp.exa.ai) — formatted plain-text payloads -------------------------
  193. def _after(line: str, prefix: str) -> str:
  194. return line[len(prefix):].strip()
  195. _EXA_LABELS = ("Title:", "URL:", "Highlights:", "Published:", "Author:")
  196. def _parse_exa_search_text(text: str, limit: int) -> list[dict[str, Any]]:
  197. """Parse Exa's ``---``-separated ``Title:/URL:/Published:/Author:/Highlights:`` blocks."""
  198. results: list[dict[str, Any]] = []
  199. for block in text.split("\n---\n"):
  200. title = url = ""
  201. highlight_lines: list[str] = []
  202. in_highlights = False
  203. for stripped in map(str.strip, block.splitlines()):
  204. if stripped.startswith("Title:"):
  205. title = _after(stripped, "Title:")
  206. elif stripped.startswith("URL:"):
  207. url = _after(stripped, "URL:")
  208. elif in_highlights and stripped and not stripped.startswith(_EXA_LABELS):
  209. highlight_lines.append(stripped)
  210. # Highlights run until the next labelled field.
  211. if stripped.startswith(_EXA_LABELS):
  212. in_highlights = stripped.startswith("Highlights:")
  213. if url:
  214. results.append(_row(url, title, " ".join(highlight_lines), len(results) + 1))
  215. if limit and len(results) >= limit:
  216. break
  217. return results
  218. def exa_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
  219. return _search("exa", lambda: _parse_exa_search_text(mcp_call(EXA_MCP_URL, "web_search_exa", {"query": query, "numResults": max(1, int(limit))}), limit))
  220. def exa_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
  221. """Called per-URL; the tool returns one combined text payload."""
  222. def _fetch(url: str) -> dict[str, Any]:
  223. try:
  224. text = mcp_call(EXA_MCP_URL, "web_fetch_exa", {"urls": [url]})
  225. except KeylessMCPError as exc:
  226. # ``...: CRAWL_HTTP_429`` is the page's answer, not Exa throttling this client.
  227. if origin := origin_http_error(exa_crawl_status(exc)):
  228. return _page_error(url, origin)
  229. raise
  230. # Title: first markdown H1 or ``Title:`` line, whichever comes first.
  231. titles = (_after(s, "# " if s.startswith("# ") else "Title:") for s in map(str.strip, text.splitlines()) if s.startswith(("# ", "Title:")))
  232. return _page(url, next(titles, ""), text)
  233. return _per_url(urls, _fetch, "exa", catch=KeylessMCPError, hint=True)
  234. # --- Firecrawl keyless (public cloud API, no auth header) ---------------------
  235. def firecrawl_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
  236. from plugins.web.firecrawl.provider import _KeylessFirecrawlClient, _extract_web_search_results
  237. rows = lambda: _extract_web_search_results(_KeylessFirecrawlClient().search(query=query, limit=limit))
  238. return _search("firecrawl", rows, Exception, lambda exc: _fail_msg("firecrawl", "search", exc))
  239. def firecrawl_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
  240. from plugins.web.firecrawl.provider import (
  241. _KeylessFirecrawlClient, _extract_scrape_payload, _final_url, _with_scrape_status,
  242. )
  243. client = _KeylessFirecrawlClient()
  244. def _fetch(url: str) -> dict[str, Any]:
  245. payload = _extract_scrape_payload(client.scrape(url=url, formats=["markdown"])) or {}
  246. metadata = payload.get("metadata")
  247. metadata = metadata if isinstance(metadata, dict) else {}
  248. content = payload.get("markdown") or payload.get("html") or ""
  249. # Report the post-redirect page as ``url`` so the shared dispatch re-gates it.
  250. page = _page(_final_url(metadata, url), metadata.get("title") or "", content, source_url=url)
  251. return _with_scrape_status(page, metadata)
  252. return _per_url(urls, _fetch, "firecrawl")
  253. # --- Keenable keyless (api.keenable.ai public endpoints) ----------------------
  254. def _keenable_request(method: str, path: str, **kwargs: Any) -> dict[str, Any]:
  255. """Call a Keenable public endpoint with the mandatory X-Keenable-Title app id."""
  256. import requests
  257. headers = {"X-Keenable-Title": _KEENABLE_TITLE}
  258. if method == "post":
  259. headers["Content-Type"] = "application/json"
  260. response = getattr(requests, method)(f"{KEENABLE_API_URL}{path}", headers=headers, timeout=_TIMEOUT_SECONDS, **kwargs)
  261. if response.status_code >= 400:
  262. body = _response_text(response).strip()
  263. raise KeylessMCPError(keenable_origin_error(response.status_code, body) or f"HTTP {response.status_code}: {body[:300]}")
  264. return response.json()
  265. def keenable_search_keyless(query: str, limit: int = 5) -> dict[str, Any]:
  266. def _rows() -> list[dict[str, Any]]:
  267. data = _keenable_request("post", "/v1/search/public", json={"query": query, "max_results": max(1, int(limit))})
  268. return [
  269. _row(r.get("url") or "", r.get("title") or "", r.get("snippet") or r.get("description") or "", i + 1)
  270. for i, r in enumerate(data.get("results") or [])
  271. ]
  272. return _search("keenable", _rows, Exception, lambda exc: f"Keyless Keenable search failed: {exc}.")
  273. def keenable_extract_keyless(urls: list[str]) -> list[dict[str, Any]]:
  274. def _fetch(url: str) -> dict[str, Any]:
  275. try:
  276. data = _keenable_request("get", "/v1/fetch/public", params={"url": url})
  277. except KeylessMCPError as exc:
  278. if is_origin_http_error(str(exc)): # the target's answer, relayed by Keenable
  279. return _page_error(url, str(exc))
  280. raise
  281. return _page(data.get("url") or url, data.get("title") or "", data.get("content") or "", source_url=url)
  282. return _per_url(urls, _fetch, "keenable")
  283. # --- Round-robin ring + next-in-line failover (rate-limited free tiers) -------
  284. _KEYLESS_RING = ("exa", "parallel", "firecrawl", "keenable")
  285. # Late-bound lookups (not bare references) so ``patch.object(keyless_mcp, "<vendor>_search_keyless")``
  286. # is honored at call time. Tests also ``setitem`` these dicts directly.
  287. _KEYLESS_SEARCHERS: dict[str, Callable[[str, int], dict[str, Any]]] = {v: (lambda query, limit, _v=v: globals()[f"{_v}_search_keyless"](query, limit)) for v in _KEYLESS_RING}
  288. _KEYLESS_EXTRACTORS: dict[str, Callable[[list[str]], list[dict[str, Any]]]] = {v: (lambda urls, _v=v: globals()[f"{_v}_extract_keyless"](urls)) for v in _KEYLESS_RING}
  289. # Per-process round-robin cursor, seeded by the random session id so the fleet
  290. # spreads across vendors; advances once per unpinned keyless request.
  291. _ring_lock = threading.Lock()
  292. _ring_cursor = int(_SESSION_ID, 16) % len(_KEYLESS_RING)
  293. def _vendor_pinned(name: str) -> bool:
  294. """True when config explicitly routes web traffic to *name* (backend keys or a
  295. ``free`` tier pin). A pinned vendor starts every keyless request."""
  296. if provider_tier(name) == "free":
  297. return True
  298. try:
  299. return _web_config_selects(name)
  300. except Exception as exc:
  301. logger.debug("_vendor_pinned(%r) config read failed: %s", name, exc)
  302. return False
  303. def _ring_order(name: str) -> list[str]:
  304. """Vendor walk order: pinned → start at *name* (its ring position fixes the failover
  305. succession); else round-robin from the cursor, advancing it per request. Vendors
  306. pinned ``paid`` are excluded (explicit paid opts their free endpoint out)."""
  307. global _ring_cursor
  308. if _vendor_pinned(name):
  309. start = _KEYLESS_RING.index(name) if name in _KEYLESS_RING else 0
  310. else:
  311. with _ring_lock:
  312. start = _ring_cursor
  313. _ring_cursor = (_ring_cursor + 1) % len(_KEYLESS_RING)
  314. ordered = _KEYLESS_RING[start:] + _KEYLESS_RING[:start]
  315. return [v for v in ordered if provider_tier(v) != "paid"]
  316. _ALL_PAID_MSG = "All keyless web providers are pinned to paid tiers."
  317. def _walk_ring(name: str, kind: str, call, throttled) -> tuple:
  318. """Call each vendor from :func:`_ring_order` until a result is not ``throttled``.
  319. Returns ``(order, vendor, result, exhausted)``; ``order`` is empty (result None)
  320. when every vendor is pinned paid."""
  321. order = _ring_order(name)
  322. vendor, result = None, None
  323. for i, vendor in enumerate(order):
  324. result = call(vendor)
  325. if not throttled(result):
  326. return order, vendor, result, False
  327. if i + 1 < len(order):
  328. logger.info("keyless %s %s unavailable; failing over to %s", vendor, kind, order[i + 1])
  329. return order, vendor, result, True
  330. def search_with_failover(name: str, query: str, limit: int = 5) -> dict[str, Any]:
  331. """Rate limits and vendor refusals (HTTP 401/402/403/5xx) advance to the next
  332. vendor, other errors stop the walk (a malformed query fails everywhere).
  333. ``data.served_by`` is set when the serving vendor differs from *name*."""
  334. def _throttled(result: dict[str, Any]) -> bool:
  335. return not result.get("success") and _is_search_failover_eligible(result.get("error", ""))
  336. order, vendor, result, exhausted = _walk_ring(name, "search", lambda v: _KEYLESS_SEARCHERS[v](query, limit), _throttled)
  337. if not order:
  338. return search_fail(_ALL_PAID_MSG)
  339. if exhausted:
  340. result["error"] = f"{result.get('error', '')} (all keyless vendors unavailable: {', '.join(order)})"
  341. elif result.get("success") and vendor != name:
  342. result.setdefault("data", {})["served_by"] = vendor
  343. return result
  344. def extract_with_failover(name: str, urls: list[str]) -> list[dict[str, Any]]:
  345. """Advances to the next ring vendor only when EVERY url in a batch comes
  346. back with a rate-limit-shaped error — partial failures and HTTP 403s can
  347. be page problems, not provider throttling, and return as-is. A target's own
  348. ``HTTP 429 from origin`` is the page's answer, not the vendor's quota."""
  349. def _throttled(error: str) -> bool:
  350. return bool(error) and not is_origin_http_error(error) and _is_rate_limitish(error)
  351. def _all_throttled(results: list[dict[str, Any]]) -> bool:
  352. return bool(results) and all(_throttled(r.get("error", "")) for r in results)
  353. order, _vendor, results, _exhausted = _walk_ring(name, "extract", lambda v: _KEYLESS_EXTRACTORS[v](list(urls)), _all_throttled)
  354. if not order:
  355. return [_page_error(u, _ALL_PAID_MSG) for u in urls]
  356. return results