_common.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317
  1. """Shared plumbing for the bundled web-search provider plugins.
  2. Every helper resolves patched-in-tests collaborators (``get_provider_env``,
  3. ``tools.interrupt``, ``plugins.web.keyless_mcp``, ``tools.web_tools`` client
  4. slots) lazily at call time so monkeypatching the source module keeps working.
  5. """
  6. from __future__ import annotations
  7. import json
  8. import logging
  9. import re
  10. from typing import Any, Awaitable, Callable, Dict, List, Optional
  11. import httpx
  12. from agent.web_search_provider import WebSearchProvider
  13. SEARCH_LIMIT_CAP = 20 # every vendor here caps max_results at 20 server-side
  14. def provider_env(name: str) -> str:
  15. """Config-aware env lookup (os.environ, then ~/.hermes/.env)."""
  16. from agent.web_search_provider import get_provider_env
  17. return get_provider_env(name)
  18. def use_keyless(name: str, api_key: str) -> bool:
  19. from plugins.web.keyless_mcp import use_keyless as _use_keyless
  20. return _use_keyless(name, api_key)
  21. def _interrupted() -> bool:
  22. from tools.interrupt import is_interrupted
  23. return is_interrupted()
  24. # --- Result shapes (key order is part of the contract — it reaches the model as JSON) ---
  25. def search_ok(web_results: list[dict[str, Any]]) -> dict[str, Any]:
  26. return {"success": True, "data": {"web": web_results}}
  27. def search_fail(error: str) -> dict[str, Any]:
  28. return {"success": False, "error": error}
  29. def web_hit(url: str, title: str, description: str, position: int) -> dict[str, Any]:
  30. return {"url": url, "title": title, "description": description, "position": position}
  31. def title_hit(title: str, url: str, description: str, position: int) -> dict[str, Any]:
  32. """Title-first row — the historical wire shape of brave/searxng/ddgs/tavily/xai."""
  33. return {"title": title, "url": url, "description": description, "position": position}
  34. def document(url: str, title: str, content: str, *, source_url: Optional[str] = None) -> dict[str, Any]:
  35. """Successful extract entry; ``raw_content`` mirrors ``content`` for the legacy pipeline."""
  36. return {
  37. "url": url, "title": title, "content": content, "raw_content": content,
  38. "metadata": {"sourceURL": url if source_url is None else source_url, "title": title},
  39. }
  40. def page_error(url: str, error: str) -> dict[str, Any]:
  41. return {"url": url, "title": "", "content": "", "error": error}
  42. def extract_fail(urls: list[str], error: str) -> list[dict[str, Any]]:
  43. return [page_error(u, error) for u in urls]
  44. # --- Origin HTTP status (the target page's answer, not the vendor API's) ---
  45. # Scraping vendors report the target's own status inside a successful API reply (Firecrawl
  46. # ``metadata.statusCode``, Parallel ``errors[].http_status_code``, Exa ``statuses[].error``).
  47. # Unchecked, a 403/404 page reads as a successful (often empty) extraction: the agent can't tell
  48. # "blocked" from "blank" and the error page is cached as content. The backend did its job, so an
  49. # origin error never triggers the keyless rescue (``is_origin_http_error``).
  50. _ORIGIN_HTTP_ERROR_RE = re.compile(r"HTTP \d{3} from origin(?::|$)")
  51. def is_origin_http_error(error: Any) -> bool:
  52. """True for an ``error`` string built by ``origin_http_error`` (the target page's own answer)."""
  53. return isinstance(error, str) and _ORIGIN_HTTP_ERROR_RE.match(error) is not None
  54. def origin_http_error(status: Any, reason: Any = None) -> Optional[str]:
  55. """``HTTP <status> from origin[: reason]`` when the target answered >= 400, else None."""
  56. if isinstance(status, bool):
  57. return None
  58. try:
  59. code = int(status)
  60. except (TypeError, ValueError):
  61. return None
  62. if code < 400:
  63. return None
  64. detail = str(reason or "").strip()
  65. return f"HTTP {code} from origin: {detail}" if detail else f"HTTP {code} from origin"
  66. # Vendors that fetch the page themselves also report its status in their own words: Exa
  67. # ``CRAWL_HTTP_<code>`` / ``CRAWL_NOT_FOUND``; Keenable mirrors the target's code as its own HTTP
  68. # status with a message about the target. Read as vendor errors, an origin 429 looked like the
  69. # vendor throttling us and walked the keyless ring.
  70. _EXA_CRAWL_STATUS_RE = re.compile(r"\bCRAWL_(?:HTTP_(\d{3})|(NOT_FOUND))\s*$")
  71. _KEENABLE_ORIGIN_MESSAGE_RE = re.compile(r"the (?:target server|page does not exist)\b", re.IGNORECASE)
  72. def exa_crawl_status(text: Any) -> Optional[int]:
  73. """The origin status an Exa crawl error tag ends with (``CRAWL_HTTP_429`` → 429, ``CRAWL_NOT_FOUND`` → 404)."""
  74. match = _EXA_CRAWL_STATUS_RE.search(str(text or ""))
  75. if match is None:
  76. return None
  77. return int(match.group(1)) if match.group(1) else 404
  78. def keenable_origin_error(status: Any, body: str) -> Optional[str]:
  79. """``HTTP <status> from origin: <message>`` when a Keenable fetch error describes the target's
  80. answer; None for Keenable's own refusals (its rate limit for this client, auth, 5xx)."""
  81. try:
  82. data = json.loads(body or "")
  83. except ValueError:
  84. return None
  85. message = str(data.get("message") or "").strip() if isinstance(data, dict) else ""
  86. return origin_http_error(status, message) if _KEENABLE_ORIGIN_MESSAGE_RE.match(message) else None
  87. def with_origin_status(entry: dict[str, Any], status: Any, reason: Any = None) -> dict[str, Any]:
  88. """Set ``error`` on an extract entry whose target answered >= 400; the error page body
  89. stays in ``content`` (it can still explain the refusal)."""
  90. if error := origin_http_error(status, reason):
  91. entry["error"] = error
  92. return entry
  93. def vendor_error_entry(url: str, *, status: Any = None, reason: Any = None, body: Any = None) -> dict[str, Any]:
  94. """Per-URL failure from a vendor's own error list. Names the origin status when the vendor
  95. reports one (body kept as content; ``reason`` is a vendor tag like ``http_error`` that adds
  96. nothing to the code); otherwise the vendor's body/reason is the error text."""
  97. if origin := origin_http_error(status):
  98. entry = {**page_error(url, origin), "content": str(body or "")}
  99. else:
  100. entry = page_error(url, str(body or reason or "extraction failed"))
  101. return {**entry, "metadata": {"sourceURL": url}}
  102. # --- Keyless ring hand-off (shared by exa / parallel / keenable) ---------------
  103. def keyless_search(display: str, name: str, query: str, limit: int, logger: logging.Logger) -> dict[str, Any]:
  104. from plugins.web.keyless_mcp import search_with_failover
  105. logger.info("%s keyless search: '%s' (limit=%d)", display, query, limit)
  106. return search_with_failover(name, query, limit)
  107. def keyless_extract(display: str, name: str, urls: list[str], logger: logging.Logger) -> list[dict[str, Any]]:
  108. from plugins.web.keyless_mcp import extract_with_failover
  109. logger.info("%s keyless extract: %d URL(s)", display, len(urls))
  110. return extract_with_failover(name, list(urls))
  111. # --- Guarded execution: interrupt check + uniform failure classification ---
  112. def _failure_message(vendor: str, kind: str, exc: Exception, logger: logging.Logger, *, sdk: bool, verbatim_value_error: bool) -> str:
  113. """``verbatim_value_error``: ValueError carries a pre-formatted message (missing key,
  114. HTTP body) and is returned as-is. ``sdk``: ImportError means the lazily installed
  115. vendor SDK is missing. Anything else is logged and wrapped."""
  116. if verbatim_value_error and isinstance(exc, ValueError):
  117. return str(exc)
  118. if sdk and isinstance(exc, ImportError):
  119. return f"{vendor} SDK not installed: {exc}"
  120. logger.warning("%s %s error: %s", vendor, kind, exc)
  121. return f"{vendor} {kind} failed: {exc}"
  122. def _guarded(vendor: str, kind: str, logger: logging.Logger, body: Callable[[], Any], interrupted: Any, fail: Callable[[str], Any], sdk: bool, vve: bool) -> Any:
  123. try:
  124. if _interrupted():
  125. return interrupted
  126. return body()
  127. except Exception as exc:
  128. return fail(_failure_message(vendor, kind, exc, logger, sdk=sdk, verbatim_value_error=vve))
  129. def run_search(vendor: str, logger: logging.Logger, body: Callable[[], dict[str, Any]], *, sdk: bool = False, verbatim_value_error: bool = True) -> dict[str, Any]:
  130. return _guarded(vendor, "search", logger, body, search_fail("Interrupted"), search_fail, sdk, verbatim_value_error)
  131. def _extract_interrupted(urls: list[str]) -> list[dict[str, Any]]:
  132. return [{"url": u, "error": "Interrupted", "title": ""} for u in urls]
  133. def run_extract(
  134. vendor: str, logger: logging.Logger, urls: list[str], body: Callable[[], list[dict[str, Any]]],
  135. *, sdk: bool = False, verbatim_value_error: bool = True,
  136. ) -> list[dict[str, Any]]:
  137. """Per-URL failures are returned as entries with ``error`` — never raised."""
  138. return _guarded(vendor, "extract", logger, body, _extract_interrupted(urls), lambda m: extract_fail(urls, m), sdk, verbatim_value_error)
  139. async def run_extract_async(
  140. vendor: str, logger: logging.Logger, urls: list[str], body: Callable[[], Awaitable[list[dict[str, Any]]]],
  141. *, sdk: bool = False, verbatim_value_error: bool = True,
  142. ) -> list[dict[str, Any]]:
  143. """Async twin of :func:`run_extract` (``body`` is awaited inside the guard)."""
  144. try:
  145. if _interrupted():
  146. return _extract_interrupted(urls)
  147. return await body()
  148. except Exception as exc:
  149. return extract_fail(urls, _failure_message(vendor, "extract", exc, logger, sdk=sdk, verbatim_value_error=verbatim_value_error))
  150. # --- HTTP + SDK client helpers ---
  151. def http_status_detail(response: Any) -> str:
  152. """Response body text for a >=400 reply, or ``HTTP <code>`` when the body is empty."""
  153. return (response.text or "").strip() or f"HTTP {response.status_code}"
  154. def http_get_json(
  155. label: str, url: str, *, params: dict[str, Any], headers: dict[str, str], timeout: int,
  156. logger: logging.Logger, reach_target: Optional[str] = None,
  157. ) -> tuple[Any, Optional[dict[str, Any]]]:
  158. """GET ``url`` and parse JSON → ``(data, None)`` or ``(None, failure_dict)``.
  159. ``reach_target`` overrides the "Could not reach ..." subject (SearXNG includes its URL)."""
  160. try:
  161. resp = httpx.get(url, params=params, headers=headers, timeout=timeout)
  162. resp.raise_for_status()
  163. except httpx.HTTPStatusError as exc:
  164. logger.warning("%s HTTP error: %s", label, exc)
  165. return None, search_fail(f"{label} returned HTTP {exc.response.status_code}")
  166. except httpx.RequestError as exc:
  167. logger.warning("%s request error: %s", label, exc)
  168. return None, search_fail(f"Could not reach {reach_target or label}: {exc}")
  169. try:
  170. return resp.json(), None
  171. except Exception as exc:
  172. logger.warning("%s response parse error: %s", label, exc)
  173. return None, search_fail(f"Could not parse {label} response as JSON")
  174. def titled_rows(raw_results: list[dict[str, Any]], description_key: str) -> list[dict[str, Any]]:
  175. """Brave/SearXNG row normalizer: ``str()`` every field, 1-based positions."""
  176. return [
  177. title_hit(str(r.get("title", "")), str(r.get("url", "")), str(r.get(description_key, "")), i + 1)
  178. for i, r in enumerate(raw_results)
  179. ]
  180. def lazy_ensure(extra: str) -> None:
  181. """Best-effort ``pm.ensure_import``: a missing-but-unavailable extra raises
  182. ``pm.InstallError``, re-raised as ImportError for the caller's own handling."""
  183. import pm
  184. try:
  185. pm.ensure_import(extra)
  186. except Exception as exc:
  187. raise ImportError(str(exc))
  188. def cached_sdk_client(slot: str, env_var: str, missing_key_error: str, feature: str, factory: Callable[[str], Any]) -> Any:
  189. """Lazy-build + cache a vendor SDK client on ``tools.web_tools.<slot>`` (so tests that
  190. reset ``tools.web_tools._<vendor>_client = None`` see fresh state). Raises ValueError
  191. when the key is unset."""
  192. import tools.web_tools as _wt
  193. # Resolved before the cache is consulted: the slot is one per process, but the key can change
  194. # under it (``/reload``, or each multiplexed profile's secret scope resolving its own key).
  195. api_key = provider_env(env_var)
  196. if not api_key:
  197. raise ValueError(missing_key_error)
  198. # (key, client) as one value so concurrent builds under different keys can never leave one
  199. # key recorded beside another key's client.
  200. cached = getattr(_wt, slot, None)
  201. if cached is not None and cached[0] == api_key:
  202. return cached[1]
  203. lazy_ensure(feature)
  204. client = factory(api_key)
  205. setattr(_wt, slot, (api_key, client))
  206. return client
  207. # --- Provider base ---
  208. class BaseWebSearchProvider(WebSearchProvider):
  209. """Subclasses set ``NAME`` / ``DISPLAY_NAME`` / ``KEY_ENV`` and flip ``EXTRACT`` / ``KEYLESS``.
  210. ``is_available`` deliberately ignores the keyless tier: otherwise the legacy preference walk
  211. would route users holding a key for a lower-priority backend onto this vendor's free tier.
  212. ``is_keyless_available`` is True for keyless vendors unless pinned ``web.provider_tier.<name>: paid``."""
  213. NAME: str = ""
  214. DISPLAY_NAME: str = ""
  215. KEY_ENV: str = ""
  216. EXTRACT: bool = False
  217. KEYLESS: bool = False
  218. name = property(lambda self: self.NAME)
  219. display_name = property(lambda self: self.DISPLAY_NAME)
  220. def is_available(self) -> bool:
  221. return bool(provider_env(self.KEY_ENV))
  222. def is_keyless_available(self) -> bool:
  223. from plugins.web.keyless_mcp import keyless_enabled, provider_tier
  224. return self.KEYLESS and keyless_enabled() and provider_tier(self.NAME) != "paid"
  225. def supports_search(self) -> bool:
  226. return True
  227. def supports_extract(self) -> bool:
  228. return self.EXTRACT
  229. def setup_schema(name: str, badge: str, tag: str, key_env: str = "", prompt: str = "", url: str = "", **extra: Any) -> dict[str, Any]:
  230. """``hermes tools`` picker entry; ``env_vars`` is empty when ``key_env`` is blank."""
  231. env_vars = [{"key": key_env, "prompt": prompt, "url": url}] if key_env else []
  232. return {"name": name, "badge": badge, "tag": tag, "env_vars": env_vars, **extra}
  233. def keyless_variant_schema(display: str, key_env: str, key_url: str, *, free_tag: str, paid_tag: str) -> dict[str, Any]:
  234. """Picker entry for a keyless-ring vendor with a paid variant."""
  235. paid = setup_schema(f"{display} · Paid (API key)", "paid", paid_tag, key_env, f"{display} API key", key_url, web_tier="paid")
  236. return setup_schema(f"{display} · Free (keyless)", "free · no key", free_tag, web_tier="free", variants=[paid])