web_search_registry.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204
  1. """Web Search Provider Registry.
  2. Populated by plugins via :meth:`PluginContext.register_web_search_provider`;
  3. consumed by the ``web_search`` / ``web_extract`` wrappers in :mod:`tools.web_tools`.
  4. Active selection, in precedence order (the ``supports_search`` /
  5. ``supports_extract`` capability filter applies at every step, so a search-only
  6. provider configured as ``web.extract_backend`` falls through):
  7. 1. ``web.search_backend`` / ``web.extract_backend``, then ``web.backend``.
  8. 2. The single capability-eligible provider that is registered AND available.
  9. 3. Legacy preference walk (``_LEGACY_PREFERENCE``) filtered by availability —
  10. the historic ``tools.web_tools._get_backend()`` order, so installs that never
  11. set a config key keep landing on the same provider.
  12. 4. Keyless free-tier walk (``_KEYLESS_PREFERENCE``), last resort.
  13. 5. ``None`` — the tool points the user at ``hermes tools``.
  14. """
  15. from __future__ import annotations
  16. import logging
  17. from typing import Optional
  18. from agent.provider_registry import ProviderRegistry, is_available_safe
  19. from agent.web_search_provider import WebSearchProvider
  20. logger = logging.getLogger(__name__)
  21. _registry: ProviderRegistry[WebSearchProvider] = ProviderRegistry(
  22. label="Web", provider_cls=WebSearchProvider, logger=logger,
  23. )
  24. _registry.export(globals())
  25. def _read_config_key(*path: str) -> Optional[str]:
  26. """Resolve a dotted config key from ``config.yaml``. Returns None on miss."""
  27. try:
  28. from hermes_cli.config import load_config_readonly
  29. cur = load_config_readonly()
  30. for segment in path:
  31. if not isinstance(cur, dict):
  32. return None
  33. cur = cur.get(segment)
  34. if isinstance(cur, str) and cur.strip():
  35. return cur.strip()
  36. except Exception as exc:
  37. logger.debug("Could not read config %s: %s", ".".join(path), exc)
  38. return None
  39. def _configured_backend(capability: str) -> Optional[str]:
  40. """``web.<capability>_backend`` (preferred) or ``web.backend`` (shared fallback). The managed ``nous``
  41. selection names no provider: it is served by Perplexity (search) and Firecrawl (extract)."""
  42. configured = _read_config_key("web", f"{capability}_backend") or _read_config_key("web", "backend")
  43. if configured and configured.lower() == "nous":
  44. return "perplexity" if capability == "search" else "firecrawl"
  45. return configured
  46. # Paid providers first so existing paid setups don't get downgraded to a free
  47. # tier on upgrade; filtered by ``is_available()`` at walk time.
  48. _LEGACY_PREFERENCE = ("firecrawl", "parallel", "tavily", "perplexity", "exa", "searxng", "brave-free", "ddgs")
  49. # Anonymous public free tiers (see plugins/web/keyless_mcp.py); strictly last
  50. # resort, i.e. zero web credentials and no importable ddgs. Unpinned keyless
  51. # traffic round-robins across the ring per request; an explicit `hermes tools`
  52. # pick bypasses this walk. Disable with ``web.keyless_fallback: false``.
  53. _KEYLESS_PREFERENCE = ("exa", "parallel", "firecrawl", "keenable")
  54. def _keyless_preference() -> tuple:
  55. """Keyless walk order, starting at the ring cursor in
  56. :mod:`plugins.web.keyless_mcp` so resolution and dispatch agree on which
  57. vendor a fresh install starts at; the rest follow in ring order."""
  58. try:
  59. from plugins.web.keyless_mcp import _KEYLESS_RING, _ring_cursor
  60. start = _ring_cursor % len(_KEYLESS_RING)
  61. return tuple(_KEYLESS_RING[start:] + _KEYLESS_RING[:start])
  62. except Exception as exc:
  63. logger.debug("keyless ring order unavailable: %s", exc)
  64. return _KEYLESS_PREFERENCE
  65. def _resolve(configured: Optional[str], *, capability: str) -> Optional[WebSearchProvider]:
  66. """Resolve the active provider for a capability ("search" | "extract").
  67. Rules, in order (see module docstring): explicit config wins even when
  68. ``is_available()`` is False (the dispatcher surfaces a precise
  69. "X_API_KEY is not set" error instead of a silent switch); then the single
  70. available capable provider; then the availability-filtered legacy walk;
  71. then the keyless free-tier walk; else None.
  72. """
  73. snapshot = _registry.merged()
  74. def _capable(p: WebSearchProvider) -> bool:
  75. return bool(
  76. p.supports_search() if capability == "search"
  77. else p.supports_extract() if capability == "extract" else False
  78. )
  79. def _available(p: WebSearchProvider) -> bool:
  80. return is_available_safe(p, logger, "provider %s.is_available() raised %s")
  81. if configured:
  82. provider = snapshot.get(configured)
  83. if provider is not None and _capable(provider):
  84. return provider
  85. if provider is None:
  86. logger.debug("web backend '%s' configured but not registered; falling back", configured)
  87. else:
  88. logger.debug(
  89. "web backend '%s' configured but does not support '%s'; falling back", configured, capability
  90. )
  91. # Fallbacks are availability-filtered so a registered-but-keyless provider
  92. # never becomes "active" on a fresh install.
  93. eligible = [p for p in snapshot.values() if _capable(p) and _available(p)]
  94. if len(eligible) == 1:
  95. return eligible[0]
  96. for legacy in _LEGACY_PREFERENCE:
  97. provider = snapshot.get(legacy)
  98. if provider is not None and provider in eligible:
  99. return provider
  100. # Keyless free tier (anonymous public MCP tiers) is last-resort only: it is
  101. # reachable solely when the legacy walk found nothing, never pre-empting a
  102. # keyed setup. Disabled via ``web.keyless_fallback: false``.
  103. if _keyless_tier_enabled():
  104. for name in _keyless_preference():
  105. provider = snapshot.get(name)
  106. if provider is None or not _capable(provider):
  107. continue
  108. try:
  109. if provider.is_keyless_available():
  110. return provider
  111. except Exception as exc:
  112. logger.debug("provider %s.is_keyless_available() raised %s", name, exc)
  113. return None
  114. def _keyless_tier_enabled() -> bool:
  115. """Read ``web.keyless_fallback`` from config.yaml (default: enabled)."""
  116. try:
  117. from hermes_cli.config import load_config
  118. web_cfg = load_config().get("web") or {}
  119. return bool(web_cfg.get("keyless_fallback", True))
  120. except Exception as exc:
  121. logger.debug("keyless_fallback config read failed: %s", exc)
  122. return True
  123. def _disabled_web_plugin_for(configured: Optional[str] = None, *, capability: Optional[str] = None) -> Optional[str]:
  124. """Plugin key of a *disabled* bundled web plugin that would have provided
  125. the configured backend (``web.<capability>_backend`` → ``web.backend``), or None.
  126. Lets the dispatcher say "re-enable web-firecrawl" instead of "No web extract
  127. provider configured". Resolved from config.yaml rather than the resolved
  128. backend because a disabled provider fails the availability gate and silently
  129. drops to the default. Bundled web plugins live under ``web/<vendor>`` with
  130. the provider name differing only by hyphen/underscore, so both are normalized.
  131. When a user sets ``web.extract_backend: firecrawl`` (or the search equivalent) but also lists
  132. ``web-firecrawl`` in ``plugins.disabled``, the provider never registers and the dispatcher would
  133. otherwise emit a misleading "No web extract provider configured. Set web.extract_backend to ..." error —
  134. even though the backend IS configured correctly. This helper detects that case so the dispatcher can
  135. point the user at the actual cause (issue #40190 follow-up: pi314's disabled-plugin symptom).
  136. """
  137. def _norm(s: str) -> str:
  138. return s.strip().lower().replace("-", "_")
  139. if not configured and capability in ("search", "extract"):
  140. configured = _configured_backend(capability)
  141. if not configured:
  142. return None
  143. want = _norm(configured)
  144. try:
  145. from hermes_cli.plugins import get_plugin_manager
  146. pm = get_plugin_manager()
  147. for key, loaded in pm._plugins.items():
  148. if (
  149. isinstance(key, str) and key.startswith("web/") and not loaded.enabled
  150. and loaded.error == "disabled via config" and _norm(key.split("/", 1)[1]) == want
  151. ):
  152. return key
  153. except Exception as exc:
  154. logger.debug("disabled-web-plugin lookup failed: %s", exc)
  155. return None
  156. def get_active_search_provider() -> Optional[WebSearchProvider]:
  157. """Resolve the currently-active web search provider."""
  158. return _resolve(_configured_backend("search"), capability="search")
  159. def get_active_extract_provider() -> Optional[WebSearchProvider]:
  160. """Resolve the currently-active web extract provider."""
  161. return _resolve(_configured_backend("extract"), capability="extract")