provider.py 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990
  1. """Parallel.ai web search (sync ``Parallel`` SDK) + async extract (``AsyncParallel``).
  2. Env: ``PARALLEL_API_KEY`` (https://parallel.ai), optional
  3. ``PARALLEL_SEARCH_MODE`` = agentic (default) | fast | one-shot.
  4. """
  5. from __future__ import annotations
  6. import asyncio
  7. import logging
  8. import os
  9. from typing import Any, Dict, List
  10. from plugins.web._common import (
  11. SEARCH_LIMIT_CAP, BaseWebSearchProvider, cached_sdk_client, document, keyless_extract, keyless_search,
  12. keyless_variant_schema, provider_env, run_extract_async, run_search, search_ok, use_keyless, vendor_error_entry, web_hit,
  13. )
  14. logger = logging.getLogger(__name__)
  15. _MISSING_KEY = "PARALLEL_API_KEY environment variable not set. Get your API key at https://parallel.ai"
  16. def _client(slot: str, cls_name: str) -> Any:
  17. def _factory(api_key: str) -> Any:
  18. import parallel # deliberately lazy
  19. return getattr(parallel, cls_name)(api_key=api_key)
  20. # The key is checked before the SDK is ensured (cached_sdk_client): without a key the
  21. # error names the key, not an install the user may not be allowed to run.
  22. return cached_sdk_client(slot, "PARALLEL_API_KEY", _MISSING_KEY, "parallel-web", _factory)
  23. def _get_sync_client() -> Any:
  24. return _client("_parallel_client", "Parallel")
  25. def _get_async_client() -> Any:
  26. return _client("_async_parallel_client", "AsyncParallel")
  27. def _resolve_search_mode() -> str:
  28. mode = os.getenv("PARALLEL_SEARCH_MODE", "agentic").lower().strip()
  29. return mode if mode in {"fast", "one-shot", "agentic"} else "agentic"
  30. class ParallelWebSearchProvider(BaseWebSearchProvider):
  31. """Parallel.ai search + async extract provider."""
  32. NAME = "parallel"
  33. DISPLAY_NAME = "Parallel"
  34. KEY_ENV = "PARALLEL_API_KEY"
  35. EXTRACT = True
  36. KEYLESS = True
  37. def search(self, query: str, limit: int = 5) -> dict[str, Any]:
  38. def _body() -> dict[str, Any]:
  39. if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
  40. return keyless_search("Parallel", "parallel", query, limit, logger)
  41. mode = _resolve_search_mode()
  42. logger.info("Parallel search: '%s' (mode=%s, limit=%d)", query, mode, limit)
  43. response = _get_sync_client().beta.search(search_queries=[query], objective=query, mode=mode, max_results=min(limit, SEARCH_LIMIT_CAP))
  44. return search_ok([
  45. web_hit(r.url or "", r.title or "", " ".join(r.excerpts or []), i + 1)
  46. for i, r in enumerate(response.results or [])
  47. ])
  48. return run_search("Parallel", logger, _body, sdk=True)
  49. async def extract(self, urls: list[str], **kwargs: Any) -> list[dict[str, Any]]:
  50. async def _body() -> list[dict[str, Any]]:
  51. if use_keyless("parallel", provider_env("PARALLEL_API_KEY")):
  52. # Keyless ring is blocking HTTP — hop off the event loop.
  53. return await asyncio.to_thread(keyless_extract, "Parallel", "parallel", urls, logger)
  54. logger.info("Parallel extract: %d URL(s)", len(urls))
  55. response = await _get_async_client().beta.extract(urls=urls, full_content=True)
  56. results = [document(r.url or "", r.title or "", r.full_content or "\n\n".join(r.excerpts or [])) for r in response.results or []]
  57. return results + [
  58. vendor_error_entry(e.url or "", status=getattr(e, "http_status_code", None), reason=e.error_type, body=e.content)
  59. for e in response.errors or []
  60. ]
  61. return await run_extract_async("Parallel", logger, urls, _body, sdk=True)
  62. def get_setup_schema(self) -> dict[str, Any]:
  63. return keyless_variant_schema(
  64. "Parallel", "PARALLEL_API_KEY", "https://parallel.ai",
  65. free_tag="Objective-tuned search + page extraction on Parallel's anonymous free tier. Rate-limited under burst load.",
  66. paid_tag="Objective-tuned search + parallel page extraction via the Parallel SDK. Unthrottled, guaranteed service.",
  67. )