skills_hub_search.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. """Skills Hub discovery: the centralized Hermes index fetch (cached, stale-
  2. fallback), the source router, and parallel/unified search across source
  3. adapters.
  4. Split out of ``tools/skills_hub.py``; hub state (cache dir, ``TapsManager``, JSON
  5. cache reads) is still read from there at call time.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import httpx
  10. import json
  11. import time
  12. from pathlib import Path
  13. from typing import Any, Dict, List, Optional, Tuple
  14. from tools.skills_hub_clawhub import ClawHubSource
  15. from tools.skills_hub_github import GitHubAuth, GitHubSource, _filter_results_by_provider, _provider_filter_of
  16. from tools.skills_hub_models import SkillMeta, SkillSource, TRUST_RANK, _dedupe_by_trust, hub
  17. from tools.skills_hub_official import HermesIndexSource, OptionalSkillSource
  18. from tools.skills_hub_skillssh import SkillsShSource
  19. from tools.skills_hub_sources import BrowseShSource, LobeHubSource, UrlSource, WellKnownSkillSource
  20. # Log-record parity with the origin module.
  21. logger = logging.getLogger("tools.skills_hub")
  22. HERMES_INDEX_URL = "https://hermes-agent.nousresearch.com/docs/api/skills-index.json"
  23. HERMES_INDEX_TTL = 6 * 3600 # 6 hours
  24. def _hermes_index_cache_file() -> Path:
  25. from tools.skills_hub import _index_cache_dir
  26. return _index_cache_dir() / "hermes-index.json"
  27. def _load_hermes_index() -> Optional[dict]:
  28. """Fetch the centralized skills index (docs site, rebuilt daily), cached
  29. locally for HERMES_INDEX_TTL; on any failure serve the stale cache.
  30. Brotli is deliberately NOT negotiated: the index is tens of MB and httpx's
  31. streaming Brotli decoder (brotlicffi, pinned for Discord attachments) raises
  32. DecodingError on payloads this size — which surfaced as a silently empty
  33. Skills Hub. gzip/deflate first; the identity retry covers proxies that
  34. ignore the header and return Brotli anyway.
  35. """
  36. from tools.skills_hub import _read_json_if_fresh
  37. cache_file = _hermes_index_cache_file()
  38. cached = _read_json_if_fresh(cache_file, HERMES_INDEX_TTL)
  39. if cached is not None:
  40. return cached
  41. data = None
  42. for accept_encoding in ("gzip, deflate", "identity"):
  43. try:
  44. resp = hub()._skills_hub_http_get(
  45. HERMES_INDEX_URL, timeout=15, follow_redirects=True,
  46. headers={"Accept-Encoding": accept_encoding},
  47. )
  48. if resp.status_code != 200:
  49. logger.debug("Hermes index fetch returned %d", resp.status_code)
  50. return _load_stale_index_cache()
  51. data = resp.json()
  52. break
  53. except httpx.DecodingError as e:
  54. logger.debug("Hermes index decode failed (Accept-Encoding=%s): %s", accept_encoding, e)
  55. except (httpx.HTTPError, json.JSONDecodeError) as e:
  56. logger.debug("Hermes index fetch failed: %s", e)
  57. return _load_stale_index_cache()
  58. if not isinstance(data, dict) or "skills" not in data:
  59. return _load_stale_index_cache()
  60. try:
  61. cache_file.parent.mkdir(parents=True, exist_ok=True)
  62. cache_file.write_text(json.dumps(data), encoding="utf-8")
  63. except OSError:
  64. pass
  65. return data
  66. def _load_stale_index_cache() -> Optional[dict]:
  67. """Fall back to the cache regardless of age when the network fetch fails."""
  68. from tools.skills_hub import _read_json_if_fresh
  69. return _read_json_if_fresh(_hermes_index_cache_file(), float("inf"))
  70. # External API sources the centralized index already covers; skipped when the
  71. # index is available and no source filter is active (~70 GitHub calls/search
  72. # for unauthenticated users otherwise).
  73. _API_SOURCE_IDS = frozenset({"github", "skills-sh", "clawhub", "lobehub", "well-known"})
  74. # Consulted only when the index answered a non-empty query with nothing: the
  75. # index is rebuilt asynchronously and lags the registries, so a skill that is
  76. # live on skills.sh may not be in it yet. GitHub stays out — one miss (a typo)
  77. # would burn an unauthenticated user's whole hourly GitHub budget.
  78. _INDEX_MISS_FALLBACK_IDS = _API_SOURCE_IDS - {"github"}
  79. # Cap on the fallback pass. ClawHub can take minutes; without its own budget
  80. # every unfiltered miss (a typo) would stall CLI/TUI/dashboard for the callers'
  81. # full 30 s ``overall_timeout`` where the index alone answered instantly.
  82. _INDEX_MISS_FALLBACK_BUDGET = 8.0
  83. def create_source_router(auth: Optional[GitHubAuth] = None) -> list[SkillSource]:
  84. """All configured source adapters, in priority order."""
  85. from tools.skills_hub import TapsManager
  86. if auth is None:
  87. auth = GitHubAuth()
  88. return [
  89. OptionalSkillSource(auth=auth), # official optional skills (highest priority)
  90. HermesIndexSource(auth=auth), # centralized index (search + resolved install paths)
  91. SkillsShSource(auth=auth),
  92. WellKnownSkillSource(),
  93. UrlSource(), # direct HTTP(S) URL to a SKILL.md
  94. GitHubSource(auth=auth, extra_taps=TapsManager().list_taps()),
  95. ClawHubSource(),
  96. LobeHubSource(),
  97. BrowseShSource(), # browse.sh site-specific browser skills
  98. ]
  99. def _search_one_source(
  100. src: SkillSource, query: str, limit: int, provider_filter: str = "",
  101. ) -> tuple[str, list[SkillMeta]]:
  102. """Search a single source. Runs in a thread for parallelism."""
  103. try:
  104. # These sources mix providers in one catalog. Narrow before their top-N
  105. # cut so another provider cannot crowd every requested match out.
  106. if provider_filter and isinstance(src, (HermesIndexSource, GitHubSource)):
  107. return src.source_id(), src.search(query, limit=limit, provider_filter=provider_filter)
  108. return src.source_id(), src.search(query, limit=limit)
  109. except Exception as e:
  110. logger.debug("Search failed for %s: %s", src.source_id(), e)
  111. return src.source_id(), []
  112. def _select_active_sources(sources: list[SkillSource], source_filter: str) -> list[SkillSource]:
  113. """Sources to query for ``source_filter``.
  114. A provider filter (nvidia/openai/...) is not a source id — the data lives
  115. in the index/github source under ``extra.provider`` — so it selects like
  116. "all". Mixed-provider sources narrow before their top-N cut; the walker
  117. cuts every source's results. "official" is always queried alongside an
  118. explicit source filter.
  119. """
  120. effective = "all" if _provider_filter_of(source_filter) else source_filter
  121. index_available = effective == "all" and any(
  122. src.source_id() == "hermes-index" and getattr(src, "is_available", False) for src in sources
  123. )
  124. active: list[SkillSource] = []
  125. for src in sources:
  126. sid = src.source_id()
  127. if effective != "all" and sid != effective and sid != "official":
  128. continue
  129. if index_available and sid in _API_SOURCE_IDS:
  130. continue
  131. active.append(src)
  132. return active
  133. def _index_miss_fallback_sources(
  134. sources: list[SkillSource], active: list[SkillSource], query: str, source_counts: dict[str, int],
  135. provider_filter: str = "",
  136. ) -> list[SkillSource]:
  137. """Registries to consult after the index stood in for them and found nothing.
  138. Empty for a browse (no query), when the index was not consulted (no skip
  139. happened), when it returned matches, or when it did not answer at all
  140. (timed out: no budget is left and the registries would only be blamed as
  141. late without being asked). Also empty under a provider filter
  142. (``--source nvidia``): the fallback registries carry no ``extra.provider``,
  143. so their results would all be cut and the calls would only burn budget.
  144. """
  145. if not query.strip() or provider_filter or not any(src.source_id() == "hermes-index" for src in active):
  146. return []
  147. if source_counts.get("hermes-index") != 0:
  148. return []
  149. return [src for src in sources if src.source_id() in _INDEX_MISS_FALLBACK_IDS and src not in active]
  150. def _fan_out(
  151. active: list[SkillSource], query: str, per_source_limits: dict[str, int], provider_filter: str,
  152. deadline: float, on_source_done: Optional[Any], all_results: list[SkillMeta],
  153. source_counts: dict[str, int], timed_out_ids: list[str],
  154. ) -> None:
  155. """Query ``active`` in parallel until ``deadline`` (monotonic), merging into the accumulators."""
  156. from concurrent.futures import as_completed
  157. from tools.daemon_pool import DaemonThreadPoolExecutor
  158. remaining = deadline - time.monotonic()
  159. if remaining <= 0:
  160. timed_out_ids.extend(src.source_id() for src in active)
  161. return
  162. # Not a ``with`` block: its shutdown(wait=True) would block on a slow source
  163. # (ClawHub) for minutes and defeat ``overall_timeout``. Daemon workers so an
  164. # abandoned source cannot block interpreter exit either.
  165. pool = DaemonThreadPoolExecutor(max_workers=min(len(active), 8))
  166. futures = {
  167. pool.submit(
  168. _search_one_source, src, query, per_source_limits.get(src.source_id(), 50), provider_filter,
  169. ): src.source_id()
  170. for src in active
  171. }
  172. try:
  173. for fut in as_completed(futures, timeout=remaining):
  174. try:
  175. sid, results = fut.result(timeout=0)
  176. if provider_filter:
  177. # One owner for the merged provider cut: sources that cannot
  178. # filter per-tap (official, url, ...) are narrowed here, so
  179. # every caller (CLI, TUI gateway, dashboard) sees one rule.
  180. results = _filter_results_by_provider(results, provider_filter)
  181. source_counts[sid] = len(results)
  182. all_results.extend(results)
  183. if on_source_done:
  184. on_source_done(sid, len(results))
  185. except Exception:
  186. pass
  187. except TimeoutError:
  188. late = [futures[f] for f in futures if not f.done()]
  189. timed_out_ids.extend(late)
  190. if late:
  191. logger.debug("Skills browse timed out waiting for: %s", ", ".join(late))
  192. finally:
  193. pool.shutdown(wait=False, cancel_futures=True)
  194. def parallel_search_sources(
  195. sources: list[SkillSource], query: str = "", per_source_limits: Optional[dict[str, int]] = None,
  196. source_filter: str = "all", overall_timeout: float = 30, on_source_done: Optional[Any] = None,
  197. ) -> tuple[list[SkillMeta], dict[str, int], list[str]]:
  198. """Search all sources in parallel with an overall timeout.
  199. Returns ``(all_results, source_counts, timed_out_ids)``. *on_source_done*
  200. is an optional ``(source_id, count) -> None`` progress callback. Under a
  201. provider filter every source's results are narrowed before they are
  202. counted and merged, so callers need no provider logic of their own.
  203. When the centralized index stood in for the external registries and found
  204. nothing for a non-empty query, those registries are queried within the
  205. same ``overall_timeout`` — capped at ``_INDEX_MISS_FALLBACK_BUDGET`` so a
  206. slow registry cannot turn an instant index miss into a 30 s stall — so
  207. every caller (CLI, TUI gateway, dashboard) still finds skills the index
  208. has not picked up yet.
  209. """
  210. per_source_limits = per_source_limits or {}
  211. active = _select_active_sources(sources, source_filter)
  212. provider_filter = _provider_filter_of(source_filter)
  213. all_results: list[SkillMeta] = []
  214. source_counts: dict[str, int] = {}
  215. timed_out_ids: list[str] = []
  216. if not active:
  217. return all_results, source_counts, timed_out_ids
  218. deadline = time.monotonic() + overall_timeout
  219. _fan_out(active, query, per_source_limits, provider_filter, deadline, on_source_done,
  220. all_results, source_counts, timed_out_ids)
  221. fallback = _index_miss_fallback_sources(sources, active, query, source_counts, provider_filter)
  222. if fallback:
  223. fallback_deadline = min(deadline, time.monotonic() + _INDEX_MISS_FALLBACK_BUDGET)
  224. _fan_out(fallback, query, per_source_limits, provider_filter, fallback_deadline, on_source_done,
  225. all_results, source_counts, timed_out_ids)
  226. return all_results, source_counts, timed_out_ids
  227. def unified_search(query: str, sources: list[SkillSource],
  228. source_filter: str = "all", limit: int = 10) -> list[SkillMeta]:
  229. """Search all sources (in parallel) and merge results."""
  230. all_results, _, _ = parallel_search_sources(sources, query=query, source_filter=source_filter, overall_timeout=30)
  231. deduped = _dedupe_by_trust(all_results)
  232. # Stable-sort by trust before truncating so the limit cut never drops a
  233. # builtin/official entry because a high-volume community source finished
  234. # first; insertion order is preserved within each rank.
  235. deduped.sort(key=lambda r: -TRUST_RANK.get(r.trust_level, 0))
  236. return deduped[:limit]