skills_hub_skillssh.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. """Skills Hub skills.sh adapter: catalog discovery via skills.sh, content via GitHub."""
  2. import hashlib
  3. import logging
  4. import re
  5. import time
  6. from typing import Any, Dict, List, Optional
  7. from tools.skills_hub_github import GitHubAuth, GitHubSource, _split_repo_id
  8. from tools.skills_hub_models import (
  9. SkillBundle, SkillMeta, SkillSource, _cache_metas, _cached_metas, _get_json, _get_text, _memo_json, hub,
  10. )
  11. logger = logging.getLogger("tools.skills_hub")
  12. def _strip_html(value: str) -> str:
  13. return re.sub(r'<[^>]+>', '', value)
  14. class SkillsShSource(SkillSource):
  15. """Discover skills via skills.sh and fetch content from the underlying GitHub repo."""
  16. BASE_URL = "https://skills.sh"
  17. SEARCH_URL = f"{BASE_URL}/api/search"
  18. # The sitemap is the real catalog source: the homepage only exposes a
  19. # ~200-entry featured strip; sitemap.xml points at sitemap-skills-N.xml
  20. # files (10k URLs each) covering the full ~20k+ catalog.
  21. SITEMAP_INDEX_URL = "https://www.skills.sh/sitemap.xml"
  22. # skills.sh serves per-skill sitemaps brotli-compressed and httpx's optional
  23. # brotlicffi backend has a streaming-decode bug on them; asking for gzip
  24. # only makes the server fall back to gzip/identity on every httpx install.
  25. _SITEMAP_HEADERS = {"Accept-Encoding": "gzip"}
  26. _SITEMAP_LOC_RE = re.compile(r"<loc>([^<]+)</loc>", re.IGNORECASE)
  27. _SITEMAP_SKILL_RE = re.compile(
  28. r"^https?://(?:www\.)?skills\.sh/(?P<owner>[^/]+)/(?P<repo>[^/]+)/(?P<skill>[^/]+)/?$", re.IGNORECASE,
  29. )
  30. _SKILL_LINK_RE = re.compile(r'href=["\']/(?P<id>(?!agents/|_next/|api/)[^"\'/]+/[^"\'/]+/[^"\'/]+)["\']')
  31. _INSTALL_CMD_RE = re.compile(
  32. r'npx\s+skills\s+add\s+(?P<repo>https?://github\.com/[^\s<]+|[^\s<]+)'
  33. r'(?:\s+--skill\s+(?P<skill>[^\s<]+))?',
  34. re.IGNORECASE,
  35. )
  36. _PAGE_H1_RE = re.compile(r'<h1[^>]*>(?P<title>.*?)</h1>', re.IGNORECASE | re.DOTALL)
  37. _PROSE_H1_RE = re.compile(
  38. r'<div[^>]*class=["\'][^"\']*prose[^"\']*["\'][^>]*>.*?<h1[^>]*>(?P<title>.*?)</h1>',
  39. re.IGNORECASE | re.DOTALL,
  40. )
  41. _PROSE_P_RE = re.compile(
  42. r'<div[^>]*class=["\'][^"\']*prose[^"\']*["\'][^>]*>.*?<p[^>]*>(?P<body>.*?)</p>',
  43. re.IGNORECASE | re.DOTALL,
  44. )
  45. _WEEKLY_INSTALLS_RE = re.compile(r'Weekly Installs.*?children\\":\\"(?P<count>[0-9.,Kk]+)\\"', re.DOTALL)
  46. _ID_PREFIX_ALIASES = ("skills-sh/", "skills.sh/", "skils-sh/", "skils.sh/")
  47. # Standard skill sub-paths tried before any tree/discovery walk.
  48. _STANDARD_BASE_PATHS = ("skills/", ".agents/skills/", ".claude/skills/")
  49. _strip_html = staticmethod(_strip_html)
  50. SOURCE_ID = "skills-sh"
  51. def __init__(self, auth: GitHubAuth):
  52. self.auth, self.github = auth, GitHubSource(auth=auth)
  53. def trust_level_for(self, identifier: str) -> str:
  54. return self.github.trust_level_for(self._normalize_identifier(identifier))
  55. def _meta(self, canonical: str, *, name: str, description: str, path: str,
  56. extra: Optional[dict[str, Any]] = None) -> SkillMeta:
  57. return SkillMeta(
  58. name=name, description=description, source="skills.sh", identifier=self._wrap_identifier(canonical),
  59. trust_level=self.github.trust_level_for(canonical), repo="/".join(canonical.split("/", 2)[:2]),
  60. path=path, extra=extra if extra is not None else {},
  61. )
  62. def _urls_for(self, canonical: str, repo: str) -> dict[str, str]:
  63. return {"detail_url": f"{self.BASE_URL}/{canonical}", "repo_url": f"https://github.com/{repo}"}
  64. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  65. if not query.strip():
  66. # Empty query = bulk catalog dump (build_skills_index.py) — walk the sitemap.
  67. return self._sitemap_catalog(limit)
  68. cache_key = f"skills_sh_search_{hashlib.md5(f'{query}|{limit}'.encode()).hexdigest()}"
  69. cached = _cached_metas(cache_key)
  70. if cached is not None:
  71. return cached[:limit]
  72. data = _get_json(self.SEARCH_URL, params={"q": query, "limit": limit})
  73. if data is None:
  74. return []
  75. items = data.get("skills", []) if isinstance(data, dict) else []
  76. if not isinstance(items, list):
  77. return []
  78. results = [m for m in map(self._meta_from_search_item, items[:limit]) if m]
  79. _cache_metas(cache_key, results)
  80. return results
  81. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  82. canonical = self._normalize_identifier(identifier)
  83. detail = self._fetch_detail_page(canonical)
  84. def _relabel(github_id: Optional[str]) -> Optional[SkillBundle]:
  85. bundle = self.github.fetch(github_id) if github_id else None
  86. if bundle:
  87. bundle.source, bundle.identifier = "skills.sh", self._wrap_identifier(canonical)
  88. if bundle.name == "skills": # a generic dir: install under the slug, as the index does
  89. bundle.name = canonical.rstrip("/").rsplit("/", 1)[-1]
  90. bundle.metadata.update(self._detail_to_metadata(canonical, detail))
  91. return bundle or None
  92. for candidate in self._candidate_identifiers(canonical):
  93. bundle = _relabel(candidate)
  94. if bundle:
  95. return bundle
  96. return _relabel(self._discover_identifier(canonical, detail=detail))
  97. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  98. canonical = self._normalize_identifier(identifier)
  99. detail = self._fetch_detail_page(canonical)
  100. meta = self._resolve_github_meta(canonical, detail=detail)
  101. return self._finalize_inspect_meta(meta, canonical, detail) if meta else None
  102. def _sitemap_catalog(self, limit: int) -> list[SkillMeta]:
  103. """Enumerate the full catalog via the sitemap (cached for the index TTL —
  104. ~2 MB of XML). Falls back to ``_featured_skills`` when unreachable/empty."""
  105. cache_key = "skills_sh_sitemap_v1"
  106. cached = _cached_metas(cache_key)
  107. if cached is not None:
  108. return cached[:limit] if limit > 0 else cached
  109. # Every hop goes through the hub's guarded GET: the index is a root of trust
  110. # that may redirect, and its <loc> entries are remote-party-controlled — a
  111. # hostile index could point a sitemap at an internal address.
  112. def _xml(url: str, timeout: int) -> Optional[str]:
  113. resp = hub()._guarded_http_get(url, timeout=timeout, headers=self._SITEMAP_HEADERS)
  114. return resp.text if resp is not None and resp.status_code == 200 else None
  115. # Step 1: sitemap index -> per-skill sitemap URLs.
  116. index_xml = _xml(self.SITEMAP_INDEX_URL, 20)
  117. skill_sitemap_urls = [m.group(1).strip() for m in self._SITEMAP_LOC_RE.finditer(index_xml or "")
  118. if "sitemap-skills" in m.group(1)]
  119. if not skill_sitemap_urls:
  120. return self._featured_skills(limit)
  121. # Step 2: collect canonical "owner/repo/skill" IDs from each sitemap. A shard
  122. # ``_xml`` returns None for is a hole, not an empty shard: retry it, and
  123. # if it stays dark return the partial slice without publishing it to the cache.
  124. seen, results, partial = set(), [], False
  125. for sitemap_url in skill_sitemap_urls:
  126. for attempt in range(1, self.CATALOG_PAGE_RETRIES + 1):
  127. xml = _xml(sitemap_url, 30)
  128. if xml is not None or attempt == self.CATALOG_PAGE_RETRIES:
  129. break
  130. time.sleep(min(2 ** attempt, 8))
  131. if xml is None:
  132. partial = True
  133. continue
  134. for loc_match in self._SITEMAP_LOC_RE.finditer(xml):
  135. m = self._SITEMAP_SKILL_RE.match(loc_match.group(1).strip())
  136. if not m:
  137. continue
  138. repo, skill = f"{m.group('owner')}/{m.group('repo')}", m.group("skill")
  139. canonical = f"{repo}/{skill}"
  140. if canonical not in seen:
  141. seen.add(canonical)
  142. results.append(self._meta(canonical, name=skill, description=f"Indexed by skills.sh from {repo}",
  143. path=skill, extra=self._urls_for(canonical, repo)))
  144. if not results:
  145. return self._featured_skills(limit)
  146. if not partial:
  147. _cache_metas(cache_key, results)
  148. return results[:limit] if limit > 0 else results
  149. def _featured_skills(self, limit: int) -> list[SkillMeta]:
  150. cache_key = "skills_sh_featured"
  151. cached = _cached_metas(cache_key)
  152. if cached is not None:
  153. return cached[:limit]
  154. html = _get_text(self.BASE_URL)
  155. if html is None:
  156. return []
  157. seen, results = set(), []
  158. for match in self._SKILL_LINK_RE.finditer(html):
  159. canonical = match.group("id")
  160. split = None if canonical in seen else _split_repo_id(canonical)
  161. seen.add(canonical)
  162. if split is None:
  163. continue
  164. repo, skill_path = split
  165. results.append(self._meta(canonical, name=skill_path.split("/")[-1],
  166. description=f"Featured on skills.sh from {repo}", path=skill_path))
  167. if len(results) >= limit:
  168. break
  169. _cache_metas(cache_key, results)
  170. return results
  171. def _meta_from_search_item(self, item: dict) -> Optional[SkillMeta]:
  172. if not isinstance(item, dict):
  173. return None
  174. canonical, repo, skill_path = item.get("id"), item.get("source"), item.get("skillId")
  175. if not isinstance(canonical, str) or canonical.count("/") < 2:
  176. if not (isinstance(repo, str) and isinstance(skill_path, str)):
  177. return None
  178. canonical = f"{repo}/{skill_path}"
  179. split = _split_repo_id(canonical)
  180. if split is None:
  181. return None
  182. repo, skill_path = split
  183. installs = item.get("installs")
  184. installs_label = f" · {int(installs):,} installs" if isinstance(installs, int) else ""
  185. return self._meta(
  186. canonical, name=str(item.get("name") or skill_path.split("/")[-1]),
  187. description=f"Indexed by skills.sh from {repo}{installs_label}", path=skill_path,
  188. extra={"installs": installs, **self._urls_for(canonical, repo)},
  189. )
  190. def _fetch_detail_page(self, identifier: str) -> Optional[dict]:
  191. def compute():
  192. html = _get_text(f"{self.BASE_URL}/{identifier}")
  193. return None if html is None else self._parse_detail_page(identifier, html) or None
  194. key = f"skills_sh_detail_{hashlib.md5(identifier.encode()).hexdigest()}"
  195. return _memo_json(key, compute, valid=lambda c: isinstance(c, dict))
  196. def _parse_detail_page(self, identifier: str, html: str) -> Optional[dict]:
  197. split = _split_repo_id(identifier)
  198. if split is None:
  199. return None
  200. repo, install_skill = split
  201. install_command, install_match = None, self._INSTALL_CMD_RE.search(html)
  202. if install_match:
  203. install_command = install_match.group(0).strip()
  204. install_skill = (install_match.group("skill") or install_skill).strip()
  205. repo = self._extract_repo_slug((install_match.group("repo") or "").strip()) or repo
  206. return {
  207. "repo": repo, "install_skill": install_skill,
  208. "page_title": self._extract_first_match(self._PAGE_H1_RE, html),
  209. "body_title": self._extract_first_match(self._PROSE_H1_RE, html),
  210. "body_summary": self._extract_first_match(self._PROSE_P_RE, html),
  211. "weekly_installs": self._extract_weekly_installs(html),
  212. "install_command": install_command,
  213. **self._urls_for(identifier, repo),
  214. "security_audits": self._extract_security_audits(html, identifier),
  215. }
  216. def _discover_identifier(self, identifier: str, detail: Optional[dict] = None) -> Optional[str]:
  217. split = _split_repo_id(identifier)
  218. if split is None:
  219. return None
  220. default_repo, skill_path = split
  221. repo = detail.get("repo", default_repo) if isinstance(detail, dict) else default_repo
  222. skill_token = skill_path.split("/")[-1]
  223. tokens = [skill_token]
  224. if isinstance(detail, dict):
  225. tokens.extend(detail.get(k, "") for k in ("install_skill", "page_title", "body_title"))
  226. def _match_in(base_path: str) -> Optional[str]:
  227. try:
  228. skills = self.github._list_skills_in_repo(repo, base_path)
  229. except Exception:
  230. return None
  231. return next((m.identifier for m in skills if self._matches_skill_tokens(m, tokens)), None)
  232. # One recursive tree lookup before brute-forcing every top-level dir
  233. # (avoids request bursts on categorized repos like borghei/claude-skills).
  234. found = (next((f for f in map(_match_in, self._STANDARD_BASE_PATHS) if f), None)
  235. or self.github._find_skill_in_repo_tree(repo, skill_token)
  236. or self.github._find_repo_root_skill(repo))
  237. if found:
  238. return found
  239. # Fallback: scan repo root for directories that might contain skills.
  240. try:
  241. entries = _get_json(f"https://api.github.com/repos/{repo}/contents/",
  242. headers=self.github.auth.get_headers(), timeout=15, follow_redirects=True)
  243. for entry in entries if isinstance(entries, list) else []:
  244. if entry.get("type") != "dir":
  245. continue
  246. dir_name = entry["name"]
  247. if dir_name.startswith((".", "_")) or dir_name in {"skills", ".agents", ".claude"}:
  248. continue
  249. meta = self.github.inspect(f"{repo}/{dir_name}/{skill_token}")
  250. if meta:
  251. return meta.identifier
  252. found = _match_in(dir_name + "/")
  253. if found:
  254. return found
  255. except Exception:
  256. pass
  257. return None
  258. def _resolve_github_meta(self, identifier: str, detail: Optional[dict] = None) -> Optional[SkillMeta]:
  259. for candidate in self._candidate_identifiers(identifier):
  260. meta = self.github.inspect(candidate)
  261. if meta:
  262. return meta
  263. resolved = self._discover_identifier(identifier, detail=detail)
  264. return self.github.inspect(resolved) if resolved else None
  265. def _finalize_inspect_meta(self, meta: SkillMeta, canonical: str, detail: Optional[dict]) -> SkillMeta:
  266. meta.source, meta.identifier = "skills.sh", self._wrap_identifier(canonical)
  267. meta.trust_level = self.trust_level_for(canonical)
  268. meta.extra = {**meta.extra, **self._detail_to_metadata(canonical, detail)}
  269. if isinstance(detail, dict):
  270. body_summary, weekly_installs = detail.get("body_summary"), detail.get("weekly_installs")
  271. if body_summary:
  272. meta.description = body_summary
  273. elif meta.description and weekly_installs:
  274. meta.description = f"{meta.description} · {weekly_installs} weekly installs on skills.sh"
  275. return meta
  276. @classmethod
  277. def _matches_skill_tokens(cls, meta: SkillMeta, skill_tokens: list[str]) -> bool:
  278. candidates = (cls._token_variants(meta.name) | cls._token_variants(meta.path)
  279. | cls._token_variants(meta.identifier.split("/", 2)[-1] if meta.identifier else None))
  280. return any(cls._token_variants(token) & candidates for token in skill_tokens)
  281. @staticmethod
  282. def _token_variants(value: Optional[str]) -> set[str]:
  283. if not value:
  284. return set()
  285. plain = _strip_html(str(value)).strip().strip("/").lower()
  286. if not plain:
  287. return set()
  288. base, sanitized = plain.split("/")[-1], re.sub(r'[^a-z0-9/_-]+', '-', plain).strip('-')
  289. tail = base.lstrip('@')
  290. variants = {
  291. plain, plain.replace("_", "-"), plain.replace("/", "-"), base, base.replace("_", "-"),
  292. sanitized, sanitized.replace("/", "-"), sanitized.split("/")[-1], tail, tail.replace("_", "-"),
  293. }
  294. return {v for v in variants if v}
  295. @staticmethod
  296. def _extract_repo_slug(repo_value: str) -> Optional[str]:
  297. parts = repo_value.strip().removeprefix("https://github.com/").strip("/").split("/")
  298. return f"{parts[0]}/{parts[1]}" if len(parts) >= 2 else None
  299. @staticmethod
  300. def _extract_first_match(pattern: re.Pattern, text: str) -> Optional[str]:
  301. match = pattern.search(text)
  302. value = next((group for group in match.groups() if group), None) if match else None
  303. return (_strip_html(value).strip() or None) if value is not None else None
  304. def _detail_to_metadata(self, canonical: str, detail: Optional[dict]) -> dict[str, Any]:
  305. parts = canonical.split("/", 2)
  306. metadata = {"detail_url": f"{self.BASE_URL}/{canonical}"}
  307. if len(parts) >= 2:
  308. metadata["repo_url"] = f"https://github.com/{parts[0]}/{parts[1]}"
  309. if isinstance(detail, dict):
  310. for key in ("weekly_installs", "install_command", "repo_url", "detail_url", "security_audits"):
  311. if detail.get(key):
  312. metadata[key] = detail[key]
  313. return metadata
  314. @classmethod
  315. def _extract_weekly_installs(cls, html: str) -> Optional[str]:
  316. match = cls._WEEKLY_INSTALLS_RE.search(html)
  317. return match.group("count") if match else None
  318. @staticmethod
  319. def _extract_security_audits(html: str, identifier: str) -> dict[str, str]:
  320. audits: dict[str, str] = {}
  321. for audit in ("agent-trust-hub", "socket", "snyk"):
  322. idx = html.find(f"/security/{audit}")
  323. match = re.search(r'(Pass|Warn|Fail)', html[idx:idx + 500], re.IGNORECASE) if idx != -1 else None
  324. if match:
  325. audits[audit] = match.group(1).title()
  326. return audits
  327. @classmethod
  328. def _normalize_identifier(cls, identifier: str) -> str:
  329. prefix = next((p for p in cls._ID_PREFIX_ALIASES if identifier.startswith(p)), "")
  330. return identifier[len(prefix):]
  331. @classmethod
  332. def _candidate_identifiers(cls, identifier: str) -> list[str]:
  333. split = _split_repo_id(identifier)
  334. if split is None:
  335. return [identifier]
  336. repo, path = split[0], split[1].lstrip("/")
  337. return list(dict.fromkeys([f"{repo}/{path}"] + [f"{repo}/{b}{path}" for b in cls._STANDARD_BASE_PATHS]))
  338. @staticmethod
  339. def _wrap_identifier(identifier: str) -> str:
  340. return f"skills-sh/{identifier}"