skills_hub_sources.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430
  1. """Skills Hub adapters for well-known endpoints, direct URLs, LobeHub, and browse.sh."""
  2. import hashlib
  3. import json
  4. import logging
  5. import re
  6. from typing import Any, Dict, List, Optional, Union
  7. from urllib.parse import quote, urljoin, urlparse, urlunparse
  8. from tools.skills_hub_models import (
  9. GuardedFetchMixin, SkillBundle, SkillMeta, SkillSource, _first_matching, _get_json, _get_text,
  10. _hermes_tags, _memo_json, _parse_frontmatter, _referenced_support_paths,
  11. _validate_bundle_rel_path, _validate_skill_name, hub,
  12. )
  13. logger = logging.getLogger("tools.skills_hub")
  14. # --- Well-known Agent Skills endpoint source adapter ------------------------
  15. class WellKnownSkillSource(GuardedFetchMixin, SkillSource):
  16. """Read skills from a domain exposing /.well-known/skills/index.json."""
  17. SOURCE_ID = "well-known"
  18. BASE_PATH = "/.well-known/skills"
  19. def _meta(self, parsed: dict, skill_name: str, description: str, files: Any, **extra) -> SkillMeta:
  20. return SkillMeta(
  21. name=skill_name, description=description, source="well-known",
  22. identifier=self._wrap_identifier(parsed["base_url"], skill_name), trust_level="community",
  23. path=skill_name,
  24. extra={"index_url": parsed["index_url"], "base_url": parsed["base_url"], "files": files, **extra},
  25. )
  26. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  27. index_url = self._query_to_index_url(query)
  28. parsed = self._parse_index(index_url) if index_url else None
  29. if not parsed:
  30. return []
  31. results: list[SkillMeta] = []
  32. for entry in parsed["skills"][:limit]:
  33. name = entry.get("name")
  34. if not isinstance(name, str) or not name:
  35. continue
  36. files = entry.get("files", ["SKILL.md"])
  37. results.append(self._meta(parsed, name, str(entry.get("description", "")),
  38. files if isinstance(files, list) else ["SKILL.md"]))
  39. return results
  40. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  41. parsed = self._parse_identifier(identifier)
  42. entry = self._index_entry(parsed["index_url"], parsed["skill_name"]) if parsed else None
  43. skill_md = self._fetch_text(f"{parsed['skill_url']}/SKILL.md") if entry else None
  44. if skill_md is None:
  45. return None
  46. fm = _parse_frontmatter(skill_md)
  47. return self._meta(parsed, str(fm.get("name") or parsed["skill_name"]),
  48. str(fm.get("description") or entry.get("description") or ""),
  49. entry.get("files", ["SKILL.md"]), endpoint=parsed["skill_url"])
  50. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  51. parsed = self._parse_identifier(identifier)
  52. if not parsed:
  53. return None
  54. try:
  55. skill_name = _validate_skill_name(parsed["skill_name"])
  56. except ValueError:
  57. logger.warning("Well-known skill identifier contained unsafe skill name: %s", identifier)
  58. return None
  59. entry = self._index_entry(parsed["index_url"], parsed["skill_name"])
  60. if not entry:
  61. return None
  62. files = entry.get("files", ["SKILL.md"])
  63. if not isinstance(files, list) or not files:
  64. files = ["SKILL.md"]
  65. downloaded: dict[str, str] = {}
  66. for rel_path in files:
  67. if not isinstance(rel_path, str) or not rel_path:
  68. continue
  69. try:
  70. safe_rel_path = _validate_bundle_rel_path(rel_path)
  71. except ValueError:
  72. logger.warning("Well-known skill %s advertised unsafe file path: %r", identifier, rel_path)
  73. return None
  74. text = self._fetch_text(f"{parsed['skill_url']}/{safe_rel_path}")
  75. if text is None:
  76. return None
  77. downloaded[safe_rel_path] = text
  78. if "SKILL.md" not in downloaded:
  79. return None
  80. return SkillBundle(
  81. name=skill_name, files=downloaded, source="well-known",
  82. identifier=self._wrap_identifier(parsed["base_url"], skill_name), trust_level="community",
  83. metadata={"index_url": parsed["index_url"], "base_url": parsed["base_url"],
  84. "endpoint": parsed["skill_url"], "files": files},
  85. )
  86. def _query_to_index_url(self, query: str) -> Optional[str]:
  87. query = query.strip()
  88. if not query.startswith(("http://", "https://")):
  89. return None
  90. if query.endswith("/index.json"):
  91. return query
  92. if f"{self.BASE_PATH}/" in query:
  93. return query.split(f"{self.BASE_PATH}/", 1)[0] + f"{self.BASE_PATH}/index.json"
  94. return query.rstrip("/") + f"{self.BASE_PATH}/index.json"
  95. def _parse_identifier(self, identifier: str) -> Optional[dict]:
  96. raw = identifier.removeprefix("well-known:")
  97. if not raw.startswith(("http://", "https://")):
  98. return None
  99. parsed_url = urlparse(raw)
  100. clean_url = urlunparse(parsed_url._replace(fragment=""))
  101. if clean_url.endswith("/index.json"):
  102. if not parsed_url.fragment:
  103. return None
  104. base_url, skill_name = clean_url[:-len("/index.json")], parsed_url.fragment
  105. skill_url = f"{base_url}/{skill_name}"
  106. else:
  107. skill_url = clean_url[:-len("/SKILL.md")] if clean_url.endswith("/SKILL.md") else clean_url.rstrip("/")
  108. if f"{self.BASE_PATH}/" not in skill_url:
  109. return None
  110. base_url, skill_name = skill_url.rsplit("/", 1)
  111. return {"index_url": f"{base_url}/index.json", "base_url": base_url,
  112. "skill_name": skill_name, "skill_url": skill_url}
  113. def _parse_index(self, index_url: str) -> Optional[dict]:
  114. def compute():
  115. resp = hub()._guarded_http_get(index_url, timeout=20)
  116. if resp is None or resp.status_code != 200:
  117. return None
  118. try:
  119. data = resp.json()
  120. except json.JSONDecodeError:
  121. return None
  122. skills = data.get("skills", []) if isinstance(data, dict) else []
  123. if not isinstance(skills, list):
  124. return None
  125. return {"index_url": index_url, "base_url": index_url[:-len("/index.json")], "skills": skills}
  126. return _memo_json(f"well_known_index_{hashlib.md5(index_url.encode()).hexdigest()}", compute,
  127. valid=lambda c: isinstance(c, dict) and isinstance(c.get("skills"), list))
  128. def _index_entry(self, index_url: str, skill_name: str) -> Optional[dict]:
  129. parsed = self._parse_index(index_url)
  130. skills = parsed["skills"] if parsed else []
  131. return next((e for e in skills if isinstance(e, dict) and e.get("name") == skill_name), None)
  132. @staticmethod
  133. def _wrap_identifier(base_url: str, skill_name: str) -> str:
  134. return f"well-known:{base_url.rstrip('/')}/{skill_name}"
  135. # --- Direct URL source adapter ----------------------------------------------
  136. class UrlSource(GuardedFetchMixin, SkillSource):
  137. """Fetch SKILL.md plus explicitly referenced, allowlisted support files.
  138. The identifier IS the URL (``https://example.com/path/SKILL.md``). Bare URLs cannot enumerate a
  139. repository, so only exact references below references/templates/scripts/assets are fetched. The
  140. skill name comes from frontmatter ``name:`` (URL-slug fallback); trust is always ``community``.
  141. """
  142. SOURCE_ID = "url"
  143. # Skill names must look like identifiers: lowercase letters/digits with optional hyphens/underscores.
  144. # Blocks dangerous (``../evil``) AND useless (``SKILL``, ``README``, empty) candidates before they hit the disk.
  145. _VALID_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
  146. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  147. return [] # search is meaningless for a direct URL
  148. def _matches(self, identifier: str) -> bool:
  149. """Claim bare HTTP(S) URLs ending in ``.md``; leave wrapped identifiers
  150. and ``/.well-known/skills/`` URLs to their own adapters."""
  151. if not isinstance(identifier, str):
  152. return False
  153. ident = identifier.strip()
  154. if (not ident.lower().startswith(("http://", "https://")) or "/.well-known/skills/" in ident
  155. or ident.rstrip("/").endswith("/index.json")):
  156. return False
  157. try:
  158. return urlparse(ident).path.lower().endswith(".md")
  159. except ValueError:
  160. return False
  161. def _load(self, identifier: str):
  162. """``(url, text, frontmatter, resolved name)`` for a claimed identifier, else None."""
  163. if not self._matches(identifier):
  164. return None
  165. url = identifier.strip()
  166. text = self._fetch_text(url)
  167. if text is None:
  168. return None
  169. fm = _parse_frontmatter(text)
  170. return url, text, fm, self._resolve_skill_name(fm, url)
  171. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  172. loaded = self._load(identifier)
  173. if loaded is None:
  174. return None
  175. url, _text, fm, name = loaded
  176. raw_tags = _hermes_tags(fm)
  177. return SkillMeta(
  178. name=name or "", description=str(fm.get("description") or ""), source="url", identifier=url,
  179. trust_level="community", path=name or "",
  180. tags=[str(t) for t in raw_tags] if isinstance(raw_tags, list) else [],
  181. extra={"url": url, "awaiting_name": name is None},
  182. )
  183. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  184. loaded = self._load(identifier)
  185. if loaded is None:
  186. return None
  187. url, text, _fm, name = loaded
  188. referenced = _referenced_support_paths(text)
  189. if referenced is None:
  190. return None
  191. files: dict[str, str | bytes] = {"SKILL.md": text}
  192. base_url = url.rsplit("/", 1)[0] + "/"
  193. for rel_path in sorted(referenced):
  194. support_url = urljoin(base_url, quote(rel_path, safe="/"))
  195. if urlparse(support_url).netloc != urlparse(url).netloc:
  196. return None
  197. content = self._fetch_bytes(support_url)
  198. if content is None: # A 404ing support file shouldn't sink the whole install.
  199. logger.warning("URL skill %s: referenced support file %r could not be fetched from %s; skipping it",
  200. url, rel_path, support_url)
  201. continue
  202. files[rel_path] = content
  203. # When no name resolves, return the bundle with an empty name and ``awaiting_name=True``: ``do_install``
  204. # prompts on a TTY or refuses non-interactively, without re-downloading after the user picks a name.
  205. skill_name = ""
  206. if name is not None:
  207. try:
  208. skill_name = _validate_skill_name(name)
  209. except ValueError:
  210. logger.warning("URL skill %s produced unsafe skill name: %r", url, name)
  211. return None
  212. return SkillBundle(name=skill_name, files=files, source="url", identifier=url, trust_level="community",
  213. metadata={"url": url, "source_url": url, "awaiting_name": not skill_name})
  214. @classmethod
  215. def _is_valid_skill_name(cls, name: Optional[str]) -> bool:
  216. if not isinstance(name, str):
  217. return False
  218. candidate = name.strip().lower()
  219. return bool(candidate) and candidate not in {"skill", "readme", "index", "unnamed-skill"} and bool(
  220. cls._VALID_NAME_RE.match(candidate))
  221. @classmethod
  222. def _resolve_skill_name(cls, fm: dict, url: str) -> Optional[str]:
  223. """Frontmatter ``name:`` when valid, else a URL-slug candidate (``.../<name>/SKILL.md`` -> ``<name>``,
  224. ``.../<name>.md`` -> ``<name>``). None when nothing usable — the CLI then prompts or refuses rather
  225. than auto-naming something like ``SKILL``."""
  226. fm_name = fm.get("name") if isinstance(fm, dict) else None
  227. if isinstance(fm_name, str) and cls._is_valid_skill_name(fm_name):
  228. return fm_name.strip()
  229. try:
  230. path = urlparse(url).path
  231. except ValueError:
  232. return None
  233. parts = [p for p in path.split("/") if p]
  234. if len(parts) >= 2 and parts[-1].lower() == "skill.md" and cls._is_valid_skill_name(parts[-2]):
  235. return parts[-2]
  236. candidate = re.sub(r"\.md$", "", parts[-1], flags=re.IGNORECASE) if parts else ""
  237. return candidate if cls._is_valid_skill_name(candidate) else None
  238. # --- LobeHub source adapter -------------------------------------------------
  239. class LobeHubSource(SkillSource):
  240. """LobeHub agent marketplace (14,500+ system-prompt agents, converted to
  241. SKILL.md on fetch). Data lives in GitHub: lobehub/lobe-chat-agents."""
  242. SOURCE_ID = "lobehub"
  243. INDEX_URL = "https://chat-agents.lobehub.com/index.json"
  244. def _agents(self) -> Optional[list]:
  245. index = self._fetch_index()
  246. agents = (index.get("agents", index) if isinstance(index, dict) else index) if index else None
  247. return agents if isinstance(agents, list) else None
  248. @staticmethod
  249. def _agent_id(identifier: str) -> str:
  250. return identifier.split("/", 1)[-1] if identifier.startswith("lobehub/") else identifier
  251. @staticmethod
  252. def _agent_meta(agent: dict, name: str, description: str) -> SkillMeta:
  253. tags = agent.get("meta", agent).get("tags", [])
  254. return SkillMeta(name=name, description=description, source="lobehub", identifier=f"lobehub/{name}",
  255. trust_level="community", tags=tags if isinstance(tags, list) else [])
  256. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  257. agents = self._agents()
  258. if agents is None:
  259. return []
  260. def fields(agent):
  261. meta = agent.get("meta", agent)
  262. tags = meta.get("tags", [])
  263. return (meta.get("title", agent.get("identifier", "")), meta.get("description", ""),
  264. tags if isinstance(tags, list) else "")
  265. def to_meta(agent):
  266. meta = agent.get("meta", agent)
  267. title = meta.get("title", agent.get("identifier", ""))
  268. identifier = agent.get("identifier", title.lower().replace(" ", "-"))
  269. return self._agent_meta(agent, identifier, meta.get("description", "")[:200])
  270. return _first_matching(query.lower(), agents, fields, to_meta, limit)
  271. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  272. agent_id = self._agent_id(identifier)
  273. agent_data = self._fetch_agent(agent_id)
  274. if not agent_data:
  275. return None
  276. return SkillBundle(name=agent_id, files={"SKILL.md": self._convert_to_skill_md(agent_data)}, source="lobehub",
  277. identifier=f"lobehub/{agent_id}", trust_level="community")
  278. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  279. agent_id = self._agent_id(identifier)
  280. agent = next((a for a in self._agents() or [] if a.get("identifier") == agent_id), None)
  281. return self._agent_meta(agent, agent_id, agent.get("meta", agent).get("description", "")) if agent else None
  282. def _fetch_index(self) -> Optional[Any]:
  283. return _memo_json("lobehub_index", lambda: _get_json(self.INDEX_URL, timeout=30))
  284. def _fetch_agent(self, agent_id: str) -> Optional[dict]:
  285. return _get_json(f"https://chat-agents.lobehub.com/{agent_id}.json", timeout=15)
  286. @staticmethod
  287. def _convert_to_skill_md(agent_data: dict) -> str:
  288. """Convert a LobeHub agent JSON into SKILL.md format."""
  289. meta = agent_data.get("meta", agent_data)
  290. identifier = agent_data.get("identifier", "lobehub-agent")
  291. title = meta.get("title", identifier)
  292. description = meta.get("description", "")
  293. tags = meta.get("tags", [])
  294. tag_list = tags if isinstance(tags, list) else []
  295. system_role = agent_data.get("config", {}).get("systemRole", "")
  296. fm_lines = ["---", f"name: {identifier}", f"description: {description[:500]}", "metadata:", " hermes:",
  297. f" tags: [{', '.join(str(t) for t in tag_list)}]", " lobehub:", " source: lobehub", "---"]
  298. body_lines = [f"# {title}", "", description, "", "## Instructions", "",
  299. system_role if system_role else "(No system role defined)"]
  300. return "\n".join(fm_lines) + "\n\n" + "\n".join(body_lines) + "\n"
  301. # --- browse.sh source adapter -----------------------------------------------
  302. class BrowseShSource(SkillSource):
  303. """Browserbase's browse.sh catalog of site-specific browser-automation SKILL.md files.
  304. The catalog is ``/api/skills``; content comes from ``/api/skills/{slug}``'s ``skillMdUrl`` (CDN blob).
  305. The catalog's ``sourceUrl`` is a GitHub HTML URL whose repo is not always public, so it is not used for content.
  306. """
  307. SOURCE_ID = "browse-sh"
  308. CATALOG_URL = "https://browse.sh/api/skills"
  309. SKILL_DETAIL_URL = "https://browse.sh/api/skills/{slug}"
  310. _CACHE_KEY = "browse_sh_catalog"
  311. def _fetch_catalog(self) -> list[dict]:
  312. def compute():
  313. data = _get_json(self.CATALOG_URL)
  314. skills = data.get("skills", []) if isinstance(data, dict) else []
  315. return skills if isinstance(skills, list) else None
  316. return _memo_json(self._CACHE_KEY, compute) or []
  317. def _item_to_meta(self, item: dict) -> Optional[SkillMeta]:
  318. slug = item.get("slug", "")
  319. name = item.get("name", "")
  320. description = item.get("description", item.get("title", name))
  321. if not slug or not name:
  322. return None
  323. if len(description) > 1024:
  324. description = description[:1021] + "..."
  325. return SkillMeta(
  326. name=name, description=description, source="browse-sh", identifier=f"browse-sh/{slug}",
  327. trust_level="community", tags=item.get("tags", []),
  328. extra={"slug": slug, "hostname": item.get("hostname", ""), "category": item.get("category", ""),
  329. "source_url": item.get("sourceUrl", ""), "recommended_method": item.get("recommendedMethod", ""),
  330. "proxies": item.get("proxies", False), "install_count": item.get("installCount", 0)},
  331. )
  332. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  333. def fields(item):
  334. return (item.get("name", ""), item.get("title", ""), item.get("description", ""),
  335. item.get("hostname", ""), item.get("category", ""), item.get("tags", []))
  336. return _first_matching(query.lower(), self._fetch_catalog(), fields, self._item_to_meta, limit)
  337. def _catalog_item(self, identifier: str) -> Optional[dict]:
  338. slug = self._slug_from_identifier(identifier)
  339. return next((i for i in self._fetch_catalog() if i.get("slug") == slug), None) if slug else None
  340. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  341. item = self._catalog_item(identifier)
  342. return self._item_to_meta(item) if item else None
  343. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  344. item = self._catalog_item(identifier)
  345. if not item:
  346. return None
  347. slug = item["slug"]
  348. md_url = self._resolve_skill_md_url(slug, item)
  349. content = _get_text(md_url, follow_redirects=True) if md_url else None
  350. if content is None:
  351. return None
  352. meta = self._item_to_meta(item)
  353. return SkillBundle(
  354. name=meta.name if meta else slug.split("/")[-1], files={"SKILL.md": content}, source="browse-sh",
  355. identifier=identifier, trust_level="community",
  356. metadata={"slug": slug, "hostname": item.get("hostname", ""), "source_url": item.get("sourceUrl", ""),
  357. "skill_md_url": md_url},
  358. )
  359. def _resolve_skill_md_url(self, slug: str, item: dict) -> Optional[str]:
  360. """``skillMdUrl`` from ``/api/skills/{slug}``; fallback to a ``raw.githubusercontent.com`` ``sourceUrl``."""
  361. data = _get_json(self.SKILL_DETAIL_URL.format(slug=slug), follow_redirects=True)
  362. md_url = data.get("skillMdUrl") if isinstance(data, dict) else None
  363. if isinstance(md_url, str) and md_url.startswith("http"):
  364. return md_url
  365. source_url = item.get("sourceUrl", "") if isinstance(item, dict) else ""
  366. from utils import base_url_host_matches
  367. return source_url if source_url and base_url_host_matches(source_url, "raw.githubusercontent.com") else None
  368. def _slug_from_identifier(self, identifier: str) -> str:
  369. """'browse-sh/airbnb.com/search-listings-abc' -> 'airbnb.com/search-listings-abc'."""
  370. return identifier.removeprefix("browse-sh/")