skills_hub_official.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400
  1. """Skills Hub official sources: repo-shipped optional skills and the centralized Hermes index."""
  2. import logging
  3. from pathlib import Path, PurePosixPath
  4. from typing import Dict, List, Optional, Tuple, Union
  5. from agent.skill_utils import is_excluded_skill_path
  6. from tools.skills_hub_github import GitHubAuth, GitHubSource, _skip_bundle_file, _tree_members
  7. from tools.skills_hub_models import (
  8. SkillBundle, SkillMeta, SkillSource, _hermes_tags, _matches_query, _memo_json, _parse_frontmatter, hub,
  9. )
  10. logger = logging.getLogger("tools.skills_hub")
  11. # Identifier prefixes stripped when matching index entries loosely.
  12. _INDEX_ID_PREFIXES = ("skills-sh/", "skills.sh/", "official/", "github/", "clawhub/")
  13. def _strip_prefix(value: str, prefixes) -> str:
  14. return next((value[len(p):] for p in prefixes if value.startswith(p)), value)
  15. def _clean_rel_parts(path: str) -> Optional[list[str]]:
  16. """Split a relative path, dropping ``.``/empty parts; None on traversal or empty."""
  17. parts = [p for p in path.split("/") if p not in ("", ".")]
  18. return None if not parts or ".." in parts else parts
  19. def _entry_provider(entry: dict) -> str:
  20. """Normalized ``extra.provider`` label of a raw index entry."""
  21. return str((entry.get("extra") or {}).get("provider", "")).lower()
  22. class OptionalSkillSource(SkillSource):
  23. """Skills from the repo's ``optional-skills/`` directory: official (Nous-maintained) but not
  24. activated by default — absent from the system prompt and not copied to ~/.hermes/skills/ at
  25. setup. Discoverable via the Skills Hub as source "official" with "builtin" trust."""
  26. SOURCE_ID = "official"
  27. TRUST_LEVEL = "builtin"
  28. OFFICIAL_REPO = "NousResearch/hermes-agent"
  29. OPTIONAL_SKILLS_PREFIX = "optional-skills"
  30. _parse_frontmatter = staticmethod(_parse_frontmatter)
  31. def __init__(self, auth: Optional[GitHubAuth] = None):
  32. from hermes_constants import get_optional_skills_dir
  33. self._optional_dir = get_optional_skills_dir(Path(__file__).parent.parent / "optional-skills")
  34. self._auth = auth
  35. # GitHubSource for the live-repo fallback, created only when a skill is missing locally.
  36. self._github: Optional[GitHubSource] = None
  37. # "category/skill" -> True from the live repo tree; None = not fetched yet.
  38. self._remote_dirs: Optional[dict[str, bool]] = None
  39. @staticmethod
  40. def _rel(identifier: str) -> str:
  41. return identifier.split("/", 1)[-1] if identifier.startswith("official/") else identifier
  42. def _meta(self, rel_dir: str, name: str, description: str, tags: list) -> SkillMeta:
  43. return SkillMeta(
  44. name=name, description=description, source="official", identifier=f"official/{rel_dir}",
  45. trust_level="builtin", repo=self.OFFICIAL_REPO,
  46. # The centralized skills index consumes repo-root-relative paths.
  47. path=f"{self.OPTIONAL_SKILLS_PREFIX}/{rel_dir}", tags=tags,
  48. )
  49. def _remote_meta(self, rel_dir: str) -> SkillMeta:
  50. """Placeholder meta for a skill that exists on live main but not locally."""
  51. desc = "Official optional skill (from live repo; run install to fetch)"
  52. return self._meta(rel_dir, rel_dir.rsplit("/", 1)[-1], desc, [])
  53. @staticmethod
  54. def _bundle(rel_id: str, files: dict[str, str | bytes], **kwargs) -> SkillBundle:
  55. return SkillBundle(name=rel_id.rsplit("/", 1)[-1], files=files, source="official",
  56. identifier=f"official/{rel_id}", trust_level="builtin", **kwargs)
  57. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  58. results: list[SkillMeta] = []
  59. query_lower = query.lower()
  60. local_rels: set = set()
  61. for meta in self._scan_all():
  62. local_rels.add(meta.identifier.split("/", 1)[-1] if meta.identifier else "")
  63. if _matches_query(query_lower, meta.name, meta.description, meta.tags):
  64. results.append(meta)
  65. if len(results) >= limit:
  66. break
  67. # Also surface skills that landed on live main after this install was cut.
  68. if len(results) < limit:
  69. for rel_dir in sorted(self._list_remote_skill_dirs()):
  70. if rel_dir in local_rels or (query_lower and query_lower not in rel_dir.lower()):
  71. continue
  72. results.append(self._remote_meta(rel_dir))
  73. if len(results) >= limit:
  74. break
  75. return results
  76. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  77. # identifier format: "official/category/skill" or "official/skill"
  78. rel = self._rel(identifier)
  79. # Guard against path traversal (e.g. "official/../../etc")
  80. try:
  81. resolved = (self._optional_dir / rel).resolve()
  82. optional_root = self._optional_dir.resolve()
  83. if not resolved.is_relative_to(optional_root):
  84. return None
  85. except (OSError, ValueError):
  86. return None
  87. # Else try by bare skill name; if still absent, the skill may have landed on main
  88. # after this install was cut — use the live repo.
  89. skill_dir = resolved if resolved.is_dir() else self._find_skill_dir(rel.rsplit("/", 1)[-1])
  90. if not skill_dir:
  91. return self._fetch_from_live_repo(rel)
  92. rel_id = skill_dir.resolve().relative_to(optional_root).as_posix()
  93. # Catalog stubs point at the real skill in an upstream-maintained repo
  94. # (metadata.hermes.upstream); install pulls the live content from there.
  95. try:
  96. skill_md = (skill_dir / "SKILL.md").read_text(encoding="utf-8-sig")
  97. except (OSError, UnicodeDecodeError):
  98. skill_md = None
  99. upstream = None if skill_md is None else self._upstream_pointer_from_content(skill_md)
  100. if upstream is not None:
  101. return self._fetch_from_upstream(upstream, rel_id)
  102. files: dict[str, str | bytes] = {}
  103. for f in skill_dir.rglob("*"):
  104. if f.is_file() and not _skip_bundle_file(f.relative_to(skill_dir).as_posix()):
  105. try:
  106. files[str(f.relative_to(skill_dir))] = f.read_bytes()
  107. except OSError:
  108. continue
  109. return self._bundle(rel_id, files) if files else None
  110. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  111. skill_name = self._rel(identifier).rsplit("/", 1)[-1]
  112. for meta in self._scan_all():
  113. if meta.name == skill_name:
  114. return meta
  115. matches = self._remote_matches(skill_name) # not in the local checkout — check live main
  116. return self._remote_meta(matches[0]) if len(matches) == 1 else None
  117. def list_local(self) -> list[SkillMeta]:
  118. """Every optional skill in the local checkout, with frontmatter metadata
  119. (backs the dashboard/desktop "built-in optional skills" catalog)."""
  120. return self._scan_all()
  121. def _get_github(self) -> GitHubSource:
  122. if self._github is None:
  123. self._github = GitHubSource(auth=self._auth or GitHubAuth())
  124. return self._github
  125. def _remote_matches(self, name: str) -> list[str]:
  126. return [d for d in self._list_remote_skill_dirs() if d.rsplit("/", 1)[-1] == name]
  127. def _fetch_from_live_repo(self, rel: str) -> Optional[SkillBundle]:
  128. """Fetch an optional skill straight from the live default branch. Local installs lag
  129. ``main``; rather than demanding ``hermes update`` first, resolve against the live repo.
  130. ``rel`` is ``category/skill`` (used verbatim) or a bare skill name (located via the repo tree)."""
  131. parts = _clean_rel_parts(rel.strip("/"))
  132. if parts is None:
  133. return None
  134. rel = "/".join(parts)
  135. github = self._get_github()
  136. if rel not in self._list_remote_skill_dirs():
  137. # Bare name (or stale category) — locate by final path segment.
  138. matches = self._remote_matches(parts[-1])
  139. if len(matches) != 1:
  140. return None
  141. rel = matches[0]
  142. # Download the FULL directory byte-exact (root-level install scripts, LICENSE, tests/).
  143. # GitHubSource.fetch() would only pull SKILL.md + referenced support dirs.
  144. tree = github._get_repo_tree(self.OFFICIAL_REPO)
  145. if tree is None:
  146. return None
  147. files: dict[str, str | bytes] = {}
  148. for rel_file, item_path, regular in _tree_members(tree[1], f"{self.OPTIONAL_SKILLS_PREFIX}/{rel}/"):
  149. if not regular or _skip_bundle_file(rel_file):
  150. continue
  151. content = github._fetch_file_bytes(self.OFFICIAL_REPO, item_path)
  152. if content is None:
  153. logger.warning("Live-repo optional skill fetch failed for %s", item_path)
  154. return None
  155. files[rel_file] = content
  156. if "SKILL.md" not in files:
  157. return None
  158. # Live-fetched catalog stubs redirect the same way local ones do.
  159. upstream = self._upstream_pointer_from_content(files["SKILL.md"])
  160. if upstream is not None:
  161. return self._fetch_from_upstream(upstream, rel)
  162. logger.info("Optional skill '%s' fetched from live repo (not in local checkout)", rel)
  163. return self._bundle(rel, files)
  164. def _list_remote_skill_dirs(self) -> dict[str, bool]:
  165. """``category/skill`` dirs under optional-skills/ on live main. One repo-tree call (cached
  166. per-process by GitHubSource + the on-disk index cache). {} when the network/API is
  167. unavailable — callers degrade to local-only."""
  168. if self._remote_dirs is not None:
  169. return self._remote_dirs
  170. def compute():
  171. dirs: dict[str, bool] = {}
  172. if (tree := self._get_github()._get_repo_tree(self.OFFICIAL_REPO)) is None:
  173. return None
  174. prefix, suffix = f"{self.OPTIONAL_SKILLS_PREFIX}/", "/SKILL.md"
  175. for item in tree[1]:
  176. path = item.get("path", "")
  177. if item.get("type") == "blob" and path.startswith(prefix) and path.endswith(suffix):
  178. rel_dir = path[len(prefix):-len(suffix)]
  179. if rel_dir and not is_excluded_skill_path(PurePosixPath(rel_dir + suffix)):
  180. dirs[rel_dir] = True
  181. return dirs or None
  182. self._remote_dirs = _memo_json("official_optional_dirs", compute,
  183. valid=lambda c: isinstance(c, dict) and bool(c)) or {}
  184. return self._remote_dirs
  185. def _upstream_pointer_from_content(self, content: str | bytes) -> Optional[dict[str, str]]:
  186. """Parse ``metadata.hermes.upstream: {repo: owner/name, path: ...}`` out of SKILL.md content
  187. (a catalog stub); None for vendored skills."""
  188. if isinstance(content, bytes):
  189. try:
  190. content = content.decode("utf-8")
  191. except UnicodeDecodeError:
  192. return None
  193. meta_block = _parse_frontmatter(content).get("metadata")
  194. hermes_meta = meta_block.get("hermes") if isinstance(meta_block, dict) else None
  195. upstream = hermes_meta.get("upstream") if isinstance(hermes_meta, dict) else None
  196. if not isinstance(upstream, dict):
  197. return None
  198. repo = str(upstream.get("repo", "")).strip().strip("/")
  199. path = str(upstream.get("path", "")).strip().strip("/")
  200. # repo must be exactly owner/name; path must be a clean relative path.
  201. if not repo or repo.count("/") != 1 or not path:
  202. return None
  203. parts = _clean_rel_parts(path)
  204. return None if parts is None else {"repo": repo, "path": "/".join(parts)}
  205. def _fetch_from_upstream(self, upstream: dict[str, str], rel_id: str) -> Optional[SkillBundle]:
  206. """Fetch an upstream-maintained optional skill via GitHubSource.fetch() (full-tree download,
  207. symlink/unsafe-path rejection, quarantine + scan downstream) and re-label it as an official
  208. catalog entry."""
  209. bundle = self._get_github().fetch(f"{upstream['repo']}/{upstream['path']}")
  210. if bundle is None:
  211. logger.warning("Upstream fetch failed for optional skill %s (%s:%s)",
  212. rel_id, upstream["repo"], upstream["path"])
  213. return None
  214. return SkillBundle(
  215. name=bundle.name, files=bundle.files, source="official", identifier=f"official/{rel_id}",
  216. # Curated endorsement, but the content is live third-party:
  217. # "trusted", not "builtin", so a dangerous scan verdict still blocks.
  218. trust_level="trusted",
  219. metadata={**bundle.metadata, "upstream_repo": upstream["repo"], "upstream_path": upstream["path"]},
  220. )
  221. def _local_skill_mds(self):
  222. root = self._optional_dir
  223. return (md for md in (sorted(root.rglob("SKILL.md")) if root.is_dir() else [])
  224. if not is_excluded_skill_path(md.relative_to(root), root=root))
  225. def _find_skill_dir(self, name: str) -> Optional[Path]:
  226. """Find a skill directory by name anywhere in optional-skills/."""
  227. return next((md.parent for md in self._local_skill_mds() if md.parent.name == name), None)
  228. def _scan_all(self) -> list[SkillMeta]:
  229. """Enumerate all optional skills with metadata."""
  230. results: list[SkillMeta] = []
  231. for skill_md in self._local_skill_mds():
  232. parent = skill_md.parent
  233. try:
  234. content = skill_md.read_text(encoding="utf-8-sig")
  235. except (OSError, UnicodeDecodeError):
  236. continue
  237. fm = _parse_frontmatter(content)
  238. tags = _hermes_tags(fm)
  239. results.append(self._meta(parent.relative_to(self._optional_dir).as_posix(), fm.get("name", parent.name),
  240. fm.get("description", "")[:200], tags if isinstance(tags, list) else []))
  241. return results
  242. class HermesIndexSource(SkillSource):
  243. """Skill source backed by the centralized Hermes Skills Index: a JSON catalog on the docs site,
  244. rebuilt daily by CI, with metadata + resolved GitHub paths for every skill — search and path
  245. discovery cost zero GitHub API calls. When unavailable every method returns empty/None so
  246. downstream sources take over transparently."""
  247. SOURCE_ID = "hermes-index"
  248. def __init__(self, auth: GitHubAuth):
  249. self._index: Optional[dict] = None
  250. self._loaded = False
  251. self.auth = auth
  252. self._github: Optional[GitHubSource] = None # only needed for fetch
  253. def _ensure_loaded(self) -> dict:
  254. if not self._loaded:
  255. from tools.skills_hub_search import _load_hermes_index
  256. self._index, self._loaded = _load_hermes_index(), True
  257. return self._index or {}
  258. def _skills(self) -> list:
  259. return self._ensure_loaded().get("skills", [])
  260. def _get_github(self) -> GitHubSource:
  261. if self._github is None:
  262. self._github = GitHubSource(auth=self.auth)
  263. return self._github
  264. @property
  265. def is_available(self) -> bool:
  266. """Whether the index is loaded and has skills."""
  267. return bool(self._skills())
  268. def trust_level_for(self, identifier: str) -> str:
  269. entry = next((s for s in self._skills() if s.get("identifier") == identifier), None)
  270. return entry.get("trust_level", "community") if entry else "community"
  271. def search(self, query: str, limit: int = 10, *, provider_filter: str = "") -> list[SkillMeta]:
  272. """Search the cached index (zero API calls). Matches name, description, tags, identifier and
  273. ``extra.provider`` (so ``nvidia`` finds ``NVIDIA/skills/...`` entries stored as source
  274. "github"). Ranked exact name > name prefix > provider > whole-word > name substring > other,
  275. index order as tiebreaker — a raw break-at-limit slice buried the most relevant skills.
  276. Provider filters narrow the catalog before ranking and limiting."""
  277. skills = self._skills()
  278. want = provider_filter.strip().lower()
  279. if want:
  280. skills = [s for s in skills if _entry_provider(s) == want]
  281. if not skills:
  282. return []
  283. if not query.strip():
  284. return [self._to_meta(s) for s in skills[:limit]] # featured / index order
  285. query_lower = query.lower()
  286. scored: list[tuple[int, int, dict]] = []
  287. for i, s in enumerate(skills):
  288. name = str(s.get("name", "")).lower()
  289. provider = _entry_provider(s)
  290. haystack = " ".join([
  291. name, str(s.get("description", "")).lower(), " ".join(str(t).lower() for t in s.get("tags", [])),
  292. str(s.get("identifier", "")).lower(), provider,
  293. ])
  294. if query_lower not in haystack:
  295. continue
  296. ranks = (
  297. name == query_lower, name.startswith(query_lower), provider == query_lower,
  298. query_lower in name.split() or query_lower in provider.split(), query_lower in name, True,
  299. )
  300. scored.append((ranks.index(True), i, s))
  301. scored.sort(key=lambda x: (x[0], x[1]))
  302. return [self._to_meta(s) for _, _, s in scored[:limit]]
  303. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  304. """Fetch via the index's ``resolved_github_id`` (skipping the whole
  305. candidate/discovery chain), falling back to ``repo/path``."""
  306. entry = self._find_entry(identifier)
  307. if not entry:
  308. return None
  309. repo, path = entry.get("repo", ""), entry.get("path", "")
  310. github, slug = self._get_github(), path.rstrip("/").rsplit("/", 1)[-1]
  311. skills_sh = entry.get("source") in {"skills.sh", "skills-sh"}
  312. candidates = [entry.get("resolved_github_id")] + ([f"{repo}/{path}"] if repo and path else [])
  313. for github_id in filter(None, candidates):
  314. if bundle := github.fetch(github_id):
  315. break
  316. else:
  317. # skills.sh rows carry the slug as ``path``; when no build resolved it, find the skill in
  318. # the repo tree (a slug-named dir, a lone generic ``skills/``, or the repo root) instead
  319. # of reporting a live skill as a stale entry (#130129). A concrete GitHub path that is
  320. # gone stays gone: never substitute whatever skill remains in that repo.
  321. found = skills_sh and repo and slug and (
  322. github._find_skill_in_repo_tree(repo, slug) or github._find_repo_root_skill(repo))
  323. if not (found and (bundle := github.fetch(found))):
  324. return None
  325. if skills_sh and bundle.name == "skills": # same name resolved or not, never a shared "skills/"
  326. bundle.name = slug
  327. bundle.source = entry.get("source", "hermes-index")
  328. bundle.identifier = identifier
  329. return bundle
  330. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  331. """Return metadata from the index (zero API calls)."""
  332. entry = self._find_entry(identifier)
  333. return self._to_meta(entry) if entry else None
  334. def _find_entry(self, identifier: str) -> Optional[dict]:
  335. """Exact identifier match first, then match with source prefixes stripped."""
  336. skills = self._skills()
  337. normalized = _strip_prefix(identifier, _INDEX_ID_PREFIXES)
  338. return next((s for s in skills if s.get("identifier") == identifier), None) or next(
  339. (s for s in skills if _strip_prefix(s.get("identifier", ""), _INDEX_ID_PREFIXES) == normalized), None,
  340. )
  341. @staticmethod
  342. def _to_meta(entry: dict) -> SkillMeta:
  343. return SkillMeta(
  344. name=entry.get("name", ""), description=entry.get("description", ""),
  345. source=entry.get("source", "hermes-index"), identifier=entry.get("identifier", ""),
  346. trust_level=entry.get("trust_level", "community"), repo=entry.get("repo"), path=entry.get("path"),
  347. tags=entry.get("tags", []), extra=entry.get("extra", {}),
  348. )