skills_hub_models.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340
  1. """Skills Hub data models, path validators, and SKILL.md helpers. Leaf module (no imports from
  2. tools.skills_hub) so every source adapter module can import it at top level without cycles."""
  3. import json
  4. import logging
  5. import re
  6. from abc import ABC, abstractmethod
  7. from dataclasses import dataclass, field
  8. from pathlib import PurePosixPath
  9. from typing import Any, Callable, Dict, Iterable, List, Optional, Union
  10. from urllib.parse import unquote, urlsplit
  11. import httpx
  12. import hermes_yaml as yaml
  13. logger = logging.getLogger("tools.skills_hub")
  14. def hub():
  15. """``tools.skills_hub`` resolved at call time: its cache / HTTP / index helpers are the test-patch targets
  16. (``patch("tools.skills_hub._read_index_cache")`` ...), so adapters look them up on every call, not at import."""
  17. import tools.skills_hub as mod
  18. return mod
  19. @dataclass
  20. class SkillMeta:
  21. """Minimal metadata returned by search results."""
  22. name: str
  23. description: str
  24. source: str # "official", "github", "clawhub", "lobehub"
  25. identifier: str # source-specific ID (e.g. "openai/skills/skill-creator")
  26. trust_level: str # "builtin" | "trusted" | "community"
  27. repo: Optional[str] = None
  28. path: Optional[str] = None
  29. tags: list[str] = field(default_factory=list)
  30. extra: dict[str, Any] = field(default_factory=dict)
  31. @dataclass
  32. class SkillBundle:
  33. """A downloaded skill ready for quarantine/scanning/installation."""
  34. name: str
  35. files: dict[str, str | bytes] # relative_path -> file content
  36. source: str
  37. identifier: str
  38. trust_level: str
  39. metadata: dict[str, Any] = field(default_factory=dict)
  40. def _skill_meta_to_dict(meta: SkillMeta) -> dict:
  41. return dict(vars(meta))
  42. def _cached_metas(key: str) -> Optional[list[SkillMeta]]:
  43. """SkillMeta list from the shared index cache, or None on miss/expiry."""
  44. cached = hub()._read_index_cache(key)
  45. return None if cached is None else [SkillMeta(**item) for item in cached]
  46. def _cache_metas(key: str, metas: list[SkillMeta]) -> None:
  47. hub()._write_index_cache(key, [_skill_meta_to_dict(m) for m in metas])
  48. def _memo_json(key: str, compute: Callable[[], Any], valid: Callable[[Any], bool] = lambda c: c is not None) -> Any:
  49. """Shared-index-cache memo: a cached value passing ``valid`` is returned as-is; otherwise ``compute()``
  50. runs and a non-None result is written back."""
  51. cached = hub()._read_index_cache(key)
  52. if valid(cached):
  53. return cached
  54. data = compute()
  55. if data is not None:
  56. hub()._write_index_cache(key, data)
  57. return data
  58. def _get_json(url: str, *, timeout: int = 20, **kwargs) -> Optional[Any]:
  59. """Plain (unguarded) GET + JSON decode; None on non-200 or transport/decode error."""
  60. try:
  61. resp = hub()._skills_hub_http_get(url, timeout=timeout, **kwargs)
  62. return resp.json() if resp.status_code == 200 else None
  63. except (httpx.HTTPError, json.JSONDecodeError):
  64. return None
  65. def _get_text(url: str, *, timeout: int = 20, **kwargs) -> Optional[str]:
  66. """Plain (unguarded) GET; body text on 200, None on any other status or transport error."""
  67. try:
  68. resp = hub()._skills_hub_http_get(url, timeout=timeout, **kwargs)
  69. except httpx.HTTPError:
  70. return None
  71. return resp.text if resp.status_code == 200 else None
  72. def _matches_query(query_lower: str, *fields: Any) -> bool:
  73. """Case-insensitive substring match against joined fields (lists space-joined; empty query matches all)."""
  74. parts = [" ".join(str(t) for t in f) if isinstance(f, list) else str(f) for f in fields]
  75. return query_lower in " ".join(parts).lower()
  76. def _first_matching(query_lower: str, items: Iterable[Any], fields_of: Callable[[Any], tuple],
  77. to_meta: Callable[[Any], Optional[SkillMeta]], limit: int) -> list[SkillMeta]:
  78. """Substring-search ``items`` in order, converting hits with ``to_meta`` until ``limit``."""
  79. results: list[SkillMeta] = []
  80. for item in items:
  81. if _matches_query(query_lower, *fields_of(item)) and (meta := to_meta(item)):
  82. results.append(meta)
  83. if len(results) >= limit:
  84. break
  85. return results
  86. TRUST_RANK = {"builtin": 2, "trusted": 1, "community": 0}
  87. def _dedupe_by_trust(results: Iterable[SkillMeta]) -> list[SkillMeta]:
  88. """Dedupe by identifier, keeping the higher-trust copy (first wins on ties). identifier is unique per
  89. skill; name is not — two taps can publish same-named skills, and browse-sh reuses task names across sites."""
  90. seen: dict[str, SkillMeta] = {}
  91. for r in results:
  92. kept = seen.get(r.identifier)
  93. if kept is None or TRUST_RANK.get(r.trust_level, 0) > TRUST_RANK.get(kept.trust_level, 0):
  94. seen[r.identifier] = r
  95. return list(seen.values())
  96. class SkillSource(ABC):
  97. """Abstract base for all skill registry adapters. ``SOURCE_ID`` is the unique source id (e.g. 'github',
  98. 'clawhub'); ``TRUST_LEVEL`` the trust every identifier gets unless ``trust_level_for`` is overridden."""
  99. SOURCE_ID: str = ""
  100. TRUST_LEVEL: str = "community"
  101. # Consecutive failed fetches of one catalog page/shard before a walk gives up as partial.
  102. CATALOG_PAGE_RETRIES = 5
  103. @abstractmethod
  104. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  105. """Search for skills matching a query string."""
  106. @abstractmethod
  107. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  108. """Download a skill bundle by identifier."""
  109. @abstractmethod
  110. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  111. """Fetch metadata for a skill without downloading all files."""
  112. def source_id(self) -> str:
  113. return self.SOURCE_ID
  114. def trust_level_for(self, identifier: str) -> str:
  115. return self.TRUST_LEVEL
  116. def current_revision(self, identifier: str) -> str:
  117. """Upstream revision the skill would be fetched at; "" when the registry has no cheap
  118. revision probe, which keeps update checks on the full-fetch path."""
  119. return ""
  120. class GuardedFetchMixin:
  121. """SSRF/policy-guarded GETs, routed through ``tools.skills_hub`` (test-patchable)."""
  122. @staticmethod
  123. def _fetch_text(url: str) -> Optional[str]:
  124. resp = hub()._guarded_http_get(url, timeout=20)
  125. return resp.text if resp is not None and resp.status_code == 200 else None
  126. @staticmethod
  127. def _fetch_bytes(url: str) -> Optional[bytes]:
  128. resp = hub()._guarded_http_get(url, timeout=20)
  129. return resp.content if resp is not None and resp.status_code == 200 else None
  130. # --- SKILL.md frontmatter ---------------------------------------------------
  131. def _parse_frontmatter(content: str) -> dict:
  132. """Parse YAML frontmatter from SKILL.md content ({} when absent/invalid)."""
  133. content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors)
  134. match = re.search(r'\n---\s*\n', content[3:]) if content.startswith("---") else None
  135. if not match:
  136. return {}
  137. try:
  138. parsed = yaml.safe_load(content[3:match.start() + 3])
  139. return parsed if isinstance(parsed, dict) else {}
  140. except yaml.YAMLError:
  141. return {}
  142. def _hermes_tags(fm: dict) -> Any:
  143. """``metadata.hermes.tags`` from parsed frontmatter, or ``[]`` (unvalidated type)."""
  144. metadata = fm.get("metadata", {})
  145. hermes_meta = metadata.get("hermes", {}) if isinstance(metadata, dict) else None
  146. return hermes_meta.get("tags", []) if isinstance(hermes_meta, dict) else []
  147. def source_url_for_bundle(bundle: SkillBundle) -> str:
  148. """Best available human-facing immutable-source provenance URL."""
  149. explicit = bundle.metadata.get("source_url") or bundle.metadata.get("url")
  150. if explicit:
  151. return str(explicit)
  152. if bundle.source == "github":
  153. parts = bundle.identifier.split("/", 2)
  154. if len(parts) >= 2:
  155. suffix = f"/tree/main/{parts[2]}" if len(parts) == 3 else ""
  156. return f"https://github.com/{parts[0]}/{parts[1]}{suffix}"
  157. return bundle.identifier
  158. # --- Bundle path validation -------------------------------------------------
  159. def _normalize_bundle_path(path_value: str, *, field_name: str, allow_nested: bool) -> str:
  160. """Normalize and validate bundle-controlled paths before touching disk."""
  161. if not isinstance(path_value, str):
  162. raise ValueError(f"Unsafe {field_name}: expected a string")
  163. raw = path_value.strip()
  164. if not raw:
  165. raise ValueError(f"Unsafe {field_name}: empty path")
  166. normalized = raw.replace("\\", "/")
  167. path = PurePosixPath(normalized)
  168. parts = [part for part in path.parts if part not in {"", "."}]
  169. # A colon in any component is rejected: on Windows it marks a drive (``C:foo``) or an NTFS Alternate Data
  170. # Stream (``file.py:payload`` writes scanner-invisible bytes); ``/`` is the only legal separator once normalized.
  171. if (normalized.startswith("/") or path.is_absolute() or not parts or any(part == ".." for part in parts)
  172. or any(":" in part for part in parts) or (not allow_nested and len(parts) != 1)):
  173. raise ValueError(f"Unsafe {field_name}: {path_value}")
  174. return "/".join(parts)
  175. def _validate_skill_name(name: str) -> str:
  176. return _normalize_bundle_path(name, field_name="skill name", allow_nested=False)
  177. def _validate_install_parent_path(category: str) -> str:
  178. return _normalize_bundle_path(category, field_name="install parent path", allow_nested=True)
  179. def _validate_bundle_rel_path(rel_path: str) -> str:
  180. return _normalize_bundle_path(rel_path, field_name="bundle file path", allow_nested=True)
  181. def _normalize_lock_install_path(install_path: str, skill_name: str) -> str:
  182. """Validate a lock-file ``install_path`` (the ``uninstall_skill`` rmtree target).
  183. Must be relative, traversal-free, and end with ``<skill_name>`` — nested official skills legitimately
  184. live at ``mlops/training/<skill_name>``; an empty/``"."``/absolute/mismatched entry could point rmtree
  185. at the whole ``skills/`` tree or outside it.
  186. """
  187. safe_skill_name = _validate_skill_name(skill_name)
  188. normalized = _normalize_bundle_path(install_path, field_name="install path", allow_nested=True)
  189. if normalized.split("/")[-1] != safe_skill_name:
  190. raise ValueError(f"Unsafe install path: {install_path}")
  191. return normalized
  192. # --- Referenced support-file extraction from SKILL.md -----------------------
  193. _ALLOWED_SUPPORT_DIRS = frozenset({"references", "templates", "scripts", "assets", "examples"})
  194. _LOCAL_LINK_RE = re.compile(
  195. r"(?:\]\(|`|(?:^|[\s\"']))((?:references|templates|scripts|assets|examples)/[^\s)`\"'<>]+)", re.MULTILINE)
  196. _SUSPICIOUS_LOCAL_REF_RE = re.compile(
  197. r"(?:references|templates|scripts|assets|examples)/(?:[^\s)`\"'<>]*/)?\.\.(?:/|$)")
  198. _VALUELESS_QUERY_FLAG_RE = re.compile(r"(?:[A-Za-z0-9_~-]|%[0-9A-Fa-f]{2})+\Z")
  199. # Same-directory links (``](./FILE.ext)`` / ``](FILE.ext)``): siblings of SKILL.md the document links
  200. # explicitly (e.g. ./CONTEXT-FORMAT.md). Dropping them made the install "succeed" with unresolved links.
  201. # The extension requirement keeps prose words out; support-dir links stay on _LOCAL_LINK_RE.
  202. # Skills legitimately ship supporting docs next to SKILL.md instead of under a support directory (e.g.
  203. # mattpocock/skills' domain-modeling links ./CONTEXT-FORMAT.md); dropping them made the install "succeed"
  204. # while the bundle came out with unresolved links (#96310).
  205. _SAMEDIR_LINK_RE = re.compile(r"\]\(([^)\s\"'<>]+)")
  206. _SAMEDIR_NAME_RE = re.compile(r"^(?:\./)?[A-Za-z0-9][A-Za-z0-9._-]*$")
  207. def _query_is_concrete(query: str) -> bool:
  208. """Whether a URL query is real URL syntax rather than glob prose.
  209. A non-empty ``key=value`` part is always concrete. Valueless flags are accepted only when RFC 3986
  210. unreserved-token shaped (percent escapes ok); ``.``, brackets and extra ``?`` are excluded because
  211. ``?x.md`` / ``?.md`` is indistinguishable from a single-char glob finishing a filename in prose.
  212. """
  213. return all(("=" in part and bool(part.split("=", 1)[0])) or bool(_VALUELESS_QUERY_FLAG_RE.fullmatch(part))
  214. for part in query.split("&"))
  215. def _referenced_support_paths(skill_md: str) -> Optional[set[str]]:
  216. """Extract safe referenced paths; return None on a traversal attempt (fail closed)."""
  217. normalized = skill_md.replace("\\", "/")
  218. if _SUSPICIOUS_LOCAL_REF_RE.search(normalized):
  219. return None
  220. paths: set[str] = set()
  221. for match in _LOCAL_LINK_RE.finditer(normalized):
  222. candidate = match.group(1).rstrip(".,;:")
  223. parsed = urlsplit(candidate)
  224. raw = unquote(parsed.path)
  225. if (candidate.endswith("?") or any(char in raw for char in "*?[]")
  226. or (parsed.query and not _query_is_concrete(parsed.query))):
  227. continue
  228. try:
  229. safe = _validate_bundle_rel_path(raw)
  230. except ValueError:
  231. return None
  232. if safe.split("/", 1)[0] in _ALLOWED_SUPPORT_DIRS:
  233. # Prose placeholders (``references/type-<name>.md``, truncated at ``<`` to
  234. # ``references/type-``) are instructions, not files: a basename ending in a
  235. # separator is skipped. No extension requirement — ``references/LICENSE`` is legitimate.
  236. base = safe.rsplit("/", 1)[-1]
  237. if re.search(r"[*?<>]", safe) or not re.search(r"[A-Za-z0-9]$", base):
  238. continue
  239. paths.add(safe)
  240. for match in _SAMEDIR_LINK_RE.finditer(normalized):
  241. raw = match.group(1).rstrip(".,;:")
  242. # Canonicalize like the support-dir branch (drop query/fragment, percent-decode), strip leading ``./``.
  243. name = unquote(urlsplit(raw).path)
  244. name = name.removeprefix("./")
  245. # External URLs, anchors, mailto and site-absolute targets are not same-directory file links.
  246. if not name or "://" in raw or raw.startswith(("mailto:", "#", "/")):
  247. continue
  248. if name.startswith(".."):
  249. # A repo-relative link to a doc outside the skill directory (``../../tools/REGISTRY.md``
  250. # in a multi-skill repo) is prose, never a bundle path: nothing is fetched or written for
  251. # it, so refusing the whole bundle protected nothing and made every skill that links a
  252. # sibling doc uninstallable with a misleading "files no longer exist upstream" (#115171).
  253. # The link is left dangling in the installed copy, like an absent support file.
  254. logger.warning("SKILL.md links outside the skill directory; installing without it: %s", raw)
  255. continue
  256. # Only unambiguous file links: an extension, no internal slash, never SKILL.md itself (casefolded —
  257. # a ``skill.md`` entry would collide with the bundle root on macOS/Windows; skipped, not merged).
  258. if ("/" in name or name.casefold() == "skill.md" or "." not in name.lstrip(".")
  259. or not _SAMEDIR_NAME_RE.match(name)):
  260. continue
  261. try:
  262. safe = _validate_bundle_rel_path(name)
  263. except ValueError:
  264. return None
  265. paths.add(safe)
  266. # Case-folded collisions among accepted same-dir names (``A.md`` + ``a.md``)
  267. # would collide on install — drop the pair rather than guess.
  268. folded: dict[str, list[str]] = {}
  269. for p in sorted(paths):
  270. folded.setdefault(p.casefold(), []).append(p)
  271. for group in folded.values():
  272. if len(group) > 1:
  273. paths.difference_update(group)
  274. return paths