| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340 |
- """Skills Hub data models, path validators, and SKILL.md helpers. Leaf module (no imports from
- tools.skills_hub) so every source adapter module can import it at top level without cycles."""
- import json
- import logging
- import re
- from abc import ABC, abstractmethod
- from dataclasses import dataclass, field
- from pathlib import PurePosixPath
- from typing import Any, Callable, Dict, Iterable, List, Optional, Union
- from urllib.parse import unquote, urlsplit
- import httpx
- import hermes_yaml as yaml
- logger = logging.getLogger("tools.skills_hub")
- def hub():
- """``tools.skills_hub`` resolved at call time: its cache / HTTP / index helpers are the test-patch targets
- (``patch("tools.skills_hub._read_index_cache")`` ...), so adapters look them up on every call, not at import."""
- import tools.skills_hub as mod
- return mod
- @dataclass
- class SkillMeta:
- """Minimal metadata returned by search results."""
- name: str
- description: str
- source: str # "official", "github", "clawhub", "lobehub"
- identifier: str # source-specific ID (e.g. "openai/skills/skill-creator")
- trust_level: str # "builtin" | "trusted" | "community"
- repo: Optional[str] = None
- path: Optional[str] = None
- tags: list[str] = field(default_factory=list)
- extra: dict[str, Any] = field(default_factory=dict)
- @dataclass
- class SkillBundle:
- """A downloaded skill ready for quarantine/scanning/installation."""
- name: str
- files: dict[str, str | bytes] # relative_path -> file content
- source: str
- identifier: str
- trust_level: str
- metadata: dict[str, Any] = field(default_factory=dict)
- def _skill_meta_to_dict(meta: SkillMeta) -> dict:
- return dict(vars(meta))
- def _cached_metas(key: str) -> Optional[list[SkillMeta]]:
- """SkillMeta list from the shared index cache, or None on miss/expiry."""
- cached = hub()._read_index_cache(key)
- return None if cached is None else [SkillMeta(**item) for item in cached]
- def _cache_metas(key: str, metas: list[SkillMeta]) -> None:
- hub()._write_index_cache(key, [_skill_meta_to_dict(m) for m in metas])
- def _memo_json(key: str, compute: Callable[[], Any], valid: Callable[[Any], bool] = lambda c: c is not None) -> Any:
- """Shared-index-cache memo: a cached value passing ``valid`` is returned as-is; otherwise ``compute()``
- runs and a non-None result is written back."""
- cached = hub()._read_index_cache(key)
- if valid(cached):
- return cached
- data = compute()
- if data is not None:
- hub()._write_index_cache(key, data)
- return data
- def _get_json(url: str, *, timeout: int = 20, **kwargs) -> Optional[Any]:
- """Plain (unguarded) GET + JSON decode; None on non-200 or transport/decode error."""
- try:
- resp = hub()._skills_hub_http_get(url, timeout=timeout, **kwargs)
- return resp.json() if resp.status_code == 200 else None
- except (httpx.HTTPError, json.JSONDecodeError):
- return None
- def _get_text(url: str, *, timeout: int = 20, **kwargs) -> Optional[str]:
- """Plain (unguarded) GET; body text on 200, None on any other status or transport error."""
- try:
- resp = hub()._skills_hub_http_get(url, timeout=timeout, **kwargs)
- except httpx.HTTPError:
- return None
- return resp.text if resp.status_code == 200 else None
- def _matches_query(query_lower: str, *fields: Any) -> bool:
- """Case-insensitive substring match against joined fields (lists space-joined; empty query matches all)."""
- parts = [" ".join(str(t) for t in f) if isinstance(f, list) else str(f) for f in fields]
- return query_lower in " ".join(parts).lower()
- def _first_matching(query_lower: str, items: Iterable[Any], fields_of: Callable[[Any], tuple],
- to_meta: Callable[[Any], Optional[SkillMeta]], limit: int) -> list[SkillMeta]:
- """Substring-search ``items`` in order, converting hits with ``to_meta`` until ``limit``."""
- results: list[SkillMeta] = []
- for item in items:
- if _matches_query(query_lower, *fields_of(item)) and (meta := to_meta(item)):
- results.append(meta)
- if len(results) >= limit:
- break
- return results
- TRUST_RANK = {"builtin": 2, "trusted": 1, "community": 0}
- def _dedupe_by_trust(results: Iterable[SkillMeta]) -> list[SkillMeta]:
- """Dedupe by identifier, keeping the higher-trust copy (first wins on ties). identifier is unique per
- skill; name is not — two taps can publish same-named skills, and browse-sh reuses task names across sites."""
- seen: dict[str, SkillMeta] = {}
- for r in results:
- kept = seen.get(r.identifier)
- if kept is None or TRUST_RANK.get(r.trust_level, 0) > TRUST_RANK.get(kept.trust_level, 0):
- seen[r.identifier] = r
- return list(seen.values())
- class SkillSource(ABC):
- """Abstract base for all skill registry adapters. ``SOURCE_ID`` is the unique source id (e.g. 'github',
- 'clawhub'); ``TRUST_LEVEL`` the trust every identifier gets unless ``trust_level_for`` is overridden."""
- SOURCE_ID: str = ""
- TRUST_LEVEL: str = "community"
- # Consecutive failed fetches of one catalog page/shard before a walk gives up as partial.
- CATALOG_PAGE_RETRIES = 5
- @abstractmethod
- def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
- """Search for skills matching a query string."""
- @abstractmethod
- def fetch(self, identifier: str) -> Optional[SkillBundle]:
- """Download a skill bundle by identifier."""
- @abstractmethod
- def inspect(self, identifier: str) -> Optional[SkillMeta]:
- """Fetch metadata for a skill without downloading all files."""
- def source_id(self) -> str:
- return self.SOURCE_ID
- def trust_level_for(self, identifier: str) -> str:
- return self.TRUST_LEVEL
- def current_revision(self, identifier: str) -> str:
- """Upstream revision the skill would be fetched at; "" when the registry has no cheap
- revision probe, which keeps update checks on the full-fetch path."""
- return ""
- class GuardedFetchMixin:
- """SSRF/policy-guarded GETs, routed through ``tools.skills_hub`` (test-patchable)."""
- @staticmethod
- def _fetch_text(url: str) -> Optional[str]:
- resp = hub()._guarded_http_get(url, timeout=20)
- return resp.text if resp is not None and resp.status_code == 200 else None
- @staticmethod
- def _fetch_bytes(url: str) -> Optional[bytes]:
- resp = hub()._guarded_http_get(url, timeout=20)
- return resp.content if resp is not None and resp.status_code == 200 else None
- # --- SKILL.md frontmatter ---------------------------------------------------
- def _parse_frontmatter(content: str) -> dict:
- """Parse YAML frontmatter from SKILL.md content ({} when absent/invalid)."""
- content = content.lstrip("\ufeff") # tolerate UTF-8 BOM (Windows editors)
- match = re.search(r'\n---\s*\n', content[3:]) if content.startswith("---") else None
- if not match:
- return {}
- try:
- parsed = yaml.safe_load(content[3:match.start() + 3])
- return parsed if isinstance(parsed, dict) else {}
- except yaml.YAMLError:
- return {}
- def _hermes_tags(fm: dict) -> Any:
- """``metadata.hermes.tags`` from parsed frontmatter, or ``[]`` (unvalidated type)."""
- metadata = fm.get("metadata", {})
- hermes_meta = metadata.get("hermes", {}) if isinstance(metadata, dict) else None
- return hermes_meta.get("tags", []) if isinstance(hermes_meta, dict) else []
- def source_url_for_bundle(bundle: SkillBundle) -> str:
- """Best available human-facing immutable-source provenance URL."""
- explicit = bundle.metadata.get("source_url") or bundle.metadata.get("url")
- if explicit:
- return str(explicit)
- if bundle.source == "github":
- parts = bundle.identifier.split("/", 2)
- if len(parts) >= 2:
- suffix = f"/tree/main/{parts[2]}" if len(parts) == 3 else ""
- return f"https://github.com/{parts[0]}/{parts[1]}{suffix}"
- return bundle.identifier
- # --- Bundle path validation -------------------------------------------------
- def _normalize_bundle_path(path_value: str, *, field_name: str, allow_nested: bool) -> str:
- """Normalize and validate bundle-controlled paths before touching disk."""
- if not isinstance(path_value, str):
- raise ValueError(f"Unsafe {field_name}: expected a string")
- raw = path_value.strip()
- if not raw:
- raise ValueError(f"Unsafe {field_name}: empty path")
- normalized = raw.replace("\\", "/")
- path = PurePosixPath(normalized)
- parts = [part for part in path.parts if part not in {"", "."}]
- # A colon in any component is rejected: on Windows it marks a drive (``C:foo``) or an NTFS Alternate Data
- # Stream (``file.py:payload`` writes scanner-invisible bytes); ``/`` is the only legal separator once normalized.
- if (normalized.startswith("/") or path.is_absolute() or not parts or any(part == ".." for part in parts)
- or any(":" in part for part in parts) or (not allow_nested and len(parts) != 1)):
- raise ValueError(f"Unsafe {field_name}: {path_value}")
- return "/".join(parts)
- def _validate_skill_name(name: str) -> str:
- return _normalize_bundle_path(name, field_name="skill name", allow_nested=False)
- def _validate_install_parent_path(category: str) -> str:
- return _normalize_bundle_path(category, field_name="install parent path", allow_nested=True)
- def _validate_bundle_rel_path(rel_path: str) -> str:
- return _normalize_bundle_path(rel_path, field_name="bundle file path", allow_nested=True)
- def _normalize_lock_install_path(install_path: str, skill_name: str) -> str:
- """Validate a lock-file ``install_path`` (the ``uninstall_skill`` rmtree target).
- Must be relative, traversal-free, and end with ``<skill_name>`` — nested official skills legitimately
- live at ``mlops/training/<skill_name>``; an empty/``"."``/absolute/mismatched entry could point rmtree
- at the whole ``skills/`` tree or outside it.
- """
- safe_skill_name = _validate_skill_name(skill_name)
- normalized = _normalize_bundle_path(install_path, field_name="install path", allow_nested=True)
- if normalized.split("/")[-1] != safe_skill_name:
- raise ValueError(f"Unsafe install path: {install_path}")
- return normalized
- # --- Referenced support-file extraction from SKILL.md -----------------------
- _ALLOWED_SUPPORT_DIRS = frozenset({"references", "templates", "scripts", "assets", "examples"})
- _LOCAL_LINK_RE = re.compile(
- r"(?:\]\(|`|(?:^|[\s\"']))((?:references|templates|scripts|assets|examples)/[^\s)`\"'<>]+)", re.MULTILINE)
- _SUSPICIOUS_LOCAL_REF_RE = re.compile(
- r"(?:references|templates|scripts|assets|examples)/(?:[^\s)`\"'<>]*/)?\.\.(?:/|$)")
- _VALUELESS_QUERY_FLAG_RE = re.compile(r"(?:[A-Za-z0-9_~-]|%[0-9A-Fa-f]{2})+\Z")
- # Same-directory links (``](./FILE.ext)`` / ``](FILE.ext)``): siblings of SKILL.md the document links
- # explicitly (e.g. ./CONTEXT-FORMAT.md). Dropping them made the install "succeed" with unresolved links.
- # The extension requirement keeps prose words out; support-dir links stay on _LOCAL_LINK_RE.
- # Skills legitimately ship supporting docs next to SKILL.md instead of under a support directory (e.g.
- # mattpocock/skills' domain-modeling links ./CONTEXT-FORMAT.md); dropping them made the install "succeed"
- # while the bundle came out with unresolved links (#96310).
- _SAMEDIR_LINK_RE = re.compile(r"\]\(([^)\s\"'<>]+)")
- _SAMEDIR_NAME_RE = re.compile(r"^(?:\./)?[A-Za-z0-9][A-Za-z0-9._-]*$")
- def _query_is_concrete(query: str) -> bool:
- """Whether a URL query is real URL syntax rather than glob prose.
- A non-empty ``key=value`` part is always concrete. Valueless flags are accepted only when RFC 3986
- unreserved-token shaped (percent escapes ok); ``.``, brackets and extra ``?`` are excluded because
- ``?x.md`` / ``?.md`` is indistinguishable from a single-char glob finishing a filename in prose.
- """
- return all(("=" in part and bool(part.split("=", 1)[0])) or bool(_VALUELESS_QUERY_FLAG_RE.fullmatch(part))
- for part in query.split("&"))
- def _referenced_support_paths(skill_md: str) -> Optional[set[str]]:
- """Extract safe referenced paths; return None on a traversal attempt (fail closed)."""
- normalized = skill_md.replace("\\", "/")
- if _SUSPICIOUS_LOCAL_REF_RE.search(normalized):
- return None
- paths: set[str] = set()
- for match in _LOCAL_LINK_RE.finditer(normalized):
- candidate = match.group(1).rstrip(".,;:")
- parsed = urlsplit(candidate)
- raw = unquote(parsed.path)
- if (candidate.endswith("?") or any(char in raw for char in "*?[]")
- or (parsed.query and not _query_is_concrete(parsed.query))):
- continue
- try:
- safe = _validate_bundle_rel_path(raw)
- except ValueError:
- return None
- if safe.split("/", 1)[0] in _ALLOWED_SUPPORT_DIRS:
- # Prose placeholders (``references/type-<name>.md``, truncated at ``<`` to
- # ``references/type-``) are instructions, not files: a basename ending in a
- # separator is skipped. No extension requirement — ``references/LICENSE`` is legitimate.
- base = safe.rsplit("/", 1)[-1]
- if re.search(r"[*?<>]", safe) or not re.search(r"[A-Za-z0-9]$", base):
- continue
- paths.add(safe)
- for match in _SAMEDIR_LINK_RE.finditer(normalized):
- raw = match.group(1).rstrip(".,;:")
- # Canonicalize like the support-dir branch (drop query/fragment, percent-decode), strip leading ``./``.
- name = unquote(urlsplit(raw).path)
- name = name.removeprefix("./")
- # External URLs, anchors, mailto and site-absolute targets are not same-directory file links.
- if not name or "://" in raw or raw.startswith(("mailto:", "#", "/")):
- continue
- if name.startswith(".."):
- # A repo-relative link to a doc outside the skill directory (``../../tools/REGISTRY.md``
- # in a multi-skill repo) is prose, never a bundle path: nothing is fetched or written for
- # it, so refusing the whole bundle protected nothing and made every skill that links a
- # sibling doc uninstallable with a misleading "files no longer exist upstream" (#115171).
- # The link is left dangling in the installed copy, like an absent support file.
- logger.warning("SKILL.md links outside the skill directory; installing without it: %s", raw)
- continue
- # Only unambiguous file links: an extension, no internal slash, never SKILL.md itself (casefolded —
- # a ``skill.md`` entry would collide with the bundle root on macOS/Windows; skipped, not merged).
- if ("/" in name or name.casefold() == "skill.md" or "." not in name.lstrip(".")
- or not _SAMEDIR_NAME_RE.match(name)):
- continue
- try:
- safe = _validate_bundle_rel_path(name)
- except ValueError:
- return None
- paths.add(safe)
- # Case-folded collisions among accepted same-dir names (``A.md`` + ``a.md``)
- # would collide on install — drop the pair rather than guess.
- folded: dict[str, list[str]] = {}
- for p in sorted(paths):
- folded.setdefault(p.casefold(), []).append(p)
- for group in folded.values():
- if len(group) > 1:
- paths.difference_update(group)
- return paths
|