skills_hub_github.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628
  1. """Skills Hub GitHub adapter: API auth, tap providers, and the Contents/Trees source."""
  2. import json
  3. import logging
  4. import subprocess
  5. import time
  6. from pathlib import Path
  7. from typing import Dict, List, Optional, Tuple, Union
  8. from urllib.parse import quote
  9. import httpx
  10. from hermes_cli._subprocess_compat import windows_hide_flags
  11. from agent.retry_utils import parse_retry_after_seconds
  12. from tools.skills_guard import TRUSTED_REPOS
  13. from tools.skills_hub_models import (
  14. SkillBundle, SkillMeta, SkillSource, _cache_metas, _cached_metas, _dedupe_by_trust,
  15. _hermes_tags, _matches_query, _parse_frontmatter, _referenced_support_paths, hub,
  16. _validate_bundle_rel_path,
  17. )
  18. logger = logging.getLogger("tools.skills_hub")
  19. # GitHub tap repo (owner/repo) -> provider label used by the docs-site catalog
  20. # (website/scripts/extract-skills.py::GITHUB_TAP_LABELS). The runtime index collapses every tap into
  21. # source="github"; ``extra.provider`` keeps per-tap identity searchable/filterable without disturbing
  22. # dedup / floor / index-skip logic keyed on the bare source id.
  23. GITHUB_TAP_PROVIDERS = {
  24. "openai/skills": "OpenAI", "anthropics/skills": "Anthropic", "huggingface/skills": "HuggingFace",
  25. "nvidia/skills": "NVIDIA", "voltagent/awesome-agent-skills": "VoltAgent", "garrytan/gstack": "gstack",
  26. "minimax-ai/cli": "MiniMax", "wandb/skills": "Weights & Biases", "qdrant/skills": "Qdrant",
  27. "pinecone-io/skills": "Pinecone", "makenotion/skills": "Notion", "airtable/skills": "Airtable",
  28. "shopify/shopify-ai-toolkit": "Shopify",
  29. }
  30. # Accepted ``--source`` provider filters (lowercased). Not real source ids —
  31. # they narrow merged results to GitHub-tap skills carrying that ``extra.provider``.
  32. _PROVIDER_FILTER_VALUES = frozenset(v.lower() for v in GITHUB_TAP_PROVIDERS.values())
  33. _API = "https://api.github.com/repos"
  34. _ACCEPT_JSON = "application/vnd.github.v3+json"
  35. def github_provider_for(repo: str) -> Optional[str]:
  36. """Provider label for an ``owner/repo`` tap (case-insensitive), or None."""
  37. return GITHUB_TAP_PROVIDERS.get(repo.strip().lower()) if repo else None
  38. def _filter_results_by_provider(results: list[SkillMeta], provider: str) -> list[SkillMeta]:
  39. """Keep only results whose ``extra.provider`` matches ``provider``. An explicit provider filter
  40. (``--source nvidia``) narrows to exactly that provider — the official catalog is NOT injected the
  41. way unfiltered browse does."""
  42. want = provider.strip().lower()
  43. return [r for r in results if str((r.extra or {}).get("provider", "")).lower() == want]
  44. def _provider_filter_of(source_filter: str) -> str:
  45. """Normalized provider filter when ``--source`` names one (nvidia/openai/...), else ``""``."""
  46. value = source_filter.strip().lower()
  47. return value if value in _PROVIDER_FILTER_VALUES else ""
  48. def _tap_cache_key(repo: str, path: str, bucket: Optional[str] = None) -> str:
  49. """Disk-cache key for one tap's skill listing (tests seed that cache through it too)."""
  50. return f"{repo}_{path}_{bucket or ''}".replace("/", "_").replace(" ", "_")
  51. def _is_rate_limit_response(resp: httpx.Response) -> bool:
  52. """403 with exhausted quota, or any 429."""
  53. return resp.status_code == 429 or (
  54. resp.status_code == 403 and resp.headers.get("X-RateLimit-Remaining", "") == "0"
  55. )
  56. class GitHubAuth:
  57. """GitHub API authentication, tried in priority order: GITHUB_TOKEN / GH_TOKEN (PAT), `gh auth token`
  58. (gh CLI), GitHub App JWT + installation token, then unauthenticated (60 req/hr, public repos only)."""
  59. def __init__(self):
  60. self._cached_token: Optional[str] = None
  61. self._cached_method: Optional[str] = None
  62. self._app_token_expiry: float = 0
  63. # Credentials GitHub answered 401 to: skipped from then on so the chain falls through (#98725).
  64. self._rejected_tokens: set = set()
  65. self._rejected_methods: set = set()
  66. self.rejected: list[str] = [] # human labels, for the install error
  67. def get_headers(self) -> dict[str, str]:
  68. token = self._resolve_token()
  69. return {"Accept": _ACCEPT_JSON, **({"Authorization": f"token {token}"} if token else {})}
  70. def is_authenticated(self) -> bool:
  71. return self._resolve_token() is not None
  72. def auth_method(self) -> str:
  73. """'pat', 'gh-cli', 'github-app', or 'anonymous'."""
  74. self._resolve_token()
  75. return self._cached_method or "anonymous"
  76. def _resolve_token(self) -> Optional[str]:
  77. if self._cached_token and (self._cached_method != "github-app" or time.time() < self._app_token_expiry):
  78. return self._cached_token
  79. if self._cached_method == "anonymous":
  80. # The chain already came up empty: re-running `gh auth token` (5 s timeout when gh has no
  81. # login / no keyring) before every API call stretched one install past four minutes.
  82. return None
  83. for method, resolve in (
  84. ("pat", self._try_pat), ("gh-cli", self._try_gh_cli), ("github-app", self._try_github_app),
  85. ):
  86. token = None if method in self._rejected_methods else resolve()
  87. if token and token not in self._rejected_tokens:
  88. self._cached_token, self._cached_method = token, method
  89. if method == "github-app":
  90. self._app_token_expiry = time.time() + 3500 # ~58 min (tokens last 1 hour)
  91. return token
  92. self._cached_method = "anonymous"
  93. return None
  94. def reject(self, token: str) -> None:
  95. """GitHub answered 401 Bad credentials to *token*: stop sending it, so the next method and
  96. finally anonymous are tried. A revoked GITHUB_TOKEN in .env otherwise shadows a working
  97. `gh auth login` (and anonymous access to public repos) for the whole process."""
  98. if token in self._rejected_tokens:
  99. return
  100. self._rejected_tokens.add(token)
  101. if token == self._cached_token and self._cached_method:
  102. label = {"pat": "GITHUB_TOKEN/GH_TOKEN", "gh-cli": "`gh auth token`"}.get(
  103. self._cached_method, "the GitHub App token")
  104. self._rejected_methods.add(self._cached_method)
  105. self.rejected.append(label)
  106. self._cached_token = self._cached_method = None
  107. logger.warning("GitHub rejected %s (401 Bad credentials); trying the next credential", label)
  108. @staticmethod
  109. def _try_pat() -> Optional[str]:
  110. # Profile-scoped secret lookup (multiplexed gateway safe).
  111. from agent.secret_scope import get_secret
  112. return get_secret("GITHUB_TOKEN") or get_secret("GH_TOKEN")
  113. def _try_gh_cli(self) -> Optional[str]:
  114. # Shared with git clones: strips GH_TOKEN/GITHUB_TOKEN, which gh would otherwise echo back
  115. # instead of its keyring login (the dead .env token would come back as the "gh" credential).
  116. from hermes_cli.git_credentials import _gh_cli_token
  117. return _gh_cli_token()
  118. def _try_github_app(self) -> Optional[str]:
  119. from agent.secret_scope import get_secret
  120. app_id, key_path = get_secret("GITHUB_APP_ID"), get_secret("GITHUB_APP_PRIVATE_KEY_PATH")
  121. installation_id = get_secret("GITHUB_APP_INSTALLATION_ID")
  122. if not all([app_id, key_path, installation_id]):
  123. return None
  124. try:
  125. import jwt # PyJWT
  126. except ImportError:
  127. logger.debug("PyJWT not installed, skipping GitHub App auth")
  128. return None
  129. try:
  130. key_file = Path(key_path)
  131. if not key_file.exists():
  132. return None
  133. now = int(time.time())
  134. encoded_jwt = jwt.encode(
  135. {"iat": now - 60, "exp": now + (10 * 60), "iss": app_id},
  136. key_file.read_text(encoding="utf-8-sig"), algorithm="RS256",
  137. )
  138. resp = httpx.post(
  139. f"https://api.github.com/app/installations/{installation_id}/access_tokens",
  140. headers={"Authorization": f"Bearer {encoded_jwt}", "Accept": _ACCEPT_JSON}, timeout=10,
  141. )
  142. if resp.status_code == 201:
  143. return resp.json().get("token")
  144. except Exception as e:
  145. logger.debug("GitHub App auth failed: %s", e)
  146. return None
  147. def _split_repo_id(identifier: str) -> Optional[tuple[str, str]]:
  148. """``owner/repo/path/to/skill`` -> ``(owner/repo, path/to/skill)``; None when too short."""
  149. parts = identifier.split("/", 2)
  150. return (f"{parts[0]}/{parts[1]}", parts[2]) if len(parts) >= 3 else None
  151. def _skill_file_path(skill_path: str, filename: str = "SKILL.md") -> str:
  152. """Path of ``filename`` inside a skill directory. An empty ``skill_path`` means the skill
  153. directory IS the repo root — the single-skill layout some skills.sh repos use (``SKILL.md``
  154. and its ``references/``/``scripts/`` next to ``README.md``)."""
  155. return f"{skill_path}/{filename}" if skill_path else filename
  156. def _skip_bundle_file(rel_path: str) -> bool:
  157. """Dotfiles, bytecode and __pycache__ never ship in a bundle."""
  158. base = rel_path.rsplit("/", 1)[-1]
  159. return base.startswith(".") or base.endswith(".pyc") or "__pycache__" in rel_path.split("/")
  160. def _tree_members(entries: list[dict], prefix: str):
  161. """``(rel_path, item_path, is_regular_blob)`` for every git-tree entry under ``prefix``. Symlinks
  162. (mode 120000) and non-blobs report ``is_regular_blob=False`` so callers can reject a SKILL.md-linked
  163. symlink instead of silently following it."""
  164. for item in entries:
  165. item_path = item.get("path", "")
  166. if item_path.startswith(prefix):
  167. yield item_path[len(prefix):], item_path, item.get("type") == "blob" and item.get("mode") != "120000"
  168. class GitHubSource(SkillSource):
  169. """Fetch skills from GitHub repos via the Contents API."""
  170. DEFAULT_TAPS = [
  171. # openai/skills keeps content under skills/.curated/ + skills/.system/; _list_skills_in_repo
  172. # skips "."/"_" directories, so both entries point at the inner paths.
  173. {"repo": "openai/skills", "path": "skills/.curated/"},
  174. {"repo": "openai/skills", "path": "skills/.system/"},
  175. {"repo": "anthropics/skills", "path": "skills/"},
  176. {"repo": "huggingface/skills", "path": "skills/"},
  177. # NVIDIA-verified skills (CUDA-X, NeMo, cuOpt, ...), each with a signed skill.oms.sig
  178. # + governance card; `trusted` via tools/skills_guard.py::TRUSTED_REPOS.
  179. {"repo": "NVIDIA/skills", "path": "skills/"},
  180. # Vendor-maintained skills for their own products; they replaced our copied
  181. # weights-and-biases / qdrant / pinecone / notion / airtable / shopify optional skills.
  182. {"repo": "wandb/skills", "path": "skills/"},
  183. {"repo": "qdrant/skills", "path": "skills/"},
  184. {"repo": "pinecone-io/skills", "path": "skills/"},
  185. {"repo": "makenotion/skills", "path": "skills/"},
  186. {"repo": "Airtable/skills", "path": "plugins/airtable/skills/"},
  187. {"repo": "Shopify/Shopify-AI-Toolkit", "path": "skills/"},
  188. {"repo": "garrytan/gstack", "path": ""},
  189. # --- Science bucket ---
  190. # Two scientific-skill repos share one hub category via the tap-level "bucket" key so
  191. # their skills surface together. Both stay `community` trust on purpose (NOT in
  192. # tools/skills_guard.py::TRUSTED_REPOS): the guard scans every skill and INSTALL_POLICY
  193. # auto-installs only "safe" ones. Skills wrap third-party tools with their OWN licenses
  194. # (some GPL; KEGG is commercial for non-academic use) — surfaced per skill, not vetted here.
  195. # K-Dense-AI/scientific-agent-skills: flat skills/<name>/, MIT.
  196. {"repo": "K-Dense-AI/scientific-agent-skills", "path": "skills/", "bucket": "science"},
  197. # synthetic-sciences/openscience: Apache-2.0, nested backend/cli/skills/<category>/<name>/.
  198. # _list_skills_in_repo walks ONE level under a tap path, so each category is its own tap
  199. # (same one-entry-per-inner-path pattern as openai/skills above).
  200. *(
  201. {"repo": "synthetic-sciences/openscience", "path": f"backend/cli/skills/{_cat}/", "bucket": "science"}
  202. for _cat in (
  203. "biology", "chemistry", "cloud-compute", "coding", "data-engineering", "databases",
  204. "document-parsing", "llm-tools", "ml-inference", "ml-training", "other", "physics",
  205. "quantum", "research", "scholar-evaluation", "visualization", "writing",
  206. )
  207. ),
  208. ]
  209. SOURCE_ID = "github"
  210. _parse_frontmatter_quick = staticmethod(_parse_frontmatter)
  211. def __init__(self, auth: GitHubAuth, extra_taps: Optional[list[dict]] = None):
  212. self.auth = auth
  213. self.taps = list(self.DEFAULT_TAPS) + list(extra_taps or [])
  214. # Per-instance repo -> (default_branch, tree_entries); lives for one
  215. # search/install flow so repeated tree lookups cost no API calls.
  216. self._tree_cache: dict[str, Optional[tuple[str, list[dict]]]] = {}
  217. self._tree_revisions: dict[str, str] = {}
  218. # repo -> skills.sh.json grouping map; None = fetched, no sidecar.
  219. self._skillsh_groupings: dict[str, Optional[dict[str, str]]] = {}
  220. self._rate_limited: bool = False
  221. self._raw_unusable = False # raw host unreachable here: later files go straight to the API
  222. @property
  223. def is_rate_limited(self) -> bool: # whether the GitHub API rate limit was hit during operations
  224. return self._rate_limited
  225. def trust_level_for(self, identifier: str) -> str:
  226. # identifier format: "owner/repo/path/to/skill"
  227. parts = identifier.split("/", 2)
  228. return "trusted" if len(parts) >= 2 and f"{parts[0]}/{parts[1]}" in TRUSTED_REPOS else "community"
  229. def search(self, query: str, limit: int = 10, *, provider_filter: str = "") -> list[SkillMeta]:
  230. """Substring-match taps, skip taps outside a provider filter, then dedupe by identifier
  231. preferring higher trust and limit."""
  232. results: list[SkillMeta] = []
  233. query_lower = query.lower()
  234. want = provider_filter.strip().lower()
  235. for tap in self.taps:
  236. # The tap repo fixes every result's provider, so wrong-provider taps can be
  237. # skipped before their enumeration cost (cache reads or GitHub API calls).
  238. if want and (github_provider_for(tap["repo"]) or "").lower() != want:
  239. continue
  240. try:
  241. for skill in self._list_skills_in_repo(tap["repo"], tap.get("path", ""), tap.get("bucket")):
  242. if _matches_query(query_lower, skill.name, skill.description, skill.tags):
  243. results.append(skill)
  244. except Exception as e:
  245. logger.debug("Failed to search %s: %s", tap['repo'], e)
  246. return _dedupe_by_trust(results)[:limit]
  247. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  248. """Download a skill; identifier format: "owner/repo/path/to/skill-dir"."""
  249. if (split := _split_repo_id(identifier)) is None:
  250. return None
  251. repo, skill_path = split
  252. skill_dir = skill_path.rstrip("/")
  253. # Resolve the tree FIRST so every byte fetch — SKILL.md included — is pinned to the
  254. # same revision; an unpinned /contents fetch floats to HEAD and can serve bytes newer
  255. # than the tree the paths were validated against (TOCTOU). Idempotent + cached.
  256. tree = self._get_repo_tree(repo)
  257. pinned_ref = self._tree_revisions.get(repo)
  258. skill_md = self._fetch_file_content(repo, _skill_file_path(skill_dir), ref=pinned_ref)
  259. if skill_md is None:
  260. return None
  261. referenced = _referenced_support_paths(skill_md)
  262. if referenced is None:
  263. return None
  264. files: dict[str, str | bytes] = {"SKILL.md": skill_md}
  265. if tree is not None:
  266. complete = self._collect_tree_files(repo, skill_dir, tree[1], pinned_ref, referenced, files)
  267. if complete is None:
  268. return None
  269. # A bundle with a transiently failed blob fetch must not record the tree sha: the
  270. # update check would otherwise see "same revision" and never re-fetch the gap (#101454).
  271. revision = (pinned_ref or tree[0]) if complete else ""
  272. else:
  273. for rel_path in referenced:
  274. self._add_support_file(repo, _skill_file_path(skill_dir, rel_path), rel_path, files, rel_path)
  275. revision = ""
  276. url = (f"https://github.com/{repo}/tree/{revision}" + (f"/{skill_path}" if skill_path else "")
  277. if revision else f"https://github.com/{repo}/{skill_path}")
  278. return SkillBundle(
  279. name=skill_dir.split("/")[-1] or repo.split("/")[-1], files=files, source="github", identifier=identifier,
  280. trust_level=self.trust_level_for(identifier), metadata={"source_url": url, "source_revision": revision},
  281. )
  282. def current_revision(self, identifier: str) -> str:
  283. """Tree sha the default branch currently resolves to — one cached tree lookup per repo,
  284. no blob downloads — so an update check can skip refetching unchanged skills."""
  285. if (split := _split_repo_id(identifier)) is None:
  286. return ""
  287. repo, _ = split
  288. self._get_repo_tree(repo) # populates _tree_revisions
  289. return self._tree_revisions.get(repo, "")
  290. def _add_support_file(self, repo: str, item_path: str, rel_path: str, files: dict, shown: str, **kw) -> bool:
  291. """Fetch one support file into ``files``; a failed fetch warns (naming ``shown``), is skipped, and
  292. returns False."""
  293. content = self._fetch_file_bytes(repo, item_path, **kw)
  294. if content is None:
  295. logger.warning("Failed to fetch referenced skill support file; continuing without it: %s", shown)
  296. return False
  297. files[rel_path] = content
  298. return True
  299. def _collect_tree_files(
  300. self, repo: str, skill_path: str, entries: list[dict], ref: Optional[str], referenced: set,
  301. files: dict[str, str | bytes],
  302. ) -> Optional[bool]:
  303. """Download the FULL skill directory from the pinned tree into ``files``. Link-driven fetching
  304. silently dropped support files under non-canonical dirs (``reference/``, ``agents/``, root
  305. LICENSE); everything still goes through quarantine + scan, and the scanner sees MORE this way.
  306. Returns None (bundle rejected) on an unsafe path or a SKILL.md-linked path that exists in the
  307. tree as a symlink/non-blob — that shape is an escape attempt. A linked path that is simply absent
  308. is a dangling link (repo-only dev tool, prose over-match): warn and install without it. Returns
  309. False when a blob fetch failed (installed with a gap the next update check must be able to fill).
  310. An empty ``skill_path`` is the repo-root skill layout, so the whole repo root is its directory."""
  311. prefix = f"{skill_path}/" if skill_path else ""
  312. symlinked: set = set()
  313. complete = True
  314. for rel_path, item_path, regular in _tree_members(entries, prefix):
  315. if not regular:
  316. symlinked.add(rel_path)
  317. continue
  318. if rel_path == "SKILL.md" or _skip_bundle_file(rel_path):
  319. continue
  320. try:
  321. rel_path = _validate_bundle_rel_path(rel_path)
  322. except ValueError:
  323. logger.warning("Rejected unsafe file path in skill bundle: %s", item_path)
  324. return None
  325. complete &= self._add_support_file(repo, item_path, rel_path, files, item_path, ref=ref)
  326. for rel_path in sorted(referenced):
  327. # A SKILL.md-linked support path that isn't in the tree is a dangling link — a repo-only dev
  328. # tool, prose over-match, or a file the author forgot to push. Warn and install without it
  329. # rather than aborting the whole install (#66760/#90081): the skill body still works, and the
  330. # gap is visible in the log. A referenced path that IS in the tree but as a symlink (or any
  331. # non-regular entry) stays a hard rejection — that shape is an escape attempt, not a forgotten
  332. # file.
  333. if rel_path in symlinked:
  334. logger.warning("Rejected non-regular referenced file in skill bundle: %s%s", prefix, rel_path)
  335. return None
  336. if rel_path not in files:
  337. logger.warning(
  338. "Referenced skill support file is missing; continuing without it: %s%s", prefix, rel_path)
  339. return complete
  340. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  341. """Fetch just the SKILL.md metadata for preview."""
  342. if (split := _split_repo_id(identifier)) is None:
  343. return None
  344. repo, skill_path = split[0], split[1].rstrip("/")
  345. content = self._fetch_file_content(repo, _skill_file_path(skill_path))
  346. if not content:
  347. return None
  348. fm = _parse_frontmatter(content)
  349. tags = _hermes_tags(fm) or (fm["tags"] if isinstance(fm.get("tags"), list) else [])
  350. provider = github_provider_for(repo)
  351. return SkillMeta(
  352. name=fm.get("name", skill_path.split("/")[-1] or repo.split("/")[-1]),
  353. description=str(fm.get("description", "")),
  354. source="github", identifier=identifier, trust_level=self.trust_level_for(identifier),
  355. repo=repo, path=skill_path, tags=[str(t) for t in tags],
  356. extra={"provider": provider} if provider else {},
  357. )
  358. # -- Internal helpers --
  359. def _list_skills_in_repo(self, repo: str, path: str, bucket: Optional[str] = None) -> list[SkillMeta]:
  360. """List skill directories in a GitHub repo path, using cached index. ``bucket`` labels every
  361. skill from a tap whose repo ships no ``skills.sh.json`` grouping, so several repos can share one
  362. hub category (e.g. "science"); a sidecar grouping still wins when present."""
  363. cache_key = _tap_cache_key(repo, path, bucket)
  364. cached = _cached_metas(cache_key)
  365. if cached is not None:
  366. return cached
  367. resp = self._github_get(f"{_API}/{repo}/contents/{path.rstrip('/')}")
  368. if resp is None or resp.status_code != 200:
  369. return []
  370. entries = resp.json()
  371. if not isinstance(entries, list):
  372. return []
  373. skills: list[SkillMeta] = []
  374. groupings = self._get_skillsh_groupings(repo)
  375. prefix = path.rstrip("/")
  376. for entry in entries:
  377. if entry.get("type") != "dir" or entry["name"].startswith((".", "_")):
  378. continue
  379. dir_name = entry["name"]
  380. meta = self.inspect(f"{repo}/{prefix}/{dir_name}" if prefix else f"{repo}/{dir_name}")
  381. if meta:
  382. category = (groupings and (groupings.get(meta.name) or groupings.get(dir_name))) or bucket
  383. if category:
  384. meta.extra["category"] = category
  385. skills.append(meta)
  386. _cache_metas(cache_key, skills)
  387. return skills
  388. def _get_repo_tree(self, repo: str) -> Optional[tuple[str, list[dict]]]:
  389. """Cached ``(default_branch, tree_entries)`` for a repo, or None. One install may need the tree
  390. several times; caching saves the ``GET /repos/{repo}`` + ``GET .../git/trees/{branch}`` pair each
  391. time (~12 of the 60/hr unauthenticated budget before)."""
  392. if repo in self._tree_cache:
  393. return self._tree_cache[repo]
  394. # Misses are cached too: within one command a truncated/unreachable tree stays that way,
  395. # and the update check now probes the tree before every fetch (#101454).
  396. self._tree_cache[repo] = None
  397. repo_data = self._github_json(f"{_API}/{repo}")
  398. if repo_data is None:
  399. return None
  400. default_branch = repo_data.get("default_branch", "main")
  401. tree_data = self._github_json(
  402. f"{_API}/{repo}/git/trees/{default_branch}", params={"recursive": "1"}, timeout=30.0,
  403. )
  404. if tree_data is None:
  405. return None
  406. if tree_data.get("truncated"):
  407. logger.debug("Git tree truncated for %s", repo)
  408. return None
  409. if isinstance(tree_data.get("sha"), str) and tree_data["sha"]:
  410. self._tree_revisions[repo] = tree_data["sha"]
  411. self._tree_cache[repo] = tree = (default_branch, tree_data.get("tree", []))
  412. return tree
  413. def _github_json(self, url: str, **kwargs) -> Optional[dict]:
  414. """Decoded JSON body of a 200 ``_github_get`` (which flags rate-limit exhaustion), else None."""
  415. resp = self._github_get(url, **kwargs)
  416. try:
  417. return resp.json() if resp is not None and resp.status_code == 200 else None
  418. except ValueError:
  419. return None
  420. def _github_get(
  421. self, url: str, *, params: Optional[dict] = None, headers: Optional[dict] = None,
  422. timeout: float = 15.0, max_retries: int = 3,
  423. ) -> Optional[httpx.Response]:
  424. """GET against the GitHub API with retry/backoff on transient failures. Returns the final
  425. response (caller inspects status) or None when every attempt raised a transport error.
  426. Retries rate-limit 403/429 (waiting until ``Retry-After`` / ``X-RateLimit-Reset`` when present,
  427. capped 60s — one shared limit zeroes every GitHub tap at once during an index build), 5xx, and
  428. transport errors with exponential backoff. Terminal rate-limit exhaustion flags the instance so
  429. an index build fails loud instead of silently shipping zero GitHub skills."""
  430. hdrs = headers if headers is not None else self.auth.get_headers()
  431. backoff = 1.0
  432. last_resp: Optional[httpx.Response] = None
  433. for attempt in range(max_retries):
  434. last_attempt = attempt >= max_retries - 1
  435. wait = backoff
  436. try:
  437. resp = hub()._skills_hub_http_get(
  438. url, params=params, headers=hdrs, timeout=timeout, follow_redirects=True
  439. )
  440. except httpx.HTTPError as e:
  441. logger.debug("GitHub GET %s failed (attempt %d/%d): %s", url, attempt + 1, max_retries, e)
  442. if last_attempt:
  443. return None
  444. else:
  445. last_resp = resp
  446. if resp.status_code == 200:
  447. return resp
  448. sent = hdrs.get("Authorization", "")
  449. if resp.status_code == 401 and sent:
  450. # Bad credentials: same request with the next credential, then anonymous.
  451. self.auth.reject(sent.split(" ", 1)[-1])
  452. retry = {k: v for k, v in hdrs.items() if k != "Authorization"}
  453. retry.update((k, v) for k, v in self.auth.get_headers().items() if k == "Authorization")
  454. return self._github_get(url, params=params, headers=retry, timeout=timeout,
  455. max_retries=max_retries)
  456. if resp.status_code in (403, 429):
  457. limited = _is_rate_limit_response(resp)
  458. if not limited or last_attempt:
  459. if limited: # terminal exhaustion: flag the instance so callers fail loud
  460. self._rate_limited = True
  461. logger.warning("GitHub API rate limit exhausted (unauthenticated: 60 req/hr). "
  462. "Set GITHUB_TOKEN or install the gh CLI to raise the limit to 5,000/hr.")
  463. return resp
  464. reset = resp.headers.get("X-RateLimit-Reset", "")
  465. retry_after = parse_retry_after_seconds(resp.headers)
  466. if retry_after is not None:
  467. wait = min(retry_after, 60.0)
  468. elif reset.isdigit():
  469. delta = float(reset) - time.time()
  470. if 0 < delta <= 60.0:
  471. wait = delta
  472. logger.debug("GitHub rate limited on %s, waiting %.1fs (attempt %d/%d)",
  473. url, wait, attempt + 1, max_retries)
  474. elif not (500 <= resp.status_code < 600) or last_attempt:
  475. return resp
  476. time.sleep(wait)
  477. backoff = min(backoff * 2, 30.0)
  478. return last_resp
  479. def _find_skill_in_repo_tree(self, repo: str, skill_name: str) -> Optional[str]:
  480. """Locate ``<skill_name>/SKILL.md`` anywhere in the repo tree (one API call); full identifier or None."""
  481. if (cached := self._get_repo_tree(repo)) is None:
  482. return None
  483. skill_md_suffix = f"/{skill_name}/SKILL.md"
  484. skill_dirs = []
  485. for entry in cached[1]:
  486. path = entry.get("path", "")
  487. if entry.get("type") != "blob" or entry.get("mode") == "120000" or not f"/{path}".endswith("/SKILL.md"):
  488. continue
  489. if path.endswith(skill_md_suffix) or path == skill_md_suffix[1:]:
  490. return f"{repo}/{path[: -len('/SKILL.md')]}"
  491. skill_dirs.append(path[: -len("SKILL.md")].rstrip("/"))
  492. # A single-skill repo may keep it in a generic dir (``skills/SKILL.md``) no slug matches; a lone
  493. # NAMED dir (``skills/bar``) is another skill, and a lone root SKILL.md is ``_find_repo_root_skill``'s.
  494. return f"{repo}/{skill_dirs[0]}" if skill_dirs in (["skills"], [".agents/skills"], [".claude/skills"]) else None
  495. def _find_repo_root_skill(self, repo: str) -> Optional[str]:
  496. """Identifier for a single-skill repo whose ``SKILL.md`` sits at the repo ROOT (no skill
  497. directory) — e.g. ``orzcls/win-disk-cleaner``. The empty path segment (``owner/repo/``)
  498. denotes the skill directory being the repo root. Only repos with EXACTLY ONE SKILL.md in
  499. the whole tree qualify, so a categorized multi-skill repo never resolves here."""
  500. tree = self._get_repo_tree(repo)
  501. if tree is None:
  502. return None
  503. skill_mds = [
  504. entry.get("path", "") for entry in tree[1]
  505. if entry.get("type") == "blob" and entry.get("mode") != "120000"
  506. and (entry.get("path", "") == "SKILL.md" or entry.get("path", "").endswith("/SKILL.md"))
  507. ]
  508. return f"{repo}/" if skill_mds == ["SKILL.md"] else None
  509. def _fetch_file_content(self, repo: str, path: str, ref: Optional[str] = None) -> Optional[str]:
  510. """Fetch a single text file from GitHub (None on miss or non-UTF-8)."""
  511. content = self._fetch_file_bytes(repo, path, ref=ref)
  512. try:
  513. return None if content is None else content.decode("utf-8")
  514. except UnicodeDecodeError:
  515. return None
  516. def _fetch_file_bytes(self, repo: str, path: str, ref: Optional[str] = None) -> Optional[bytes]:
  517. """Fetch exact file bytes. ``ref`` pins to a tree SHA (see ``fetch`` on
  518. the TOCTOU); None keeps the legacy unpinned behavior. A pinned file comes from
  519. raw.githubusercontent.com first: the same bytes at the same commit, outside the REST rate
  520. limit. Through the Contents API every file in a skill folder cost one call, so an anonymous
  521. user (60/h) could never install a skill with more than ~58 files. A private repo or a raw
  522. miss falls back to the API. A transport failure (DNS-poisoned or NXDOMAIN raw host surfaces as
  523. SSRFConnectionBlocked, a dropping firewall as a 15 s timeout) also falls back, and marks raw
  524. unusable for the rest of this source's flow."""
  525. if ref and not self._raw_unusable:
  526. try:
  527. raw = hub()._skills_hub_http_get(
  528. f"https://raw.githubusercontent.com/{repo}/{ref}/{quote(path, safe='/')}",
  529. timeout=15.0, follow_redirects=False)
  530. except (httpx.HTTPError, OSError, ValueError): # ValueError: SSRFConnectionBlocked
  531. self._raw_unusable = True
  532. else:
  533. if raw.status_code == 200:
  534. return raw.content
  535. resp = self._github_get(
  536. f"{_API}/{repo}/contents/{quote(path, safe='/')}", params={"ref": ref} if ref else None,
  537. headers={**self.auth.get_headers(), "Accept": "application/vnd.github.v3.raw"},
  538. )
  539. return resp.content if resp is not None and resp.status_code == 200 else None
  540. def _get_skillsh_groupings(self, repo: str) -> Optional[dict[str, str]]:
  541. """Repo-root ``skills.sh.json`` groupings flattened to ``{skill_name: title}``. ``skills.sh.json``
  542. is a cross-ecosystem standard (``$schema: https://skills.sh/schemas/skills.sh.schema.json``); any
  543. tap shipping it gets category pills for free. None when absent/unparsable; cached per repo."""
  544. if repo not in self._skillsh_groupings:
  545. content = self._fetch_file_content(repo, "skills.sh.json")
  546. self._skillsh_groupings[repo] = self._parse_skillsh_groupings(content) if content else None
  547. return self._skillsh_groupings[repo]
  548. @staticmethod
  549. def _parse_skillsh_groupings(content: str) -> Optional[dict[str, str]]:
  550. """Flatten ``{"groupings": [{"title", "skills": [...]}]}``; None if not usable."""
  551. try:
  552. data = json.loads(content)
  553. except (json.JSONDecodeError, TypeError):
  554. return None
  555. groupings = data.get("groupings") if isinstance(data, dict) else None
  556. if not isinstance(groupings, list):
  557. return None
  558. mapping: dict[str, str] = {}
  559. for group in groupings:
  560. if not isinstance(group, dict):
  561. continue
  562. title, members = group.get("title"), group.get("skills")
  563. if not isinstance(title, str) or not isinstance(members, list):
  564. continue
  565. for member in members:
  566. if isinstance(member, str) and member:
  567. mapping.setdefault(member, title) # first grouping wins
  568. return mapping