skills_hub_install.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325
  1. """Skills Hub install/uninstall/update operations: quarantine staging, install-
  2. target safety (symlink/junction, category-bucket and nested-skill checks),
  3. lock-file-backed uninstall, bundle hashing and upstream update checks.
  4. Split out of ``tools/skills_hub.py``; hub state (paths, ``HubLockFile``, audit log)
  5. is still read from there at call time.
  6. """
  7. from __future__ import annotations
  8. import logging
  9. import hashlib
  10. import shutil
  11. from pathlib import Path
  12. from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
  13. from agent.skill_utils import is_excluded_skill_path
  14. from tools.skills_guard import ScanResult, content_hash
  15. from tools.skills_hub_github import GitHubAuth
  16. from tools.skills_hub_models import (
  17. SkillBundle, SkillSource, _normalize_lock_install_path, _validate_bundle_rel_path,
  18. _validate_install_parent_path, _validate_skill_name,
  19. )
  20. if TYPE_CHECKING: # origin class; runtime use is via the lazy origin import
  21. from tools.skills_hub import HubLockFile
  22. # Log-record parity with the origin module.
  23. logger = logging.getLogger("tools.skills_hub")
  24. def _is_path_redirect(path: Path) -> bool:
  25. """True when ``path`` is a symlink or (on Windows) a directory junction —
  26. either lets a writer in ``skills/`` redirect a later ``rmtree`` outside it.
  27. ``is_junction`` only exists on Python 3.12+ Windows; gate with ``hasattr``."""
  28. return path.is_symlink() or (hasattr(path, "is_junction") and path.is_junction())
  29. def _resolve_lock_install_path(install_path: str, skill_name: str) -> Path:
  30. """Resolve a lock-file install path without allowing escapes from ``SKILLS_DIR``.
  31. Walks component-by-component refusing symlink/junction redirects (which
  32. ``Path.resolve`` would silently follow), then rejects both escape-out and
  33. ``resolved == SKILLS_DIR`` — an empty/``"."`` install_path resolves to the
  34. skills root and ``rmtree`` there would wipe every installed skill.
  35. """
  36. from tools.skills_hub import _skills_dir
  37. normalized = _normalize_lock_install_path(install_path, skill_name)
  38. target = skills_dir = _skills_dir()
  39. skills_root = skills_dir.resolve()
  40. for part in normalized.split("/"):
  41. target = target / part
  42. if _is_path_redirect(target):
  43. raise ValueError(f"Unsafe install path: {install_path}")
  44. target = target.resolve()
  45. if target == skills_root or not target.is_relative_to(skills_root):
  46. raise ValueError(f"Unsafe install path: {install_path}")
  47. return target
  48. def quarantine_bundle(bundle: SkillBundle) -> Path:
  49. """Write a skill bundle to the quarantine directory for scanning."""
  50. from tools.skills_hub import _quarantine_dir, ensure_hub_dirs
  51. ensure_hub_dirs()
  52. skill_name = _validate_skill_name(bundle.name)
  53. # Validate every path before touching disk so a bad member aborts cleanly.
  54. validated_files = [(_validate_bundle_rel_path(rel_path), content) for rel_path, content in bundle.files.items()]
  55. dest = _quarantine_dir() / skill_name
  56. if dest.exists():
  57. shutil.rmtree(dest)
  58. dest.mkdir(parents=True)
  59. for rel_path, file_content in validated_files:
  60. file_dest = dest.joinpath(*rel_path.split("/"))
  61. file_dest.parent.mkdir(parents=True, exist_ok=True)
  62. if isinstance(file_content, bytes):
  63. file_dest.write_bytes(file_content)
  64. else:
  65. # newline="" keeps the bundle's LF bytes verbatim; the default None mode would
  66. # translate to os.linesep on Windows and desync content_hash from bundle_content_hash.
  67. file_dest.write_text(file_content, encoding="utf-8", newline="")
  68. return dest
  69. def _category_skill_dirs(directory: Path) -> list[str]:
  70. """Names of non-hidden child dirs holding at least one active SKILL.md
  71. anywhere below (nested layouts like ``mlops/training/<skill>`` count).
  72. Vendored/cache/progressive-disclosure paths are pruned via
  73. :func:`is_excluded_skill_path` so a lone ``node_modules`` or
  74. ``references/pkg/SKILL.md`` does not make the directory a category.
  75. Shared with ``hermes_cli.skills_hub._existing_categories``.
  76. """
  77. return [
  78. entry.name for entry in directory.iterdir()
  79. if entry.is_dir() and not entry.name.startswith(".") and any(
  80. not is_excluded_skill_path(skill_md.relative_to(directory), root=directory)
  81. for skill_md in entry.rglob("SKILL.md")
  82. )
  83. ]
  84. def _check_install_target(install_dir: Path) -> None:
  85. """Raise ValueError when installing at ``install_dir`` would destroy other skills.
  86. - Nesting inside an existing skill dir (``--category <existing-skill>``)
  87. would make a later update/uninstall of the outer skill rmtree the inner one.
  88. - A stray regular file at the target: rmtree would raise NotADirectoryError.
  89. - A category bucket (dir without SKILL.md that holds other skills) must never
  90. be silently wiped; a dir that directly contains SKILL.md is an existing
  91. install and stays overwritable (hub installs are lock-guarded in do_install).
  92. """
  93. from tools.skills_hub import _skills_dir
  94. # Refuse to nest a skill inside an existing skill directory. Installing with ``--category
  95. # <name-of-an-existing-skill>`` would create a hybrid skill-plus-category directory; a later update or
  96. # uninstall of the outer skill would then rmtree the inner one — the sibling case of the category-bucket
  97. # wipe reported in issue #75983.
  98. skills_root = _skills_dir().resolve()
  99. ancestor = install_dir.parent
  100. while ancestor != skills_root and ancestor.is_relative_to(skills_root):
  101. if (ancestor / "SKILL.md").is_file():
  102. raise ValueError(f"Refusing to install into '{ancestor.name}': it is an "
  103. f"existing skill directory, not a category. Choose a different category.")
  104. ancestor = ancestor.parent
  105. if not install_dir.exists():
  106. return
  107. if not install_dir.is_dir():
  108. raise ValueError(f"Refusing to install: '{install_dir.name}' already exists "
  109. f"and is not a directory. Remove it or choose a different skill name.")
  110. # Guard against silent data loss when the install target collides with an existing category bucket (a
  111. # directory that holds other skills). This was reported as GitHub issue #75983: installing a skill with
  112. # --name matching an existing category directory caused rmtree to wipe all sibling skills. A directory
  113. # that directly contains SKILL.md is an existing skill installation and stays overwritable
  114. # (hub-installed skills are additionally guarded by the lock-file check in do_install()). But a
  115. # directory that contains *other* skill directories is a category bucket and must NOT be silently
  116. # deleted.
  117. if not (install_dir / "SKILL.md").exists():
  118. skill_dirs_in = _category_skill_dirs(install_dir)
  119. if skill_dirs_in:
  120. raise ValueError(f"Refusing to overwrite category directory '{install_dir}' "
  121. f"which contains {len(skill_dirs_in)} skill(s): {', '.join(sorted(skill_dirs_in))}. "
  122. f"Use a different --name or install into a subcategory.")
  123. def install_from_quarantine(
  124. quarantine_path: Path, skill_name: str, category: str, bundle: SkillBundle, scan_result: ScanResult,
  125. scan_provenance: Optional[dict[str, Any]] = None,
  126. ) -> Path:
  127. """Move a scanned skill from quarantine into the skills directory."""
  128. from tools.skills_hub import HubLockFile, _quarantine_dir, _skills_dir, append_audit_log
  129. safe_skill_name = _validate_skill_name(skill_name)
  130. safe_category = _validate_install_parent_path(category) if category else ""
  131. quarantine_resolved = quarantine_path.resolve()
  132. if not quarantine_resolved.is_relative_to(_quarantine_dir().resolve()):
  133. raise ValueError(f"Unsafe quarantine path: {quarantine_path}")
  134. install_rel_path = f"{safe_category}/{safe_skill_name}" if safe_category else safe_skill_name
  135. # Same validator the uninstaller uses, so a lock entry can never point at a
  136. # symlink-redirected target.
  137. install_dir = _resolve_lock_install_path(install_rel_path, safe_skill_name)
  138. _check_install_target(install_dir)
  139. if install_dir.exists():
  140. shutil.rmtree(install_dir)
  141. try:
  142. skill_size = (quarantine_path / "SKILL.md").stat().st_size
  143. except OSError:
  144. skill_size = 0
  145. if skill_size > 100_000:
  146. logger.warning(
  147. "Skill '%s' has a large SKILL.md (%s chars). "
  148. "Large skills consume significant context when loaded. "
  149. "Consider asking the author to split it into smaller files.",
  150. safe_skill_name, f"{skill_size:,}",
  151. )
  152. # A symlink in the bundle would copy its target into skills/ and leak it
  153. # to the agent on the next skill_view.
  154. for entry in quarantine_path.rglob("*"):
  155. if _is_path_redirect(entry):
  156. try:
  157. rel = entry.relative_to(quarantine_resolved)
  158. except ValueError:
  159. rel = entry
  160. raise ValueError(f"Installed skill contains symlinks, which is not allowed: {rel}")
  161. install_dir.parent.mkdir(parents=True, exist_ok=True)
  162. shutil.move(str(quarantine_path), str(install_dir))
  163. installed_hash = content_hash(install_dir)
  164. HubLockFile().record_install(
  165. name=safe_skill_name, source=bundle.source, identifier=bundle.identifier, trust_level=bundle.trust_level,
  166. scan_verdict=scan_result.verdict, skill_hash=installed_hash,
  167. install_path=install_dir.resolve().relative_to(_skills_dir().resolve()).as_posix(),
  168. files=list(bundle.files.keys()), metadata=bundle.metadata,
  169. scan_provenance=scan_provenance or getattr(scan_result, "scan_provenance", None),
  170. )
  171. append_audit_log("INSTALL", safe_skill_name, bundle.source, bundle.trust_level, scan_result.verdict,
  172. installed_hash)
  173. try:
  174. from tools.skill_usage import record_installed
  175. record_installed(safe_skill_name)
  176. except Exception:
  177. logger.debug("Unable to record skill install lifecycle for %s", safe_skill_name, exc_info=True)
  178. return install_dir
  179. def uninstall_skill(skill_name: str) -> tuple[bool, str]:
  180. """Remove a hub-installed skill. Refuses to remove builtins."""
  181. from tools.skills_hub import HubLockFile, append_audit_log
  182. lock = HubLockFile()
  183. entry = lock.get_installed(skill_name)
  184. if not entry:
  185. return False, f"'{skill_name}' is not a hub-installed skill (may be a builtin)"
  186. # The destructive boundary: whatever reaches rmtree MUST be inside
  187. # SKILLS_DIR and MUST NOT be SKILLS_DIR itself (see _resolve_lock_install_path).
  188. try:
  189. install_path = _resolve_lock_install_path(entry.get("install_path", ""), skill_name)
  190. except ValueError as exc:
  191. return False, f"Refusing to uninstall '{skill_name}': {exc}"
  192. if install_path.exists():
  193. shutil.rmtree(install_path)
  194. lock.record_uninstall(skill_name)
  195. append_audit_log("UNINSTALL", skill_name, entry["source"], entry["trust_level"], "n/a", "user_request")
  196. from hermes_cli.observability.shared_metrics_disabled import record_skill_removed
  197. record_skill_removed(skill_name)
  198. return True, f"Uninstalled '{skill_name}' from {entry['install_path']}"
  199. def bundle_content_hash(bundle: SkillBundle) -> str:
  200. """Deterministic hash of an in-memory bundle.
  201. MUST stay symmetric with ``tools.skills_guard.content_hash`` (same skill
  202. from disk), which keys files by POSIX relative path. Windows bundle keys
  203. carry backslashes, which changed both bytes and sort order and made every
  204. skill report ``update_available`` forever — normalize before hashing. The
  205. path is hashed too so swapping contents between two files changes the hash.
  206. That function keys files by ``relative_to(...).as_posix()`` — forward slashes on every OS. See #62310.
  207. """
  208. h = hashlib.sha256()
  209. normalized = {rel_path.replace("\\", "/"): content for rel_path, content in bundle.files.items()}
  210. for rel_path in sorted(normalized):
  211. h.update(rel_path.encode("utf-8"))
  212. h.update(b"\x00")
  213. content = normalized[rel_path]
  214. h.update(content if isinstance(content, bytes) else content.encode("utf-8"))
  215. return f"sha256:{h.hexdigest()[:16]}"
  216. _SOURCE_ID_ALIASES = {"skills.sh": "skills-sh"}
  217. def _source_matches(source: SkillSource, source_name: str) -> bool:
  218. return source.source_id() == _SOURCE_ID_ALIASES.get(source_name, source_name)
  219. def _current_revision_or_empty(src, identifier: str) -> str:
  220. """A probe failure must degrade one row to the full-fetch path, not abort the whole check."""
  221. try:
  222. return src.current_revision(identifier)
  223. except Exception:
  224. logger.debug("current_revision probe failed for %s", identifier, exc_info=True)
  225. return ""
  226. def check_for_skill_updates(
  227. name: Optional[str] = None, *, lock: Optional[HubLockFile] = None,
  228. sources: Optional[list[SkillSource]] = None, auth: Optional[GitHubAuth] = None,
  229. ) -> list[dict]:
  230. """Check installed hub skills for upstream changes.
  231. Each entry is fetched ONLY from adapters matching its recorded source.
  232. Falling back to all sources let a same-named skill in a different registry
  233. satisfy the fetch and silently reassign provenance (names are not
  234. namespaced across registries), so a missing adapter reports "unavailable".
  235. """
  236. from tools.skills_hub import HubLockFile
  237. from tools.skills_hub_search import create_source_router
  238. lock = lock or HubLockFile()
  239. installed = lock.list_installed()
  240. if name:
  241. installed = [entry for entry in installed if entry.get("name") == name]
  242. if sources is None:
  243. sources = create_source_router(auth=auth)
  244. results: list[dict] = []
  245. for entry in installed:
  246. identifier, source_name = entry.get("identifier", ""), entry.get("source", "")
  247. row = {"name": entry.get("name", ""), "identifier": identifier, "source": source_name}
  248. try:
  249. install_dir = _resolve_lock_install_path(
  250. entry.get("install_path", ""), entry.get("name", "skill"))
  251. orphaned = not install_dir.is_dir()
  252. except (ValueError, OSError, RuntimeError):
  253. results.append({**row, "status": "invalid_install"})
  254. continue
  255. if orphaned:
  256. # The lock-file entry points at a directory that is gone: the fetched
  257. # bundle could never be applied, so skip the network cost entirely
  258. # instead of re-paying it on every update run (#104291).
  259. results.append({**row, "status": "orphaned"})
  260. continue
  261. matching = [src for src in sources if _source_matches(src, source_name)]
  262. recorded = (entry.get("metadata") or {}).get("source_revision", "")
  263. if recorded and any(_current_revision_or_empty(src, identifier) == recorded for src in matching):
  264. # Same upstream tree as at install time: the bundle bytes cannot differ, so
  265. # skip downloading SKILL.md + every support blob just to re-hash them (#101454).
  266. content_hash = entry.get("content_hash", "")
  267. results.append({**row, "status": "up_to_date", "current_hash": content_hash, "latest_hash": content_hash})
  268. continue
  269. bundle = None
  270. for src in matching:
  271. try:
  272. bundle = src.fetch(identifier)
  273. except Exception:
  274. bundle = None
  275. if bundle:
  276. break
  277. if not bundle:
  278. results.append({**row, "status": "unavailable"})
  279. continue
  280. current_hash, latest_hash = entry.get("content_hash", ""), bundle_content_hash(bundle)
  281. results.append({
  282. **row, "status": "up_to_date" if current_hash == latest_hash else "update_available",
  283. "current_hash": current_hash, "latest_hash": latest_hash, "bundle": bundle,
  284. })
  285. return results