skills_hub.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484
  1. #!/usr/bin/env python3
  2. """
  3. Skills Hub — hub state management and the public facade for the source adapters.
  4. Library module (not an agent tool). Owns the hub paths, guarded HTTP, index
  5. cache, lock file, taps and audit log. Install/uninstall/update live in
  6. ``skills_hub_install``, the index fetch/source router/search in
  7. ``skills_hub_search``, and the adapters in the other ``tools.skills_hub_*``
  8. siblings; import each name from its defining module.
  9. Used by hermes_cli/skills_hub.py for CLI commands and the /skills slash command.
  10. """
  11. import json
  12. import logging
  13. import threading
  14. import time
  15. from contextvars import ContextVar
  16. from contextlib import ExitStack, contextmanager
  17. from datetime import datetime, timezone, UTC
  18. from pathlib import Path
  19. from typing import Any, Callable, Dict, Iterator, List, Optional
  20. from urllib.parse import urljoin
  21. import httpx
  22. from hermes_constants import get_hermes_home
  23. from tools.url_safety import is_safe_url
  24. from tools.url_safety import create_ssrf_safe_client
  25. from tools.website_policy import check_website_access
  26. from tools.skills_hub_models import _normalize_lock_install_path, _validate_skill_name
  27. from utils import atomic_write_text
  28. logger = logging.getLogger(__name__)
  29. # ---------------------------------------------------------------------------
  30. # Paths
  31. # ---------------------------------------------------------------------------
  32. # Resolved per-call (not frozen at import) so the profile override is honored;
  33. # import-time constants leaked across profiles in single-process multi-profile
  34. # runtimes. The path names (SKILLS_DIR, ...) resolve live via __getattr__ below.
  35. INDEX_CACHE_TTL = 3600 # 1 hour
  36. def _path_resolver(name: str, parent: str, leaf: str):
  37. """Resolver for hub path ``<parent>/<leaf>``: a test-injected real module attribute
  38. (patch.object/monkeypatch on SKILLS_DIR etc.) wins over live resolution."""
  39. def resolve() -> Path:
  40. forced = globals().get(name)
  41. return Path(forced) if forced is not None else _DYNAMIC_PATH_RESOLVERS[parent]() / leaf
  42. resolve.__name__ = f"_{name.lower()}"
  43. return resolve
  44. def _hermes_home() -> Path:
  45. return get_hermes_home()
  46. _skills_dir = _path_resolver("SKILLS_DIR", "HERMES_HOME", "skills")
  47. _hub_dir = _path_resolver("HUB_DIR", "SKILLS_DIR", ".hub")
  48. _lock_file = _path_resolver("LOCK_FILE", "HUB_DIR", "lock.json")
  49. _quarantine_dir = _path_resolver("QUARANTINE_DIR", "HUB_DIR", "quarantine")
  50. _audit_log = _path_resolver("AUDIT_LOG", "HUB_DIR", "audit.log")
  51. _taps_file = _path_resolver("TAPS_FILE", "HUB_DIR", "taps.json")
  52. _index_cache_dir = _path_resolver("INDEX_CACHE_DIR", "HUB_DIR", "index-cache")
  53. _DYNAMIC_PATH_RESOLVERS = {"HERMES_HOME": _hermes_home, **{
  54. r.__name__[1:].upper(): r
  55. for r in (_skills_dir, _hub_dir, _lock_file, _quarantine_dir, _audit_log, _taps_file, _index_cache_dir)
  56. }}
  57. def __getattr__(name: str):
  58. """Resolve legacy path constants dynamically (PEP 562) so they reflect the
  59. active profile override; a test's patch.object-set real attribute shadows it."""
  60. resolver = _DYNAMIC_PATH_RESOLVERS.get(name)
  61. if resolver is not None:
  62. return resolver()
  63. raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
  64. # ---------------------------------------------------------------------------
  65. # Install-path safety + guarded HTTP
  66. # ---------------------------------------------------------------------------
  67. _REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
  68. _MAX_SKILL_FETCH_REDIRECTS = 5
  69. # Default per-request timeout every one-shot hub GET used before pooling existed.
  70. _DEFAULT_HTTP_TIMEOUT = 20
  71. # An inspect resolves metadata and then the preview bundle through the same
  72. # source adapter. Keeping this context local to that operation lets httpx reuse
  73. # its verified connection without extending a client beyond the CLI request.
  74. _skills_hub_http_client: ContextVar[Optional[Any]] = ContextVar(
  75. "skills_hub_http_client", default=None
  76. )
  77. @contextmanager
  78. def skills_hub_http_session() -> Iterator[None]:
  79. """Reuse one SSRF-safe HTTP client for a single skills-hub resolution."""
  80. if _skills_hub_http_client.get() is not None:
  81. yield
  82. return
  83. with create_ssrf_safe_client(timeout=_DEFAULT_HTTP_TIMEOUT, follow_redirects=False) as client:
  84. token = _skills_hub_http_client.set(client)
  85. try:
  86. yield
  87. finally:
  88. _skills_hub_http_client.reset(token)
  89. def _skills_hub_http_get(url: str, **kwargs: Any) -> httpx.Response:
  90. """GET through the active resolution pool, or preserve one-shot behavior."""
  91. client = _skills_hub_http_client.get()
  92. if client is not None:
  93. return client.get(url, **kwargs)
  94. return httpx.get(url, **kwargs)
  95. def _ssrf_safe_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
  96. headers: Optional[dict[str, str]] = None) -> httpx.Response:
  97. """Fetch one URL with connect-time SSRF validation and no automatic redirects."""
  98. with skills_hub_http_session():
  99. return _skills_hub_http_client.get().get(url, timeout=timeout, headers=headers)
  100. def _guarded_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
  101. headers: Optional[dict[str, str]] = None) -> Optional[httpx.Response]:
  102. """Fetch a URL with SSRF and redirect-target validation (each hop re-checked).
  103. *headers* are plain request headers (no credentials) and are sent on every hop."""
  104. from tools.url_safety import SSRFConnectionBlocked
  105. current_url = url
  106. for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
  107. if not is_safe_url(current_url):
  108. logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
  109. return None
  110. blocked = check_website_access(current_url)
  111. if blocked:
  112. logger.info(
  113. "Blocked Skills Hub fetch for %s by rule %s",
  114. blocked["host"],
  115. blocked["rule"],
  116. )
  117. return None
  118. try:
  119. resp = _ssrf_safe_http_get(current_url, timeout=timeout, headers=headers)
  120. except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
  121. logger.debug("Skills Hub fetch failed for %s: %s", current_url, exc)
  122. return None
  123. if resp.status_code in _REDIRECT_STATUS_CODES:
  124. location = getattr(resp, "headers", {}).get("location")
  125. if not location:
  126. return None
  127. current_url = urljoin(current_url, location)
  128. continue
  129. return resp
  130. logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
  131. return None
  132. @contextmanager
  133. def _guarded_http_stream(
  134. url: str,
  135. *,
  136. params: Optional[dict[str, str]] = None,
  137. timeout: int = _DEFAULT_HTTP_TIMEOUT,
  138. ) -> Iterator[Optional[httpx.Response]]:
  139. """Stream one response with bounded, policy-checked redirects."""
  140. from tools.url_safety import SSRFConnectionBlocked, create_ssrf_safe_client
  141. current_url = url
  142. current_params = params
  143. response: Optional[httpx.Response] = None
  144. stack = ExitStack()
  145. try:
  146. for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
  147. if not is_safe_url(current_url):
  148. logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
  149. response = None
  150. break
  151. blocked = check_website_access(current_url)
  152. if blocked:
  153. logger.info(
  154. "Blocked Skills Hub fetch for %s by rule %s",
  155. blocked["host"],
  156. blocked["rule"],
  157. )
  158. response = None
  159. break
  160. stack.close()
  161. stack = ExitStack()
  162. try:
  163. client = stack.enter_context(
  164. create_ssrf_safe_client(timeout=timeout, follow_redirects=False)
  165. )
  166. response = stack.enter_context(
  167. client.stream("GET", current_url, params=current_params)
  168. )
  169. except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
  170. logger.debug("Skills Hub stream failed for %s: %s", current_url, exc)
  171. response = None
  172. break
  173. if response.status_code not in _REDIRECT_STATUS_CODES:
  174. break
  175. location = response.headers.get("location")
  176. if not location:
  177. response = None
  178. break
  179. current_url = urljoin(current_url, location)
  180. current_params = None
  181. else:
  182. logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
  183. response = None
  184. yield response
  185. finally:
  186. stack.close()
  187. # ---------------------------------------------------------------------------
  188. # Shared index cache (used by every adapter)
  189. # ---------------------------------------------------------------------------
  190. def _read_json_if_fresh(path: Path, ttl: float) -> Optional[Any]:
  191. """Parsed JSON from ``path`` when it exists and is younger than ``ttl`` seconds."""
  192. try:
  193. if time.time() - path.stat().st_mtime > ttl:
  194. return None
  195. return json.loads(path.read_text(encoding="utf-8-sig"))
  196. except (OSError, json.JSONDecodeError):
  197. return None
  198. def _read_index_cache(key: str) -> Optional[Any]:
  199. return _read_json_if_fresh(_index_cache_dir() / f"{key}.json", INDEX_CACHE_TTL)
  200. def _write_index_cache(key: str, data: Any) -> None:
  201. index_cache_dir = _index_cache_dir()
  202. index_cache_dir.mkdir(parents=True, exist_ok=True)
  203. # Cache files hold unvetted community text (possible prompt injection);
  204. # a .ignore keeps ripgrep and .ignore-aware tools out of the hub dir.
  205. ignore_file = _hub_dir() / ".ignore"
  206. if not ignore_file.exists():
  207. try:
  208. ignore_file.write_text("# Exclude hub internals from search tools\n*\n", encoding="utf-8")
  209. except OSError:
  210. pass
  211. try:
  212. (index_cache_dir / f"{key}.json").write_text(
  213. json.dumps(data, ensure_ascii=False, default=str), encoding="utf-8"
  214. )
  215. except OSError as e:
  216. logger.debug("Could not write cache: %s", e)
  217. # ---------------------------------------------------------------------------
  218. # Hub state files: lock.json, taps.json, audit.log
  219. # ---------------------------------------------------------------------------
  220. class _JsonStateFile:
  221. """A JSON file under the hub dir with a fixed empty shape (``EMPTY``, deep-copied
  222. on every miss/corrupt read); ``DEFAULT_PATH`` is the hub path resolver."""
  223. EMPTY: dict = {}
  224. DEFAULT_PATH: Any = None
  225. _LOCK_HOLDER: threading.local
  226. def __init__(self, path: Optional[Path] = None):
  227. self.path = path if path is not None else type(self).DEFAULT_PATH()
  228. def _read(self) -> dict:
  229. try:
  230. return json.loads(self.path.read_text(encoding="utf-8-sig"))
  231. except (json.JSONDecodeError, OSError):
  232. return json.loads(json.dumps(self.EMPTY))
  233. def _write(self, data: dict, **dumps_kwargs) -> None:
  234. self.path.parent.mkdir(parents=True, exist_ok=True)
  235. # Temp + rename: another process (CLI vs desktop install) reading mid-write would see an
  236. # empty file, load EMPTY and save a lock that drops every other installed skill.
  237. atomic_write_text(self.path, json.dumps(data, indent=2, **dumps_kwargs) + "\n", preserve_mode=True)
  238. def _update(self, mutate: Callable[[dict], bool], **dumps_kwargs) -> bool:
  239. """Re-read, ``mutate`` and save under a cross-process lock; the file is rewritten only
  240. when ``mutate`` returns True. The CLI and desktop rewrite the same file, and a save from a
  241. snapshot loaded outside the lock drops whatever the other process recorded meanwhile.
  242. An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
  243. update rather than failing the install that triggered it."""
  244. from hermes_cli.auth import _file_lock
  245. with ExitStack() as stack:
  246. try:
  247. stack.enter_context(_file_lock(
  248. self.path.with_name(self.path.name + ".lock"), type(self)._LOCK_HOLDER, 10.0,
  249. f"Timed out waiting for the skills hub state lock ({self.path})",
  250. ))
  251. except (OSError, TimeoutError):
  252. logger.warning("Updating %s without its lock", self.path, exc_info=True)
  253. data = self._read()
  254. changed = mutate(data)
  255. if changed:
  256. self._write(data, **dumps_kwargs)
  257. return changed
  258. class HubLockFile(_JsonStateFile):
  259. """skills/.hub/lock.json — provenance of installed hub skills."""
  260. EMPTY = {"version": 1, "installed": {}}
  261. DEFAULT_PATH = staticmethod(_lock_file)
  262. _LOCK_HOLDER = threading.local()
  263. def load(self) -> dict:
  264. return self._read()
  265. def save(self, data: dict) -> None:
  266. self._write(data, ensure_ascii=False)
  267. def record_install(
  268. self,
  269. name: str,
  270. source: str,
  271. identifier: str,
  272. trust_level: str,
  273. scan_verdict: str,
  274. skill_hash: str,
  275. install_path: str,
  276. files: list[str],
  277. metadata: Optional[dict[str, Any]] = None,
  278. scan_provenance: Optional[dict[str, Any]] = None,
  279. ) -> None:
  280. # Validate name and install-path SHAPE at write time: a poisoned lock
  281. # entry is the precondition for the uninstall_skill rmtree-escape.
  282. safe_name = _validate_skill_name(name)
  283. safe_install_path = _normalize_lock_install_path(install_path, safe_name)
  284. def record(data: dict) -> bool:
  285. now = datetime.now(UTC).isoformat()
  286. data.setdefault("installed", {})[safe_name] = {
  287. "source": source,
  288. "identifier": identifier,
  289. "trust_level": trust_level,
  290. "scan_verdict": scan_verdict,
  291. "content_hash": skill_hash,
  292. "install_path": safe_install_path,
  293. "files": files,
  294. "metadata": metadata or {},
  295. "scan_provenance": scan_provenance or {},
  296. "installed_at": now,
  297. "updated_at": now,
  298. }
  299. return True
  300. self._update(record, ensure_ascii=False)
  301. def record_uninstall(self, name: str) -> None:
  302. def forget(data: dict) -> bool:
  303. data.setdefault("installed", {}).pop(name, None)
  304. return True
  305. self._update(forget, ensure_ascii=False)
  306. def get_installed(self, name: str) -> Optional[dict]:
  307. return self.load()["installed"].get(name)
  308. def list_installed(self) -> list[dict]:
  309. return [{"name": name, **entry} for name, entry in self.load()["installed"].items()]
  310. def by_identifier(self) -> dict[str, dict]:
  311. """Installed entries keyed by identifier. A ClawHub row is also keyed by the
  312. ``clawhub/<slug>`` id a catalog card installs, since the lock records ``@owner/slug``.
  313. That alias names its source, so a ClawHub install never answers for another source's
  314. id. Exact identifiers win."""
  315. from tools.skills_hub_clawhub import ClawHubSource
  316. rows = [entry for entry in self.list_installed() if entry.get("identifier")]
  317. installed = {entry["identifier"]: entry for entry in rows}
  318. for entry in rows:
  319. if entry.get("source") == "clawhub":
  320. if alias := ClawHubSource.catalog_identifier(entry["identifier"]):
  321. installed.setdefault(alias, entry)
  322. return installed
  323. def installed_entry(installed: dict[str, dict], identifier: str,
  324. source: Optional[str] = None) -> Optional[dict]:
  325. """The ``HubLockFile.by_identifier`` entry a hub result names. Search and the index list a
  326. ClawHub skill by its bare slug, so a ``source="clawhub"`` result also matches through its
  327. ``clawhub/<slug>`` card id. Any other source matches its exact id only."""
  328. if identifier in installed:
  329. return installed[identifier]
  330. if source != "clawhub":
  331. return None
  332. from tools.skills_hub_clawhub import ClawHubSource
  333. alias = ClawHubSource.catalog_identifier(identifier)
  334. return installed.get(alias) if alias else None
  335. class TapsManager(_JsonStateFile):
  336. """skills/.hub/taps.json — custom GitHub repo sources."""
  337. EMPTY = {"taps": []}
  338. DEFAULT_PATH = staticmethod(_taps_file)
  339. _LOCK_HOLDER = threading.local()
  340. def load(self) -> list[dict]:
  341. return self._read().get("taps", [])
  342. def save(self, taps: list[dict]) -> None:
  343. self._write({"taps": taps})
  344. def add(self, repo: str, path: str = "skills/") -> bool:
  345. """Add a tap. Returns False if already exists."""
  346. def append(data: dict) -> bool:
  347. taps = data.setdefault("taps", [])
  348. if any(t["repo"] == repo for t in taps):
  349. return False
  350. taps.append({"repo": repo, "path": path})
  351. return True
  352. return self._update(append)
  353. def remove(self, repo: str) -> bool:
  354. """Remove a tap by repo name. Returns False if not found."""
  355. def drop(data: dict) -> bool:
  356. taps = data.get("taps", [])
  357. kept = [t for t in taps if t["repo"] != repo]
  358. data["taps"] = kept
  359. return len(kept) != len(taps)
  360. return self._update(drop)
  361. list_taps = load
  362. def append_audit_log(action: str, skill_name: str, source: str,
  363. trust_level: str, verdict: str, extra: str = "") -> None:
  364. """Append one space-separated line to the audit log (best-effort)."""
  365. audit_log = _audit_log()
  366. audit_log.parent.mkdir(parents=True, exist_ok=True)
  367. timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
  368. parts = [timestamp, action, skill_name, f"{source}:{trust_level}", verdict]
  369. if extra:
  370. parts.append(extra)
  371. try:
  372. with open(audit_log, "a", encoding="utf-8") as f:
  373. f.write(" ".join(parts) + "\n")
  374. except OSError as e:
  375. logger.debug("Could not write audit log: %s", e)
  376. # ---------------------------------------------------------------------------
  377. # Hub operations (high-level)
  378. # ---------------------------------------------------------------------------
  379. def ensure_hub_dirs() -> None:
  380. """Create the .hub directory structure if it doesn't exist."""
  381. _hub_dir().mkdir(parents=True, exist_ok=True)
  382. _quarantine_dir().mkdir(exist_ok=True)
  383. _index_cache_dir().mkdir(exist_ok=True)
  384. for path, initial in (
  385. (_lock_file(), '{"version": 1, "installed": {}}\n'),
  386. (_audit_log(), ""),
  387. (_taps_file(), '{"taps": []}\n'),
  388. ):
  389. if not path.exists():
  390. path.write_text(initial, encoding="utf-8")