| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484 |
- #!/usr/bin/env python3
- """
- Skills Hub — hub state management and the public facade for the source adapters.
- Library module (not an agent tool). Owns the hub paths, guarded HTTP, index
- cache, lock file, taps and audit log. Install/uninstall/update live in
- ``skills_hub_install``, the index fetch/source router/search in
- ``skills_hub_search``, and the adapters in the other ``tools.skills_hub_*``
- siblings; import each name from its defining module.
- Used by hermes_cli/skills_hub.py for CLI commands and the /skills slash command.
- """
- import json
- import logging
- import threading
- import time
- from contextvars import ContextVar
- from contextlib import ExitStack, contextmanager
- from datetime import datetime, timezone, UTC
- from pathlib import Path
- from typing import Any, Callable, Dict, Iterator, List, Optional
- from urllib.parse import urljoin
- import httpx
- from hermes_constants import get_hermes_home
- from tools.url_safety import is_safe_url
- from tools.url_safety import create_ssrf_safe_client
- from tools.website_policy import check_website_access
- from tools.skills_hub_models import _normalize_lock_install_path, _validate_skill_name
- from utils import atomic_write_text
- logger = logging.getLogger(__name__)
- # ---------------------------------------------------------------------------
- # Paths
- # ---------------------------------------------------------------------------
- # Resolved per-call (not frozen at import) so the profile override is honored;
- # import-time constants leaked across profiles in single-process multi-profile
- # runtimes. The path names (SKILLS_DIR, ...) resolve live via __getattr__ below.
- INDEX_CACHE_TTL = 3600 # 1 hour
- def _path_resolver(name: str, parent: str, leaf: str):
- """Resolver for hub path ``<parent>/<leaf>``: a test-injected real module attribute
- (patch.object/monkeypatch on SKILLS_DIR etc.) wins over live resolution."""
- def resolve() -> Path:
- forced = globals().get(name)
- return Path(forced) if forced is not None else _DYNAMIC_PATH_RESOLVERS[parent]() / leaf
- resolve.__name__ = f"_{name.lower()}"
- return resolve
- def _hermes_home() -> Path:
- return get_hermes_home()
- _skills_dir = _path_resolver("SKILLS_DIR", "HERMES_HOME", "skills")
- _hub_dir = _path_resolver("HUB_DIR", "SKILLS_DIR", ".hub")
- _lock_file = _path_resolver("LOCK_FILE", "HUB_DIR", "lock.json")
- _quarantine_dir = _path_resolver("QUARANTINE_DIR", "HUB_DIR", "quarantine")
- _audit_log = _path_resolver("AUDIT_LOG", "HUB_DIR", "audit.log")
- _taps_file = _path_resolver("TAPS_FILE", "HUB_DIR", "taps.json")
- _index_cache_dir = _path_resolver("INDEX_CACHE_DIR", "HUB_DIR", "index-cache")
- _DYNAMIC_PATH_RESOLVERS = {"HERMES_HOME": _hermes_home, **{
- r.__name__[1:].upper(): r
- for r in (_skills_dir, _hub_dir, _lock_file, _quarantine_dir, _audit_log, _taps_file, _index_cache_dir)
- }}
- def __getattr__(name: str):
- """Resolve legacy path constants dynamically (PEP 562) so they reflect the
- active profile override; a test's patch.object-set real attribute shadows it."""
- resolver = _DYNAMIC_PATH_RESOLVERS.get(name)
- if resolver is not None:
- return resolver()
- raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
- # ---------------------------------------------------------------------------
- # Install-path safety + guarded HTTP
- # ---------------------------------------------------------------------------
- _REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
- _MAX_SKILL_FETCH_REDIRECTS = 5
- # Default per-request timeout every one-shot hub GET used before pooling existed.
- _DEFAULT_HTTP_TIMEOUT = 20
- # An inspect resolves metadata and then the preview bundle through the same
- # source adapter. Keeping this context local to that operation lets httpx reuse
- # its verified connection without extending a client beyond the CLI request.
- _skills_hub_http_client: ContextVar[Optional[Any]] = ContextVar(
- "skills_hub_http_client", default=None
- )
- @contextmanager
- def skills_hub_http_session() -> Iterator[None]:
- """Reuse one SSRF-safe HTTP client for a single skills-hub resolution."""
- if _skills_hub_http_client.get() is not None:
- yield
- return
- with create_ssrf_safe_client(timeout=_DEFAULT_HTTP_TIMEOUT, follow_redirects=False) as client:
- token = _skills_hub_http_client.set(client)
- try:
- yield
- finally:
- _skills_hub_http_client.reset(token)
- def _skills_hub_http_get(url: str, **kwargs: Any) -> httpx.Response:
- """GET through the active resolution pool, or preserve one-shot behavior."""
- client = _skills_hub_http_client.get()
- if client is not None:
- return client.get(url, **kwargs)
- return httpx.get(url, **kwargs)
- def _ssrf_safe_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
- headers: Optional[dict[str, str]] = None) -> httpx.Response:
- """Fetch one URL with connect-time SSRF validation and no automatic redirects."""
- with skills_hub_http_session():
- return _skills_hub_http_client.get().get(url, timeout=timeout, headers=headers)
- def _guarded_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
- headers: Optional[dict[str, str]] = None) -> Optional[httpx.Response]:
- """Fetch a URL with SSRF and redirect-target validation (each hop re-checked).
- *headers* are plain request headers (no credentials) and are sent on every hop."""
- from tools.url_safety import SSRFConnectionBlocked
- current_url = url
- for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
- if not is_safe_url(current_url):
- logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
- return None
- blocked = check_website_access(current_url)
- if blocked:
- logger.info(
- "Blocked Skills Hub fetch for %s by rule %s",
- blocked["host"],
- blocked["rule"],
- )
- return None
- try:
- resp = _ssrf_safe_http_get(current_url, timeout=timeout, headers=headers)
- except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
- logger.debug("Skills Hub fetch failed for %s: %s", current_url, exc)
- return None
- if resp.status_code in _REDIRECT_STATUS_CODES:
- location = getattr(resp, "headers", {}).get("location")
- if not location:
- return None
- current_url = urljoin(current_url, location)
- continue
- return resp
- logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
- return None
- @contextmanager
- def _guarded_http_stream(
- url: str,
- *,
- params: Optional[dict[str, str]] = None,
- timeout: int = _DEFAULT_HTTP_TIMEOUT,
- ) -> Iterator[Optional[httpx.Response]]:
- """Stream one response with bounded, policy-checked redirects."""
- from tools.url_safety import SSRFConnectionBlocked, create_ssrf_safe_client
- current_url = url
- current_params = params
- response: Optional[httpx.Response] = None
- stack = ExitStack()
- try:
- for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
- if not is_safe_url(current_url):
- logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
- response = None
- break
- blocked = check_website_access(current_url)
- if blocked:
- logger.info(
- "Blocked Skills Hub fetch for %s by rule %s",
- blocked["host"],
- blocked["rule"],
- )
- response = None
- break
- stack.close()
- stack = ExitStack()
- try:
- client = stack.enter_context(
- create_ssrf_safe_client(timeout=timeout, follow_redirects=False)
- )
- response = stack.enter_context(
- client.stream("GET", current_url, params=current_params)
- )
- except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
- logger.debug("Skills Hub stream failed for %s: %s", current_url, exc)
- response = None
- break
- if response.status_code not in _REDIRECT_STATUS_CODES:
- break
- location = response.headers.get("location")
- if not location:
- response = None
- break
- current_url = urljoin(current_url, location)
- current_params = None
- else:
- logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
- response = None
- yield response
- finally:
- stack.close()
- # ---------------------------------------------------------------------------
- # Shared index cache (used by every adapter)
- # ---------------------------------------------------------------------------
- def _read_json_if_fresh(path: Path, ttl: float) -> Optional[Any]:
- """Parsed JSON from ``path`` when it exists and is younger than ``ttl`` seconds."""
- try:
- if time.time() - path.stat().st_mtime > ttl:
- return None
- return json.loads(path.read_text(encoding="utf-8-sig"))
- except (OSError, json.JSONDecodeError):
- return None
- def _read_index_cache(key: str) -> Optional[Any]:
- return _read_json_if_fresh(_index_cache_dir() / f"{key}.json", INDEX_CACHE_TTL)
- def _write_index_cache(key: str, data: Any) -> None:
- index_cache_dir = _index_cache_dir()
- index_cache_dir.mkdir(parents=True, exist_ok=True)
- # Cache files hold unvetted community text (possible prompt injection);
- # a .ignore keeps ripgrep and .ignore-aware tools out of the hub dir.
- ignore_file = _hub_dir() / ".ignore"
- if not ignore_file.exists():
- try:
- ignore_file.write_text("# Exclude hub internals from search tools\n*\n", encoding="utf-8")
- except OSError:
- pass
- try:
- (index_cache_dir / f"{key}.json").write_text(
- json.dumps(data, ensure_ascii=False, default=str), encoding="utf-8"
- )
- except OSError as e:
- logger.debug("Could not write cache: %s", e)
- # ---------------------------------------------------------------------------
- # Hub state files: lock.json, taps.json, audit.log
- # ---------------------------------------------------------------------------
- class _JsonStateFile:
- """A JSON file under the hub dir with a fixed empty shape (``EMPTY``, deep-copied
- on every miss/corrupt read); ``DEFAULT_PATH`` is the hub path resolver."""
- EMPTY: dict = {}
- DEFAULT_PATH: Any = None
- _LOCK_HOLDER: threading.local
- def __init__(self, path: Optional[Path] = None):
- self.path = path if path is not None else type(self).DEFAULT_PATH()
- def _read(self) -> dict:
- try:
- return json.loads(self.path.read_text(encoding="utf-8-sig"))
- except (json.JSONDecodeError, OSError):
- return json.loads(json.dumps(self.EMPTY))
- def _write(self, data: dict, **dumps_kwargs) -> None:
- self.path.parent.mkdir(parents=True, exist_ok=True)
- # Temp + rename: another process (CLI vs desktop install) reading mid-write would see an
- # empty file, load EMPTY and save a lock that drops every other installed skill.
- atomic_write_text(self.path, json.dumps(data, indent=2, **dumps_kwargs) + "\n", preserve_mode=True)
- def _update(self, mutate: Callable[[dict], bool], **dumps_kwargs) -> bool:
- """Re-read, ``mutate`` and save under a cross-process lock; the file is rewritten only
- when ``mutate`` returns True. The CLI and desktop rewrite the same file, and a save from a
- snapshot loaded outside the lock drops whatever the other process recorded meanwhile.
- An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
- update rather than failing the install that triggered it."""
- from hermes_cli.auth import _file_lock
- with ExitStack() as stack:
- try:
- stack.enter_context(_file_lock(
- self.path.with_name(self.path.name + ".lock"), type(self)._LOCK_HOLDER, 10.0,
- f"Timed out waiting for the skills hub state lock ({self.path})",
- ))
- except (OSError, TimeoutError):
- logger.warning("Updating %s without its lock", self.path, exc_info=True)
- data = self._read()
- changed = mutate(data)
- if changed:
- self._write(data, **dumps_kwargs)
- return changed
- class HubLockFile(_JsonStateFile):
- """skills/.hub/lock.json — provenance of installed hub skills."""
- EMPTY = {"version": 1, "installed": {}}
- DEFAULT_PATH = staticmethod(_lock_file)
- _LOCK_HOLDER = threading.local()
- def load(self) -> dict:
- return self._read()
- def save(self, data: dict) -> None:
- self._write(data, ensure_ascii=False)
- def record_install(
- self,
- name: str,
- source: str,
- identifier: str,
- trust_level: str,
- scan_verdict: str,
- skill_hash: str,
- install_path: str,
- files: list[str],
- metadata: Optional[dict[str, Any]] = None,
- scan_provenance: Optional[dict[str, Any]] = None,
- ) -> None:
- # Validate name and install-path SHAPE at write time: a poisoned lock
- # entry is the precondition for the uninstall_skill rmtree-escape.
- safe_name = _validate_skill_name(name)
- safe_install_path = _normalize_lock_install_path(install_path, safe_name)
- def record(data: dict) -> bool:
- now = datetime.now(UTC).isoformat()
- data.setdefault("installed", {})[safe_name] = {
- "source": source,
- "identifier": identifier,
- "trust_level": trust_level,
- "scan_verdict": scan_verdict,
- "content_hash": skill_hash,
- "install_path": safe_install_path,
- "files": files,
- "metadata": metadata or {},
- "scan_provenance": scan_provenance or {},
- "installed_at": now,
- "updated_at": now,
- }
- return True
- self._update(record, ensure_ascii=False)
- def record_uninstall(self, name: str) -> None:
- def forget(data: dict) -> bool:
- data.setdefault("installed", {}).pop(name, None)
- return True
- self._update(forget, ensure_ascii=False)
- def get_installed(self, name: str) -> Optional[dict]:
- return self.load()["installed"].get(name)
- def list_installed(self) -> list[dict]:
- return [{"name": name, **entry} for name, entry in self.load()["installed"].items()]
- def by_identifier(self) -> dict[str, dict]:
- """Installed entries keyed by identifier. A ClawHub row is also keyed by the
- ``clawhub/<slug>`` id a catalog card installs, since the lock records ``@owner/slug``.
- That alias names its source, so a ClawHub install never answers for another source's
- id. Exact identifiers win."""
- from tools.skills_hub_clawhub import ClawHubSource
- rows = [entry for entry in self.list_installed() if entry.get("identifier")]
- installed = {entry["identifier"]: entry for entry in rows}
- for entry in rows:
- if entry.get("source") == "clawhub":
- if alias := ClawHubSource.catalog_identifier(entry["identifier"]):
- installed.setdefault(alias, entry)
- return installed
- def installed_entry(installed: dict[str, dict], identifier: str,
- source: Optional[str] = None) -> Optional[dict]:
- """The ``HubLockFile.by_identifier`` entry a hub result names. Search and the index list a
- ClawHub skill by its bare slug, so a ``source="clawhub"`` result also matches through its
- ``clawhub/<slug>`` card id. Any other source matches its exact id only."""
- if identifier in installed:
- return installed[identifier]
- if source != "clawhub":
- return None
- from tools.skills_hub_clawhub import ClawHubSource
- alias = ClawHubSource.catalog_identifier(identifier)
- return installed.get(alias) if alias else None
- class TapsManager(_JsonStateFile):
- """skills/.hub/taps.json — custom GitHub repo sources."""
- EMPTY = {"taps": []}
- DEFAULT_PATH = staticmethod(_taps_file)
- _LOCK_HOLDER = threading.local()
- def load(self) -> list[dict]:
- return self._read().get("taps", [])
- def save(self, taps: list[dict]) -> None:
- self._write({"taps": taps})
- def add(self, repo: str, path: str = "skills/") -> bool:
- """Add a tap. Returns False if already exists."""
- def append(data: dict) -> bool:
- taps = data.setdefault("taps", [])
- if any(t["repo"] == repo for t in taps):
- return False
- taps.append({"repo": repo, "path": path})
- return True
- return self._update(append)
- def remove(self, repo: str) -> bool:
- """Remove a tap by repo name. Returns False if not found."""
- def drop(data: dict) -> bool:
- taps = data.get("taps", [])
- kept = [t for t in taps if t["repo"] != repo]
- data["taps"] = kept
- return len(kept) != len(taps)
- return self._update(drop)
- list_taps = load
- def append_audit_log(action: str, skill_name: str, source: str,
- trust_level: str, verdict: str, extra: str = "") -> None:
- """Append one space-separated line to the audit log (best-effort)."""
- audit_log = _audit_log()
- audit_log.parent.mkdir(parents=True, exist_ok=True)
- timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
- parts = [timestamp, action, skill_name, f"{source}:{trust_level}", verdict]
- if extra:
- parts.append(extra)
- try:
- with open(audit_log, "a", encoding="utf-8") as f:
- f.write(" ".join(parts) + "\n")
- except OSError as e:
- logger.debug("Could not write audit log: %s", e)
- # ---------------------------------------------------------------------------
- # Hub operations (high-level)
- # ---------------------------------------------------------------------------
- def ensure_hub_dirs() -> None:
- """Create the .hub directory structure if it doesn't exist."""
- _hub_dir().mkdir(parents=True, exist_ok=True)
- _quarantine_dir().mkdir(exist_ok=True)
- _index_cache_dir().mkdir(exist_ok=True)
- for path, initial in (
- (_lock_file(), '{"version": 1, "installed": {}}\n'),
- (_audit_log(), ""),
- (_taps_file(), '{"taps": []}\n'),
- ):
- if not path.exists():
- path.write_text(initial, encoding="utf-8")
|