| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430 |
- """Skills Hub adapters for well-known endpoints, direct URLs, LobeHub, and browse.sh."""
- import hashlib
- import json
- import logging
- import re
- from typing import Any, Dict, List, Optional, Union
- from urllib.parse import quote, urljoin, urlparse, urlunparse
- from tools.skills_hub_models import (
- GuardedFetchMixin, SkillBundle, SkillMeta, SkillSource, _first_matching, _get_json, _get_text,
- _hermes_tags, _memo_json, _parse_frontmatter, _referenced_support_paths,
- _validate_bundle_rel_path, _validate_skill_name, hub,
- )
- logger = logging.getLogger("tools.skills_hub")
- # --- Well-known Agent Skills endpoint source adapter ------------------------
- class WellKnownSkillSource(GuardedFetchMixin, SkillSource):
- """Read skills from a domain exposing /.well-known/skills/index.json."""
- SOURCE_ID = "well-known"
- BASE_PATH = "/.well-known/skills"
- def _meta(self, parsed: dict, skill_name: str, description: str, files: Any, **extra) -> SkillMeta:
- return SkillMeta(
- name=skill_name, description=description, source="well-known",
- identifier=self._wrap_identifier(parsed["base_url"], skill_name), trust_level="community",
- path=skill_name,
- extra={"index_url": parsed["index_url"], "base_url": parsed["base_url"], "files": files, **extra},
- )
- def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
- index_url = self._query_to_index_url(query)
- parsed = self._parse_index(index_url) if index_url else None
- if not parsed:
- return []
- results: list[SkillMeta] = []
- for entry in parsed["skills"][:limit]:
- name = entry.get("name")
- if not isinstance(name, str) or not name:
- continue
- files = entry.get("files", ["SKILL.md"])
- results.append(self._meta(parsed, name, str(entry.get("description", "")),
- files if isinstance(files, list) else ["SKILL.md"]))
- return results
- def inspect(self, identifier: str) -> Optional[SkillMeta]:
- parsed = self._parse_identifier(identifier)
- entry = self._index_entry(parsed["index_url"], parsed["skill_name"]) if parsed else None
- skill_md = self._fetch_text(f"{parsed['skill_url']}/SKILL.md") if entry else None
- if skill_md is None:
- return None
- fm = _parse_frontmatter(skill_md)
- return self._meta(parsed, str(fm.get("name") or parsed["skill_name"]),
- str(fm.get("description") or entry.get("description") or ""),
- entry.get("files", ["SKILL.md"]), endpoint=parsed["skill_url"])
- def fetch(self, identifier: str) -> Optional[SkillBundle]:
- parsed = self._parse_identifier(identifier)
- if not parsed:
- return None
- try:
- skill_name = _validate_skill_name(parsed["skill_name"])
- except ValueError:
- logger.warning("Well-known skill identifier contained unsafe skill name: %s", identifier)
- return None
- entry = self._index_entry(parsed["index_url"], parsed["skill_name"])
- if not entry:
- return None
- files = entry.get("files", ["SKILL.md"])
- if not isinstance(files, list) or not files:
- files = ["SKILL.md"]
- downloaded: dict[str, str] = {}
- for rel_path in files:
- if not isinstance(rel_path, str) or not rel_path:
- continue
- try:
- safe_rel_path = _validate_bundle_rel_path(rel_path)
- except ValueError:
- logger.warning("Well-known skill %s advertised unsafe file path: %r", identifier, rel_path)
- return None
- text = self._fetch_text(f"{parsed['skill_url']}/{safe_rel_path}")
- if text is None:
- return None
- downloaded[safe_rel_path] = text
- if "SKILL.md" not in downloaded:
- return None
- return SkillBundle(
- name=skill_name, files=downloaded, source="well-known",
- identifier=self._wrap_identifier(parsed["base_url"], skill_name), trust_level="community",
- metadata={"index_url": parsed["index_url"], "base_url": parsed["base_url"],
- "endpoint": parsed["skill_url"], "files": files},
- )
- def _query_to_index_url(self, query: str) -> Optional[str]:
- query = query.strip()
- if not query.startswith(("http://", "https://")):
- return None
- if query.endswith("/index.json"):
- return query
- if f"{self.BASE_PATH}/" in query:
- return query.split(f"{self.BASE_PATH}/", 1)[0] + f"{self.BASE_PATH}/index.json"
- return query.rstrip("/") + f"{self.BASE_PATH}/index.json"
- def _parse_identifier(self, identifier: str) -> Optional[dict]:
- raw = identifier.removeprefix("well-known:")
- if not raw.startswith(("http://", "https://")):
- return None
- parsed_url = urlparse(raw)
- clean_url = urlunparse(parsed_url._replace(fragment=""))
- if clean_url.endswith("/index.json"):
- if not parsed_url.fragment:
- return None
- base_url, skill_name = clean_url[:-len("/index.json")], parsed_url.fragment
- skill_url = f"{base_url}/{skill_name}"
- else:
- skill_url = clean_url[:-len("/SKILL.md")] if clean_url.endswith("/SKILL.md") else clean_url.rstrip("/")
- if f"{self.BASE_PATH}/" not in skill_url:
- return None
- base_url, skill_name = skill_url.rsplit("/", 1)
- return {"index_url": f"{base_url}/index.json", "base_url": base_url,
- "skill_name": skill_name, "skill_url": skill_url}
- def _parse_index(self, index_url: str) -> Optional[dict]:
- def compute():
- resp = hub()._guarded_http_get(index_url, timeout=20)
- if resp is None or resp.status_code != 200:
- return None
- try:
- data = resp.json()
- except json.JSONDecodeError:
- return None
- skills = data.get("skills", []) if isinstance(data, dict) else []
- if not isinstance(skills, list):
- return None
- return {"index_url": index_url, "base_url": index_url[:-len("/index.json")], "skills": skills}
- return _memo_json(f"well_known_index_{hashlib.md5(index_url.encode()).hexdigest()}", compute,
- valid=lambda c: isinstance(c, dict) and isinstance(c.get("skills"), list))
- def _index_entry(self, index_url: str, skill_name: str) -> Optional[dict]:
- parsed = self._parse_index(index_url)
- skills = parsed["skills"] if parsed else []
- return next((e for e in skills if isinstance(e, dict) and e.get("name") == skill_name), None)
- @staticmethod
- def _wrap_identifier(base_url: str, skill_name: str) -> str:
- return f"well-known:{base_url.rstrip('/')}/{skill_name}"
- # --- Direct URL source adapter ----------------------------------------------
- class UrlSource(GuardedFetchMixin, SkillSource):
- """Fetch SKILL.md plus explicitly referenced, allowlisted support files.
- The identifier IS the URL (``https://example.com/path/SKILL.md``). Bare URLs cannot enumerate a
- repository, so only exact references below references/templates/scripts/assets are fetched. The
- skill name comes from frontmatter ``name:`` (URL-slug fallback); trust is always ``community``.
- """
- SOURCE_ID = "url"
- # Skill names must look like identifiers: lowercase letters/digits with optional hyphens/underscores.
- # Blocks dangerous (``../evil``) AND useless (``SKILL``, ``README``, empty) candidates before they hit the disk.
- _VALID_NAME_RE = re.compile(r"^[a-z][a-z0-9_-]*$")
- def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
- return [] # search is meaningless for a direct URL
- def _matches(self, identifier: str) -> bool:
- """Claim bare HTTP(S) URLs ending in ``.md``; leave wrapped identifiers
- and ``/.well-known/skills/`` URLs to their own adapters."""
- if not isinstance(identifier, str):
- return False
- ident = identifier.strip()
- if (not ident.lower().startswith(("http://", "https://")) or "/.well-known/skills/" in ident
- or ident.rstrip("/").endswith("/index.json")):
- return False
- try:
- return urlparse(ident).path.lower().endswith(".md")
- except ValueError:
- return False
- def _load(self, identifier: str):
- """``(url, text, frontmatter, resolved name)`` for a claimed identifier, else None."""
- if not self._matches(identifier):
- return None
- url = identifier.strip()
- text = self._fetch_text(url)
- if text is None:
- return None
- fm = _parse_frontmatter(text)
- return url, text, fm, self._resolve_skill_name(fm, url)
- def inspect(self, identifier: str) -> Optional[SkillMeta]:
- loaded = self._load(identifier)
- if loaded is None:
- return None
- url, _text, fm, name = loaded
- raw_tags = _hermes_tags(fm)
- return SkillMeta(
- name=name or "", description=str(fm.get("description") or ""), source="url", identifier=url,
- trust_level="community", path=name or "",
- tags=[str(t) for t in raw_tags] if isinstance(raw_tags, list) else [],
- extra={"url": url, "awaiting_name": name is None},
- )
- def fetch(self, identifier: str) -> Optional[SkillBundle]:
- loaded = self._load(identifier)
- if loaded is None:
- return None
- url, text, _fm, name = loaded
- referenced = _referenced_support_paths(text)
- if referenced is None:
- return None
- files: dict[str, str | bytes] = {"SKILL.md": text}
- base_url = url.rsplit("/", 1)[0] + "/"
- for rel_path in sorted(referenced):
- support_url = urljoin(base_url, quote(rel_path, safe="/"))
- if urlparse(support_url).netloc != urlparse(url).netloc:
- return None
- content = self._fetch_bytes(support_url)
- if content is None: # A 404ing support file shouldn't sink the whole install.
- logger.warning("URL skill %s: referenced support file %r could not be fetched from %s; skipping it",
- url, rel_path, support_url)
- continue
- files[rel_path] = content
- # When no name resolves, return the bundle with an empty name and ``awaiting_name=True``: ``do_install``
- # prompts on a TTY or refuses non-interactively, without re-downloading after the user picks a name.
- skill_name = ""
- if name is not None:
- try:
- skill_name = _validate_skill_name(name)
- except ValueError:
- logger.warning("URL skill %s produced unsafe skill name: %r", url, name)
- return None
- return SkillBundle(name=skill_name, files=files, source="url", identifier=url, trust_level="community",
- metadata={"url": url, "source_url": url, "awaiting_name": not skill_name})
- @classmethod
- def _is_valid_skill_name(cls, name: Optional[str]) -> bool:
- if not isinstance(name, str):
- return False
- candidate = name.strip().lower()
- return bool(candidate) and candidate not in {"skill", "readme", "index", "unnamed-skill"} and bool(
- cls._VALID_NAME_RE.match(candidate))
- @classmethod
- def _resolve_skill_name(cls, fm: dict, url: str) -> Optional[str]:
- """Frontmatter ``name:`` when valid, else a URL-slug candidate (``.../<name>/SKILL.md`` -> ``<name>``,
- ``.../<name>.md`` -> ``<name>``). None when nothing usable — the CLI then prompts or refuses rather
- than auto-naming something like ``SKILL``."""
- fm_name = fm.get("name") if isinstance(fm, dict) else None
- if isinstance(fm_name, str) and cls._is_valid_skill_name(fm_name):
- return fm_name.strip()
- try:
- path = urlparse(url).path
- except ValueError:
- return None
- parts = [p for p in path.split("/") if p]
- if len(parts) >= 2 and parts[-1].lower() == "skill.md" and cls._is_valid_skill_name(parts[-2]):
- return parts[-2]
- candidate = re.sub(r"\.md$", "", parts[-1], flags=re.IGNORECASE) if parts else ""
- return candidate if cls._is_valid_skill_name(candidate) else None
- # --- LobeHub source adapter -------------------------------------------------
- class LobeHubSource(SkillSource):
- """LobeHub agent marketplace (14,500+ system-prompt agents, converted to
- SKILL.md on fetch). Data lives in GitHub: lobehub/lobe-chat-agents."""
- SOURCE_ID = "lobehub"
- INDEX_URL = "https://chat-agents.lobehub.com/index.json"
- def _agents(self) -> Optional[list]:
- index = self._fetch_index()
- agents = (index.get("agents", index) if isinstance(index, dict) else index) if index else None
- return agents if isinstance(agents, list) else None
- @staticmethod
- def _agent_id(identifier: str) -> str:
- return identifier.split("/", 1)[-1] if identifier.startswith("lobehub/") else identifier
- @staticmethod
- def _agent_meta(agent: dict, name: str, description: str) -> SkillMeta:
- tags = agent.get("meta", agent).get("tags", [])
- return SkillMeta(name=name, description=description, source="lobehub", identifier=f"lobehub/{name}",
- trust_level="community", tags=tags if isinstance(tags, list) else [])
- def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
- agents = self._agents()
- if agents is None:
- return []
- def fields(agent):
- meta = agent.get("meta", agent)
- tags = meta.get("tags", [])
- return (meta.get("title", agent.get("identifier", "")), meta.get("description", ""),
- tags if isinstance(tags, list) else "")
- def to_meta(agent):
- meta = agent.get("meta", agent)
- title = meta.get("title", agent.get("identifier", ""))
- identifier = agent.get("identifier", title.lower().replace(" ", "-"))
- return self._agent_meta(agent, identifier, meta.get("description", "")[:200])
- return _first_matching(query.lower(), agents, fields, to_meta, limit)
- def fetch(self, identifier: str) -> Optional[SkillBundle]:
- agent_id = self._agent_id(identifier)
- agent_data = self._fetch_agent(agent_id)
- if not agent_data:
- return None
- return SkillBundle(name=agent_id, files={"SKILL.md": self._convert_to_skill_md(agent_data)}, source="lobehub",
- identifier=f"lobehub/{agent_id}", trust_level="community")
- def inspect(self, identifier: str) -> Optional[SkillMeta]:
- agent_id = self._agent_id(identifier)
- agent = next((a for a in self._agents() or [] if a.get("identifier") == agent_id), None)
- return self._agent_meta(agent, agent_id, agent.get("meta", agent).get("description", "")) if agent else None
- def _fetch_index(self) -> Optional[Any]:
- return _memo_json("lobehub_index", lambda: _get_json(self.INDEX_URL, timeout=30))
- def _fetch_agent(self, agent_id: str) -> Optional[dict]:
- return _get_json(f"https://chat-agents.lobehub.com/{agent_id}.json", timeout=15)
- @staticmethod
- def _convert_to_skill_md(agent_data: dict) -> str:
- """Convert a LobeHub agent JSON into SKILL.md format."""
- meta = agent_data.get("meta", agent_data)
- identifier = agent_data.get("identifier", "lobehub-agent")
- title = meta.get("title", identifier)
- description = meta.get("description", "")
- tags = meta.get("tags", [])
- tag_list = tags if isinstance(tags, list) else []
- system_role = agent_data.get("config", {}).get("systemRole", "")
- fm_lines = ["---", f"name: {identifier}", f"description: {description[:500]}", "metadata:", " hermes:",
- f" tags: [{', '.join(str(t) for t in tag_list)}]", " lobehub:", " source: lobehub", "---"]
- body_lines = [f"# {title}", "", description, "", "## Instructions", "",
- system_role if system_role else "(No system role defined)"]
- return "\n".join(fm_lines) + "\n\n" + "\n".join(body_lines) + "\n"
- # --- browse.sh source adapter -----------------------------------------------
- class BrowseShSource(SkillSource):
- """Browserbase's browse.sh catalog of site-specific browser-automation SKILL.md files.
- The catalog is ``/api/skills``; content comes from ``/api/skills/{slug}``'s ``skillMdUrl`` (CDN blob).
- The catalog's ``sourceUrl`` is a GitHub HTML URL whose repo is not always public, so it is not used for content.
- """
- SOURCE_ID = "browse-sh"
- CATALOG_URL = "https://browse.sh/api/skills"
- SKILL_DETAIL_URL = "https://browse.sh/api/skills/{slug}"
- _CACHE_KEY = "browse_sh_catalog"
- def _fetch_catalog(self) -> list[dict]:
- def compute():
- data = _get_json(self.CATALOG_URL)
- skills = data.get("skills", []) if isinstance(data, dict) else []
- return skills if isinstance(skills, list) else None
- return _memo_json(self._CACHE_KEY, compute) or []
- def _item_to_meta(self, item: dict) -> Optional[SkillMeta]:
- slug = item.get("slug", "")
- name = item.get("name", "")
- description = item.get("description", item.get("title", name))
- if not slug or not name:
- return None
- if len(description) > 1024:
- description = description[:1021] + "..."
- return SkillMeta(
- name=name, description=description, source="browse-sh", identifier=f"browse-sh/{slug}",
- trust_level="community", tags=item.get("tags", []),
- extra={"slug": slug, "hostname": item.get("hostname", ""), "category": item.get("category", ""),
- "source_url": item.get("sourceUrl", ""), "recommended_method": item.get("recommendedMethod", ""),
- "proxies": item.get("proxies", False), "install_count": item.get("installCount", 0)},
- )
- def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
- def fields(item):
- return (item.get("name", ""), item.get("title", ""), item.get("description", ""),
- item.get("hostname", ""), item.get("category", ""), item.get("tags", []))
- return _first_matching(query.lower(), self._fetch_catalog(), fields, self._item_to_meta, limit)
- def _catalog_item(self, identifier: str) -> Optional[dict]:
- slug = self._slug_from_identifier(identifier)
- return next((i for i in self._fetch_catalog() if i.get("slug") == slug), None) if slug else None
- def inspect(self, identifier: str) -> Optional[SkillMeta]:
- item = self._catalog_item(identifier)
- return self._item_to_meta(item) if item else None
- def fetch(self, identifier: str) -> Optional[SkillBundle]:
- item = self._catalog_item(identifier)
- if not item:
- return None
- slug = item["slug"]
- md_url = self._resolve_skill_md_url(slug, item)
- content = _get_text(md_url, follow_redirects=True) if md_url else None
- if content is None:
- return None
- meta = self._item_to_meta(item)
- return SkillBundle(
- name=meta.name if meta else slug.split("/")[-1], files={"SKILL.md": content}, source="browse-sh",
- identifier=identifier, trust_level="community",
- metadata={"slug": slug, "hostname": item.get("hostname", ""), "source_url": item.get("sourceUrl", ""),
- "skill_md_url": md_url},
- )
- def _resolve_skill_md_url(self, slug: str, item: dict) -> Optional[str]:
- """``skillMdUrl`` from ``/api/skills/{slug}``; fallback to a ``raw.githubusercontent.com`` ``sourceUrl``."""
- data = _get_json(self.SKILL_DETAIL_URL.format(slug=slug), follow_redirects=True)
- md_url = data.get("skillMdUrl") if isinstance(data, dict) else None
- if isinstance(md_url, str) and md_url.startswith("http"):
- return md_url
- source_url = item.get("sourceUrl", "") if isinstance(item, dict) else ""
- from utils import base_url_host_matches
- return source_url if source_url and base_url_host_matches(source_url, "raw.githubusercontent.com") else None
- def _slug_from_identifier(self, identifier: str) -> str:
- """'browse-sh/airbnb.com/search-listings-abc' -> 'airbnb.com/search-listings-abc'."""
- return identifier.removeprefix("browse-sh/")
|