skills_hub_clawhub.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585
  1. """Skills Hub ClawHub adapter (clawhub.ai HTTP API)."""
  2. import hashlib
  3. import json
  4. import logging
  5. import re
  6. import time
  7. from typing import Any, Dict, List, Optional, Tuple
  8. import httpx
  9. from agent.retry_utils import parse_retry_after_seconds
  10. from tools.skills_hub import _guarded_http_stream
  11. from tools.skills_hub_models import (
  12. GuardedFetchMixin, SkillBundle, SkillMeta, SkillSource, _cache_metas, _cached_metas, _get_json,
  13. _validate_bundle_rel_path, hub,
  14. )
  15. logger = logging.getLogger("tools.skills_hub")
  16. def _query_terms(query: str) -> list[str]:
  17. return [term for term in re.split(r"[^a-z0-9]+", query.lower()) if term]
  18. def _dedupe_results(results: list[SkillMeta]) -> list[SkillMeta]:
  19. """Dedupe by lowercased identifier (name fallback), first wins."""
  20. seen: dict[str, SkillMeta] = {}
  21. for result in results:
  22. seen.setdefault((result.identifier or result.name).lower(), result)
  23. return list(seen.values())
  24. def _search_score(query: str, meta: SkillMeta) -> int:
  25. query_norm = query.strip().lower()
  26. if not query_norm:
  27. return 1
  28. identifier, name = (meta.identifier or "").lower(), (meta.name or "").lower()
  29. description = (meta.description or "").lower()
  30. query_terms, identifier_terms, name_terms = _query_terms(query_norm), _query_terms(identifier), _query_terms(name)
  31. normalized_identifier, normalized_name = " ".join(identifier_terms), " ".join(name_terms)
  32. checks = (
  33. (140, query_norm == identifier), (130, query_norm == name),
  34. (125, normalized_identifier == query_norm), (120, normalized_name == query_norm),
  35. (95, normalized_identifier.startswith(query_norm)), (90, normalized_name.startswith(query_norm)),
  36. (70, bool(query_terms) and identifier_terms[: len(query_terms)] == query_terms),
  37. (65, bool(query_terms) and name_terms[: len(query_terms)] == query_terms),
  38. (40, query_norm in identifier), (35, query_norm in name), (10, query_norm in description),
  39. )
  40. score = sum(points for points, hit in checks if hit)
  41. for term in query_terms:
  42. score += 15 * (term in identifier_terms) + 12 * (term in name_terms) + 3 * (term in description)
  43. return score
  44. def _first_str(*values: Any) -> Optional[str]:
  45. """First non-empty ``str`` among ``values`` (dicts/None are skipped)."""
  46. return next((v for v in values if isinstance(v, str) and v), None)
  47. class ClawHubSource(GuardedFetchMixin, SkillSource):
  48. """ClawHub (clawhub.ai) HTTP API. Every skill is community trust — the ClawHavoc
  49. incident (341 malicious skills, Feb 2026) showed their vetting is insufficient."""
  50. SOURCE_ID = "clawhub"
  51. BASE_URL = "https://clawhub.ai/api/v1"
  52. # Wall-clock budget for a full catalog walk: 50k+ skills, sequential
  53. # (~250 requests each under timeout=30), so unbounded it blocks for minutes.
  54. CATALOG_WALK_BUDGET_SECONDS = 12
  55. ZIP_DOWNLOAD_MAX_BYTES = 25 * 1024 * 1024
  56. ZIP_DOWNLOAD_CHUNK_BYTES = 64 * 1024
  57. _SLUG_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*$")
  58. _query_terms = staticmethod(_query_terms)
  59. _dedupe_results = staticmethod(_dedupe_results)
  60. _search_score = staticmethod(_search_score)
  61. _get_json = staticmethod(_get_json)
  62. @staticmethod
  63. def _normalize_tags(tags: Any) -> list[str]:
  64. if isinstance(tags, list):
  65. return [str(t) for t in tags]
  66. if isinstance(tags, dict):
  67. return [str(k) for k in tags if str(k) != "latest"]
  68. return []
  69. @staticmethod
  70. def _coerce_skill_payload(data: Any) -> Optional[dict[str, Any]]:
  71. """Flatten ``{"skill": {...}, "latestVersion", "owner"}`` listing shapes."""
  72. if not isinstance(data, dict):
  73. return None
  74. nested = data.get("skill")
  75. if not isinstance(nested, dict):
  76. return data
  77. merged = dict(nested)
  78. # latestVersion and owner (needed for valid detail URLs) live beside the skill.
  79. if data.get("latestVersion") is not None and "latestVersion" not in merged:
  80. merged["latestVersion"] = data["latestVersion"]
  81. if "owner" in data and "owner" not in merged:
  82. merged["owner"] = data["owner"]
  83. return merged
  84. @staticmethod
  85. def _owner_from_payload(data: Optional[dict[str, Any]]) -> Optional[str]:
  86. owner = data.get("owner") if isinstance(data, dict) else None
  87. if isinstance(owner, dict):
  88. owner = owner.get("handle")
  89. return owner.strip() if isinstance(owner, str) and owner.strip() else None
  90. @classmethod
  91. def _owner_matches(cls, expected_owner: Optional[str], data: Optional[dict[str, Any]]) -> bool:
  92. if not expected_owner:
  93. return True
  94. actual = cls._owner_from_payload(data)
  95. return not actual or actual.lower() == expected_owner.lower()
  96. @classmethod
  97. def _item_to_meta(cls, item: dict[str, Any]) -> Optional[SkillMeta]:
  98. """Listing item -> SkillMeta (None without a slug)."""
  99. slug = item.get("slug")
  100. if not isinstance(slug, str) or not slug:
  101. return None
  102. owner = cls._owner_from_payload(item)
  103. return SkillMeta(
  104. name=item.get("displayName") or item.get("name") or slug,
  105. description=item.get("summary") or item.get("description") or "",
  106. source="clawhub", identifier=slug, trust_level="community",
  107. tags=cls._normalize_tags(item.get("tags", [])), extra={"owner": owner} if owner else {},
  108. )
  109. def _skill_detail(self, identifier: str) -> Optional[tuple[str, dict[str, Any]]]:
  110. """``(slug, payload)`` for an identifier, or None when unparsable,
  111. missing, or owned by someone other than the ``@owner`` requested."""
  112. parsed = self._parse_identifier(identifier)
  113. if parsed is None:
  114. return None
  115. slug, expected_owner = parsed
  116. # ClawHub answers a slug claimed by several owners with 409
  117. # AMBIGUOUS_SKILL_SLUG; the ?owner= query picks ours (#104117).
  118. data = self._coerce_skill_payload(
  119. self._get_json(
  120. f"{self.BASE_URL}/skills/{slug}",
  121. params={"owner": expected_owner} if expected_owner else None,
  122. )
  123. )
  124. if not isinstance(data, dict) or not self._owner_matches(expected_owner, data):
  125. return None
  126. return slug, data
  127. def _exact_slug_meta(self, query: str) -> Optional[SkillMeta]:
  128. query = query.strip()
  129. parsed, query_terms = self._parse_identifier(query), _query_terms(query)
  130. candidates = [parsed[0]] if parsed else [query] if "/" not in query and self._SLUG_RE.fullmatch(query) else []
  131. if query_terms:
  132. base_slug = "-".join(query_terms)
  133. if len(query_terms) >= 2:
  134. candidates.extend(f"{base_slug}-{s}" for s in ("agent", "skill", "tool", "assistant", "playbook"))
  135. candidates.append(base_slug)
  136. return next((m for m in map(self.inspect, dict.fromkeys(candidates)) if m), None)
  137. def _finalize_search_results(self, query: str, results: list[SkillMeta], limit: int) -> list[SkillMeta]:
  138. query_norm = query.strip()
  139. if not query_norm:
  140. return _dedupe_results(results)[:limit]
  141. filtered = [meta for meta in results if _search_score(query_norm, meta) > 0]
  142. filtered.sort(key=lambda meta: (-_search_score(query_norm, meta), meta.name.lower(), meta.identifier.lower()))
  143. filtered = _dedupe_results(filtered)
  144. exact = self._exact_slug_meta(query_norm)
  145. if exact:
  146. filtered = _dedupe_results([exact] + [m for m in filtered if _search_score(query_norm, m) >= 20])
  147. if filtered or re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._/-]*", query_norm):
  148. return filtered[:limit]
  149. return _dedupe_results(results)[:limit]
  150. def search(self, query: str, limit: int = 10) -> list[SkillMeta]:
  151. query = query.strip()
  152. if query:
  153. if len(_query_terms(query)) >= 2:
  154. direct = self._exact_slug_meta(query)
  155. if direct:
  156. return [direct]
  157. results = self._search_catalog(query, limit=limit)
  158. if results:
  159. return results
  160. else:
  161. # Empty query: paginating catalog walker. A disk-cached full catalog
  162. # is returned whole (caller paginates); on a cold cache the walk is
  163. # bounded to `limit` so browse renders page one without walking
  164. # 50k+ skills (max_items=0 = unbounded, offline index builder only).
  165. catalog = self._load_catalog_index(max_items=max(limit, 0))
  166. if catalog:
  167. deduped = _dedupe_results(catalog)
  168. return deduped[:limit] if limit > 0 else deduped
  169. # Catalog miss / walker failure: best-effort lightweight listing API.
  170. cache_key = f"clawhub_search_listing_v1_{hashlib.md5(query.encode()).hexdigest()}_{limit}"
  171. cached = _cached_metas(cache_key)
  172. if cached is not None:
  173. return self._finalize_search_results(query, cached, limit)
  174. data = self._get_json(f"{self.BASE_URL}/skills", timeout=15, params={"search": query, "limit": limit})
  175. skills_data = data.get("items", data) if isinstance(data, dict) else data
  176. if not isinstance(skills_data, list):
  177. return []
  178. results = [m for m in map(self._item_to_meta, skills_data[:limit]) if m]
  179. final_results = self._finalize_search_results(query, results, limit)
  180. _cache_metas(cache_key, final_results)
  181. return final_results
  182. @classmethod
  183. def _parse_identifier(cls, identifier: str) -> Optional[tuple[str, Optional[str]]]:
  184. """``(slug, expected_owner)`` for a bare slug, ``clawhub/<slug>``,
  185. ``@owner/slug``, or the URL path ``owner/skills/slug``.
  186. GitHub-style ``owner/repo/skill`` identifiers are NOT ClawHub's —
  187. claiming them by last segment would install a same-named skill from a
  188. different author.
  189. """
  190. raw = (identifier or "").strip()
  191. if not raw:
  192. return None
  193. had_at = raw.startswith("@")
  194. stripped = raw.removeprefix("@").removeprefix("clawhub/")
  195. if stripped.startswith("@"): # clawhub/@owner/slug — @ surfaces after the prefix
  196. had_at = True
  197. stripped = stripped.removeprefix("@")
  198. parts = [part for part in stripped.split("/") if part]
  199. if len(parts) == 1:
  200. owner, slug = None, parts[0]
  201. elif (len(parts) == 2 and had_at) or (len(parts) == 3 and parts[1].lower() == "skills"):
  202. owner, slug = parts[0], parts[-1]
  203. else:
  204. return None
  205. if not cls._SLUG_RE.fullmatch(slug) or (owner is not None and not cls._SLUG_RE.fullmatch(owner)):
  206. return None
  207. return slug, owner
  208. @classmethod
  209. def catalog_identifier(cls, identifier: Any) -> Optional[str]:
  210. """``clawhub/<slug>``, the owner-less id a catalog card installs, for any ClawHub
  211. ``identifier`` (``fetch`` records ``@owner/slug``). None when it is not a ClawHub id,
  212. including a non-string value from a hand-edited lock."""
  213. parsed = cls._parse_identifier(identifier) if isinstance(identifier, str) else None
  214. return f"clawhub/{parsed[0]}" if parsed else None
  215. def fetch(self, identifier: str) -> Optional[SkillBundle]:
  216. detail = self._skill_detail(identifier)
  217. if detail is None:
  218. return None
  219. slug, skill_data = detail
  220. # Keep the requested identity even when the detail payload omits its owner.
  221. owner = self._parse_identifier(identifier)[1] or self._owner_from_payload(skill_data)
  222. owner_params = {"owner": owner} if owner else None
  223. latest_version = self._resolve_latest_version(slug, skill_data, owner=owner)
  224. if not latest_version:
  225. logger.warning("ClawHub fetch failed for %s: could not resolve latest version", slug)
  226. return None
  227. # Primary: ZIP bundle from /download. Fallback: version metadata with
  228. # inline/raw content (files may sit under version_data["version"]).
  229. files = self._download_zip(slug, latest_version, owner=owner)
  230. if "SKILL.md" not in files:
  231. version_data = self._get_json(
  232. f"{self.BASE_URL}/skills/{slug}/versions/{latest_version}", params=owner_params,
  233. )
  234. if isinstance(version_data, dict) and self._owner_matches(owner, version_data):
  235. files = self._extract_files(version_data) or files
  236. nested = version_data.get("version", {})
  237. if "SKILL.md" not in files and isinstance(nested, dict):
  238. files = self._extract_files(nested) or files
  239. if "SKILL.md" not in files:
  240. logger.warning(
  241. "ClawHub fetch for %s resolved version %s but could not retrieve file content", slug, latest_version,
  242. )
  243. return None
  244. return SkillBundle(name=slug, files=files, source="clawhub",
  245. identifier=f"@{owner}/{slug}" if owner else slug, trust_level="community")
  246. def inspect(self, identifier: str) -> Optional[SkillMeta]:
  247. detail = self._skill_detail(identifier)
  248. if detail is None:
  249. return None
  250. slug, data = detail
  251. meta = self._item_to_meta({**data, "slug": data.get("slug") or slug})
  252. owner = self._parse_identifier(identifier)[1] or self._owner_from_payload(data)
  253. if meta is not None and owner:
  254. meta.identifier = f"@{owner}/{slug}"
  255. meta.extra["owner"] = owner
  256. return meta
  257. def _search_catalog(self, query: str, limit: int = 10) -> list[SkillMeta]:
  258. cache_key = f"clawhub_search_catalog_v1_{hashlib.md5(f'{query}|{limit}'.encode()).hexdigest()}"
  259. cached = _cached_metas(cache_key)
  260. if cached is not None:
  261. return cached[:limit]
  262. catalog = self._load_catalog_index()
  263. if not catalog:
  264. return []
  265. results = self._finalize_search_results(query, catalog, limit)
  266. _cache_metas(cache_key, results)
  267. return results
  268. def _load_catalog_index(self, max_items: int = 0) -> list[SkillMeta]:
  269. """Walk the ClawHub catalog via cursor pagination.
  270. ``max_items`` stops the walk early once that many distinct skills are
  271. gathered (browse's cold-start fallback renders one page); ``0`` walks
  272. to exhaustion (offline index builder). Only a COMPLETE walk (cursor
  273. exhausted or page cap) is written to the shared ``clawhub_catalog_v1``
  274. cache — a walk cut by ``max_items``, the wall-clock budget, or a
  275. failed page fetch would poison it with a partial slice.
  276. ``_get_json`` returns ``None`` on timeout/non-200. That is a hole in the
  277. walk, not catalog exhaustion: the same cursor is retried a few times
  278. before the walk gives up (partial, uncached).
  279. """
  280. cache_key = "clawhub_catalog_v1"
  281. cached = _cached_metas(cache_key)
  282. if cached is not None:
  283. return cached
  284. cursor: Optional[str] = None
  285. results: list[SkillMeta] = []
  286. seen: set[str] = set()
  287. # 750 pages * 200/page = 150k ceiling over the ~50k catalog; a safety
  288. # rail against an infinite-cursor loop, normally ended by nextCursor=None.
  289. # Wall-clock budget applies to interactive browse only: the index builder
  290. # (max_items=0) must walk everything or it trips the deploy health floor.
  291. deadline = time.monotonic() + self.CATALOG_WALK_BUDGET_SECONDS if max_items > 0 else None
  292. partial = False
  293. fetch_failures = 0
  294. for _ in range(750):
  295. if deadline is not None and time.monotonic() > deadline:
  296. partial = True
  297. break
  298. params: dict[str, Any] = {"limit": 200, "cursor": cursor} if cursor else {"limit": 200}
  299. data = self._get_json(f"{self.BASE_URL}/skills", timeout=30, params=params)
  300. if data is None:
  301. fetch_failures += 1
  302. if fetch_failures >= self.CATALOG_PAGE_RETRIES:
  303. partial = True
  304. break
  305. # Interactive browse stays inside its 12 s budget; the index builder backs off.
  306. time.sleep(0.5 if deadline is not None else min(2 ** fetch_failures, 8))
  307. continue
  308. fetch_failures = 0
  309. items = data.get("items", []) if isinstance(data, dict) else []
  310. if not isinstance(items, list) or not items:
  311. break
  312. for item in items:
  313. slug = item.get("slug")
  314. if isinstance(slug, str) and slug and slug not in seen:
  315. seen.add(slug)
  316. meta = self._item_to_meta(item)
  317. if meta:
  318. results.append(meta)
  319. cursor = data.get("nextCursor") if isinstance(data, dict) else None
  320. if not isinstance(cursor, str) or not cursor:
  321. break
  322. if max_items > 0 and len(results) >= max_items:
  323. partial = True
  324. break
  325. if not partial:
  326. _cache_metas(cache_key, results)
  327. return results
  328. def _resolve_latest_version(self, slug: str, skill_data: dict[str, Any],
  329. owner: Optional[str] = None) -> Optional[str]:
  330. latest, tags = skill_data.get("latestVersion"), skill_data.get("tags")
  331. version = _first_str(
  332. latest.get("version") if isinstance(latest, dict) else None,
  333. tags.get("latest") if isinstance(tags, dict) else None,
  334. )
  335. if version:
  336. return version
  337. vd = self._get_json(f"{self.BASE_URL}/skills/{slug}/versions",
  338. params={"owner": owner} if owner else None)
  339. if isinstance(vd, dict):
  340. vd = vd.get("items")
  341. return _first_str(vd[0].get("version")) if isinstance(vd, list) and vd and isinstance(vd[0], dict) else None
  342. def _fetch_owner_handle(self, slug: str) -> Optional[str]:
  343. """Owner handle from the detail API (the listing API lacks it), or None.
  344. Bounded retry: 3 attempts total. 429 honours ``Retry-After`` else
  345. exponential backoff (2s -> 4s); 5xx and transport errors back off;
  346. other 4xx means the resource doesn't exist — no retry.
  347. """
  348. url = f"{self.BASE_URL}/skills/{slug}"
  349. max_attempts = 3
  350. for attempt in range(max_attempts):
  351. delay = 2.0 * (2 ** attempt)
  352. try:
  353. resp = hub()._skills_hub_http_get(url, timeout=20)
  354. except (httpx.HTTPError, OSError):
  355. reason = "transport error"
  356. else:
  357. if resp.status_code == 200:
  358. try:
  359. raw = resp.json()
  360. except (json.JSONDecodeError, ValueError):
  361. return None
  362. return self._owner_from_payload(self._coerce_skill_payload(raw))
  363. if resp.status_code == 429:
  364. retry_after = parse_retry_after_seconds(resp.headers)
  365. if retry_after is not None:
  366. delay = retry_after
  367. reason = "HTTP 429"
  368. elif 500 <= resp.status_code < 600:
  369. reason = f"HTTP {resp.status_code}"
  370. else:
  371. return None # 4xx (non-429): doesn't exist / bad request
  372. if attempt >= max_attempts - 1:
  373. return None
  374. logger.debug("_fetch_owner_handle(%s): %s on attempt %d/%d, retrying in %.1fs",
  375. slug, reason, attempt + 1, max_attempts, delay)
  376. time.sleep(delay)
  377. return None
  378. def enrich_owners(self, skills: list[SkillMeta], max_workers: int = 30,
  379. budget_seconds: Optional[float] = None) -> int:
  380. """Batch-fetch owner handles for ClawHub skills missing ``extra["owner"]``
  381. (in-place; returns the number enriched).
  382. ``budget_seconds`` makes this best-effort: the detail API answers in ~2s, so the
  383. full catalog (78k+ skills) needs well over an hour at 30 workers — unbounded, it
  384. was the phase that pushed the index build past its CI timeout for two months.
  385. Skills left un-enriched simply ship without a "View source" owner link.
  386. Safety rails: aborts after 50 consecutive failures (systemic outage),
  387. per-request 429 backoff, progress log every 1000 skills.
  388. """
  389. needs_enrichment = [s for s in skills if s.source == "clawhub" and not (s.extra or {}).get("owner")]
  390. if not needs_enrichment:
  391. return 0
  392. enriched = consecutive_failures = processed = 0
  393. max_consecutive_failures = 50
  394. deadline = time.monotonic() + budget_seconds if budget_seconds is not None else None
  395. from concurrent.futures import ThreadPoolExecutor, as_completed
  396. with ThreadPoolExecutor(max_workers=max_workers) as pool:
  397. futures = {pool.submit(self._fetch_owner_handle, s.identifier): s for s in needs_enrichment}
  398. for future in as_completed(futures):
  399. meta = futures[future]
  400. processed += 1
  401. if deadline is not None and time.monotonic() > deadline:
  402. logger.warning("ClawHub owner enrichment: budget of %.0fs exhausted after %d/%d "
  403. "(%d enriched) — shipping the rest without owner handles.",
  404. budget_seconds, processed, len(needs_enrichment), enriched)
  405. for f in futures:
  406. f.cancel()
  407. break
  408. try:
  409. handle = future.result()
  410. except Exception:
  411. handle = None
  412. consecutive_failures = 0 if handle else consecutive_failures + 1
  413. if handle:
  414. meta.extra = meta.extra or {}
  415. meta.extra["owner"] = handle
  416. enriched += 1
  417. if processed % 1000 == 0:
  418. logger.info("ClawHub owner enrichment: %d/%d processed, %d enriched",
  419. processed, len(needs_enrichment), enriched)
  420. if consecutive_failures >= max_consecutive_failures:
  421. logger.warning(
  422. "ClawHub owner enrichment: %d consecutive failures — "
  423. "aborting early (%d/%d processed, %d enriched). "
  424. "The ClawHub API may be down or rate-limited.",
  425. max_consecutive_failures, processed, len(needs_enrichment), enriched,
  426. )
  427. for f in futures:
  428. f.cancel()
  429. break
  430. return enriched
  431. def _extract_files(self, version_data: dict[str, Any]) -> dict[str, str]:
  432. files: dict[str, str] = {}
  433. file_list = version_data.get("files")
  434. if isinstance(file_list, dict):
  435. return {k: v for k, v in file_list.items() if isinstance(v, str)}
  436. for file_meta in file_list if isinstance(file_list, list) else []:
  437. if not isinstance(file_meta, dict):
  438. continue
  439. fname, inline = file_meta.get("path") or file_meta.get("name"), file_meta.get("content")
  440. raw_url = file_meta.get("rawUrl") or file_meta.get("downloadUrl") or file_meta.get("url")
  441. if not fname or not isinstance(fname, str):
  442. continue
  443. if isinstance(inline, str):
  444. files[fname] = inline
  445. elif isinstance(raw_url, str) and raw_url.startswith("http"):
  446. content = self._fetch_text(raw_url)
  447. if content is not None:
  448. files[fname] = content
  449. return files
  450. def _download_zip(self, slug: str, version: str, owner: Optional[str] = None) -> dict[str, str]:
  451. """Download the skill ZIP from /download (bounded, streamed) and extract its text files."""
  452. import io
  453. import zipfile
  454. files: dict[str, str] = {}
  455. params = {"slug": slug, "version": version}
  456. if owner:
  457. params["owner"] = owner
  458. max_retries = 3
  459. for attempt in range(max_retries):
  460. retry_after_delay: Optional[int] = None
  461. try:
  462. with _guarded_http_stream(
  463. f"{self.BASE_URL}/download",
  464. params=params,
  465. timeout=30,
  466. ) as resp:
  467. if resp is None:
  468. return files
  469. if resp.status_code == 429:
  470. parsed = parse_retry_after_seconds(resp.headers)
  471. retry_after = min(int(5 if parsed is None else parsed), 15) # Cap wait time
  472. logger.debug(
  473. "ClawHub download rate-limited for %s, retrying in %ds (attempt %d/%d)",
  474. slug, retry_after, attempt + 1, max_retries,
  475. )
  476. retry_after_delay = retry_after
  477. else:
  478. if resp.status_code != 200:
  479. logger.debug("ClawHub ZIP download for %s v%s returned %s", slug, version, resp.status_code)
  480. return files
  481. content_length = resp.headers.get("content-length")
  482. if content_length:
  483. try:
  484. declared_size = int(content_length)
  485. except (ValueError, TypeError):
  486. declared_size = 0
  487. if declared_size > self.ZIP_DOWNLOAD_MAX_BYTES:
  488. logger.debug(
  489. "Skipping oversized ClawHub ZIP for %s v%s: %d bytes",
  490. slug, version, declared_size,
  491. )
  492. return files
  493. archive = io.BytesIO()
  494. total = 0
  495. for chunk in resp.iter_bytes(chunk_size=self.ZIP_DOWNLOAD_CHUNK_BYTES):
  496. if not chunk:
  497. continue
  498. total += len(chunk)
  499. if total > self.ZIP_DOWNLOAD_MAX_BYTES:
  500. logger.debug(
  501. "Skipping oversized ClawHub ZIP for %s v%s: exceeded %d bytes",
  502. slug, version, self.ZIP_DOWNLOAD_MAX_BYTES,
  503. )
  504. return files
  505. archive.write(chunk)
  506. archive.seek(0)
  507. if retry_after_delay is not None:
  508. if attempt < max_retries - 1:
  509. time.sleep(retry_after_delay)
  510. continue
  511. with zipfile.ZipFile(archive) as zf:
  512. for info in zf.infolist():
  513. if info.is_dir():
  514. continue
  515. try:
  516. name = _validate_bundle_rel_path(info.filename)
  517. except ValueError:
  518. logger.debug("Skipping unsafe ZIP member path: %s", info.filename)
  519. continue
  520. if info.file_size > 500_000: # skip large binaries
  521. logger.debug("Skipping large file in ZIP: %s (%d bytes)", name, info.file_size)
  522. continue
  523. try:
  524. files[name] = zf.read(info.filename).decode("utf-8")
  525. except (UnicodeDecodeError, KeyError):
  526. logger.debug("Skipping non-text file in ZIP: %s", name)
  527. return files
  528. except zipfile.BadZipFile:
  529. logger.warning("ClawHub returned invalid ZIP for %s v%s", slug, version)
  530. return files
  531. except httpx.HTTPError as exc:
  532. logger.debug("ClawHub ZIP download failed for %s v%s: %s", slug, version, exc)
  533. return files
  534. logger.debug("ClawHub ZIP download exhausted retries for %s v%s", slug, version)
  535. return files