model_catalog.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405
  1. """Remote model catalog fetcher.
  2. ``get_catalog()`` returns the parsed manifest: in-process cache (TTL) → disk cache at
  3. ``~/.hermes/cache/model_catalog.json`` → master URL fetch; any fetch failure keeps the stale copy
  4. (or ``{}``, and the URL is not retried for ``FETCH_RETRY_DELAY_SECONDS``). ``get_curated_openrouter_models()`` / ``get_curated_nous_models()`` are thin accessors
  5. whose callers fall back to the in-repo lists on ``None``.
  6. """
  7. from __future__ import annotations
  8. import contextvars
  9. import json
  10. import logging
  11. import threading
  12. import time
  13. import urllib.error
  14. import urllib.request
  15. from pathlib import Path
  16. from typing import Any
  17. from hermes_cli.version_info import hermes_user_agent
  18. from utils import atomic_json_write
  19. logger = logging.getLogger(__name__)
  20. DEFAULT_CATALOG_URL = (
  21. "https://hermes-agent.nousresearch.com/docs/api/model-catalog.json")
  22. # The Docusaurus site sits behind Vercel, which occasionally 403s non-browser clients (bot
  23. # challenge); the raw GitHub copy is the same manifest and is not bot-gated.
  24. DEFAULT_CATALOG_FALLBACK_URLS: tuple[str, ...] = (
  25. "https://raw.githubusercontent.com/NousResearch/hermes-agent/main/website/static/api/model-catalog.json",
  26. )
  27. DEFAULT_TTL_MINUTES = 20
  28. # Legacy key, honoured only when the user set it explicitly; ``ttl_minutes`` is the shipped default.
  29. DEFAULT_TTL_HOURS = DEFAULT_TTL_MINUTES / 60.0
  30. DEFAULT_FETCH_TIMEOUT = 8.0
  31. # After a failed fetch with no cached manifest to serve, the same URL is not retried in this process
  32. # for this long: an offline user would otherwise wait out the timeout (primary + fallback URL, ~16 s)
  33. # on every picker build. Same window as the models.dev registry's failure backoff.
  34. FETCH_RETRY_DELAY_SECONDS = 300.0
  35. SUPPORTED_SCHEMA_VERSION = 1
  36. # In-process cache, invalidated against the disk file's path + mtime and TTL. The path matters:
  37. # under a multiplexed gateway each profile has its own ``<home>/cache/model_catalog.json``, and
  38. # mtime alone cannot tell two profiles' files apart.
  39. _catalog_cache: dict[str, Any] | None = None
  40. _catalog_cache_source_mtime: float = 0.0
  41. _catalog_cache_source_path: str = ""
  42. # ``monotonic()`` deadline before which a failed fetch is not retried (see FETCH_RETRY_DELAY_SECONDS),
  43. # keyed by (profile home key, "manifest" | "override", URL): one gateway process serves many profiles,
  44. # and a dead override must not skip the manifest's fallback chain when both share a URL. Pickers and
  45. # the gateway's refresh thread touch it concurrently, hence the lock.
  46. _fetch_retry_after: dict[tuple[str, str, str], float] = {}
  47. _fetch_retry_lock = threading.Lock()
  48. def _backoff_key(kind: str, url: str) -> tuple[str, str, str]:
  49. from hermes_constants import hermes_home_key
  50. return (hermes_home_key(), kind, url)
  51. def _backing_off(kind: str, url: str) -> bool:
  52. key = _backoff_key(kind, url)
  53. with _fetch_retry_lock:
  54. return time.monotonic() < _fetch_retry_after.get(key, 0.0)
  55. def _note_fetch_result(kind: str, url: str, ok: bool) -> None:
  56. key = _backoff_key(kind, url)
  57. with _fetch_retry_lock:
  58. if ok:
  59. _fetch_retry_after.pop(key, None)
  60. else:
  61. _fetch_retry_after[key] = time.monotonic() + FETCH_RETRY_DELAY_SECONDS
  62. def clear_fetch_backoff() -> None:
  63. """Forget the active profile's fetch failures so an explicit refresh retries every URL."""
  64. from hermes_constants import hermes_home_key
  65. home = hermes_home_key()
  66. with _fetch_retry_lock:
  67. for key in [k for k in _fetch_retry_after if k[0] == home]:
  68. del _fetch_retry_after[key]
  69. def _load_catalog_config() -> dict[str, Any]:
  70. """Load the ``model_catalog`` config block with defaults filled in."""
  71. try:
  72. from hermes_cli.config import load_config
  73. cfg = load_config() or {}
  74. except Exception:
  75. cfg = {}
  76. raw = cfg.get("model_catalog")
  77. if not isinstance(raw, dict):
  78. raw = {}
  79. # ``ttl_hours`` (legacy) is honoured only when ``ttl_minutes`` is still at its default —
  80. # load_config() deep-merges the default in, so "present" alone doesn't mean "user-set".
  81. ttl_minutes = raw.get("ttl_minutes")
  82. try:
  83. ttl_minutes = float(ttl_minutes) if ttl_minutes not in (None, "") else DEFAULT_TTL_MINUTES
  84. except (TypeError, ValueError):
  85. ttl_minutes = DEFAULT_TTL_MINUTES
  86. if ttl_minutes == DEFAULT_TTL_MINUTES and raw.get("ttl_hours"):
  87. try:
  88. ttl_minutes = float(raw["ttl_hours"]) * 60.0
  89. except (TypeError, ValueError):
  90. pass
  91. if ttl_minutes <= 0:
  92. ttl_minutes = DEFAULT_TTL_MINUTES
  93. return {
  94. "enabled": bool(raw.get("enabled", True)),
  95. "url": str(raw.get("url") or DEFAULT_CATALOG_URL),
  96. "ttl_hours": ttl_minutes / 60.0,
  97. "providers": raw.get("providers") if isinstance(raw.get("providers"), dict) else {}}
  98. def _cache_path() -> Path:
  99. """Disk cache path; imported lazily so tests can monkeypatch home."""
  100. from hermes_constants import get_hermes_home
  101. return get_hermes_home() / "cache" / "model_catalog.json"
  102. def _fetch_manifest(url: str, timeout: float) -> dict[str, Any] | None:
  103. """HTTP GET the manifest URL and return a validated dict, or None on failure."""
  104. try:
  105. req = urllib.request.Request(url, headers={"Accept": "application/json", "User-Agent": hermes_user_agent()})
  106. with urllib.request.urlopen(req, timeout=timeout) as resp:
  107. data = json.loads(resp.read().decode())
  108. except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError) as exc:
  109. logger.info("model catalog fetch failed (%s): %s", url, exc)
  110. return None
  111. except Exception as exc: # pragma: no cover — defensive
  112. logger.info("model catalog fetch errored (%s): %s", url, exc)
  113. return None
  114. if not _validate_manifest(data):
  115. logger.info("model catalog at %s failed schema validation", url)
  116. return None
  117. return data
  118. def _fetch_manifest_with_fallback(
  119. primary_url: str, timeout: float, fallback_urls: tuple[str, ...] = DEFAULT_CATALOG_FALLBACK_URLS
  120. ) -> dict[str, Any] | None:
  121. """First manifest that fetches and validates from ``primary_url`` then ``fallback_urls`` (skipping
  122. any equal to the primary so a raw-GitHub-configured operator doesn't double-fetch), or None."""
  123. data = _fetch_manifest(primary_url, timeout)
  124. if data is not None:
  125. return data
  126. for url in fallback_urls:
  127. if not url or url == primary_url:
  128. continue
  129. data = _fetch_manifest(url, timeout)
  130. if data is not None:
  131. logger.info("model catalog primary URL failed; using fallback %s", url)
  132. return data
  133. return None
  134. def _validate_manifest(data: Any) -> bool:
  135. """Return True when ``data`` matches the minimum manifest shape."""
  136. if not isinstance(data, dict):
  137. return False
  138. version = data.get("version")
  139. if not isinstance(version, int) or version > SUPPORTED_SCHEMA_VERSION:
  140. return False # future schema we don't understand — refuse rather than guess
  141. providers = data.get("providers")
  142. if not isinstance(providers, dict):
  143. return False
  144. for pname, pblock in providers.items():
  145. if not isinstance(pname, str) or not isinstance(pblock, dict):
  146. return False
  147. models = pblock.get("models")
  148. if not isinstance(models, list):
  149. return False
  150. if not all(isinstance(m, dict) and isinstance(m.get("id"), str) and m["id"].strip() for m in models):
  151. return False
  152. return True
  153. def _read_disk_cache() -> tuple[dict[str, Any] | None, float]:
  154. """Return ``(data_or_none, mtime)``. mtime is 0 if file is missing."""
  155. path = _cache_path()
  156. try:
  157. mtime = path.stat().st_mtime
  158. except (OSError, FileNotFoundError):
  159. return (None, 0.0)
  160. try:
  161. with open(path, encoding="utf-8-sig") as fh:
  162. data = json.load(fh)
  163. except (OSError, json.JSONDecodeError):
  164. return (None, 0.0)
  165. return (data, mtime) if _validate_manifest(data) else (None, 0.0)
  166. def _write_disk_cache(data: dict[str, Any]) -> None:
  167. try:
  168. atomic_json_write(_cache_path(), data)
  169. except OSError as exc:
  170. logger.info("model catalog cache write failed: %s", exc)
  171. # Stale-while-revalidate: at most one background manifest refresh in flight per cache file (i.e.
  172. # per profile home — profile A's refresh must not suppress profile B's). The refreshed manifest
  173. # lands on disk; the NEXT get_catalog() call picks it up via the mtime check.
  174. _catalog_swr_lock = threading.Lock()
  175. _catalog_swr_inflight: set[str] = set()
  176. def _spawn_catalog_swr_refresh(url: str) -> None:
  177. """Refresh the catalog manifest off-thread (fire-and-forget, deduped per cache path)."""
  178. inflight_key = str(_cache_path())
  179. with _catalog_swr_lock:
  180. if inflight_key in _catalog_swr_inflight:
  181. return
  182. _catalog_swr_inflight.add(inflight_key)
  183. def _refresh() -> None:
  184. try:
  185. fetched = _fetch_manifest_with_fallback(url, DEFAULT_FETCH_TIMEOUT)
  186. if fetched is not None:
  187. _write_disk_cache(fetched)
  188. except Exception:
  189. logger.debug("catalog SWR refresh failed", exc_info=True)
  190. finally:
  191. with _catalog_swr_lock:
  192. _catalog_swr_inflight.discard(inflight_key)
  193. # copy_context: the picker may be serving a profile scoped by the HERMES_HOME ContextVar
  194. # (tui_gateway ``_profile_scoped``), so the worker must write THAT profile's cache file.
  195. context = contextvars.copy_context()
  196. threading.Thread(target=lambda: context.run(_refresh), daemon=True, name="model-catalog-swr").start()
  197. def _remember(data: dict[str, Any], mtime: float) -> dict[str, Any]:
  198. global _catalog_cache, _catalog_cache_source_mtime, _catalog_cache_source_path
  199. _catalog_cache, _catalog_cache_source_mtime = data, mtime
  200. _catalog_cache_source_path = str(_cache_path())
  201. return data
  202. def _in_process_catalog() -> dict[str, Any] | None:
  203. """The in-process copy when it mirrors the ACTIVE profile's cache file, else None."""
  204. if _catalog_cache is not None and _catalog_cache_source_path == str(_cache_path()):
  205. return _catalog_cache
  206. return None
  207. def get_catalog(*, force_refresh: bool = False) -> dict[str, Any]:
  208. """Parsed model catalog manifest, or ``{}`` on failure — never raises, so the CLI works offline
  209. (callers treat a missing provider/model as "use the in-repo fallback")."""
  210. cfg = _load_catalog_config()
  211. if not cfg["enabled"]:
  212. return {}
  213. ttl_seconds = max(0.0, cfg["ttl_hours"] * 3600.0)
  214. disk_data, disk_mtime = _read_disk_cache()
  215. now = time.time()
  216. disk_fresh = disk_data is not None and (now - disk_mtime) < ttl_seconds
  217. if not force_refresh and disk_data is not None:
  218. cached = _in_process_catalog()
  219. if disk_fresh and cached is not None and disk_mtime == _catalog_cache_source_mtime:
  220. return cached
  221. if not disk_fresh:
  222. # Stale-while-revalidate: serve the expired disk copy now and refresh off-thread so the
  223. # /model picker (which calls this on every open) never blocks on the manifest fetch.
  224. # Only a cold cache (no disk copy at all) still blocks.
  225. _spawn_catalog_swr_refresh(cfg["url"])
  226. return _remember(disk_data, disk_mtime)
  227. if not force_refresh and disk_data is None and _backing_off("manifest", cfg["url"]):
  228. return {}
  229. fetched = _fetch_manifest_with_fallback(cfg["url"], DEFAULT_FETCH_TIMEOUT)
  230. _note_fetch_result("manifest", cfg["url"], fetched is not None)
  231. if fetched is not None:
  232. _write_disk_cache(fetched)
  233. new_disk_data, new_mtime = _read_disk_cache()
  234. if new_disk_data is not None:
  235. return _remember(new_disk_data, new_mtime)
  236. return _remember(fetched, now)
  237. if disk_data is not None:
  238. return _remember(disk_data, disk_mtime)
  239. return {}
  240. def refresh_interval_seconds() -> float:
  241. """Return the configured catalog TTL in seconds (the gateway poll cadence)."""
  242. return max(60.0, _load_catalog_config()["ttl_hours"] * 3600.0)
  243. def refresh_catalogs() -> bool:
  244. """Force-refresh every remote catalog the picker reads (manifest, OpenRouter live list, and the
  245. Nous Portal recommendations for a Nous account, the only reader), writing each disk cache so the
  246. next ``/model`` open in ANY process sees them. Blocking; run it off the event loop."""
  247. if not _load_catalog_config()["enabled"]:
  248. return False
  249. clear_fetch_backoff() # provider overrides are re-asked too, not only the forced manifest
  250. catalog = get_catalog(force_refresh=True)
  251. try:
  252. from agent.secret_scope import get_secret
  253. from hermes_cli.anon_auth import current_nous_state, is_guest_state
  254. from hermes_cli.auth import read_credential_pool
  255. from hermes_cli.models import _resolve_nous_portal_url, fetch_nous_recommended_models, fetch_openrouter_models
  256. fetch_openrouter_models(force_refresh=True)
  257. logins = [current_nous_state(), *read_credential_pool("nous")]
  258. if any(e and not is_guest_state(e) for e in logins) or get_secret("NOUS_API_KEY"):
  259. fetch_nous_recommended_models(_resolve_nous_portal_url(), force_refresh=True)
  260. except Exception:
  261. logger.debug("provider catalog refresh failed", exc_info=True)
  262. return bool(catalog)
  263. def _fetch_provider_override(provider: str) -> dict[str, Any] | None:
  264. """If ``model_catalog.providers.<name>.url`` is set, fetch that instead."""
  265. cfg = _load_catalog_config()
  266. if not cfg["enabled"]:
  267. return None
  268. provider_cfg = cfg["providers"].get(provider)
  269. if not isinstance(provider_cfg, dict):
  270. return None
  271. override_url = provider_cfg.get("url")
  272. if not isinstance(override_url, str) or not override_url.strip():
  273. return None
  274. # Overrides are usually third-party self-hosted: skip the disk cache, re-request every call
  275. # (except during a failure backoff; the caller then falls back to the main manifest).
  276. override_url = override_url.strip()
  277. if _backing_off("override", override_url):
  278. return None
  279. data = _fetch_manifest(override_url, DEFAULT_FETCH_TIMEOUT)
  280. _note_fetch_result("override", override_url, data is not None)
  281. return data
  282. def _block_of(manifest: dict[str, Any] | None, provider: str) -> dict[str, Any] | None:
  283. block = (manifest or {}).get("providers", {}).get(provider)
  284. return block if isinstance(block, dict) else None
  285. def _get_provider_block(provider: str) -> dict[str, Any] | None:
  286. """Return the provider's manifest block, respecting per-provider overrides."""
  287. return _block_of(_fetch_provider_override(provider), provider) or _block_of(get_catalog(), provider)
  288. def _block_ids(block: dict[str, Any] | None) -> list[tuple[str, dict[str, Any]]]:
  289. """``(id, entry)`` for every model entry of ``block`` with a non-empty id."""
  290. models = (block or {}).get("models", [])
  291. return [(mid, m) for m in models if isinstance(m, dict) and (mid := str(m.get("id") or "").strip())]
  292. def get_curated_openrouter_models() -> list[tuple[str, str]] | None:
  293. """OpenRouter's curated ``[(id, description), ...]`` from the manifest."""
  294. rows = _block_ids(_get_provider_block("openrouter"))
  295. return [(mid, str(m.get("description") or "")) for mid, m in rows] or None
  296. def get_curated_nous_models() -> list[str] | None:
  297. """Nous Portal's curated model ids from the manifest."""
  298. return [mid for mid, _ in _block_ids(_get_provider_block("nous"))] or None
  299. def _default_model_from_block(block: dict[str, Any] | None) -> str | None:
  300. """Id of the model entry labeled ``"default": true``, or None."""
  301. return next((mid for mid, m in _block_ids(block) if m.get("default")), None)
  302. def get_default_model_from_cache(provider: str) -> str | None:
  303. """The manifest's labeled default for ``provider`` (the model Hermes silently lands on when the
  304. user never picked one) — in-process then disk cache only, never a fetch."""
  305. cached = _in_process_catalog()
  306. found = _default_model_from_block(_block_of(cached, provider)) if cached is not None else None
  307. if found:
  308. return found
  309. disk_data, _mtime = _read_disk_cache()
  310. return _default_model_from_block(_block_of(disk_data, provider)) if disk_data is not None else None
  311. def seed_cache_from_checkout(project_root: Path | str) -> bool:
  312. """Overwrite the disk cache with the checkout's ``website/static/api/model-catalog.json``.
  313. After ``hermes update`` that file IS the newest catalog, so the picker stays current even when
  314. the remote fetch is bot-gated. Validated, then written via the same atomic writer."""
  315. src = Path(project_root) / "website" / "static" / "api" / "model-catalog.json"
  316. try:
  317. with open(src, encoding="utf-8-sig") as fh:
  318. data = json.load(fh)
  319. except (OSError, json.JSONDecodeError) as exc:
  320. logger.debug("model catalog seed from checkout skipped (%s): %s", src, exc)
  321. return False
  322. if not _validate_manifest(data):
  323. logger.debug("model catalog seed from checkout skipped: invalid manifest at %s", src)
  324. return False
  325. _write_disk_cache(data)
  326. reset_cache() # drop the in-process copy so the next read picks up the seed
  327. return True
  328. def reset_cache() -> None:
  329. """Clear the in-process cache and fetch-failure backoff. Used by tests and ``hermes model --refresh``."""
  330. global _catalog_cache, _catalog_cache_source_mtime, _catalog_cache_source_path
  331. with _fetch_retry_lock:
  332. _fetch_retry_after.clear()
  333. _catalog_cache = None
  334. _catalog_cache_source_mtime = 0.0
  335. _catalog_cache_source_path = ""