catalog.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418
  1. """Curated starter catalog for the managed local runtime.
  2. Every entry carries the estimator inputs (measured on real GGUFs) so the picker can price a model
  3. BEFORE the user downloads gigabytes; once a file is on disk, profile_from_gguf() is the authority.
  4. Builds proven end-to-end on real hardware are marked validated; day-0 entries ship without the flag
  5. — ensure_model_ready's touch generation still gates every first load at runtime.
  6. """
  7. from __future__ import annotations
  8. import json
  9. import logging
  10. import threading
  11. import time
  12. import urllib.request
  13. from dataclasses import dataclass, field
  14. from pathlib import PurePosixPath
  15. from hermes_cli.local_runtime.context_policy import (
  16. FLOOR, RUNTIME_OVERHEAD_BYTES, TARGET_WINDOW, LaunchPlan, plan_launch)
  17. from hermes_cli.local_runtime.estimator import (
  18. HardwareBudget, LayerKind, ModelProfile, PhysicsRefusal, as_loaded)
  19. from hermes_cli.local_runtime.gguf import model_id_from_stem
  20. from hermes_platform.host.products import is_nvidia_n1x_pci_id
  21. logger = logging.getLogger(__name__)
  22. @dataclass(frozen=True)
  23. class AssetFile:
  24. """One downloadable file: repo-relative path and exact bytes (feeds the estimator and the
  25. progress bar; no download-time integrity check by design — a corrupt file surfaces as a
  26. llama.cpp load error). ``local`` overrides the on-disk name (repos reuse generic names like
  27. mmproj-BF16.gguf). Non-model extras live under the models dir's assets/ subdirectory so the
  28. router never lists them.
  29. """
  30. path: str # repo-relative (may include a subdir)
  31. size_bytes: int
  32. local: str | None = None
  33. repo: str | None = None # HF repo when it is not the entry's own (a companion published elsewhere)
  34. revision: str = "main" # commit whose bytes size_bytes and the entry's pricing describe
  35. @property
  36. def local_name(self) -> str:
  37. return self.local or PurePosixPath(self.path).name
  38. @dataclass(frozen=True)
  39. class QuantVariant:
  40. """One downloadable build. Split GGUFs list every part in files; the model loads from the
  41. first part."""
  42. quant: str # e.g. "UD-Q4_K_M"
  43. files: tuple # AssetFile, first = the load target
  44. validated: bool = False # proven end-to-end on real hardware
  45. # Bytes of tensors the engine reads from the file on demand instead of loading (see
  46. # gguf._LAZY_READ_TENSORS), taken from the build's tensor table at authoring time.
  47. lazy_bytes: int = 0
  48. @property
  49. def model_id(self) -> str:
  50. return model_id_from_stem(PurePosixPath(self.files[0].path).name.removesuffix(".gguf"))
  51. @property
  52. def size_bytes(self) -> int:
  53. return sum(f.size_bytes for f in self.files)
  54. @property
  55. def weights_bytes(self) -> int:
  56. """Pre-download estimate of the weights the engine loads: GGUF bytes ≈ tensor bytes + a <2%
  57. header, less the tensors it reads from disk on demand — slightly conservative until
  58. profile_from_gguf reads the real table."""
  59. return self.size_bytes - self.lazy_bytes
  60. @dataclass(frozen=True)
  61. class CatalogEntry:
  62. id: str # stable family id (variant-independent)
  63. display_name: str
  64. description: str # one line, plain language
  65. repo: str # HF repo
  66. variants: tuple # QuantVariant (exactly one, Q4-class)
  67. # Estimator inputs (measured or config-derived; quant changes weights, never KV). Gated
  68. # upstream configs carry a conservative same-family prior — the GGUF header is the authority
  69. # after download.
  70. n_ctx_train: int
  71. full_layers: int
  72. recurrent_layers: int
  73. per_layer_f16: int # KV bytes/token per full-attention layer
  74. swa_layers: int = 0
  75. swa_window: int = 0
  76. moe: bool = False
  77. mtp: bool = False # ships MTP heads (spec decode when loaded)
  78. # Speculative draft depth for MTP models: per-model and measured — deeper drafting pays only
  79. # while draft acceptance holds, and the break-even depth differs by model.
  80. mtp_draft_depth: int = 3
  81. # Vocab size prices the GPU logits buffers (ubatch x vocab x fp32, doubled under MTP backend
  82. # sampling) — a multi-GiB term at large vocabs that a weights-only fit would miss.
  83. n_vocab: int = 0
  84. # GGUF general.architecture; prices architecture-specific buffers (estimator._WINDOW_COMPUTE_BYTES)
  85. # before the file is on disk.
  86. architecture: str = ""
  87. mmproj: AssetFile | None = None # vision projector, downloads with model
  88. draft: AssetFile | None = None # spec-decode draft model (e.g. DSpark)
  89. # How the draft decodes. draft-dspark drafts only for a model spilled past GPU memory;
  90. # draft-dflash replaces MTP whenever it fits, on one slot (each slot keeps a recurrent-state
  91. # copy per draft position). Optional: a missing draft leaves the model as it was.
  92. draft_spec_type: str = "draft-dspark"
  93. draft_depth: int = 3
  94. draft_min_engine: str = "" # oldest llama.cpp release that loads the draft
  95. # MTP head shipped as its own file (the model carries none): the engine loads it as the
  96. # draft for MTP spec decode. Downloads with the model.
  97. mtp_head: AssetFile | None = None
  98. sampling: dict = field(default_factory=dict) # INI long-form launch defaults
  99. # Oldest llama.cpp release tag that can load this model (day-0 architectures need the release
  100. # where their support landed). Empty means any installed engine.
  101. min_engine: str = ""
  102. # Editorial quality ordering (higher = smarter), authored once at catalog time — Artificial
  103. # Analysis-informed where covered (scripts/aa_quality_sync.py proposes, the commit decides).
  104. # Ranks entries for the per-machine recommendation; never displayed as a score (it grades the
  105. # full-precision model, not our Q4 build).
  106. quality: int = 0
  107. # Fraction of the build's bytes read per decoded token: 1.0 for dense, the active slice for
  108. # MoE. With memory bandwidth this predicts decode speed — the physics half of the
  109. # recommendation.
  110. decode_fraction: float = 1.0
  111. # False lists the entry for explicit choice only: it is never the automatic recommendation or
  112. # quickstart's pick on any machine, so it needs no quality ordering.
  113. recommend: bool = True
  114. @property
  115. def mtp_capable(self) -> bool:
  116. """MTP spec decode runs, from heads built into the model or shipped beside it."""
  117. return self.mtp or self.mtp_head is not None
  118. @property
  119. def companion_bytes(self) -> int:
  120. """Companion files the engine loads beside the weights (vision projector, MTP head)."""
  121. return sum(a.size_bytes for a in (self.mmproj, self.mtp_head) if a is not None)
  122. def profile(self, variant: QuantVariant) -> ModelProfile:
  123. layers = ([(LayerKind.FULL, self.per_layer_f16)] * self.full_layers
  124. + [(LayerKind.SWA, self.per_layer_f16)] * self.swa_layers
  125. + [(LayerKind.RECURRENT, 0)] * self.recurrent_layers)
  126. return ModelProfile(
  127. name=variant.model_id, weights_bytes=variant.weights_bytes, embd_table_bytes=0,
  128. n_ctx_train=self.n_ctx_train, layers=layers, swa_window=self.swa_window, moe=self.moe,
  129. architecture=self.architecture, n_vocab=self.n_vocab,
  130. kv_scale=1.2 if self.mtp_capable else 1.0, lazy_bytes=variant.lazy_bytes)
  131. def launch_plan(self, variant: QuantVariant, budget: HardwareBudget) -> LaunchPlan:
  132. # Optional external drafts may use spare memory after download, never reduce this grant.
  133. return plan_launch(as_loaded(self.profile(variant), budget), budget, mtp_capable=self.mtp_capable,
  134. fixed_overhead=RUNTIME_OVERHEAD_BYTES + self.companion_bytes)
  135. def download_files(self, variant: QuantVariant) -> tuple:
  136. """Everything a download job fetches for this variant, in order."""
  137. extras = tuple(a for a in (self.mmproj, self.draft, self.mtp_head) if a is not None)
  138. return tuple(variant.files) + extras
  139. def download_bytes(self, variant: QuantVariant) -> int:
  140. return sum(f.size_bytes for f in self.download_files(variant))
  141. def install_files(self, variant: QuantVariant) -> tuple:
  142. """The files a variant needs to count as downloaded. The draft is optional: a model
  143. installed before its draft joined the catalog stays installed."""
  144. extras = tuple(a for a in (self.mmproj, self.mtp_head) if a is not None)
  145. return tuple(variant.files) + extras
  146. @dataclass(frozen=True)
  147. class VariantChoice:
  148. """Which build this machine should download and why. reason_key is a UI-copy
  149. discriminator, not display text."""
  150. variant: QuantVariant
  151. zero_spill: bool
  152. reason_key: str # "best-large-window" | "best-fits" | "smallest-fits-spilled"
  153. def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None:
  154. """Fit the entry's one Q4-class build to this machine; headroom buys a bigger window, never a
  155. bigger quant.
  156. "best-large-window": zero-spill at TARGET_WINDOW; "best-fits": zero-spill at the 64K floor;
  157. "smallest-fits-spilled": weights spill to host RAM, priced honestly; None: physics refuses.
  158. """
  159. variant = entry.variants[-1]
  160. decision = entry.launch_plan(variant, budget).decision
  161. if isinstance(decision, PhysicsRefusal):
  162. return None
  163. if decision.spilled:
  164. return VariantChoice(variant, zero_spill=False, reason_key="smallest-fits-spilled")
  165. reason = "best-large-window" if decision.window >= min(TARGET_WINDOW, entry.n_ctx_train or FLOOR) else "best-fits"
  166. return VariantChoice(variant, zero_spill=True, reason_key=reason)
  167. # ── recommendation: best quality that fits and isn't miserably slow ──
  168. #
  169. # QUALITY is a judgment made once at authoring time (entry.quality). SPEED is physics per machine:
  170. # decode is memory-bound, so predicted tok/s ≈ bandwidth / bytes-read-per-token (build size scaled
  171. # by decode fraction). The bandwidth axis is the `uma` flag: every discrete card that matters is
  172. # 900+ GB/s GDDR while the unified-memory class measures ~1/5th of that. A measured per-machine
  173. # bandwidth could replace these class constants without touching the rule; predictions order
  174. # candidates and gate the floor — they are not display values.
  175. _DISCRETE_BANDWIDTH_GB_S = 1000.0 # representative GDDR6X/GDDR7 class
  176. _UMA_BANDWIDTH_GB_S = 210.0 # measured on unified-memory NVIDIA
  177. _HOST_BANDWIDTH_GB_S = 80.0 # spilled weights stream over host DRAM
  178. # Below this predicted decode speed a model stops feeling pleasant for agentic use (roughly
  179. # reading speed with headroom for tool-call bursts). Distinct from the growth policy's 6 tok/s
  180. # compress floor, which marks unusable, not unpleasant.
  181. PLEASANT_FLOOR_TOK_S = 20.0
  182. # Shipped short-context reference rates, not benchmarks run during recommendation.
  183. # Windows N1X / b10964 CUDA: MTP2 measured 21.76–21.96 tok/s over four 512-token
  184. # prose probes; a four-slot smoke measured 21.89. At 32K input the reference was
  185. # 18.82 tok/s: this is a baseline estimate, not a context-independent guarantee.
  186. # Unmatched hardware, backend, quant or draft depth retains the bandwidth estimate.
  187. _MEASURED_DECODE_TOK_S = {
  188. ("win32", "cuda", "NVIDIA RTX Spark N1X",
  189. "qwen3.8-27b", "UD-Q4_K_M", 2): 21.9,
  190. }
  191. # Defaults the maker of a recognized product chose for it, keyed like the measured rates. The
  192. # derived rule still decides when the chosen entry is ineligible or cannot run resident, and every
  193. # other entry that fits stays available. The RTX Spark ships the 27B: Flash Next fits 128 GB, but
  194. # Windows backs GPU memory with commit outside the carve-out, so it loads only with the carve-out
  195. # lowered.
  196. _PRODUCT_DEFAULTS = {
  197. ("win32", "cuda", "NVIDIA RTX Spark N1X"): "qwen3.8-27b",
  198. }
  199. def _hardware_key(budget: HardwareBudget, backend: str) -> tuple[str, str, str]:
  200. """(platform, backend, reference GPU name) for the shipped tables; the name is empty when the
  201. GPU is not recognized."""
  202. # Drivers may append a parenthesized description to the stable device name.
  203. gpu_name = budget.gpu_name.partition(" (")[0]
  204. # Resolve PCI identity to the existing reference key; names only backfill missing IDs.
  205. if budget.gpu_pci_id is not None:
  206. gpu_name = "NVIDIA RTX Spark N1X" if is_nvidia_n1x_pci_id(budget.gpu_pci_id) else ""
  207. effective_backend = "cuda" if backend == "auto" and gpu_name else backend
  208. return (budget.platform, effective_backend, gpu_name)
  209. def predicted_decode_tok_s(entry: CatalogEntry, variant: QuantVariant, budget: HardwareBudget, *,
  210. spilled: bool = False, backend: str = "auto") -> float:
  211. """Shipped measured baseline where matched, otherwise the memory-bound estimate."""
  212. key = (*_hardware_key(budget, backend), entry.id, variant.quant, entry.mtp_draft_depth)
  213. if budget.uma and entry.mtp and not spilled and (measured := _MEASURED_DECODE_TOK_S.get(key)) is not None:
  214. return measured
  215. bandwidth = (_HOST_BANDWIDTH_GB_S if spilled
  216. else _UMA_BANDWIDTH_GB_S if budget.uma
  217. else _DISCRETE_BANDWIDTH_GB_S)
  218. bytes_per_token = max(1.0, variant.size_bytes * entry.decode_fraction)
  219. return bandwidth * 1e9 / bytes_per_token
  220. def recommended_entry(budget: HardwareBudget,
  221. entries: tuple[CatalogEntry, ...] | None = None, *, backend: str = "auto"
  222. ) -> tuple[CatalogEntry, str] | None:
  223. """The catalog's default pick for THIS machine, with its reason key.
  224. Callers pass pre-filtered entries when some are ineligible for reasons the catalog can't know
  225. (engine too old). Entries marked ``recommend: false`` are never picked. Reasons:
  226. product-default (the product's maker chose this entry and it runs resident);
  227. best-quality-resident (quality won among resident entries clearing the pleasant floor);
  228. speed-gated-quality (same, but the floor eliminated a HIGHER quality candidate).
  229. Returns None when no eligible entry runs resident at the pleasant floor; slower and spilled
  230. models remain available for explicit selection.
  231. """
  232. pool = tuple(e for e in (CATALOG if entries is None else entries) if e.recommend)
  233. fitting = [(e, c) for e in pool if (c := select_variant(e, budget)) is not None]
  234. if not fitting:
  235. return None
  236. def speed(t, spilled=False):
  237. return predicted_decode_tok_s(t[0], t[1].variant, budget, spilled=spilled, backend=backend)
  238. resident = [(e, c) for e, c in fitting if c.zero_spill]
  239. product_default = _PRODUCT_DEFAULTS.get(_hardware_key(budget, backend))
  240. for entry, _ in resident:
  241. if entry.id == product_default:
  242. return (entry, "product-default")
  243. pleasant = [t for t in resident if speed(t) >= PLEASANT_FLOOR_TOK_S]
  244. if pleasant:
  245. pick = max(pleasant, key=lambda t: (t[0].quality, -t[1].variant.size_bytes))[0]
  246. floor_gated = any(e.quality > pick.quality for e, _ in resident)
  247. return (pick, "speed-gated-quality" if floor_gated else "best-quality-resident")
  248. # A slow or spilled model may be usable, but it is not a recommendation. Keep it
  249. # discoverable through Browse so the user can opt in with the degradation visible.
  250. return None
  251. # ── catalog data: packaged JSON, refreshed from GitHub in memory ─
  252. #
  253. # catalog.json ships as package data and is loaded at import (no network on the import path). A
  254. # TTL-gated background refresh fetches the same file from the repo's main branch and swaps it in
  255. # MEMORY only: nothing on disk changes, so a git checkout never sees a dirty tracked file and the
  256. # packaged copy remains the offline truth. A reverted commit on main heals every install on its
  257. # next fetch, and day-0 entries reach users without an app release.
  258. _CATALOG_URL = ("https://raw.githubusercontent.com/NousResearch/hermes-agent"
  259. "/main/hermes_cli/local_runtime/catalog.json")
  260. _SCHEMA_VERSION = 3 # bump when an older app would read a newer catalog but launch its entries wrongly
  261. _REFRESH_TTL_S = 6 * 3600
  262. _refresh_lock = threading.Lock()
  263. # None = never attempted: monotonic() counts from boot, so a 0.0 sentinel read as "refreshed at
  264. # boot" and suppressed the first fetch for the first six awake hours after every reboot.
  265. _last_refresh_attempt: float | None = None
  266. def _asset_from(d: dict | None) -> AssetFile | None:
  267. if not d:
  268. return None
  269. return AssetFile(path=d["path"], size_bytes=int(d["size_bytes"]), local=d.get("local"),
  270. repo=d.get("repo"), revision=d.get("revision", "main"))
  271. # Scalar CatalogEntry fields parsed from JSON: key -> (coerce, default); None default = required.
  272. _SCALAR_FIELDS = {
  273. "n_ctx_train": (int, None), "full_layers": (int, None),
  274. "recurrent_layers": (int, None), "per_layer_f16": (int, None),
  275. "swa_layers": (int, 0), "swa_window": (int, 0),
  276. "moe": (bool, False), "mtp": (bool, False), "mtp_draft_depth": (int, 3),
  277. "n_vocab": (int, 0), "sampling": (dict, {}), "min_engine": (str, ""),
  278. "quality": (int, 0), "decode_fraction": (float, 1.0), "architecture": (str, ""),
  279. "recommend": (bool, True), "draft_spec_type": (str, "draft-dspark"), "draft_depth": (int, 3),
  280. "draft_min_engine": (str, ""),
  281. }
  282. def _load_catalog(doc: dict) -> tuple[CatalogEntry, ...]:
  283. """Parse a catalog document. Unknown fields are ignored (newer catalogs stay readable by older
  284. apps); a major schema bump is the signal that they wouldn't be, and the caller skips it."""
  285. if int(doc.get("schema_version", 0)) != _SCHEMA_VERSION:
  286. raise ValueError(f"catalog schema {doc.get('schema_version')!r} "
  287. f"(this build reads {_SCHEMA_VERSION})")
  288. entries = []
  289. for m in doc["models"]:
  290. variants = tuple(QuantVariant(quant=v["quant"], validated=bool(v.get("validated")),
  291. files=tuple(_asset_from(f) for f in v["files"]),
  292. lazy_bytes=int(v.get("lazy_bytes", 0)))
  293. for v in m["variants"])
  294. scalars = {k: coerce(m[k] if default is None else m.get(k, default))
  295. for k, (coerce, default) in _SCALAR_FIELDS.items()}
  296. entries.append(CatalogEntry(
  297. id=m["id"], display_name=m["display_name"],
  298. description=m["description"], repo=m["repo"], variants=variants,
  299. mmproj=_asset_from(m.get("mmproj")), draft=_asset_from(m.get("draft")),
  300. mtp_head=_asset_from(m.get("mtp_head")),
  301. **scalars))
  302. return tuple(entries)
  303. def _packaged_catalog() -> tuple[CatalogEntry, ...]:
  304. from importlib.resources import files
  305. raw = files("hermes_cli.local_runtime").joinpath("catalog.json").read_text(encoding="utf-8-sig")
  306. return _load_catalog(json.loads(raw))
  307. CATALOG: tuple[CatalogEntry, ...] = _packaged_catalog()
  308. def refresh_catalog(force: bool = False) -> bool:
  309. """Fetch the current catalog from the repo and swap it in memory. Best-effort: any failure
  310. (offline, GitHub down, unreadable schema) leaves the running catalog untouched and retries
  311. after the TTL. Returns True when a fetched document replaced the catalog."""
  312. global CATALOG, _last_refresh_attempt
  313. now = time.monotonic()
  314. with _refresh_lock:
  315. if not force and _last_refresh_attempt is not None and now - _last_refresh_attempt < _REFRESH_TTL_S:
  316. return False
  317. _last_refresh_attempt = now
  318. try:
  319. req = urllib.request.Request(_CATALOG_URL, headers={"User-Agent": "hermes-local-runtime"})
  320. with urllib.request.urlopen(req, timeout=10) as r:
  321. fetched = _load_catalog(json.load(r))
  322. except Exception as exc:
  323. logger.debug("catalog refresh skipped: %s", exc)
  324. return False
  325. if fetched != CATALOG:
  326. logger.info("catalog refreshed from repo (%d models)", len(fetched))
  327. CATALOG = fetched
  328. return True
  329. def refresh_catalog_soon() -> None:
  330. """TTL-gated background refresh; returns immediately. The current request serves the catalog
  331. it already has — the refresh lands for the next one."""
  332. last = _last_refresh_attempt
  333. if last is not None and time.monotonic() - last < _REFRESH_TTL_S:
  334. return
  335. threading.Thread(target=refresh_catalog, daemon=True, name="catalog-refresh").start()
  336. def catalog_by_id() -> dict[str, CatalogEntry]:
  337. return {entry.id: entry for entry in CATALOG}
  338. def find_entry_for_model(model_id: str) -> tuple[CatalogEntry, QuantVariant] | None:
  339. """Locate the entry + variant that owns a staged model id."""
  340. for entry in CATALOG:
  341. for variant in entry.variants:
  342. if variant.model_id == model_id:
  343. return entry, variant
  344. return None
  345. def entry_for_model(model_id: str) -> CatalogEntry | None:
  346. hit = find_entry_for_model(model_id)
  347. return hit[0] if hit is not None else None