presets.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429
  1. """Per-model preset generation (--models-preset INI) — the router-side carrier for context-policy
  2. launch decisions.
  3. """
  4. from __future__ import annotations
  5. import json
  6. import logging
  7. from dataclasses import dataclass, replace
  8. from pathlib import Path
  9. from hermes_cli.local_runtime.context_policy import (
  10. RUNTIME_OVERHEAD_BYTES, fit_to_free_memory, launch_args, plan_launch, ub_logits_bytes)
  11. from hermes_cli.local_runtime.estimator import (
  12. HardwareBudget, PhysicsRefusal, as_loaded, ctx_bytes, footprint_bytes, profile_from_gguf)
  13. from hermes_cli.local_runtime.gguf import model_id_from_stem, read_gguf_header
  14. logger = logging.getLogger(__name__)
  15. # args list -> INI keys. Flags the policy owns; everything else stays out of the preset.
  16. _FLAG_TO_KEY = {
  17. "-c": "ctx-size", "-b": "batch-size", "-ub": "ubatch-size",
  18. "-ctk": "cache-type-k", "-ctv": "cache-type-v", "-fa": "flash-attn",
  19. "-ot": "override-tensor", "--spec-type": "spec-type", "--spec-draft-n-max": "spec-draft-n-max",
  20. }
  21. @dataclass
  22. class PresetEntry:
  23. model_id: str
  24. window: int
  25. spilled: bool
  26. refusal: str | None = None
  27. keys: dict[str, str] | None = None
  28. def _args_to_keys(args: list[str]) -> dict[str, str]:
  29. keys: dict[str, str] = {}
  30. i = 0
  31. while i < len(args):
  32. key = _FLAG_TO_KEY.get(args[i])
  33. if key is None:
  34. i += 1
  35. continue
  36. keys[key] = args[i + 1]
  37. i += 2
  38. return keys
  39. def _asset_path(asset) -> Path | None:
  40. """On-disk path of a catalog companion asset, or None when it isn't downloaded."""
  41. from hermes_cli.local_runtime.bootstrap import assets_dir
  42. if asset is None:
  43. return None
  44. path = assets_dir() / asset.local_name
  45. return path if path.exists() else None
  46. @dataclass(frozen=True)
  47. class _Companions:
  48. """The files a catalog entry loads beside its weights, as far as they are on disk."""
  49. mmproj: Path | None = None
  50. mtp_head: Path | None = None
  51. nbytes: int = 0
  52. def _companions(entry) -> _Companions:
  53. if entry is None:
  54. return _Companions()
  55. mmproj, head = _asset_path(entry.mmproj), _asset_path(entry.mtp_head)
  56. nbytes = sum(a.size_bytes for a, p in ((entry.mmproj, mmproj), (entry.mtp_head, head)) if p)
  57. return _Companions(mmproj=mmproj, mtp_head=head, nbytes=nbytes)
  58. def _runs_mtp(entry, companions: _Companions) -> bool:
  59. """MTP spec decode runs for a catalog entry: built-in heads, or a shipped head on disk."""
  60. return entry is not None and (entry.mtp or companions.mtp_head is not None)
  61. def _draft_fits(path: Path, profile, budget: HardwareBudget, window: int, overhead: int) -> bool:
  62. """Optional draft never shrinks the advertised window or displaces its GPU buffers.
  63. The catalog does not know the draft's context layout. Admit it only after reading the file;
  64. draft KV defaults to f16, independently of the target's q8 cache.
  65. """
  66. try:
  67. draft = profile_from_gguf(read_gguf_header(path))
  68. draft_need = footprint_bytes(
  69. draft, window, flash_attention=False,
  70. overhead_bytes=RUNTIME_OVERHEAD_BYTES + ub_logits_bytes(draft.n_vocab, mtp_capable=False))
  71. except (ValueError, OSError) as exc:
  72. logger.warning("draft omitted %s: %s", path.name, exc)
  73. return False
  74. return (footprint_bytes(profile, window, overhead_bytes=overhead) + draft_need
  75. <= budget.usable_vram_bytes + budget.ram_available_bytes
  76. and ctx_bytes(profile, window) + overhead + draft_need <= budget.usable_vram_bytes)
  77. # The draft model's own context beside the target's: its KV and compute buffers. Without it the
  78. # price below still covered every measured server (b11671, -ub 2048, one slot) by 0.2 GiB
  79. # (Nemotron 3.5 Bolt) to 2.7 GiB (Qwen3.8 27B); this keeps the thinnest case clear.
  80. _DRAFT_CONTEXT_BYTES = 512 << 20
  81. def _engine_build(tag: str | None) -> int | None:
  82. try:
  83. return int(str(tag).removeprefix("b")) if tag else None
  84. except ValueError:
  85. return None
  86. def _dflash_extra_bytes(entry, profile) -> int:
  87. """Fixed bytes a one-slot DFlash draft adds beside its target: the draft's weights, a copy of
  88. the recurrent state per draft position, and the draft's own context."""
  89. copies = entry.draft_depth + 1
  90. return entry.draft.size_bytes + profile.recurrent_state_bytes * copies + _DRAFT_CONTEXT_BYTES
  91. def _runs_dflash(entry) -> Path | None:
  92. """The entry's DFlash draft on disk, or None."""
  93. if entry is None or entry.draft is None or entry.draft_spec_type != "draft-dflash":
  94. return None
  95. return _asset_path(entry.draft)
  96. def _dflash_plan(entry, profile, budget: HardwareBudget, live: HardwareBudget | None, *,
  97. fixed_overhead: int, requested_window: int | None, engine_tag: str | None,
  98. window_without: int, residents: int = 1):
  99. """The launch plan with the entry's DFlash draft in place of MTP, or None.
  100. The draft runs on one slot: each slot keeps a recurrent-state copy per draft position, and
  101. the router's default four slots hold enough copies to push weights off the GPU. It may
  102. shrink the window to half native, never below what the model gets without it when that is
  103. smaller; a requested (grown) window it can't hold keeps the model as it was. On one slot a
  104. side request takes the chat's slot and the prompt cache brings the chat back, so where free
  105. commit sizes that cache the draft keeps its smallest useful size.
  106. """
  107. if _runs_dflash(entry) is None:
  108. return None
  109. build = _engine_build(engine_tag)
  110. if build is None or build < (_engine_build(entry.draft_min_engine) or 0):
  111. return None
  112. lean = replace(profile, kv_scale=1.0)
  113. overhead = fixed_overhead + _dflash_extra_bytes(entry, lean)
  114. plan = plan_launch(lean, budget, fixed_overhead=overhead, requested_window=requested_window)
  115. if live is not None and live.commit_room_bytes is not None:
  116. from hermes_cli.local_runtime.hardware import usable_beside_prompt_cache
  117. live = replace(live, usable_vram_bytes=usable_beside_prompt_cache(live, residents))
  118. if live is not None:
  119. # fit_to_free_memory holds a too-busy card at the floor rather than move weights off it on a
  120. # live reading. The draft never rides on that hold: it runs only where it fits now.
  121. now = plan_launch(lean, live, fixed_overhead=overhead)
  122. if isinstance(now.decision, PhysicsRefusal) or now.decision.spilled:
  123. return None
  124. plan = fit_to_free_memory(plan, lean, live, mtp_capable=False, fixed_overhead=overhead)
  125. decision = plan.decision
  126. if isinstance(decision, PhysicsRefusal) or decision.spilled:
  127. return None
  128. native = profile.n_ctx_train or decision.window
  129. if requested_window and decision.window < min(requested_window, native):
  130. return None
  131. if decision.window < min(window_without, native // 2):
  132. return None
  133. return plan
  134. def preset_for_model(gguf: Path, budget: HardwareBudget,
  135. mtp_capable: set[str], *, requested_window: int | None = None,
  136. live: HardwareBudget | None = None, residents: int = 1,
  137. engine_tag: str | None = None) -> PresetEntry | None:
  138. """The launch decision for one staged model, or None when its header is unreadable.
  139. ``budget`` is the card's capacity; ``live`` (from ``hardware.launch_budget``) narrows the
  140. window to what fits beside other programs now. ``residents`` is how many models the router may
  141. hold at once; on Windows unified memory their prompt caches share the commit they leave.
  142. ``engine_tag`` is the installed llama.cpp release; without it no draft that needs a minimum
  143. release runs.
  144. """
  145. from hermes_cli.local_runtime.catalog import entry_for_model
  146. from hermes_cli.local_runtime.growth import load_window_overrides
  147. model_id = model_id_from_stem(gguf.stem)
  148. try:
  149. header = read_gguf_header(gguf)
  150. profile = as_loaded(profile_from_gguf(header), budget)
  151. except (ValueError, OSError) as exc:
  152. logger.warning("preset skip %s: %s", gguf.name, exc)
  153. return None
  154. entry = entry_for_model(model_id)
  155. companions = _companions(entry)
  156. is_mtp = _runs_mtp(entry, companions) if entry is not None else model_id in mtp_capable
  157. fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
  158. requested = load_window_overrides().get(model_id) if requested_window is None else requested_window
  159. plan = plan_launch(profile, budget, mtp_capable=is_mtp, fixed_overhead=fixed_overhead,
  160. requested_window=requested)
  161. if live is not None:
  162. plan = fit_to_free_memory(plan, profile, live, mtp_capable=is_mtp,
  163. fixed_overhead=fixed_overhead)
  164. decision = plan.decision
  165. if isinstance(decision, PhysicsRefusal):
  166. return PresetEntry(model_id=model_id, window=0, spilled=False, refusal=decision.message)
  167. dflash = None if decision.spilled else _dflash_plan(
  168. entry, profile, budget, live, fixed_overhead=fixed_overhead, requested_window=requested,
  169. engine_tag=engine_tag, window_without=decision.window, residents=residents)
  170. if dflash is not None:
  171. plan, decision = dflash, dflash.decision
  172. # Router discovery is preset-only: refused files must never autoload with stock fit.
  173. keys = _args_to_keys(launch_args(
  174. profile, decision, mtp_capable=is_mtp and dflash is None, uma=budget.uma,
  175. mtp_prefill=plan.mtp_prefill,
  176. mtp_draft_depth=entry.mtp_draft_depth if entry is not None else 3))
  177. keys["model"] = str(gguf)
  178. from hermes_cli.local_runtime.hardware import prompt_cache_bytes
  179. cache = prompt_cache_bytes(live, footprint_bytes(plan.profile, decision.window,
  180. overhead_bytes=plan.overhead_bytes), residents)
  181. if cache is not None:
  182. keys["cache-ram"] = str(cache >> 20) # MiB; 0 turns the cache off
  183. if profile.lazy_bytes:
  184. # The price above leaves these tensors on disk; llama.cpp's own default loads them on
  185. # integrated GPUs.
  186. keys["lazy-mode"] = "on"
  187. if entry is not None and is_mtp:
  188. # Integrated-MTP targets sample on the backend, and so does the draft (pairing validated
  189. # against the vendor's published llama.cpp recipes).
  190. keys["backend-sampling"] = "on"
  191. keys["spec-draft-backend-sampling"] = "on"
  192. # Sampling deference ladder, under the policy keys (policy wins on clash): the GGUF's own
  193. # general.sampling.* metadata is the publisher's recommendation and covers models the catalog
  194. # has never heard of; catalog sampling applies only where the file is silent; a model
  195. # carrying neither runs llama.cpp defaults.
  196. for k, v in header.sampling_defaults.items():
  197. keys.setdefault(k, v)
  198. if entry is not None:
  199. for k, v in (entry.sampling or {}).items():
  200. keys.setdefault(k, v)
  201. if companions.mmproj is not None:
  202. keys["mmproj"] = str(companions.mmproj)
  203. if companions.mtp_head is not None:
  204. # The model carries no MTP layers: draft-mtp drafts from the shipped head instead.
  205. keys["model-draft"] = str(companions.mtp_head)
  206. if dflash is not None:
  207. keys["model-draft"] = str(_runs_dflash(entry))
  208. keys["spec-type"] = "draft-dflash"
  209. keys["spec-draft-n-max"] = str(entry.draft_depth)
  210. keys["parallel"] = "1"
  211. draft_path = (_asset_path(entry.draft) if decision.spilled
  212. and entry.draft_spec_type == "draft-dspark" else None)
  213. if draft_path is not None and _draft_fits(draft_path, plan.profile, budget, decision.window,
  214. plan.overhead_bytes):
  215. keys["model-draft"] = str(draft_path)
  216. keys["spec-type"] = "draft-dspark"
  217. # Unsloth's measured cliff: acceptance 83% at 2-3 drafts, collapses at 4.
  218. keys["spec-draft-n-max"] = "3"
  219. return PresetEntry(model_id=model_id, window=decision.window,
  220. spilled=decision.spilled, keys=keys)
  221. def resident_footprint(gguf: Path, budget: HardwareBudget, window: int, *, drafting: bool = False) -> int | None:
  222. """Estimated bytes one staged model holds while loaded at ``window``, or None when unreadable.
  223. ``drafting`` prices the one-slot DFlash launch its preset runs instead of MTP."""
  224. from hermes_cli.local_runtime.catalog import entry_for_model
  225. model_id = model_id_from_stem(gguf.stem)
  226. try:
  227. profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
  228. except (ValueError, OSError) as exc:
  229. logger.debug("footprint skip %s: %s", gguf.name, exc)
  230. return None
  231. entry = entry_for_model(model_id)
  232. companions = _companions(entry)
  233. fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
  234. if drafting and _runs_dflash(entry) is not None:
  235. lean = replace(profile, kv_scale=1.0)
  236. plan = plan_launch(lean, budget, fixed_overhead=fixed_overhead + _dflash_extra_bytes(entry, lean),
  237. requested_window=window)
  238. else:
  239. plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),
  240. fixed_overhead=fixed_overhead, requested_window=window)
  241. return footprint_bytes(plan.profile, window, overhead_bytes=plan.overhead_bytes)
  242. def _launch_footprint(gguf: Path, budget: HardwareBudget, engine_tag: str | None = None) -> int | None:
  243. """Estimated resident bytes for one staged model at the window this policy grants it, or None
  244. when it cannot be priced: an unreadable header, or a model the physics check refuses outright
  245. (it never loads, so it must not shrink the residency cap)."""
  246. from hermes_cli.local_runtime.catalog import entry_for_model
  247. from hermes_cli.local_runtime.growth import load_window_overrides
  248. model_id = model_id_from_stem(gguf.stem)
  249. try:
  250. profile = as_loaded(profile_from_gguf(read_gguf_header(gguf)), budget)
  251. except (ValueError, OSError) as exc:
  252. logger.debug("footprint skip %s: %s", gguf.name, exc)
  253. return None
  254. entry = entry_for_model(model_id)
  255. companions = _companions(entry)
  256. fixed_overhead = RUNTIME_OVERHEAD_BYTES + companions.nbytes
  257. requested = load_window_overrides().get(model_id)
  258. plan = plan_launch(profile, budget, mtp_capable=_runs_mtp(entry, companions),
  259. fixed_overhead=fixed_overhead, requested_window=requested)
  260. if isinstance(plan.decision, PhysicsRefusal):
  261. return None
  262. # Priced whole even when the plan spills: a spilled model still holds part of its weights on
  263. # the device, and over-counting errs toward the side that cannot thrash.
  264. price = footprint_bytes(plan.profile, plan.decision.window, overhead_bytes=plan.overhead_bytes)
  265. # Where the engine loads the draft, the preset runs it in place of MTP: the model holds the
  266. # larger of the two launches, never both.
  267. dflash = None if plan.decision.spilled else _dflash_plan(
  268. entry, profile, budget, None, fixed_overhead=fixed_overhead, requested_window=requested,
  269. engine_tag=engine_tag, window_without=plan.decision.window)
  270. if dflash is not None:
  271. price = max(price, footprint_bytes(dflash.profile, dflash.decision.window,
  272. overhead_bytes=dflash.overhead_bytes))
  273. return price
  274. def admitted_residency_count(models_dir: Path, budget: HardwareBudget, configured: int,
  275. live: HardwareBudget | None = None, engine_tag: str | None = None) -> int:
  276. """How many models the card may hold resident at once: priced against the budget, not a count.
  277. Residency used to be bounded by a count alone, so a second model was admitted against an
  278. already-full card. On Windows/WDDM that over-commit is not refused — the allocation is paged
  279. to host memory, and that child decodes at a third of its speed for the rest of its life: no
  280. error, no UI hint, and ejecting the incumbent afterwards does not repair it (only a clean
  281. reload does). Capping the count instead has llama.cpp evict its LRU *before* the incoming
  282. child allocates, which is the only placement that fits.
  283. The cap rises above one only while the LARGEST staged model still fits TWICE — any pair of
  284. staged models then fits by construction. ``configured`` stays a ceiling (a user's smaller
  285. number is honoured), and an unpriceable input (no usable device memory, no readable model)
  286. keeps today's behaviour. On Windows unified memory ``live`` (free commit) bounds it too: the
  287. models the router holds together must fit in commit, not just in the pool.
  288. """
  289. from hermes_cli.local_runtime.bootstrap import staged_in
  290. if configured <= 1 or budget.usable_vram_bytes <= 0:
  291. return configured
  292. largest = 0
  293. for gguf in staged_in(models_dir):
  294. need = _launch_footprint(gguf, budget, engine_tag)
  295. if need:
  296. largest = max(largest, need)
  297. if largest <= 0:
  298. return configured
  299. usable = budget.usable_vram_bytes
  300. if live is not None and live.commit_room_bytes is not None:
  301. usable = min(usable, live.usable_vram_bytes)
  302. return max(1, min(configured, usable // largest))
  303. def plan_presets(models_dir: Path, budget: HardwareBudget, mtp_capable: set[str] | None = None,
  304. *, live: HardwareBudget | None = None, residents: int = 1,
  305. engine_tag: str | None = None) -> list[PresetEntry]:
  306. """The launch decision for every staged model; unreadable headers are skipped."""
  307. from hermes_cli.local_runtime.bootstrap import staged_in
  308. entries = (preset_for_model(gguf, budget, mtp_capable or set(), live=live, residents=residents,
  309. engine_tag=engine_tag)
  310. for gguf in staged_in(models_dir))
  311. return [entry for entry in entries if entry is not None]
  312. def generate_presets(models_dir: Path, budget: HardwareBudget, preset_path: Path,
  313. mtp_capable: set[str] | None = None, *,
  314. live: HardwareBudget | None = None, residents: int = 1,
  315. engine_tag: str | None = None) -> list[PresetEntry]:
  316. """Plan every staged model and write one INI. Refused models get no section (the picker
  317. surfaces the refusal from the returned entries)."""
  318. entries = plan_presets(models_dir, budget, mtp_capable, live=live, residents=residents,
  319. engine_tag=engine_tag)
  320. write_presets(entries, preset_path)
  321. return entries
  322. def write_presets(entries: list[PresetEntry], preset_path: Path) -> None:
  323. from utils import atomic_write_text
  324. atomic_write_text(preset_path, render_presets(entries), tmp_prefix=f".{preset_path.name}_", mode=0o600)
  325. logger.info("wrote %d preset sections to %s", sum(e.keys is not None for e in entries), preset_path)
  326. def render_presets(entries: list[PresetEntry]) -> str:
  327. sections: list[str] = []
  328. for entry in entries:
  329. # INI comments preserve non-flag facts atomically with the launch policy.
  330. sections.append("# hermes-decision: " + json.dumps({
  331. "model_id": entry.model_id, "window": entry.window,
  332. "spilled": entry.spilled, "refusal": entry.refusal}) + "\n")
  333. if entry.keys is not None:
  334. body = "\n".join(f"{k} = {v}" for k, v in entry.keys.items())
  335. sections.append(f"[{entry.model_id}]\n{body}\n")
  336. return "\n".join(sections)
  337. def read_preset_decisions(preset_path: Path | None = None) -> dict[str, PresetEntry]:
  338. """The launch decisions the running server was actually given, read back from the preset INI
  339. (the INI is the record — it's what spawned the children). Missing/unparseable -> {}."""
  340. import configparser
  341. if preset_path is None:
  342. from hermes_cli.local_runtime.binaries import runtimes_root
  343. preset_path = runtimes_root() / "presets.ini"
  344. out: dict[str, PresetEntry] = {}
  345. try:
  346. parser = configparser.ConfigParser(interpolation=None)
  347. text = preset_path.read_text(encoding="utf-8-sig")
  348. parser.read_string(text)
  349. recorded = {}
  350. for line in text.splitlines():
  351. if line.startswith("# hermes-decision: "):
  352. fact = json.loads(line.removeprefix("# hermes-decision: "))
  353. recorded[fact["model_id"]] = fact
  354. if fact.get("refusal"):
  355. out[fact["model_id"]] = PresetEntry(**fact)
  356. for section in parser.sections():
  357. out[section] = PresetEntry(
  358. model_id=section, window=parser.getint(section, "ctx-size", fallback=0),
  359. spilled=recorded.get(section, {}).get("spilled", parser.has_option(section, "override-tensor")),
  360. keys=dict(parser[section]))
  361. except Exception as exc:
  362. logger.debug("preset read-back failed: %s", exc)
  363. return out