bootstrap.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533
  1. """Bootstrap for the managed runtime: config -> installed binaries -> running supervised server.
  2. One public call, ``ensure_local_runtime(config)``, safe at any session start: disabled or
  3. already-running -> no-op; enabled -> serve the installed build under a supervisor. Kept
  4. import-light: callers gate on config before importing so disabled sessions never pay the import.
  5. """
  6. from __future__ import annotations
  7. from contextlib import contextmanager, suppress
  8. import logging
  9. import os
  10. import time
  11. from pathlib import Path
  12. from hermes_cli.local_runtime.binaries import runtimes_root
  13. from hermes_cli.local_runtime.gguf import SPLIT_PART_RE, model_id_from_stem
  14. logger = logging.getLogger(__name__)
  15. _SUPERVISOR = None # process-wide singleton; one router per Hermes process
  16. # The engine that singleton runs (or is booting). Budgets price ITS devices: the server is shared by
  17. # every profile this process hosts, so another profile's ``local_runtime.backend`` must not resize it.
  18. _SERVING_ENGINE = None
  19. _EXIT_HOOKED = False
  20. def _stop_at_exit() -> None:
  21. """The process that boots the server stops it on a clean exit, whatever surface it is (the
  22. desktop backend also stops it in its shutdown handler; the second call is a no-op). Processes
  23. that only adopted another's server never stop it. A hard kill still leaves an orphan, which the
  24. next boot adopts or replaces."""
  25. global _EXIT_HOOKED
  26. if not _EXIT_HOOKED:
  27. import atexit
  28. atexit.register(shutdown_local_runtime)
  29. _EXIT_HOOKED = True
  30. def _detect_gpu_vendor() -> str | None:
  31. """Best-effort GPU vendor for backend selection.
  32. Preserve NVIDIA's detailed name when nvidia-smi is available, then use the
  33. platform host facts so Intel and AMD do not silently demote ``auto`` to CPU.
  34. """
  35. from hermes_cli.local_runtime.hardware import _cached_nvidia_gpu_query
  36. from hermes_platform.host.facts import gpu_class
  37. query = _cached_nvidia_gpu_query()
  38. if query is not None and query.get("gpu_name"):
  39. return "nvidia " + query["gpu_name"]
  40. detected = gpu_class()
  41. return detected if detected in {"nvidia", "amd", "intel"} else None
  42. def models_dir() -> Path:
  43. """Machine-scoped, deliberately NOT profile-scoped: a 20 GB GGUF is a machine asset, and every
  44. profile shares the one managed server that serves it (same rule as runtimes_root())."""
  45. from hermes_constants import get_default_hermes_root
  46. return get_default_hermes_root() / "models"
  47. def assets_dir() -> Path:
  48. """Non-model companion files (mmproj projectors, spec-decode drafts). A subdirectory so the
  49. router's model listing — and staged_models() — never mistakes an asset for a servable model."""
  50. return models_dir() / "assets"
  51. def staged_in(models_dir: Path, *, require_complete: bool = True) -> list[Path]:
  52. """Servable GGUFs in a directory: single files, plus split GGUFs once by their first part.
  53. With ``require_complete`` a split counts only when EVERY part is on disk — a mid-download split
  54. is not servable and must not surface anywhere as a model."""
  55. files = sorted(models_dir.glob("*.gguf"))
  56. names = {p.name for p in files}
  57. out = []
  58. for p in files:
  59. m = SPLIT_PART_RE.search(p.name)
  60. if m is None:
  61. out.append(p)
  62. continue
  63. if m.group(1) != "00001":
  64. continue
  65. stem, total = p.name[: m.start()], int(m.group(2))
  66. if not require_complete or all(f"{stem}-{i:05d}-of-{m.group(2)}.gguf" in names
  67. for i in range(2, total + 1)):
  68. out.append(p)
  69. return out
  70. def adopt_legacy_models() -> list[Path]:
  71. """Move GGUFs left in the old per-profile ``<profile home>/models`` layout (and its assets/)
  72. into the machine-scoped dirs, so everything downstream keeps reading one directory.
  73. ``os.rename`` only: within one filesystem it is instant even for a 20 GB model, while
  74. ``shutil.move`` silently degrades to a copy across devices. A cross-device profile dir is left
  75. in place with a warning rather than copying tens of GB at session start. A name that already
  76. exists in the destination is left alone (check-then-rename: the only window is two processes
  77. adopting two profiles' same-named file at once, and a same name is the same catalog variant).
  78. Two processes racing on one file are harmless: the loser's rename finds the source gone and
  79. skips it. Returns the new paths of the moved files."""
  80. from hermes_constants import get_default_hermes_root, named_profile_has_identity
  81. profiles_root = get_default_hermes_root() / "profiles"
  82. if not profiles_root.is_dir():
  83. return []
  84. moved: list[Path] = []
  85. for home in sorted(profiles_root.iterdir()):
  86. old = home / "models"
  87. if home.name.startswith(".") or not old.is_dir() or not named_profile_has_identity(home):
  88. continue
  89. for src_dir, dest_dir in ((old, models_dir()), (old / "assets", assets_dir())):
  90. for src in sorted(src_dir.glob("*.gguf")):
  91. dest = dest_dir / src.name
  92. if dest.exists():
  93. logger.warning("legacy model %s not moved: %s already exists", src, dest)
  94. continue
  95. try:
  96. dest_dir.mkdir(parents=True, exist_ok=True)
  97. os.rename(src, dest)
  98. except FileNotFoundError:
  99. continue
  100. except OSError as exc:
  101. logger.warning("legacy model %s not moved to %s: %s", src, dest_dir, exc)
  102. continue
  103. moved.append(dest)
  104. for emptied in (old / "assets", old):
  105. with suppress(OSError):
  106. emptied.rmdir()
  107. if moved:
  108. logger.info("moved %d legacy model file(s) into %s", len(moved), models_dir())
  109. return moved
  110. def staged_models() -> list[Path]:
  111. """Servable staged models (continuation parts, incomplete splits and assets/ never count)."""
  112. return staged_in(models_dir())
  113. def staged_model_ids() -> list[str]:
  114. return [model_id_from_stem(p.stem) for p in staged_models()]
  115. def _presets_stale() -> bool:
  116. """True when a staged model has no section in the preset INI — it would autoload with stock
  117. fit instead of a policy decision."""
  118. with suppress(Exception):
  119. from hermes_cli.local_runtime.presets import read_preset_decisions
  120. known = read_preset_decisions()
  121. return any(mid not in known or (not known[mid].refusal and not (known[mid].keys or {}).get("model"))
  122. for mid in staged_model_ids())
  123. return False
  124. def _stop_state_server() -> None:
  125. """Stop the server the state file points at before this process boots a replacement, but only
  126. an orphan (recorded owner gone): a live owner's watchdog would respawn its router, so that one
  127. is left running and the replacement boots beside it, as it always has. The process comes from
  128. the identity-guarded reader, never a bare PID — the endpoint dict callers hold has none."""
  129. from hermes_cli.local_runtime.recovery import (
  130. _owner_is_dead,
  131. is_modern,
  132. legacy_recorded_process,
  133. read_state,
  134. recorded_process,
  135. )
  136. from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
  137. state = read_state()
  138. if is_modern(state):
  139. if not _owner_is_dead(state):
  140. logger.info("llama-server pid=%s belongs to a live Hermes process; leaving it", state.get("pid"))
  141. return
  142. proc = recorded_process(state)
  143. else:
  144. proc = legacy_recorded_process(state)
  145. if proc is None:
  146. return
  147. try:
  148. # Waits for the tree to exit, so the port and the GPU are free for the replacement.
  149. LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
  150. except Exception as exc:
  151. logger.warning("could not stop the incumbent llama-server (pid=%s): %s", proc.pid, exc)
  152. def refresh_local_runtime() -> bool:
  153. """Restart the managed server so it rescans the models directory. The router's model list is
  154. SPAWN-ONLY: a GGUF added after start is invisible to GET /models and 400s on completion, so
  155. anything that changes the staged set while the server runs must bounce it."""
  156. global _SUPERVISOR
  157. try:
  158. from hermes_cli.config import load_config
  159. if _SUPERVISOR is None:
  160. from hermes_cli.local_runtime.endpoint import _state_endpoint
  161. state = _state_endpoint()
  162. if state is None:
  163. return False
  164. _stop_state_server()
  165. else:
  166. shutdown_local_runtime()
  167. return ensure_local_runtime(load_config(), force=True) is not None
  168. except Exception as exc:
  169. logger.warning("local runtime refresh failed: %s", exc)
  170. return False
  171. def _admitted_models_max(mdir: Path, configured: int, engine_tag: str | None = None) -> int:
  172. """Residency cap to hand the router: derived from the hardware budget, ``models_max`` as a ceiling.
  173. A cap of "four" on a card that holds one model is how a second child ends up paged (WDDM) and
  174. silently slow — llama.cpp evicts its LRU before an incoming load only when the cap says the
  175. card is full. A probe miss or an unpriceable model keeps the configured count: this must never
  176. block a boot. On Windows unified memory, free commit bounds the count too.
  177. """
  178. try:
  179. from hermes_cli.local_runtime.hardware import probe_budget
  180. from hermes_cli.local_runtime.presets import admitted_residency_count
  181. capacity = probe_budget(planning=True)
  182. live = _launch_budget(capacity) if capacity.uma else None
  183. cap = admitted_residency_count(mdir, capacity, configured, live=live, engine_tag=engine_tag)
  184. except Exception as exc:
  185. logger.warning("residency cap probe failed (%s); using models_max=%s", exc, configured)
  186. return configured
  187. if cap != configured:
  188. logger.info("residency cap: %s resident model(s) on this card (models_max=%s)",
  189. cap, configured)
  190. return cap
  191. def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1, *,
  192. engine_tag: str | None = None) -> Path | None:
  193. """Write the launch-policy INI for every staged model; returns the path to hand the router.
  194. Priced against CAPACITY, not live free VRAM: this runs while the outgoing server instance may
  195. still hold the card (restart, refresh after a download), and its memory is freed before the new
  196. instance loads anything. Pricing against live-free once pinned a fitting model's weights to CPU.
  197. The window is then narrowed to what fits beside other programs' GPU memory
  198. (``hardware.launch_budget``), which never moves weights to the CPU.
  199. Degradation ladder on failure: a STALE policy still beats no policy — stock fit (f16 KV at max
  200. context, no placement) is the silent-busy-wait failure on Windows. Keep serving with the
  201. previous INI when one exists; only a first boot with no INI at all falls to stock fit.
  202. ``residents`` is the router's model count, which splits the prompt-cache room on Windows
  203. unified memory.
  204. """
  205. from hermes_cli.local_runtime.hardware import probe_budget
  206. from hermes_cli.local_runtime.presets import generate_presets
  207. try:
  208. capacity = probe_budget(planning=True)
  209. for entry in generate_presets(mdir, capacity, preset_path, live=_launch_budget(capacity),
  210. residents=residents, engine_tag=engine_tag):
  211. if entry.refusal:
  212. logger.warning("model refused by physics check: %s", entry.refusal)
  213. return preset_path
  214. except Exception as exc:
  215. if preset_path.exists():
  216. logger.error("preset generation failed (%s); serving with the "
  217. "PREVIOUS launch policies — models staged since "
  218. "the last successful generation run unpoliced "
  219. "until this is fixed", exc)
  220. return preset_path
  221. logger.error("preset generation failed (%s) and no previous "
  222. "policy file exists; router runs stock fit", exc)
  223. return None
  224. def _launch_budget(capacity, own_bytes: int = 0):
  225. """``hardware.launch_budget``, or None when the probe fails: a boot never waits on it."""
  226. from hermes_cli.local_runtime.hardware import launch_budget
  227. try:
  228. return launch_budget(capacity, own_bytes=own_bytes)
  229. except Exception as exc:
  230. logger.warning("free GPU memory probe failed (%s); launch windows use capacity", exc)
  231. return None
  232. # Router statuses that hold GPU memory. The preset file changes only while every model is out of
  233. # them: the router unloads a loaded model whose launch flags change on reload.
  234. _HOLDS_MEMORY = frozenset({"loaded", "loading", "sleeping", "ready"})
  235. def refit_idle_presets(sup) -> bool:
  236. """Re-plan launch windows against free GPU memory while no model is loaded.
  237. Boot plans every window once, but models load later (on demand, after an idle unload), and by
  238. then other programs may hold more or less of the card. Returns True when the router was given
  239. new presets. A request that starts a load between the last status check and the reload is
  240. unloaded by the router and fails once; that gap is a few milliseconds, and only when a window
  241. changes.
  242. """
  243. from hermes_cli.local_runtime.hardware import probe_budget
  244. from hermes_cli.local_runtime.presets import plan_presets, render_presets
  245. path = sup.preset_path
  246. if path is None or not path.exists() or _any_holds_memory(sup):
  247. return False
  248. capacity = probe_budget(planning=True)
  249. live = _launch_budget(capacity)
  250. if live is None:
  251. return False
  252. last = sup._refit_usable
  253. if last is not None and abs(live.usable_vram_bytes - last) < _REFIT_STEP_BYTES:
  254. return False
  255. text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max,
  256. engine_tag=_SERVING_ENGINE.tag if _SERVING_ENGINE is not None else None))
  257. # Held across the write and the reload so a restart can't spawn between them; everything
  258. # slow ran above.
  259. with sup._lifecycle_lock:
  260. if sup.proc is None or sup.proc.poll() is not None or _any_holds_memory(sup):
  261. return False
  262. sup._refit_usable = live.usable_vram_bytes
  263. if text == path.read_text(encoding="utf-8-sig"):
  264. return False
  265. from utils import atomic_write_text
  266. atomic_write_text(path, text, tmp_prefix=f".{path.name}_", mode=0o600)
  267. sup.reload_presets()
  268. logger.info("launch windows re-planned for %.1f GiB of free GPU memory",
  269. live.usable_vram_bytes / (1 << 30))
  270. return True
  271. # Smaller changes in free memory don't move a window by a ladder rung; skipping them keeps the
  272. # idle loop from re-reading every model header each pass.
  273. _REFIT_STEP_BYTES = 256 << 20
  274. def _any_holds_memory(sup) -> bool:
  275. return any(status in _HOLDS_MEMORY for status in sup.models(timeout_s=5).values())
  276. def _try_lock_boot_fd(fd: int) -> bool:
  277. """Non-blocking exclusive attempt; portable across fcntl/msvcrt."""
  278. if os.name == "nt":
  279. import msvcrt
  280. if os.fstat(fd).st_size == 0:
  281. os.write(fd, b"\0")
  282. os.lseek(fd, 0, os.SEEK_SET)
  283. try:
  284. msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
  285. return True
  286. except OSError:
  287. return False
  288. else:
  289. import fcntl
  290. try:
  291. fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
  292. return True
  293. except OSError:
  294. return False
  295. def _unlock_boot_fd(fd: int) -> None:
  296. if os.name == "nt":
  297. import msvcrt
  298. os.lseek(fd, 0, os.SEEK_SET)
  299. msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
  300. else:
  301. import fcntl
  302. fcntl.flock(fd, fcntl.LOCK_UN)
  303. @contextmanager
  304. def _cross_process_boot_lock(timeout_s: float = 130.0):
  305. """Serialize the state-check-then-spawn sequence across every Hermes process on this
  306. machine — the ``_SUPERVISOR`` singleton above only rules out a race within ONE process.
  307. Two profiles booting in the same second each see no ``server.json`` yet and each spawn a
  308. router on the stable port (#116682); an OS-held lock makes the second caller wait for the
  309. first to publish its state file, so it re-checks and adopts instead of spawning a duplicate.
  310. Bounded, not indefinite: never hang session start dead if the lock is somehow stuck, and
  311. never raise into session start: an unwritable runtimes dir (or a foreign-owned lock file)
  312. proceeds unlocked with a warning, like the contention timeout."""
  313. path = runtimes_root() / "boot.lock"
  314. try:
  315. path.parent.mkdir(parents=True, exist_ok=True)
  316. fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
  317. except OSError as exc:
  318. logger.warning("boot lock unavailable (%s); proceeding without it", exc)
  319. yield
  320. return
  321. try:
  322. deadline = time.monotonic() + timeout_s
  323. while not _try_lock_boot_fd(fd):
  324. if time.monotonic() >= deadline:
  325. logger.warning("boot lock contended past %.0fs; proceeding without it", timeout_s)
  326. break
  327. time.sleep(0.2)
  328. try:
  329. yield
  330. finally:
  331. with suppress(OSError):
  332. _unlock_boot_fd(fd)
  333. finally:
  334. os.close(fd)
  335. def ensure_local_runtime(config: dict, force: bool = False) -> object | None:
  336. """Idempotent boot of the managed runtime. Returns the supervisor (or None when
  337. disabled/unavailable). Never raises into a session start — failures log and return None; chat
  338. falls back to configured providers."""
  339. global _SUPERVISOR, _SERVING_ENGINE
  340. section = (config or {}).get("local_runtime") or {}
  341. if not force and not section.get("enabled"):
  342. return None
  343. if _SUPERVISOR is not None:
  344. return _SUPERVISOR
  345. try:
  346. adopt_legacy_models()
  347. except OSError as exc: # an unreadable profiles dir must not block serving what's staged
  348. logger.warning("legacy model adoption failed: %s", exc)
  349. # Residency: no staged models means nothing to serve — don't boot an empty server (delete
  350. # your last model and boots stop). force boots as ever.
  351. if not force and not staged_models():
  352. logger.info("local runtime enabled but no models staged; not booting")
  353. return None
  354. # Another Hermes process may already be supervising — reuse via state, but ONLY while its
  355. # launch policy still covers every staged model. A server whose preset file predates a
  356. # download serves the new model with no policy at all (--models-autoload + stock fit). A stale
  357. # incumbent gets stopped and replaced by a fresh boot with regenerated presets; sessions ride
  358. # through like any other supervised restart (stable port + persisted key).
  359. #
  360. # The state check and the spawn below run under a cross-process lock: two backends racing
  361. # to boot (#116682) must not both find no state file and both spawn a router on the stable
  362. # port — the loser waits here, then re-checks state and adopts the winner's server instead.
  363. from hermes_cli.local_runtime.endpoint import _state_endpoint
  364. with _cross_process_boot_lock():
  365. state = _state_endpoint()
  366. if state is not None:
  367. if not _presets_stale():
  368. logger.info("managed llama-server already running (another process)")
  369. return None
  370. logger.info("running server's presets predate the staged models; "
  371. "replacing it so every model launches with a policy")
  372. _stop_state_server()
  373. try:
  374. from hermes_cli.local_runtime.binaries import installed_engine
  375. from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
  376. engine = installed_engine(section.get("backend", "auto"))
  377. if engine is None:
  378. logger.info("local runtime enabled but no PM engine installed; use the Local Models pane")
  379. return None
  380. _SERVING_ENGINE = engine
  381. mdir = models_dir()
  382. mdir.mkdir(parents=True, exist_ok=True)
  383. models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)), engine.tag)
  384. preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,
  385. engine_tag=engine.tag)
  386. sup = LlamaServerSupervisor(engine.binary, mdir, preset_path=preset_path,
  387. models_max=models_max,
  388. port=int(section.get("port", 0)) or None)
  389. try:
  390. sup.start()
  391. except Exception:
  392. # start() can fail after the router process exists (health timeout): leaving it
  393. # running unsupervised strands its VRAM behind a port nothing will clean up.
  394. with suppress(Exception):
  395. sup.stop()
  396. raise
  397. _SUPERVISOR = sup
  398. _stop_at_exit()
  399. logger.info("managed llama-server up at %s (backend=%s tag=%s)", sup.base_url, engine.backend, engine.tag)
  400. _start_idle_sweeper(sup)
  401. return sup
  402. except Exception as exc:
  403. _SERVING_ENGINE = None
  404. logger.warning("managed local runtime unavailable: %s", exc)
  405. return None
  406. def shutdown_local_runtime() -> None:
  407. global _SUPERVISOR, _SERVING_ENGINE
  408. if _SUPERVISOR is not None:
  409. _SUPERVISOR.stop()
  410. _SUPERVISOR = None
  411. _SERVING_ENGINE = None
  412. def get_supervisor():
  413. """The process-local supervisor, or None (a server may still run under another process —
  414. check the state file)."""
  415. return _SUPERVISOR
  416. def serving_engine():
  417. """The engine the process-local server runs or is booting, or None when there is none."""
  418. return _SERVING_ENGINE
  419. _SWEEP_EVERY_S = 120
  420. _REFIT_EVERY_S = 30
  421. def _start_idle_sweeper(sup) -> None:
  422. """Idle-residency loop: every couple of minutes, unload models idle past the supervisor's
  423. threshold; every 30 s with nothing loaded, re-plan launch windows against free GPU memory.
  424. Daemon thread tied to the supervisor's lifetime — exits when the server stops."""
  425. import threading
  426. def _loop():
  427. last_sweep = time.monotonic()
  428. while sup.proc is not None and sup.proc.poll() is None:
  429. time.sleep(_REFIT_EVERY_S)
  430. if time.monotonic() - last_sweep >= _SWEEP_EVERY_S:
  431. last_sweep = time.monotonic()
  432. try:
  433. sup.sweep_idle()
  434. except Exception as exc:
  435. logger.debug("idle sweep skipped: %s", exc)
  436. try:
  437. refit_idle_presets(sup)
  438. except Exception as exc:
  439. logger.debug("launch window re-plan skipped: %s", exc)
  440. threading.Thread(target=_loop, daemon=True, name="local-runtime-idle-sweep").start()