| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533 |
- """Bootstrap for the managed runtime: config -> installed binaries -> running supervised server.
- One public call, ``ensure_local_runtime(config)``, safe at any session start: disabled or
- already-running -> no-op; enabled -> serve the installed build under a supervisor. Kept
- import-light: callers gate on config before importing so disabled sessions never pay the import.
- """
- from __future__ import annotations
- from contextlib import contextmanager, suppress
- import logging
- import os
- import time
- from pathlib import Path
- from hermes_cli.local_runtime.binaries import runtimes_root
- from hermes_cli.local_runtime.gguf import SPLIT_PART_RE, model_id_from_stem
- logger = logging.getLogger(__name__)
- _SUPERVISOR = None # process-wide singleton; one router per Hermes process
- # The engine that singleton runs (or is booting). Budgets price ITS devices: the server is shared by
- # every profile this process hosts, so another profile's ``local_runtime.backend`` must not resize it.
- _SERVING_ENGINE = None
- _EXIT_HOOKED = False
- def _stop_at_exit() -> None:
- """The process that boots the server stops it on a clean exit, whatever surface it is (the
- desktop backend also stops it in its shutdown handler; the second call is a no-op). Processes
- that only adopted another's server never stop it. A hard kill still leaves an orphan, which the
- next boot adopts or replaces."""
- global _EXIT_HOOKED
- if not _EXIT_HOOKED:
- import atexit
- atexit.register(shutdown_local_runtime)
- _EXIT_HOOKED = True
- def _detect_gpu_vendor() -> str | None:
- """Best-effort GPU vendor for backend selection.
- Preserve NVIDIA's detailed name when nvidia-smi is available, then use the
- platform host facts so Intel and AMD do not silently demote ``auto`` to CPU.
- """
- from hermes_cli.local_runtime.hardware import _cached_nvidia_gpu_query
- from hermes_platform.host.facts import gpu_class
- query = _cached_nvidia_gpu_query()
- if query is not None and query.get("gpu_name"):
- return "nvidia " + query["gpu_name"]
- detected = gpu_class()
- return detected if detected in {"nvidia", "amd", "intel"} else None
- def models_dir() -> Path:
- """Machine-scoped, deliberately NOT profile-scoped: a 20 GB GGUF is a machine asset, and every
- profile shares the one managed server that serves it (same rule as runtimes_root())."""
- from hermes_constants import get_default_hermes_root
- return get_default_hermes_root() / "models"
- def assets_dir() -> Path:
- """Non-model companion files (mmproj projectors, spec-decode drafts). A subdirectory so the
- router's model listing — and staged_models() — never mistakes an asset for a servable model."""
- return models_dir() / "assets"
- def staged_in(models_dir: Path, *, require_complete: bool = True) -> list[Path]:
- """Servable GGUFs in a directory: single files, plus split GGUFs once by their first part.
- With ``require_complete`` a split counts only when EVERY part is on disk — a mid-download split
- is not servable and must not surface anywhere as a model."""
- files = sorted(models_dir.glob("*.gguf"))
- names = {p.name for p in files}
- out = []
- for p in files:
- m = SPLIT_PART_RE.search(p.name)
- if m is None:
- out.append(p)
- continue
- if m.group(1) != "00001":
- continue
- stem, total = p.name[: m.start()], int(m.group(2))
- if not require_complete or all(f"{stem}-{i:05d}-of-{m.group(2)}.gguf" in names
- for i in range(2, total + 1)):
- out.append(p)
- return out
- def adopt_legacy_models() -> list[Path]:
- """Move GGUFs left in the old per-profile ``<profile home>/models`` layout (and its assets/)
- into the machine-scoped dirs, so everything downstream keeps reading one directory.
- ``os.rename`` only: within one filesystem it is instant even for a 20 GB model, while
- ``shutil.move`` silently degrades to a copy across devices. A cross-device profile dir is left
- in place with a warning rather than copying tens of GB at session start. A name that already
- exists in the destination is left alone (check-then-rename: the only window is two processes
- adopting two profiles' same-named file at once, and a same name is the same catalog variant).
- Two processes racing on one file are harmless: the loser's rename finds the source gone and
- skips it. Returns the new paths of the moved files."""
- from hermes_constants import get_default_hermes_root, named_profile_has_identity
- profiles_root = get_default_hermes_root() / "profiles"
- if not profiles_root.is_dir():
- return []
- moved: list[Path] = []
- for home in sorted(profiles_root.iterdir()):
- old = home / "models"
- if home.name.startswith(".") or not old.is_dir() or not named_profile_has_identity(home):
- continue
- for src_dir, dest_dir in ((old, models_dir()), (old / "assets", assets_dir())):
- for src in sorted(src_dir.glob("*.gguf")):
- dest = dest_dir / src.name
- if dest.exists():
- logger.warning("legacy model %s not moved: %s already exists", src, dest)
- continue
- try:
- dest_dir.mkdir(parents=True, exist_ok=True)
- os.rename(src, dest)
- except FileNotFoundError:
- continue
- except OSError as exc:
- logger.warning("legacy model %s not moved to %s: %s", src, dest_dir, exc)
- continue
- moved.append(dest)
- for emptied in (old / "assets", old):
- with suppress(OSError):
- emptied.rmdir()
- if moved:
- logger.info("moved %d legacy model file(s) into %s", len(moved), models_dir())
- return moved
- def staged_models() -> list[Path]:
- """Servable staged models (continuation parts, incomplete splits and assets/ never count)."""
- return staged_in(models_dir())
- def staged_model_ids() -> list[str]:
- return [model_id_from_stem(p.stem) for p in staged_models()]
- def _presets_stale() -> bool:
- """True when a staged model has no section in the preset INI — it would autoload with stock
- fit instead of a policy decision."""
- with suppress(Exception):
- from hermes_cli.local_runtime.presets import read_preset_decisions
- known = read_preset_decisions()
- return any(mid not in known or (not known[mid].refusal and not (known[mid].keys or {}).get("model"))
- for mid in staged_model_ids())
- return False
- def _stop_state_server() -> None:
- """Stop the server the state file points at before this process boots a replacement, but only
- an orphan (recorded owner gone): a live owner's watchdog would respawn its router, so that one
- is left running and the replacement boots beside it, as it always has. The process comes from
- the identity-guarded reader, never a bare PID — the endpoint dict callers hold has none."""
- from hermes_cli.local_runtime.recovery import (
- _owner_is_dead,
- is_modern,
- legacy_recorded_process,
- read_state,
- recorded_process,
- )
- from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
- state = read_state()
- if is_modern(state):
- if not _owner_is_dead(state):
- logger.info("llama-server pid=%s belongs to a live Hermes process; leaving it", state.get("pid"))
- return
- proc = recorded_process(state)
- else:
- proc = legacy_recorded_process(state)
- if proc is None:
- return
- try:
- # Waits for the tree to exit, so the port and the GPU are free for the replacement.
- LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
- except Exception as exc:
- logger.warning("could not stop the incumbent llama-server (pid=%s): %s", proc.pid, exc)
- def refresh_local_runtime() -> bool:
- """Restart the managed server so it rescans the models directory. The router's model list is
- SPAWN-ONLY: a GGUF added after start is invisible to GET /models and 400s on completion, so
- anything that changes the staged set while the server runs must bounce it."""
- global _SUPERVISOR
- try:
- from hermes_cli.config import load_config
- if _SUPERVISOR is None:
- from hermes_cli.local_runtime.endpoint import _state_endpoint
- state = _state_endpoint()
- if state is None:
- return False
- _stop_state_server()
- else:
- shutdown_local_runtime()
- return ensure_local_runtime(load_config(), force=True) is not None
- except Exception as exc:
- logger.warning("local runtime refresh failed: %s", exc)
- return False
- def _admitted_models_max(mdir: Path, configured: int, engine_tag: str | None = None) -> int:
- """Residency cap to hand the router: derived from the hardware budget, ``models_max`` as a ceiling.
- A cap of "four" on a card that holds one model is how a second child ends up paged (WDDM) and
- silently slow — llama.cpp evicts its LRU before an incoming load only when the cap says the
- card is full. A probe miss or an unpriceable model keeps the configured count: this must never
- block a boot. On Windows unified memory, free commit bounds the count too.
- """
- try:
- from hermes_cli.local_runtime.hardware import probe_budget
- from hermes_cli.local_runtime.presets import admitted_residency_count
- capacity = probe_budget(planning=True)
- live = _launch_budget(capacity) if capacity.uma else None
- cap = admitted_residency_count(mdir, capacity, configured, live=live, engine_tag=engine_tag)
- except Exception as exc:
- logger.warning("residency cap probe failed (%s); using models_max=%s", exc, configured)
- return configured
- if cap != configured:
- logger.info("residency cap: %s resident model(s) on this card (models_max=%s)",
- cap, configured)
- return cap
- def _generate_presets(mdir: Path, preset_path: Path, residents: int = 1, *,
- engine_tag: str | None = None) -> Path | None:
- """Write the launch-policy INI for every staged model; returns the path to hand the router.
- Priced against CAPACITY, not live free VRAM: this runs while the outgoing server instance may
- still hold the card (restart, refresh after a download), and its memory is freed before the new
- instance loads anything. Pricing against live-free once pinned a fitting model's weights to CPU.
- The window is then narrowed to what fits beside other programs' GPU memory
- (``hardware.launch_budget``), which never moves weights to the CPU.
- Degradation ladder on failure: a STALE policy still beats no policy — stock fit (f16 KV at max
- context, no placement) is the silent-busy-wait failure on Windows. Keep serving with the
- previous INI when one exists; only a first boot with no INI at all falls to stock fit.
- ``residents`` is the router's model count, which splits the prompt-cache room on Windows
- unified memory.
- """
- from hermes_cli.local_runtime.hardware import probe_budget
- from hermes_cli.local_runtime.presets import generate_presets
- try:
- capacity = probe_budget(planning=True)
- for entry in generate_presets(mdir, capacity, preset_path, live=_launch_budget(capacity),
- residents=residents, engine_tag=engine_tag):
- if entry.refusal:
- logger.warning("model refused by physics check: %s", entry.refusal)
- return preset_path
- except Exception as exc:
- if preset_path.exists():
- logger.error("preset generation failed (%s); serving with the "
- "PREVIOUS launch policies — models staged since "
- "the last successful generation run unpoliced "
- "until this is fixed", exc)
- return preset_path
- logger.error("preset generation failed (%s) and no previous "
- "policy file exists; router runs stock fit", exc)
- return None
- def _launch_budget(capacity, own_bytes: int = 0):
- """``hardware.launch_budget``, or None when the probe fails: a boot never waits on it."""
- from hermes_cli.local_runtime.hardware import launch_budget
- try:
- return launch_budget(capacity, own_bytes=own_bytes)
- except Exception as exc:
- logger.warning("free GPU memory probe failed (%s); launch windows use capacity", exc)
- return None
- # Router statuses that hold GPU memory. The preset file changes only while every model is out of
- # them: the router unloads a loaded model whose launch flags change on reload.
- _HOLDS_MEMORY = frozenset({"loaded", "loading", "sleeping", "ready"})
- def refit_idle_presets(sup) -> bool:
- """Re-plan launch windows against free GPU memory while no model is loaded.
- Boot plans every window once, but models load later (on demand, after an idle unload), and by
- then other programs may hold more or less of the card. Returns True when the router was given
- new presets. A request that starts a load between the last status check and the reload is
- unloaded by the router and fails once; that gap is a few milliseconds, and only when a window
- changes.
- """
- from hermes_cli.local_runtime.hardware import probe_budget
- from hermes_cli.local_runtime.presets import plan_presets, render_presets
- path = sup.preset_path
- if path is None or not path.exists() or _any_holds_memory(sup):
- return False
- capacity = probe_budget(planning=True)
- live = _launch_budget(capacity)
- if live is None:
- return False
- last = sup._refit_usable
- if last is not None and abs(live.usable_vram_bytes - last) < _REFIT_STEP_BYTES:
- return False
- text = render_presets(plan_presets(models_dir(), capacity, live=live, residents=sup.models_max,
- engine_tag=_SERVING_ENGINE.tag if _SERVING_ENGINE is not None else None))
- # Held across the write and the reload so a restart can't spawn between them; everything
- # slow ran above.
- with sup._lifecycle_lock:
- if sup.proc is None or sup.proc.poll() is not None or _any_holds_memory(sup):
- return False
- sup._refit_usable = live.usable_vram_bytes
- if text == path.read_text(encoding="utf-8-sig"):
- return False
- from utils import atomic_write_text
- atomic_write_text(path, text, tmp_prefix=f".{path.name}_", mode=0o600)
- sup.reload_presets()
- logger.info("launch windows re-planned for %.1f GiB of free GPU memory",
- live.usable_vram_bytes / (1 << 30))
- return True
- # Smaller changes in free memory don't move a window by a ladder rung; skipping them keeps the
- # idle loop from re-reading every model header each pass.
- _REFIT_STEP_BYTES = 256 << 20
- def _any_holds_memory(sup) -> bool:
- return any(status in _HOLDS_MEMORY for status in sup.models(timeout_s=5).values())
- def _try_lock_boot_fd(fd: int) -> bool:
- """Non-blocking exclusive attempt; portable across fcntl/msvcrt."""
- if os.name == "nt":
- import msvcrt
- if os.fstat(fd).st_size == 0:
- os.write(fd, b"\0")
- os.lseek(fd, 0, os.SEEK_SET)
- try:
- msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
- return True
- except OSError:
- return False
- else:
- import fcntl
- try:
- fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
- return True
- except OSError:
- return False
- def _unlock_boot_fd(fd: int) -> None:
- if os.name == "nt":
- import msvcrt
- os.lseek(fd, 0, os.SEEK_SET)
- msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
- else:
- import fcntl
- fcntl.flock(fd, fcntl.LOCK_UN)
- @contextmanager
- def _cross_process_boot_lock(timeout_s: float = 130.0):
- """Serialize the state-check-then-spawn sequence across every Hermes process on this
- machine — the ``_SUPERVISOR`` singleton above only rules out a race within ONE process.
- Two profiles booting in the same second each see no ``server.json`` yet and each spawn a
- router on the stable port (#116682); an OS-held lock makes the second caller wait for the
- first to publish its state file, so it re-checks and adopts instead of spawning a duplicate.
- Bounded, not indefinite: never hang session start dead if the lock is somehow stuck, and
- never raise into session start: an unwritable runtimes dir (or a foreign-owned lock file)
- proceeds unlocked with a warning, like the contention timeout."""
- path = runtimes_root() / "boot.lock"
- try:
- path.parent.mkdir(parents=True, exist_ok=True)
- fd = os.open(path, os.O_CREAT | os.O_RDWR, 0o600)
- except OSError as exc:
- logger.warning("boot lock unavailable (%s); proceeding without it", exc)
- yield
- return
- try:
- deadline = time.monotonic() + timeout_s
- while not _try_lock_boot_fd(fd):
- if time.monotonic() >= deadline:
- logger.warning("boot lock contended past %.0fs; proceeding without it", timeout_s)
- break
- time.sleep(0.2)
- try:
- yield
- finally:
- with suppress(OSError):
- _unlock_boot_fd(fd)
- finally:
- os.close(fd)
- def ensure_local_runtime(config: dict, force: bool = False) -> object | None:
- """Idempotent boot of the managed runtime. Returns the supervisor (or None when
- disabled/unavailable). Never raises into a session start — failures log and return None; chat
- falls back to configured providers."""
- global _SUPERVISOR, _SERVING_ENGINE
- section = (config or {}).get("local_runtime") or {}
- if not force and not section.get("enabled"):
- return None
- if _SUPERVISOR is not None:
- return _SUPERVISOR
- try:
- adopt_legacy_models()
- except OSError as exc: # an unreadable profiles dir must not block serving what's staged
- logger.warning("legacy model adoption failed: %s", exc)
- # Residency: no staged models means nothing to serve — don't boot an empty server (delete
- # your last model and boots stop). force boots as ever.
- if not force and not staged_models():
- logger.info("local runtime enabled but no models staged; not booting")
- return None
- # Another Hermes process may already be supervising — reuse via state, but ONLY while its
- # launch policy still covers every staged model. A server whose preset file predates a
- # download serves the new model with no policy at all (--models-autoload + stock fit). A stale
- # incumbent gets stopped and replaced by a fresh boot with regenerated presets; sessions ride
- # through like any other supervised restart (stable port + persisted key).
- #
- # The state check and the spawn below run under a cross-process lock: two backends racing
- # to boot (#116682) must not both find no state file and both spawn a router on the stable
- # port — the loser waits here, then re-checks state and adopts the winner's server instead.
- from hermes_cli.local_runtime.endpoint import _state_endpoint
- with _cross_process_boot_lock():
- state = _state_endpoint()
- if state is not None:
- if not _presets_stale():
- logger.info("managed llama-server already running (another process)")
- return None
- logger.info("running server's presets predate the staged models; "
- "replacing it so every model launches with a policy")
- _stop_state_server()
- try:
- from hermes_cli.local_runtime.binaries import installed_engine
- from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
- engine = installed_engine(section.get("backend", "auto"))
- if engine is None:
- logger.info("local runtime enabled but no PM engine installed; use the Local Models pane")
- return None
- _SERVING_ENGINE = engine
- mdir = models_dir()
- mdir.mkdir(parents=True, exist_ok=True)
- models_max = _admitted_models_max(mdir, int(section.get("models_max", 4)), engine.tag)
- preset_path = _generate_presets(mdir, runtimes_root() / "presets.ini", models_max,
- engine_tag=engine.tag)
- sup = LlamaServerSupervisor(engine.binary, mdir, preset_path=preset_path,
- models_max=models_max,
- port=int(section.get("port", 0)) or None)
- try:
- sup.start()
- except Exception:
- # start() can fail after the router process exists (health timeout): leaving it
- # running unsupervised strands its VRAM behind a port nothing will clean up.
- with suppress(Exception):
- sup.stop()
- raise
- _SUPERVISOR = sup
- _stop_at_exit()
- logger.info("managed llama-server up at %s (backend=%s tag=%s)", sup.base_url, engine.backend, engine.tag)
- _start_idle_sweeper(sup)
- return sup
- except Exception as exc:
- _SERVING_ENGINE = None
- logger.warning("managed local runtime unavailable: %s", exc)
- return None
- def shutdown_local_runtime() -> None:
- global _SUPERVISOR, _SERVING_ENGINE
- if _SUPERVISOR is not None:
- _SUPERVISOR.stop()
- _SUPERVISOR = None
- _SERVING_ENGINE = None
- def get_supervisor():
- """The process-local supervisor, or None (a server may still run under another process —
- check the state file)."""
- return _SUPERVISOR
- def serving_engine():
- """The engine the process-local server runs or is booting, or None when there is none."""
- return _SERVING_ENGINE
- _SWEEP_EVERY_S = 120
- _REFIT_EVERY_S = 30
- def _start_idle_sweeper(sup) -> None:
- """Idle-residency loop: every couple of minutes, unload models idle past the supervisor's
- threshold; every 30 s with nothing loaded, re-plan launch windows against free GPU memory.
- Daemon thread tied to the supervisor's lifetime — exits when the server stops."""
- import threading
- def _loop():
- last_sweep = time.monotonic()
- while sup.proc is not None and sup.proc.poll() is None:
- time.sleep(_REFIT_EVERY_S)
- if time.monotonic() - last_sweep >= _SWEEP_EVERY_S:
- last_sweep = time.monotonic()
- try:
- sup.sweep_idle()
- except Exception as exc:
- logger.debug("idle sweep skipped: %s", exc)
- try:
- refit_idle_presets(sup)
- except Exception as exc:
- logger.debug("launch window re-plan skipped: %s", exc)
- threading.Thread(target=_loop, daemon=True, name="local-runtime-idle-sweep").start()
|