supervisor.py 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499
  1. """Supervision of one llama-server in router mode.
  2. The router process is ours (restart with backoff on crash); router children are its problem — child
  3. failures surface via GET /models exit_code, never auto-retried here. Learned on real hardware:
  4. health-200 is NOT readiness — every readiness claim requires a touch generation (temp-0, expected
  5. token, generous budget, reasoning_content scanned); always dial 127.0.0.1 — resolving localhost adds
  6. ~2s per request on Windows via IPv6 fallback.
  7. """
  8. from __future__ import annotations
  9. from contextlib import suppress
  10. from functools import lru_cache
  11. import json
  12. import logging
  13. import os
  14. import secrets
  15. import socket
  16. import subprocess
  17. import sys
  18. import threading
  19. import time
  20. import urllib.error
  21. import urllib.request
  22. from pathlib import Path
  23. from hermes_cli.local_runtime.binaries import runtimes_root
  24. from hermes_cli.local_runtime.endpoint import child_query
  25. from hermes_cli.local_runtime.processes import server_child_env, spawn_server
  26. from hermes_cli.process_identity import release_process_group_once_empty, stop_owned_group_survivors
  27. logger = logging.getLogger(__name__)
  28. TOUCH_PROMPT = "Reply with exactly one word: the capital of France."
  29. TOUCH_EXPECT = "paris"
  30. _RESTART_BACKOFF_S = (1, 5, 15, 60)
  31. _RESIDENT = ("loaded", "ready")
  32. # Chosen once and reused across restarts: sessions persist the resolved base_url as a snapshot, and
  33. # every resume path re-resolves llamacpp-alias sessions to the live endpoint (a stale port is
  34. # recoverable, but a stable one keeps external tooling pointed at the right place). Deliberately NOT
  35. # 8080 so we never collide with a user's own llama-server/Ollama-adjacent stack.
  36. _DEFAULT_PORT = 18434
  37. def state_path() -> Path:
  38. """Endpoint state for other Hermes processes (provider resolution routes llamacpp-alias
  39. requests at the managed server from this)."""
  40. return runtimes_root() / "server.json"
  41. def _quiet(fn) -> None:
  42. """Best-effort call; a child that vanished mid-walk is not an error."""
  43. with suppress(Exception):
  44. fn()
  45. def _free_port() -> int:
  46. with socket.socket() as s:
  47. s.bind(("127.0.0.1", 0))
  48. return s.getsockname()[1]
  49. def _stable_port() -> int:
  50. """The stable default port, or an ephemeral one only when something else already listens
  51. there (a leftover managed server would have been cleaned up by stop())."""
  52. try:
  53. with socket.socket() as s:
  54. s.bind(("127.0.0.1", _DEFAULT_PORT))
  55. return _DEFAULT_PORT
  56. except OSError:
  57. logger.warning(
  58. "port %d busy; managed llama-server falling back to an ephemeral "
  59. "port — resumed sessions follow the live endpoint", _DEFAULT_PORT)
  60. return _free_port()
  61. def _stable_api_key() -> str:
  62. """One key for the life of the install, persisted beside the runtimes.
  63. Endpoint identity must survive restarts as a UNIT — sessions persist base_url + api_key, so a
  64. per-boot key strands every resumed session on HTTP 401 exactly as a per-boot port would on
  65. connection errors.
  66. """
  67. key_path = runtimes_root() / ".api_key"
  68. with suppress(OSError):
  69. existing = key_path.read_text(encoding="utf-8-sig").strip()
  70. if len(existing) >= 16:
  71. return existing
  72. key = secrets.token_urlsafe(24)
  73. try:
  74. key_path.parent.mkdir(parents=True, exist_ok=True)
  75. key_path.write_text(key, encoding="utf-8")
  76. except OSError as exc:
  77. logger.warning("could not persist api key (%s); sessions will need "
  78. "a re-pick after restart", exc)
  79. return key
  80. @lru_cache(maxsize=16)
  81. def _direct_io_args(executable: Path) -> tuple[str, ...]:
  82. """Select the loading option supported by this engine, including older pinned builds."""
  83. result = subprocess.run([str(executable), "--help"], capture_output=True,
  84. text=True, encoding="utf-8", errors="replace", check=True,
  85. timeout=15, cwd=str(executable.parent))
  86. help_text = result.stdout + result.stderr
  87. if "--load-mode" in help_text:
  88. return ("--load-mode", "dio")
  89. return ("-dio",) if "--direct-io" in help_text else ()
  90. class LlamaServerSupervisor:
  91. """Own one llama-server router process for the life of a Hermes session."""
  92. # A model that has gone quiet gets its VRAM back after this long. A constant, not a knob:
  93. # long enough that an active conversation never trips it, short enough that a wandered-off
  94. # session frees ~20 GiB within the hour. No exemptions: demand reloads anything the user
  95. # comes back to.
  96. IDLE_UNLOAD_S = 15 * 60
  97. def __init__(self, binary: Path, models_dir: Path, *,
  98. models_max: int = 4, port: int | None = None,
  99. extra_args: list[str] | None = None,
  100. log_path: Path | None = None,
  101. preset_path: Path | None = None):
  102. # The exact engine binary (PM store path, backend-selected), handed
  103. # in by boot — the supervisor never discovers binaries itself: a
  104. # legacy-directory scan could resurrect bytes pm did not pin.
  105. self.binary = Path(binary)
  106. self.models_dir = Path(models_dir)
  107. self.models_max = models_max
  108. self.port = port or _stable_port()
  109. self.api_key = _stable_api_key()
  110. self.extra_args = list(extra_args or [])
  111. self.log_path = log_path or (self.models_dir.parent / "logs" / "llama-server.log")
  112. self.preset_path = preset_path
  113. self.proc: subprocess.Popen | None = None
  114. self._job = None
  115. self.primary_model: str | None = None
  116. self._restarts = 0
  117. self._stopping = False
  118. self._stop_event = threading.Event()
  119. self._lifecycle_lock = threading.RLock()
  120. self._state: dict | None = None
  121. self._watchdog: threading.Thread | None = None
  122. self._log_handle = None
  123. self._idle_since: dict[str, float] = {}
  124. # Launch budget the preset file was last planned against (bootstrap.refit_idle_presets).
  125. self._refit_usable: int | None = None
  126. # ── endpoints ────────────────────────────────────────────
  127. @property
  128. def base_url(self) -> str:
  129. return f"http://127.0.0.1:{self.port}/v1"
  130. def _url(self, route: str) -> str:
  131. return f"http://127.0.0.1:{self.port}{route}"
  132. def _open(self, route: str, body: dict | None = None, timeout_s: int = 30,
  133. *, json_type: bool = True):
  134. headers = {"Authorization": f"Bearer {self.api_key}"}
  135. if json_type:
  136. headers["Content-Type"] = "application/json"
  137. req = urllib.request.Request(self._url(route), headers=headers,
  138. data=json.dumps(body).encode() if body is not None else None)
  139. return urllib.request.urlopen(req, timeout=timeout_s)
  140. def _request(self, route: str, body: dict | None = None, timeout_s: int = 30) -> dict:
  141. with self._open(route, body, timeout_s) as r:
  142. raw = r.read()
  143. return json.loads(raw) if raw else {}
  144. # ── lifecycle ────────────────────────────────────────────
  145. def _spawn(self) -> None:
  146. exe = self.binary
  147. cmd = [
  148. str(exe),
  149. "--host", "127.0.0.1",
  150. "--port", str(self.port),
  151. "--api-key", self.api_key,
  152. "--models-max", str(self.models_max),
  153. # Residency contract: a chat request to a staged-but-unloaded model loads it (slow
  154. # first token) instead of a bare 400/404 after an eject.
  155. "--models-autoload",
  156. "--metrics", # opt-in flag; supervisor telemetry needs it
  157. "--slots", # /slots endpoint is also opt-in; is_idle reads it
  158. "--no-ui",
  159. "--jinja",
  160. # Direct I/O on model load bypasses the page cache so a multi-GB load doesn't evict
  161. # half the OS cache — measured faster on NVMe, and our router bounces reload often.
  162. *_direct_io_args(exe),
  163. ]
  164. if self.preset_path and self.preset_path.exists():
  165. cmd += ["--models-preset", str(self.preset_path)]
  166. else:
  167. cmd += ["--models-dir", str(self.models_dir)]
  168. cmd += self.extra_args
  169. self.log_path.parent.mkdir(parents=True, exist_ok=True)
  170. if self._log_handle is not None:
  171. # The crash-restart loop calls _spawn repeatedly; each restart would leak one fd.
  172. _quiet(self._log_handle.close)
  173. self._log_handle = open(self.log_path, "a", encoding="utf-8", errors="replace")
  174. self._log_handle.write(f"\n# spawn: {cmd}\n")
  175. self._log_handle.flush()
  176. # list-args, never a shell: spaced paths (user homes) must survive. owned: the router
  177. # tree dies with this process on every OS, however this process dies (#126631).
  178. self.proc, self._job = spawn_server(cmd, owned=True, stdout=self._log_handle, stderr=subprocess.STDOUT,
  179. cwd=str(exe.parent), env=server_child_env(os.environ))
  180. logger.info("llama-server router spawned pid=%s port=%s", self.proc.pid, self.port)
  181. # State goes down at SPAWN, not after health: endpoint resolution treats a
  182. # live-pid-but-not-yet-healthy server as "starting" rather than "unconfigured", so a
  183. # readiness probe racing the boot doesn't throw the app back to onboarding.
  184. self._write_state()
  185. def start(self, timeout_s: int = 120) -> None:
  186. with self._lifecycle_lock:
  187. self._stopping = False
  188. self._stop_event.clear()
  189. self._spawn()
  190. self._wait_health(timeout_s)
  191. self._watchdog = threading.Thread(target=self._watch, daemon=True, name="llamacpp-supervisor")
  192. self._watchdog.start()
  193. def _write_state(self) -> None:
  194. import os
  195. import psutil
  196. from gateway.status import get_process_start_time
  197. from utils import atomic_json_write
  198. proc = psutil.Process(self.proc.pid)
  199. # create_time stays for runtimes that predate start_time.
  200. self._state = {"base_url": self.base_url, "api_key": self.api_key,
  201. "pid": proc.pid, "create_time": proc.create_time(),
  202. "start_time": get_process_start_time(proc.pid),
  203. "executable": proc.exe(), "owner_pid": os.getpid(),
  204. "owner_create_time": psutil.Process().create_time(),
  205. "owner_start_time": get_process_start_time(os.getpid())}
  206. path = state_path()
  207. from hermes_constants import mkdir_under_hermes_home
  208. mkdir_under_hermes_home(path.parent)
  209. atomic_json_write(path, self._state, mode=0o600)
  210. def _wait_health(self, timeout_s: int) -> None:
  211. deadline = time.monotonic() + timeout_s
  212. while time.monotonic() < deadline:
  213. if self._stop_event.is_set():
  214. raise RuntimeError("llama-server startup cancelled")
  215. if self.proc and self.proc.poll() is not None:
  216. raise RuntimeError(f"llama-server exited rc={self.proc.returncode} during startup "
  217. f"(log: {self.log_path})")
  218. with suppress(urllib.error.URLError, OSError, TimeoutError):
  219. with urllib.request.urlopen(self._url("/health"), timeout=3) as r:
  220. if r.status == 200:
  221. return
  222. time.sleep(1)
  223. raise TimeoutError(f"llama-server not healthy after {timeout_s}s (log: {self.log_path})")
  224. def _watch(self) -> None:
  225. """Restart the router (not its children) on crash, with backoff."""
  226. while not self._stopping:
  227. proc = self.proc
  228. if proc is None:
  229. return
  230. rc = proc.poll()
  231. if rc is None:
  232. time.sleep(2)
  233. continue
  234. if self._stopping:
  235. return
  236. # An already-empty group is released now, not after the backoff: its number is free for
  237. # reuse meanwhile, and the survivor stop below would then signal a stranger's group.
  238. release_process_group_once_empty(proc.pid, proc=proc, timeout=0)
  239. backoff = _RESTART_BACKOFF_S[min(self._restarts, len(_RESTART_BACKOFF_S) - 1)]
  240. logger.warning("llama-server exited rc=%s; restart #%s in %ss", rc, self._restarts + 1, backoff)
  241. if self._stop_event.wait(backoff):
  242. return
  243. self._restarts += 1
  244. try:
  245. with self._lifecycle_lock:
  246. if self._stopping:
  247. return
  248. # The crashed router's group: stop what is left of it and release its number
  249. # before the replacement registers its own (survivors stay covered).
  250. stop_owned_group_survivors(proc.pid, proc=proc)
  251. self._reap_orphaned_children()
  252. self._spawn()
  253. self._wait_health(120)
  254. if self.primary_model:
  255. self.ensure_model_ready(self.primary_model)
  256. except Exception as exc:
  257. logger.error("llama-server restart failed: %s", exc)
  258. def stop(self) -> None:
  259. with self._lifecycle_lock:
  260. self._stopping = True
  261. self._stop_event.set()
  262. try:
  263. if self.proc and self.proc.poll() is None:
  264. self._terminate_tree(self.proc)
  265. finally:
  266. if self._job is not None:
  267. self._job.close()
  268. self._job = None
  269. if self.proc is not None:
  270. # A router that crashed (the watcher's restart backoff) can leave its model children
  271. # running in its group: stop them, then release the group only once it is empty, so
  272. # a tree we failed to kill stays covered by the owner-death supervisor.
  273. stop_owned_group_survivors(self.proc.pid, proc=self.proc)
  274. # Retain state: deleting it could race a replacement publication.
  275. if self._log_handle:
  276. self._log_handle.close()
  277. self._log_handle = None
  278. @staticmethod
  279. def _terminate_tree(proc: subprocess.Popen, *, verified_root: bool = False) -> None:
  280. """Terminate the router AND its model children.
  281. Each child holds gigabytes of VRAM; terminating only the router (TerminateProcess on
  282. Windows does no cleanup) orphans them with the weights still resident. Enumerate children
  283. FIRST (the parent must be alive to walk them), terminate all, escalate to kill.
  284. """
  285. children: list = []
  286. timeouts = (subprocess.TimeoutExpired,)
  287. with suppress(ImportError):
  288. import psutil
  289. timeouts += (psutil.TimeoutExpired,)
  290. if verified_root:
  291. # Recovery retains the birth identity; never rebuild it from a PID.
  292. children = proc.children(recursive=True)
  293. if not proc.is_running():
  294. raise psutil.NoSuchProcess(proc.pid)
  295. else:
  296. with suppress(psutil.Error):
  297. children = psutil.Process(proc.pid).children(recursive=True)
  298. try:
  299. for child in children:
  300. _quiet(child.terminate)
  301. proc.terminate()
  302. try:
  303. proc.wait(timeout=15)
  304. except timeouts:
  305. proc.kill()
  306. finally:
  307. for child in children:
  308. _quiet(lambda: child.is_running() and child.kill())
  309. def _reap_orphaned_children(self) -> None:
  310. """Kill model children orphaned by a router crash, before respawn.
  311. A dead parent can't be walked, so match by identity: any process running OUR
  312. llama-server binary whose parent is gone is an orphan of a previous router. Its VRAM must
  313. come back before the new router loads models next to the ghosts.
  314. """
  315. if self._job is not None:
  316. self._job.close()
  317. self._job = None
  318. return
  319. if sys.platform == "win32":
  320. return # Unrecorded processes are not ours merely because the binary matches.
  321. try:
  322. import psutil
  323. exe = str(self.binary)
  324. except Exception:
  325. return
  326. own_pid = self.proc.pid if self.proc is not None else None
  327. for p in psutil.process_iter(["exe", "ppid"]):
  328. with suppress(psutil.NoSuchProcess, psutil.AccessDenied):
  329. ppid = p.info.get("ppid") or 0
  330. if (p.info.get("exe") != exe or p.pid == own_pid
  331. or (ppid and psutil.pid_exists(ppid))):
  332. continue
  333. logger.warning("reaping orphaned llama-server child pid=%s", p.pid)
  334. p.kill()
  335. # ── model management (router endpoints) ──────────────────
  336. def models(self, timeout_s: int = 30) -> dict:
  337. """{model_id: status_value} from GET /models."""
  338. return {m["id"]: m.get("status", {}).get("value", "unknown")
  339. for m in self._request("/models", timeout_s=timeout_s).get("data", [])}
  340. def load_model(self, model_id: str, timeout_s: int = 600) -> None:
  341. self._request("/models/load", {"model": model_id}, timeout_s=timeout_s)
  342. def reload_presets(self) -> None:
  343. """Have the router re-read the preset file (GET /models?reload=1). It applies new launch
  344. flags to models that aren't loaded and unloads any loaded model whose flags changed, so
  345. callers rewrite the file only while nothing is loaded."""
  346. self._request("/models?reload=1", timeout_s=10)
  347. def unload_model(self, model_id: str) -> None:
  348. """Free the child's VRAM now (POST /models/unload; bogus name -> 400). Momentary: never
  349. touches primary_model — the declaration is durable, an eject is not."""
  350. self._request("/models/unload", {"model": model_id}, timeout_s=120)
  351. deadline = time.monotonic() + 15
  352. while time.monotonic() < deadline:
  353. try:
  354. if self.models().get(model_id) not in (*_RESIDENT, "unloading"):
  355. return
  356. except Exception:
  357. return
  358. time.sleep(0.3)
  359. def sweep_idle(self, now: float | None = None) -> list[str]:
  360. """Unload models idle past IDLE_UNLOAD_S; returns their ids. Idle = no busy slots and no
  361. queued work, tracked per model across calls; a model seen busy resets its clock. A
  362. failed telemetry probe is neither idle nor busy: the clock is kept, so a flaky probe
  363. cannot pin a resident model (and its VRAM) indefinitely."""
  364. now = time.monotonic() if now is None else now
  365. unloaded: list[str] = []
  366. try:
  367. statuses = self.models()
  368. except Exception:
  369. return unloaded
  370. for model_id, status in statuses.items():
  371. if status not in _RESIDENT:
  372. self._idle_since.pop(model_id, None)
  373. continue
  374. probe = self._probe_idle(model_id)
  375. if probe is None:
  376. logger.info("idle probe for %s failed; keeping idle clock (idle %ds)", model_id,
  377. int(now - self._idle_since.get(model_id, now)))
  378. continue
  379. if probe is False:
  380. self._idle_since.pop(model_id, None)
  381. continue
  382. first_idle = self._idle_since.setdefault(model_id, now)
  383. if now - first_idle < self.IDLE_UNLOAD_S:
  384. continue
  385. try:
  386. self.unload_model(model_id)
  387. self._idle_since.pop(model_id, None)
  388. unloaded.append(model_id)
  389. logger.info("idle-unloaded %s (idle %ds)", model_id, int(now - first_idle))
  390. except Exception as exc:
  391. logger.warning("idle unload of %s failed: %s", model_id, exc)
  392. return unloaded
  393. def touch_generate(self, model_id: str, timeout_s: int = 300) -> bool:
  394. """The readiness proof. Generous budget + reasoning_content scan — small token budgets
  395. false-fail reasoning models, which spend their first tokens thinking."""
  396. try:
  397. resp = self._request("/v1/chat/completions", {
  398. "model": model_id, "messages": [{"role": "user", "content": TOUCH_PROMPT}],
  399. "max_tokens": 512, "temperature": 0}, timeout_s=timeout_s)
  400. msg = resp["choices"][0]["message"]
  401. blob = (msg.get("content") or "") + " " + (msg.get("reasoning_content") or "")
  402. return TOUCH_EXPECT in blob.lower()
  403. except Exception as exc:
  404. logger.warning("touch generation failed for %s: %s", model_id, exc)
  405. return False
  406. def ensure_model_ready(self, model_id: str, timeout_s: int = 600) -> bool:
  407. """Load if needed, then prove readiness with a touch generation."""
  408. status = self.models().get(model_id)
  409. if status is None:
  410. raise KeyError(f"model {model_id} not present in models dir")
  411. if status not in _RESIDENT:
  412. self.load_model(model_id, timeout_s=timeout_s)
  413. return self.touch_generate(model_id)
  414. # ── telemetry ────────────────────────────────────────────
  415. def is_idle(self, model_id: str | None = None) -> bool:
  416. """No processing requests and no busy slots. Router quirk: /slots and /metrics are
  417. per-child and require ?model= (bare calls 400). With ``model_id`` checks that one child;
  418. without, every loaded child."""
  419. return self._probe_idle(model_id) is True
  420. def _probe_idle(self, model_id: str | None = None) -> bool | None:
  421. """Tri-state idle probe for the sweeper: True = confirmed idle, False = confirmed
  422. busy, None = the probe itself failed. The sweeper must never mistake a dead probe
  423. for activity — that resets the idle clock and pins the model's VRAM. A model that
  424. isn't loaded is idle: the router answers 400 for it (the reads don't autoload)."""
  425. try:
  426. loaded = ([model_id] if model_id is not None
  427. else [m for m, status in self.models().items() if status in _RESIDENT])
  428. for mid in loaded:
  429. try:
  430. slots = self._request(f"/slots?{child_query(mid)}")
  431. except urllib.error.HTTPError as exc:
  432. if exc.code == 400:
  433. continue
  434. raise
  435. if any(s.get("is_processing") for s in slots):
  436. return False
  437. with self._open(f"/metrics?{child_query(mid)}", timeout_s=10, json_type=False) as r:
  438. text = r.read().decode()
  439. for line in text.splitlines():
  440. if (line.startswith("llamacpp:requests_processing")
  441. and float(line.split()[-1]) != 0.0):
  442. return False
  443. return True
  444. except Exception:
  445. return None