| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200 |
- """Conservative recovery of the one router recorded by this managed runtime."""
- from __future__ import annotations
- import json
- import logging
- import math
- import os
- from pathlib import Path
- import psutil
- logger = logging.getLogger(__name__)
- def read_state() -> dict:
- from hermes_cli.local_runtime.supervisor import state_path
- try:
- state = json.loads(state_path().read_text(encoding="utf-8-sig"))
- return state if isinstance(state, dict) else {}
- except (OSError, ValueError):
- return {}
- _MODERN_FIELDS = ("create_time", "executable", "owner_pid", "owner_create_time")
- def is_modern(state: dict) -> bool:
- return any(key in state for key in _MODERN_FIELDS)
- def _valid_pid(value) -> bool:
- return type(value) is int and value > 0
- def _valid_birth(value) -> bool:
- return type(value) in (int, float) and math.isfinite(value) and value > 0
- def _same_birth(proc, state: dict, prefix: str = "") -> bool:
- """Linux derives psutil's create_time from /proc/stat's boot time, which moves whenever the
- clock is stepped (WSL re-syncs within minutes), so a live process stops matching its record.
- The start_time fingerprint is boot-relative there; create_time is for records that predate it."""
- from gateway.status import get_process_start_time, start_time_fingerprints_match
- recorded = state.get(f"{prefix}start_time")
- if not _valid_birth(recorded):
- return proc.create_time() == state[f"{prefix}create_time"]
- current = get_process_start_time(proc.pid)
- return current is not None and start_time_fingerprints_match(recorded, current)
- def recorded_process(state: dict):
- """Match a modern process incarnation, never just its reusable PID."""
- try:
- pid, created = state["pid"], state["create_time"]
- owner_pid, owner_created = state["owner_pid"], state["owner_create_time"]
- exe = state["executable"]
- if (not _valid_pid(pid) or not _valid_birth(created)
- or not _valid_pid(owner_pid) or not _valid_birth(owner_created)
- or owner_created > created or not isinstance(exe, str) or not exe):
- return None
- proc = psutil.Process(pid)
- if (not proc.is_running() or not _same_birth(proc, state)
- or Path(proc.exe()) != Path(exe)):
- return None
- # Windows retains the original parent PID; POSIX reparents orphans.
- if os.name == "nt" and proc.ppid() != owner_pid:
- return None
- parent = proc.parent()
- if parent is not None:
- if parent.pid == owner_pid:
- if not _same_birth(parent, state, "owner_"):
- return None
- elif os.name == "nt" or not _owner_is_dead(state):
- return None # POSIX may reparent a router whose recorded owner exited.
- return proc
- except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
- return None
- def _owner_is_dead(state: dict) -> bool:
- try:
- pid, created = state["owner_pid"], state["owner_create_time"]
- if not _valid_pid(pid) or not _valid_birth(created):
- return False
- try:
- owner = psutil.Process(pid)
- if not owner.is_running():
- return True
- if _valid_birth(state.get("owner_start_time")):
- return not _same_birth(owner, state, "owner_")
- # Records that predate start_time: only a NEWER incarnation proves the owner exited. A
- # stepped clock (WSL) moves create_time, so inequality alone would bury a live owner.
- return owner.create_time() > created
- except psutil.NoSuchProcess:
- return True
- except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
- return False
- def legacy_recorded_process(state: dict):
- """Match an older record only to the managed binary and its exact launch arguments.
- Legacy records lack process birth identity. A live PID alone is not evidence because the OS
- can reuse it after llama-server exits. The executable location, state-file age, endpoint key,
- and launch arguments together keep old records usable without adopting an unrelated process.
- """
- from urllib.parse import urlsplit
- from hermes_cli.local_runtime.bootstrap import models_dir
- from hermes_cli.local_runtime.supervisor import state_path
- # A damaged new record must not fall back to weaker legacy evidence.
- if is_modern(state):
- return None
- try:
- if not _valid_pid(state["pid"]):
- return None
- proc = psutil.Process(state["pid"])
- root = state_path().parent
- exe = Path(proc.exe())
- exe.relative_to(root)
- if exe.name.lower() not in ("llama-server", "llama-server.exe"):
- return None
- if proc.create_time() > state_path().stat().st_mtime:
- return None # the PID was reused after this record was written
- url = urlsplit(state["base_url"])
- if url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port or not state["api_key"]:
- return None
- argv = proc.cmdline()
- def value(flag):
- if argv.count(flag) != 1:
- return None
- index = argv.index(flag) + 1
- return argv[index] if index < len(argv) else None
- if (value("--host") != "127.0.0.1" or value("--port") != str(url.port)
- or value("--api-key") != state["api_key"]):
- return None
- preset, directory = value("--models-preset"), value("--models-dir")
- if not ((preset and Path(preset) == root / "presets.ini" and directory is None)
- or (directory and Path(directory) == models_dir() and preset is None)):
- return None
- return proc
- except (KeyError, TypeError, ValueError, OSError, psutil.Error):
- return None
- def _legacy_orphan_process(state: dict):
- """Windows-only legacy stop recovery after the process identity is established."""
- from hermes_cli.local_runtime.supervisor import state_path
- if os.name != "nt":
- return None
- proc = legacy_recorded_process(state)
- if proc is None:
- return None
- try:
- relative = Path(proc.exe()).relative_to(state_path().parent)
- if len(relative.parts) != 3 or relative.name.lower() != "llama-server.exe":
- return None
- if proc.parent() is not None or proc.ppid() <= 0:
- return None
- return proc
- except (OSError, psutil.Error):
- return None
- def stop_recorded_orphan() -> bool:
- """Explicit user stop only. Refuse uncertain identity or a living recorded owner."""
- from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor, state_path
- try:
- state = json.loads(state_path().read_text(encoding="utf-8-sig"))
- except FileNotFoundError:
- return True # already stopped and no record
- except (OSError, ValueError):
- return False
- if not isinstance(state, dict) or not _valid_pid(state.get("pid")):
- return False
- try:
- if not psutil.pid_exists(state["pid"]):
- return True # keep the dead record; endpoint resolution rejects it
- if is_modern(state):
- proc = recorded_process(state)
- if proc is None or not _owner_is_dead(state):
- return False
- else:
- proc = _legacy_orphan_process(state)
- if proc is None or read_state() != state or not proc.is_running():
- return False
- # Retain the psutil incarnation object: destructive methods guard PID reuse.
- LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
- proc.wait(timeout=5)
- logger.info("stopped orphaned managed llama-server pid=%s", proc.pid)
- return True
- except (OSError, psutil.Error):
- logger.warning("could not verify or stop recorded managed llama-server")
- return False
|