"""Conservative recovery of the one router recorded by this managed runtime.""" from __future__ import annotations import json import logging import math import os from pathlib import Path import psutil logger = logging.getLogger(__name__) def read_state() -> dict: from hermes_cli.local_runtime.supervisor import state_path try: state = json.loads(state_path().read_text(encoding="utf-8-sig")) return state if isinstance(state, dict) else {} except (OSError, ValueError): return {} _MODERN_FIELDS = ("create_time", "executable", "owner_pid", "owner_create_time") def is_modern(state: dict) -> bool: return any(key in state for key in _MODERN_FIELDS) def _valid_pid(value) -> bool: return type(value) is int and value > 0 def _valid_birth(value) -> bool: return type(value) in (int, float) and math.isfinite(value) and value > 0 def _same_birth(proc, state: dict, prefix: str = "") -> bool: """Linux derives psutil's create_time from /proc/stat's boot time, which moves whenever the clock is stepped (WSL re-syncs within minutes), so a live process stops matching its record. The start_time fingerprint is boot-relative there; create_time is for records that predate it.""" from gateway.status import get_process_start_time, start_time_fingerprints_match recorded = state.get(f"{prefix}start_time") if not _valid_birth(recorded): return proc.create_time() == state[f"{prefix}create_time"] current = get_process_start_time(proc.pid) return current is not None and start_time_fingerprints_match(recorded, current) def recorded_process(state: dict): """Match a modern process incarnation, never just its reusable PID.""" try: pid, created = state["pid"], state["create_time"] owner_pid, owner_created = state["owner_pid"], state["owner_create_time"] exe = state["executable"] if (not _valid_pid(pid) or not _valid_birth(created) or not _valid_pid(owner_pid) or not _valid_birth(owner_created) or owner_created > created or not isinstance(exe, str) or not exe): return None proc = psutil.Process(pid) if (not proc.is_running() or not _same_birth(proc, state) or Path(proc.exe()) != Path(exe)): return None # Windows retains the original parent PID; POSIX reparents orphans. if os.name == "nt" and proc.ppid() != owner_pid: return None parent = proc.parent() if parent is not None: if parent.pid == owner_pid: if not _same_birth(parent, state, "owner_"): return None elif os.name == "nt" or not _owner_is_dead(state): return None # POSIX may reparent a router whose recorded owner exited. return proc except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error): return None def _owner_is_dead(state: dict) -> bool: try: pid, created = state["owner_pid"], state["owner_create_time"] if not _valid_pid(pid) or not _valid_birth(created): return False try: owner = psutil.Process(pid) if not owner.is_running(): return True if _valid_birth(state.get("owner_start_time")): return not _same_birth(owner, state, "owner_") # Records that predate start_time: only a NEWER incarnation proves the owner exited. A # stepped clock (WSL) moves create_time, so inequality alone would bury a live owner. return owner.create_time() > created except psutil.NoSuchProcess: return True except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error): return False def legacy_recorded_process(state: dict): """Match an older record only to the managed binary and its exact launch arguments. Legacy records lack process birth identity. A live PID alone is not evidence because the OS can reuse it after llama-server exits. The executable location, state-file age, endpoint key, and launch arguments together keep old records usable without adopting an unrelated process. """ from urllib.parse import urlsplit from hermes_cli.local_runtime.bootstrap import models_dir from hermes_cli.local_runtime.supervisor import state_path # A damaged new record must not fall back to weaker legacy evidence. if is_modern(state): return None try: if not _valid_pid(state["pid"]): return None proc = psutil.Process(state["pid"]) root = state_path().parent exe = Path(proc.exe()) exe.relative_to(root) if exe.name.lower() not in ("llama-server", "llama-server.exe"): return None if proc.create_time() > state_path().stat().st_mtime: return None # the PID was reused after this record was written url = urlsplit(state["base_url"]) if url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port or not state["api_key"]: return None argv = proc.cmdline() def value(flag): if argv.count(flag) != 1: return None index = argv.index(flag) + 1 return argv[index] if index < len(argv) else None if (value("--host") != "127.0.0.1" or value("--port") != str(url.port) or value("--api-key") != state["api_key"]): return None preset, directory = value("--models-preset"), value("--models-dir") if not ((preset and Path(preset) == root / "presets.ini" and directory is None) or (directory and Path(directory) == models_dir() and preset is None)): return None return proc except (KeyError, TypeError, ValueError, OSError, psutil.Error): return None def _legacy_orphan_process(state: dict): """Windows-only legacy stop recovery after the process identity is established.""" from hermes_cli.local_runtime.supervisor import state_path if os.name != "nt": return None proc = legacy_recorded_process(state) if proc is None: return None try: relative = Path(proc.exe()).relative_to(state_path().parent) if len(relative.parts) != 3 or relative.name.lower() != "llama-server.exe": return None if proc.parent() is not None or proc.ppid() <= 0: return None return proc except (OSError, psutil.Error): return None def stop_recorded_orphan() -> bool: """Explicit user stop only. Refuse uncertain identity or a living recorded owner.""" from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor, state_path try: state = json.loads(state_path().read_text(encoding="utf-8-sig")) except FileNotFoundError: return True # already stopped and no record except (OSError, ValueError): return False if not isinstance(state, dict) or not _valid_pid(state.get("pid")): return False try: if not psutil.pid_exists(state["pid"]): return True # keep the dead record; endpoint resolution rejects it if is_modern(state): proc = recorded_process(state) if proc is None or not _owner_is_dead(state): return False else: proc = _legacy_orphan_process(state) if proc is None or read_state() != state or not proc.is_running(): return False # Retain the psutil incarnation object: destructive methods guard PID reuse. LlamaServerSupervisor._terminate_tree(proc, verified_root=True) proc.wait(timeout=5) logger.info("stopped orphaned managed llama-server pid=%s", proc.pid) return True except (OSError, psutil.Error): logger.warning("could not verify or stop recorded managed llama-server") return False