recovery.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. """Conservative recovery of the one router recorded by this managed runtime."""
  2. from __future__ import annotations
  3. import json
  4. import logging
  5. import math
  6. import os
  7. from pathlib import Path
  8. import psutil
  9. logger = logging.getLogger(__name__)
  10. def read_state() -> dict:
  11. from hermes_cli.local_runtime.supervisor import state_path
  12. try:
  13. state = json.loads(state_path().read_text(encoding="utf-8-sig"))
  14. return state if isinstance(state, dict) else {}
  15. except (OSError, ValueError):
  16. return {}
  17. _MODERN_FIELDS = ("create_time", "executable", "owner_pid", "owner_create_time")
  18. def is_modern(state: dict) -> bool:
  19. return any(key in state for key in _MODERN_FIELDS)
  20. def _valid_pid(value) -> bool:
  21. return type(value) is int and value > 0
  22. def _valid_birth(value) -> bool:
  23. return type(value) in (int, float) and math.isfinite(value) and value > 0
  24. def _same_birth(proc, state: dict, prefix: str = "") -> bool:
  25. """Linux derives psutil's create_time from /proc/stat's boot time, which moves whenever the
  26. clock is stepped (WSL re-syncs within minutes), so a live process stops matching its record.
  27. The start_time fingerprint is boot-relative there; create_time is for records that predate it."""
  28. from gateway.status import get_process_start_time, start_time_fingerprints_match
  29. recorded = state.get(f"{prefix}start_time")
  30. if not _valid_birth(recorded):
  31. return proc.create_time() == state[f"{prefix}create_time"]
  32. current = get_process_start_time(proc.pid)
  33. return current is not None and start_time_fingerprints_match(recorded, current)
  34. def recorded_process(state: dict):
  35. """Match a modern process incarnation, never just its reusable PID."""
  36. try:
  37. pid, created = state["pid"], state["create_time"]
  38. owner_pid, owner_created = state["owner_pid"], state["owner_create_time"]
  39. exe = state["executable"]
  40. if (not _valid_pid(pid) or not _valid_birth(created)
  41. or not _valid_pid(owner_pid) or not _valid_birth(owner_created)
  42. or owner_created > created or not isinstance(exe, str) or not exe):
  43. return None
  44. proc = psutil.Process(pid)
  45. if (not proc.is_running() or not _same_birth(proc, state)
  46. or Path(proc.exe()) != Path(exe)):
  47. return None
  48. # Windows retains the original parent PID; POSIX reparents orphans.
  49. if os.name == "nt" and proc.ppid() != owner_pid:
  50. return None
  51. parent = proc.parent()
  52. if parent is not None:
  53. if parent.pid == owner_pid:
  54. if not _same_birth(parent, state, "owner_"):
  55. return None
  56. elif os.name == "nt" or not _owner_is_dead(state):
  57. return None # POSIX may reparent a router whose recorded owner exited.
  58. return proc
  59. except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
  60. return None
  61. def _owner_is_dead(state: dict) -> bool:
  62. try:
  63. pid, created = state["owner_pid"], state["owner_create_time"]
  64. if not _valid_pid(pid) or not _valid_birth(created):
  65. return False
  66. try:
  67. owner = psutil.Process(pid)
  68. if not owner.is_running():
  69. return True
  70. if _valid_birth(state.get("owner_start_time")):
  71. return not _same_birth(owner, state, "owner_")
  72. # Records that predate start_time: only a NEWER incarnation proves the owner exited. A
  73. # stepped clock (WSL) moves create_time, so inequality alone would bury a live owner.
  74. return owner.create_time() > created
  75. except psutil.NoSuchProcess:
  76. return True
  77. except (KeyError, TypeError, ValueError, OverflowError, OSError, psutil.Error):
  78. return False
  79. def legacy_recorded_process(state: dict):
  80. """Match an older record only to the managed binary and its exact launch arguments.
  81. Legacy records lack process birth identity. A live PID alone is not evidence because the OS
  82. can reuse it after llama-server exits. The executable location, state-file age, endpoint key,
  83. and launch arguments together keep old records usable without adopting an unrelated process.
  84. """
  85. from urllib.parse import urlsplit
  86. from hermes_cli.local_runtime.bootstrap import models_dir
  87. from hermes_cli.local_runtime.supervisor import state_path
  88. # A damaged new record must not fall back to weaker legacy evidence.
  89. if is_modern(state):
  90. return None
  91. try:
  92. if not _valid_pid(state["pid"]):
  93. return None
  94. proc = psutil.Process(state["pid"])
  95. root = state_path().parent
  96. exe = Path(proc.exe())
  97. exe.relative_to(root)
  98. if exe.name.lower() not in ("llama-server", "llama-server.exe"):
  99. return None
  100. if proc.create_time() > state_path().stat().st_mtime:
  101. return None # the PID was reused after this record was written
  102. url = urlsplit(state["base_url"])
  103. if url.scheme != "http" or url.hostname != "127.0.0.1" or not url.port or not state["api_key"]:
  104. return None
  105. argv = proc.cmdline()
  106. def value(flag):
  107. if argv.count(flag) != 1:
  108. return None
  109. index = argv.index(flag) + 1
  110. return argv[index] if index < len(argv) else None
  111. if (value("--host") != "127.0.0.1" or value("--port") != str(url.port)
  112. or value("--api-key") != state["api_key"]):
  113. return None
  114. preset, directory = value("--models-preset"), value("--models-dir")
  115. if not ((preset and Path(preset) == root / "presets.ini" and directory is None)
  116. or (directory and Path(directory) == models_dir() and preset is None)):
  117. return None
  118. return proc
  119. except (KeyError, TypeError, ValueError, OSError, psutil.Error):
  120. return None
  121. def _legacy_orphan_process(state: dict):
  122. """Windows-only legacy stop recovery after the process identity is established."""
  123. from hermes_cli.local_runtime.supervisor import state_path
  124. if os.name != "nt":
  125. return None
  126. proc = legacy_recorded_process(state)
  127. if proc is None:
  128. return None
  129. try:
  130. relative = Path(proc.exe()).relative_to(state_path().parent)
  131. if len(relative.parts) != 3 or relative.name.lower() != "llama-server.exe":
  132. return None
  133. if proc.parent() is not None or proc.ppid() <= 0:
  134. return None
  135. return proc
  136. except (OSError, psutil.Error):
  137. return None
  138. def stop_recorded_orphan() -> bool:
  139. """Explicit user stop only. Refuse uncertain identity or a living recorded owner."""
  140. from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor, state_path
  141. try:
  142. state = json.loads(state_path().read_text(encoding="utf-8-sig"))
  143. except FileNotFoundError:
  144. return True # already stopped and no record
  145. except (OSError, ValueError):
  146. return False
  147. if not isinstance(state, dict) or not _valid_pid(state.get("pid")):
  148. return False
  149. try:
  150. if not psutil.pid_exists(state["pid"]):
  151. return True # keep the dead record; endpoint resolution rejects it
  152. if is_modern(state):
  153. proc = recorded_process(state)
  154. if proc is None or not _owner_is_dead(state):
  155. return False
  156. else:
  157. proc = _legacy_orphan_process(state)
  158. if proc is None or read_state() != state or not proc.is_running():
  159. return False
  160. # Retain the psutil incarnation object: destructive methods guard PID reuse.
  161. LlamaServerSupervisor._terminate_tree(proc, verified_root=True)
  162. proc.wait(timeout=5)
  163. logger.info("stopped orphaned managed llama-server pid=%s", proc.pid)
  164. return True
  165. except (OSError, psutil.Error):
  166. logger.warning("could not verify or stop recorded managed llama-server")
  167. return False