local_server.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418
  1. """Process ownership and lifecycle for one profile's Quick Local server."""
  2. from __future__ import annotations
  3. import hashlib
  4. import json
  5. import os
  6. import time
  7. from contextlib import contextmanager
  8. from dataclasses import dataclass
  9. from pathlib import Path
  10. import psutil
  11. from utils import atomic_json_write
  12. @dataclass(frozen=True)
  13. class StartedServer:
  14. endpoint: str
  15. process: object | None
  16. reused: bool
  17. created: float | None = None
  18. class _RecordedProcess:
  19. def __init__(self, process):
  20. self.process = process
  21. self.created = process.create_time()
  22. self.pid = process.pid
  23. def poll(self):
  24. try:
  25. if self.process.create_time() != self.created:
  26. return -1
  27. return (
  28. None
  29. if self.process.is_running() and self.process.status() != psutil.STATUS_ZOMBIE
  30. else -1
  31. )
  32. except psutil.NoSuchProcess:
  33. return -1
  34. def _wait_stopped(processes, timeout):
  35. # A separate CLI cannot reap Hermes's children. A zombie has already
  36. # stopped; wait_procs() can otherwise keep reporting it as alive.
  37. deadline = time.monotonic() + timeout
  38. while True:
  39. alive = []
  40. for process in processes:
  41. try:
  42. if process.is_running() and process.status() != psutil.STATUS_ZOMBIE:
  43. alive.append(process)
  44. except psutil.NoSuchProcess:
  45. pass
  46. if not alive or time.monotonic() >= deadline:
  47. return alive
  48. time.sleep(min(0.05, max(0, deadline - time.monotonic())))
  49. processes = alive
  50. def stop_process_tree(process, *, timeout=5):
  51. """Stop an owned parent and its existing children, including Windows launchers."""
  52. from .quick_local import QuickLocalSetupError
  53. try:
  54. processes = [*process.children(recursive=True), process]
  55. for child in processes:
  56. try:
  57. child.terminate()
  58. except psutil.NoSuchProcess:
  59. pass
  60. alive = _wait_stopped(processes, timeout)
  61. for child in alive:
  62. try:
  63. child.kill()
  64. except psutil.NoSuchProcess:
  65. pass
  66. alive = _wait_stopped(alive, timeout)
  67. if alive:
  68. raise QuickLocalSetupError(
  69. "Quick Local could not stop its server. Review the server log."
  70. )
  71. except psutil.NoSuchProcess:
  72. pass
  73. class LocalServer:
  74. """Never stop a process unless its identity and profile config match."""
  75. def __init__(self, hermes_home: Path):
  76. from . import quick_local as ql
  77. self.ql = ql
  78. self.paths = ql.managed_paths(hermes_home)
  79. self.record_path = self.paths.root / "server-process.json"
  80. @contextmanager
  81. def locked(self, timeout=10.0):
  82. ql = self.ql
  83. ql.repair_private_paths(self.paths)
  84. path = self.paths.root / "server.lock"
  85. if path.is_symlink():
  86. raise ql.QuickLocalSetupError("Quick Local lock must belong to this profile.")
  87. with path.open("a+b") as file:
  88. if os.name != "nt":
  89. path.chmod(0o600)
  90. file.seek(0, 2)
  91. if not file.tell():
  92. file.write(b"\0")
  93. file.flush()
  94. deadline = time.monotonic() + timeout
  95. while True:
  96. try:
  97. file.seek(0)
  98. if os.name == "nt":
  99. import msvcrt
  100. msvcrt.locking(file.fileno(), msvcrt.LK_NBLCK, 1)
  101. else:
  102. import fcntl
  103. fcntl.flock(file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  104. break
  105. except OSError:
  106. if time.monotonic() >= deadline:
  107. raise ql.QuickLocalSetupError(
  108. "Quick Local is busy in another process. Retry shortly."
  109. )
  110. time.sleep(0.05)
  111. try:
  112. yield
  113. finally:
  114. file.seek(0)
  115. if os.name == "nt":
  116. msvcrt.locking(file.fileno(), msvcrt.LK_UNLCK, 1)
  117. else:
  118. fcntl.flock(file.fileno(), fcntl.LOCK_UN)
  119. def _config(self):
  120. self.ql.connection_config({}, self.paths.root.parent)
  121. return json.loads(self.paths.server_config.read_text(encoding="utf-8"))
  122. def _fingerprint(self, config):
  123. return hashlib.sha256(json.dumps(config, sort_keys=True).encode()).hexdigest()
  124. def _read_record(self):
  125. try:
  126. value = json.loads(self.record_path.read_text(encoding="utf-8"))
  127. return value if isinstance(value, dict) else {}
  128. except (OSError, ValueError):
  129. return {}
  130. def _restart_stamp(self):
  131. try:
  132. return self.paths.restart_required_marker.stat().st_mtime_ns
  133. except FileNotFoundError:
  134. return None
  135. def _matches_desired(self, record, config):
  136. return (
  137. record.get("fingerprint") == self._fingerprint(config)
  138. and self.ql._paths_equivalent(record.get("command"), self.paths.server_command)
  139. and (
  140. self._restart_stamp() is None
  141. or record.get("restart_stamp") == self._restart_stamp()
  142. )
  143. )
  144. def _verified_process(self, record):
  145. try:
  146. process = psutil.Process(record["pid"])
  147. if (
  148. process.create_time() != record["created"]
  149. or process.status() == psutil.STATUS_ZOMBIE
  150. ):
  151. return None
  152. args = process.cmdline()
  153. config = args[args.index("--config") + 1]
  154. command = Path(record["command"]).absolute()
  155. # PM generations can change during an upgrade. The old command must
  156. # still be inside this profile's private runtime, never another one.
  157. command.relative_to(self.paths.runtime.absolute())
  158. if not self.ql._paths_equivalent(config, self.paths.server_config):
  159. return None
  160. if not any(self.ql._paths_equivalent(arg, command) for arg in args[:2]):
  161. return None
  162. if args[args.index("--port") + 1] != str(record["port"]):
  163. return None
  164. return process
  165. except (psutil.Error, KeyError, ValueError, IndexError, TypeError, OSError):
  166. return None
  167. def _adopt(self, config, endpoint):
  168. # Older Quick Local builds did not save a process record. Authenticated
  169. # ownership plus an exact executable/config match permits adoption.
  170. if not self.ql.server_belongs_to_profile(self.paths, endpoint):
  171. return None
  172. for process in psutil.process_iter(["pid", "create_time", "cmdline"]):
  173. try:
  174. args = process.info["cmdline"] or []
  175. # A runtime upgrade may select a new PM generation before
  176. # setup recovers the OLD server's lost record. Adopt only an
  177. # OpenViking entrypoint inside this profile's private runtime.
  178. command = next(
  179. (
  180. arg
  181. for arg in args[:2]
  182. if Path(arg).name
  183. in {"openviking-server", "openviking-server.exe", "openviking-server.cmd"}
  184. and Path(arg).absolute().is_relative_to(self.paths.runtime.absolute())
  185. ),
  186. None,
  187. )
  188. if command is None:
  189. continue
  190. record = {
  191. "pid": process.pid,
  192. "created": process.info["create_time"],
  193. "command": command,
  194. "port": self.ql._endpoint_port(endpoint),
  195. "fingerprint": self._fingerprint(config),
  196. }
  197. if "--config" in args and self._verified_process(record):
  198. atomic_json_write(self.record_path, record, mode=0o600)
  199. return process
  200. except psutil.Error:
  201. continue
  202. return None
  203. def _stop(self, process):
  204. stop_process_tree(process)
  205. def configure(self, config, *, runtime_changed=False):
  206. with self.locked():
  207. changed = not self.ql._stored_server_config_matches(self.paths, config)
  208. if (changed or runtime_changed) and self.paths.server_config.is_file():
  209. # Authenticate/adopt against the OLD config. Publishing a new
  210. # key first makes a lost process record impossible to recover.
  211. # Setup can repair an incomplete ovcli file; only the old
  212. # server config/key is needed for authenticated adoption.
  213. old_config = json.loads(self.paths.server_config.read_text(encoding="utf-8"))
  214. endpoint = self.ql._configured_endpoint(self.paths)
  215. process = self._verified_process(self._read_record())
  216. if process is None:
  217. process = self._adopt(old_config, endpoint)
  218. if process is None and self.ql.server_belongs_to_profile(self.paths, endpoint):
  219. raise self.ql.QuickLocalSetupError(
  220. "Quick Local cannot verify the server process. Existing settings were retained. "
  221. "Stop that server manually, then retry."
  222. )
  223. if changed or runtime_changed:
  224. self.ql._mark_server_restart_required(self.paths)
  225. atomic_json_write(self.paths.server_config, config, mode=0o600)
  226. self.ql._write_ovcli_profile(
  227. self.paths.ovcli_config, f"http://127.0.0.1:{config['server']['port']}", config
  228. )
  229. def is_running(self, endpoint):
  230. """Cheap owned-process check for a retained provider's cached client."""
  231. record = self._read_record()
  232. if record.get("port") == self.ql._endpoint_port(endpoint) and self._verified_process(
  233. record
  234. ):
  235. return True
  236. # A server started outside the controller may have no record. Keep
  237. # serving from it while its key still proves profile ownership.
  238. return self.ql.server_belongs_to_profile(self.paths, endpoint)
  239. def start(self, *, restart=False):
  240. ql = self.ql
  241. with self.locked():
  242. config = self._config()
  243. endpoint = ql._configured_endpoint(self.paths)
  244. record = self._read_record()
  245. process = self._verified_process(record)
  246. if process is None:
  247. process = self._adopt(config, endpoint)
  248. record = self._read_record()
  249. changed = (
  250. not self._matches_desired(record, config)
  251. if process is not None
  252. else self.paths.restart_required_marker.is_file()
  253. )
  254. if process is not None:
  255. if not restart and not changed:
  256. return StartedServer(
  257. endpoint, _RecordedProcess(process), True, process.create_time()
  258. )
  259. self._stop(process)
  260. elif ql.server_belongs_to_profile(self.paths, endpoint):
  261. if changed or restart:
  262. raise ql.QuickLocalSetupError(
  263. "Quick Local cannot verify the server process. Stop that server manually, then retry."
  264. )
  265. return StartedServer(endpoint, None, True)
  266. self.record_path.unlink(missing_ok=True)
  267. port = ql.find_available_port(preferred_endpoint=endpoint)
  268. if port is None:
  269. raise ql.QuickLocalSetupError("No free Quick Local port was found (1934-1953).")
  270. endpoint = f"http://127.0.0.1:{port}"
  271. config["server"]["port"] = port
  272. atomic_json_write(self.paths.server_config, config, mode=0o600)
  273. ql._write_ovcli_profile(self.paths.ovcli_config, endpoint, config)
  274. process = ql._start_validation_server(
  275. endpoint,
  276. self.paths.server_config,
  277. self.paths.root.parent,
  278. self.paths.server_command,
  279. )
  280. try:
  281. created = psutil.Process(process.pid).create_time()
  282. atomic_json_write(
  283. self.record_path,
  284. {
  285. "pid": process.pid,
  286. "created": created,
  287. "command": str(self.paths.server_command),
  288. "port": port,
  289. "restart_stamp": self._restart_stamp(),
  290. "fingerprint": self._fingerprint(config),
  291. },
  292. mode=0o600,
  293. )
  294. except BaseException:
  295. ql._stop_process(process)
  296. raise
  297. return StartedServer(endpoint, process, False, created)
  298. def wait_ready(self, started, health_check, *, should_stop=lambda: False, timeout=None):
  299. """Retry a port race after an early exit, without stopping another server."""
  300. ql = self.ql
  301. deadline = time.monotonic() + (ql._HEALTH_TIMEOUT_SECONDS if timeout is None else timeout)
  302. for attempt in range(3):
  303. while time.monotonic() < deadline and not should_stop():
  304. if started.process is not None and started.process.poll() is not None:
  305. break
  306. healthy, _message = health_check(started.endpoint)
  307. if healthy and ql.server_belongs_to_profile(self.paths, started.endpoint):
  308. with self.locked():
  309. # Do not clear a restart requested by a concurrent setup.
  310. record = self._read_record()
  311. current = self._matches_desired(record, self._config())
  312. if started.process is not None:
  313. current = (
  314. current
  315. and record.get("pid") == started.process.pid
  316. and record.get("created") == started.created
  317. )
  318. if current:
  319. ql.clear_server_restart_required(self.paths.server_config)
  320. elif (
  321. not self.paths.restart_required_marker.is_file()
  322. and started.process is None
  323. ):
  324. current = True
  325. if current:
  326. return started
  327. # Setup changed the saved config during this health probe.
  328. # Reconcile it before publishing a stale running server.
  329. started = self.start()
  330. continue
  331. time.sleep(ql._HEALTH_POLL_INTERVAL_SECONDS)
  332. if should_stop():
  333. return None
  334. host, port = ql._endpoint_bind(started.endpoint)
  335. if (
  336. attempt == 2
  337. or ql._can_bind_local_port(host, port)
  338. or ql.server_belongs_to_profile(self.paths, started.endpoint)
  339. ):
  340. break
  341. started = self.start()
  342. # A failed start must not leave a hung child behind. Recheck the record
  343. # under the lock so a newer start by another caller is never stopped.
  344. with self.locked():
  345. record = self._read_record()
  346. if (
  347. started.process is not None
  348. and record.get("pid") == started.process.pid
  349. and record.get("created") == started.created
  350. ):
  351. process = self._verified_process(record)
  352. if process is not None:
  353. self._stop(process)
  354. self.record_path.unlink(missing_ok=True)
  355. raise ql.QuickLocalSetupError(
  356. f"Quick Local did not become ready. Review {ql._server_log_path(self.paths.root.parent)} and retry."
  357. )
  358. def stop(self):
  359. with self.locked():
  360. config = self._config()
  361. endpoint = self.ql._configured_endpoint(self.paths)
  362. process = self._verified_process(self._read_record()) or self._adopt(config, endpoint)
  363. if process is None:
  364. if self.ql.server_belongs_to_profile(self.paths, endpoint):
  365. raise self.ql.QuickLocalSetupError(
  366. "Quick Local cannot verify the server process; it was not stopped."
  367. )
  368. return False
  369. self._stop(process)
  370. self.record_path.unlink(missing_ok=True)
  371. return True
  372. def status(self):
  373. with self.locked():
  374. config = self._config()
  375. endpoint = self.ql._configured_endpoint(self.paths)
  376. process = self._verified_process(self._read_record()) or self._adopt(config, endpoint)
  377. ready = self.ql.server_belongs_to_profile(self.paths, endpoint)
  378. return {
  379. "endpoint": endpoint,
  380. "pid": process.pid if process else None,
  381. "state": "ready" if ready else "starting" if process else "stopped",
  382. "restart_required": self.paths.restart_required_marker.is_file(),
  383. }