status.py 102 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105
  1. """Gateway runtime status helpers: PID/lock/marker files under ``{HERMES_HOME}`` (one set per
  2. home/profile) that tell whether the gateway daemon is running."""
  3. import asyncio
  4. import contextlib
  5. import copy
  6. import hashlib
  7. import json
  8. import logging
  9. import math
  10. import os
  11. import re
  12. import shlex
  13. import signal
  14. import subprocess
  15. import sys
  16. import threading
  17. import time
  18. from dataclasses import dataclass
  19. from datetime import datetime, timezone, UTC
  20. from pathlib import Path
  21. from typing import Any, Callable, Iterable, NamedTuple, Optional
  22. from hermes_dir_acl import make_user_dir
  23. from gateway.status_inline_source import (
  24. command_line_runs_inline_source,
  25. inline_bootstrap_argv,
  26. inline_source_flag_index,
  27. )
  28. from hermes_constants import _get_platform_default_hermes_home, get_hermes_home, get_process_hermes_home
  29. from hermes_cli._subprocess_compat import pid_exists_stdlib
  30. from gateway.status_lock_dir import unusable_lock_dir, unusable_lock_dir_recovered
  31. from utils import atomic_json_write
  32. if sys.platform == "win32":
  33. import msvcrt
  34. else:
  35. import fcntl
  36. _GATEWAY_KIND = "hermes-gateway"
  37. _RUNTIME_STATUS_FILE = "gateway_state.json"
  38. _LOCKS_DIRNAME = "gateway-locks"
  39. _IS_WINDOWS = sys.platform == "win32"
  40. _UNSET = object()
  41. _GATEWAY_LOCK_FILENAME = "gateway.lock"
  42. _gateway_lock_handle = None
  43. # Windows byte-range locks are mandatory for other readers: lock a byte well past
  44. # the JSON payload so status/PID readers can read while another process holds it.
  45. _WINDOWS_LOCK_OFFSET = 1024 * 1024
  46. _GATEWAY_RUNNING_PID_CACHE_TTL_SECONDS = 1.0
  47. _gateway_running_pid_cache_lock = threading.Lock()
  48. # key: (pid_path, cleanup_stale, include_runtime_status) -> (cached_at, file signature, pid)
  49. _gateway_running_pid_cache: dict[tuple[str, bool, bool], tuple[float, tuple, Optional[int]]] = {}
  50. logger = logging.getLogger(__name__)
  51. class _RuntimeStatusWriter:
  52. """Persist the latest complete status snapshot on one daemon thread.
  53. Runtime status is diagnostic state, not an event log. While one write is
  54. blocked in filesystem I/O, newer submissions replace the single pending
  55. snapshot. This bounds memory and keeps every status producer off asyncio.
  56. """
  57. def __init__(self, write_fn: Optional[Callable[[Path, dict[str, Any]], None]] = None):
  58. self._write_fn = write_fn
  59. self._condition = threading.Condition()
  60. self._pending: Optional[tuple[int, Path, dict[str, Any]]] = None
  61. self._writing_generation = 0
  62. self._submitted_generation = 0
  63. self._completed_generation = 0
  64. self._successful_generation = 0
  65. self._last_error: Optional[BaseException] = None
  66. self._failure_logged = False
  67. self._thread: Optional[threading.Thread] = None
  68. def submit(self, path: Path, payload: dict[str, Any]) -> int:
  69. with self._condition:
  70. self._submitted_generation += 1
  71. generation = self._submitted_generation
  72. self._pending = (generation, path, copy.deepcopy(payload))
  73. if self._thread is None or not self._thread.is_alive():
  74. self._thread = threading.Thread(
  75. target=self._run, daemon=True, name="gateway-runtime-status-writer")
  76. self._thread.start()
  77. self._condition.notify_all()
  78. return generation
  79. def wait(self, generation: int, timeout: Optional[float] = None) -> bool:
  80. """Block until ``generation`` (or a later snapshot) is persisted; ``False`` on timeout/failure."""
  81. deadline = None if timeout is None else time.monotonic() + max(timeout, 0.0)
  82. with self._condition:
  83. while (state := self.settled(generation)) is None:
  84. if deadline is None:
  85. self._condition.wait()
  86. continue
  87. remaining = deadline - time.monotonic()
  88. if remaining <= 0:
  89. return False
  90. self._condition.wait(timeout=remaining)
  91. if not state and timeout is None and self._last_error is not None:
  92. raise self._last_error
  93. return state
  94. def flush(self, timeout: float = 2.0) -> bool:
  95. with self._condition:
  96. generation = self._submitted_generation
  97. return generation == 0 or self.wait(generation, timeout=timeout)
  98. def settled(self, generation: int) -> Optional[bool]:
  99. """``True`` once ``generation`` persisted, ``False`` once it can no longer, else ``None``."""
  100. with self._condition:
  101. if self._successful_generation >= generation:
  102. return True
  103. no_more_work = (
  104. self._completed_generation >= generation
  105. and self._writing_generation == 0
  106. and self._pending is None
  107. )
  108. return False if no_more_work else None
  109. def _run(self) -> None:
  110. while True:
  111. with self._condition:
  112. while self._pending is None:
  113. self._condition.wait()
  114. generation, path, payload = self._pending
  115. self._pending = None
  116. self._writing_generation = generation
  117. error: Optional[BaseException] = None
  118. try:
  119. (self._write_fn or _write_json_file)(path, _merge_over_on_disk(path, payload))
  120. except BaseException as exc:
  121. error = exc
  122. with self._condition:
  123. self._writing_generation = 0
  124. self._completed_generation = max(self._completed_generation, generation)
  125. if error is None:
  126. self._successful_generation = max(self._successful_generation, generation)
  127. self._last_error = None
  128. else:
  129. self._last_error = error
  130. self._condition.notify_all()
  131. if error is None:
  132. if self._failure_logged:
  133. logger.info("Gateway runtime-status persistence recovered")
  134. self._failure_logged = False
  135. elif not self._failure_logged:
  136. logger.warning(
  137. "Failed to persist gateway runtime status; later updates will retry: %s", error)
  138. self._failure_logged = True
  139. else:
  140. logger.debug("Failed to persist gateway runtime status: %s", error)
  141. _runtime_status_state_lock = threading.RLock()
  142. _runtime_status_state_path: Optional[Path] = None
  143. _runtime_status_state: Optional[dict[str, Any]] = None
  144. def _merge_over_on_disk(path: Path, payload: dict[str, Any]) -> dict[str, Any]:
  145. """Lay the canonical snapshot over whatever is on disk right before writing. Out-of-process
  146. writers (the migration's compensator clearing multiplex-owned status, container_boot
  147. seeding ``desired_state``) stamp this file directly; the gateway's fields win, theirs survive."""
  148. existing = _read_json_file(path)
  149. return {**existing, **payload} if isinstance(existing, dict) else payload
  150. _runtime_status_writer: Optional[_RuntimeStatusWriter] = None
  151. def _get_runtime_status_writer() -> _RuntimeStatusWriter:
  152. """Lazily create the single writer; callers serialise on ``_runtime_status_state_lock``."""
  153. global _runtime_status_writer
  154. with _runtime_status_state_lock:
  155. if _runtime_status_writer is None:
  156. _runtime_status_writer = _RuntimeStatusWriter()
  157. return _runtime_status_writer
  158. def flush_runtime_status(timeout: float = 2.0) -> bool:
  159. """Wait boundedly for all runtime-status updates submitted so far."""
  160. writer = _runtime_status_writer
  161. return True if writer is None else writer.flush(timeout=timeout)
  162. async def flush_runtime_status_async(timeout: float = 2.0) -> bool:
  163. """Await the current writer generation without blocking the event loop."""
  164. writer = _runtime_status_writer
  165. if writer is None:
  166. return True
  167. # ``flush()`` returns at its deadline, so the worker thread is bounded.
  168. return await asyncio.to_thread(writer.flush, timeout=max(float(timeout), 0.0))
  169. class StormInfo(NamedTuple):
  170. """Respawn-storm check result: start count, window, and backoff to sleep."""
  171. count: int
  172. window_s: float
  173. backoff_s: float
  174. def record_start_and_check_storm(
  175. max_starts: int = 5, window_s: float = 120.0, *, backoff_cap_s: float = 300.0
  176. ) -> Optional[StormInfo]:
  177. """Record this start; :class:`StormInfo` when > ``max_starts`` landed in ``window_s``.
  178. Best-effort: a broken ``gateway-starts.log`` ledger is logged and swallowed, never fatal."""
  179. try:
  180. path = get_hermes_home() / "gateway-starts.log"
  181. make_user_dir(path.parent, parents=True, exist_ok=True)
  182. now = datetime.now(UTC).timestamp()
  183. existing: list[float] = []
  184. if path.exists():
  185. for line in path.read_text(encoding="utf-8-sig").splitlines():
  186. line = line.strip()
  187. if not line:
  188. continue
  189. try:
  190. existing.append(float(line))
  191. except ValueError:
  192. pass # corrupt/truncated ledger line — skip, never fatal
  193. existing.append(now)
  194. recent = [ts for ts in existing if now - ts <= window_s]
  195. # Ring-buffer the persisted file so it stays bounded.
  196. to_write = existing[-max(max_starts * 4, 40):]
  197. tmp = path.with_suffix(".tmp")
  198. tmp.write_text("\n".join(repr(ts) for ts in to_write) + "\n", encoding="utf-8")
  199. os.replace(tmp, path)
  200. if len(recent) <= max_starts:
  201. return None
  202. backoff = min(backoff_cap_s, 5.0 * (2 ** min(len(recent) - max_starts, 6)))
  203. return StormInfo(count=len(recent), window_s=window_s, backoff_s=backoff)
  204. except Exception as _e:
  205. logger.debug("respawn-storm breaker bookkeeping failed (non-fatal): %s", _e)
  206. return None
  207. def _get_process_hermes_home() -> Path:
  208. """Launch-home HERMES_HOME for identity files (PID, lock, status, markers):
  209. ``get_hermes_home()`` honors the per-session ``_HERMES_HOME_OVERRIDE`` and would misroute
  210. them."""
  211. return get_process_hermes_home()
  212. def _canonical_hermes_home(path: Path | str) -> Path:
  213. """Stable absolute HERMES_HOME path for persisted identity data."""
  214. return Path(path).expanduser().resolve(strict=False)
  215. def _same_hermes_home(left: Path | str, right: Path | str) -> bool:
  216. """Compare HERMES_HOME paths with the host platform's case semantics."""
  217. left_c = os.path.normcase(str(_canonical_hermes_home(left)))
  218. return left_c == os.path.normcase(str(_canonical_hermes_home(right)))
  219. def recorded_gateway_home_conflicts(
  220. record: Optional[dict[str, Any]], *, expected_home: Optional[Path | str] = None
  221. ) -> bool:
  222. """True when a persisted gateway record names a DIFFERENT HERMES_HOME (cross-profile kill guard:
  223. profile B's stop must never SIGTERM profile A). ``expected_home`` overrides the comparison base.
  224. Legacy records without ``hermes_home`` prove nothing -> False; a comparison failure fails
  225. closed -> True."""
  226. recorded_home = record.get("hermes_home") if isinstance(record, dict) else None
  227. if not isinstance(recorded_home, str) or not recorded_home.strip():
  228. return False
  229. try:
  230. base = expected_home if expected_home is not None else _get_process_hermes_home()
  231. return not _same_hermes_home(recorded_home, base)
  232. except Exception:
  233. return True
  234. # Mirrors hermes_cli.profiles._PROFILE_ID_RE -- duplicated so gateway identity code
  235. # stays import-light (hermes_constants + stdlib only).
  236. _PROFILE_LABEL_RE = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
  237. def _profile_label_for_home(home: Path | str) -> Optional[str]:
  238. """Best-effort label: ``<root>/profiles/<name>`` -> name, root home -> "default", else None."""
  239. try:
  240. canonical = _canonical_hermes_home(home)
  241. except Exception:
  242. return None
  243. if canonical.parent.name == "profiles" and _PROFILE_LABEL_RE.match(canonical.name):
  244. return canonical.name
  245. import hermes_constants
  246. default_homes = (hermes_constants.get_default_hermes_root, _get_platform_default_hermes_home)
  247. for default_home in default_homes:
  248. with contextlib.suppress(Exception):
  249. if _same_hermes_home(canonical, default_home()):
  250. return "default"
  251. return None
  252. def scoped_lock_owner_label(record: Optional[dict[str, Any]]) -> Optional[str]:
  253. """Profile label of a scoped-lock owner (None: PID-only wording): the validated ``profile``
  254. field stamped by :func:`acquire_scoped_lock`, else inferred from ``hermes_home`` (old locks)."""
  255. if not isinstance(record, dict):
  256. return None
  257. profile = record.get("profile")
  258. if isinstance(profile, str) and _PROFILE_LABEL_RE.match(profile.strip()):
  259. return profile.strip()
  260. home = record.get("hermes_home")
  261. return _profile_label_for_home(home) if isinstance(home, str) and home.strip() else None
  262. def _get_pid_path() -> Path:
  263. return _get_process_hermes_home() / "gateway.pid"
  264. def _get_gateway_lock_path(pid_path: Optional[Path] = None) -> Path:
  265. return (pid_path or _get_pid_path()).with_name(_GATEWAY_LOCK_FILENAME)
  266. def _get_runtime_status_path() -> Path:
  267. return _get_process_hermes_home() / _RUNTIME_STATUS_FILE
  268. def _get_lock_dir() -> Path:
  269. """Cross-profile rendezvous dir for machine-local locks; ``HERMES_GATEWAY_LOCK_DIR`` overrides.
  270. Scope is the **OS user**, not the kernel host: separate users have separate ``$HOME``s,
  271. separate ``~/.hermes`` profile roots and separate credentials, so "one gateway per host"
  272. means "one per host per OS user". Holds the token-scoped locks (:func:`acquire_scoped_lock`)
  273. and the host-role lock + rendezvous record (``gateway/host_rendezvous.py``); the per-home
  274. ``gateway.pid``/``gateway.lock`` above deliberately stay under each profile's HERMES_HOME.
  275. """
  276. override = os.getenv("HERMES_GATEWAY_LOCK_DIR")
  277. if override:
  278. return Path(override)
  279. # XDG spec: a relative $XDG_STATE_HOME is INVALID and must be ignored. Honouring one made the
  280. # lock dir CWD-relative, so two serves started from different directories shared no singleton.
  281. state_home_env = os.getenv("XDG_STATE_HOME") or ""
  282. state_home = Path(state_home_env) if os.path.isabs(state_home_env) else Path.home() / ".local" / "state"
  283. return state_home / "hermes" / _LOCKS_DIRNAME
  284. def _utc_now_iso() -> str:
  285. return datetime.now(UTC).isoformat()
  286. # Epochs before 2000-01-01 are corrupt/hand-edited state (e.g. an accidental 0).
  287. _EPOCH_MIN_PLAUSIBLE = 946684800.0 # 2000-01-01T00:00:00Z
  288. def normalize_updated_at(value: Any) -> Optional[str]:
  289. """Coerce a persisted ``updated_at`` (ISO string, legacy epoch, hand edit, garbage) to the
  290. RFC3339 ``string | null`` that ``/api/status`` promises. ``str``: iff fromisoformat parses
  291. (trailing ``Z`` tolerated; naive -> UTC). Epoch: before 2000-01-01, > 1 day ahead or
  292. non-finite -> None. ``bool``/other -> None."""
  293. if isinstance(value, str):
  294. raw = value.strip()
  295. # Python < 3.11 fromisoformat rejects a trailing 'Z'; tolerate it.
  296. if raw.endswith(("Z", "z")):
  297. raw = raw[:-1] + "+00:00"
  298. try:
  299. parsed = datetime.fromisoformat(raw)
  300. except ValueError:
  301. return None
  302. return (parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)).isoformat()
  303. if isinstance(value, (int, float)) and not isinstance(value, bool):
  304. seconds = float(value)
  305. now = datetime.now(UTC).timestamp()
  306. if not math.isfinite(seconds) or seconds < _EPOCH_MIN_PLAUSIBLE or seconds > now + 86400:
  307. return None
  308. try:
  309. return datetime.fromtimestamp(seconds, tz=UTC).isoformat()
  310. except (OverflowError, OSError, ValueError):
  311. return None
  312. return None
  313. # ``exit_reason`` values the out-of-loop watchdogs (gateway/shutdown_watchdog.py) stamp together with
  314. # ``gateway_state: degraded`` right before they hard-exit a wedged process (#113372).
  315. WATCHDOG_EXIT_REASONS = frozenset({"loop_liveness_watchdog", "shutdown_watchdog"})
  316. def retained_gateway_state(runtime: Any) -> str:
  317. """What a NOT-running gateway's retained ``gateway_state.json`` says about it now:
  318. ``"startup_failed"`` (or a watchdog-stamped ``"degraded"``) only while the operator still
  319. wants it running, else ``"stopped"``.
  320. ``hermes gateway stop`` keeps the last ``startup_failed`` + ``exit_reason`` on disk for
  321. diagnostics and records the durable stop intent as ``desired_state``; a profile the operator
  322. stopped is "stopped", not a current failure. A watchdog exit (``degraded`` + an exit_reason in
  323. ``WATCHDOG_EXIT_REASONS``) is the same kind of current failure as ``startup_failed`` and is kept
  324. under the same rule, so the dashboard agrees with ``hermes gateway status``. Any other retained
  325. state of a dead process (``running``, ``starting``, missing) is just "stopped". Shared by
  326. ``/api/status`` and ``/api/messaging/platforms`` so the sidebar strip and the Channels page
  327. cannot disagree."""
  328. rt = runtime if isinstance(runtime, dict) else {}
  329. if rt.get("desired_state") != "stopped":
  330. if rt.get("gateway_state") == "startup_failed":
  331. return "startup_failed"
  332. if rt.get("gateway_state") == "degraded" and rt.get("exit_reason") in WATCHDOG_EXIT_REASONS:
  333. return "degraded"
  334. return "stopped"
  335. def terminate_pid(
  336. pid: int, *, force: bool = False, expected_start_time: Optional[float] = None
  337. ) -> None:
  338. """Terminate a PID; POSIX SIGTERM/SIGKILL, Windows taskkill /T /F for force. Identity guard:
  339. Windows ``force`` REQUIRES a matching ``expected_start_time`` (taskkill on a recycled PID has
  340. killed svchost.exe); POSIX optional, but a provided mismatch refuses the kill everywhere.
  341. On POSIX an expectation is optional, but when the caller provides one and it no longer matches the live
  342. process, the kill is refused on every platform — a mismatched fingerprint always means the PID was
  343. recycled. See #89614.
  344. """
  345. if force and (_IS_WINDOWS or expected_start_time is not None):
  346. if expected_start_time is None:
  347. raise OSError(f"refusing to force-kill PID {pid} without a process start-time guard")
  348. current_start_time = _get_process_start_time(pid)
  349. if current_start_time is None:
  350. raise OSError(f"refusing to force-kill PID {pid}; process start time is unavailable")
  351. try:
  352. if not _start_times_agree(current_start_time, expected_start_time):
  353. raise OSError(f"refusing to force-kill PID {pid}; process identity changed")
  354. except (TypeError, ValueError) as exc:
  355. raise OSError(f"refusing to force-kill PID {pid}; malformed start time") from exc
  356. if not (force and _IS_WINDOWS):
  357. os.kill(pid, signal.SIGTERM if not force else getattr(signal, "SIGKILL", signal.SIGTERM))
  358. return
  359. # Hide flags: a bare taskkill spawn from windowless pythonw.exe would flash a conhost window.
  360. from hermes_cli._subprocess_compat import windows_hide_flags
  361. try:
  362. result = subprocess.run(
  363. ["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True, text=True,
  364. encoding="utf-8", errors="replace", timeout=10, creationflags=windows_hide_flags(),
  365. check=False,
  366. )
  367. except FileNotFoundError:
  368. os.kill(pid, signal.SIGTERM)
  369. return
  370. if result.returncode != 0:
  371. details = (result.stderr or result.stdout or "").strip()
  372. raise OSError(details or f"taskkill failed for PID {pid}")
  373. def _start_times_agree(current: Any, *recorded: Any) -> bool:
  374. """Same process object: all fingerprints > 0 and within 1ms of ``current``; raises on junk."""
  375. cur = float(current)
  376. return cur > 0 and all(r > 0 and abs(r - cur) <= 0.001 for r in map(float, recorded))
  377. # Same-host start-time readings can drift by ~1 s between the claim-time and a later liveness read
  378. # (macOS ``kern.boottime`` adjustment, #117505). Both fingerprint scales are ×100 (Linux /proc ticks,
  379. # psutil centiseconds), so 200 means 2 s on either platform — a recycled PID is essentially never
  380. # that close to the original's start time.
  381. START_TIME_DRIFT_TOLERANCE = 200
  382. def start_time_fingerprints_match(recorded: Any, current: Any, tolerance: int = START_TIME_DRIFT_TOLERANCE) -> bool:
  383. """Liveness-reconciliation comparator for :func:`get_process_start_time` fingerprints: the
  384. recorded owner and the current reading are the same incarnation when they agree within
  385. ``tolerance``. Raises on junk; callers decide what an unreadable (``None``) side means."""
  386. return abs(int(current) - int(recorded)) <= tolerance
  387. def _scope_hash(identity: str) -> str:
  388. return hashlib.sha256(identity.encode("utf-8")).hexdigest()[:16]
  389. def _get_scope_lock_path(scope: str, identity: str) -> Path:
  390. return _get_lock_dir() / f"{scope}-{_scope_hash(identity)}.lock"
  391. def _get_process_start_time(pid: int) -> Optional[int]:
  392. """Return a stable per-process start-time fingerprint, or None.
  393. Used as a PID-reuse guard: a ``(pid, start_time)`` pair uniquely identifies
  394. a process, so a recycled PID (same number, different process) yields a
  395. different value and is never mistaken for the original.
  396. On Linux this is field 22 of ``/proc/<pid>/stat`` (start time in clock
  397. ticks since boot, an int). On platforms without ``/proc`` (macOS, Windows)
  398. we fall back to ``psutil.Process(pid).create_time()`` — a float epoch
  399. timestamp — quantized to an int (centiseconds) for stable equality.
  400. The two sources are never mixed on a single platform: ``/proc`` always
  401. succeeds first on Linux, and always fails on macOS/Windows so psutil is
  402. always used there. Because the guard only compares the value recorded at
  403. spawn against the live value *on the same host*, the differing units across
  404. platforms are irrelevant — only same-source equality matters.
  405. """
  406. stat_path = Path(f"/proc/{pid}/stat")
  407. try:
  408. # Field 22 (ticks) after comm's last ")": comm may hold spaces or non-UTF-8 bytes.
  409. return int(stat_path.read_bytes().rsplit(b")", 1)[1].split()[19])
  410. except (FileNotFoundError, IndexError, PermissionError, ValueError, OSError):
  411. pass
  412. # No /proc (macOS / Windows): psutil, quantized to centiseconds for stable equality.
  413. # process_birth: macOS's clock-step-stable kernel start (#140580); unchanged with no step.
  414. try:
  415. import psutil # type: ignore
  416. from hermes_cli.process_identity import process_birth
  417. return round(process_birth(psutil.Process(pid)) * 100)
  418. except Exception:
  419. return None
  420. def get_process_start_time(pid: int) -> Optional[int]:
  421. """Public wrapper for retrieving a process start time when available."""
  422. return _get_process_start_time(pid)
  423. def _read_process_cmdline(pid: int) -> Optional[str]:
  424. """Process command line as one string: /proc, then psutil, then ``ps``.
  425. Order is by cost, and this runs per live gateway on every roster/status poll. ``psutil`` reads
  426. the process table in-process (a ``sysctl`` on macOS) where ``ps`` costs a fork+exec — measured
  427. 0.02ms against 4.2ms on macOS for the same string. It cannot always answer: on macOS it raises
  428. ``AccessDenied`` for a process owned by another user, which ``ps`` still reports, so ``ps``
  429. stays as the fallback rather than being replaced."""
  430. with contextlib.suppress(OSError):
  431. raw = Path(f"/proc/{pid}/cmdline").read_bytes()
  432. if raw:
  433. return raw.replace(b"\x00", b" ").decode("utf-8", errors="ignore").strip()
  434. with contextlib.suppress(Exception):
  435. import psutil # type: ignore
  436. cmdline_parts = psutil.Process(pid).cmdline()
  437. if cmdline_parts:
  438. return " ".join(cmdline_parts)
  439. if not _IS_WINDOWS:
  440. with contextlib.suppress(OSError, subprocess.TimeoutExpired):
  441. result = subprocess.run(
  442. ["ps", "-p", str(pid), "-o", "command="],
  443. capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=5,
  444. check=False,
  445. )
  446. if result.returncode == 0 and result.stdout.strip():
  447. return result.stdout.strip()
  448. return None
  449. def _gateway_command_subcommand(command: str | None) -> str | None:
  450. """Hermes gateway lifecycle subcommand from a command line, or None. No loose substring matches
  451. (``"gateway" in cmdline`` also matched ``gateway status`` / ``python -m tui_gateway``): needs a
  452. Hermes entrypoint plus the ``gateway`` subcommand, or a gateway-dedicated entrypoint. Tokenizes
  453. quote-aware (Windows paths with spaces); ``--profile``/``-p`` selectors are stripped anywhere in
  454. argv since ``_apply_profile_override`` removes them before argparse."""
  455. if not command:
  456. return None
  457. try:
  458. raw_tokens = shlex.split(command, posix=False)
  459. except ValueError:
  460. raw_tokens = command.split()
  461. # Strip surrounding quotes, normalize slashes + case per token.
  462. cased_tokens = [t.strip("\"'").replace("\\", "/") for t in raw_tokens]
  463. tokens = [t.lower() for t in cased_tokens]
  464. if not tokens:
  465. return None
  466. basenames = [t.rsplit("/", 1)[-1] for t in tokens]
  467. # ``python -c <src> … -m hermes_cli.main gateway run``: the trailing argv belongs to the program
  468. # the inline source will spawn later, not to this process (#107002). Case-preserving tokens:
  469. # the operand-taking ``-X``/``-W``/``-Q`` must not be conflated with ``-q``/``-b``.
  470. if command_line_runs_inline_source(cased_tokens):
  471. # …unless the source is a Hermes bootstrap running the entry point in THIS process (store
  472. # launcher, launcher script, venv_sync re-entry): then its argv is this process's (#124318).
  473. cased_tokens = inline_bootstrap_argv(cased_tokens)
  474. if cased_tokens is None:
  475. return None
  476. tokens = [t.lower() for t in cased_tokens]
  477. basenames = [t.rsplit("/", 1)[-1] for t in tokens]
  478. # The launchd job's osascript wrapper (gateway_launchd.launchd_program_arguments) carries the gateway argv
  479. # inside one JXA script string; the gateway itself is its child and is matched on its own command line.
  480. if basenames[0] == "osascript":
  481. return None
  482. # Gateway-dedicated entrypoints carry no subcommand to inspect (``python -m gateway.run`` is the
  483. # module spelling of the same script: its PID record renders argv as ``gateway/run.py``, and a
  484. # liveness probe that read the live ``-m`` cmdline as "not a gateway" made every fixture-launched
  485. # standalone owner invisible to the multiplex preflight).
  486. if any(t == "gateway/run.py" or t.endswith("/gateway/run.py") for t in tokens):
  487. return "run"
  488. if any(tokens[i] == "-m" and tokens[i + 1] == "gateway.run" for i in range(len(tokens) - 1)):
  489. return "run"
  490. # Atomic Hermes' bundled desktop runner shares HERMES_HOME with the CLI; without this,
  491. # `gateway run --replace` does not recognise it as a running gateway, skips the
  492. # terminate-and-scoped-lock-handoff path, and collides with its still-held scoped locks
  493. # (e.g. the Discord bot-token lock). See #22418.
  494. if any(b == "desktop-gateway.py" for b in basenames):
  495. return "run"
  496. if any(b in ("hermes-gateway", "hermes-gateway.exe") for b in basenames):
  497. return "run"
  498. joined = " ".join(tokens)
  499. if "hermes_cli.main" not in joined and "hermes_cli/main.py" not in joined and not any(
  500. b in ("hermes", "hermes.exe") for b in basenames
  501. ):
  502. return None
  503. # Drop --profile X / -p X / --profile=X / -p=X (consumes a VALUE of "gateway" too).
  504. filtered: list[str] = []
  505. skip_next = False
  506. for token in tokens:
  507. if skip_next:
  508. skip_next = False
  509. elif token in ("--profile", "-p"):
  510. skip_next = True
  511. elif not token.startswith(("--profile=", "-p=")):
  512. filtered.append(token)
  513. for i, token in enumerate(filtered):
  514. if token == "gateway":
  515. # Bare `hermes gateway` defaults to `run`.
  516. return filtered[i + 1] if i + 1 < len(filtered) else "run"
  517. return None
  518. def gateway_spawn_intent_subcommand(command: str | None) -> str | None:
  519. """Gateway lifecycle subcommand a command line would EVENTUALLY launch, or None.
  520. The identity matcher (``_gateway_command_subcommand``) deliberately refuses ``python -c <src>
  521. …``: the trailing argv is the inline program's data, not that process's own identity (#107002).
  522. Callers that inspect a command line as SPAWN INTENT — "if I launch this, does a gateway runtime
  523. eventually appear?" — need the opposite answer, because
  524. ``gateway._spawn_gateway_restart_watcher`` hides a real ``… -m hermes_cli.main gateway run``
  525. behind exactly that wrapper. ``tests/_fixtures/live_system_guard.py`` is the canonical caller.
  526. Still no substring matching: the wrapper is peeled token-wise and each remaining suffix is
  527. handed to the same canonical matcher.
  528. """
  529. direct = _gateway_command_subcommand(command)
  530. if direct is not None or not command:
  531. return direct
  532. try:
  533. raw_tokens = shlex.split(command, posix=False)
  534. except ValueError:
  535. raw_tokens = command.split()
  536. cased_tokens = [t.strip("\"'").replace("\\", "/") for t in raw_tokens]
  537. flag_index = inline_source_flag_index(cased_tokens)
  538. if flag_index is None:
  539. return None
  540. # Skip the interpreter, its options, ``-c`` and the source literal; then try every suffix —
  541. # the embedded argv starts at an unknown offset (the watcher prefixes it with the old PID).
  542. start = flag_index + 2
  543. for i in range(start, len(cased_tokens)):
  544. nested = _gateway_command_subcommand(" ".join(raw_tokens[i:]))
  545. if nested is not None:
  546. return nested
  547. return None
  548. def looks_like_gateway_command_line(command: str | None) -> bool:
  549. """True only for a real ``gateway run`` process command line."""
  550. return _gateway_command_subcommand(command) == "run"
  551. def looks_like_gateway_runtime_command_line(command: str | None) -> bool:
  552. """True for command lines that can host the runtime (``run`` or ``restart``: without a service
  553. manager the manual restart fallback runs ``run_gateway()`` in-process). For validating
  554. Hermes-owned records / cleanup scans only; ``looks_like_gateway_command_line`` stays strict."""
  555. return _gateway_command_subcommand(command) in {"run", "restart"}
  556. def _looks_like_gateway_process(pid: int) -> bool:
  557. """True when the live PID still looks like the Hermes gateway."""
  558. cmdline = _read_process_cmdline(pid)
  559. return bool(cmdline) and looks_like_gateway_command_line(cmdline)
  560. def _record_looks_like_gateway(record: dict[str, Any]) -> bool:
  561. """Validate gateway identity from PID-file metadata when cmdline is unavailable."""
  562. argv = record.get("argv")
  563. if record.get("kind") != _GATEWAY_KIND or not isinstance(argv, list) or not argv:
  564. return False
  565. return looks_like_gateway_runtime_command_line(" ".join(str(part) for part in argv))
  566. def _profile_name_for_home(profile_home: Path) -> Optional[str]:
  567. """Profile id for ``<root>/profiles/<name>``; None for the root/default home (bare gateway)."""
  568. return profile_home.name if profile_home.parent.name == "profiles" else None
  569. def profile_flag_value(command: str) -> Optional[str]:
  570. """Lowercased ``-p``/``--profile`` argument of a case-preserved command line (a bootstrap's interpreter
  571. ``-P`` is no ``-p``, #127865), or None. Token equality: ``-p ops`` must not claim ``-p ops-2``."""
  572. tokens = [t.lower() for t in inline_bootstrap_argv(command.split()) or command.split()]
  573. for i, tok in enumerate(tokens):
  574. if tok.startswith("--profile="):
  575. return tok.partition("=")[2]
  576. if tok in ("-p", "--profile") and i + 1 < len(tokens):
  577. return tokens[i + 1]
  578. return None
  579. _HERMES_HOME_ASSIGNMENT_RE = re.compile(r"(?:^|\s)hermes_home=(?:\"([^\"]*)\"|'([^']*)'|(\S+))")
  580. def hermes_home_assignments(command: str) -> list[str]:
  581. """Values of every ``HERMES_HOME=<value>`` assignment in ``command`` (the caller lowercases
  582. and normalizes separators). Values are token-bounded, quotes stripped: the substring test
  583. this replaces let ``HERMES_HOME=/root/profiles/ops`` claim a ``/root/profiles/ops2`` gateway.
  584. The name is token-bounded too (``FOO=hermes_home=/x`` is not an assignment), and a trailing
  585. separator on the value is stripped -- ``HERMES_HOME=/root/.hermes/`` (systemd ``Environment=``
  586. or a shell wrapper spelling) is the same home as ``/root/.hermes``; callers strip the profile
  587. home the same way."""
  588. return [
  589. next(g for g in m.groups() if g is not None).rstrip("/")
  590. for m in _HERMES_HOME_ASSIGNMENT_RE.finditer(command)
  591. ]
  592. def command_line_names_hermes_home(command_lc: str, home_lc: str) -> bool:
  593. """True when ``command_lc`` carries ``HERMES_HOME=<home_lc>`` (both lowercased, ``/``-separated,
  594. no trailing separator). Argv reaches us space-joined, so an unquoted value with a space in it
  595. (``HERMES_HOME=C:/Users/John Doe/.hermes``) is cut at the space by the token parser; a
  596. token-bounded literal match of the whole home recovers that spelling."""
  597. if home_lc in hermes_home_assignments(command_lc):
  598. return True
  599. return re.search(rf"(?:^|\s)hermes_home={re.escape(home_lc)}/?(?=\s|$)", command_lc) is not None
  600. def _command_line_belongs_to_profile(command: str, profile_home: Path) -> bool:
  601. """True when a gateway command line belongs to ``profile_home`` (mirrors
  602. ``hermes_cli.gateway._matches_current_profile``): a stale state file can record a PID recycled
  603. onto ANOTHER profile's live gateway. Named profiles carry ``-p``/``--profile <name>`` or
  604. ``HERMES_HOME=`` on argv; the default gateway runs bare. Separators normalized."""
  605. command_lc = command.lower().replace("\\", "/")
  606. profile_name = _profile_name_for_home(profile_home)
  607. home_lc = str(profile_home).lower().replace("\\", "/").rstrip("/")
  608. if profile_name is not None and profile_name != "default":
  609. if profile_flag_value(command) == profile_name.lower():
  610. return True
  611. return command_line_names_hermes_home(command_lc, home_lc)
  612. # Default profile: accept unless argv names ANOTHER profile (any spelling the CLI pre-parser
  613. # accepts, ``--profile=ops`` included -- a substring test let that gateway pass as the default's)
  614. # or a conflicting explicit HERMES_HOME= (its absence is not disqualifying -- HERMES_HOME usually
  615. # arrives via the env). ``--profile default`` names this profile: a hand-written launchd plist
  616. # mirrors the named-profile service shape, and rejecting it reported a live default gateway
  617. # as stopped (#100817).
  618. if profile_flag_value(command) not in (None, "default"):
  619. return False
  620. return not hermes_home_assignments(command_lc) or command_line_names_hermes_home(command_lc, home_lc)
  621. def _host_gateway_serves_home(pid: int, profile_home: Path) -> bool:
  622. """Does the ONE host gateway — PID ``pid`` — serve ``profile_home``'s profile?
  623. Argv cannot answer this: the host singleton runs ONE home's (usually bare/default) command line
  624. while multiplexing every profile, so :func:`_command_line_belongs_to_profile` rejects every
  625. secondary and the profile reads as "not running" while its messages are being served. The live
  626. served set is the only proof; the argv rule stays as the fallback when no record exists.
  627. """
  628. try:
  629. from gateway.host_attach import host_gateway, profile_name_for_home
  630. owner = host_gateway()
  631. except Exception:
  632. return False
  633. return owner is not None and owner.pid == pid and owner.serves(profile_name_for_home(profile_home))
  634. def _record_matches_live_gateway_pid(
  635. record: dict[str, Any], pid: int, *, expected_home: Optional[Path] = None
  636. ) -> bool:
  637. """True when a live PID still identifies as this gateway record. The live command line wins (a
  638. stale record's argv must not make a recycled PID count as a gateway; with ``expected_home`` it
  639. must also belong to that profile — or serve it as the host multiplexer); unreadable cmdline
  640. (Windows/EACCES) -> persisted record."""
  641. live_cmdline = _read_process_cmdline(pid)
  642. if not live_cmdline:
  643. return _record_looks_like_gateway(record)
  644. if not looks_like_gateway_runtime_command_line(live_cmdline):
  645. return False
  646. if expected_home is not None and _host_gateway_serves_home(pid, expected_home):
  647. return True
  648. return expected_home is None or _command_line_belongs_to_profile(live_cmdline, expected_home)
  649. def _record_argv() -> list[str]:
  650. """``sys.argv`` with the inline-source placeholder replaced by the entry point this process runs.
  651. The published launcher script (``_launchers._launcher_script``: the POSIX ``bin/hermes`` shell
  652. launcher and the Windows ``.cmd``) imports ``hermes_cli.main`` inside ``python -I -c <script>``,
  653. so ``sys.argv`` is ``["-c", "gateway", "run"]`` — a record the argv matcher can never accept
  654. once the live command line is unreadable (Windows/EACCES fallback in
  655. ``_record_matches_live_gateway_pid``). Recording the module path follows runpy's ``alter_sys``
  656. convention, which is what the ``--run-module`` and store-launcher forms already persist."""
  657. argv = list(sys.argv)
  658. entry = sys.modules.get("hermes_cli.main")
  659. if argv[:1] == ["-c"] and getattr(entry, "__file__", None):
  660. argv[0] = entry.__file__
  661. return argv
  662. def _build_pid_record() -> dict:
  663. return {
  664. "pid": os.getpid(), "kind": _GATEWAY_KIND, "argv": _record_argv(),
  665. "start_time": _get_process_start_time(os.getpid()),
  666. # Scoped locks are machine-global; the owner's home lets a cross-profile
  667. # --replace place its takeover marker where the target will read it.
  668. "hermes_home": str(_canonical_hermes_home(_get_process_hermes_home())),
  669. }
  670. def _get_code_identity_fields() -> dict[str, Any]:
  671. """Code identity of THIS process for ``gateway_state.json`` (restart picked up new code?).
  672. Lazy import keeps ``gateway.status`` free of ``hermes_cli`` at import time. Never raises.
  673. A gateway keeps serving the module versions it imported at startup, so stamping the identity into
  674. ``gateway_state.json`` lets `hermes update` (and the dashboard) prove whether a running gateway actually
  675. picked up new code after the restart phase — instead of assuming it did (#88654, #69754). Never raises;
  676. degrades to absent fields.
  677. """
  678. try:
  679. from hermes_cli.version_info import get_code_identity
  680. identity = get_code_identity()
  681. return {"code_sha": identity.get("sha"), "code_version": identity.get("version")}
  682. except Exception:
  683. return {}
  684. def _pid_record_belongs_to_current_profile(record: Optional[dict[str, Any]]) -> bool:
  685. """True when the record's ``hermes_home`` matches the current process (legacy records: True);
  686. another HERMES_HOME's record must be ignored or the default gateway assumes its identity."""
  687. if not isinstance(record, dict):
  688. return False
  689. record_home = record.get("hermes_home")
  690. return not record_home or _same_hermes_home(record_home, _get_process_hermes_home())
  691. def _build_runtime_status_record() -> dict[str, Any]:
  692. return {
  693. **_build_pid_record(), "gateway_state": "starting", "exit_reason": None,
  694. "restart_requested": False, "active_agents": 0, "platforms": {},
  695. "session_store": {"status": "unknown"}, "updated_at": _utc_now_iso(),
  696. **_get_code_identity_fields(),
  697. }
  698. def _read_json_file(path: Path, *, bare_pid_ok: bool = False) -> Optional[dict[str, Any]]:
  699. """JSON object at ``path``, or None when absent/empty/unreadable/invalid. ``bare_pid_ok`` also
  700. accepts legacy bare-integer PID files as ``{"pid": N}``."""
  701. try:
  702. raw = path.read_text(encoding="utf-8-sig").strip()
  703. except (OSError, UnicodeDecodeError):
  704. # OSError: file vanished or permission flipped between exists() and
  705. # read. UnicodeDecodeError: file holds non-UTF-8 / binary garbage
  706. # (a truncated or clobbered status file). Either way it's unusable.
  707. return None
  708. if not raw:
  709. return None
  710. try:
  711. payload = json.loads(raw)
  712. except json.JSONDecodeError:
  713. payload = None
  714. if bare_pid_ok:
  715. with contextlib.suppress(ValueError):
  716. payload = int(raw)
  717. if bare_pid_ok and isinstance(payload, int):
  718. return {"pid": payload}
  719. return payload if isinstance(payload, dict) else None
  720. def _write_json_file(path: Path, payload: dict[str, Any]) -> None:
  721. atomic_json_write(path, payload, indent=None, separators=(",", ":"))
  722. def _unlink_quietly(path: Path) -> None:
  723. with contextlib.suppress(OSError):
  724. path.unlink(missing_ok=True)
  725. def _read_pid_record(pid_path: Optional[Path] = None) -> Optional[dict]:
  726. return _read_json_file(pid_path or _get_pid_path(), bare_pid_ok=True)
  727. def _read_gateway_lock_record(lock_path: Optional[Path] = None) -> Optional[dict[str, Any]]:
  728. return _read_json_file(lock_path or _get_gateway_lock_path(), bare_pid_ok=True)
  729. def _pid_from_record(record: Optional[dict[str, Any]], key: str = "pid") -> Optional[int]:
  730. try:
  731. return int(record[key])
  732. except (KeyError, TypeError, ValueError):
  733. return None
  734. def _start_times_conflict(recorded_start: Any, current_start: Any) -> bool:
  735. """PID-reuse guard: True only when BOTH start times are known and differ."""
  736. return None not in (recorded_start, current_start) and current_start != recorded_start
  737. def _live_pid_from_record(record: Optional[dict[str, Any]]) -> Optional[int]:
  738. """Record's PID when it is alive and passes the start-time PID-reuse guard, else None."""
  739. pid = _pid_from_record(record)
  740. if pid is None or not _pid_exists(pid):
  741. return None
  742. if _start_times_conflict(record.get("start_time"), _get_process_start_time(pid)):
  743. return None
  744. return pid
  745. def _clear_running_pid_cache() -> None:
  746. with _gateway_running_pid_cache_lock:
  747. _gateway_running_pid_cache.clear()
  748. def _file_cache_signature(path: Path) -> tuple[bool, Optional[int], Optional[int]]:
  749. try:
  750. st = path.stat()
  751. except OSError:
  752. return (False, None, None)
  753. return (True, st.st_mtime_ns, st.st_size)
  754. def _cleanup_invalid_pid_path(
  755. pid_path: Path, *, cleanup_stale: bool, unlink_lock: bool = True
  756. ) -> None:
  757. """Force-unlink a stale PID file (lock confirmed inactive, so no pid check). The sibling lock
  758. file is never unlinked, held or not (``unlink_lock`` is therefore moot): another contender may
  759. already have opened that inode, and a fresh one would let two processes each hold "the" lock."""
  760. if not cleanup_stale:
  761. return
  762. _clear_running_pid_cache()
  763. # Keep the inode even when stale: another contender may already have opened it.
  764. for path in (pid_path,):
  765. with contextlib.suppress(Exception):
  766. path.unlink(missing_ok=True)
  767. def _try_acquire_file_lock(handle) -> bool:
  768. try:
  769. if _IS_WINDOWS:
  770. handle.seek(0, os.SEEK_END)
  771. if handle.tell() == 0:
  772. handle.write("\n")
  773. handle.flush()
  774. handle.seek(_WINDOWS_LOCK_OFFSET)
  775. msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1)
  776. else:
  777. fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
  778. return True
  779. except (BlockingIOError, OSError):
  780. return False
  781. def _pid_exists(pid: int) -> bool:
  782. """Cross-platform "is this PID alive" check that does NOT kill the target. CRITICAL on Windows:
  783. ``os.kill(pid, 0)`` sends ``CTRL_C_EVENT`` to the whole console group (bpo-14484), so prefer
  784. psutil, then ctypes ``OpenProcess`` (Windows) / ``os.kill(pid, 0)`` (POSIX). Zombies report
  785. dead: treating one as alive makes --replace wait forever under systemd Restart=always."""
  786. pid = int(pid)
  787. if os.name == "nt":
  788. # ctypes keeps psutil off this hot path (no zombies here). Not a GIL fix: pid_exists measured ~4 us; the
  789. # #142085 stall is Process.status() (process_identity). PID 0 is alive to psutil, dead here.
  790. return pid_exists_stdlib(pid)
  791. try:
  792. import psutil # type: ignore
  793. try:
  794. # A zombie (defunct) process is still in the process table, so ``psutil.pid_exists()`` returns
  795. # True for it — but it is already dead: SIGKILL has no effect and it cannot be a running
  796. # gateway. Treating a zombie as alive makes ``--replace`` wait for the old PID to die (it never
  797. # does, until its parent reaps it), then abort with exit 1 — a silent crash loop under systemd
  798. # ``Restart=always``, which respawns the gateway before reaping the previous process (issue
  799. # #42126). Report zombies as dead so the takeover proceeds. Best-effort: any failure to read
  800. # status (partial/stub psutil, access denied, transient race) falls through to the authoritative
  801. # ``pid_exists()`` below rather than raising.
  802. if psutil.Process(pid).status() == psutil.STATUS_ZOMBIE:
  803. return False
  804. except getattr(psutil, "NoSuchProcess", ()):
  805. return False
  806. except Exception:
  807. pass
  808. return bool(psutil.pid_exists(pid))
  809. except ImportError:
  810. pass # Fall through to stdlib fallback.
  811. return pid_exists_stdlib(pid)
  812. def _release_file_lock(handle) -> None:
  813. with contextlib.suppress(OSError):
  814. if _IS_WINDOWS:
  815. handle.seek(_WINDOWS_LOCK_OFFSET)
  816. msvcrt.locking(handle.fileno(), msvcrt.LK_UNLCK, 1)
  817. else:
  818. fcntl.flock(handle.fileno(), fcntl.LOCK_UN)
  819. def acquire_gateway_runtime_lock() -> bool:
  820. """Reserve the launch profile without unlinking inaccessible lock inodes."""
  821. from gateway.runtime_ownership import process_ownership, OwnershipConflict
  822. try:
  823. process_ownership.reserve([_get_process_hermes_home()])
  824. except (OwnershipConflict, OSError):
  825. return False
  826. _clear_running_pid_cache()
  827. return True
  828. def release_gateway_runtime_lock() -> None:
  829. """Release only this process's profile reservations, never their lock files."""
  830. from gateway.runtime_ownership import process_ownership
  831. process_ownership.close()
  832. _clear_running_pid_cache()
  833. def owns_gateway_runtime_lock() -> bool:
  834. from gateway.runtime_ownership import process_ownership
  835. return process_ownership.owns(_get_process_hermes_home())
  836. def _probe_lock_file(handle) -> bool:
  837. """True when another process holds the lock (a won probe is released); closes ``handle``."""
  838. try:
  839. held = not _try_acquire_file_lock(handle)
  840. if not held:
  841. _release_file_lock(handle)
  842. return held
  843. finally:
  844. with contextlib.suppress(OSError):
  845. handle.close()
  846. def is_gateway_runtime_lock_active(lock_path: Optional[Path] = None) -> bool:
  847. """True when some process currently owns the gateway runtime lock."""
  848. resolved_lock_path = lock_path or _get_gateway_lock_path()
  849. if owns_gateway_runtime_lock() and resolved_lock_path == _get_gateway_lock_path():
  850. return True
  851. if not resolved_lock_path.exists():
  852. return False
  853. try:
  854. handle = open(resolved_lock_path, "a+", encoding="utf-8")
  855. except PermissionError:
  856. # Unknown ownership is not permission to delete a potentially held inode.
  857. return True
  858. return _probe_lock_file(handle)
  859. def _strict_path_exists(path: Path, label: str) -> bool:
  860. """Like ``path.exists()`` but raises RuntimeError instead of False on EACCES-style errors."""
  861. try:
  862. path.stat()
  863. return True
  864. except FileNotFoundError:
  865. return False
  866. except OSError as exc:
  867. raise RuntimeError(f"{label} metadata is not inspectable: {exc}") from exc
  868. def _is_gateway_runtime_lock_active_strict(lock_path: Path) -> bool:
  869. """Probe ownership without treating access failures as absence."""
  870. try:
  871. handle = open(lock_path, "r+", encoding="utf-8")
  872. except FileNotFoundError:
  873. return False
  874. except OSError as exc:
  875. raise RuntimeError(f"gateway runtime lock is not inspectable: {exc}") from exc
  876. try:
  877. return _probe_lock_file(handle)
  878. except OSError as exc:
  879. raise RuntimeError(f"gateway runtime lock probe failed: {exc}") from exc
  880. def write_pid_file() -> None:
  881. """Write this process's PID record via O_CREAT|O_EXCL; a racing gateway's FileExistsError
  882. propagates for the caller to decide."""
  883. path = _get_pid_path()
  884. make_user_dir(path.parent, parents=True, exist_ok=True)
  885. _write_json_excl(path, _build_pid_record())
  886. _clear_running_pid_cache()
  887. def _write_json_excl(path: Path, record: dict[str, Any]) -> None:
  888. """Create ``path`` with O_CREAT|O_EXCL and dump ``record``; unlinks on a failed write."""
  889. fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
  890. try:
  891. with os.fdopen(fd, "w", encoding="utf-8") as handle:
  892. json.dump(record, handle)
  893. except Exception:
  894. _unlink_quietly(path)
  895. raise
  896. def _apply_set_fields(target: dict[str, Any], fields) -> None:
  897. """Assign each ``(key, value, coerce)`` whose value was explicitly passed (not ``_UNSET``)."""
  898. for key, value, coerce in fields:
  899. if value is not _UNSET:
  900. target[key] = coerce(value) if coerce is not None else value
  901. def _coerce_session_store(session_store: Any) -> dict[str, str]:
  902. state = str(session_store.get("status") or "") if isinstance(session_store, dict) else ""
  903. return {"status": state if state in {"ok", "unavailable", "retrying"} else "unknown"}
  904. def _prepare_runtime_status_update(
  905. *, gateway_state: Any = _UNSET, exit_reason: Any = _UNSET, restart_requested: Any = _UNSET,
  906. active_agents: Any = _UNSET, active_work: Any = _UNSET, platform: Any = _UNSET, platform_state: Any = _UNSET,
  907. error_code: Any = _UNSET, error_message: Any = _UNSET, needs_attention: Any = _UNSET,
  908. retrying_since: Any = _UNSET, served_profiles: Any = _UNSET, session_store: Any = _UNSET,
  909. parked_profiles: Any = _UNSET, multiplex_standalone_reason: Any = _UNSET,
  910. platform_metrics: Any = _UNSET,
  911. ingress_url: Any = _UNSET, listener_base: Any = _UNSET, clear_profile_platforms: bool = False,
  912. drop_profile_platforms: Optional[str] = None, keep_platforms: Iterable[str] = (),
  913. load_existing: bool = True, reload_existing: bool = False,
  914. ) -> tuple[Path, dict[str, Any], dict[str, Any]]:
  915. """Merge one update into the process-wide canonical status snapshot."""
  916. global _runtime_status_state_path, _runtime_status_state
  917. path = _get_runtime_status_path()
  918. with _runtime_status_state_lock:
  919. if reload_existing or _runtime_status_state_path != path or _runtime_status_state is None:
  920. _runtime_status_state_path = path
  921. _runtime_status_state = (
  922. (_read_json_file(path) if load_existing else None) or _build_runtime_status_record())
  923. # The module snapshot is only ever reassigned (never mutated in place) and
  924. # submit() copies again, so the previous snapshot can be handed out as-is.
  925. previous_payload = _runtime_status_state
  926. payload = copy.deepcopy(previous_payload)
  927. current_record = _build_pid_record()
  928. payload.setdefault("platforms", {})
  929. if not isinstance(payload["platforms"], dict):
  930. payload["platforms"] = {}
  931. if clear_profile_platforms or drop_profile_platforms:
  932. drop_prefix = f"{drop_profile_platforms}:" if drop_profile_platforms else None
  933. payload["platforms"] = {
  934. k: v for k, v in payload["platforms"].items()
  935. if not isinstance(k, str) or ":" not in k
  936. or (drop_prefix is not None and not k.startswith(drop_prefix)) or k in keep_platforms
  937. }
  938. payload.update({key: current_record[key] for key in ("kind", "pid", "argv", "start_time")})
  939. payload["updated_at"] = _utc_now_iso()
  940. payload.update(_get_code_identity_fields())
  941. _apply_set_fields(payload, (
  942. ("gateway_state", gateway_state, None), ("exit_reason", exit_reason, None),
  943. ("restart_requested", restart_requested, bool),
  944. ("active_agents", active_agents, parse_active_agents),
  945. ("active_work", active_work, lambda v: list(v) if v else None),
  946. ("served_profiles", served_profiles, lambda v: list(v or [])),
  947. # Profiles the multiplexer could not serve, name -> reason; clients fail fast on them.
  948. ("parked_profiles", parked_profiles, lambda v: {str(k): str(r) for k, r in dict(v or {}).items()}),
  949. ("multiplex_standalone_reason", multiplex_standalone_reason, lambda v: str(v) if v else None),
  950. ("session_store", session_store, _coerce_session_store),
  951. ))
  952. if platform is not _UNSET:
  953. platform_payload = copy.deepcopy(payload["platforms"].get(platform, {}))
  954. if not isinstance(platform_payload, dict):
  955. platform_payload = {}
  956. if platform_state == "connected":
  957. needs_attention = False if needs_attention is _UNSET else needs_attention
  958. retrying_since = None if retrying_since is _UNSET else retrying_since
  959. _apply_set_fields(platform_payload, (
  960. ("state", platform_state, None), ("error_code", error_code, None),
  961. ("error_message", error_message, None),
  962. ("needs_attention", needs_attention, bool),
  963. ("retrying_since", retrying_since, None),
  964. ("metrics", platform_metrics, None),
  965. ("ingress_url", ingress_url, None),
  966. ("listener_base", listener_base, None),
  967. ))
  968. platform_payload.update(
  969. updated_at=_utc_now_iso(), writer_pid=current_record["pid"],
  970. writer_start_time=current_record["start_time"])
  971. payload["platforms"][platform] = platform_payload
  972. _runtime_status_state = payload
  973. return path, payload, previous_payload
  974. def _emit_runtime_status_transition(
  975. previous_payload: dict[str, Any], payload: dict[str, Any]
  976. ) -> None:
  977. with contextlib.suppress(Exception):
  978. from agent.monitoring.gateway_health import emit_runtime_status_transition
  979. emit_runtime_status_transition(previous_payload, payload)
  980. def write_runtime_status(
  981. *, reload_existing: bool = False, wait_timeout: Optional[float] = None, **fields: Any,
  982. ) -> bool:
  983. """Synchronously persist status for CLI callers and off-loop startup.
  984. ``wait_timeout`` bounds how long the caller waits for durable persistence.
  985. A timed-out update remains queued for the single background writer.
  986. Keyword ``fields`` are those of ``_prepare_runtime_status_update``.
  987. """
  988. with _runtime_status_state_lock:
  989. path, payload, previous_payload = _prepare_runtime_status_update(
  990. reload_existing=reload_existing, **fields)
  991. writer = _get_runtime_status_writer()
  992. generation = writer.submit(path, payload)
  993. # Report the transition once it is queued (matching ``publish_runtime_status``): a
  994. # timed-out update is still written by the background writer, so its transition happened.
  995. _emit_runtime_status_transition(previous_payload, payload)
  996. return writer.wait(generation, timeout=wait_timeout)
  997. def publish_runtime_status(**fields: Any) -> int:
  998. """Merge and enqueue status without waiting for filesystem persistence.
  999. Keyword ``fields`` are those of ``_prepare_runtime_status_update``.
  1000. """
  1001. with _runtime_status_state_lock:
  1002. path, payload, previous_payload = _prepare_runtime_status_update(
  1003. load_existing=False, **fields)
  1004. generation = _get_runtime_status_writer().submit(path, payload)
  1005. _emit_runtime_status_transition(previous_payload, payload)
  1006. return generation
  1007. def read_runtime_status(path: Optional[Path] = None) -> Optional[dict[str, Any]]:
  1008. """Read ``gateway_state.json``; ``path`` lets callers inspect another profile's file."""
  1009. return _read_json_file(path or _get_runtime_status_path())
  1010. # Max age of a ``gateway_state.json`` snapshot before its liveness claim is suspect: an older record
  1011. # outlived an ungracefully-killed writer (taskkill /F, OOM, power loss) — or, with the PID alive, the
  1012. # housekeeping thread that re-stamps ``updated_at`` every tick has wedged (#113372). 2x the 60 s
  1013. # housekeeping interval.
  1014. _RUNTIME_STATUS_STALE_TTL_S = 120
  1015. def runtime_status_is_stale(
  1016. record: Optional[dict[str, Any]], ttl_s: int = _RUNTIME_STATUS_STALE_TTL_S
  1017. ) -> bool:
  1018. """True when the snapshot's ``updated_at`` is older than ``ttl_s`` (or missing/unparseable)."""
  1019. return not isinstance(record, dict) or _marker_is_stale(record.get("updated_at") or "", ttl_s)
  1020. def runtime_status_heartbeat_age_s(record: Optional[dict[str, Any]]) -> Optional[int]:
  1021. """Whole seconds since the snapshot's ``updated_at``; None when missing/unparseable (an
  1022. unparseable stamp is a stale *file*, not a wedged heartbeat)."""
  1023. updated_at = normalize_updated_at(record.get("updated_at")) if isinstance(record, dict) else None
  1024. if not updated_at:
  1025. return None
  1026. return max(0, int((datetime.now(UTC) - datetime.fromisoformat(updated_at)).total_seconds()))
  1027. def runtime_status_pid_is_live(record: Optional[dict[str, Any]]) -> bool:
  1028. """True when the snapshot's PID is alive and passes the start-time PID-reuse guard."""
  1029. return _live_pid_from_record(record) is not None
  1030. def parse_active_agents(raw: Any) -> int:
  1031. """Coerce ``active_agents`` to a non-negative int; shared by writer and both HTTP readers."""
  1032. try:
  1033. return max(0, int(raw))
  1034. except (TypeError, ValueError):
  1035. return 0
  1036. # Live, serving states: a valid begin-drain target. ``degraded`` is a serving gateway with a parked
  1037. # platform (a dead watchdog-stamped ``degraded`` is already excluded by ``gateway_running=False``).
  1038. _DRAINABLE_GATEWAY_STATES = frozenset({"running", "degraded"})
  1039. def derive_gateway_busy(*, gateway_running: bool, gateway_state: Any, active_agents: Any) -> bool:
  1040. """Busy iff live, serving (``running``/``degraded``), and ``active_agents > 0`` -- the contract NAS gates on. Liveness
  1041. keys off ``gateway_running``, NEVER ``updated_at`` (a stale heartbeat is a health warning, not death)."""
  1042. if not derive_gateway_drainable(gateway_running=gateway_running, gateway_state=gateway_state):
  1043. return False
  1044. return parse_active_agents(active_agents) > 0
  1045. def derive_gateway_drainable(*, gateway_running: bool, gateway_state: Any) -> bool:
  1046. """Drainable iff live and serving; independent of ``active_agents`` (idle drains finish)."""
  1047. return bool(gateway_running) and gateway_state in _DRAINABLE_GATEWAY_STATES
  1048. @dataclass(frozen=True)
  1049. class GatewayLiveness:
  1050. """Resolved gateway liveness for one dashboard surface. ``source``: which ladder rung answered
  1051. (logging/tests; product code reads it only via ``status_served.local_liveness_pid``).
  1052. ``probe_error``: a rung raised; lets fail-open callers tell "down" from "unknown"."""
  1053. running: bool
  1054. pid: Optional[int]
  1055. source: str
  1056. health_body: Optional[dict[str, Any]] = None
  1057. probe_error: bool = False
  1058. # The multiplexer's own ``gateway_state.json`` when the ``multiplexer`` rung answered: a served
  1059. # profile writes no runtime record of its own, so its platform states live there under
  1060. # ``<profile>:<platform>`` keys.
  1061. runtime: Optional[dict[str, Any]] = None
  1062. def profile_name_for_home(profile_home: Path) -> Optional[str]:
  1063. """Profile id of any Hermes home: ``<root>/profiles/<name>`` → ``<name>``, the default root →
  1064. ``"default"``, anything else → None. Multiplex-only makes ``default`` an ordinary served
  1065. profile, so reporting surfaces need a name for it too."""
  1066. home = Path(profile_home)
  1067. named = _profile_name_for_home(home)
  1068. if named:
  1069. return named
  1070. try:
  1071. from hermes_constants import get_default_hermes_root
  1072. if home.resolve() == Path(get_default_hermes_root()).resolve():
  1073. return "default"
  1074. except Exception:
  1075. return None
  1076. return None
  1077. def multiplexer_liveness_for_profile(profile_dir: Path) -> Optional[tuple[int, dict[str, Any]]]:
  1078. """``(pid, host gateway_state.json)`` when the ONE live host gateway serves the profile whose home
  1079. is ``profile_dir``; None for a home it does not serve or when no gateway owns the host role.
  1080. Multiplex-only: ``default`` is just another served profile, not the owner of a private topology —
  1081. resolving from the host rendezvous record (``gateway/host_topology.py``) is what lets it be
  1082. reported as SERVED rather than only as owner. A served profile owns no
  1083. ``gateway.pid``/``gateway_state.json`` (#97120), so every PID-file rung of the dashboard ladder
  1084. reports it stopped while ``hermes -p X status`` says running — the two must agree.
  1085. """
  1086. name = profile_name_for_home(Path(profile_dir))
  1087. if not name:
  1088. return None
  1089. from gateway.host_topology import host_gateway_topology
  1090. from hermes_cli.gateway import named_profile_served_by_running_multiplexer
  1091. from hermes_cli.gateway_multiplex_served import live_default_gateway_pid
  1092. from hermes_constants import get_default_hermes_root
  1093. # The roster is matched by NAME, and the multiplexer only serves ``<default root>/profiles/<name>``:
  1094. # a profile directory copied to another root (sandbox, restore-from-backup) keeps the name but is
  1095. # not the home being served, so it must not borrow the multiplexer's PID.
  1096. if name != "default" and not _same_hermes_home(profile_dir, get_default_hermes_root() / "profiles" / name):
  1097. return None
  1098. topology = host_gateway_topology()
  1099. # The multiplexer's record lives in the home that LAUNCHED it; a named-hosted multiplexer leaves
  1100. # a possibly stale standalone record at the default root, which must not be projected.
  1101. launch_home = get_default_hermes_root()
  1102. if topology is not None and topology.serves(name):
  1103. pid: Optional[int] = topology.pid
  1104. launch_home = topology.home or launch_home
  1105. elif name != "default" and named_profile_served_by_running_multiplexer(name):
  1106. # Config-derived fallback for a record that predates ``served_profiles``.
  1107. pid = live_default_gateway_pid()
  1108. else:
  1109. return None
  1110. if pid is None:
  1111. return None
  1112. return pid, read_runtime_status(launch_home / "gateway_state.json") or {}
  1113. def shared_listener_mirror_platforms(runtime: Optional[dict[str, Any]], profile: str) -> dict[str, Any]:
  1114. """Entries for the api_server/webhook mirrors a served ``profile`` gets from the DEFAULT's
  1115. listener. The multiplexer never builds those adapters for a secondary (``gateway.run_adapters``
  1116. skips them: ``SHARED_LISTENER_MIRROR_PLATFORMS``), so the record has no ``<profile>:api_server``
  1117. entry and every reader fell through to ``pending_restart`` — "Restart needed" forever while
  1118. ``/p/<profile>/v1/...`` answered. Only a live default entry is mirrored; its state is the profile's
  1119. state, plus the ``/p/<profile>`` URL the client must actually call.
  1120. """
  1121. from gateway.config import SHARED_LISTENER_MIRROR_PATHS, SHARED_LISTENER_MIRROR_PLATFORMS
  1122. plats = (runtime or {}).get("platforms")
  1123. if not profile or profile == "default" or not isinstance(plats, dict):
  1124. return {}
  1125. mirrored: dict[str, Any] = {}
  1126. for name in sorted(SHARED_LISTENER_MIRROR_PLATFORMS):
  1127. entry = plats.get(name)
  1128. if not isinstance(entry, dict) or entry.get("state") not in {"connected", "connecting", "retrying"}:
  1129. continue
  1130. # api_server and webhook bind separate ports; each mirror hangs off its own listener. A record
  1131. # from an older gateway carries no ``listener_base``: connected, URL unknown.
  1132. base = entry.get("listener_base")
  1133. url = f"{base}/p/{profile}{SHARED_LISTENER_MIRROR_PATHS.get(name, '')}" if isinstance(base, str) and base else None
  1134. mirrored[name] = {k: v for k, v in entry.items() if k != "listener_base"}
  1135. mirrored[name].update(ingress_url=url, mirrored_from="default")
  1136. return mirrored
  1137. def profile_platforms_from_multiplexer(runtime: Optional[dict[str, Any]], profile: str) -> dict[str, Any]:
  1138. """The ``<profile>:<platform>`` entries of a multiplexer record, re-keyed to bare platform names — the
  1139. same shape a standalone gateway for ``profile`` writes into its own ``gateway_state.json`` — plus the
  1140. default listener's api_server/webhook mirrors the profile is served through (``ingress_url`` set)."""
  1141. plats = (runtime or {}).get("platforms")
  1142. if not isinstance(plats, dict):
  1143. return {}
  1144. prefix = f"{profile}:"
  1145. own = {key[len(prefix):]: value for key, value in plats.items()
  1146. if isinstance(key, str) and key.startswith(prefix) and isinstance(value, dict)}
  1147. if profile == "default":
  1148. # The flat keys ARE the default's own adapters (a multiplex host's primary map is always
  1149. # ``default``, whoever launched it; a standalone gateway writes only flat keys). Dropping
  1150. # them projected ``{}`` for the one profile the record never prefixes → "Restart needed"
  1151. # forever on /api/status and the Messaging card (#123088, #123869).
  1152. own.update({key: value for key, value in plats.items()
  1153. if isinstance(key, str) and ":" not in key and isinstance(value, dict)})
  1154. return {**shared_listener_mirror_platforms(runtime, profile), **own}
  1155. def resolve_gateway_liveness(
  1156. *, profile_dir: Optional[Path] = None, runtime: Any = _UNSET,
  1157. health_probe: Optional[Callable[[], tuple[bool, Optional[dict[str, Any]]]]] = None,
  1158. use_cache: bool = True, pid_probe: Optional[Callable[..., Optional[int]]] = None,
  1159. runtime_reader: Optional[Callable[..., Optional[dict[str, Any]]]] = None,
  1160. runtime_pid_probe: Optional[Callable[..., Optional[int]]] = None,
  1161. ) -> GatewayLiveness:
  1162. """Single source of truth for "is the gateway up?" across dashboard surfaces. Ladder, most to
  1163. least authoritative: (1) PID file + runtime lock (scoped to ``profile_dir``; cached by default
  1164. so polling does not re-flock ``gateway.lock``); (2) caller-supplied HTTP health probe (gateway
  1165. in another container); (3) LOCAL runtime status PID validated against the live process table
  1166. with ``expected_home`` (a recycled PID of another profile never counts; pass ``runtime`` if
  1167. already read); (4) for a named ``profile_dir`` only, the live default multiplexer that records the
  1168. profile in ``served_profiles`` (a served profile writes no identity files of its own). ``*_probe``/
  1169. ``runtime_reader`` are the dashboard's injection/test seam. A rung
  1170. that raises degrades to the next (never 500 a status endpoint) and sets ``probe_error``.
  1171. Before this existed, ``/api/status`` and ``/api/messaging/platforms`` each open-coded their own ladder
  1172. and disagreed on the same page load — the sidebar read "running" while the Channels page rendered "The
  1173. gateway is not running." Three deployments hit it: a cross-container gateway (only ``/api/status`` ran
  1174. the HTTP health probe), a profile-scoped dashboard (only ``/api/status`` passed the profile's paths, so
  1175. messaging borrowed another profile's runtime state — issue #71211), and a launch-service-managed gateway
  1176. with no PID file (only some callers used the runtime-status fallback).
  1177. """
  1178. _pid_probe = pid_probe or (get_running_pid_cached if use_cache else get_running_pid)
  1179. _runtime_reader = runtime_reader or read_runtime_status
  1180. _runtime_pid_probe = runtime_pid_probe or get_runtime_status_running_pid
  1181. probe_error = False
  1182. scoped = profile_dir is not None
  1183. def guarded(fn, *args, fallback=None, **kwargs):
  1184. nonlocal probe_error
  1185. try:
  1186. return fn(*args, **kwargs)
  1187. except Exception:
  1188. probe_error = True
  1189. return fallback
  1190. # Zero-arg call when unscoped: callers monkeypatch with zero-arg lambdas and
  1191. # /api/status's cache signature is keyed on the call shape.
  1192. pid = guarded(_pid_probe, profile_dir / "gateway.pid") if scoped else guarded(_pid_probe)
  1193. if pid is not None:
  1194. return GatewayLiveness(running=True, pid=pid, source="pid")
  1195. health_body: Optional[dict[str, Any]] = None
  1196. if health_probe is not None:
  1197. alive, health_body = guarded(health_probe, fallback=(False, None))
  1198. if alive:
  1199. # Display-only PID: it belongs to the remote container.
  1200. remote_pid = health_body.get("pid") if health_body else None
  1201. return GatewayLiveness(
  1202. running=True, pid=remote_pid, source="health", health_body=health_body
  1203. )
  1204. if runtime is _UNSET:
  1205. reader_kwargs = {"path": profile_dir / "gateway_state.json"} if scoped else {}
  1206. runtime = guarded(_runtime_reader, **reader_kwargs)
  1207. # A scoped home with no ``gateway_state.json`` gets an EMPTY record, never ``None``: ``None`` makes
  1208. # the probe re-read the PROCESS home's record and lend that gateway's PID to a home it does not
  1209. # own (a profile directory copied out of another root).
  1210. probe_kwargs = {"expected_home": profile_dir} if scoped else {}
  1211. runtime_pid = guarded(_runtime_pid_probe, {} if (scoped and runtime is None) else runtime, **probe_kwargs)
  1212. if runtime_pid is not None:
  1213. return GatewayLiveness(
  1214. running=True, pid=runtime_pid, source="runtime_status", health_body=health_body
  1215. )
  1216. # (4) A named profile served by the live default multiplexer: no identity files of its own, but
  1217. # the multiplexer IS its gateway (mirrors `hermes -p X status` / `gateway list`). Unscoped, the
  1218. # question is about the process's OWN home — which is a named profile inside a pooled
  1219. # `hermes --profile X serve` (the Desktop's per-profile backend answers its REST without
  1220. # `?profile=`), so it takes the same rung instead of reporting the served profile stopped.
  1221. own_home = profile_dir if scoped else _get_process_hermes_home()
  1222. served = guarded(multiplexer_liveness_for_profile, own_home)
  1223. if served is not None:
  1224. mux_pid, mux_runtime = served
  1225. return GatewayLiveness(
  1226. running=True, pid=mux_pid, source="multiplexer", health_body=health_body, runtime=mux_runtime
  1227. )
  1228. return GatewayLiveness(
  1229. running=False, pid=None, source="none", health_body=health_body, probe_error=probe_error
  1230. )
  1231. def get_runtime_status_running_pid(
  1232. runtime: Optional[dict[str, Any]] = None, *, expected_home: Optional[Path] = None
  1233. ) -> Optional[int]:
  1234. """Live gateway PID from the runtime status record, or None: the ``get_running_pid()`` fallback
  1235. for launch-service-managed gateways with a fresh ``gateway_state.json`` but no ``gateway.pid``.
  1236. ``expected_home`` scopes the OS-identity check to another profile's home so a PID recycled onto
  1237. a different profile's gateway is not reported running for the dead one."""
  1238. payload = runtime if runtime is not None else read_runtime_status()
  1239. if not isinstance(payload, dict):
  1240. return None
  1241. if payload.get("gateway_state") in {None, "stopped", "startup_failed"}:
  1242. return None
  1243. pid = _live_pid_from_record(payload)
  1244. if pid is None:
  1245. return None
  1246. # The record's hermes_home must match the home asked about (this process unscoped) so a stale
  1247. # or copied record cannot lend another home's gateway identity; legacy records without the
  1248. # stamp prove nothing either way and fall through to the live command-line check.
  1249. if expected_home is None and not _pid_record_belongs_to_current_profile(payload):
  1250. return None
  1251. if expected_home is not None and recorded_gateway_home_conflicts(payload, expected_home=expected_home):
  1252. return None
  1253. if not _record_matches_live_gateway_pid(payload, pid, expected_home=expected_home):
  1254. return None
  1255. return pid
  1256. def live_gateway_pid_for_home(home: Path) -> Optional[int]:
  1257. """Verified PID of the gateway owned by ``home`` (pid file + runtime lock first, then the runtime
  1258. status record), or None. Every reader of another home's gateway identity goes through this so
  1259. they all prove the same thing: the PID passes the start-time reuse guard, its live command line is
  1260. a gateway's belonging to ``home``, and the record is not ``stopped``. Bare PID existence is not
  1261. identity -- a stale record whose PID was recycled by an unrelated process lent it ``served_profiles``
  1262. and put phantom gateways into the update inventory (#109680) -- while a launch-service gateway whose
  1263. ``gateway.pid`` was unlinked is still live (#110166). Never unlinks ``home``'s identity files."""
  1264. home = Path(home)
  1265. # Cached: dashboard surfaces poll this for every served profile; the cache invalidates on any
  1266. # pid/lock file change, so a stopped or replaced gateway is seen at once.
  1267. pid = get_running_pid_cached(home / "gateway.pid", cleanup_stale=False)
  1268. if pid is not None:
  1269. return pid
  1270. return get_runtime_status_running_pid(read_runtime_status(home / "gateway_state.json"), expected_home=home)
  1271. def remove_pid_file() -> None:
  1272. """Remove the PID file only if it belongs to this process: during --replace the old process's
  1273. atexit can fire AFTER the new process wrote its own record."""
  1274. with contextlib.suppress(Exception):
  1275. path = _get_pid_path()
  1276. file_pid = _pid_from_record(_read_json_file(path))
  1277. if file_pid is not None and file_pid != os.getpid():
  1278. return # Belongs to a different process — leave it alone.
  1279. path.unlink(missing_ok=True)
  1280. _clear_running_pid_cache()
  1281. def _process_is_stopped(pid: int) -> bool:
  1282. """True for a stopped / tracing-stop state (T/t) in ``/proc/<pid>/status``."""
  1283. with contextlib.suppress(OSError):
  1284. for line in Path(f"/proc/{pid}/status").read_text(encoding="utf-8").splitlines():
  1285. if line.startswith("State:"):
  1286. return line.split()[1] in {"T", "t"}
  1287. return False
  1288. def acquire_scoped_lock(
  1289. scope: str, identity: str, metadata: Optional[dict[str, Any]] = None
  1290. ) -> tuple[bool, Optional[dict[str, Any]]]:
  1291. """Acquire a machine-local lock keyed by scope + identity (one Telegram token across homes).
  1292. ``metadata={"owner_kind": "process"}`` allows a non-gateway owner such as
  1293. a setup process. Its PID/start-time identity still gates stale recovery.
  1294. Raises :class:`gateway.status_lock_dir.ScopedLockDirUnusable` (logged once per dir with the repair) when this token may
  1295. not use the lock dir at all."""
  1296. lock_path = _get_scope_lock_path(scope, identity)
  1297. try:
  1298. acquired = _acquire_scoped_lock_at(lock_path, scope, identity, metadata)
  1299. except PermissionError as exc:
  1300. raise unusable_lock_dir(lock_path.parent, exc) from exc
  1301. unusable_lock_dir_recovered(lock_path.parent)
  1302. return acquired
  1303. def _acquire_scoped_lock_at(
  1304. lock_path: Path, scope: str, identity: str, metadata: Optional[dict[str, Any]]
  1305. ) -> tuple[bool, Optional[dict[str, Any]]]:
  1306. from gateway.status_scoped_locks import _scoped_lock_record_is_stale
  1307. make_user_dir(lock_path.parent, parents=True, exist_ok=True)
  1308. # An existing lock this token may not read (left by an elevated gateway) must raise here:
  1309. # ``_read_json_file`` would call it empty and O_EXCL "in use", a conflict with no holder.
  1310. with contextlib.suppress(FileNotFoundError), open(lock_path, "rb"):
  1311. pass
  1312. record = {
  1313. **_build_pid_record(), "scope": scope, "identity_hash": _scope_hash(identity),
  1314. "metadata": metadata or {}, "updated_at": _utc_now_iso(),
  1315. }
  1316. # Profile label for cross-profile conflict diagnostics ("token already in use (PID 559)" alone
  1317. # does not say WHICH profile). Omitted when not inferable; readers fall back to hermes_home.
  1318. profile = _profile_label_for_home(_get_process_hermes_home())
  1319. if profile:
  1320. record["profile"] = profile
  1321. existing = _read_json_file(lock_path)
  1322. if existing is None and lock_path.exists():
  1323. # Empty/invalid JSON: previous process died between O_EXCL create and json.dump().
  1324. _unlink_quietly(lock_path)
  1325. if existing:
  1326. existing_pid = _pid_from_record(existing)
  1327. # Our own PID: always self-reacquire. start_time guards reuse of OTHER PIDs; requiring
  1328. # equality here rejects reconnects when the on-disk record has start_time null.
  1329. # Same live PID as this process: always self-reacquire. ``start_time`` is a PID-reuse guard for
  1330. # *other* PIDs; it cannot distinguish two processes that share the caller's own PID (impossible
  1331. # while we are alive). Requiring start_time equality here falsely rejects reconnects when the
  1332. # on-disk record has ``start_time: null`` (older writers / psutil failure at first write) while the
  1333. # freshly built record has a real value — the gateway then reports itself as the foreign squatter of
  1334. # its own token (#81468).
  1335. if existing_pid == os.getpid():
  1336. _write_json_file(lock_path, record)
  1337. return True, existing
  1338. if not _scoped_lock_record_is_stale(existing, existing_pid):
  1339. return False, existing
  1340. # Rename to a tombstone instead of unlink(): with unlink()+O_EXCL two racing starters
  1341. # could both win. os.replace() lets exactly one claim it; a failed replace means another
  1342. # racer claimed it and O_EXCL below decides.
  1343. with contextlib.suppress(OSError):
  1344. tombstone = lock_path.with_name(lock_path.name + ".stale")
  1345. os.replace(lock_path, tombstone)
  1346. _unlink_quietly(tombstone)
  1347. try:
  1348. _write_json_excl(lock_path, record)
  1349. except FileExistsError:
  1350. return False, _read_json_file(lock_path)
  1351. return True, None
  1352. # ── --replace takeover marker ─────────────────────────────────────────
  1353. # SIGTERM exits the gateway with code 1 so Restart=on-failure revives it after unexpected
  1354. # kills -- which would also revive a --replace target (flap loop against the replacer). The
  1355. # replacer therefore writes a short-lived marker naming the target PID + start_time BEFORE
  1356. # SIGTERM; the target's shutdown handler treats a matching marker as a planned takeover and
  1357. # exits 0. Unlinked once consumed, so a stale one can grief at most one future shutdown on
  1358. # the same PID, within _TAKEOVER_MARKER_TTL_S.
  1359. # When a new gateway starts with ``--replace``, it SIGTERMs the existing gateway so it can take over the bot
  1360. # token. ``hermes.service`` + ``hermes- gateway.service``). See #5646.
  1361. _TAKEOVER_MARKER_FILENAME = ".gateway-takeover.json"
  1362. _TAKEOVER_MARKER_TTL_S = 60 # Marker older than this is treated as stale
  1363. _PLANNED_STOP_MARKER_FILENAME = ".gateway-planned-stop.json"
  1364. _PLANNED_STOP_MARKER_TTL_S = 60
  1365. def _get_takeover_marker_path(hermes_home: Optional[Path] = None) -> Path:
  1366. """Takeover marker path; ``hermes_home`` is given only for a verified cross-home handoff."""
  1367. home = _canonical_hermes_home(hermes_home or _get_process_hermes_home())
  1368. return home / _TAKEOVER_MARKER_FILENAME
  1369. def _get_planned_stop_marker_path() -> Path:
  1370. return _get_process_hermes_home() / _PLANNED_STOP_MARKER_FILENAME
  1371. def _marker_is_stale(written_at: str, ttl_s: int) -> bool:
  1372. try:
  1373. age = datetime.now(UTC) - datetime.fromisoformat(written_at)
  1374. return age.total_seconds() > ttl_s
  1375. except (TypeError, ValueError):
  1376. return True
  1377. def _read_live_pid_marker(path: Path, ttl_s: int) -> Optional[tuple[dict[str, Any], int, Any]]:
  1378. """``(record, target_pid, target_start_time)`` for a usable marker, else None. Malformed/expired
  1379. markers can never match anyone, so they are unlinked here (must not wedge a new instance) --
  1380. except a request stamped ``accepted`` (``update_pause_record.mark_stop_accepted``): expired, it
  1381. matches nobody, but it can be the only trace that a still-draining gateway accepted an update's
  1382. stop, so it stays until that pause is settled."""
  1383. record = _read_json_file(path)
  1384. if not record:
  1385. return None
  1386. target_pid = _pid_from_record(record, "target_pid")
  1387. if target_pid is None or _marker_is_stale(record.get("written_at") or "", ttl_s):
  1388. if record.get("accepted") is not True or not _update_pause_on_disk():
  1389. _unlink_quietly(path)
  1390. return None
  1391. return record, target_pid, record.get("target_start_time")
  1392. def _pid_marker_names_self(target_pid: int, target_start_time: Any) -> bool:
  1393. """PID match with an optional start-time PID-reuse guard (watcher probe + consume). Both start
  1394. times known -> must match; either unknown -> PID equality decides (bounded by the marker TTL):
  1395. ``_get_process_start_time`` is None without /proc (macOS, native Windows -- where the
  1396. planned-stop watcher matters most) and requiring a match there would misclassify a legitimate
  1397. ``hermes gateway stop`` as an unexpected exit revived by the service manager."""
  1398. if target_pid != os.getpid():
  1399. return False
  1400. our_start_time = _get_process_start_time(target_pid)
  1401. return None in (target_start_time, our_start_time) or target_start_time == our_start_time
  1402. def _consume_pid_marker_for_self(path: Path, *, ttl_s: int, on_consume=None, keep: bool = False) -> bool:
  1403. parsed = _read_live_pid_marker(path, ttl_s)
  1404. if parsed is None:
  1405. return False
  1406. record, target_pid, target_start_time = parsed
  1407. # Cross-profile guard: new markers name the verified TARGET home, which permits a deliberate
  1408. # cross-HERMES_HOME --replace while ignoring a marker accidentally written into another
  1409. # profile's directory. Legacy markers have no target field: keep the same-replacer-home rule.
  1410. # See #29092.
  1411. our_home = _get_process_hermes_home()
  1412. target_home = record.get("target_hermes_home")
  1413. if target_home is not None:
  1414. if not isinstance(target_home, str) or not _same_hermes_home(target_home, our_home):
  1415. return False
  1416. else:
  1417. replacer_home = record.get("replacer_hermes_home")
  1418. if replacer_home is not None and not _same_hermes_home(replacer_home, our_home):
  1419. return False
  1420. matches = _pid_marker_names_self(target_pid, target_start_time)
  1421. if matches and on_consume is not None:
  1422. on_consume(path, record)
  1423. if not keep:
  1424. _unlink_quietly(path)
  1425. return matches
  1426. def write_takeover_marker(
  1427. target_pid: int, *, target_home: Optional[Path] = None, target_start_time: Any = _UNSET
  1428. ) -> bool:
  1429. """Record that ``target_pid`` is being replaced by this process; True on success. Captures the
  1430. target's ``start_time`` (PID-reuse guard) + a timestamp for TTL. A verified cross-home handoff
  1431. passes ``target_home`` + validated ``target_start_time`` so the marker lands in the target's
  1432. home; such callers must fail closed on False (the target's supervisor could revive it)."""
  1433. try:
  1434. marker_home = _canonical_hermes_home(target_home or _get_process_hermes_home())
  1435. if target_start_time is _UNSET:
  1436. target_start_time = _get_process_start_time(target_pid)
  1437. return _write_marker(_get_takeover_marker_path(marker_home), {
  1438. "target_pid": target_pid, "target_start_time": target_start_time,
  1439. "target_hermes_home": str(marker_home), "replacer_pid": os.getpid(),
  1440. "replacer_hermes_home": str(_canonical_hermes_home(_get_process_hermes_home())),
  1441. "written_at": _utc_now_iso(),
  1442. })
  1443. except OSError:
  1444. return False
  1445. def _write_marker(path: Path, record: dict[str, Any]) -> bool:
  1446. """Atomically write a marker record; False (never raise) on OS failure."""
  1447. try:
  1448. _write_json_file(path, record)
  1449. return True
  1450. except OSError:
  1451. return False
  1452. def consume_takeover_marker_for_self() -> bool:
  1453. """Consume the takeover marker; True => planned takeover (exit 0); unlinked on match/stale."""
  1454. return _consume_pid_marker_for_self(_get_takeover_marker_path(), ttl_s=_TAKEOVER_MARKER_TTL_S)
  1455. def clear_takeover_marker(target_home: Optional[Path] = None) -> None:
  1456. """Remove the takeover marker unconditionally. Safe to call repeatedly."""
  1457. _unlink_quietly(_get_takeover_marker_path(target_home))
  1458. def _validated_scoped_lock_gateway_owner(record: dict[str, Any]) -> Optional[tuple[int, int, Path]]:
  1459. """Resolve a live scoped-lock owner to a verified ``(pid, start_time, home)``. A lock file is
  1460. only a claim: the record, the target home's PID record, and the live process must agree on
  1461. PID, start-time, gateway identity, and home. Missing legacy metadata fails closed."""
  1462. if not isinstance(record, dict) or not _record_looks_like_gateway(record):
  1463. return None
  1464. owner_pid = _pid_from_record(record)
  1465. owner_start_time = record.get("start_time")
  1466. raw_home = record.get("hermes_home")
  1467. if (
  1468. owner_pid is None or owner_pid <= 0 or owner_pid == os.getpid()
  1469. or not isinstance(owner_start_time, int) or isinstance(owner_start_time, bool)
  1470. or not isinstance(raw_home, str) or not raw_home.strip()
  1471. or not Path(raw_home).expanduser().is_absolute()
  1472. ):
  1473. return None
  1474. target_home = _canonical_hermes_home(raw_home)
  1475. if _scoped_lock_owner_state(owner_pid, owner_start_time) != "same":
  1476. return None
  1477. live_cmdline = _read_process_cmdline(owner_pid)
  1478. if live_cmdline is not None and not looks_like_gateway_runtime_command_line(live_cmdline):
  1479. return None
  1480. # The target home's own PID record must corroborate the claim.
  1481. pid_record = _read_json_file(target_home / "gateway.pid") or {}
  1482. pid_record_home = pid_record.get("hermes_home")
  1483. if (
  1484. not _record_looks_like_gateway(pid_record)
  1485. or _pid_from_record(pid_record) != owner_pid
  1486. or pid_record.get("start_time") != owner_start_time
  1487. or not isinstance(pid_record_home, str)
  1488. or not _same_hermes_home(pid_record_home, target_home)
  1489. ):
  1490. return None
  1491. return owner_pid, owner_start_time, target_home
  1492. def _scoped_lock_owner_state(owner_pid: int, owner_start_time: int) -> str:
  1493. """Return ``same``, ``exited``, or ``unknown`` for a validated owner."""
  1494. if not _pid_exists(owner_pid):
  1495. return "exited"
  1496. live_start_time = _get_process_start_time(owner_pid)
  1497. # A different start time means the PID was recycled; never signal the replacement.
  1498. if live_start_time is None:
  1499. return "unknown"
  1500. return "same" if live_start_time == owner_start_time else "exited"
  1501. def _wait_for_scoped_lock_owner_exit(
  1502. owner_pid: int, owner_start_time: int, *, attempts: int, delay: float
  1503. ) -> tuple[bool, bool]:
  1504. """Return ``(exited, safe_to_force)`` after bounded identity-aware waits."""
  1505. for _ in range(max(0, attempts)):
  1506. state = _scoped_lock_owner_state(owner_pid, owner_start_time)
  1507. if state == "exited":
  1508. return True, False
  1509. if state == "unknown":
  1510. return False, False
  1511. time.sleep(max(0.0, delay))
  1512. return False, _scoped_lock_owner_state(owner_pid, owner_start_time) == "same"
  1513. def _snapshot_gateway_children(pid: int) -> list:
  1514. """Best-effort snapshot of ``pid``'s live descendants (POSIX only; never raises). Take it while
  1515. the parent is alive -- once it exits the children are reparented and undiscoverable. ``[]`` on
  1516. Windows (taskkill /T tree-kills)."""
  1517. if _IS_WINDOWS:
  1518. return []
  1519. try:
  1520. import psutil # type: ignore
  1521. return psutil.Process(int(pid)).children(recursive=True)
  1522. except Exception:
  1523. logger.debug("Could not snapshot children of gateway PID %d", pid, exc_info=True)
  1524. return []
  1525. def reap_gateway_children(children: list, *, parent_pid: int, timeout: float = 5.0) -> int:
  1526. """Best-effort reap of a dead gateway's orphaned descendants (POSIX; surviving adapter
  1527. subprocesses keep holding token locks); returns count signalled. Call only AFTER the parent is
  1528. confirmed dead, with a :func:`_snapshot_gateway_children` snapshot. ``is_running()`` is
  1529. identity-aware so a recycled child PID is never signalled; a child whose ppid still equals
  1530. ``parent_pid`` is skipped (parent alive => not an orphan). SIGTERM, bounded wait, SIGKILL
  1531. survivors. Never raises."""
  1532. if _IS_WINDOWS or not children:
  1533. return 0
  1534. reaped = 0
  1535. try:
  1536. import psutil # type: ignore
  1537. live = []
  1538. for child in children:
  1539. try:
  1540. if not child.is_running() or child.status() == psutil.STATUS_ZOMBIE:
  1541. continue
  1542. if child.ppid() == parent_pid:
  1543. logger.debug("Skipping child PID %d of old gateway %d: parent still appears "
  1544. "alive", child.pid, parent_pid)
  1545. continue
  1546. child.terminate()
  1547. live.append(child)
  1548. except psutil.NoSuchProcess:
  1549. continue
  1550. except Exception:
  1551. logger.debug("Could not terminate child PID %s of old gateway %d",
  1552. getattr(child, "pid", "?"), parent_pid, exc_info=True)
  1553. if not live:
  1554. return 0
  1555. gone, alive = psutil.wait_procs(live, timeout=max(0.0, timeout))
  1556. reaped = len(gone)
  1557. for child in alive:
  1558. try:
  1559. child.kill()
  1560. reaped += 1
  1561. except Exception:
  1562. logger.debug("Could not force-kill child PID %s of old gateway %d",
  1563. getattr(child, "pid", "?"), parent_pid, exc_info=True)
  1564. if reaped:
  1565. logger.info("Reaped %d orphaned child process(es) of replaced gateway PID %d.",
  1566. reaped, parent_pid)
  1567. except Exception:
  1568. logger.debug("Child reap for replaced gateway PID %d failed", parent_pid, exc_info=True)
  1569. return reaped
  1570. def take_over_scoped_lock_holder(
  1571. record: dict[str, Any], *, graceful_attempts: int = 20, force_attempts: int = 20
  1572. ) -> Optional[int]:
  1573. """Terminate one verified scoped-lock holder for explicit ``--replace``. Returns the owner PID
  1574. only after that exact PID/start-time identity exited; validation or marker-write failure returns
  1575. None without signalling (a cross-home handoff must place a consumable marker in the target's
  1576. home or its supervisor revives it: flap loop). On POSIX the snapshotted children are reaped."""
  1577. owner = _validated_scoped_lock_gateway_owner(record)
  1578. if owner is None:
  1579. return None
  1580. owner_pid, owner_start_time, target_home = owner
  1581. # Snapshot while the owner is alive; afterwards children are reparented.
  1582. owner_children = _snapshot_gateway_children(owner_pid)
  1583. if not write_takeover_marker(
  1584. owner_pid, target_home=target_home, target_start_time=owner_start_time
  1585. ):
  1586. return None
  1587. try:
  1588. replaced = _terminate_verified_owner(
  1589. owner_pid, owner_start_time, graceful_attempts=graceful_attempts,
  1590. force_attempts=force_attempts,
  1591. )
  1592. finally:
  1593. # The target normally consumes the marker; clean up any remainder.
  1594. clear_takeover_marker(target_home)
  1595. if replaced is not None:
  1596. reap_gateway_children(owner_children, parent_pid=owner_pid)
  1597. return replaced
  1598. def _terminate_verified_owner(
  1599. owner_pid: int, owner_start_time: int, *, graceful_attempts: int, force_attempts: int
  1600. ) -> Optional[int]:
  1601. """Bounded identity-aware SIGTERM-then-SIGKILL of a verified owner; the PID once it exited, else
  1602. None. Per signal step: ``ProcessLookupError`` => already gone; other ``OSError`` => refuse."""
  1603. state = _scoped_lock_owner_state(owner_pid, owner_start_time)
  1604. if state == "exited":
  1605. return owner_pid
  1606. if state != "same":
  1607. return None
  1608. for attempts, delay, kwargs in (
  1609. (graceful_attempts, 0.5, {"force": False}),
  1610. (force_attempts, 0.25, {"force": True, "expected_start_time": owner_start_time}),
  1611. ):
  1612. try:
  1613. terminate_pid(owner_pid, **kwargs)
  1614. except ProcessLookupError:
  1615. return owner_pid
  1616. except OSError:
  1617. return None
  1618. exited, safe_to_force = _wait_for_scoped_lock_owner_exit(
  1619. owner_pid, owner_start_time, attempts=attempts, delay=delay
  1620. )
  1621. if exited:
  1622. return owner_pid
  1623. if not safe_to_force:
  1624. return None
  1625. return None
  1626. def write_planned_stop_marker(target_pid: int) -> bool:
  1627. """Record that ``target_pid`` is being stopped intentionally: unexpected SIGTERM exits non-zero
  1628. so service managers revive the gateway; the CLI writes this first so a deliberate stop exits
  1629. cleanly."""
  1630. return _write_marker(_get_planned_stop_marker_path(), {
  1631. "target_pid": target_pid, "target_start_time": _get_process_start_time(target_pid),
  1632. "stopper_pid": os.getpid(), "written_at": _utc_now_iso(),
  1633. })
  1634. _PLANNED_STOP_MUTEX_WAIT_S = 2.0
  1635. def _update_pause_on_disk() -> bool:
  1636. """A Windows update pause of this checkout is on disk (its record or a recovery's claim), or the
  1637. record directory cannot be listed. Only then does a consume need the pause lock and a checkpoint,
  1638. and only then can accepted-stop evidence beside a request still be owed: with none, every pause
  1639. was settled by recovery or the updater (or never made: every non-Windows host)."""
  1640. from hermes_cli import update_pause_record
  1641. record = update_pause_record.record_path()
  1642. try:
  1643. with os.scandir(record.parent) as entries:
  1644. return any(e.name == record.name or (e.name.startswith(record.name + ".") and e.name.endswith(".claim"))
  1645. for e in entries)
  1646. except FileNotFoundError:
  1647. return False
  1648. except OSError:
  1649. return True
  1650. def _checkpoint_planned_stop(path: Path, record: dict[str, Any]) -> None:
  1651. """``on_consume``, decided after the request was read: an update writes its record before its
  1652. request, so a pause that appeared since the consumer first looked is checkpointed too."""
  1653. from hermes_cli import update_pause_record
  1654. if _update_pause_on_disk():
  1655. with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S):
  1656. update_pause_record.mark_stop_consumed(path, record)
  1657. def consume_planned_stop_marker_for_self() -> bool:
  1658. """Return True when the current process is being intentionally stopped."""
  1659. from hermes_cli import update_pause_record
  1660. path = _get_planned_stop_marker_path()
  1661. try:
  1662. paused = _update_pause_on_disk()
  1663. if not paused: # an earlier drain's receipt: its debt is settled
  1664. _unlink_quietly(update_pause_record._accepted_path(path))
  1665. # With a pause on disk, recovery must see either the request or its checkpoint, never the
  1666. # gap between validating/consuming the marker and scheduling asynchronous stop.
  1667. with update_pause_record._mutex(_PLANNED_STOP_MUTEX_WAIT_S) if paused else contextlib.nullcontext():
  1668. return _consume_pid_marker_for_self(path, ttl_s=_PLANNED_STOP_MARKER_TTL_S,
  1669. on_consume=_checkpoint_planned_stop)
  1670. except OSError as exc:
  1671. # The pause bookkeeping (a busy mutex, a checkpoint that cannot be written) never decides
  1672. # whether this stop was planned: classify without consuming, and leave a receipt beside the
  1673. # request so recovery keeps this drain owed after the request's TTL (mark_stop_accepted).
  1674. logger.warning("Planned-stop checkpoint skipped (%s); the stop request stays on disk", exc)
  1675. return _consume_pid_marker_for_self(_get_planned_stop_marker_path(), ttl_s=_PLANNED_STOP_MARKER_TTL_S,
  1676. keep=True, on_consume=update_pause_record.mark_stop_accepted)
  1677. def planned_stop_marker_targets_self() -> bool:
  1678. """Non-destructive watcher probe: True when a live planned-stop marker names us. Never unlinks a
  1679. matching marker (the shutdown handler does the authoritative consume); malformed/expired ones
  1680. are still cleaned up; markers naming another PID are left alone."""
  1681. parsed = _read_live_pid_marker(_get_planned_stop_marker_path(), _PLANNED_STOP_MARKER_TTL_S)
  1682. return parsed is not None and _pid_marker_names_self(parsed[1], parsed[2])
  1683. def get_running_pid(
  1684. pid_path: Optional[Path] = None, *, cleanup_stale: bool = True
  1685. ) -> Optional[int]:
  1686. """PID of a running gateway (lock + PID file verified against the live process), or None.
  1687. An explicit ``pid_path`` is a scoped query into that home's identity files: records are
  1688. validated against the probed home (not the serve process's) (#106406). While the runtime lock
  1689. is HELD, a live SAME-home record the identity matcher rejects is never cleanup-unlinked, scoped
  1690. or not: the holder is a gateway whatever its command line reads as, and unlinking a held
  1691. ``gateway.lock`` leaves it locking a deleted inode so the next starter double-runs (#125610,
  1692. #123109). Unscoped, a live ``gateway.pid`` that truthfully names ANOTHER home's gateway is
  1693. poison inside this home and is unlinked on refusal — the held lock stays (#89315). Dead
  1694. records and inactive-lock metadata still take the full poison-file cleanup."""
  1695. resolved_pid_path = pid_path or _get_pid_path()
  1696. resolved_lock_path = _get_gateway_lock_path(resolved_pid_path)
  1697. if is_gateway_runtime_lock_active(resolved_lock_path):
  1698. records = (
  1699. _read_pid_record(resolved_pid_path), _read_gateway_lock_record(resolved_lock_path),
  1700. )
  1701. expected_home = pid_path.parent if pid_path is not None else None
  1702. saw_live_pid = False
  1703. foreign_live_pid = False
  1704. for record in records:
  1705. pid = _live_pid_from_record(record)
  1706. if pid is None:
  1707. continue
  1708. home_ok = (
  1709. _pid_record_belongs_to_current_profile(record) if expected_home is None
  1710. else not recorded_gateway_home_conflicts(record, expected_home=expected_home)
  1711. )
  1712. if home_ok and _record_matches_live_gateway_pid(
  1713. record, pid, expected_home=expected_home
  1714. ):
  1715. return pid
  1716. # A same-home record the matcher could not adopt may still be a real gateway (an
  1717. # identity matcher that lags a new launcher shape, a record written by an older
  1718. # version): that rejection is not stale-file authority while the PID is alive and the
  1719. # lock held. A scoped poll never unlinks the other home's files either (#106406).
  1720. if home_ok or expected_home is not None:
  1721. saw_live_pid = True
  1722. else:
  1723. foreign_live_pid = True
  1724. if not saw_live_pid:
  1725. _cleanup_invalid_pid_path(
  1726. resolved_pid_path, cleanup_stale=cleanup_stale, unlink_lock=not foreign_live_pid
  1727. )
  1728. return get_runtime_status_running_pid() if pid_path is None else None
  1729. # Lock inactive: the runtime-status fallback runs BEFORE cleanup here. A record naming THIS
  1730. # process is one we wrote earlier in our own boot (the multiplex verdict is persisted before
  1731. # the PID claim), not a rival: treating it as live skipped the stale gateway.pid cleanup after
  1732. # a SIGKILLed owner and the O_EXCL claim then lost the "PID file race" to itself.
  1733. runtime_pid = get_runtime_status_running_pid() if pid_path is None else None
  1734. if runtime_pid is None or runtime_pid == os.getpid():
  1735. _cleanup_invalid_pid_path(resolved_pid_path, cleanup_stale=cleanup_stale)
  1736. return runtime_pid
  1737. def get_running_pid_identity_strict(pid_path: Path) -> Optional[tuple[int, float]]:
  1738. """Return a verified process identity or fail on ambiguous runtime state."""
  1739. resolved_pid_path = Path(pid_path)
  1740. resolved_lock_path = _get_gateway_lock_path(resolved_pid_path)
  1741. pid_exists = _strict_path_exists(resolved_pid_path, "gateway PID")
  1742. # A stale PID file without a lock is not a live gateway; the lock probe is authoritative
  1743. # for absence.
  1744. if not _strict_path_exists(resolved_lock_path, "gateway lock"):
  1745. return None
  1746. if not _is_gateway_runtime_lock_active_strict(resolved_lock_path):
  1747. return None
  1748. lock_record = _read_gateway_lock_record(resolved_lock_path)
  1749. # The PID file is advisory beside a HELD lock: a launch-service gateway keeps serving after its
  1750. # gateway.pid was unlinked (#110166) and --replace force-unlinks the old one (#123430) or leaves
  1751. # it behind (#143331); the lock record validated against the live process below is the proof.
  1752. from gateway.status_pid_records import strict_identity_records
  1753. records = strict_identity_records(resolved_pid_path, lock_record, pid_exists)
  1754. if not all(records):
  1755. raise RuntimeError("gateway PID or lock metadata is malformed")
  1756. pid = _pid_from_record(records[0])
  1757. if pid is None or pid <= 0 or any(_pid_from_record(record) != pid for record in records[1:]):
  1758. raise RuntimeError("gateway PID and lock identities disagree")
  1759. if not _pid_exists(pid):
  1760. raise RuntimeError("gateway identity is not live")
  1761. current_start = _get_process_start_time(pid)
  1762. starts = tuple(record.get("start_time") for record in records)
  1763. if current_start is None or any(start is None for start in starts):
  1764. raise RuntimeError("gateway creation time is unavailable")
  1765. try:
  1766. if not _start_times_agree(current_start, *starts):
  1767. raise RuntimeError("gateway process identity changed")
  1768. except (TypeError, ValueError) as exc:
  1769. raise RuntimeError("gateway creation time is malformed") from exc
  1770. if not all(_record_matches_live_gateway_pid(record, pid) for record in records):
  1771. raise RuntimeError("runtime metadata does not identify a live gateway")
  1772. current = float(current_start)
  1773. if not _IS_WINDOWS:
  1774. return pid, current
  1775. # Windows persists a centisecond fingerprint; SCM checks need the exact psutil epoch.
  1776. # Re-read only after validation and prove it rounds to the same value.
  1777. try:
  1778. import psutil # type: ignore
  1779. exact_create_time = float(psutil.Process(pid).create_time())
  1780. except Exception as exc:
  1781. raise RuntimeError("exact gateway creation time is unavailable") from exc
  1782. if round(exact_create_time * 100) != int(current):
  1783. raise RuntimeError("gateway process identity changed")
  1784. return pid, exact_create_time
  1785. def get_running_pid_cached(
  1786. pid_path: Optional[Path] = None, *, cleanup_stale: bool = True,
  1787. ttl_seconds: float = _GATEWAY_RUNNING_PID_CACHE_TTL_SECONDS,
  1788. ) -> Optional[int]:
  1789. """Cached ``get_running_pid()`` for dashboard polling: short TTL, invalidated on PID/lock/
  1790. runtime-status file changes, so status endpoints do not re-flock ``gateway.lock`` constantly."""
  1791. if ttl_seconds <= 0:
  1792. return get_running_pid(pid_path, cleanup_stale=cleanup_stale)
  1793. resolved_pid_path = pid_path or _get_pid_path()
  1794. include_runtime_status = pid_path is None
  1795. # The signature covers the PID file, its sibling lock and (unscoped) the runtime status file.
  1796. watched = [resolved_pid_path, _get_gateway_lock_path(resolved_pid_path)]
  1797. if include_runtime_status:
  1798. watched.append(_get_runtime_status_path())
  1799. signature = tuple(_file_cache_signature(p) for p in watched)
  1800. key = (str(resolved_pid_path), bool(cleanup_stale), include_runtime_status)
  1801. now = time.monotonic()
  1802. with _gateway_running_pid_cache_lock:
  1803. cached = _gateway_running_pid_cache.get(key)
  1804. if cached is not None and now - cached[0] <= ttl_seconds and cached[1] == signature:
  1805. return cached[2]
  1806. pid = get_running_pid(pid_path, cleanup_stale=cleanup_stale)
  1807. refreshed_signature = tuple(_file_cache_signature(p) for p in watched)
  1808. with _gateway_running_pid_cache_lock:
  1809. _gateway_running_pid_cache[key] = (time.monotonic(), refreshed_signature, pid)
  1810. return pid