process_identity.py 51 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145
  1. """Process identity: spawn tags, the machine-wide spawn ledger, the Windows job-object self-attach,
  2. and the POSIX owner-death supervisor.
  3. Four layers make every long-lived Hermes process positively identifiable, so reapers (``hermes
  4. update``, Desktop startup sweeps) never guess lineage from PPID archaeology or cmdline matching,
  5. and make every long-lived CHILD die with the Hermes process that owns it:
  6. 1. spawn tags (``HERMES_SPAWN`` env stamped by the spawner); 2. a ``(pid, create_time)`` ledger;
  7. 3. Windows job-object self-attach with ``KILL_ON_JOB_CLOSE`` so the whole child tree dies with the
  8. root — no launcher→worker chains left holding ``.pyd`` locks; 4. on POSIX, one supervisor per
  9. Hermes process that reaps the owned children's process groups the moment Hermes dies.
  10. """
  11. from __future__ import annotations
  12. import functools
  13. import hashlib
  14. import json
  15. import logging
  16. import os
  17. import platform
  18. import signal
  19. import subprocess
  20. import sys
  21. import threading
  22. import time
  23. from contextlib import ExitStack, contextmanager, suppress
  24. from contextvars import ContextVar
  25. from dataclasses import asdict, dataclass
  26. from pathlib import Path
  27. from typing import Iterator, Optional, Sequence
  28. from utils import atomic_json_write
  29. logger = logging.getLogger(__name__)
  30. SPAWN_ENV_VAR = "HERMES_SPAWN"
  31. _TAG_VERSION = "v1"
  32. LEDGER_FILENAME = "spawn-ledger.json"
  33. #: Purposes a reaper may treat as "safe to kill when the owner is gone".
  34. #: Interactive processes (chat, REPLs) are deliberately NOT in this set.
  35. REAPABLE_PURPOSES = frozenset({"serve", "dashboard", "gateway", "mcp-helper"})
  36. #: Host-role owners a fresh backend may reap on a HELD_BY_OTHER conflict (#121964): the
  37. #: serve/dashboard/gateway subset. The mcp-helper rung stays owned by reap_orphaned_mcp_helpers.
  38. _BACKEND_OWNER_PURPOSES = REAPABLE_PURPOSES - {"mcp-helper"}
  39. _IS_WINDOWS = platform.system() == "Windows"
  40. # Module-global job handle: must live exactly as long as this process so the
  41. # kernel closes it (and kills the job) when we die. Never close it manually.
  42. _JOB_HANDLE = None
  43. _LEDGER_LOCK = threading.Lock()
  44. def install_id(project_root: Optional[Path] = None) -> str:
  45. """Stable 12-hex identifier for THIS install (derived from its path)."""
  46. if project_root is None:
  47. try:
  48. from hermes_constants import PROJECT_ROOT as _root
  49. project_root = Path(_root)
  50. except Exception:
  51. project_root = Path(__file__).resolve().parent.parent
  52. try:
  53. canonical = str(Path(project_root).resolve()).lower()
  54. except OSError:
  55. canonical = str(project_root).lower()
  56. return hashlib.sha256(canonical.encode("utf-8", "replace")).hexdigest()[:12]
  57. def process_birth(proc) -> float:
  58. """A managed worker's identity birth for the ``psutil.Process`` *proc*, the same in every process.
  59. On macOS psutil shifts ``create_time()`` by ``abs(INIT_BOOT_TIME - boot_time())``
  60. (``psutil._psosx.adjust_proc_create_time``), with a boot-time reference frozen when psutil
  61. was imported in the CALLING process. After a clock step (NTP, sleep/wake) a long-lived
  62. gateway owner and a freshly spawned worker read different births for the same process, and
  63. the exact identity match refused every managed worker (#140580). The kernel's own start
  64. (``monotonic=True``: the unadjusted ``kinfo_proc`` epoch start) is what every process sees.
  65. Other platforms keep ``create_time()``: Linux and Windows apply no per-process reference.
  66. Raises ``psutil.Error`` like ``create_time()``.
  67. """
  68. import psutil
  69. if getattr(psutil, "MACOS", False):
  70. platform_proc = getattr(proc, "_proc", None)
  71. if platform_proc is not None:
  72. try:
  73. return float(platform_proc.create_time(monotonic=True))
  74. except TypeError: # a psutil without the monotonic read: fall back to the adjusted one
  75. pass
  76. return float(proc.create_time())
  77. def worker_birth(proc) -> float:
  78. """The birth a managed worker's identity fences compare: its hello, its scope and the Kanban
  79. owner receipt. :func:`process_birth`, except on Linux, where psutil's ``create_time()`` adds the
  80. ``/proc/stat`` btime it re-reads on every call. WSL2 re-anchors btime after host sleep, so the
  81. owner read a live worker's birth shifted from the one it reported and refused it (KB-12). The
  82. kernel's start ticks since boot (``monotonic=True``) never move. The spawn ledger keeps
  83. :func:`process_birth`: its persisted epoch births must keep matching across an update.
  84. Raises ``psutil.Error`` like ``create_time()``.
  85. """
  86. import psutil
  87. if getattr(psutil, "LINUX", False):
  88. return float(proc._proc.create_time(monotonic=True))
  89. return process_birth(proc)
  90. def boot_start(proc) -> Optional[float]:
  91. """Seconds from boot to the start of the ``psutil.Process`` *proc* on Linux; ``None`` elsewhere.
  92. The ledger's epoch ``create_time`` is :func:`process_birth`, which on Linux is the kernel's
  93. start ticks plus the ``/proc/stat`` btime psutil re-reads on every call, so an NTP or VM-resume
  94. clock step moves it by whole seconds and a live process stops matching its entry. Ledger
  95. entries carry this alongside it (``boot_start`` / ``spawner_boot``) and compare it exactly.
  96. Raises ``psutil.Error`` like ``create_time()``.
  97. """
  98. import psutil
  99. if getattr(psutil, "LINUX", False):
  100. return float(proc._proc.create_time(monotonic=True))
  101. return None
  102. def _process_create_time(pid: Optional[int] = None) -> Optional[float]:
  103. """:func:`process_birth` of ``pid`` (default: this process); ``None`` when psutil can't say."""
  104. try:
  105. import psutil
  106. return process_birth(psutil.Process(os.getpid() if pid is None else pid))
  107. except Exception:
  108. return None
  109. @functools.lru_cache(maxsize=1)
  110. def _live_boot_id() -> Optional[str]:
  111. """The kernel's id for the running boot (Linux); ``None`` elsewhere or when unreadable."""
  112. try:
  113. return Path("/proc/sys/kernel/random/boot_id").read_text(encoding="utf-8").strip() or None
  114. except OSError:
  115. return None
  116. # No live process has a negative boot_start, so an entry from an earlier boot never matches.
  117. _EARLIER_BOOT = -1.0
  118. def _entry_boot(entry: dict, key: str) -> Optional[float]:
  119. """The entry's boot-relative birth ``key`` (``boot_start``/``spawner_boot``) as exact proof.
  120. Boot ticks restart at every boot while the ledger survives reboots (and early-boot services
  121. get the same pid at the same tick), so they identify a process only within the boot that
  122. recorded them: an entry from another boot is :data:`_EARLIER_BOOT`, matching nothing alive.
  123. ``None`` (epoch match) when either boot id is unknown.
  124. """
  125. recorded, live = entry.get("boot_id"), _live_boot_id()
  126. if not isinstance(recorded, str) or live is None:
  127. return None
  128. return entry.get(key) if recorded == live else _EARLIER_BOOT
  129. def _verified_boot_start(pid: Optional[int], create_time: Optional[float]) -> Optional[float]:
  130. """:func:`boot_start` of ``pid`` while it still matches ``create_time``; ``None`` when unproven
  131. or when the boot is unidentifiable (ticks alone repeat across reboots)."""
  132. if not isinstance(pid, int) or pid <= 0 or create_time is None or _live_boot_id() is None:
  133. return None
  134. try:
  135. import psutil
  136. proc = psutil.Process(pid)
  137. return boot_start(proc) if _same_incarnation(proc, create_time) else None
  138. except Exception: # health: allow BLE001 -- best-effort probe like _process_create_time: any failure is "unproven"
  139. return None
  140. # Layer 1 — spawn tags
  141. @dataclass(frozen=True)
  142. class SpawnTag:
  143. install: str
  144. purpose: str
  145. spawner_pid: int
  146. spawner_create: Optional[float]
  147. def build_spawn_tag(purpose: str, *, project_root: Optional[Path] = None) -> str:
  148. """Value for the child's ``HERMES_SPAWN`` env var, stamped by the spawner."""
  149. create = _process_create_time()
  150. create_part = f"{create:.3f}" if create is not None else "-"
  151. return ":".join((_TAG_VERSION, install_id(project_root), purpose, str(os.getpid()), create_part))
  152. def spawn_env(purpose: str, *, project_root: Optional[Path] = None) -> dict[str, str]:
  153. """Env fragment a spawner merges into a child's environment."""
  154. return {SPAWN_ENV_VAR: build_spawn_tag(purpose, project_root=project_root)}
  155. def parse_spawn_tag(raw: object) -> Optional[SpawnTag]:
  156. """Parse a ``HERMES_SPAWN`` value; ``None`` for anything malformed."""
  157. parts = raw.split(":") if isinstance(raw, str) else []
  158. if len(parts) != 5 or parts[0] != _TAG_VERSION:
  159. return None
  160. _, install, purpose, pid_s, create_s = parts
  161. if not install or not purpose:
  162. return None
  163. try:
  164. pid = int(pid_s)
  165. create = None if create_s == "-" else float(create_s)
  166. except ValueError:
  167. return None
  168. return SpawnTag(install, purpose, pid, create) if pid > 0 else None
  169. def spawned_by_this_process(pid: int) -> bool:
  170. """True when ``pid`` carries a spawn tag naming THIS process (pid and create time).
  171. A shared daemon (one session or pid file every Hermes of this user can reach) is owned only by
  172. the Hermes whose command actually started it: the CLI it ran was given ``spawn_env`` and the
  173. daemon inherited it. A pid file proves what a process is, not who started it. Fails closed:
  174. an unreadable environment (another user, a race) is not ours."""
  175. import psutil
  176. try:
  177. tag = parse_spawn_tag(psutil.Process(int(pid)).environ().get(SPAWN_ENV_VAR))
  178. except (psutil.Error, OSError):
  179. return False
  180. if tag is None or tag.spawner_pid != os.getpid():
  181. return False
  182. mine = _process_create_time()
  183. return tag.spawner_create is None or mine is None or abs(tag.spawner_create - mine) < 2.0
  184. # Layer 2 — spawn ledger
  185. @dataclass
  186. class LedgerEntry:
  187. pid: int
  188. create_time: Optional[float]
  189. purpose: str
  190. install: str
  191. spawner_pid: Optional[int]
  192. spawner_create: Optional[float]
  193. registered_at: float
  194. argv: str
  195. # Structured launch identity a relauncher needs after an update, without parsing argv. Empty
  196. # for purposes that don't supply it; readers must use .get() — older ledger files predate these.
  197. host: str = ""
  198. port: Optional[int] = None
  199. profile: str = ""
  200. hermes_home: str = ""
  201. # `serve --isolated`: opted out of the host singleton (Desktop's SSH backend for another
  202. # machine). Attach-first readers must never adopt it; argv is truncated, so this is canonical.
  203. isolated: bool = False
  204. # Linux boot-relative births (:func:`boot_start`) of the process and its spawner: exact
  205. # identity across clock steps, within the boot named by ``boot_id`` (:func:`_entry_boot`).
  206. # ``None`` elsewhere and in older entries (epoch match only).
  207. boot_start: Optional[float] = None
  208. spawner_boot: Optional[float] = None
  209. boot_id: Optional[str] = None
  210. def _ledger_path() -> Path:
  211. """Machine-root ledger path (shared by every profile of this install)."""
  212. try:
  213. from hermes_constants import get_default_hermes_root
  214. return Path(get_default_hermes_root()) / LEDGER_FILENAME
  215. except Exception:
  216. from hermes_cli.config import get_hermes_home
  217. return Path(get_hermes_home()) / LEDGER_FILENAME
  218. def _read_ledger(path: Path) -> Optional[list[dict]]:
  219. """Entries list, ``[]`` for empty/missing, ``None`` for CORRUPT (never silently an empty roster).
  220. Mirrors the #89298 contract: corrupt is a distinct state that must never be silently treated as an empty
  221. roster.
  222. """
  223. try:
  224. text = path.read_text(encoding="utf-8-sig")
  225. except FileNotFoundError:
  226. return []
  227. except (OSError, UnicodeError):
  228. return None
  229. if not text.strip():
  230. return []
  231. try:
  232. parsed = json.loads(text)
  233. except (ValueError, TypeError):
  234. return None
  235. return [e for e in parsed if isinstance(e, dict)] if isinstance(parsed, list) else None
  236. def _read_ledger_or_quarantine(path: Path) -> Optional[list[dict]]:
  237. """Ledger entries; ``None`` after parking a corrupt file. Caller holds ``_LEDGER_LOCK``."""
  238. entries = _read_ledger(path)
  239. if entries is None:
  240. parked = path.with_suffix(path.suffix + ".corrupt")
  241. try:
  242. os.replace(path, parked)
  243. logger.warning("spawn ledger was unreadable; moved to %s", parked)
  244. except OSError:
  245. pass
  246. return entries
  247. def _same_incarnation(proc, create_time: Optional[float], boot: Optional[float] = None) -> bool:
  248. """Does the live ``proc`` match a recorded birth? A recorded ``boot`` (:func:`boot_start`)
  249. compares exactly where the live one is readable; otherwise ``create_time`` (2 s tolerance;
  250. ``None`` matches)."""
  251. if boot is not None and (live_boot := boot_start(proc)) is not None:
  252. return live_boot == float(boot)
  253. return create_time is None or abs(process_birth(proc) - float(create_time)) < 2.0
  254. def _pid_alive_matches(
  255. pid: int, create_time: Optional[float], *, strict: bool = False, boot: Optional[float] = None,
  256. ) -> Optional[bool]:
  257. """True/False when provable; ``None`` when psutil can't say."""
  258. try:
  259. import psutil
  260. except Exception:
  261. return None
  262. try:
  263. proc = psutil.Process(int(pid))
  264. # ``strict`` is positive proof (an exact epoch birth); a clock step only withholds it.
  265. if strict:
  266. # Windows: skip the status() read entirely. psutil's Windows status path
  267. # (NtQuerySystemInformation) does not release the GIL, so under load it freezes every
  268. # other Python thread in the process for minutes (#142085) — and Windows has no POSIX
  269. # zombies, so the STATUS_ZOMBIE comparison had nothing to contribute there anyway. The
  270. # non-strict branch right below already makes this exact skip.
  271. if os.name == "nt":
  272. return (create_time is not None and process_birth(proc) == create_time
  273. and proc.is_running())
  274. return (create_time is not None and process_birth(proc) == create_time
  275. and proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE)
  276. # A zombie keeps its create_time until its parent reaps it, but it is already dead: the
  277. # dashboard stop check (``gateway.status._pid_exists``) books it stopped, so the ledger must
  278. # not report it as a live pre-update survivor. Windows has no zombies (and status() is slow there).
  279. return _same_incarnation(proc, create_time, boot) and (
  280. os.name == "nt" or proc.status() != getattr(psutil, "STATUS_ZOMBIE", "zombie"))
  281. except psutil.NoSuchProcess:
  282. return False
  283. except Exception:
  284. return None
  285. def register_self(purpose: str, *, project_root: Optional[Path] = None, detail: Optional[dict] = None) -> bool:
  286. """Record this process in the machine spawn ledger. Best-effort.
  287. Called at the top of every long-lived entry point; dead ``(pid, create_time)`` entries are
  288. pruned on every write. ``detail`` may carry ``host``/``port``/``profile`` so the update
  289. pipeline can relaunch a manually-started serve with its real bind address, and ``isolated``
  290. so attach-first discovery skips a backend that opted out of the host singleton.
  291. """
  292. from hermes_constants import hermes_home_key
  293. tag = parse_spawn_tag(os.environ.get(SPAWN_ENV_VAR))
  294. spawner_pid, spawner_create = (tag.spawner_pid, tag.spawner_create) if tag else _desktop_spawner_identity()
  295. entry = _new_entry(os.getpid(), _process_create_time(), purpose, project_root, spawner_pid, spawner_create)
  296. entry.spawner_boot = _verified_boot_start(spawner_pid, spawner_create)
  297. entry.hermes_home = hermes_home_key()
  298. if detail:
  299. try:
  300. entry.host = str(detail.get("host") or "")
  301. entry.port = int(detail["port"]) if detail.get("port") is not None else None
  302. entry.profile = str(detail.get("profile") or "")
  303. entry.isolated = bool(detail.get("isolated"))
  304. except (TypeError, ValueError):
  305. pass
  306. try:
  307. import sys as _sys
  308. # 10 tokens: enough for `hermes serve --host X --port N --profile P` while bounding
  309. # pathological argv. Structured detail is canonical; argv is the human-readable fallback.
  310. entry.argv = " ".join(_sys.argv[:10])
  311. except Exception:
  312. pass
  313. return _append_entry(entry)
  314. def is_desktop_owned_backend(argv: Optional[Sequence[str]] = None) -> bool:
  315. """Whether this process is the backend Desktop spawned and owns.
  316. ``HERMES_DESKTOP=1`` is inherited by every shell and agent child the app launches, so the
  317. flag alone is not ownership proof (same class as #116107). Desktop hands its backend a
  318. per-spawn credential the terminal pane never receives (and the terminal tool's env policy
  319. strips from agent children): the local pool spawn mints ``HERMES_DASHBOARD_SESSION_TOKEN``,
  320. the SSH spawn passes a 0600 token FILE on argv and deliberately sets no token env var.
  321. ``argv`` defaults to this process's own.
  322. """
  323. if os.environ.get("HERMES_DESKTOP") != "1":
  324. return False
  325. if os.environ.get("HERMES_DASHBOARD_SESSION_TOKEN"):
  326. return True
  327. from hermes_cli._startup_fast import is_desktop_ssh_backend_argv
  328. return is_desktop_ssh_backend_argv(list(sys.argv[1:] if argv is None else argv))
  329. # Set while this process works for the Desktop app on its own machine: one request from, or one turn of
  330. # a chat created by, a client ``local_desktop_client`` accepted. The gateway serves the Desktop, the CLI,
  331. # the TUI and bots from one process, so this is a property of the activity, never of the process.
  332. _SERVING_LOCAL_DESKTOP: ContextVar[bool] = ContextVar("hermes_serving_local_desktop", default=False)
  333. @contextmanager
  334. def serving_local_desktop(active: bool) -> Iterator[None]:
  335. """Mark the enclosed activity (and threads it hands its context to) as work for the local Desktop."""
  336. token = _SERVING_LOCAL_DESKTOP.set(bool(active) or _SERVING_LOCAL_DESKTOP.get())
  337. try:
  338. yield
  339. finally:
  340. _SERVING_LOCAL_DESKTOP.reset(token)
  341. def serves_local_desktop_now() -> bool:
  342. return _SERVING_LOCAL_DESKTOP.get()
  343. def local_desktop_client(identity: Optional[dict], declared: object) -> bool:
  344. """A connected client is the Desktop app on this machine when it says so (it knows its connection
  345. mode) AND it redeemed a native ticket, which only this machine's private control socket mints, for
  346. this OS user. Neither alone: the CLI and TUI hold native tickets too, and the claim is the client's
  347. word. A remote Desktop reaches a backend through SSH, URL or Cloud credentials, never a native ticket."""
  348. return (declared is True and isinstance(identity, dict) and identity.get("provider") == "local"
  349. and identity.get("native_bootstrap") is True)
  350. def _desktop_spawner_identity() -> tuple[Optional[int], Optional[float]]:
  351. """Spawner ``(pid, create_time)`` from the Electron app's HERMES_PARENT_PID (+ optional
  352. ``winms:<ms>`` start marker) parent-death watchdog vars, so ledger lineage works with every
  353. Desktop version without a TS change. ``(None, None)`` when absent/malformed."""
  354. try:
  355. spawner_pid = int(os.environ.get("HERMES_PARENT_PID", ""))
  356. except (TypeError, ValueError):
  357. spawner_pid = 0
  358. if spawner_pid <= 0:
  359. return None, None
  360. marker = os.environ.get("HERMES_PARENT_START_MARKER", "")
  361. if not marker.startswith("winms:"):
  362. return spawner_pid, None
  363. try:
  364. return spawner_pid, float(marker.split(":", 1)[1]) / 1000.0
  365. except (ValueError, IndexError):
  366. return spawner_pid, None
  367. def _new_entry(
  368. pid: int, create_time: Optional[float], purpose: str, project_root: Optional[Path],
  369. spawner_pid: Optional[int], spawner_create: Optional[float],
  370. ) -> LedgerEntry:
  371. return LedgerEntry(
  372. pid, create_time, purpose, install_id(project_root), spawner_pid, spawner_create, time.time(), argv="",
  373. boot_start=_verified_boot_start(pid, create_time), boot_id=_live_boot_id(),
  374. )
  375. # A ledger writer stopped mid-write (SIGSTOP, a debugger) must not stall every spawn_owned on the
  376. # machine: past this the write goes ahead under the thread lock alone, as when no lock can be taken.
  377. _LEDGER_LOCK_TIMEOUT_S = 2.0
  378. _ledger_lock_warned = False
  379. def _append_entry(entry: LedgerEntry) -> bool:
  380. """Prune dead entries and append ``entry`` — the ONLY ledger write path.
  381. Serialized under ``_LEDGER_LOCK`` and, across processes, an exclusive lock on a sidecar file
  382. (every ``spawn_owned`` writes here, so concurrent Hermes processes otherwise dropped each
  383. other's rows), with an atomic tmp+replace; no writer touches the file outside this function.
  384. See #91660.
  385. """
  386. from hermes_cli.active_sessions import _FileLock
  387. path = _ledger_path()
  388. with _LEDGER_LOCK, ExitStack() as held:
  389. try:
  390. held.enter_context(_FileLock(path.with_name(path.name + ".lock"), timeout=_LEDGER_LOCK_TIMEOUT_S))
  391. except (RuntimeError, OSError):
  392. # Best-effort like the write itself: the thread lock is then the only guard, so a row another
  393. # Hermes process writes meanwhile can be lost. Say so once where it shows (agent.log).
  394. global _ledger_lock_warned
  395. if not _ledger_lock_warned:
  396. _ledger_lock_warned = True
  397. logger.warning("spawn ledger lock unavailable (a stalled holder, or no file locks here); writing "
  398. "without it, so concurrent Hermes processes may drop each other's rows")
  399. logger.debug("spawn ledger lock unavailable", exc_info=True)
  400. entries = _read_ledger_or_quarantine(path) or []
  401. # Drop malformed entries, our own stale entry, and provably dead pids.
  402. pruned = [
  403. e for e in entries
  404. if isinstance(e.get("pid"), int)
  405. and e["pid"] != entry.pid
  406. and entry_still_running(e)
  407. ]
  408. pruned.append(asdict(entry))
  409. try:
  410. from hermes_constants import mkdir_under_hermes_home
  411. mkdir_under_hermes_home(path.parent)
  412. # argv may carry surrogate-escaped bytes (non-UTF-8 paths); ensure_ascii keeps the
  413. # utf-8 text handle from raising UnicodeEncodeError (a ValueError, not an OSError).
  414. atomic_json_write(path, pruned, mode=0o600, ensure_ascii=True)
  415. return True
  416. except OSError:
  417. logger.debug("spawn ledger write failed", exc_info=True)
  418. return False
  419. def register_child(pid: int, purpose: str, *, project_root: Optional[Path] = None) -> bool:
  420. """Record a CHILD process this process just spawned. Best-effort.
  421. Mirror of :func:`register_self` for children that cannot register themselves (stdio MCP
  422. helpers: arbitrary ``npx``/binary servers never import Hermes code). Records the child's
  423. ``(pid, create_time)`` with THIS process as spawner, so a helper whose spawner is provably gone
  424. is a reapable orphan and one whose spawner is alive is never reaped.
  425. """
  426. try:
  427. pid = int(pid)
  428. except (TypeError, ValueError):
  429. return False
  430. child_create = _process_create_time(pid) if pid > 0 else None
  431. if child_create is None:
  432. return False
  433. entry = _new_entry(pid, child_create, purpose, project_root, os.getpid(), _process_create_time())
  434. entry.spawner_boot = _verified_boot_start(os.getpid(), entry.spawner_create)
  435. try:
  436. import psutil
  437. entry.argv = " ".join(psutil.Process(pid).cmdline()[:10])
  438. except Exception:
  439. pass
  440. return _append_entry(entry)
  441. def ledger_entries(
  442. *, project_root: Optional[Path] = None, all_installs: bool = False, verified_only: bool = False,
  443. ) -> list[dict]:
  444. """Ledger entries for this install, or all installs when explicitly requested.
  445. ``verified_only`` requires an exact live PID/create-time pair. The default preserves
  446. unknown processes for reapers, which must not mistake missing proof for a dead process.
  447. Entries whose ``(pid, create_time)`` no longer matches a live process are excluded (PID reuse reads as
  448. dead, thanks to the create-time pair). A corrupt ledger is quarantined and read as empty — identical
  449. philosophy to the backend-ownership fix (#89298): never let corruption erase or fake a roster; never let
  450. it block the caller either.
  451. """
  452. want_install = install_id(project_root)
  453. with _LEDGER_LOCK:
  454. entries = _read_ledger_or_quarantine(_ledger_path())
  455. if entries is None:
  456. return []
  457. return [
  458. e for e in entries
  459. if (all_installs or e.get("install") == want_install)
  460. and isinstance(e.get("pid"), int)
  461. and (_pid_alive_matches(e["pid"], e.get("create_time"), strict=True) is True
  462. if verified_only else _pid_alive_matches(e["pid"], e.get("create_time"), boot=_entry_boot(e, "boot_start")) is not False)
  463. ]
  464. def entry_still_running(entry: dict) -> bool:
  465. """Is the ledger *entry*'s process still the recorded incarnation (``None`` from psutil
  466. counts as running)? Exact boot ticks within their boot survive a clock step; an entry
  467. without them keeps the 2 s epoch match, so a step reads it as a reused pid."""
  468. pid = entry.get("pid")
  469. if not isinstance(pid, int) or pid <= 0:
  470. return False
  471. return _pid_alive_matches(pid, entry.get("create_time"), boot=_entry_boot(entry, "boot_start")) is not False
  472. def spawner_is_dead(entry: dict) -> Optional[bool]:
  473. """Is the recorded spawner of this entry provably gone? ``None`` when unrecorded/unprovable."""
  474. spawner_pid = entry.get("spawner_pid")
  475. if not isinstance(spawner_pid, int) or spawner_pid <= 0:
  476. return None
  477. # A spawner recorded in an earlier boot is dead (:func:`_entry_boot`), as the epoch match says.
  478. boot = _entry_boot(entry, "spawner_boot")
  479. alive = _pid_alive_matches(spawner_pid, entry.get("spawner_create"), boot=boot)
  480. return None if alive is None else not alive
  481. def _terminate_then_kill(pid: int, create_time: Optional[float], boot: Optional[float] = None) -> bool:
  482. """SIGTERM, 2 s grace, then SIGKILL. False when the incarnation moved on or is already gone."""
  483. try:
  484. import psutil
  485. proc = psutil.Process(pid)
  486. if not _same_incarnation(proc, create_time, boot):
  487. return False # PID reused since registration — never signal a stranger
  488. proc.terminate()
  489. try:
  490. proc.wait(timeout=2.0)
  491. except psutil.TimeoutExpired:
  492. proc.kill()
  493. return True
  494. except Exception:
  495. logger.debug("orphan terminate failed for pid %s", pid, exc_info=True)
  496. return False
  497. def _reparented_orphan(pid: int) -> bool:
  498. """Null-spawner owner with no live supervisor: reparented to init, old, lock-unclaimed.
  499. Mirrors ``_reap_orphaned_desktop_local_serves``: operator-managed remote backends legitimately
  500. sit at ppid 1 (systemd, an exited sshd), so ppid alone is never proof — the SSH-lock claim
  501. and the 180 s lock-write grace exclude live-supervised backends. Windows: the ppid probe is
  502. unavailable there, so this rung never fires (desktop tree-kill reaps instead).
  503. """
  504. try:
  505. from hermes_cli.dashboard_procs import (
  506. _REAP_MIN_AGE_SECONDS, _lock_owned_serve_pids, _process_ppid)
  507. if _process_ppid(pid) not in (0, 1):
  508. return False
  509. if pid in (_lock_owned_serve_pids() or ()):
  510. return False
  511. import time as _time
  512. import psutil
  513. return max(0.0, _time.time() - psutil.Process(pid).create_time()) >= _REAP_MIN_AGE_SECONDS
  514. except Exception:
  515. return False # unprovable → never touch
  516. def reap_orphaned_backend_owner(
  517. pid: int, create_time: Optional[float], *, kill_fn=None) -> Optional[int]:
  518. """Kill the conflicting host owner when the ledger PROVES it is a dead session's orphan.
  519. HELD_BY_OTHER rung for #121964: the owner PID is alive (re-parented to init), so
  520. ``record_is_stale()`` never fires and a fresh backend would loop observe-only against the
  521. same orphan forever. Reaped only with a live ``serve``/``dashboard``/``gateway`` entry for
  522. THIS install whose spawner is provably dead — or whose spawner was never recorded and which
  523. is reparented to init, old, and claimed by no Desktop SSH lock. Live-spawned, lock-claimed,
  524. young, interactive, self, or unprovable owners are never touched. Returns the reaped pid,
  525. else ``None``; never raises.
  526. # ponytail: a dead session's surviving backend.lock.json still shields its orphan (the lock
  527. # has no session liveness); clearing stale locks is a Desktop-side lifecycle job, not this rung.
  528. """
  529. try:
  530. pid = int(pid)
  531. except (TypeError, ValueError):
  532. return None
  533. if pid <= 0 or pid == os.getpid():
  534. return None
  535. try:
  536. entry = next(
  537. (e for e in ledger_entries()
  538. if e.get("pid") == pid
  539. and e.get("purpose") in _BACKEND_OWNER_PURPOSES
  540. and (e.get("create_time") is None or create_time is None
  541. or abs(float(e["create_time"]) - float(create_time)) < 2.0)),
  542. None)
  543. except Exception:
  544. return None
  545. if entry is None:
  546. return None
  547. if spawner_is_dead(entry) is not True and not (
  548. entry.get("spawner_pid") is None and _reparented_orphan(pid)):
  549. return None # live or unprovable spawner → never touch
  550. try:
  551. if kill_fn is not None:
  552. kill_fn(pid)
  553. elif not _terminate_then_kill(pid, entry.get("create_time"), _entry_boot(entry, "boot_start")):
  554. return None
  555. except Exception:
  556. logger.debug("backend owner reap failed for %s", entry, exc_info=True)
  557. return None
  558. logger.info("reaped orphaned %s backend owner pid %s", entry.get("purpose"), pid)
  559. return pid
  560. def reap_orphaned_mcp_helpers(*, project_root: Optional[Path] = None, kill_fn=None) -> list[int]:
  561. """Kill ledger-registered stdio MCP helpers whose spawner is provably dead.
  562. Ledger-driven startup-sweep rung (not cmdline-heuristic): a helper is reaped ONLY when it has a
  563. live ``mcp-helper`` entry for THIS install AND ``spawner_is_dead`` is ``True`` — never
  564. ``None``/unprovable, never a live spawner.
  565. """
  566. reaped: list[int] = []
  567. try:
  568. entries = ledger_entries(project_root=project_root)
  569. except Exception:
  570. return reaped
  571. own_pid = os.getpid()
  572. for entry in entries:
  573. try:
  574. pid = entry.get("pid")
  575. if entry.get("purpose") != "mcp-helper" or not isinstance(pid, int) or pid <= 0 or pid == own_pid:
  576. continue
  577. if spawner_is_dead(entry) is not True:
  578. continue # live or unprovable spawner → never touch
  579. if kill_fn is not None:
  580. kill_fn(pid)
  581. else:
  582. import psutil
  583. proc = psutil.Process(pid)
  584. if not _same_incarnation(proc, entry.get("create_time"), _entry_boot(entry, "boot_start")):
  585. continue # PID reused since registration
  586. # Windows: descendants (npx.cmd → node.exe) have no pgid to group-kill and
  587. # reparent with ParentId=null when the direct child exits first (#61059), so
  588. # reap the whole tree. On POSIX the killpg-based sweep already reaches them.
  589. if _IS_WINDOWS:
  590. _kill_process_tree_windows(proc)
  591. else:
  592. proc.terminate()
  593. try:
  594. proc.wait(timeout=2.0)
  595. except psutil.TimeoutExpired:
  596. proc.kill()
  597. reaped.append(pid)
  598. except Exception:
  599. logger.debug("mcp-helper orphan reap failed for %s", entry, exc_info=True)
  600. if reaped:
  601. logger.info("reaped %d orphaned stdio MCP helper(s): %s", len(reaped), reaped)
  602. return reaped
  603. # Layer 3 — Windows job-object self-attach
  604. def _kill_process_tree_windows(proc) -> None:
  605. """Terminate *proc* and every still-alive descendant (npx.cmd → node.exe), Windows-only
  606. (#61059): without a pgid there is no group-kill, and grandchildren reparent to nothing
  607. (ParentId=null) once the direct child exits, so they must be reached through the tree."""
  608. import psutil
  609. try:
  610. descendants = proc.children(recursive=True)
  611. except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
  612. descendants = []
  613. for child in descendants:
  614. try:
  615. child.terminate()
  616. except Exception:
  617. pass
  618. try:
  619. proc.terminate()
  620. except Exception:
  621. pass
  622. try:
  623. _, alive = psutil.wait_procs(descendants + [proc], timeout=2.0)
  624. except Exception:
  625. return
  626. for survivor in alive:
  627. try:
  628. survivor.kill()
  629. except Exception:
  630. pass
  631. def attach_self_to_kill_on_close_job() -> bool:
  632. """Place this process in a job that dies (whole tree) when we die. Windows-only, idempotent.
  633. ``BREAKAWAY_OK`` keeps children spawned with ``CREATE_BREAKAWAY_FROM_JOB`` (gateway relaunch
  634. during update, detached watchers) escaping exactly as before.
  635. """
  636. global _JOB_HANDLE
  637. if not _IS_WINDOWS or _JOB_HANDLE is not None:
  638. return _JOB_HANDLE is not None
  639. try:
  640. import ctypes
  641. from ctypes import wintypes
  642. kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
  643. JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE = 0x2000
  644. JOB_OBJECT_LIMIT_BREAKAWAY_OK = 0x0800
  645. JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK = 0x1000
  646. JobObjectExtendedLimitInformation = 9
  647. class IO_COUNTERS(ctypes.Structure):
  648. _fields_ = [(n, ctypes.c_ulonglong) for n in (
  649. "ReadOperationCount", "WriteOperationCount", "OtherOperationCount",
  650. "ReadTransferCount", "WriteTransferCount", "OtherTransferCount")]
  651. class JOBOBJECT_BASIC_LIMIT_INFORMATION(ctypes.Structure):
  652. _fields_ = [
  653. ("PerProcessUserTimeLimit", wintypes.LARGE_INTEGER), ("PerJobUserTimeLimit", wintypes.LARGE_INTEGER),
  654. ("LimitFlags", wintypes.DWORD), ("MinimumWorkingSetSize", ctypes.c_size_t),
  655. ("MaximumWorkingSetSize", ctypes.c_size_t), ("ActiveProcessLimit", wintypes.DWORD),
  656. ("Affinity", ctypes.POINTER(wintypes.ULONG)), ("PriorityClass", wintypes.DWORD),
  657. ("SchedulingClass", wintypes.DWORD),
  658. ]
  659. class JOBOBJECT_EXTENDED_LIMIT_INFORMATION(ctypes.Structure):
  660. _fields_ = [
  661. ("BasicLimitInformation", JOBOBJECT_BASIC_LIMIT_INFORMATION),
  662. ("IoInfo", IO_COUNTERS),
  663. *((n, ctypes.c_size_t) for n in (
  664. "ProcessMemoryLimit", "JobMemoryLimit", "PeakProcessMemoryUsed", "PeakJobMemoryUsed")),
  665. ]
  666. job = kernel32.CreateJobObjectW(None, None)
  667. if not job:
  668. return False
  669. info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION()
  670. info.BasicLimitInformation.LimitFlags = (
  671. JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | JOB_OBJECT_LIMIT_BREAKAWAY_OK | JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK
  672. )
  673. ok = kernel32.SetInformationJobObject(job, JobObjectExtendedLimitInformation, ctypes.byref(info), ctypes.sizeof(info))
  674. if not ok or not kernel32.AssignProcessToJobObject(job, kernel32.GetCurrentProcess()):
  675. kernel32.CloseHandle(job)
  676. return False
  677. _JOB_HANDLE = job # keep alive for the life of the process — never close
  678. logger.debug("attached to kill-on-close job object")
  679. return True
  680. except Exception:
  681. logger.debug("job object self-attach failed", exc_info=True)
  682. return False
  683. # Layer 4 — POSIX owner-death supervisor
  684. #
  685. # If this process dies without running its cleanup path (SIGKILL, OOM killer, crash, force-quit),
  686. # its long-lived children reparent to init or the nearest subreaper (systemd --user) and run
  687. # forever; macOS has no PR_SET_PDEATHSIG, and on Linux it fires when the spawning THREAD exits,
  688. # which kills children spawned from short-lived worker threads while the owner is alive. So ONE
  689. # supervisor per Hermes process (``process_supervisor.sh``) holds the read end of a pipe whose only
  690. # write end we hold: EOF means we died, by any cause, and it killpg()s every group still registered.
  691. # A registered group that was never unregistered IS the orphan set, so a clean exit that stopped its
  692. # children first ends with nothing to kill, and one that didn't takes them along.
  693. #
  694. # ``spawn_owned`` is THE way to start a long-lived child that must die with this process: the child
  695. # leads its own process group, that group is handed to the supervisor, and the child is recorded in
  696. # the spawn ledger. Children something else spawns (the MCP SDK, the browser-harness daemon) hand their
  697. # group to ``own_process_groups`` where we learn the pgid; a descendant that leaves the group on
  698. # purpose is registered by its own pgid the same way. Stopping an owned child yourself ends with
  699. # ``release_process_group_once_empty`` (survivors stay covered); ``release_process_groups`` hands a
  700. # group back on purpose. ``scripts/check_detached_spawns.py`` flags new detached spawns
  701. # (start_new_session / process_group / setsid / setpgid; its docstring lists what it misses). POSIX-only: Windows children die
  702. # through the layer-3 job object instead.
  703. _supervisor = None # Optional[subprocess.Popen]; tests rebind it
  704. _supervisor_lock = threading.Lock()
  705. # Groups the supervisor is reaping on our behalf; replayed verbatim on respawn so a respawn never
  706. # silently drops coverage for children that are still running.
  707. _supervised_pgids: set = set()
  708. _SUPERVISOR_SCRIPT = Path(__file__).with_name("process_supervisor.sh")
  709. # Android (Termux) has no /bin/sh; its system shell is the same POSIX sh the script needs.
  710. _POSIX_SH = "/bin/sh" if os.path.exists("/bin/sh") else "/system/bin/sh"
  711. def _spawn_supervisor():
  712. """Start the per-process supervisor, or None if it cannot be started."""
  713. try:
  714. # start_new_session=True is load-bearing: shutdown paths killpg this process's own group,
  715. # which would kill the supervisor before it could reap anything. close_fds keeps the write
  716. # end of every OTHER pipe we hold out of it, so it never delays another child's EOF.
  717. proc = subprocess.Popen(
  718. [_POSIX_SH, str(_SUPERVISOR_SCRIPT), str(os.getpgid(0))],
  719. stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
  720. start_new_session=True, close_fds=True, text=True, encoding="utf-8")
  721. except Exception:
  722. # Never let supervisor bookkeeping block a real child: graceful shutdown paths still reap
  723. # normally; only the ungraceful-exit safety net is lost.
  724. logger.debug("Could not start the owner-death supervisor", exc_info=True)
  725. return None
  726. _watch_supervisor(proc)
  727. return proc
  728. # A supervisor that dies sooner than this after starting may not be able to stay up (a broken install:
  729. # sh exiting 2 or 127; a sandbox or OOM killer that keeps killing it): the waiter leaves it to the next
  730. # registration change instead of restarting it in a loop. One killed by a signal gets one young restart
  731. # (a new supervisor starts after every idle period, so a `pkill -f hermes` often meets a young one); if
  732. # its replacement is killed young as well, that one waits for the next registration change too.
  733. _SUPERVISOR_RESTART_MIN_UPTIME_S = 5.0
  734. # Set (under _supervisor_lock) while the waiter starts the young restart, so that replacement's own
  735. # waiter knows it already had one.
  736. _starting_young_restart = False
  737. def _watch_supervisor(proc) -> None:
  738. """One waiter thread per supervisor, blocked in the untimed ``wait()`` (waitpid, no polling). It
  739. reaps the supervisor the moment it exits, so no zombie child is left for psutil walkers of this
  740. process's children to trip on. A supervisor that died on its own while groups are still registered
  741. (``pkill -f hermes``, the OOM killer) is restarted and replayed at once, so coverage does not wait
  742. for the next registration change; a released one (EOF with an empty set) is only reaped."""
  743. started = time.monotonic()
  744. young_restart = _starting_young_restart
  745. def wait_then_restart() -> None:
  746. global _supervisor, _starting_young_restart
  747. proc.wait()
  748. with _supervisor_lock:
  749. if _supervisor is not None and _supervisor is not proc:
  750. return # already replaced: only reaped
  751. # Still current, or dropped by a caller whose write hit its broken pipe: restart below if
  752. # groups are still registered (a deliberate release leaves the set empty).
  753. _supervisor = None
  754. _prune_dead_supervised_pgids()
  755. if not _supervised_pgids:
  756. return
  757. old_enough = time.monotonic() - started >= _SUPERVISOR_RESTART_MIN_UPTIME_S
  758. young_signal = not old_enough and proc.returncode < 0 and not young_restart
  759. restart = old_enough or young_signal
  760. logger.warning(
  761. "owner-death supervisor exited (status %s) with %d process group(s) registered; %s",
  762. proc.returncode, len(_supervised_pgids),
  763. "restarting it" if restart else "it could not stay up, retrying at the next registration")
  764. if restart:
  765. _starting_young_restart = young_signal
  766. try:
  767. _start_supervisor_locked()
  768. finally:
  769. _starting_young_restart = False
  770. try:
  771. # health: allow HX012 -- the thread waitpid()s one child and touches only process-global supervisor state
  772. threading.Thread(target=wait_then_restart, name="owner-death-supervisor-waiter", daemon=True).start()
  773. except RuntimeError: # "can't start new thread" (thread limit, interpreter shutdown)
  774. # The supervisor still works; it is left a zombie until we exit and is not restarted if killed.
  775. logger.debug("Could not start the owner-death supervisor's waiter", exc_info=True)
  776. # An idle supervisor (nothing registered) exits this long after the set last emptied, unless a new
  777. # registration arrives first.
  778. _SUPERVISOR_IDLE_EXIT_S = 30.0
  779. _idle_release: Optional[threading.Timer] = None
  780. def _release_when_idle_locked(proc) -> None:
  781. """Close ``proc``'s pipe once the set has stayed empty for ``_SUPERVISOR_IDLE_EXIT_S``; the caller
  782. holds ``_supervisor_lock``. Closing our write end is the same EOF our death sends; with an empty
  783. set it exits, and its waiter thread reaps it. The next registration respawns and replays."""
  784. global _idle_release
  785. _cancel_idle_release_locked()
  786. if _SUPERVISOR_IDLE_EXIT_S <= 0:
  787. _close_idle_supervisor_locked(proc)
  788. return
  789. def release() -> None:
  790. with _supervisor_lock:
  791. _close_idle_supervisor_locked(proc)
  792. try:
  793. _idle_release = threading.Timer(_SUPERVISOR_IDLE_EXIT_S, release)
  794. _idle_release.daemon = True
  795. _idle_release.start()
  796. except RuntimeError:
  797. # No thread for the grace (thread limit, interpreter shutdown): release at once, as before the
  798. # grace existed. Losing the grace must never fail the live child this call is about.
  799. _idle_release = None
  800. logger.debug("Could not start the supervisor idle timer; releasing it now", exc_info=True)
  801. _close_idle_supervisor_locked(proc)
  802. def _close_idle_supervisor_locked(proc) -> None:
  803. global _supervisor
  804. if _supervisor is not proc or _supervised_pgids:
  805. return # replaced, or something registered meanwhile
  806. with suppress(BrokenPipeError, ValueError, OSError):
  807. proc.stdin.close()
  808. _supervisor = None
  809. def _cancel_idle_release_locked() -> None:
  810. global _idle_release
  811. if _idle_release is not None:
  812. _idle_release.cancel()
  813. _idle_release = None
  814. def _start_supervisor_locked():
  815. """Start a supervisor and replay every live registration to it; the caller holds
  816. ``_supervisor_lock``. Returns it, or None when it cannot start or its pipe broke at once."""
  817. global _supervisor
  818. proc = _supervisor = _spawn_supervisor()
  819. if proc is None:
  820. return None
  821. try:
  822. proc.stdin.write("".join(f"register {pgid}\n" for pgid in _supervised_pgids))
  823. proc.stdin.flush()
  824. except (BrokenPipeError, ValueError, OSError):
  825. _supervisor = None # its waiter (if it has one) reaps it; the next registration change retries
  826. return None
  827. return proc
  828. def is_owner_death_supervisor(argv: Sequence[str]) -> bool:
  829. """True when a direct child's ``argv`` is this install's supervisor (``sh <script> <pgid>``).
  830. Another family's child discovery (MCP's before/after child-pid delta) can catch it starting and
  831. must never adopt, and later kill, it. Matched on the script path, not ``_supervisor.pid``: that
  832. is assigned only after the spawn returns, so a scan racing the spawn would miss it."""
  833. return list(argv[1:2]) == [str(_SUPERVISOR_SCRIPT)]
  834. def _prune_dead_supervised_pgids() -> set:
  835. """Forget supervised groups with no members left; return what went. Caller holds
  836. ``_supervisor_lock``. Signal 0 is a pure existence probe (cannot terminate anything). It
  837. narrows, but cannot close, the window where a dead group's pgid is recycled before we notice."""
  838. stale = set()
  839. for pgid in list(_supervised_pgids):
  840. try:
  841. os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only, callers return early on nt
  842. except ProcessLookupError:
  843. stale.add(pgid)
  844. except (PermissionError, OSError):
  845. # Exists but not ours to signal, or the probe failed: keep it — dropping coverage on
  846. # an ambiguous answer is the more expensive mistake.
  847. pass
  848. _supervised_pgids.difference_update(stale)
  849. return stale
  850. def own_process_groups(pgids) -> None:
  851. """Make these process groups die with this process, however it dies (see layer 4 above)."""
  852. _update_supervised_pgids("register", pgids)
  853. def release_process_groups(pgids) -> None:
  854. """Stop covering these groups: this process stopped them itself."""
  855. _update_supervised_pgids("unregister", pgids)
  856. def release_process_group_once_empty(pgid: int, *, proc=None, timeout: float = 1.0) -> bool:
  857. """Release ``pgid`` once no member is left; True if released. Polls ``proc`` (its leader) so a
  858. dead-but-unreaped leader does not keep the group alive, and waits up to ``timeout`` for killed
  859. members to be reaped by init. A member still there after that keeps the group covered, so it
  860. still dies with this process. ``timeout=0`` probes once (async callers)."""
  861. if os.name != "posix":
  862. return False
  863. deadline = time.monotonic() + timeout
  864. while True:
  865. if proc is not None:
  866. try:
  867. proc.poll()
  868. except OSError:
  869. pass
  870. try:
  871. os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only, returned above; existence probe
  872. except ProcessLookupError:
  873. release_process_groups([pgid])
  874. return True
  875. except OSError:
  876. pass # exists but not ours to probe: keep covering it
  877. if time.monotonic() >= deadline:
  878. return False
  879. time.sleep(0.05)
  880. def stop_owned_group_survivors(pgid: int, *, proc=None, grace: float = 3.0) -> bool:
  881. """Stop what is left of owned group ``pgid`` after its leader exited, then release it; True once
  882. released. Members that outlived a crashed leader (a router's model children) keep the group, and
  883. so its number, alive: TERM, wait ``grace``, KILL. Only groups this process registered are
  884. signalled, so a number never registered here (or already released) is never touched."""
  885. if os.name != "posix":
  886. return False
  887. with _supervisor_lock:
  888. owned = pgid in _supervised_pgids
  889. if owned:
  890. for sig, wait in ((signal.SIGTERM, grace), (signal.SIGKILL, 1.0)): # windows-footgun: ok — POSIX-only, returned above
  891. if release_process_group_once_empty(pgid, proc=proc, timeout=0):
  892. return True
  893. try:
  894. os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only, returned above
  895. except ProcessLookupError:
  896. pass
  897. except OSError:
  898. return False # not ours to signal: keep covering it
  899. if release_process_group_once_empty(pgid, proc=proc, timeout=wait):
  900. return True
  901. return False
  902. return release_process_group_once_empty(pgid, proc=proc, timeout=0)
  903. def spawn_owned(argv, *, purpose: str, **popen_kwargs) -> subprocess.Popen:
  904. """Start a long-lived child that dies with this process: the one way to own a child.
  905. POSIX: the child leads a new session and process group, and that group goes to the owner-death
  906. supervisor, so the child and every descendant that stays in its group die the moment this
  907. process dies by any cause (SIGKILL, OOM, crash). The child is recorded in the spawn ledger under
  908. ``purpose``. After stopping it yourself, ``release_process_group_once_empty(proc.pid, proc=proc)``.
  909. Windows: a plain ``Popen`` with the caller's kwargs (the layer-3 job object covers ownership).
  910. """
  911. if os.name != "posix":
  912. return subprocess.Popen(argv, **popen_kwargs)
  913. proc = subprocess.Popen(argv, start_new_session=True, **popen_kwargs)
  914. own_process_groups([proc.pid])
  915. register_child(proc.pid, purpose)
  916. return proc
  917. def _update_supervised_pgids(verb: str, pgids) -> None:
  918. """Send ``register``/``unregister`` for ``pgids`` to this process's supervisor. Failures are
  919. swallowed: losing the safety net must never fail a live child."""
  920. if os.name != "posix":
  921. return
  922. wanted = {int(pgid) for pgid in pgids}
  923. if not wanted:
  924. return
  925. global _supervisor
  926. with _supervisor_lock:
  927. if verb == "register":
  928. _supervised_pgids.update(wanted)
  929. else:
  930. _supervised_pgids.difference_update(wanted)
  931. # A registration outlives its child only while some member survives (e.g. an orphaned
  932. # grandchild teardown failed to kill, deliberately kept registered). Once that group is
  933. # empty its pgid can be recycled by a stranger, so prune here too.
  934. stale = _prune_dead_supervised_pgids()
  935. proc = _supervisor
  936. if proc is None or proc.poll() is not None:
  937. if not _supervised_pgids:
  938. # Nothing left to cover: nothing to tell and nothing to respawn for. Keyed on the
  939. # SET, not the verb: after a broken-pipe write dropped the supervisor with groups
  940. # still registered, an unregister must still rebuild coverage for the survivors.
  941. return
  942. # A fresh supervisor knows nothing: replay live coverage (already reflects this call's
  943. # mutation and the prune, so pruned groups never reach the replacement).
  944. _start_supervisor_locked()
  945. return
  946. if _supervised_pgids:
  947. _cancel_idle_release_locked()
  948. payload = "".join(f"{verb} {pgid}\n" for pgid in wanted)
  949. payload += "".join(f"unregister {pgid}\n" for pgid in stale)
  950. try:
  951. proc.stdin.write(payload)
  952. proc.stdin.flush()
  953. except (BrokenPipeError, ValueError, OSError):
  954. # It exited between poll() and write(). Drop it so the next call respawns and replays
  955. # from ``_supervised_pgids`` (the set, not the pipe, is the record of what to reap). Its
  956. # waiter thread reaps it.
  957. _supervisor = None
  958. return
  959. if not _supervised_pgids:
  960. # Nothing left to reap: release the supervisor after an idle grace rather than keep a
  961. # process and a pipe resident for the life of a gateway. Not at once: a foreground
  962. # command owns and releases its group every time, and a supervisor per command costs a
  963. # /bin/sh spawn (~11 ms) under this lock.
  964. _release_when_idle_locked(proc)