dashboard_procs.py 50 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042
  1. """Dashboard process-hygiene helpers — extracted from ``hermes_cli/main.py``.
  2. Helpers defined in ``hermes_cli.main_dashboard`` / ``hermes_cli.main_install_repair`` are imported at
  3. call time so imports stay one-way (both of those modules import this one lazily).
  4. """
  5. import contextlib
  6. import os
  7. import subprocess
  8. import sys
  9. from pathlib import Path
  10. from hermes_cli._startup_fast import is_desktop_ssh_backend_argv
  11. _PS_RUN_KWARGS = dict(capture_output=True, text=True, encoding="utf-8", errors="replace")
  12. def _empty_result() -> dict[str, list]:
  13. return {"matched": [], "killed": [], "failed": []}
  14. def _append_row(rows: list[tuple[int, str]], pid_text: str, command: str) -> None:
  15. try:
  16. rows.append((int(pid_text), command))
  17. except ValueError:
  18. pass
  19. def _iter_process_table() -> list[tuple[int, str]]:
  20. """``(pid, cmdline)`` for every process, via wmic (Windows) or ps. Raises on scan failure."""
  21. rows: list[tuple[int, str]] = []
  22. if sys.platform == "win32":
  23. # errors="ignore": wmic may emit the system code page. bounded_probe_run, not run():
  24. # run()'s post-timeout cleanup joins pipe readers unbounded and a conhost descendant
  25. # holding duplicated handles wedges it forever.
  26. # In text mode, subprocess output decoding depends on Python's configuration (locale-dependent by
  27. # default, or UTF-8 in UTF-8 mode). The important protection here is errors="ignore": it prevents a
  28. # reader-thread UnicodeDecodeError from leaving result.stdout=None and turning the later .split()
  29. # into an AttributeError (#17049). bounded_probe_run (rather than subprocess.run with a timeout)
  30. # keeps a slow scan from wedging the caller forever: run()'s post-timeout cleanup joins the pipe
  31. # reader threads unbounded, and a conhost.exe descendant holding duplicated pipe handles blocks that
  32. # join indefinitely (#87134). It also passes CREATE_NO_WINDOW: this scan can run from the windowless
  33. # pythonw.exe desktop/gateway backend during an update, where a bare wmic spawn would pop a console
  34. # window.
  35. from hermes_cli._subprocess_compat import bounded_probe_run
  36. result = bounded_probe_run(
  37. ["wmic", "process", "get", "ProcessId,CommandLine", "/FORMAT:LIST"],
  38. timeout=10, errors="ignore")
  39. if result is None or result.returncode != 0 or result.stdout is None:
  40. return rows
  41. current_cmd = ""
  42. for line in result.stdout.split("\n"):
  43. line = line.strip()
  44. if line.startswith("CommandLine="):
  45. current_cmd = line[len("CommandLine=") :]
  46. elif line.startswith("ProcessId="):
  47. _append_row(rows, line[len("ProcessId=") :], current_cmd)
  48. return rows
  49. # ps, not `pgrep -f "hermes.*dashboard"` (greedy regex; consistent with gateway pid scan).
  50. result = subprocess.run(["ps", "-A", "-o", "pid=,command="], timeout=10, **_PS_RUN_KWARGS, check=False)
  51. if result.returncode == 0:
  52. for line in getattr(result, "stdout", "").split("\n"):
  53. parts = line.strip().split(None, 1)
  54. if len(parts) == 2 and "grep" not in line:
  55. _append_row(rows, parts[0], parts[1])
  56. return rows
  57. def _scan_dashboard_processes(*, exclude_pids: set[int] | None = None) -> list[tuple[int, str]]:
  58. """``(pid, cmdline)`` of running ``dashboard``/``serve`` processes; empty on any scan error.
  59. A forgotten dashboard keeps the old Python backend against the new JS bundle after
  60. ``hermes update`` (every API call 401s). *exclude_pids* (Desktop's HERMES_DESKTOP_CHILD_PID
  61. backends) are never returned.
  62. *exclude_pids* is an optional set of PIDs that must never be returned. This is used by the Hermes
  63. Desktop Electron app to protect its own backend child process: when the desktop spawns ``hermes serve``
  64. as a backend and triggers an auto-update, the update must not kill the backend that the desktop itself
  65. manages. The desktop sets the environment variable ``HERMES_DESKTOP_CHILD_PID`` on the spawned backend
  66. process; ``_kill_stale_dashboard_processes`` reads it and passes it here. (#37532)
  67. """
  68. skip = {os.getpid(), *(exclude_pids or ())}
  69. # Canonical token matcher, never argv substrings: ``hermes serve`` is a prefix of ``hermes
  70. # server`` and this list decides a SIGTERM — ``herdr --session hermes server`` (a terminal
  71. # multiplexer) was killed and its unit restarted by ``hermes update`` (#121156).
  72. from hermes_cli.update_cmd_windows import _hermes_holder_subcommand
  73. try:
  74. found = [(pid, cmd) for pid, cmd in _iter_process_table()
  75. if pid not in skip and _hermes_holder_subcommand(cmd) in ("dashboard", "serve")]
  76. except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
  77. return []
  78. # Spawn-ledger augmentation: an argv scan misses a truncated or unreadable cmdline; the ledger
  79. # holds live-verified pids. Unavailable ledger → scan-only.
  80. with contextlib.suppress(Exception):
  81. # Every serve/ dashboard registers itself in the machine spawn ledger at startup with live-verified
  82. # (pid, create_time), so ledger rows are positive identity, not argv guessing. Add any live ledger
  83. # serve/dashboard the scan missed; prefer the ledger's recorded argv (full launch args) over the
  84. # scan's truncated view. See #81564.
  85. from hermes_cli.process_identity import ledger_entries
  86. seen = {pid for pid, _ in found} | skip
  87. for entry in ledger_entries():
  88. pid = entry.get("pid")
  89. if (entry.get("purpose") in ("serve", "dashboard") and isinstance(pid, int)
  90. and pid not in seen):
  91. found.append((pid, str(entry.get("argv") or "")))
  92. return found
  93. def _ledger_serve_binds() -> dict[int, tuple[str, int]]:
  94. """``pid -> (host, port)`` recorded in the spawn ledger for live serve/dashboard backends.
  95. The entry is written after the bind, so it carries the real port where argv only says
  96. ``--port 0`` (Desktop SSH backends ask the OS for a port). Empty when the ledger is unavailable.
  97. """
  98. binds: dict[int, tuple[str, int]] = {}
  99. with contextlib.suppress(Exception):
  100. from hermes_cli.process_identity import ledger_entries
  101. for entry in ledger_entries():
  102. pid, port = entry.get("pid"), entry.get("port")
  103. if (entry.get("purpose") in ("serve", "dashboard") and isinstance(pid, int)
  104. and isinstance(port, int) and port > 0):
  105. binds[pid] = (str(entry.get("host") or ""), port)
  106. return binds
  107. def _pid_environ(pid: int) -> dict[str, str] | None:
  108. """Exec-time environment of *pid* (psutil, then /proc); ``None`` when unreadable."""
  109. with contextlib.suppress(Exception):
  110. import psutil
  111. return dict(psutil.Process(pid).environ())
  112. try:
  113. raw = Path(f"/proc/{pid}/environ").read_bytes()
  114. except OSError:
  115. return None
  116. env: dict[str, str] = {}
  117. for part in raw.split(b"\x00"):
  118. key, sep, value = part.partition(b"=")
  119. if sep:
  120. env[key.decode("utf-8", errors="replace")] = value.decode("utf-8", errors="replace")
  121. return env
  122. def _pid_passwd_home(pid: int) -> str | None:
  123. """Login home of the user *pid* runs as, from the password database (psutil, then /proc).
  124. A service unit with a scrubbed environment exports no ``HOME``; the target resolves its own
  125. default home through ``Path.home()``, which falls back to this entry. The inspecting process's
  126. home belongs to a different user and must never stand in for it. ``None`` when the owner or the
  127. entry is unreadable, leaving the caller its existing fallback.
  128. """
  129. uid: int | None = None
  130. with contextlib.suppress(Exception):
  131. import psutil
  132. uid = psutil.Process(pid).uids().real
  133. if uid is None:
  134. with contextlib.suppress(OSError):
  135. uid = os.stat(f"/proc/{pid}").st_uid
  136. if uid is None:
  137. return None
  138. with contextlib.suppress(Exception):
  139. import pwd
  140. return pwd.getpwuid(uid).pw_dir or None
  141. return None
  142. def _hermes_home_for_pid(pid: int, *, require_launch_identity: bool = False) -> str | None:
  143. """Resolve a process home for discovery, or require pinned launch identity for termination.
  144. By default, ``None`` means an unreadable environment (another user, hardened ``/proc``).
  145. ``require_launch_identity`` also refuses missing argv, relative/missing home anchors and
  146. any profile selection that would depend on today's sticky ``active_profile``.
  147. A readable environment always resolves, replaying ``_apply_profile_override`` on the target's
  148. exec-time env + argv (``hermes -p X serve`` rewrites ``HERMES_HOME`` in ``os.environ`` AFTER
  149. startup, which ``/proc/<pid>/environ`` never reflects): a profile-shaped ``HERMES_HOME``
  150. without a flag is the home; otherwise the root is ``HERMES_HOME`` (its grandparent when
  151. profile-shaped) or the platform default of the process's own ``HOME`` / ``LOCALAPPDATA``
  152. (its owner's password-database home when a scrubbed unit environment exports neither), and
  153. the profile is the ``--profile``/``-p`` flag, else the root's sticky ``active_profile`` unless
  154. the process has a fixed identity (supervised child, post-swap updater, Desktop SSH backend).
  155. """
  156. # Destructive callers require launch evidence; discovery may still use sticky selection.
  157. env = _pid_environ(pid)
  158. if env is None:
  159. return None
  160. from hermes_cli.main_dashboard import _dashboard_cmdline_for_pid
  161. from hermes_cli.profiles import get_active_profile, normalize_profile_name, profile_root_for_env_home
  162. argv = _dashboard_cmdline_for_pid(pid)
  163. if require_launch_identity and argv is None:
  164. # The dashboard argv reader has no Windows implementation; psutil does.
  165. import psutil
  166. argv = psutil.Process(pid).cmdline()
  167. if require_launch_identity and not argv:
  168. return None
  169. argv = argv or []
  170. env_home = env.get("HERMES_HOME", "").strip()
  171. profile = _profile_flag_value(argv)
  172. if require_launch_identity:
  173. anchor = (env.get('LOCALAPPDATA') or env.get('USERPROFILE')) if sys.platform == 'win32' else env.get('HOME')
  174. if env_home and not Path(env_home).is_absolute():
  175. return None
  176. if not env_home and (not anchor or not Path(anchor).is_absolute()):
  177. return None
  178. if profile is None and env_home and (
  179. Path(env_home).parent.name == "profiles" or env.get("HERMES_UPDATE_POST_SWAP") == "1"
  180. ):
  181. return env_home
  182. if sys.platform == "win32":
  183. local_appdata = env.get("LOCALAPPDATA", "").strip()
  184. base = Path(local_appdata) if local_appdata else Path(env.get("USERPROFILE") or Path.home()) / "AppData" / "Local"
  185. default_home = base / "hermes"
  186. else:
  187. default_home = Path(env.get("HOME") or _pid_passwd_home(pid) or Path.home()) / ".hermes"
  188. root = profile_root_for_env_home(env_home, default_home)
  189. fixed_identity = any(env.get(k) for k in ("HERMES_SUPERVISED_CHILD", "HERMES_S6_SUPERVISED_CHILD",
  190. "HERMES_GATEWAY_EXTERNAL_SUPERVISOR",
  191. "HERMES_GATEWAY_PROFILE_PIN")) or is_desktop_ssh_backend_argv(argv)
  192. if profile is None and not fixed_identity:
  193. if require_launch_identity:
  194. return None # The sticky profile at launch is unknowable; never read today's value.
  195. profile = get_active_profile(root)
  196. canon = normalize_profile_name(profile) if profile else "default"
  197. return str(root) if canon == "default" else str(root / "profiles" / canon)
  198. def _dashboard_subcommand_index(argv: list[str]) -> int | None:
  199. return next((i for i, tok in enumerate(argv) if tok in ("serve", "dashboard")), None)
  200. def _profile_flag_value(argv: list[str]) -> str | None:
  201. """Value of the first ``--profile X`` / ``-p X`` / ``--profile=X`` in *argv*."""
  202. for i, tok in enumerate(argv):
  203. if tok in ("--profile", "-p") and i + 1 < len(argv):
  204. return str(argv[i + 1])
  205. if tok.startswith("--profile="):
  206. return tok.split("=", 1)[1]
  207. return None
  208. def _is_ephemeral_port_zero_backend(argv: list[str]) -> bool:
  209. """True for Desktop-style ``serve|dashboard --port 0`` backends — replaying them after
  210. ``hermes update`` multiplies listening backends because ``--port 0`` binds a fresh port.
  211. See #78821.
  212. """
  213. if _dashboard_subcommand_index(argv) is None:
  214. return False
  215. return any((tok == "--port" and i + 1 < len(argv) and str(argv[i + 1]) == "0")
  216. or (tok.startswith("--port=") and tok.split("=", 1)[1].strip() == "0")
  217. for i, tok in enumerate(argv))
  218. def _normalize_dashboard_cmdline(argv: list[str]) -> tuple[str, ...]:
  219. """Collapse argv to profile flags + serve/dashboard tail for dedupe."""
  220. idx = _dashboard_subcommand_index(argv)
  221. if idx is None:
  222. return tuple(argv)
  223. prefix: list[str] = []
  224. i = 0
  225. while i < idx:
  226. tok = argv[i]
  227. if tok in ("--profile", "-p") and i + 1 < idx:
  228. prefix.extend([tok, argv[i + 1]])
  229. i += 2
  230. continue
  231. if tok.startswith("--profile="):
  232. prefix.append(tok)
  233. i += 1
  234. return tuple(prefix + list(argv[idx:]))
  235. def _resolved_home(home: str) -> Path:
  236. try:
  237. return Path(home).resolve()
  238. except (OSError, RuntimeError, ValueError):
  239. return Path(home)
  240. def _normalized_home_for_compare(home: str) -> str:
  241. """Install-identity key for *home*: symlinked / differently-spelled roots compare equal.
  242. See #94030.
  243. """
  244. return os.path.normcase(str(_resolved_home(home)))
  245. def _pids_owned_by_hermes_home(pids: list[int], home: str) -> list[int]:
  246. """Return only *pids* whose resolved Hermes home (``_hermes_home_for_pid``) is ``home``.
  247. Dashboard argv is discovery-only: it is not an ownership proof because
  248. several Hermes installs and profiles can run the same command on one
  249. machine. An unreadable process environment is deliberately not treated
  250. as a match, so a stop request fails closed rather than taking down an
  251. unrelated backend.
  252. """
  253. target = _normalized_home_for_compare(home)
  254. return [
  255. pid for pid in pids
  256. if (pid_home := _hermes_home_for_pid(pid))
  257. and _normalized_home_for_compare(pid_home) == target
  258. ]
  259. def _profile_key_for_respawn(argv: list[str], hermes_home: str | None = None) -> str:
  260. """Stable owner key: ``HERMES_HOME`` when known, else ``--profile`` / ``-p``.
  261. A home ending in ``profiles/<name>`` → ``profile:<name>`` (shares a cap with an explicit
  262. ``--profile``); other homes keep a ``home:`` key so unrelated installs never collapse.
  263. See #78821.
  264. """
  265. if hermes_home:
  266. parts = _resolved_home(hermes_home).parts
  267. if len(parts) >= 2 and parts[-2] == "profiles" and parts[-1]:
  268. return f"profile:{parts[-1]}"
  269. return f"home:{_normalized_home_for_compare(hermes_home)}"
  270. return f"profile:{_profile_flag_value(argv) or 'default'}"
  271. def _filter_dashboard_respawn_candidates(
  272. candidates: list[tuple[int, list[str], str | None]], *, own_home: str | None = None
  273. ) -> list[list[str]]:
  274. """Select which killed manual backends ``(pid, argv, hermes_home)`` to respawn after update.
  275. Rules: never resurrect Desktop ``--port 0`` backends; never replay a backend from a
  276. **foreign** ``HERMES_HOME`` (the argv-only respawn would come back on this install's home
  277. and steal the foreign install's fixed port → EADDRINUSE crash-loop; unreadable ``None``
  278. stays eligible); dedupe by normalized cmdline; one backend per profile / home. PPID-1 is
  279. NOT skipped: a prior respawn detaches, so fixed-port manual backends sit under init.
  280. 1. Never resurrect Desktop ephemeral ``serve|dashboard --port 0`` backends — Desktop
  281. (``HERMES_DESKTOP_CHILD_PID``) owns their lifecycle. These are also the PPID-1 orphans that previously
  282. multiplied across updates because ``--port 0`` always binds a fresh free port. 2. A foreign install's
  283. backend is owned by that install's supervisor/user. 3. 4. See #78821, #94030.
  284. Intentionally does **not** blanket-skip every PPID-1 process: a prior ``hermes update`` respawn detaches
  285. with ``start_new_session=True``, so fixed-port manual backends are reparented to init and must still be
  286. eligible for the next update's #40449 restart.
  287. """
  288. if own_home is None:
  289. try:
  290. from hermes_constants import get_hermes_home
  291. own_home = str(get_hermes_home())
  292. except Exception:
  293. own_home = ""
  294. own_key = _normalized_home_for_compare(own_home) if own_home else ""
  295. selected: list[list[str]] = []
  296. seen_cmdlines: set[tuple[str, ...]] = set()
  297. seen_profiles: set[str] = set()
  298. for _pid, argv, hermes_home in candidates:
  299. if not argv or _is_ephemeral_port_zero_backend(argv):
  300. continue
  301. if own_key and hermes_home and _normalized_home_for_compare(hermes_home) != own_key:
  302. continue
  303. norm = _normalize_dashboard_cmdline(argv)
  304. profile_key = _profile_key_for_respawn(argv, hermes_home)
  305. if norm in seen_cmdlines or profile_key in seen_profiles:
  306. continue
  307. seen_cmdlines.add(norm)
  308. seen_profiles.add(profile_key)
  309. selected.append(list(argv))
  310. return selected
  311. def _exclude_pids_from_env() -> set[int]:
  312. """PIDs Desktop marks as live backends (``HERMES_DESKTOP_CHILD_PID``, comma-separated)."""
  313. out: set[int] = set()
  314. for part in os.environ.get("HERMES_DESKTOP_CHILD_PID", "").split(","):
  315. with contextlib.suppress(ValueError):
  316. out.add(int(part))
  317. return out
  318. #: Executables that only *carry* a hermes command line. A process headed by one of these
  319. #: never serves traffic itself; when its argv matches the dashboard patterns it is a
  320. #: wrapper around the command (``bash -c 'hermes dashboard --stop'``), not a backend.
  321. _WRAPPER_HEAD_COMMANDS = frozenset({
  322. "ash", "bash", "csh", "dash", "fish", "ksh", "sh", "tcsh", "zsh",
  323. "env", "nohup", "nice", "stdbuf", "timeout", "watch", "xargs",
  324. "screen", "tmux", "sudo",
  325. })
  326. def _caller_ancestor_pids() -> set[int]:
  327. """PIDs of THIS process's ancestors (self excluded), best-effort; empty on any failure.
  328. ``--stop`` and the update sweep must never kill the process tree they were invoked
  329. from. psutil is primary; the ``/proc`` walk keeps the answer when psutil is unusable.
  330. """
  331. try:
  332. import psutil
  333. return {p.pid for p in psutil.Process().parents()}
  334. except Exception:
  335. pass
  336. ancestors: set[int] = set()
  337. cur = os.getpid()
  338. for _ in range(2048): # cycle / corrupt-PPid guard
  339. try:
  340. status_text = Path(f"/proc/{cur}/status").read_text(
  341. encoding="utf-8", errors="replace")
  342. for line in status_text.splitlines():
  343. if line.startswith("PPid:"):
  344. cur = int(line.split()[1])
  345. break
  346. else:
  347. return ancestors
  348. except (OSError, ValueError, IndexError):
  349. return ancestors
  350. if cur <= 1:
  351. return ancestors
  352. ancestors.add(cur)
  353. return ancestors
  354. def _argv_head_command(pid: int) -> str | None:
  355. """Basename of *pid*'s first argv token, best-effort; ``None`` when unreadable."""
  356. try:
  357. import psutil
  358. argv = psutil.Process(pid).cmdline()
  359. if argv:
  360. return os.path.basename(str(argv[0]))
  361. except Exception:
  362. pass
  363. try:
  364. raw = Path(f"/proc/{pid}/cmdline").read_bytes()
  365. except OSError:
  366. return None
  367. head = raw.split(b"\x00", 1)[0].decode("utf-8", "replace")
  368. return head.rsplit("/", 1)[-1] or None
  369. def _is_caller_wrapper_shell(pid: int, ancestors: set[int]) -> bool:
  370. """True when *pid* is a caller ancestor headed by a wrapper executable.
  371. Root selection is a substring match, so the shell a ``--stop`` was typed into (or a
  372. ``bash -c 'hermes dashboard --stop'`` wrapper) matches on its own argv. Ancestor alone
  373. is not a spare: the backend hosting a shell-escaped TUI is also the caller's ancestor
  374. and must stay stoppable — only a wrapper-headed ancestor is spared.
  375. """
  376. if pid not in ancestors:
  377. return False
  378. return (_argv_head_command(pid) or "") in _WRAPPER_HEAD_COMMANDS
  379. def _kill_pids_windows(pids: list[int], killed: list[int], failed: list[tuple[int, str]]) -> None:
  380. """``taskkill /F`` each PID after re-verifying its identity."""
  381. from gateway.status import get_process_start_time
  382. from hermes_cli._subprocess_compat import pid_is_hermes, windows_hide_flags
  383. # Identity captured right after discovery: a PID reused before the kill fails the check.
  384. pid_start_times = {pid: get_process_start_time(pid) for pid in pids}
  385. for pid in pids:
  386. try:
  387. expected_start_time = pid_start_times.get(pid)
  388. if expected_start_time is None:
  389. failed.append((pid, "could not verify process identity"))
  390. elif not pid_is_hermes(pid, expected_start_time=expected_start_time):
  391. failed.append((pid, "not hermes-owned or process identity changed"))
  392. else:
  393. result = subprocess.run(
  394. ["taskkill", "/PID", str(pid), "/F"], capture_output=True,
  395. stdin=subprocess.DEVNULL, text=True, encoding="utf-8",
  396. errors="replace", timeout=10, creationflags=windows_hide_flags(), check=False)
  397. if result.returncode == 0:
  398. killed.append(pid)
  399. else:
  400. failed.append((pid, (result.stderr or result.stdout or "").strip()))
  401. except (FileNotFoundError, subprocess.TimeoutExpired, OSError) as e:
  402. failed.append((pid, str(e)))
  403. # SIGTERM → SIGKILL grace for the dashboard/serve backend. Must outlast the lifespan teardown in
  404. # hermes_cli/web_server.py::_lifespan: stop_hosted_room_service(timeout=5.0) + the startup-thread
  405. # join(1.0) + PTY_REGISTRY.close_all() (concurrent; ≤ ~4s per PTY, see pty_bridge._MAX_HELPER_SHUTDOWN_GRACE_S). A SIGKILL inside
  406. # that window skips close_all(), so the ui-tui / tui_gateway.entry children outlive the backend
  407. # and keep the deleted state.db-wal inode open — the next hermes start refuses with a FATAL
  408. # DeletedWalGenerationError (#111912). The orphan reaper's 1.5s (`_reap_orphaned_desktop_local_serves`)
  409. # is deliberately shorter: it runs on the Desktop boot path under a 10s ready-probe.
  410. _POSIX_TERM_GRACE_SECONDS = 10.0
  411. # Grace for a descendant that outlived the backend's own teardown. It already got the backend's
  412. # SIGTERM forwarded (or SIGHUP from its PTY master closing); anything still up is wedged, and a
  413. # wedged ui-tui keeps the deleted state.db-wal inode open until the next start refuses with
  414. # DeletedWalGenerationError (#112631) — no finite root grace can cover an unbounded teardown.
  415. _POSIX_DESCENDANT_GRACE_SECONDS = 2.0
  416. _NO_TTY = ("?", "??", "-") # Linux / macOS / BSD spellings of "no controlling terminal"
  417. def _is_detached_session_leader(pid: int, tty: str) -> bool:
  418. """True for a process the dashboard launched with ``start_new_session`` (own session, no tty).
  419. Messaging-gateway bots and profile actions started from ``/api/gateway/*`` are such processes:
  420. they are the user's, not the dashboard's, and must survive a dashboard stop. A hosted
  421. ``hermes --tui`` child is a session leader too (``pty.fork``) but owns the pts whose master the
  422. dashboard held, so its tty column is set and it stays in the sweep.
  423. Known gap: the turn-isolation ``tui_gateway.compute_host`` and ``slash_worker`` children are
  424. launched the same way (``start_new_session=True``, no tty; ``compute_host`` also holds
  425. ``state.db``), so a wedged WAL holder of that class is spared here too. Their exit relies on
  426. ``compute_host._parent_guard_loop`` and on the gateway's owner-death supervisor
  427. (``process_identity.spawn_owned``) respectively, not on this sweep. Discriminating by WAL-holder
  428. identity (``iter_deleted_sqlite_sidecar_holders``) was deliberately not done in this change.
  429. """
  430. if tty not in _NO_TTY:
  431. return False
  432. try:
  433. return os.getsid(pid) == pid
  434. except OSError:
  435. return False
  436. def _posix_descendants(roots: list[int]) -> dict[int, tuple[int, int | None]]:
  437. """``{pid: (root, start_time)}`` of every dashboard-owned descendant of *roots*, snapshotted
  438. BEFORE the kill: once the root dies its children are reparented and the PPID link is gone.
  439. Detached session leaders (see ``_is_detached_session_leader``) are pruned together with their own
  440. subtrees. So is the calling process with its subtree and its ancestor chain: ``hermes dashboard
  441. --stop`` / ``hermes update`` run from a shell escape inside the hosted Chat TUI are same-session
  442. descendants of the backend, and sweeping them would SIGTERM the caller mid-run (POSIX twin of the
  443. Windows #98814 hazard). The start-time fingerprint is the PID-reuse guard (same one
  444. ``_kill_pids_windows`` uses). Empty on scan failure → root-only kill, the historical behaviour.
  445. """
  446. from gateway.status import get_process_start_time
  447. try:
  448. result = subprocess.run(["ps", "-A", "-o", "pid=,ppid=,tty="], timeout=10, **_PS_RUN_KWARGS, check=False)
  449. except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
  450. return {}
  451. children: dict[int, list[tuple[int, str]]] = {}
  452. parent: dict[int, int] = {}
  453. for line in (result.stdout or "").splitlines():
  454. parts = line.split()
  455. if len(parts) == 3 and parts[0].isdigit() and parts[1].isdigit():
  456. children.setdefault(int(parts[1]), []).append((int(parts[0]), parts[2]))
  457. parent[int(parts[0])] = int(parts[1])
  458. me = os.getpid()
  459. ancestors: set[int] = set()
  460. cur = me
  461. while (cur := parent.get(cur, 0)) > 1 and cur not in ancestors:
  462. ancestors.add(cur)
  463. found: dict[int, tuple[int, int | None]] = {}
  464. pending = [(root, root) for root in roots]
  465. while pending:
  466. root, cur = pending.pop()
  467. for pid, tty in children.get(cur, ()):
  468. if pid in found or pid in roots or pid == me or _is_detached_session_leader(pid, tty):
  469. continue
  470. if pid not in ancestors: # an ancestor of the caller is spared, but its other children are not
  471. found[pid] = (root, get_process_start_time(pid))
  472. pending.append((root, pid))
  473. return found
  474. def _wait_gone(pids: list[int], seconds: float) -> list[int]:
  475. """Poll up to *seconds*; return the PIDs still alive (zombies count as gone)."""
  476. import time as _time
  477. from gateway.status import _pid_exists
  478. deadline = _time.monotonic() + seconds
  479. alive = list(pids)
  480. while alive and _time.monotonic() < deadline:
  481. _time.sleep(0.1)
  482. alive = [p for p in alive if _pid_exists(p)] # os.kill(pid, 0) breaks on Windows
  483. return alive
  484. def _kill_pids_posix(pids: list[int], killed: list[int], failed: list[tuple[int, str]]) -> None:
  485. """SIGTERM, wait up to ``_POSIX_TERM_GRACE_SECONDS`` for graceful exit, SIGKILL survivors, then
  486. sweep the dashboard-owned descendants that outlived the root and wait for the tree to be gone.
  487. *killed* reports the roots only; swept descendants are the roots' own teardown debt. A descendant
  488. still alive after its SIGKILL grace is appended to *failed*: the stop must not be declared
  489. complete while a wedged ui-tui still holds the deleted state.db-wal inode (#112631).
  490. """
  491. import signal as _signal
  492. from gateway.status import get_process_start_time
  493. descendants = _posix_descendants(pids)
  494. def _send(pid: int, sig) -> None:
  495. try:
  496. os.kill(pid, sig)
  497. if sig == _signal.SIGKILL:
  498. killed.append(pid)
  499. except ProcessLookupError:
  500. killed.append(pid) # already gone — count as killed
  501. except (PermissionError, OSError) as e:
  502. failed.append((pid, str(e)))
  503. for pid in pids:
  504. _send(pid, _signal.SIGTERM)
  505. pending = [p for p in pids if p not in killed and p not in {f[0] for f in failed}]
  506. alive = _wait_gone(pending, _POSIX_TERM_GRACE_SECONDS)
  507. killed.extend(p for p in pending if p not in alive)
  508. for pid in alive:
  509. _send(pid, _signal.SIGKILL)
  510. # Snapshot identity must still match: a PID recycled during the grace is not ours to signal.
  511. survivors = [p for p, (_root, start) in descendants.items()
  512. if start is not None and get_process_start_time(p) == start]
  513. for sig in (_signal.SIGTERM, _signal.SIGKILL):
  514. for pid in survivors:
  515. with contextlib.suppress(OSError):
  516. os.kill(pid, sig)
  517. survivors = _wait_gone(survivors, _POSIX_DESCENDANT_GRACE_SECONDS)
  518. failed.extend((pid, "descendant of the stopped backend still alive after SIGKILL")
  519. for pid in survivors)
  520. def _kill_stale_dashboard_processes(
  521. reason: str = "the running backend no longer matches the updated frontend", *,
  522. restart_managed: bool = False, already_restarted_units: set[str] | None = None,
  523. scope_home: str | None = None,
  524. ) -> dict[str, list]:
  525. """Kill running ``hermes dashboard`` / ``hermes serve`` processes (update end, ``--stop``).
  526. With ``restart_managed`` (update only) systemd-owned PIDs get their unit restarted after the
  527. kill (systemd treats our SIGTERM as a clean stop, so ``Restart=on-failure`` never fires) and
  528. manual PIDs are respawned from captured argv. PIDs owned by *already_restarted_units* (no
  529. ``.service`` suffix) are left untouched, not killed twice.
  530. When *scope_home* is supplied, only processes with that exact live
  531. ``HERMES_HOME`` are candidates; unknown ownership fails closed. This is
  532. used by ``dashboard --stop`` and the per-profile update cleanup.
  533. Manually-started dashboards are not auto-restarted because we don't know the original launch args
  534. (--host, --port, --insecure, --tui, --no-open). See #68934.
  535. *already_restarted_units* names units (no ``.service`` suffix) the caller already handled directly —
  536. e.g. ``hermes update``'s systemd fleet-restart loop, which restarts, declines (without root) or fails
  537. ``hermes-serve*``/``hermes-dashboard*`` units, reporting each, before this function runs. Without excluding them,
  538. a Serve-only install's freshly restarted process is found again here and restarted a second time for
  539. no benefit (review on #83595), and a declined system unit is retried into a polkit prompt (#141021).
  540. """
  541. from hermes_cli import main_dashboard as _dash
  542. _dash_unit = str(getattr(_dash, "_DASHBOARD_SYSTEMD_UNIT", "hermes-dashboard.service")).removesuffix(".service")
  543. # The update's fleet loop already handled the dashboard unit (restarted it, or declined for lack
  544. # of root and printed the manual command): a second attempt here restarts a healthy unit twice,
  545. # or retries without --no-ask-password and dumps a polkit/tty error under the same advice (#141021).
  546. if (restart_managed and _dash_unit not in (already_restarted_units or ())
  547. and _dash._restart_managed_dashboard_service(reason)):
  548. # The dashboard unit is handled but other backends (e.g. hermes-serve.service) are not:
  549. # mark the unit handled so the filter below drops its PIDs, and keep going.
  550. already_restarted_units = set(already_restarted_units or ()) | {_dash_unit}
  551. exclude = _exclude_pids_from_env()
  552. if restart_managed:
  553. # An SSH-owned backend belongs to an attached Desktop client; killing it strands that
  554. # client's fixed SSH port-forward. Same ownership records as the reaper.
  555. exclude |= _lock_owned_serve_pids()
  556. pids = _dash._find_stale_dashboard_pids(exclude_pids=exclude or None, scope_home=scope_home)
  557. if not pids:
  558. return _empty_result()
  559. # Snapshot systemd unit/cgroup and argv BEFORE killing (the cgroup dies with the process).
  560. pid_cgroup: dict[int, str | None] = {}
  561. pid_service: dict[int, str | None] = {}
  562. pid_launchd: dict[int, tuple[str, str, int | None]] = {}
  563. pid_cmdline: dict[int, list[str]] = {}
  564. pid_home: dict[int, str | None] = {}
  565. # macOS: a backend supervised by a launchd job (LaunchAgent / LaunchDaemon) must come back
  566. # through launchd, never as a detached argv respawn — the respawn holds the job's port, the
  567. # job then fails every KeepAlive restart with "port already in use", and the running backend
  568. # is left unsupervised. Snapshot the loaded jobs once, before the kill; ``--stop`` reads them
  569. # too, so it can say that a KeepAlive job will undo the stop.
  570. launchd_jobs = _dash._loaded_launchd_backend_jobs() if sys.platform != "win32" else []
  571. def _launchd_owner(pid: int, cmdline: list[str] | None):
  572. return _dash._launchd_job_owning_backend(pid, cmdline, launchd_jobs, ancestors=_process_ancestors(pid))
  573. if restart_managed and sys.platform != "win32":
  574. for pid in pids:
  575. pid_cgroup[pid] = _dash._get_pid_cgroup_path(pid)
  576. pid_service[pid] = _dash._get_systemd_service_for_pid(pid)
  577. if pid_service[pid]:
  578. continue
  579. cmdline = _dash._dashboard_cmdline_for_pid(pid)
  580. if launchd_jobs and (job := _launchd_owner(pid, cmdline)):
  581. pid_launchd[pid] = job
  582. elif cmdline:
  583. # Manual process: exact argv + HERMES_HOME for the respawn and its profile cap.
  584. # Manually-started process: preserve its exact argv so we can respawn it after the update
  585. # (#40449, #68934). Snapshot HERMES_HOME before the kill so per-profile caps still work
  586. # after the process is gone (#78821).
  587. pid_cmdline[pid] = cmdline
  588. pid_home[pid] = _hermes_home_for_pid(pid)
  589. if already_restarted_units:
  590. pids = [pid for pid in pids if (pid_service.get(pid) or "").removesuffix(".service")
  591. not in already_restarted_units]
  592. if not pids:
  593. return _empty_result()
  594. elif launchd_jobs:
  595. for pid in pids:
  596. if job := _launchd_owner(pid, _dash._dashboard_cmdline_for_pid(pid)):
  597. pid_launchd[pid] = job
  598. print(f"\n⟲ Stopping {len(pids)} dashboard process(es) ({reason})")
  599. killed: list[int] = []
  600. failed: list[tuple[int, str]] = []
  601. (_kill_pids_windows if sys.platform == "win32" else _kill_pids_posix)(pids, killed, failed)
  602. for pid in killed:
  603. print(f" ✓ stopped PID {pid}")
  604. for pid, err_msg in failed:
  605. print(f" ✗ failed to stop PID {pid}: {err_msg}")
  606. if killed and restart_managed:
  607. unrecovered = _restart_killed_backends(
  608. killed, pid_service, pid_cgroup, pid_cmdline, pid_home, pid_launchd=pid_launchd)
  609. else:
  610. unrecovered = list(killed)
  611. # A stopped launchd job with KeepAlive restarts itself: say so instead of a misleading
  612. # "restart it yourself" hint, and give the command that actually keeps it down.
  613. for target in sorted({f"{d}/{l}" for p in killed if (j := pid_launchd.get(p)) for d, l, _ in (j,)}):
  614. print(f" ⚠ PID(s) supervised by launchd job {target}: a KeepAlive job restarts itself.\n"
  615. f" To keep it down: launchctl bootout {target}")
  616. if any(p not in pid_launchd for p in killed):
  617. print(" Restart the dashboard when you're ready:\n hermes dashboard --port <port>")
  618. return {"matched": list(pids), "killed": list(killed), "failed": list(failed),
  619. "unrecovered": list(unrecovered)}
  620. def _restart_killed_backends(
  621. killed: list[int], pid_service: dict[int, str | None], pid_cgroup: dict[int, str | None],
  622. pid_cmdline: dict[int, list[str]], pid_home: dict[int, str | None], *,
  623. pid_launchd: dict[int, tuple[str, str, int | None]] | None = None) -> list[int]:
  624. """Update path: restart systemd units, kickstart launchd jobs (macOS), respawn manual argv
  625. (detached, headless, logged to logs/dashboard-restart.log; one per profile, no ``--port 0``).
  626. Returns PIDs not brought back."""
  627. # Two categories: Without this, a remote backend (hermes serve) under Restart=on-failure never comes
  628. # back after our clean SIGTERM, and the Desktop can't reconnect (#68934). Filtered so Desktop
  629. # ``serve|dashboard --port 0`` backends are not resurrected and duplicates collapse to one per profile
  630. # (#78821).
  631. from hermes_cli import main_dashboard as _dash
  632. unrecovered: list[int] = []
  633. failed_restarts: list[tuple[str, str]] = []
  634. seen_services: set[str] = set()
  635. respawn_candidates: list[tuple[int, list[str], str | None]] = []
  636. for pid in killed:
  637. svc_name = pid_service.get(pid)
  638. launchd_job = (pid_launchd or {}).get(pid)
  639. if svc_name:
  640. if svc_name in seen_services:
  641. continue
  642. seen_services.add(svc_name)
  643. if _dash._try_restart_systemd_service(svc_name, pid_cgroup.get(pid)):
  644. print(f" ✓ restarted systemd service {svc_name}")
  645. else:
  646. failed_restarts.append((svc_name, "systemctl restart returned non-zero"))
  647. unrecovered.append(pid)
  648. elif launchd_job:
  649. # launchd owns the backend: the job brings it back (KeepAlive, or the kickstart below),
  650. # and success means launchd reports a fresh supervised PID — an argv respawn would sit
  651. # on the job's port and leave it failing forever.
  652. domain, label, old_pid = launchd_job
  653. target = f"{domain}/{label}"
  654. if target in seen_services:
  655. continue
  656. seen_services.add(target)
  657. if _dash._restart_launchd_job(domain, label, old_pid):
  658. print(f" ✓ restarted launchd job {target}")
  659. else:
  660. # A LaunchDaemon (system domain) can only be kickstarted by root; the hint must
  661. # be the command that works from the shell the operator is actually in.
  662. sudo = "sudo " if domain.startswith("system") and os.geteuid() != 0 else "" # windows-footgun: ok — launchd jobs exist only on macOS
  663. failed_restarts.append(
  664. (target, f"launchd is not supervising a fresh process; run: {sudo}launchctl kickstart -k {target}"))
  665. unrecovered.append(pid)
  666. elif pid in pid_cmdline:
  667. respawn_candidates.append((pid, pid_cmdline[pid], pid_home.get(pid)))
  668. else:
  669. unrecovered.append(pid)
  670. for svc, err in failed_restarts:
  671. print(f" ⚠ {svc}: {err}")
  672. respawn_cmds = _filter_dashboard_respawn_candidates(respawn_candidates)
  673. failed_cmds = _dash._respawn_dashboard_processes(respawn_cmds) if respawn_cmds else None
  674. if failed_cmds:
  675. unrecovered.extend(p for p in killed if pid_cmdline.get(p) in failed_cmds)
  676. if failed_restarts or unrecovered:
  677. print(" Restart anything not auto-restarted when you're ready:\n hermes dashboard --port <port>")
  678. return unrecovered
  679. def _norm_exe(path) -> str:
  680. """Canonical lower-cased executable path for comparison."""
  681. try:
  682. return str(Path(path).resolve()).lower()
  683. except (OSError, ValueError):
  684. return str(path).lower()
  685. def _detect_concurrent_hermes_instances(
  686. scripts_dir: Path, *, exclude_pid: int | None = None) -> list[tuple[int, str]]:
  687. """Historical main export: stop old updaters without scanning live shims.
  688. PM stages a fresh generation instead of replacing a mapped hermes.exe.
  689. Returning an empty list would let old callers continue into that mutation.
  690. """
  691. from hermes_cli._old_updater import stop_for_relaunch
  692. stop_for_relaunch()
  693. def _is_desktop_local_serve_cmdline(command: str) -> bool:
  694. """True for the Desktop-local shape ``hermes serve [--isolated] --host 127.0.0.1 --port 0``.
  695. Long-lived headless serves (``--host <tailscale-ip> --port 9119``) must never match —
  696. those are operator-managed remote backends that legitimately run with ppid 1.
  697. """
  698. from hermes_cli.update_cmd_windows import _hermes_holder_subcommand
  699. # Canonical token matcher, never argv substrings: ``kanban --preserve-cache`` contains "serve" and
  700. # ``vim notes about hermes serve`` contains both markers — this predicate decides a kill.
  701. if _hermes_holder_subcommand(command) != "serve":
  702. return False
  703. tokens = command.lower().split()
  704. host = _flag_value(tokens, "--host")
  705. return host in ("127.0.0.1", "localhost") and _flag_value(tokens, "--port") == "0"
  706. def _flag_value(tokens: list[str], flag: str) -> str | None:
  707. """``--flag value`` / ``--flag=value`` from split argv, or None."""
  708. for i, tok in enumerate(tokens):
  709. if tok == flag and i + 1 < len(tokens):
  710. return tokens[i + 1]
  711. if tok.startswith(flag + "="):
  712. return tok.partition("=")[2]
  713. return None
  714. def _process_ancestors(pid: int, *, max_depth: int = 8) -> list[int]:
  715. """Parent chain of *pid* (nearest first), stopping at init / a lookup failure / *max_depth*."""
  716. chain: list[int] = []
  717. current = pid
  718. while len(chain) < max_depth:
  719. parent = _process_ppid(current)
  720. if parent is None or parent <= 1 or parent == pid or parent in chain:
  721. break
  722. chain.append(parent)
  723. current = parent
  724. return chain
  725. def _process_ppid(pid: int) -> int | None:
  726. """Best-effort parent pid; None on failure (always None on Windows: desktop tree-kill reaps)."""
  727. try:
  728. if sys.platform == "win32":
  729. return None
  730. result = subprocess.run(["ps", "-o", "ppid=", "-p", str(pid)], timeout=5, **_PS_RUN_KWARGS, check=False)
  731. if result.returncode != 0 or not result.stdout:
  732. return None
  733. return int(result.stdout.strip().split()[0])
  734. except (ValueError, FileNotFoundError, subprocess.TimeoutExpired, OSError):
  735. return None
  736. # SSH remote-backend lock ownership: ``backend.lock.json`` is written by the Desktop SSH runtime
  737. # (apps/desktop/electron/remote-lifecycle.ts) for every ``hermes serve`` it spawns. Such a backend
  738. # is legitimate even at ppid 1 (sshd exited); the reap must NEVER kill a PID a valid lock claims
  739. # — that once killed a production backend. Schema mirrors the writer; mismatches are ignored.
  740. _LOCKFILE_SCHEMA_VERSION = 2
  741. _PROTOCOL_VERSION = 1
  742. _REMOTE_LOCK_SUBDIR = "desktop-ssh"
  743. BACKEND_LOCK_NAME = "backend.lock.json"
  744. _HEX32 = set("0123456789abcdef")
  745. def _hermes_home_dir() -> Path:
  746. """The process's Hermes home: remote-backend locks are a process-level asset, so a request scoped
  747. to another profile must still see the same lock dir."""
  748. from hermes_constants import get_process_hermes_home
  749. return get_process_hermes_home()
  750. def _is_hex(value: object, length: int) -> bool:
  751. return isinstance(value, str) and len(value) == length and not (set(value) - _HEX32)
  752. def _valid_lockfile_payload(parsed: object, ownership_id: str) -> bool:
  753. """Validate a parsed ``backend.lock.json`` body, mirroring readLockfile()."""
  754. if (
  755. not isinstance(parsed, dict)
  756. or parsed.get("schemaVersion") != _LOCKFILE_SCHEMA_VERSION
  757. or parsed.get("protocolVersion") != _PROTOCOL_VERSION
  758. or parsed.get("ownershipId") != ownership_id
  759. or not _is_hex(parsed.get("spawnNonce"), 16)
  760. or not _is_hex(parsed.get("tokenFingerprint"), 32)):
  761. return False
  762. pid, port = parsed.get("pid"), parsed.get("port")
  763. if not (isinstance(pid, int) and 0 < pid <= 4194304 and isinstance(port, int)
  764. and 0 <= port <= 65535):
  765. return False
  766. # String fields must be present and bounded (the writer enforces <=1024).
  767. if any(not isinstance(parsed.get(f), str) or len(parsed[f]) > 1024
  768. for f in ("profile", "hermesPath", "hermesHome", "logPath", "startedAt")):
  769. return False
  770. # Suffix-only check of logPath so a relocated HERMES_HOME can't reject a legitimate backend.
  771. return parsed["logPath"].endswith(f"/{ownership_id}/{parsed['spawnNonce']}.log")
  772. def _remote_lock_roots(base_dir: Path | None) -> list[Path]:
  773. """Every dir the Desktop may have written ``desktop-ssh/<ownershipId>/backend.lock.json`` under.
  774. The Desktop writes SSH locks beneath the ROOT home (``~/.hermes/desktop-ssh``), but a profile
  775. backend (``hermes --profile X serve``) runs with ``HERMES_HOME=<root>/profiles/X`` — scanning only
  776. the process home found no lock there and its reaper killed the sibling profile's live SSH
  777. backend on every profile switch (#89811)."""
  778. if base_dir is not None:
  779. return [base_dir]
  780. from hermes_constants import get_default_hermes_root
  781. roots: list[Path] = []
  782. for home in (_hermes_home_dir(), get_default_hermes_root()):
  783. root = home / _REMOTE_LOCK_SUBDIR
  784. if root not in roots:
  785. roots.append(root)
  786. return roots
  787. def _lock_owned_serve_pids(base_dir: Path | None = None) -> set[int]:
  788. """PIDs claimed by valid ``{hermes_home}/desktop-ssh/<ownershipId>/backend.lock.json`` records
  789. (best-effort: a bad record contributes no PID; never raises)."""
  790. owned: set[int] = set()
  791. entries: list[Path] = []
  792. for root in _remote_lock_roots(base_dir):
  793. try:
  794. entries.extend(root.iterdir() if root.is_dir() else [])
  795. except OSError:
  796. continue
  797. for entry in entries:
  798. lock = read_valid_backend_lock(entry / BACKEND_LOCK_NAME)
  799. if lock is not None:
  800. owned.add(lock["pid"]) # validated as int
  801. return owned
  802. def read_valid_backend_lock(lock_path: Path) -> dict | None:
  803. """The validated body of one ``desktop-ssh/<ownershipId>/backend.lock.json``, or None when it is
  804. missing, unreadable, oversized or does not match the writer's schema (never raises)."""
  805. import json
  806. ownership_id = lock_path.parent.name
  807. try: # validateOwnershipId(): exactly 32 lowercase hex chars
  808. if not _is_hex(ownership_id, 32) or not lock_path.is_file():
  809. return None
  810. with lock_path.open("rb") as f: # bounded: this runs on every owner-watchdog poll
  811. data = f.read(65537)
  812. if len(data) > 65536:
  813. return None
  814. parsed = json.loads(data)
  815. except (OSError, UnicodeDecodeError, ValueError):
  816. return None
  817. return parsed if _valid_lockfile_payload(parsed, ownership_id) else None
  818. # Covers the gap between process start and the Desktop client writing backend.lock.json.
  819. _REAP_MIN_AGE_SECONDS = 180.0
  820. def _process_age_seconds(pid: int) -> float:
  821. """Process age from psutil's cross-platform start timestamp."""
  822. import time as _time
  823. import psutil as _psutil
  824. return max(0.0, _time.time() - _psutil.Process(pid).create_time())
  825. def _reap_orphaned_desktop_local_serves(
  826. *, reason: str = "orphaned desktop-local hermes serve", signal_term=None, signal_kill=None,
  827. sleep_fn=None, lock_owned_pids_fn=None, process_age_seconds_fn=None) -> dict[str, list]:
  828. """Kill leftover Desktop-local ``hermes serve`` backends with no parent. Never raises.
  829. When Electron dies uncleanly its ``serve --host 127.0.0.1 --port 0`` children are
  830. reparented to pid 1 with their MCP trees alive; each Desktop boot then stacks a fresh
  831. backend on the corpses until EMFILE. Reaped only if ALL hold: Desktop-local shape; ppid
  832. 0/1; not self / parent / HERMES_DESKTOP_CHILD_PID; not claimed by a valid
  833. ``backend.lock.json`` (SSH backends other clients started legitimately sit at ppid 1);
  834. older than ``_REAP_MIN_AGE_SECONDS`` with a determinable age (Desktop writes the lock only
  835. after HERMES_BACKEND_READY, so a live sibling mid-startup is briefly unowned).
  836. """
  837. import signal as _signal
  838. import time as _time
  839. signal_term = _signal.SIGTERM if signal_term is None else signal_term
  840. signal_kill = getattr(_signal, "SIGKILL", _signal.SIGTERM) if signal_kill is None else signal_kill
  841. sleep_fn = sleep_fn or _time.sleep
  842. lock_owned_pids_fn = lock_owned_pids_fn or _lock_owned_serve_pids
  843. process_age_seconds_fn = process_age_seconds_fn or _process_age_seconds
  844. if sys.platform == "win32": # Windows desktop uses taskkill tree teardown
  845. return _empty_result()
  846. def _owned_pids() -> set[int]:
  847. try:
  848. return set(lock_owned_pids_fn())
  849. except Exception:
  850. return set() # never let lock scanning block or widen the reap
  851. def _is_stale_orphan(pid: int) -> bool:
  852. try: # never let a liveness probe failure widen the reap
  853. return process_age_seconds_fn(pid) >= _REAP_MIN_AGE_SECONDS
  854. except Exception:
  855. return False
  856. exclude = _exclude_pids_from_env() | {os.getpid()} | _owned_pids()
  857. with contextlib.suppress(Exception):
  858. exclude.add(os.getppid()) # the desktop / sshd wrapper
  859. try:
  860. scanned = _scan_dashboard_processes(exclude_pids=exclude)
  861. except Exception:
  862. return _empty_result()
  863. owned_now = _owned_pids() # re-read: a lock may have been written since the scan
  864. matched = [pid for pid, cmd in scanned
  865. if _is_desktop_local_serve_cmdline(cmd) and pid not in owned_now
  866. and _process_ppid(pid) in (0, 1) and _is_stale_orphan(pid)]
  867. if not matched:
  868. return _empty_result()
  869. descendants = _posix_descendants(matched) # before the kill: the root's death reparents them
  870. killed: list[int] = []
  871. failed: list[int] = []
  872. for pid in matched:
  873. try:
  874. os.kill(pid, signal_term)
  875. except ProcessLookupError:
  876. continue
  877. except OSError:
  878. failed.append(pid)
  879. # Brief grace, then SIGKILL survivors (psutil.pid_exists: os.kill(pid, 0) is a Windows footgun).
  880. sleep_fn(1.5)
  881. import psutil
  882. for pid in matched:
  883. if pid in failed:
  884. continue
  885. try:
  886. if psutil.pid_exists(pid):
  887. os.kill(pid, signal_kill)
  888. killed.append(pid)
  889. except ProcessLookupError:
  890. killed.append(pid)
  891. except OSError:
  892. failed.append(pid)
  893. # A SIGKILLed backend never ran PTY_REGISTRY.close_all(): its hosted ui-tui / MCP trees would
  894. # keep the deleted state.db-wal inode open (#112631). The boot-path budget leaves no second
  895. # grace, and these trees already lost their Electron and their backend.
  896. # A root whose own kill raised (EPERM: not ours) keeps its subtree — do not orphan it half-way.
  897. from gateway.status import get_process_start_time
  898. for pid, (root, start) in descendants.items():
  899. if root not in failed and start is not None and get_process_start_time(pid) == start:
  900. with contextlib.suppress(OSError):
  901. os.kill(pid, signal_kill)
  902. with contextlib.suppress(Exception):
  903. print(f"⟲ Reaped {len(killed)} orphaned desktop-local serve backend(s) ({reason}): {killed or matched}")
  904. return {"matched": matched, "killed": killed, "failed": failed}