mcp_tool_lifecycle.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. """MCP process lifecycle: stdio child PID tracking and orphan cleanup, graceful
  2. server shutdown and draining of the background MCP loop."""
  3. import logging
  4. import asyncio
  5. import os
  6. import time
  7. from typing import Dict, Optional
  8. from hermes_cli.process_identity import is_owner_death_supervisor, release_process_groups
  9. from tools.mcp_tool_common import _core
  10. from tools import mcp_tool_loop as _loop
  11. logger = logging.getLogger("tools.mcp_tool")
  12. # Live stdio MCP children (pid -> server_name), added after connection and removed on normal
  13. # shutdown, so they can be force-killed if SDK teardown fails.
  14. _stdio_pids: dict[int, str] = {}
  15. # PIDs that survived their session context exit (detected in _run_stdio's finally, reaped by
  16. # _kill_orphaned_mcp_children). Separate from _stdio_pids so sweeps never race active sessions.
  17. _orphan_stdio_pids: set = set()
  18. _orphan_stdio_pid_servers: dict[int, str] = {}
  19. # pid -> pgid captured at spawn. The SDK spawns with start_new_session=True (PGID == PID);
  20. # grandchildren keep that PGID after the direct child exits, so killpg still reaches them.
  21. # Separate from _stdio_pids so the PGID survives the child's removal. Empty on Windows.
  22. _stdio_pgids: dict[int, int] = {}
  23. # Spawn-time start-time fingerprints of each stdio child's pgroup leader, captured
  24. # alongside the PGID (the psutil fallback means every platform has a baseline, macOS
  25. # included). PIDs/PGIDs are recycled by the kernel once the original process exits and
  26. # is reaped, so a long-lived tracker holding a bare PGID is unsafe: by the time a sweep
  27. # runs, that number may name an unrelated process group (observed in the wild: a
  28. # desktop browser whose session leader happened to reuse a dead MCP child's PID —
  29. # #43044). We re-check the leader's start time — drift-tolerantly, since same-host
  30. # readings drift ~1 s on macOS (#117505) — before signalling so a recycled PGID is
  31. # never killed. None entries are dropped: a capture that raced the child's exit keeps
  32. # the legacy best-effort behaviour.
  33. _stdio_starttimes: dict[int, int] = {} # pid -> leader start ticks
  34. def _leader_start_time(pid: int) -> Optional[int]:
  35. """Start-time fingerprint of the pgroup leader (PGID == leader PID on setsid spawn);
  36. ``None`` only when the reading is genuinely unavailable (already-reaped PID, no
  37. /proc AND no psutil) — the psutil fallback covers macOS/Windows."""
  38. from gateway.status import get_process_start_time
  39. try:
  40. return get_process_start_time(pid)
  41. except Exception:
  42. return None
  43. def _snapshot_child_pids() -> set:
  44. """Current direct-child PIDs: /proc on Linux, else psutil, else empty set."""
  45. my_pid = os.getpid()
  46. # /proc/<pid>/task/<tid>/children is per-THREAD, and stdio_client() spawns from the MCP
  47. # loop thread, so union every task's children — reading only the main thread's file
  48. # returns an empty set on every Linux install.
  49. try:
  50. # ``/proc/<pid>/task/<tid>/children`` is per-THREAD — a child forked from thread T is listed only
  51. # under T's task dir. stdio_client() spawns from the background MCP loop thread, so reading only the
  52. # main thread's file (``task/<pid>/children``) returned an empty set on every Linux install and left
  53. # ``_stdio_child_pids`` / ``_stdio_pids`` empty: the #81995 dead-child fast-fail, the #96452 respawn
  54. # signal, and the killpg shutdown sweep never saw the subprocess.
  55. task_dir = f"/proc/{my_pid}/task"
  56. found: set = set()
  57. for tid in os.listdir(task_dir):
  58. try:
  59. with open(f"{task_dir}/{tid}/children", encoding="utf-8-sig") as f:
  60. found.update(int(p) for p in f.read().split() if p.strip())
  61. except (FileNotFoundError, OSError, ValueError):
  62. continue # thread exited between listdir and open
  63. return found
  64. except (FileNotFoundError, OSError, ValueError):
  65. pass
  66. try:
  67. import psutil
  68. return {c.pid for c in psutil.Process(my_pid).children()}
  69. except Exception:
  70. return set()
  71. # argv markers of non-MCP gateway children that can race into the snapshot delta during an
  72. # MCP spawn (defense-in-depth: LSP servers and the slash worker lead their own groups under the
  73. # owner-death supervisor, see spawn_owned). Matched against argv[1:] because Python/Java children
  74. # start with the interpreter path.
  75. _NON_MCP_CHILD_CMDLINE_MARKERS: tuple[str, ...] = (
  76. "tui_gateway.slash_worker", "tui_gateway.entry",
  77. "-dorg.eclipse.equinox.launcher", "eclipse.jdt.ls", "org.eclipse.equinox.launcher_", # jdtls
  78. )
  79. def _filter_mcp_children(pids: set) -> set:
  80. """Drop non-MCP children from a PID snapshot delta. Tracking a stray child in _stdio_pgids
  81. is catastrophic if it lacks start_new_session: its pgid can be the TUI parent's, so the
  82. shutdown killpg() would kill the TUI itself."""
  83. if not pids:
  84. return pids
  85. try:
  86. import psutil
  87. except ImportError:
  88. return pids # keep all PIDs (prior behavior)
  89. kept = set()
  90. for pid in pids:
  91. try:
  92. argv = psutil.Process(pid).cmdline()
  93. except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
  94. continue # raced away or zombie — cannot be our fresh server, unsafe to track
  95. if is_owner_death_supervisor(argv):
  96. continue # the shared reaper of every owned child; MCP teardown must never kill it
  97. if not any(marker in arg for arg in argv[1:] for marker in _NON_MCP_CHILD_CMDLINE_MARKERS):
  98. kept.add(pid)
  99. return kept
  100. def _clear_connect_cooldowns(keys=None) -> None:
  101. """Drop connect-retry cooldowns: a restart must re-attempt every server immediately, not
  102. honour a stale per-server backoff. Caller holds ``_core._lock``."""
  103. if keys is None:
  104. _core._server_connect_retry_after.clear()
  105. _core._server_connect_failures.clear()
  106. else:
  107. for key in keys:
  108. _core._server_connect_retry_after.pop(key, None)
  109. _core._server_connect_failures.pop(key, None)
  110. def _reregister_orphaned_adopters() -> None:
  111. """Re-run MCP registration for profiles whose ADOPTED shared connection an owner's
  112. ``/reload-mcp`` just tore down. Their tools vanished with the owner's teardown and nothing
  113. re-runs their discovery until THEY reload, so they sat tool-less behind a healthy-looking
  114. status (#106005). Runs after the owner's rediscovery, under each adopter's own home + secret
  115. scope (its ``${VAR}`` refs must resolve to ITS credentials): the adopter re-adopts the owner's
  116. new identical connection or connects its own."""
  117. with _core._lock:
  118. pending = dict(_core._orphaned_adopters)
  119. _core._orphaned_adopters.clear()
  120. if not pending:
  121. return
  122. from pathlib import Path
  123. from agent.secret_scope import build_profile_secret_scope, reset_secret_scope, set_secret_scope
  124. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  125. from tools import mcp_tool_discovery as _discovery
  126. from tools.mcp_tool_config import _load_mcp_config
  127. for adopter, names in pending.items():
  128. home_token = secret_token = None
  129. try:
  130. home_token = set_hermes_home_override(adopter)
  131. secret_token = set_secret_scope(build_profile_secret_scope(Path(adopter)), profile_home=adopter)
  132. servers = {n: c for n, c in (_load_mcp_config() or {}).items() if n in names}
  133. if servers:
  134. _discovery.register_mcp_servers(servers)
  135. except Exception:
  136. logger.debug("MCP: re-registration for profile scope %s failed", adopter, exc_info=True)
  137. finally:
  138. if secret_token is not None:
  139. reset_secret_scope(secret_token)
  140. if home_token is not None:
  141. reset_hermes_home_override(home_token)
  142. def shutdown_mcp_servers(*, scope: Optional[str] = None, names: Optional[set] = None,
  143. timeout: float = 15.0):
  144. """Close MCP server connections (in parallel) and stop the background loop. Each server
  145. Task is signalled to exit its own ``async with`` so the anyio cancel-scope cleanup runs in
  146. the Task that opened it. ``scope`` restricts teardown to one multiplexed profile's servers
  147. (its ``/reload-mcp`` must not kill other profiles') and leaves the shared loop running if
  148. anything else is still connected. ``names`` restricts it further to those server names
  149. (dropped-from-config pruning); other servers' bookkeeping is untouched. Only the bare call
  150. (no ``scope``, no ``names``) is the process-wide wildcard: the launch profile's registry
  151. scope IS ``None``, so ``scope=None, names={...}`` prunes that unscoped owner's servers and
  152. must leave a served profile's same-named ``(B, name)`` connection alone. ``timeout`` bounds
  153. the wait for the close to land on the MCP loop — a caller running one pass per served
  154. profile under a total budget divides it, or N profiles × 15s starve the wildcard pass that
  155. actually stops the loop."""
  156. from tools.mcp_tool_scope import _key_name
  157. wildcard = scope is None and names is None
  158. with _core._lock:
  159. selected = [key for key in _core._servers if wildcard or _core._server_scope_keys.get(key) == scope]
  160. if names is not None:
  161. selected = [key for key in selected if _key_name(key) in names]
  162. servers_snapshot = [_core._servers[key] for key in selected]
  163. if names is not None:
  164. selected_status = set(selected)
  165. elif wildcard:
  166. selected_status = (
  167. set(_core._servers) | set(_core._server_scope_keys)
  168. | set(_core._server_tool_scopes)
  169. | set(_core._server_connecting) | set(_core._server_connect_errors))
  170. else:
  171. selected_status = {key for key, owner in _core._server_scope_keys.items() if owner == scope}
  172. # Adopters of the connections being torn down lose their overlays with the tasks' own
  173. # ``_deregister_tools``; remember them so the next discovery pass re-registers them
  174. # (``_reregister_orphaned_adopters``).
  175. if not wildcard:
  176. for key in selected:
  177. for adopter in _core._server_tool_scopes.get(key, ()):
  178. if adopter != scope:
  179. _core._orphaned_adopters.setdefault(adopter, set()).add(_key_name(key))
  180. def clear_selected_status():
  181. _core._server_connecting.difference_update(selected_status)
  182. for key in selected_status:
  183. _core._server_connect_errors.pop(key, None)
  184. _core._server_scope_keys.pop(key, None)
  185. _core._server_tool_scopes.pop(key, None)
  186. # Fast path: nothing to shut down. The connect-cooldown maps can still be populated here — a server that
  187. # failed to connect is never recorded in ``_servers`` (that is the very premise of the #50394 cooldown),
  188. # so "no live servers" is the MOST likely state in which stale backoff entries exist. Clear them so a
  189. # post-shutdown restart re-attempts every configured server immediately.
  190. if servers_snapshot:
  191. async def _shutdown():
  192. results = await asyncio.gather(*(server.shutdown() for server in servers_snapshot), return_exceptions=True)
  193. for server, result in zip(servers_snapshot, results):
  194. if isinstance(result, Exception):
  195. logger.debug("Error closing MCP server '%s': %s", server.name, result)
  196. with _core._lock:
  197. for key in selected:
  198. _core._servers.pop(key, None)
  199. _core._server_scope_keys.pop(key, None)
  200. clear_selected_status()
  201. _clear_connect_cooldowns(None if wildcard else selected_status)
  202. with _core._lock:
  203. loop = _core._mcp_loop
  204. if loop is not None and loop.is_running():
  205. from agent.async_utils import safe_schedule_threadsafe
  206. future = safe_schedule_threadsafe(_shutdown(), loop, logger=logger, log_message="MCP shutdown: failed to schedule")
  207. if future is not None:
  208. try:
  209. future.result(timeout=timeout)
  210. except BaseException as exc:
  211. logger.debug("Error during MCP shutdown: %s", exc)
  212. # Unconditional final sweep: whether ``_shutdown`` ran, timed out, or was never scheduled
  213. # (a server that failed to connect is never in ``_servers`` — the most likely state for
  214. # stale backoff entries), no connect-cooldown state may survive shutdown.
  215. with _core._lock:
  216. if not servers_snapshot:
  217. clear_selected_status()
  218. _clear_connect_cooldowns(None if wildcard else selected_status)
  219. _loop._stop_mcp_loop(only_if_idle=not wildcard)
  220. # A removed subset still shares its profile's log with the remaining servers.
  221. # Full/profile shutdown must also release handles left by completed CLI/UI probes.
  222. if names is None:
  223. from tools.mcp_tool_config import _close_mcp_stderr_logs
  224. _close_mcp_stderr_logs(scope=scope)
  225. def _take_reapable_pids(include_active: bool, server_name: Optional[str]) -> tuple[dict[int, str], dict[int, int], dict[int, int]]:
  226. """Pop the PIDs to reap (and their spawn-time pgids) out of the ledgers under the lock, so
  227. a future spawn can't collide with stale state. Returns ``(pid -> owner, pid -> pgid)``."""
  228. def _owned(entries: dict[int, str]) -> dict[int, str]:
  229. return {pid: owner for pid, owner in entries.items() if server_name is None or owner == server_name}
  230. with _core._lock:
  231. pids = _owned({opid: _orphan_stdio_pid_servers.get(opid, "orphan") for opid in _orphan_stdio_pids})
  232. _orphan_stdio_pids.difference_update(pids)
  233. for opid in pids:
  234. _orphan_stdio_pid_servers.pop(opid, None)
  235. if include_active:
  236. active = _owned(_stdio_pids)
  237. pids.update(active)
  238. for pid in active:
  239. _stdio_pids.pop(pid, None)
  240. pgids = {pid: _stdio_pgids.pop(pid) for pid in pids if pid in _stdio_pgids}
  241. starts = {pid: _stdio_starttimes.pop(pid) for pid in pids if pid in _stdio_starttimes}
  242. return pids, pgids, starts
  243. def _signal_mcp_process(pid: int, sig: int, server_name: str, pgid: Optional[int], my_pgid: Optional[int],
  244. expected_start: Optional[int] = None) -> None:
  245. """SIGTERM/SIGKILL via the spawn-time pgroup on POSIX (reaches reparented grandchildren),
  246. falling back to a per-pid signal.
  247. PID-reuse guard (#43044): only signal if ``pid`` still names the process we spawned. Once
  248. an MCP child exits and is reaped the kernel may recycle its PID/PGID onto an unrelated
  249. process group; signalling the stale number would kill a stranger (observed: a recycled
  250. PGID landing on a desktop browser's session leader). When ``expected_start`` was captured
  251. at spawn and no longer matches — compared drift-tolerantly, because same-host readings
  252. drift ~1 s on macOS (#117505) and exact equality skipped live, legitimately-owned servers
  253. — skip entirely. Without a baseline (the capture raced the child's exit), or when the
  254. current reading is unreadable (leader reaped: POSIX never reuses a PGID while a member
  255. lives, so its reparented grandchildren are still ours), fall through to the legacy
  256. best-effort path."""
  257. if expected_start is not None:
  258. current = _leader_start_time(pid)
  259. if current is not None:
  260. try:
  261. from gateway.status import start_time_fingerprints_match
  262. if not start_time_fingerprints_match(expected_start, current):
  263. logger.debug(
  264. "Skip signalling MCP pid %d (%s): start-time mismatch — PID was recycled; "
  265. "refusing to kill an unrelated process group.", pid, server_name)
  266. return
  267. except (TypeError, ValueError):
  268. pass # junk fingerprints: best-effort, never break signalling
  269. killpg = getattr(os, "killpg", None)
  270. if pgid is not None and killpg is not None:
  271. if my_pgid is not None and pgid == my_pgid:
  272. # Child shares the gateway's pgroup: killpg would kill the gateway too, so use
  273. # per-pid kill. Warn because per-pid kill can't reach grandchildren in this group.
  274. logger.warning("MCP server '%s' pgid %d matches gateway pgid; skipping "
  275. # Fall through to the per-pid kill() path instead. Warn because per-pid kill
  276. # cannot reach grandchildren in this shared group — if the direct child has
  277. # already exited, they may leak (inherent: group-killing them would also kill the
  278. # gateway). See #47134.
  279. "killpg to avoid self-kill and using per-pid kill — any "
  280. "grandchildren in this group may not be reaped", server_name, pgid)
  281. else:
  282. try:
  283. killpg(pgid, sig)
  284. return
  285. except (ProcessLookupError, PermissionError, OSError) as exc:
  286. # Pgroup gone or refused — still try the direct child.
  287. logger.debug("killpg(%d, %d) failed for MCP server '%s': %s; falling back to kill(pid)",
  288. pgid, sig, server_name, exc)
  289. try:
  290. os.kill(pid, sig)
  291. except (ProcessLookupError, PermissionError, OSError):
  292. pass
  293. if os.name == "nt": # Windows has no pgid reaching reparented grandchildren — kill the tree
  294. _kill_windows_process_tree(pid, sig)
  295. def _kill_windows_process_tree(pid: int, sig: int) -> None:
  296. """Windows counterpart of the POSIX killpg path (#61059): after the direct child is signalled,
  297. terminate every still-alive descendant (npx.cmd → node.exe) so graceful teardown cannot leave
  298. orphans reparented with ParentId=null. Best-effort, per-descendant; never raises."""
  299. import signal as _signal
  300. try:
  301. import psutil
  302. except ImportError:
  303. return
  304. try:
  305. parent = psutil.Process(pid)
  306. descendants = parent.children(recursive=True)
  307. except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
  308. return
  309. for child in descendants:
  310. try:
  311. child.terminate()
  312. except Exception:
  313. pass
  314. if sig == getattr(_signal, "SIGKILL", _signal.SIGTERM): # force pass: don't wait for graceful exit
  315. _, alive = psutil.wait_procs(descendants, timeout=0)
  316. for child in alive:
  317. try:
  318. child.kill()
  319. except Exception:
  320. pass
  321. def _group_alive(pgid: Optional[int], my_pgid: Optional[int]) -> bool:
  322. """A reaped leader's descendants that ignored SIGTERM keep its group alive, so the
  323. SIGKILL pass must probe the group, not only the leader PID."""
  324. if pgid is None or pgid == my_pgid or not hasattr(os, "killpg"):
  325. return False
  326. try:
  327. os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only, guarded by hasattr
  328. return True
  329. except OSError:
  330. return False
  331. def _kill_orphaned_mcp_children(include_active: bool = False, server_name: Optional[str] = None) -> None:
  332. """Best-effort reap of stdio MCP subprocesses: SIGTERM, wait 2s, SIGKILL survivors. By
  333. default only ``_orphan_stdio_pids`` are reaped so concurrent cron jobs / live sessions are
  334. untouched; ``include_active=True`` also kills every ``_stdio_pids`` entry and is only for
  335. final shutdown after the MCP loop has stopped. ``server_name`` limits the sweep to one
  336. server (stdio reconnects cleaning up their old transport)."""
  337. import signal as _signal
  338. pids, pgids, starts = _take_reapable_pids(include_active, server_name)
  339. if not pids: # skip the 2s sleep every MCP-free shutdown would otherwise pay
  340. return
  341. try: # our own pgid, so we never killpg() the gateway itself
  342. my_pgid = os.getpgrp()
  343. except (AttributeError, OSError):
  344. my_pgid = None # Windows or restricted environment
  345. for pid, owner in pids.items():
  346. _signal_mcp_process(pid, _signal.SIGTERM, owner, pgids.get(pid), my_pgid, starts.get(pid))
  347. logger.debug("Sent SIGTERM to orphaned MCP process %d (%s)", pid, owner)
  348. time.sleep(2)
  349. sigkill = getattr(_signal, "SIGKILL", _signal.SIGTERM)
  350. from gateway.status import _pid_exists # ``os.kill(pid, 0)`` is NOT a no-op on Windows
  351. for pid, owner in pids.items():
  352. if _pid_exists(pid) or _group_alive(pgids.get(pid), my_pgid): # leader or descendants survived SIGTERM
  353. _signal_mcp_process(pid, sigkill, owner, pgids.get(pid), my_pgid, starts.get(pid))
  354. logger.warning("Force-killed MCP process %d (%s) after SIGTERM timeout", pid, owner)
  355. # These groups are reaped. Release them last, so a crash partway through the SIGTERM/SIGKILL
  356. # dance still leaves the supervisor holding them.
  357. release_process_groups(pgids.values())
  358. def _stop_mcp_loop_if_idle() -> bool:
  359. """Stop the MCP loop only when no registered server still owns it. Probe paths create
  360. temporary MCPServerTasks not placed in ``_servers``; they may clean up an idle loop but
  361. must not tear down the process-global loop under live agent tools."""
  362. return _loop._stop_mcp_loop(only_if_idle=True)
  363. async def _drain_mcp_loop_tasks(*, timeout: Optional[float] = None) -> None:
  364. """Cancel every task still pending on the MCP loop and reap it. ``Task.cancel()`` only
  365. schedules the throw, so tasks need a cancellation cycle before the loop goes away; wait
  366. for them here, on their owning loop, bounded so a task that suppresses cancellation
  367. cannot hang process exit."""
  368. if timeout is None:
  369. timeout = _core._MCP_LOOP_DRAIN_TIMEOUT
  370. current = asyncio.current_task()
  371. pending = [t for t in asyncio.all_tasks() if t is not current and not t.done()]
  372. if not pending:
  373. return
  374. logger.debug("Draining %d pending task(s) from the MCP loop", len(pending))
  375. for task in pending:
  376. task.cancel()
  377. done, still_pending = await asyncio.wait(pending, timeout=timeout)
  378. for task in done:
  379. if not task.cancelled():
  380. task.exception() # mark retrieved so asyncio doesn't warn "exception was never retrieved"
  381. if still_pending:
  382. logger.warning("%d MCP loop task(s) still pending after %.1fs drain", len(still_pending), timeout)
  383. async def _drain_and_stop_mcp_loop() -> None:
  384. """Drain pending tasks, then stop the loop from its owning thread. Both must run as one
  385. loop-owned sequence: a ``loop.stop`` queued separately by a timed-out caller can overtake
  386. the scheduled drain, leaving the drain coroutine itself pending when the loop is closed."""
  387. loop = asyncio.get_running_loop()
  388. try:
  389. await _drain_mcp_loop_tasks(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT)
  390. finally:
  391. loop.call_soon(loop.stop)