| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432 |
- """MCP process lifecycle: stdio child PID tracking and orphan cleanup, graceful
- server shutdown and draining of the background MCP loop."""
- import logging
- import asyncio
- import os
- import time
- from typing import Dict, Optional
- from hermes_cli.process_identity import is_owner_death_supervisor, release_process_groups
- from tools.mcp_tool_common import _core
- from tools import mcp_tool_loop as _loop
- logger = logging.getLogger("tools.mcp_tool")
- # Live stdio MCP children (pid -> server_name), added after connection and removed on normal
- # shutdown, so they can be force-killed if SDK teardown fails.
- _stdio_pids: dict[int, str] = {}
- # PIDs that survived their session context exit (detected in _run_stdio's finally, reaped by
- # _kill_orphaned_mcp_children). Separate from _stdio_pids so sweeps never race active sessions.
- _orphan_stdio_pids: set = set()
- _orphan_stdio_pid_servers: dict[int, str] = {}
- # pid -> pgid captured at spawn. The SDK spawns with start_new_session=True (PGID == PID);
- # grandchildren keep that PGID after the direct child exits, so killpg still reaches them.
- # Separate from _stdio_pids so the PGID survives the child's removal. Empty on Windows.
- _stdio_pgids: dict[int, int] = {}
- # Spawn-time start-time fingerprints of each stdio child's pgroup leader, captured
- # alongside the PGID (the psutil fallback means every platform has a baseline, macOS
- # included). PIDs/PGIDs are recycled by the kernel once the original process exits and
- # is reaped, so a long-lived tracker holding a bare PGID is unsafe: by the time a sweep
- # runs, that number may name an unrelated process group (observed in the wild: a
- # desktop browser whose session leader happened to reuse a dead MCP child's PID —
- # #43044). We re-check the leader's start time — drift-tolerantly, since same-host
- # readings drift ~1 s on macOS (#117505) — before signalling so a recycled PGID is
- # never killed. None entries are dropped: a capture that raced the child's exit keeps
- # the legacy best-effort behaviour.
- _stdio_starttimes: dict[int, int] = {} # pid -> leader start ticks
- def _leader_start_time(pid: int) -> Optional[int]:
- """Start-time fingerprint of the pgroup leader (PGID == leader PID on setsid spawn);
- ``None`` only when the reading is genuinely unavailable (already-reaped PID, no
- /proc AND no psutil) — the psutil fallback covers macOS/Windows."""
- from gateway.status import get_process_start_time
- try:
- return get_process_start_time(pid)
- except Exception:
- return None
- def _snapshot_child_pids() -> set:
- """Current direct-child PIDs: /proc on Linux, else psutil, else empty set."""
- my_pid = os.getpid()
- # /proc/<pid>/task/<tid>/children is per-THREAD, and stdio_client() spawns from the MCP
- # loop thread, so union every task's children — reading only the main thread's file
- # returns an empty set on every Linux install.
- try:
- # ``/proc/<pid>/task/<tid>/children`` is per-THREAD — a child forked from thread T is listed only
- # under T's task dir. stdio_client() spawns from the background MCP loop thread, so reading only the
- # main thread's file (``task/<pid>/children``) returned an empty set on every Linux install and left
- # ``_stdio_child_pids`` / ``_stdio_pids`` empty: the #81995 dead-child fast-fail, the #96452 respawn
- # signal, and the killpg shutdown sweep never saw the subprocess.
- task_dir = f"/proc/{my_pid}/task"
- found: set = set()
- for tid in os.listdir(task_dir):
- try:
- with open(f"{task_dir}/{tid}/children", encoding="utf-8-sig") as f:
- found.update(int(p) for p in f.read().split() if p.strip())
- except (FileNotFoundError, OSError, ValueError):
- continue # thread exited between listdir and open
- return found
- except (FileNotFoundError, OSError, ValueError):
- pass
- try:
- import psutil
- return {c.pid for c in psutil.Process(my_pid).children()}
- except Exception:
- return set()
- # argv markers of non-MCP gateway children that can race into the snapshot delta during an
- # MCP spawn (defense-in-depth: LSP servers and the slash worker lead their own groups under the
- # owner-death supervisor, see spawn_owned). Matched against argv[1:] because Python/Java children
- # start with the interpreter path.
- _NON_MCP_CHILD_CMDLINE_MARKERS: tuple[str, ...] = (
- "tui_gateway.slash_worker", "tui_gateway.entry",
- "-dorg.eclipse.equinox.launcher", "eclipse.jdt.ls", "org.eclipse.equinox.launcher_", # jdtls
- )
- def _filter_mcp_children(pids: set) -> set:
- """Drop non-MCP children from a PID snapshot delta. Tracking a stray child in _stdio_pgids
- is catastrophic if it lacks start_new_session: its pgid can be the TUI parent's, so the
- shutdown killpg() would kill the TUI itself."""
- if not pids:
- return pids
- try:
- import psutil
- except ImportError:
- return pids # keep all PIDs (prior behavior)
- kept = set()
- for pid in pids:
- try:
- argv = psutil.Process(pid).cmdline()
- except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
- continue # raced away or zombie — cannot be our fresh server, unsafe to track
- if is_owner_death_supervisor(argv):
- continue # the shared reaper of every owned child; MCP teardown must never kill it
- if not any(marker in arg for arg in argv[1:] for marker in _NON_MCP_CHILD_CMDLINE_MARKERS):
- kept.add(pid)
- return kept
- def _clear_connect_cooldowns(keys=None) -> None:
- """Drop connect-retry cooldowns: a restart must re-attempt every server immediately, not
- honour a stale per-server backoff. Caller holds ``_core._lock``."""
- if keys is None:
- _core._server_connect_retry_after.clear()
- _core._server_connect_failures.clear()
- else:
- for key in keys:
- _core._server_connect_retry_after.pop(key, None)
- _core._server_connect_failures.pop(key, None)
- def _reregister_orphaned_adopters() -> None:
- """Re-run MCP registration for profiles whose ADOPTED shared connection an owner's
- ``/reload-mcp`` just tore down. Their tools vanished with the owner's teardown and nothing
- re-runs their discovery until THEY reload, so they sat tool-less behind a healthy-looking
- status (#106005). Runs after the owner's rediscovery, under each adopter's own home + secret
- scope (its ``${VAR}`` refs must resolve to ITS credentials): the adopter re-adopts the owner's
- new identical connection or connects its own."""
- with _core._lock:
- pending = dict(_core._orphaned_adopters)
- _core._orphaned_adopters.clear()
- if not pending:
- return
- from pathlib import Path
- from agent.secret_scope import build_profile_secret_scope, reset_secret_scope, set_secret_scope
- from hermes_constants import reset_hermes_home_override, set_hermes_home_override
- from tools import mcp_tool_discovery as _discovery
- from tools.mcp_tool_config import _load_mcp_config
- for adopter, names in pending.items():
- home_token = secret_token = None
- try:
- home_token = set_hermes_home_override(adopter)
- secret_token = set_secret_scope(build_profile_secret_scope(Path(adopter)), profile_home=adopter)
- servers = {n: c for n, c in (_load_mcp_config() or {}).items() if n in names}
- if servers:
- _discovery.register_mcp_servers(servers)
- except Exception:
- logger.debug("MCP: re-registration for profile scope %s failed", adopter, exc_info=True)
- finally:
- if secret_token is not None:
- reset_secret_scope(secret_token)
- if home_token is not None:
- reset_hermes_home_override(home_token)
- def shutdown_mcp_servers(*, scope: Optional[str] = None, names: Optional[set] = None,
- timeout: float = 15.0):
- """Close MCP server connections (in parallel) and stop the background loop. Each server
- Task is signalled to exit its own ``async with`` so the anyio cancel-scope cleanup runs in
- the Task that opened it. ``scope`` restricts teardown to one multiplexed profile's servers
- (its ``/reload-mcp`` must not kill other profiles') and leaves the shared loop running if
- anything else is still connected. ``names`` restricts it further to those server names
- (dropped-from-config pruning); other servers' bookkeeping is untouched. Only the bare call
- (no ``scope``, no ``names``) is the process-wide wildcard: the launch profile's registry
- scope IS ``None``, so ``scope=None, names={...}`` prunes that unscoped owner's servers and
- must leave a served profile's same-named ``(B, name)`` connection alone. ``timeout`` bounds
- the wait for the close to land on the MCP loop — a caller running one pass per served
- profile under a total budget divides it, or N profiles × 15s starve the wildcard pass that
- actually stops the loop."""
- from tools.mcp_tool_scope import _key_name
- wildcard = scope is None and names is None
- with _core._lock:
- selected = [key for key in _core._servers if wildcard or _core._server_scope_keys.get(key) == scope]
- if names is not None:
- selected = [key for key in selected if _key_name(key) in names]
- servers_snapshot = [_core._servers[key] for key in selected]
- if names is not None:
- selected_status = set(selected)
- elif wildcard:
- selected_status = (
- set(_core._servers) | set(_core._server_scope_keys)
- | set(_core._server_tool_scopes)
- | set(_core._server_connecting) | set(_core._server_connect_errors))
- else:
- selected_status = {key for key, owner in _core._server_scope_keys.items() if owner == scope}
- # Adopters of the connections being torn down lose their overlays with the tasks' own
- # ``_deregister_tools``; remember them so the next discovery pass re-registers them
- # (``_reregister_orphaned_adopters``).
- if not wildcard:
- for key in selected:
- for adopter in _core._server_tool_scopes.get(key, ()):
- if adopter != scope:
- _core._orphaned_adopters.setdefault(adopter, set()).add(_key_name(key))
- def clear_selected_status():
- _core._server_connecting.difference_update(selected_status)
- for key in selected_status:
- _core._server_connect_errors.pop(key, None)
- _core._server_scope_keys.pop(key, None)
- _core._server_tool_scopes.pop(key, None)
- # Fast path: nothing to shut down. The connect-cooldown maps can still be populated here — a server that
- # failed to connect is never recorded in ``_servers`` (that is the very premise of the #50394 cooldown),
- # so "no live servers" is the MOST likely state in which stale backoff entries exist. Clear them so a
- # post-shutdown restart re-attempts every configured server immediately.
- if servers_snapshot:
- async def _shutdown():
- results = await asyncio.gather(*(server.shutdown() for server in servers_snapshot), return_exceptions=True)
- for server, result in zip(servers_snapshot, results):
- if isinstance(result, Exception):
- logger.debug("Error closing MCP server '%s': %s", server.name, result)
- with _core._lock:
- for key in selected:
- _core._servers.pop(key, None)
- _core._server_scope_keys.pop(key, None)
- clear_selected_status()
- _clear_connect_cooldowns(None if wildcard else selected_status)
- with _core._lock:
- loop = _core._mcp_loop
- if loop is not None and loop.is_running():
- from agent.async_utils import safe_schedule_threadsafe
- future = safe_schedule_threadsafe(_shutdown(), loop, logger=logger, log_message="MCP shutdown: failed to schedule")
- if future is not None:
- try:
- future.result(timeout=timeout)
- except BaseException as exc:
- logger.debug("Error during MCP shutdown: %s", exc)
- # Unconditional final sweep: whether ``_shutdown`` ran, timed out, or was never scheduled
- # (a server that failed to connect is never in ``_servers`` — the most likely state for
- # stale backoff entries), no connect-cooldown state may survive shutdown.
- with _core._lock:
- if not servers_snapshot:
- clear_selected_status()
- _clear_connect_cooldowns(None if wildcard else selected_status)
- _loop._stop_mcp_loop(only_if_idle=not wildcard)
- # A removed subset still shares its profile's log with the remaining servers.
- # Full/profile shutdown must also release handles left by completed CLI/UI probes.
- if names is None:
- from tools.mcp_tool_config import _close_mcp_stderr_logs
- _close_mcp_stderr_logs(scope=scope)
- def _take_reapable_pids(include_active: bool, server_name: Optional[str]) -> tuple[dict[int, str], dict[int, int], dict[int, int]]:
- """Pop the PIDs to reap (and their spawn-time pgids) out of the ledgers under the lock, so
- a future spawn can't collide with stale state. Returns ``(pid -> owner, pid -> pgid)``."""
- def _owned(entries: dict[int, str]) -> dict[int, str]:
- return {pid: owner for pid, owner in entries.items() if server_name is None or owner == server_name}
- with _core._lock:
- pids = _owned({opid: _orphan_stdio_pid_servers.get(opid, "orphan") for opid in _orphan_stdio_pids})
- _orphan_stdio_pids.difference_update(pids)
- for opid in pids:
- _orphan_stdio_pid_servers.pop(opid, None)
- if include_active:
- active = _owned(_stdio_pids)
- pids.update(active)
- for pid in active:
- _stdio_pids.pop(pid, None)
- pgids = {pid: _stdio_pgids.pop(pid) for pid in pids if pid in _stdio_pgids}
- starts = {pid: _stdio_starttimes.pop(pid) for pid in pids if pid in _stdio_starttimes}
- return pids, pgids, starts
- def _signal_mcp_process(pid: int, sig: int, server_name: str, pgid: Optional[int], my_pgid: Optional[int],
- expected_start: Optional[int] = None) -> None:
- """SIGTERM/SIGKILL via the spawn-time pgroup on POSIX (reaches reparented grandchildren),
- falling back to a per-pid signal.
- PID-reuse guard (#43044): only signal if ``pid`` still names the process we spawned. Once
- an MCP child exits and is reaped the kernel may recycle its PID/PGID onto an unrelated
- process group; signalling the stale number would kill a stranger (observed: a recycled
- PGID landing on a desktop browser's session leader). When ``expected_start`` was captured
- at spawn and no longer matches — compared drift-tolerantly, because same-host readings
- drift ~1 s on macOS (#117505) and exact equality skipped live, legitimately-owned servers
- — skip entirely. Without a baseline (the capture raced the child's exit), or when the
- current reading is unreadable (leader reaped: POSIX never reuses a PGID while a member
- lives, so its reparented grandchildren are still ours), fall through to the legacy
- best-effort path."""
- if expected_start is not None:
- current = _leader_start_time(pid)
- if current is not None:
- try:
- from gateway.status import start_time_fingerprints_match
- if not start_time_fingerprints_match(expected_start, current):
- logger.debug(
- "Skip signalling MCP pid %d (%s): start-time mismatch — PID was recycled; "
- "refusing to kill an unrelated process group.", pid, server_name)
- return
- except (TypeError, ValueError):
- pass # junk fingerprints: best-effort, never break signalling
- killpg = getattr(os, "killpg", None)
- if pgid is not None and killpg is not None:
- if my_pgid is not None and pgid == my_pgid:
- # Child shares the gateway's pgroup: killpg would kill the gateway too, so use
- # per-pid kill. Warn because per-pid kill can't reach grandchildren in this group.
- logger.warning("MCP server '%s' pgid %d matches gateway pgid; skipping "
- # Fall through to the per-pid kill() path instead. Warn because per-pid kill
- # cannot reach grandchildren in this shared group — if the direct child has
- # already exited, they may leak (inherent: group-killing them would also kill the
- # gateway). See #47134.
- "killpg to avoid self-kill and using per-pid kill — any "
- "grandchildren in this group may not be reaped", server_name, pgid)
- else:
- try:
- killpg(pgid, sig)
- return
- except (ProcessLookupError, PermissionError, OSError) as exc:
- # Pgroup gone or refused — still try the direct child.
- logger.debug("killpg(%d, %d) failed for MCP server '%s': %s; falling back to kill(pid)",
- pgid, sig, server_name, exc)
- try:
- os.kill(pid, sig)
- except (ProcessLookupError, PermissionError, OSError):
- pass
- if os.name == "nt": # Windows has no pgid reaching reparented grandchildren — kill the tree
- _kill_windows_process_tree(pid, sig)
- def _kill_windows_process_tree(pid: int, sig: int) -> None:
- """Windows counterpart of the POSIX killpg path (#61059): after the direct child is signalled,
- terminate every still-alive descendant (npx.cmd → node.exe) so graceful teardown cannot leave
- orphans reparented with ParentId=null. Best-effort, per-descendant; never raises."""
- import signal as _signal
- try:
- import psutil
- except ImportError:
- return
- try:
- parent = psutil.Process(pid)
- descendants = parent.children(recursive=True)
- except (psutil.NoSuchProcess, psutil.AccessDenied, OSError):
- return
- for child in descendants:
- try:
- child.terminate()
- except Exception:
- pass
- if sig == getattr(_signal, "SIGKILL", _signal.SIGTERM): # force pass: don't wait for graceful exit
- _, alive = psutil.wait_procs(descendants, timeout=0)
- for child in alive:
- try:
- child.kill()
- except Exception:
- pass
- def _group_alive(pgid: Optional[int], my_pgid: Optional[int]) -> bool:
- """A reaped leader's descendants that ignored SIGTERM keep its group alive, so the
- SIGKILL pass must probe the group, not only the leader PID."""
- if pgid is None or pgid == my_pgid or not hasattr(os, "killpg"):
- return False
- try:
- os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only, guarded by hasattr
- return True
- except OSError:
- return False
- def _kill_orphaned_mcp_children(include_active: bool = False, server_name: Optional[str] = None) -> None:
- """Best-effort reap of stdio MCP subprocesses: SIGTERM, wait 2s, SIGKILL survivors. By
- default only ``_orphan_stdio_pids`` are reaped so concurrent cron jobs / live sessions are
- untouched; ``include_active=True`` also kills every ``_stdio_pids`` entry and is only for
- final shutdown after the MCP loop has stopped. ``server_name`` limits the sweep to one
- server (stdio reconnects cleaning up their old transport)."""
- import signal as _signal
- pids, pgids, starts = _take_reapable_pids(include_active, server_name)
- if not pids: # skip the 2s sleep every MCP-free shutdown would otherwise pay
- return
- try: # our own pgid, so we never killpg() the gateway itself
- my_pgid = os.getpgrp()
- except (AttributeError, OSError):
- my_pgid = None # Windows or restricted environment
- for pid, owner in pids.items():
- _signal_mcp_process(pid, _signal.SIGTERM, owner, pgids.get(pid), my_pgid, starts.get(pid))
- logger.debug("Sent SIGTERM to orphaned MCP process %d (%s)", pid, owner)
- time.sleep(2)
- sigkill = getattr(_signal, "SIGKILL", _signal.SIGTERM)
- from gateway.status import _pid_exists # ``os.kill(pid, 0)`` is NOT a no-op on Windows
- for pid, owner in pids.items():
- if _pid_exists(pid) or _group_alive(pgids.get(pid), my_pgid): # leader or descendants survived SIGTERM
- _signal_mcp_process(pid, sigkill, owner, pgids.get(pid), my_pgid, starts.get(pid))
- logger.warning("Force-killed MCP process %d (%s) after SIGTERM timeout", pid, owner)
- # These groups are reaped. Release them last, so a crash partway through the SIGTERM/SIGKILL
- # dance still leaves the supervisor holding them.
- release_process_groups(pgids.values())
- def _stop_mcp_loop_if_idle() -> bool:
- """Stop the MCP loop only when no registered server still owns it. Probe paths create
- temporary MCPServerTasks not placed in ``_servers``; they may clean up an idle loop but
- must not tear down the process-global loop under live agent tools."""
- return _loop._stop_mcp_loop(only_if_idle=True)
- async def _drain_mcp_loop_tasks(*, timeout: Optional[float] = None) -> None:
- """Cancel every task still pending on the MCP loop and reap it. ``Task.cancel()`` only
- schedules the throw, so tasks need a cancellation cycle before the loop goes away; wait
- for them here, on their owning loop, bounded so a task that suppresses cancellation
- cannot hang process exit."""
- if timeout is None:
- timeout = _core._MCP_LOOP_DRAIN_TIMEOUT
- current = asyncio.current_task()
- pending = [t for t in asyncio.all_tasks() if t is not current and not t.done()]
- if not pending:
- return
- logger.debug("Draining %d pending task(s) from the MCP loop", len(pending))
- for task in pending:
- task.cancel()
- done, still_pending = await asyncio.wait(pending, timeout=timeout)
- for task in done:
- if not task.cancelled():
- task.exception() # mark retrieved so asyncio doesn't warn "exception was never retrieved"
- if still_pending:
- logger.warning("%d MCP loop task(s) still pending after %.1fs drain", len(still_pending), timeout)
- async def _drain_and_stop_mcp_loop() -> None:
- """Drain pending tasks, then stop the loop from its owning thread. Both must run as one
- loop-owned sequence: a ``loop.stop`` queued separately by a timed-out caller can overtake
- the scheduled drain, leaving the drain coroutine itself pending when the loop is closed."""
- loop = asyncio.get_running_loop()
- try:
- await _drain_mcp_loop_tasks(timeout=_core._MCP_LOOP_DRAIN_TIMEOUT)
- finally:
- loop.call_soon(loop.stop)
|