process_registry_termination.py 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. """Process-tree termination and post-kill survivor verification for ProcessRegistry."""
  2. import logging
  3. import os
  4. import signal
  5. import subprocess
  6. import time
  7. from contextlib import suppress
  8. from typing import TYPE_CHECKING, List, Optional
  9. from hermes_cli._subprocess_compat import windows_hide_flags
  10. if TYPE_CHECKING:
  11. from tools.process_registry import ProcessSession
  12. logger = logging.getLogger("tools.process_registry")
  13. class ProcessTerminationMixin:
  14. """The subclass supplies ``_host_pid_is_ours``, ``_is_host_pid_alive``,
  15. ``_detached_host_fate`` and ``_daemon_term_grace_seconds``. See ProcessRegistry."""
  16. @staticmethod
  17. def _proc_alive(proc) -> bool:
  18. """True if a psutil.Process is running and not a zombie (already dead, just unreaped)."""
  19. try:
  20. import psutil
  21. return proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE
  22. except Exception:
  23. return False
  24. @classmethod
  25. def _terminate_host_pid(cls, pid: int, expected_start: Optional[int] = None) -> None:
  26. """Terminate a host-visible PID and its descendants.
  27. ``expected_start`` (kernel start time at spawn) is re-validated first: a mismatch
  28. or dead PID means the number was recycled onto a stranger and we refuse to touch
  29. it — a leaked orphan beats tree-killing someone's browser. POSIX: snapshot descendants,
  30. SIGTERM the parent alone so it can perform an orderly shutdown, then clean up snapshot
  31. descendants that survive its grace window. Survivors are SIGKILLed after a second
  32. ``terminal.daemon_term_grace_seconds`` window. Windows:
  33. ``taskkill /T /F`` (psutil's stale PPID links miss orphans there); ``os.kill``
  34. is the fallback."""
  35. from tools.process_registry import _IS_WINDOWS
  36. if expected_start is not None and not cls._host_pid_is_ours(pid, expected_start):
  37. logger.warning(
  38. "Refusing to terminate host pid %d: start-time mismatch — "
  39. "PID was recycled onto an unrelated process.", pid)
  40. return
  41. def _sigterm_quietly():
  42. with suppress(OSError, ProcessLookupError, PermissionError):
  43. os.kill(pid, signal.SIGTERM)
  44. if _IS_WINDOWS:
  45. try:
  46. subprocess.run(
  47. ["taskkill", "/PID", str(pid), "/T", "/F"], capture_output=True, text=True,
  48. encoding='utf-8', errors='replace', timeout=10, creationflags=windows_hide_flags(),
  49. stdin=subprocess.DEVNULL, check=False)
  50. except (FileNotFoundError, subprocess.TimeoutExpired, OSError):
  51. _sigterm_quietly()
  52. return
  53. import psutil
  54. gone = (psutil.NoSuchProcess, psutil.AccessDenied, OSError)
  55. try:
  56. parent = psutil.Process(pid)
  57. except psutil.NoSuchProcess:
  58. return
  59. except (OSError, PermissionError):
  60. _sigterm_quietly()
  61. return
  62. # Snapshot before signalling: once the parent exits, psutil can no longer
  63. # reliably find children that it failed to reap.
  64. try:
  65. descendants = parent.children(recursive=True)
  66. except gone:
  67. descendants = []
  68. # Let self-managing parents (notably Chromium/Electron) shut down their
  69. # tree before touching children. Killing their zygotes first can turn a
  70. # graceful browser shutdown into a crash dump.
  71. with suppress(gone):
  72. parent.terminate()
  73. grace = cls._daemon_term_grace_seconds()
  74. def _wait_for_exit(targets) -> None:
  75. if grace <= 0:
  76. return
  77. deadline = time.monotonic() + grace
  78. while time.monotonic() < deadline and any(cls._proc_alive(p) for p in targets):
  79. time.sleep(0.05)
  80. # Preserve descendants during the parent's configured shutdown window.
  81. _wait_for_exit([parent])
  82. # The snapshot is an anti-orphan guarantee: only descendants still alive
  83. # after the parent had its chance are asked to terminate themselves.
  84. remaining = descendants if grace <= 0 else [
  85. proc for proc in descendants if cls._proc_alive(proc)
  86. ]
  87. for proc in remaining:
  88. with suppress(gone):
  89. proc.terminate()
  90. # Preserve the existing SIGKILL escalation semantics for every owned
  91. # process that remains after its SIGTERM grace window. The parent is
  92. # included in case it ignored the first signal.
  93. targets = [parent, *remaining]
  94. # Escalate to SIGKILL for anything that ignored SIGTERM within the grace window.
  95. # ``psutil.wait_procs``' gone/alive partition is deliberately NOT trusted: it
  96. # reaps via ``Process.wait()`` and mis-partitions across zombie transitions in a
  97. # parent/child tree, leaving survivors un-killed. Re-probing every target is
  98. # deterministic.
  99. if grace <= 0:
  100. return
  101. _wait_for_exit(targets)
  102. # A parent that ignored SIGTERM (the interactive ``bash -lic`` wrapper does) keeps
  103. # running its script through both grace windows and can spawn children the first
  104. # snapshot never saw. Re-snapshot while it is still alive: once it is SIGKILLed
  105. # they reparent to init and nothing can find them again.
  106. with suppress(gone):
  107. if cls._proc_alive(parent):
  108. known = {proc.pid for proc in targets}
  109. targets.extend(p for p in parent.children(recursive=True) if p.pid not in known)
  110. for proc in targets:
  111. with suppress(gone):
  112. if cls._proc_alive(proc):
  113. proc.kill() # SIGKILL on POSIX
  114. logger.info("Escalated to SIGKILL for pid %d (ignored SIGTERM within %.1fs grace)", proc.pid, grace)
  115. @staticmethod
  116. def _live_descendants(pid: int) -> list[int]:
  117. """PIDs of living non-zombie descendants of host PID ``pid`` (best-effort)."""
  118. try:
  119. import psutil
  120. children = psutil.Process(pid).children(recursive=True)
  121. except Exception:
  122. logger.debug("Could not list descendants of pid %s", pid, exc_info=True)
  123. return []
  124. return [c.pid for c in children if ProcessTerminationMixin._proc_alive(c)]
  125. # SIGKILL / taskkill are asynchronous: the kernel needs a scheduling tick to
  126. # tear the process down and the parent must reap it before poll()/isalive()
  127. # stop saying "alive". Verifying survivors in that window flagged every
  128. # escalated kill as incomplete.
  129. _KILL_SETTLE_SECONDS = 1.0
  130. def _post_kill_survivors(self, session: "ProcessSession") -> list[int]:
  131. """Host PIDs still alive once the kill signals have had time to land (#115490).
  132. Fail-closed: anything unverifiable counts as a survivor, so a kill
  133. that leaves a live tree can never write a killed receipt. Sandbox
  134. (env) sessions have no host-visible tree and are unverifiable by
  135. design — they return no survivors, preserving existing behavior."""
  136. deadline = time.monotonic() + self._KILL_SETTLE_SECONDS
  137. while True:
  138. survivors = self._probe_survivors(session)
  139. if not survivors or time.monotonic() >= deadline:
  140. return survivors
  141. time.sleep(0.05)
  142. def _probe_survivors(self, session: "ProcessSession") -> list[int]:
  143. survivors: list[int] = []
  144. proc = getattr(session, "process", None)
  145. if proc is not None:
  146. try:
  147. root_alive = proc.poll() is None
  148. except Exception:
  149. root_alive = True
  150. if root_alive:
  151. survivors.append(getattr(proc, "pid", None) or session.pid)
  152. pty = getattr(session, "_pty", None)
  153. if pty is not None:
  154. try:
  155. pty_alive = bool(pty.isalive())
  156. except Exception:
  157. pty_alive = self._is_host_pid_alive(session.pid)
  158. if pty_alive:
  159. survivors.append(session.pid)
  160. if session.pid_scope == "host" and session.pid:
  161. if self._detached_host_fate(session.pid, session.host_start_time) == "running":
  162. if session.pid not in survivors:
  163. survivors.append(session.pid)
  164. survivors.extend(
  165. pid for pid in self._live_descendants(session.pid)
  166. if pid not in survivors)
  167. # A dead/recycled root has no PID-scope descendants left to find:
  168. # reparented orphans are outside PID scope (systemd scope stop,
  169. # issued before this check, covers the cgroup case).
  170. return [pid for pid in survivors if pid]