pty_bridge.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432
  1. """PTY bridge for `hermes dashboard` chat tab.
  2. Wraps a child process behind a pseudo-terminal so its ANSI output can be streamed to xterm.js and
  3. keystrokes fed back in; the only caller is the ``/api/pty`` WebSocket endpoint in
  4. ``hermes_cli.web_server``. POSIX-only: depends on ``fcntl``, ``termios`` and ``ptyprocess`` (native
  5. Windows would need a separate ConPTY/``pywinpty`` implementation).
  6. """
  7. from __future__ import annotations
  8. import asyncio
  9. import errno
  10. import fcntl # windows-footgun: ok — POSIX-only module by design (see docstring)
  11. import math
  12. import os
  13. import signal
  14. import struct
  15. import sys
  16. import termios # windows-footgun: ok — POSIX-only module by design (see docstring)
  17. import threading
  18. import time
  19. from typing import Callable, Optional, Sequence
  20. try:
  21. import ptyprocess # type: ignore
  22. _PTY_AVAILABLE = not sys.platform.startswith("win")
  23. except ImportError: # pragma: no cover - dev env without ptyprocess
  24. ptyprocess = None # type: ignore
  25. _PTY_AVAILABLE = False
  26. __all__ = ["PTY_HOST_DASHBOARD", "PTY_HOST_ENV", "PtyBridge", "PtyUnavailableError"]
  27. # Set on the spawned TUI so Ink knows which emulator is hosting it. Mirrored in
  28. # ui-tui/packages/hermes-ink/src/ink/termio/host.ts — keep the two in sync.
  29. PTY_HOST_ENV = "HERMES_PTY_HOST"
  30. PTY_HOST_DASHBOARD = "dashboard"
  31. # ``struct winsize`` packs rows/cols as unsigned short; we clamp well below that ceiling because a
  32. # value above it is a broken probe (WSL2 reports columns=131072), not a genuine ultrawide. Lower
  33. # bound is 1 — a zero/negative dimension is the classic "no size yet" signal.
  34. _MIN_DIMENSION = 1
  35. _MAX_COLS = 2000
  36. _MAX_ROWS = 1000
  37. def _clamp_dimension(value: int, maximum: int) -> int:
  38. """Clamp into ``[_MIN_DIMENSION, maximum]``; non-integer / non-finite values fall back to
  39. ``_MIN_DIMENSION`` so a bad probe can never reach ``struct.pack``.
  40. """
  41. try:
  42. n = int(value)
  43. except (TypeError, ValueError, OverflowError):
  44. return _MIN_DIMENSION
  45. return max(_MIN_DIMENSION, min(n, maximum))
  46. class PtyUnavailableError(RuntimeError):
  47. """PTY cannot be created here (native Windows, or ``ptyprocess`` missing); the dashboard
  48. surfaces the message as a chat-tab banner.
  49. """
  50. # Longer than the TUI gateway's own SIGHUP shutdown grace, so a helper that is saving state on
  51. # SIGHUP finishes before it is SIGKILLed.
  52. _HELPER_SHUTDOWN_GRACE_S = 1.5
  53. # Upper bound: PTY_REGISTRY.close_all() runs inside the backend's SIGTERM -> SIGKILL budget
  54. # (dashboard_procs._POSIX_TERM_GRACE_SECONDS), so a raised gateway grace cannot stretch it.
  55. _MAX_HELPER_SHUTDOWN_GRACE_S = 2.0
  56. def _helper_shutdown_grace() -> float:
  57. # The gateway's grace is 1.0 s unless HERMES_TUI_GATEWAY_SHUTDOWN_GRACE_S raises it
  58. # (tui_gateway/entry.py); the PTY child inherits that env. Read it here rather than import
  59. # tui_gateway.entry, which installs signal handlers.
  60. from utils import env_float
  61. gateway = env_float("HERMES_TUI_GATEWAY_SHUTDOWN_GRACE_S", 1.0)
  62. if not math.isfinite(gateway) or gateway <= 0:
  63. gateway = 1.0
  64. return min(max(_HELPER_SHUTDOWN_GRACE_S, gateway + 0.5), _MAX_HELPER_SHUTDOWN_GRACE_S)
  65. def _process_group_exists(pgid: int) -> bool:
  66. try:
  67. os.killpg(pgid, 0) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  68. except OSError:
  69. return False
  70. return True
  71. def _psutil_alive(proc) -> bool:
  72. """Best-effort 'is this psutil Process still alive' for the shutdown cadence;
  73. zombies count as dead and it never raises."""
  74. try:
  75. import psutil # type: ignore
  76. return proc.is_running() and proc.status() != psutil.STATUS_ZOMBIE
  77. except Exception:
  78. return False
  79. class PtyBridge:
  80. """Thin wrapper around ``ptyprocess.PtyProcess`` for byte streaming. Not thread-safe: owned by
  81. the WebSocket handler that spawned it; reads run in an executor thread, writes are awaited on
  82. the loop (or run in a worker under a caller's ``fence``). The master fd is non-blocking so
  83. input backpressure suspends only the owning WebSocket task, never the dashboard event loop.
  84. """
  85. _waiter = None # FdWaiter for the master; None until __init__ registers it
  86. def __init__(self, proc: ptyprocess.PtyProcess): # type: ignore[name-defined]
  87. self._proc = proc
  88. self._fd: int = proc.fd
  89. self._closed = False
  90. # A fenced write runs in a worker thread; it must never reach a descriptor
  91. # number that close() has already released for reuse.
  92. self._fd_lock = threading.Lock()
  93. # Recorded at spawn: once the leader is reaped getpgid() can no longer find its group,
  94. # but the helpers it started still belong to it.
  95. try:
  96. self._pgid: Optional[int] = os.getpgid(proc.pid) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  97. except OSError:
  98. self._pgid = None
  99. os.set_blocking(self._fd, False)
  100. # A selector, never select.select() (ValueError at fd >= FD_SETSIZE, #94928) or poll()
  101. # (macOS poll() reports a PTY master as POLLNVAL, so read() would spin).
  102. from tools.environments.fd_wait import open_fd_waiter
  103. self._waiter = open_fd_waiter(self._fd)
  104. @classmethod
  105. def is_available(cls) -> bool:
  106. """True if a PTY can be spawned on this platform."""
  107. return bool(_PTY_AVAILABLE)
  108. @classmethod
  109. def spawn(
  110. cls, argv: Sequence[str], *, cwd: Optional[str] = None, env: Optional[dict] = None, cols: int = 80, rows: int = 24
  111. ) -> PtyBridge:
  112. """Spawn ``argv`` behind a new PTY and return a bridge."""
  113. if not _PTY_AVAILABLE:
  114. if sys.platform.startswith("win"):
  115. raise PtyUnavailableError("Pseudo-terminals are unavailable on this platform. "
  116. "Hermes Agent supports Windows only via WSL.")
  117. raise PtyUnavailableError("The `ptyprocess` package is missing. " # only other way _PTY_AVAILABLE is False
  118. "Run hermes pm repair, then restart Hermes.")
  119. # env=None: callers own env policy (process_registry already sanitizes), so inherit via the
  120. # factory with exact preservation. Backfill TERM when missing/blank — CI often lacks it and
  121. # probes like `tput cols` then fail before winsize reads; explicit overrides are kept.
  122. from tools.environments.local import build_subprocess_env
  123. spawn_env = build_subprocess_env(scrub_secrets=False, inherit_profile_home=False) if env is None else env.copy()
  124. if not spawn_env.get("TERM"):
  125. spawn_env["TERM"] = "xterm-256color"
  126. # Tell the child TUI it is hosted by the dashboard's xterm.js. Ink uses this to skip the
  127. # focus-in erase+repaint it does for native emulators that coalesce hidden-tab output
  128. # (xterm.js never drops frames, so under the dashboard that repaint was a visible flash on
  129. # every OS app-switch).
  130. spawn_env[PTY_HOST_ENV] = PTY_HOST_DASHBOARD
  131. proc = ptyprocess.PtyProcess.spawn(list(argv), cwd=cwd, env=spawn_env, dimensions=(rows, cols)) # type: ignore[union-attr]
  132. try:
  133. return cls(proc)
  134. except OSError: # no safe readiness wait for the master (fd_wait): don't leak the child
  135. proc.close(force=True)
  136. raise
  137. @property
  138. def pid(self) -> int:
  139. return int(self._proc.pid)
  140. def is_alive(self) -> bool:
  141. try:
  142. return not self._closed and bool(self._proc.isalive())
  143. except Exception:
  144. return False
  145. def read(self, timeout: float = 0.2) -> Optional[bytes]:
  146. """Read up to 64 KiB from the PTY master, blocking at most ``timeout`` seconds.
  147. ``b""`` = nothing yet; ``None`` = EOF / closed (also after :meth:`close`).
  148. """
  149. if self._closed or self._waiter is None:
  150. return None
  151. try:
  152. readable = self._waiter.wait(timeout)
  153. except (OSError, ValueError): # close() raced this read
  154. return None
  155. if not readable:
  156. return b""
  157. try:
  158. data = os.read(self._fd, 65536)
  159. except OSError as exc:
  160. # EIO on Linux = slave side closed. EBADF = already closed.
  161. if exc.errno in {errno.EIO, errno.EBADF}:
  162. return None
  163. # The fd is deliberately non-blocking. Readiness can disappear
  164. # between the readiness check and os.read() when close/output races occur.
  165. if exc.errno in {errno.EAGAIN, errno.EWOULDBLOCK}:
  166. return b""
  167. raise
  168. return data or None
  169. async def _wait_writable(self, timeout: Optional[float]) -> bool:
  170. """Wait without blocking the event loop until the master accepts input (``None``: no limit)."""
  171. if self._closed or (timeout is not None and timeout <= 0):
  172. return False
  173. loop = asyncio.get_running_loop()
  174. ready = loop.create_future()
  175. def _mark_ready() -> None:
  176. if not ready.done():
  177. ready.set_result(None)
  178. try:
  179. loop.add_writer(self._fd, _mark_ready)
  180. # wait_for can swallow cancellation on Python 3.11 if readiness
  181. # completes in the same tick as viewer takeover. Never retry input
  182. # from that cancelled writer after the new viewer owns the PTY.
  183. async with asyncio.timeout(timeout):
  184. await ready
  185. return not self._closed
  186. except (TimeoutError, OSError, ValueError):
  187. return False
  188. finally:
  189. try:
  190. loop.remove_writer(self._fd)
  191. except (OSError, ValueError):
  192. pass
  193. def _write_some(self, view: memoryview) -> Optional[int]:
  194. """One non-blocking write: bytes accepted (0 while the child's buffer is full), None once closed."""
  195. with self._fd_lock:
  196. if self._closed:
  197. return None
  198. try:
  199. return os.write(self._fd, view)
  200. except OSError as exc:
  201. if exc.errno in {errno.EIO, errno.EBADF, errno.EPIPE}:
  202. return None
  203. if exc.errno in {errno.EAGAIN, errno.EWOULDBLOCK}:
  204. return 0
  205. raise
  206. async def write(
  207. self, data: bytes, *, timeout: Optional[float] = 10.0,
  208. fence: Optional[Callable[[Callable[[], Optional[int]]], Optional[int]]] = None,
  209. ) -> bool:
  210. """Write all raw bytes without ever blocking the dashboard event loop.
  211. Returns ``False`` when the bridge closes or the child leaves its input
  212. buffer full for ``timeout`` seconds. Callers can then recycle only the
  213. affected terminal session while the rest of the dashboard stays live.
  214. ``timeout=None`` waits for as long as the child leaves its input unread
  215. (cancel the call to stop waiting), so ``False`` then means the bridge closed.
  216. ``fence`` wraps every non-blocking write attempt in a worker thread, so a
  217. caller can hold a lock around exactly the bytes that reach the child;
  218. whatever it raises propagates. Backpressure waits happen outside it.
  219. """
  220. if self._closed:
  221. return False
  222. if not data:
  223. return True
  224. loop = asyncio.get_running_loop()
  225. deadline = None if timeout is None else loop.time() + max(0.0, timeout)
  226. view = memoryview(data)
  227. while view:
  228. if fence is None:
  229. n = self._write_some(view)
  230. else:
  231. n = await asyncio.to_thread(fence, lambda chunk=view: self._write_some(chunk))
  232. if n is None:
  233. return False
  234. if n > 0:
  235. view = view[n:]
  236. # A very large paste can otherwise monopolize the loop while
  237. # the child drains quickly enough to keep the fd writable.
  238. if view:
  239. await asyncio.sleep(0)
  240. continue
  241. remaining = None if deadline is None else deadline - loop.time()
  242. if not await self._wait_writable(remaining):
  243. return False
  244. return True
  245. def resize(self, cols: int, rows: int) -> None:
  246. """Forward a terminal resize to the child via ``TIOCSWINSZ``.
  247. Clamped first: WSL2 via xterm.js reports garbage like ``columns=131072, rows=1`` and an
  248. unclamped unsigned-short pack raises ``struct.error`` (not ``OSError``), leaving the TUI
  249. laid out for a one-row screen — the blank/disappearing-text symptom.
  250. """
  251. if self._closed:
  252. return
  253. # struct winsize: rows, cols, xpixel, ypixel (all unsigned short)
  254. winsize = struct.pack("HHHH", _clamp_dimension(rows, _MAX_ROWS), _clamp_dimension(cols, _MAX_COLS), 0, 0)
  255. try:
  256. fcntl.ioctl(self._fd, termios.TIOCSWINSZ, winsize)
  257. except OSError:
  258. pass
  259. def size(self) -> Optional[tuple[int, int]]:
  260. """The child's ``(cols, rows)`` (``TIOCGWINSZ``), or None once the PTY is closed."""
  261. if self._closed:
  262. return None
  263. try:
  264. rows, cols, _xpixel, _ypixel = struct.unpack(
  265. "HHHH", fcntl.ioctl(self._fd, termios.TIOCGWINSZ, bytes(8)))
  266. except OSError:
  267. return None
  268. return cols, rows
  269. def _discard_output(self, timeout: float) -> None:
  270. # Exiting children flush their pending terminal output first; on macOS a process stays in
  271. # exit (and a SIGHUP save outlasts the grace) until someone drains the master. The drain
  272. # task no longer consumes the output, so read and drop while waiting.
  273. try:
  274. if self._waiter is not None and self._waiter.wait(timeout) and os.read(self._fd, 65536):
  275. return
  276. except (OSError, ValueError):
  277. pass
  278. time.sleep(timeout) # nothing read, EOF (b"") or EIO: don't spin on a readable dead fd
  279. def _wait_for_group_exit(self, pgid: int, timeout: float) -> None:
  280. deadline = time.monotonic() + timeout
  281. while _process_group_exists(pgid) and time.monotonic() < deadline:
  282. self._discard_output(0.02)
  283. def close(self) -> None:
  284. """Terminate the child's process group (SIGHUP → SIGTERM → SIGKILL, 0.5s grace each while
  285. the leader lives, then up to the helper grace for the rest of the group), reap the leader so
  286. the dashboard never leaks zombies, and close fds. Idempotent; blocks, so call it off the
  287. event loop.
  288. """
  289. with self._fd_lock:
  290. if self._closed:
  291. return
  292. self._closed = True
  293. pgid = self._pgid
  294. leader_was_alive = self._proc.isalive()
  295. # No group signal is safe for a shared-group child, so its descendants must be
  296. # snapshotted BEFORE the parent is signalled: once the parent exits they reparent
  297. # and psutil can no longer find them (process_registry._terminate_host_pid). The sweep below then kills them individually
  298. # so a SIGHUP-ignoring helper cannot keep the PTY slave open (#76759).
  299. non_leader_descendants: list = []
  300. if pgid is not None and pgid != self._proc.pid:
  301. # Not a group leader: the child shares OUR process group, so killpg would
  302. # take the TUI down with it. Signal the child directly instead.
  303. if leader_was_alive:
  304. try:
  305. import psutil # type: ignore
  306. non_leader_descendants = psutil.Process(self._proc.pid).children(recursive=True)
  307. except Exception:
  308. non_leader_descendants = []
  309. pgid = None
  310. # Signal the whole process group, not just the PTY leader: the dashboard TUI starts helper
  311. # children (e.g. the Python slash worker) and killing only the leader strands them.
  312. for sig in (signal.SIGHUP, signal.SIGTERM, signal.SIGKILL): # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  313. if not self._proc.isalive():
  314. break
  315. try:
  316. if pgid is not None:
  317. os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  318. else:
  319. self._proc.kill(sig)
  320. except Exception:
  321. pass
  322. deadline = time.monotonic() + 0.5
  323. while self._proc.isalive() and time.monotonic() < deadline:
  324. self._discard_output(0.02)
  325. # Helpers can outlive the leader and keep the PTY slave open, so no EOF ever arrives (#76759):
  326. # one that ignores SIGHUP, or any helper of a leader that died first (crash, OOM kill). When
  327. # the group already got SIGHUP above, only wait: the TUI gateway saves its sessions on SIGHUP
  328. # within its shutdown grace, and a second SIGHUP or an early SIGKILL would cut that short.
  329. # The group id cannot be reused while any member is alive; if the group emptied after the
  330. # leader was reaped, a pid wrap inside that window could hand it to a new group (killpg
  331. # then hits that group). The reaper closes a dead session within one tick, so the window is short.
  332. if pgid is not None:
  333. grace = _helper_shutdown_grace()
  334. sweep = (signal.SIGKILL,) if leader_was_alive else (signal.SIGHUP, signal.SIGKILL) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  335. if leader_was_alive:
  336. self._wait_for_group_exit(pgid, grace)
  337. for sig in sweep:
  338. try:
  339. os.killpg(pgid, sig) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  340. except OSError:
  341. break # ESRCH: the group is empty
  342. self._wait_for_group_exit(pgid, grace if sig == signal.SIGHUP else 0.5) # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  343. elif non_leader_descendants:
  344. # The shared-group branch: the helpers-outlive-leader sweep above cannot run
  345. # (no killpg is safe), so end the snapshotted descendants individually. The main
  346. # loop above signalled only the child PID, so the descendants got no SIGHUP —
  347. # send it now and allow the helper grace, then SIGKILL survivors: a helper
  348. # saving state on SIGHUP still finishes, and one that ignores SIGHUP cannot
  349. # outlive the close (mirrors the group sweep's leader-dead cadence, #76759).
  350. grace = _helper_shutdown_grace()
  351. for sig in (signal.SIGHUP, signal.SIGKILL): # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  352. for child in non_leader_descendants:
  353. try:
  354. if sig == signal.SIGHUP: # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  355. child.send_signal(sig)
  356. else:
  357. child.kill()
  358. except Exception:
  359. pass # already gone; psutil raises NoSuchProcess/Zombie
  360. if sig == signal.SIGHUP: # windows-footgun: ok — POSIX-only module (imports fcntl/termios/ptyprocess at top)
  361. deadline = time.monotonic() + grace
  362. while any(_psutil_alive(c) for c in non_leader_descendants) and time.monotonic() < deadline:
  363. self._discard_output(0.02)
  364. try:
  365. self._proc.close(force=True)
  366. except Exception:
  367. pass
  368. self._close_waiter()
  369. def _close_waiter(self) -> None:
  370. if self._waiter is not None:
  371. self._waiter.close()
  372. def __enter__(self) -> PtyBridge:
  373. return self
  374. def __exit__(self, *_exc) -> None:
  375. self.close()