pty_session.py 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611
  1. """Keep-alive PTY sessions for dashboard terminals.
  2. A PTY process outlives the WebSocket that created it: a single drain task always reads the PTY into
  3. a bounded RingBuffer and forwards to the attached socket when present. Reconnecting with the same
  4. opaque token replays the buffer and resumes live.
  5. """
  6. from __future__ import annotations
  7. import asyncio
  8. import logging
  9. import threading
  10. import time
  11. from concurrent.futures import Executor
  12. from pathlib import Path
  13. from typing import Callable, Dict, Optional, Tuple
  14. from fastapi import WebSocketDisconnect
  15. logger = logging.getLogger(__name__)
  16. WS_CLOSE_PROCESS_EXITED = 4410
  17. # 4410 also ends a session the server closed (expiry, a deleted profile, shutdown). Only a child
  18. # that exited by itself carries this reason, so the Webapp terminal can close its tab on `exit`.
  19. WS_CLOSE_PROCESS_EXITED_REASON = "Process exited"
  20. WS_CLOSE_SUPERSEDED = 4409
  21. # Going Away: the viewer stopped taking output, but its session lives on and the same
  22. # token reattaches. The dashboard chat tab and the Webapp terminal both redial on 1001;
  23. # the chat tab reads 1011 as a failed start and offers a new session instead.
  24. WS_CLOSE_VIEWER_STALLED = 1001
  25. WS_CLOSE_VIEWER_STALLED_REASON = "Terminal viewer stopped reading; reconnect"
  26. # How long one frame may wait on the viewer before it counts as stalled.
  27. _VIEWER_SEND_TIMEOUT_SECONDS = 5.0
  28. # A reattach replay is a single frame of up to the 1 MiB buffer (host-terminal clients take exactly
  29. # the next binary frame as the replay, so it is not chunked). It gets the stall limit plus the time
  30. # to cross a link this slow (about 256 kbit/s; 37 s for 1 MiB). A replay that misses that deadline is
  31. # halved for the session's next attaches and never grows back: a send returns once the socket's buffers
  32. # take the frame, so a quick one does not show that the link carries more, and growing back would
  33. # replay a size that already timed out. A viewer on a slower link redials onto a replay it carries.
  34. _REPLAY_MIN_BYTES_PER_SECOND = 32 * 1024
  35. # The smallest replay a timed-out one shrinks to, so a reattach always shows the newest output. Its
  36. # deadline needs about 3 KB/s, less than a link needs to keep up with live output (one frame per
  37. # stall limit).
  38. _REPLAY_FLOOR_BYTES = 16 * 1024
  39. # A viewer send that fails because the browser is gone: Starlette raises WebSocketDisconnect for a
  40. # dead transport and RuntimeError for a socket already closed; OSError covers the TimeoutError of
  41. # a viewer that stopped reading.
  42. _VIEWER_SEND_ERRORS = (WebSocketDisconnect, RuntimeError, OSError)
  43. TUI_FORCE_REDRAW = b"\x0c"
  44. class RingBuffer:
  45. """Keeps only the most recent ``capacity`` bytes appended to it."""
  46. def __init__(self, capacity: int) -> None:
  47. self._cap = capacity
  48. self._buf = bytearray()
  49. self.truncated = False
  50. def append(self, data: bytes) -> None:
  51. self._buf.extend(data)
  52. overflow = len(self._buf) - self._cap
  53. if overflow > 0:
  54. del self._buf[:overflow]
  55. self.truncated = True
  56. def snapshot(self) -> bytes:
  57. return bytes(self._buf)
  58. async def _close_ws(ws, code: int, reason: Optional[str] = None) -> None:
  59. try:
  60. if ws is not None:
  61. await asyncio.wait_for(ws.close(code=code, reason=reason), timeout=5.0)
  62. except Exception:
  63. pass
  64. def _process_ancestors(pid: int) -> set[int]:
  65. """Pids of ``pid``'s live ancestors; empty when the process is gone or unreadable."""
  66. import psutil # type: ignore
  67. try:
  68. return {parent.pid for parent in psutil.Process(pid).parents()}
  69. except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess, ValueError):
  70. return set()
  71. def _key_segments(key: str) -> tuple[str, str]:
  72. """``(profile, resume)`` a keep-alive key was registered under.
  73. Registry keys read ``token\\0profile\\0resume``; a key without those
  74. segments — a chat that was never resumed from — yields ``("", "")``.
  75. Session ids are only unique within a profile's store, so a resume target
  76. is identified by both.
  77. """
  78. parts = key.split("\0")
  79. return (parts[1], parts[2]) if len(parts) >= 3 else ("", "")
  80. class PtySession:
  81. def __init__(
  82. self, key: str, bridge, *, buffer_cap: int, read_timeout: float,
  83. active_session_file: Optional[Path] = None, drain_executor: Optional[Executor] = None,
  84. ) -> None:
  85. self.key = key
  86. self.bridge = bridge
  87. self.active_session_file = active_session_file
  88. self.active_session_cleanup: Optional[Callable[[], None]] = None
  89. self.buffer = RingBuffer(buffer_cap)
  90. # How many of the buffer's newest bytes a reattach replays: a replay that times out halves it,
  91. # down to _REPLAY_FLOOR_BYTES, for the rest of the session.
  92. self._replay_limit = buffer_cap
  93. self.alive = True
  94. self.attached = False
  95. # Admission can fail before the first attach (e.g. lost metadata).
  96. self.last_detached_at: Optional[float] = time.monotonic()
  97. self._read_timeout = read_timeout
  98. # Runs the blocking reads; None is the loop's default executor.
  99. self._drain_executor = drain_executor
  100. self._ws = None
  101. self._attach_generation = 0
  102. # Only final sink writes and viewer claims hold this lock. Profile
  103. # admission/backpressure waits happen outside it.
  104. self._viewer_lock = threading.Lock()
  105. self._drain_task: Optional[asyncio.Task] = None
  106. self._write_lock = asyncio.Lock()
  107. self._output_lock = asyncio.Lock()
  108. self._input_task: Optional[asyncio.Task] = None
  109. self._close_task: Optional[asyncio.Task] = None
  110. # Chunks of output read so far, and set when one arrives (``wait_for_output``).
  111. self._outputs = 0
  112. self._output_arrived = asyncio.Event()
  113. async def start(self) -> None:
  114. self._drain_task = asyncio.create_task(self._drain())
  115. async def _drain(self) -> None:
  116. loop = asyncio.get_running_loop()
  117. while self.alive:
  118. try:
  119. chunk = await loop.run_in_executor(self._drain_executor, self.bridge.read, self._read_timeout)
  120. except OSError:
  121. chunk = None
  122. if chunk is None: # EOF — the agent process exited
  123. self.alive = False
  124. ws = self._ws
  125. async with self._output_lock:
  126. if self._ws is ws:
  127. await _close_ws(ws, WS_CLOSE_PROCESS_EXITED, WS_CLOSE_PROCESS_EXITED_REASON)
  128. return
  129. if not chunk: # idle tick
  130. await asyncio.sleep(0)
  131. continue
  132. self.buffer.append(chunk)
  133. self._outputs += 1
  134. self._output_arrived.set()
  135. ws = self._ws
  136. output_lock = self._output_lock
  137. try:
  138. if ws is not None:
  139. async with output_lock:
  140. if self._ws is ws:
  141. await asyncio.wait_for(ws.send_bytes(chunk), timeout=_VIEWER_SEND_TIMEOUT_SECONDS)
  142. except Exception:
  143. # The viewer is gone; nothing else observes this failure (the handler's finally
  144. # only runs once ws.receive() sees the disconnect). detach() is a no-op when a
  145. # replacement socket attached during the send, so the new viewer keeps its session.
  146. if self._ws is ws:
  147. self.detach(ws)
  148. await _close_ws(ws, WS_CLOSE_VIEWER_STALLED, WS_CLOSE_VIEWER_STALLED_REASON)
  149. async def write(self, ws, data: bytes, *, fence=None, **bridge_options) -> bool:
  150. """Serialize input and discard bytes from a superseded socket.
  151. ``fence`` is handed to the bridge (see ``PtyBridge.write``); what it raises propagates.
  152. ``bridge_options`` (``timeout``) go to the bridge's ``write`` unchanged.
  153. """
  154. async with self._write_lock:
  155. if self._ws is not ws or not self.alive:
  156. return True
  157. generation = self._attach_generation
  158. def viewer_fence(write):
  159. def admitted_write():
  160. with self._viewer_lock:
  161. if self._ws is not ws or self._attach_generation != generation or not self.alive:
  162. return None
  163. return write()
  164. return fence(admitted_write)
  165. if fence is not None:
  166. bridge_options["fence"] = viewer_fence
  167. task = self._input_task = asyncio.create_task(self.bridge.write(data, **bridge_options))
  168. try:
  169. delivered = await task
  170. except asyncio.CancelledError:
  171. if self._attach_generation != generation:
  172. return False
  173. raise
  174. finally:
  175. if self._input_task is task:
  176. self._input_task = None
  177. # A replacement socket can attach while the bridge write is
  178. # suspended on backpressure. A late failure from the superseded
  179. # socket must not poison the replacement's shared PTY session.
  180. if (
  181. not delivered
  182. and self._ws is ws
  183. and self._attach_generation == generation
  184. ):
  185. self.alive = False
  186. return delivered
  187. def resize(self, ws, *, cols: int, rows: int) -> None:
  188. if self._ws is ws and self.alive:
  189. self.bridge.resize(cols=cols, rows=rows)
  190. def size(self) -> Optional[tuple[int, int]]:
  191. """The PTY's ``(cols, rows)``, or None where the bridge cannot tell (ConPTY, stubs)."""
  192. return self.bridge.size() if hasattr(self.bridge, "size") else None
  193. @property
  194. def outputs(self) -> int:
  195. """How many chunks of output have been read from the program so far."""
  196. return self._outputs
  197. async def wait_for_output(self, since: int, timeout: float) -> None:
  198. """Return once output past ``since`` (an ``outputs`` value) has been read, the session has
  199. ended, or ``timeout`` seconds have passed."""
  200. loop = asyncio.get_running_loop()
  201. deadline = loop.time() + timeout
  202. while self.alive and self._outputs == since:
  203. self._output_arrived.clear()
  204. remaining = deadline - loop.time()
  205. if remaining <= 0:
  206. return
  207. try:
  208. await asyncio.wait_for(self._output_arrived.wait(), remaining)
  209. except TimeoutError:
  210. return
  211. async def attach(
  212. self, ws, *, force_redraw: bool = False,
  213. initial_text: Optional[str | Callable[[bool], str]] = None,
  214. ) -> bool:
  215. """Attach a browser terminal and replay buffered PTY output.
  216. The TUI renders differentially on an alternate screen, so a bounded ANSI tail is not a
  217. self-contained frame; ``force_redraw`` asks the live TUI for one full redraw after replay.
  218. ``initial_text`` is sent before the replay; given as a function, it is called with whether
  219. the replay leaves out older output (dropped from the buffer or past the replay limit).
  220. """
  221. self._attach_generation += 1
  222. generation = self._attach_generation
  223. if self._input_task is not None:
  224. self._input_task.cancel()
  225. # Revoke queued workers first, then serialize the claim after any
  226. # already-admitted sink write. ConPTY writes can block: keep the loop
  227. # free to run their timeout/cancellation cleanup while we wait.
  228. while not self._viewer_lock.acquire(blocking=False):
  229. await asyncio.sleep(0.001)
  230. if self._attach_generation != generation:
  231. return False
  232. try:
  233. if self._attach_generation != generation:
  234. return False
  235. old_ws = self._ws
  236. self._ws = ws
  237. self.attached = True
  238. self.last_detached_at = None
  239. # Per-viewer lock: a stalled old send must not block its replacement.
  240. output_lock = self._output_lock = asyncio.Lock()
  241. finally:
  242. self._viewer_lock.release()
  243. try:
  244. async with output_lock:
  245. held = self.buffer.snapshot()
  246. snap = held[len(held) - min(len(held), self._replay_limit):]
  247. trimmed = self.buffer.truncated or len(snap) < len(held)
  248. if old_ws is not ws:
  249. await _close_ws(old_ws, WS_CLOSE_SUPERSEDED)
  250. if self._ws is not ws:
  251. return False
  252. if initial_text is not None:
  253. text = initial_text(trimmed) if callable(initial_text) else initial_text
  254. await asyncio.wait_for(ws.send_text(text), timeout=_VIEWER_SEND_TIMEOUT_SECONDS)
  255. if self._ws is not ws:
  256. return False
  257. # Metadata-bearing host clients treat exactly the next binary
  258. # frame as replay. Send an empty snapshot too, so the first live
  259. # frame never loses terminal-query replies on an empty history.
  260. if snap or initial_text is not None:
  261. replay_timeout = _VIEWER_SEND_TIMEOUT_SECONDS + len(snap) / _REPLAY_MIN_BYTES_PER_SECOND
  262. try:
  263. await asyncio.wait_for(ws.send_bytes(snap), timeout=replay_timeout)
  264. except TimeoutError:
  265. self._replay_limit = max(_REPLAY_FLOOR_BYTES, len(snap) // 2)
  266. raise
  267. except _VIEWER_SEND_ERRORS:
  268. self.detach(ws)
  269. return False
  270. except BaseException: # cancellation or a bug: never leave this socket attached
  271. self.detach(ws)
  272. raise
  273. if self._ws is not ws:
  274. return False
  275. if not self.alive:
  276. self.detach(ws)
  277. await _close_ws(ws, WS_CLOSE_PROCESS_EXITED, WS_CLOSE_PROCESS_EXITED_REASON)
  278. return False
  279. if force_redraw:
  280. return await self.write(ws, TUI_FORCE_REDRAW)
  281. return True
  282. def detach(self, ws) -> None:
  283. # Only the currently-attached socket may mark the session detached: a superseded socket's
  284. # handler also calls detach on its way out (after the new tab attached), and flipping
  285. # ``attached`` then would make a session with a live viewer look idle and reapable.
  286. if self._ws is not ws:
  287. return
  288. self._ws = None
  289. self.attached = False
  290. self.last_detached_at = time.monotonic()
  291. def hosts_pid(self, pid: Optional[int]) -> bool:
  292. """Whether ``pid`` runs inside this PTY: the leader itself or one of its descendants.
  293. The dashboard's PTY leader is ``node entry.js``; the process that takes the
  294. session lease is its ``tui_gateway`` child, so the lease pid is never the
  295. bridge's own pid and only ancestry identifies the terminal. A bridge that
  296. cannot report a pid (already reaped, or a stub) is simply not a match.
  297. """
  298. if pid is None:
  299. return False
  300. try:
  301. leader = int(self.bridge.pid)
  302. except (AttributeError, TypeError, ValueError, OSError):
  303. return False
  304. return leader == pid or leader in _process_ancestors(pid)
  305. async def close(self) -> None:
  306. # Disconnect cancellation must not strand a child after its registry
  307. # entry was removed. Cleanup has independent, idempotent ownership.
  308. if self._close_task is None:
  309. self.alive = False
  310. self._attach_generation += 1
  311. if self._input_task is not None:
  312. self._input_task.cancel()
  313. ws = self._ws
  314. self.detach(ws)
  315. self._close_task = asyncio.create_task(self._close(ws))
  316. await asyncio.shield(self._close_task)
  317. async def _close(self, ws) -> None:
  318. await _close_ws(ws, WS_CLOSE_PROCESS_EXITED)
  319. if self._drain_task is not None:
  320. self._drain_task.cancel()
  321. try:
  322. await self._drain_task
  323. except asyncio.CancelledError:
  324. pass
  325. except Exception:
  326. # The drain handles its own read and viewer errors, so this is a bug; the
  327. # child below must still be closed.
  328. logger.warning("PTY output drain failed", exc_info=True)
  329. try:
  330. # bridge.close() joins the child — blocking; keep it off the event loop.
  331. # See #53227.
  332. await asyncio.to_thread(self.bridge.close)
  333. except Exception: # health: allow BLE001 S110 -- teardown of an already-dead PTY must not mask the caller's error path
  334. pass
  335. try:
  336. if self.active_session_file is not None:
  337. self.active_session_file.unlink(missing_ok=True)
  338. except OSError:
  339. pass
  340. if self.active_session_cleanup is not None:
  341. try:
  342. self.active_session_cleanup()
  343. except Exception: # health: allow BLE001 S110 -- cleanup callback must not mask the close path; the PTY is dead either way
  344. pass
  345. self.active_session_cleanup = None
  346. class RegistryFull(Exception):
  347. """Every keep-alive slot holds a PTY that some tab is still attached to."""
  348. def __init__(self, message: str = "Too many chat terminals are open in other tabs; close one and try again.") -> None:
  349. super().__init__(message)
  350. async def run_reaper(registry: PtySessionRegistry, *, interval: float = 60.0) -> None:
  351. """Periodically reap idle/dead keep-alive sessions. Cancelled on shutdown."""
  352. while True:
  353. await asyncio.sleep(interval)
  354. try:
  355. await registry.reap_idle()
  356. except Exception:
  357. pass
  358. class PtySessionRegistry:
  359. def __init__(
  360. self, *, ttl: float, max_sessions: int, buffer_cap: int, read_timeout: float,
  361. drain_executor: Optional[Executor] = None,
  362. ) -> None:
  363. self._ttl = ttl
  364. self._max = max_sessions
  365. self._buffer_cap = buffer_cap
  366. self._read_timeout = read_timeout
  367. self._drain_executor = drain_executor
  368. self._sessions: dict[str, PtySession] = {}
  369. # One registry-wide reservation spans lookup, spawn, and registration:
  370. # racing connections with one attach token must share one tracked PTY.
  371. self._spawn_lock = asyncio.Lock()
  372. self._spawn_cleanup: set[asyncio.Task] = set()
  373. self._closed = False
  374. # Retain ownership of removed sessions until helper-process shutdown completes.
  375. self._background_closes: set[asyncio.Task] = set()
  376. async def attach_or_spawn(
  377. self, key: str, *, spawn: Callable[[], object], active_session_file: Optional[Path] = None,
  378. ) -> tuple[PtySession, bool]:
  379. # Reserve capacity and the key across blocking fork/exec. On cancellation
  380. # the registry, not the request's cancellation scope, owns that reservation
  381. # until the admission finishes and any unclaimed child has been closed.
  382. await self._spawn_lock.acquire()
  383. admission = asyncio.create_task(
  384. self._attach_or_spawn(key, spawn=spawn, active_session_file=active_session_file))
  385. try:
  386. # wait() leaves admission running on caller cancellation without
  387. # shield() reporting its late exception (Python 3.14). The registry's
  388. # discard task owns that result once the caller disconnects.
  389. await asyncio.wait({admission})
  390. result = admission.result()
  391. except asyncio.CancelledError:
  392. cleanup = asyncio.create_task(self._discard_admission(admission))
  393. self._spawn_cleanup.add(cleanup)
  394. cleanup.add_done_callback(self._spawn_cleanup.discard)
  395. raise
  396. except BaseException:
  397. self._spawn_lock.release()
  398. raise
  399. self._spawn_lock.release()
  400. return result
  401. async def _discard_admission(self, admission: asyncio.Task) -> None:
  402. try:
  403. try:
  404. session, created = await admission
  405. except Exception:
  406. # The disconnected caller cannot observe a late fork/exec
  407. # failure: retrieve it here and keep the traceback.
  408. logger.debug("PTY spawn failed after its caller disconnected", exc_info=True)
  409. return
  410. if created:
  411. if self._sessions.get(session.key) is session:
  412. self._sessions.pop(session.key)
  413. await session.close()
  414. finally:
  415. self._spawn_lock.release()
  416. async def _attach_or_spawn(
  417. self, key: str, *, spawn: Callable[[], object], active_session_file: Optional[Path] = None,
  418. ) -> tuple[PtySession, bool]:
  419. if self._closed:
  420. raise RegistryFull("Terminal service is shutting down.")
  421. await self.reap_idle()
  422. existing = self._sessions.get(key)
  423. if existing is not None and existing.alive:
  424. return existing, False
  425. if existing is not None: # dead remnant
  426. # Close in the background: ending a dead leader's helpers can take the helper
  427. # grace, and this reservation serializes every new chat.
  428. self._sessions.pop(key, None)
  429. self._close_in_background(existing)
  430. if len(self._sessions) >= self._max:
  431. self._reap_one_idle_or_raise()
  432. # PTY spawn does blocking fork/exec work — keep it off the event loop.
  433. # See #53227.
  434. bridge = await asyncio.to_thread(spawn)
  435. session = PtySession(
  436. key,
  437. bridge,
  438. buffer_cap=self._buffer_cap,
  439. read_timeout=self._read_timeout,
  440. active_session_file=active_session_file,
  441. drain_executor=self._drain_executor,
  442. )
  443. await session.start()
  444. self._sessions[key] = session
  445. return session, True
  446. async def close_other_sessions(self, prefix: str, *, keep_key: str) -> None:
  447. """Close sessions belonging to the same logical client except ``keep_key``.
  448. Dashboard profile changes keep the browser's attach token but change the
  449. canonical session key. The previous profile's detached PTY must not
  450. remain alive long enough to hold the TUI session lease and reject a
  451. later return to that chat.
  452. """
  453. async with self._spawn_lock:
  454. keys = [
  455. key for key in self._sessions
  456. if key != keep_key and (key == prefix or key.startswith(prefix + "\0"))
  457. ]
  458. for key in keys:
  459. session = self._sessions.pop(key, None)
  460. if session is not None:
  461. # A sibling tab sharing the attach token may still be viewing this
  462. # PTY: supersede it explicitly (4409) instead of leaving it silent
  463. # until its next keystroke fails with 1013. Detach first so close()
  464. # does not overwrite that code with WS_CLOSE_PROCESS_EXITED.
  465. ws = session._ws
  466. session.detach(ws)
  467. await _close_ws(ws, WS_CLOSE_SUPERSEDED)
  468. await session.close()
  469. async def close_orphaned_sessions(
  470. self, resume: Optional[str], *, keep_key: str, holder_pid: Optional[int] = None,
  471. ) -> None:
  472. """Close a keep-alive PTY stranded under a superseded attach token.
  473. Rotating the token — the dashboard's *New chat* — moves the tab to a
  474. fresh PTY while the previous one keeps the TUI's single-writer lease on
  475. its session. That terminal is out of the user's reach, so the next
  476. return to the same chat is refused as a session held elsewhere. A PTY
  477. with a live viewer is never a candidate: somebody is still using it.
  478. A chat that was never resumed from carries no resume target in its key,
  479. so ``holder_pid`` — the process holding the lease for the session being
  480. resumed — identifies that terminal instead.
  481. """
  482. if not resume:
  483. return
  484. # The requested chat is (profile, session): the same session id in
  485. # another profile's store is a different chat whose terminal stays.
  486. target = (_key_segments(keep_key)[0], resume)
  487. async with self._spawn_lock:
  488. doomed = [
  489. key for key, session in self._sessions.items()
  490. if key != keep_key and not session.attached
  491. and (_key_segments(key) == target or session.hosts_pid(holder_pid))
  492. ]
  493. sessions = [self._sessions.pop(key) for key in doomed]
  494. # Close outside the registry lock — a close can wait out its helpers' SIGHUP grace and
  495. # this lock serializes every new chat — but still before the caller spawns: the child's
  496. # session lease is only released once its close finishes.
  497. for session in sessions:
  498. await session.close()
  499. def detach(self, key: str, ws) -> None:
  500. s = self._sessions.get(key)
  501. if s is not None:
  502. s.detach(ws)
  503. def expired(self, s: PtySession, now: Optional[float] = None) -> bool:
  504. """Whether ``s`` has had no viewer for longer than the retention TTL."""
  505. return (not s.attached and s.last_detached_at is not None
  506. and (time.monotonic() if now is None else now) - s.last_detached_at > self._ttl)
  507. def _reapable(self, s: PtySession, now: Optional[float] = None) -> bool:
  508. # EOF never arrives if a helper still holds the PTY slave after the child died (#76759);
  509. # ask the process itself (a WNOHANG waitpid).
  510. return not s.alive or not s.bridge.is_alive() or self.expired(s, now)
  511. async def reap_idle(self, now: Optional[float] = None) -> None:
  512. now = time.monotonic() if now is None else now
  513. doomed = [(key, s) for key, s in self._sessions.items() if self._reapable(s, now)]
  514. for key, expected in doomed:
  515. # Reaps overlap (attach_or_spawn and the background reaper) and close()
  516. # awaits, so a concurrent reap can have popped this key already — skip
  517. # it instead of raising KeyError into the websocket handler. A viewer can
  518. # also re-attach during an earlier close, so re-check before closing.
  519. if self._sessions.get(key) is not expected or not self._reapable(expected, now):
  520. continue
  521. session = self._sessions.pop(key, None)
  522. if session is not None:
  523. await session.close()
  524. def _reap_one_idle_or_raise(self) -> None:
  525. idle = [s for s in self._sessions.values() if not s.attached and s.last_detached_at is not None]
  526. if not idle:
  527. raise RegistryFull()
  528. oldest = min(idle, key=lambda s: s.last_detached_at or 0.0)
  529. self._sessions.pop(oldest.key, None)
  530. self._close_in_background(oldest)
  531. def _close_in_background(self, session: PtySession) -> asyncio.Task:
  532. task = asyncio.create_task(session.close())
  533. self._background_closes.add(task)
  534. task.add_done_callback(self._background_closes.discard)
  535. return task
  536. def reopen(self) -> None:
  537. """Take spawns again after :meth:`close_all`: the registry is process-wide, and a lifespan can
  538. start again in one process (the gateway falling back from the Webapp's listener to its
  539. private one)."""
  540. self._closed = False
  541. async def close_all(self) -> None:
  542. self._closed = True
  543. # Wait for any admitted spawn before snapshotting shutdown ownership.
  544. async with self._spawn_lock:
  545. pass
  546. if self._spawn_cleanup:
  547. await asyncio.shield(asyncio.gather(*self._spawn_cleanup))
  548. # Helper-process grace must run concurrently inside the backend shutdown budget.
  549. sessions = [self._sessions.pop(key) for key in list(self._sessions)]
  550. await asyncio.gather(*(s.close() for s in sessions), *self._background_closes, return_exceptions=True)