| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579 |
- """Process-lifetime host shells, owned by Webapp identity and profile home.
- This is a separate capability namespace from Dashboard's client-selected chat IDs.
- The existing PTY registry owns processes/drains; this module owns host admission.
- A shell dies with its profile: once a named home is missing or deleted (main's
- tombstone rule, ``hermes_constants.assert_named_profile_home_live``) its input is
- refused and the reaper closes it.
- """
- from __future__ import annotations
- import asyncio
- from concurrent.futures import ThreadPoolExecutor
- from contextlib import asynccontextmanager, suppress
- from dataclasses import dataclass
- import json
- from pathlib import Path
- import re
- import secrets
- import time
- from fastapi import HTTPException, WebSocket, WebSocketDisconnect
- from hermes_cli.pty_session import (
- _VIEWER_SEND_ERRORS, _VIEWER_SEND_TIMEOUT_SECONDS, WS_CLOSE_VIEWER_STALLED,
- WS_CLOSE_VIEWER_STALLED_REASON, PtySession, PtySessionRegistry, RegistryFull, _close_ws, run_reaper,
- )
- from hermes_cli.web_host_terminal import META_PREFIX
- from hermes_constants import assert_named_profile_home_live, get_hermes_home
- # No survival across backend restart. Disconnects retain a bounded ANSI tail.
- RETENTION_SECONDS = 15 * 60
- _MAX_SHELLS = 16
- # Inbound input buffered while one batch is written (backpressure bound): full only once it holds
- # both ``_INPUT_QUEUE_FRAMES`` frames and ``_INPUT_QUEUE_BYTES`` bytes, each frame also costing
- # ``_FRAME_OVERHEAD_BYTES``. Until then the reader keeps reading the socket, which is where the
- # server's keepalive pongs arrive (``_InputQueue``).
- _INPUT_QUEUE_FRAMES = 64
- _INPUT_QUEUE_BYTES = 1024 * 1024
- _FRAME_OVERHEAD_BYTES = 256
- # How often a reader held up by that bound checks that the tab is still there (``_queue_frame``).
- # Once the connection is gone a check fails within about three: the first can still reach the
- # kernel before the peer's reset comes back.
- _LIVENESS_PROBE_SECONDS = 1.0
- # How long a resize that undoes the previous one waits for the program to answer that one (``_resize``).
- _RESIZE_ANSWER_SECONDS = 1.0
- TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{43}$")
- class TerminalDenied(Exception):
- pass
- class TerminalExpired(Exception):
- pass
- class TerminalExited(TerminalExpired):
- """The shell exited by itself (``exit``, Ctrl-D); the server did not end it."""
- def _home_live(home: Path) -> bool:
- """A named home must exist and carry no deletion tombstone; any other home only has to exist."""
- try:
- assert_named_profile_home_live(home)
- except FileNotFoundError:
- return False
- return home.is_dir()
- @dataclass(frozen=True)
- class HostOwner:
- identity: tuple[str, str]
- home: Path
- shell: str
- cwd: str
- def current(self) -> bool:
- """Blocking filesystem reads (home, tombstone): call off-loop."""
- return _home_live(self.home)
- def admit(self, write):
- """Run ``write`` while this owner's profile home is live, or raise ``TerminalExpired``.
- Input sent once the profile is deleted never reaches the shell. Blocking: bridges call
- it off-loop.
- """
- if not self.current():
- raise TerminalExpired()
- return write()
- def _request_identity(ws: WebSocket) -> tuple[str, str]:
- """``(provider, user)`` of the terminal ticket that admitted *ws*; ``_ws_gate`` admits no other
- credential to the host terminal."""
- identity = getattr(ws, "_hermes_auth_identity", None) or {}
- user, provider = identity.get("user_id"), identity.get("provider")
- if not isinstance(user, str) or not user or not isinstance(provider, str) or not provider:
- raise TerminalDenied()
- return provider, user
- def _request_home(profile: str | None) -> Path:
- from hermes_cli.web_server_profiles import _resolve_profile_dir
- requested = (profile or "").strip()
- return (
- _resolve_profile_dir(requested)
- if requested and requested.lower() != "current" else get_hermes_home()
- ).resolve()
- class HostTerminalRegistry(PtySessionRegistry):
- # The most exited shells remembered at once: a tab that redials after the reaper took its
- # shell still learns that it exited (``resolve``).
- _EXITS_KEPT = 4 * _MAX_SHELLS
- def __init__(self):
- # Every live shell keeps a read in flight. On the loop's default executor, retained
- # shells would queue the keystroke fences, ownership checks, uploads and SessionDB
- # hops that share it; a pool sized to capacity gives each shell its own reader.
- super().__init__(
- ttl=RETENTION_SECONDS, max_sessions=_MAX_SHELLS, buffer_cap=1024 * 1024, read_timeout=0.2,
- drain_executor=ThreadPoolExecutor(_MAX_SHELLS, thread_name_prefix="host-pty-drain"))
- self.owners: dict[str, HostOwner] = {}
- # Token -> (owner identity, when it was reaped) of shells that exited by themselves, for
- # the retention TTL.
- self.exits: dict[str, tuple[tuple[str, str], float]] = {}
- def _reap_one_idle_or_raise(self) -> None:
- # Retention is a promise, not an LRU hint; never evict a live shell to
- # make room for a different pane, even if its viewer just disconnected.
- raise RegistryFull()
- async def create(self, ws: WebSocket, identity: tuple[str, str]):
- from hermes_cli.web_host_terminal import query_dimension, resolve_argv
- from hermes_cli.web_server_chat import PtyBridge
- token = secrets.token_urlsafe(32)
- owner = None
- def spawn():
- nonlocal owner
- home = _request_home(ws.query_params.get("profile"))
- if not _home_live(home):
- raise TerminalExpired()
- argv, cwd, env, shell = resolve_argv(home=home, requested_cwd=ws.query_params.get("cwd"))
- bridge = PtyBridge.spawn(
- argv, cwd=cwd, env=env,
- cols=query_dimension(ws.query_params.get("cols"), 80, 2000),
- rows=query_dimension(ws.query_params.get("rows"), 24, 1000))
- owner = HostOwner(identity, home, shell, cwd)
- return bridge
- session, _ = await self.attach_or_spawn(token, spawn=spawn)
- assert owner is not None # spawn completed
- self.owners[token] = owner
- if not await asyncio.to_thread(owner.current):
- await self.remove(token)
- raise TerminalExpired()
- return session, owner
- async def resolve(self, token: str, ws: WebSocket, identity: tuple[str, str], *, closing=False):
- session = self._sessions.get(token)
- owner = self.owners.get(token)
- if session is None or owner is None:
- raise self._gone(token, identity)
- if owner.identity != identity:
- raise TerminalDenied()
- def validate():
- if _request_home(ws.query_params.get("profile")) != owner.home:
- raise TerminalDenied()
- owner.admit(lambda: None)
- await asyncio.to_thread(validate)
- # No await between the final check and the caller's attach claim. The
- # home was just checked off-loop; every input write checks it again.
- if self._sessions.get(token) is not session:
- # The reaper took the shell during validation; it may have exited first.
- raise self._gone(token, identity)
- if not closing and _exited(session):
- raise TerminalExited()
- if not closing and self.expired(session):
- raise TerminalExpired()
- return session, owner
- def _gone(self, token: str, identity: tuple[str, str]) -> TerminalExpired:
- """The refusal of a token with no registered shell: ``TerminalExited`` if *identity*'s shell
- exited by itself, else ``TerminalExpired``."""
- gone = self.exits.get(token)
- return TerminalExited() if gone is not None and gone[0] == identity else TerminalExpired()
- async def remove(self, token: str):
- session = self._sessions.pop(token, None)
- self.owners.pop(token, None)
- if session is not None:
- await asyncio.shield(self._close_in_background(session))
- async def reap_idle(self, now=None):
- # Durable tombstones also detect deletion by a different process; attached
- # viewers do not exempt a deleted profile.
- # One thread hop per tick keeps every shell's marker reads off the loop.
- owners = list(self.owners.items())
- retired = await asyncio.to_thread(lambda: {token for token, owner in owners if not owner.current()})
- now = time.monotonic() if now is None else now
- for token, owner in owners:
- session = self._sessions.get(token)
- if session is not None and (token in retired or self._reapable(session, now)):
- if token not in retired and _exited(session):
- self._remember_exit(token, owner, now)
- await self.remove(token)
- for token in list(self.owners):
- if token not in self._sessions:
- self.owners.pop(token, None)
- for token in [token for token, (_, at) in self.exits.items() if now - at > self._ttl]:
- del self.exits[token]
- def _remember_exit(self, token: str, owner: HostOwner, now: float) -> None:
- self.exits[token] = (owner.identity, now)
- while len(self.exits) > self._EXITS_KEPT:
- del self.exits[next(iter(self.exits))]
- async def close_all(self):
- await super().close_all()
- self.owners.clear()
- self.exits.clear()
- # Every drain is cancelled by now; a read still in flight returns within its
- # timeout, so let it finish behind us. A closed registry never spawns again.
- self._drain_executor.shutdown(wait=False)
- def _exited(session: PtySession) -> bool:
- """Whether a session still in the registry lost its shell: the server pops a session before
- it closes one, so only the shell's own exit leaves a dead one behind."""
- return not session.alive or not session.bridge.is_alive()
- def get_host_terminals(app) -> HostTerminalRegistry:
- registry = getattr(app.state, "host_terminals", None)
- if registry is None or registry._closed:
- raise RuntimeError("Host terminal service is not running; start the application lifespan first.")
- return registry
- @asynccontextmanager
- async def host_terminal_lifespan(app):
- # Overlapping lifespans must not share loop-bound sessions or teardown.
- registry = HostTerminalRegistry()
- registries = getattr(app.state, "_host_terminal_registries", None)
- if registries is None:
- registries = app.state._host_terminal_registries = []
- registries.append(registry)
- app.state.host_terminals = registry
- reaper = asyncio.create_task(run_reaper(registry, interval=1.0))
- try:
- yield
- finally:
- # Withdraw ownership before awaiting cleanup so another teardown cannot
- # restore this registry while its reaper/sessions are shutting down.
- registries.remove(registry)
- if getattr(app.state, "host_terminals", None) is registry:
- if registries:
- app.state.host_terminals = registries[-1]
- else:
- del app.state.host_terminals
- if not registries:
- del app.state._host_terminal_registries
- reaper.cancel()
- with suppress(asyncio.CancelledError):
- await reaper
- await registry.close_all()
- def _metadata(token, owner, registry, *, reconnected, truncated):
- return META_PREFIX + json.dumps({
- "shell": owner.shell, "terminalId": token, "cwd": owner.cwd,
- "reconnected": reconnected, "retentionSeconds": registry._ttl,
- "truncated": truncated,
- }, separators=(",", ":"))
- async def _send_closed(ws: WebSocket, token: str) -> None:
- await ws.send_text(META_PREFIX + json.dumps({"terminalId": token, "closed": True}))
- await ws.close(code=1000)
- async def _admit(ws: WebSocket, registry: HostTerminalRegistry):
- """Authorize the request and return ``(session, owner)`` to attach.
- Returns None once a ``close`` request has been served. Raises the ``Terminal*``
- errors ``host_terminal`` maps to close codes.
- """
- token = ws.query_params.get("attach")
- action = ws.query_params.get("action")
- identity = _request_identity(ws)
- if action not in (None, "close") or (action == "close" and token is None):
- raise TerminalDenied()
- if token is not None and not TOKEN_RE.fullmatch(token):
- raise TerminalExpired()
- if action == "close" and token not in registry._sessions:
- # Missing is idempotent, but never bypass auth/host policy.
- await _send_closed(ws, token)
- return None
- if token is None:
- return await registry.create(ws, identity)
- session, owner = await registry.resolve(token, ws, identity, closing=action == "close")
- if action == "close":
- await registry.remove(token)
- await _send_closed(ws, token)
- return None
- return session, owner
- def _frame_payload(msg: dict) -> bytes:
- """What a frame writes to the program: a binary frame's bytes, a text frame's UTF-8. The
- WebSocket library decodes a text frame strictly, so its text always encodes back."""
- raw = msg.get("bytes")
- return (msg.get("text") or "").encode("utf-8") if raw is None else raw
- def _frame_cost(item) -> int:
- """What a queued item counts against ``_INPUT_QUEUE_BYTES``: its payload and the per-frame overhead."""
- if not isinstance(item, dict): # the reader's last item: None or the error that ended reading
- return _FRAME_OVERHEAD_BYTES
- # The bytes ``_input_runs`` writes, not characters: a CJK or emoji paste is 3-4 bytes each.
- return len(_frame_payload(item)) + _FRAME_OVERHEAD_BYTES
- class _InputQueue(asyncio.Queue):
- """The frames a tab sent while a batch is written, full once it holds at least
- ``_INPUT_QUEUE_FRAMES`` frames and at least ``_INPUT_QUEUE_BYTES`` charged by ``_frame_cost``.
- A full queue stops the reader, and with it the WebSocket library's own read of the socket once
- that buffers 32 unread frames: keepalive pongs then go unread, and a bind that pings (any
- non-loopback Webapp) closes a live tab with 1011 after the ping timeout, dropping the input
- queued behind it. A frame count alone fills on a few dozen keys typed while a paste waits on a
- program that is not reading, and a byte count alone on one large paste; needing both gives
- typing far more room (one-byte keys fill it at 4,081 frames) and never stops the reader before
- the frame count alone would. Typing past that stalls the reader just as large frames do.
- So the frame count is not a strict limit: small frames can queue thousands. The payload ceiling
- stays about the frame count alone's: a frame is only taken while the queue holds fewer than
- ``_INPUT_QUEUE_FRAMES`` frames or charges under ``_INPUT_QUEUE_BYTES``, and each frame is at
- most the server's ``ws_max_size``.
- """
- def __init__(self):
- super().__init__()
- self.queued = 0
- def full(self) -> bool:
- return self.qsize() >= _INPUT_QUEUE_FRAMES and self.queued >= _INPUT_QUEUE_BYTES
- def _put(self, item):
- self.queued += _frame_cost(item)
- super()._put(item)
- def _get(self):
- item = super()._get()
- self.queued -= _frame_cost(item)
- return item
- async def _queue_frame(ws: WebSocket, inbox: asyncio.Queue, ended: asyncio.Future, msg) -> None:
- """``inbox.put(msg)``, checking that the tab is still there while the inbox stays full.
- A full inbox stops the reader taking frames. That backpressure bounds what a tab can queue, but
- it also leaves the tab's close unread behind those frames, and short of reading them only a send
- can find out that the connection is gone. So while the put waits, the tab gets an empty binary
- frame (no output, which the client ignores) every ``_LIVENESS_PROBE_SECONDS``; attach has already
- sent the replay, so it is never taken for one. A send that fails means the connection is gone:
- ``ended`` is set, which cancels a write waiting on the program, and the put goes on waiting
- unprobed.
- """
- while not ended.done():
- try:
- return await asyncio.wait_for(inbox.put(msg), _LIVENESS_PROBE_SECONDS)
- except TimeoutError:
- pass
- try:
- await asyncio.wait_for(ws.send_bytes(b""), _VIEWER_SEND_TIMEOUT_SECONDS)
- except _VIEWER_SEND_ERRORS:
- ended.set_result(None)
- await inbox.put(msg)
- async def _read_frames(ws: WebSocket, inbox: asyncio.Queue, ended: asyncio.Future) -> None:
- """Queue inbound frames. The last item is None, or the error that ended reading
- (re-raised by ``_pump_input`` so the route's handlers still see it). ``ended`` gets the same
- item as soon as reading ends, before it is queued, or None as soon as a full inbox finds the
- socket gone (``_queue_frame``): a write waiting on the shell, or a full inbox behind it, must
- not keep the pump from seeing that the socket is gone."""
- end = None
- try:
- while (msg := await ws.receive()).get("type") != "websocket.disconnect":
- await _queue_frame(ws, inbox, ended, msg)
- except RuntimeError: # a superseding/drain task already closed us
- pass
- # health: allow BLE001 -- forwarded, not swallowed: _pump_input re-raises it, and a reader that died unqueued would hang the pump
- except Exception as exc:
- end = exc
- if not ended.done():
- ended.set_result(end)
- await inbox.put(end)
- async def _write_while_open(session: PtySession, ws: WebSocket, data: bytes, fence,
- ended: asyncio.Future) -> bool | None:
- """``session.write`` of *data* with no deadline while the socket is open, or None once the
- socket has ended and the write has to wait for the program to read.
- The program may leave its input unread for as long as it likes, but not past the viewer. Input
- the tab sent before it closed still reaches the program as far as the program takes it; the
- first time the write then has to wait for room, it is cancelled (the rest is dropped, as on a
- takeover) and the session stays alive for the caller to detach.
- """
- loop = asyncio.get_running_loop()
- stalled = asyncio.Event()
- def watched(write):
- accepted = fence(write)
- # The POSIX bridge reports 0 bytes taken when the program's input queue is full; it then
- # waits for room. Runs in the bridge's worker thread.
- full = type(accepted) is int and accepted == 0
- loop.call_soon_threadsafe(stalled.set if full else stalled.clear)
- return accepted
- write = asyncio.ensure_future(session.write(ws, data, fence=watched, timeout=None))
- try:
- await asyncio.wait((write, ended), return_when=asyncio.FIRST_COMPLETED)
- if not write.done():
- stall = asyncio.ensure_future(stalled.wait())
- try:
- await asyncio.wait((write, stall), return_when=asyncio.FIRST_COMPLETED)
- finally:
- stall.cancel()
- finally:
- if not write.done():
- write.cancel()
- await asyncio.wait((write,))
- return None if write.cancelled() else write.result()
- def _input_runs(frames: list[dict], resize_re):
- """``frames`` in order: consecutive keystrokes joined into one run, resizes as ``(cols, rows)``."""
- run: list[bytes] = [] # joined once: a batch can hold megabytes of pasted input
- for msg in frames:
- raw = _frame_payload(msg)
- match = resize_re.fullmatch(raw) if raw else None
- if match is None:
- if raw:
- run.append(raw)
- continue
- if run:
- yield b"".join(run)
- run = []
- yield int(match[1]), int(match[2])
- if run:
- yield b"".join(run)
- async def _resize(session: PtySession, ws: WebSocket, size: tuple[int, int], undo):
- """Apply ``size``; return what the next resize needs to tell whether it undoes this one.
- A program can read the size well after its SIGWINCH (Node, behind its event loop, and reports a
- 'resize', which Hermes' TUI redraws on, only when the size it reads has changed). A step undone
- before it looks is never seen: both signals find the size it already had. So a resize back to
- the size before the previous one first waits until the program has written output since that
- one, as a program that redraws on a resize does, or ``_RESIZE_ANSWER_SECONDS`` for one that
- does not. ``undo`` is ``(size before the previous resize, outputs when it was applied)``, set
- only when that resize changed the size.
- """
- if undo is not None and undo[0] == size:
- await session.wait_for_output(undo[1], _RESIZE_ANSWER_SECONDS)
- before = session.size()
- session.resize(ws, cols=size[0], rows=size[1])
- return (before, session.outputs) if before is not None and before != size else None
- async def _pump_input(ws: WebSocket, registry: HostTerminalRegistry, token: str,
- session: PtySession, owner: HostOwner) -> None:
- """Forward keystrokes and resizes while ``ws`` is the attached viewer.
- Keystrokes reach the shell only inside ``owner.admit``, so input sent after the
- profile is deleted never does, even when it queued behind input that was admitted
- first. The fence reads files, so it runs off the loop, once per run of keystrokes in
- a batch (the frames that queued while the previous batch was written): a paste of
- many frames costs a few fences, not one each.
- Writes have no deadline. While the foreground program is not reading (``sleep``,
- ``npm install``) the PTY takes about 1 KB of input on macOS, so a paste waits for
- that program, in order, with any Ctrl-C behind it, as in native Desktop's terminal.
- A re-attach or the socket closing cancels the wait and keeps the shell (``host_terminal``
- then detaches it, so the retention reaper owns it again); only a closed bridge (the shell
- exited or the terminal was closed) or a deleted profile drops the session. ConPTY
- buffers input itself, so Windows writes land at once and keep the bridge's deadline.
- """
- from hermes_cli.web_server_chat import _RESIZE_RE
- inbox: asyncio.Queue = _InputQueue()
- ended = asyncio.get_running_loop().create_future()
- reader = asyncio.create_task(_read_frames(ws, inbox, ended))
- undo = None
- try:
- # No Ctrl-L/redraw input: arbitrary foreground shell programs own stdin.
- while session._ws is ws and session.alive:
- batch = [await inbox.get()]
- while not inbox.empty():
- batch.append(inbox.get_nowait())
- frames = [msg for msg in batch if isinstance(msg, dict)]
- for item in _input_runs(frames, _RESIZE_RE):
- if isinstance(item, tuple):
- undo = await _resize(session, ws, item, undo)
- continue
- try:
- delivered = await _write_while_open(session, ws, item, owner.admit, ended)
- except (TerminalExpired, FileNotFoundError):
- await registry.remove(token)
- return
- if delivered is None:
- # The socket ended and this input was left waiting for the program: stop here,
- # so the caller detaches a shell that is still running.
- if isinstance(ended.result(), Exception):
- raise ended.result()
- return
- if not delivered:
- # A closed bridge marked the session dead. A takeover only cancelled this
- # write: drop the rest of the batch and leave this socket for the new viewer
- # to supersede, so this tab does not reconnect and take the shell back.
- if not session.alive:
- await registry.remove(token)
- return
- break
- if len(frames) < len(batch):
- if isinstance(batch[-1], Exception):
- raise batch[-1]
- return
- finally:
- reader.cancel()
- with suppress(asyncio.CancelledError):
- await reader
- async def host_terminal(ws: WebSocket) -> None:
- """Called only after both the existing WS gate and host policy have passed."""
- from hermes_cli.pty_session import WS_CLOSE_PROCESS_EXITED, WS_CLOSE_PROCESS_EXITED_REASON
- from hermes_cli.web_server_chat import _pty_fail
- session = None
- try:
- registry = get_host_terminals(ws.app)
- admitted = await _admit(ws, registry)
- if admitted is None:
- return
- session, owner = admitted
- token = session.key
- reconnected = ws.query_params.get("attach") is not None
- def initial_text(truncated: bool) -> str:
- # Says whether the replay that follows leaves out older output.
- return _metadata(token, owner, registry, reconnected=reconnected, truncated=truncated)
- if not await session.attach(ws, initial_text=initial_text):
- if session._ws is None:
- await _close_ws(ws, WS_CLOSE_VIEWER_STALLED, WS_CLOSE_VIEWER_STALLED_REASON)
- return
- await _pump_input(ws, registry, token, session, owner)
- except TerminalDenied:
- await ws.close(code=4403, reason="Terminal belongs to another identity or profile")
- except TerminalExited:
- # The same close an attached tab gets when its shell exits: the tab closes too.
- await ws.close(code=WS_CLOSE_PROCESS_EXITED, reason=WS_CLOSE_PROCESS_EXITED_REASON)
- except (TerminalExpired, FileNotFoundError, HTTPException):
- await ws.close(code=4410, reason="Terminal or profile expired; open a new terminal")
- except RegistryFull:
- await ws.close(code=1013, reason="Terminal capacity reached; close a terminal and retry")
- except WebSocketDisconnect:
- pass
- except (OSError, RuntimeError) as exc:
- await _pty_fail(ws, exc, surface="Terminal")
- finally:
- if session is not None:
- session.detach(ws)
|