display.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264
  1. """``/api/display/ws`` — raw RFB over WebSocket for the Bot Desktop viewer.
  2. The Desktop renderer calls ``display.observe`` on its authenticated ``/api/ws`` connection, gets a
  3. single-use 30 s ticket pinned to that profile's RFB socket, then opens this route with
  4. ``?display_ticket=``. No websockify, no new port: the bridge splices the profile's 0600 Unix socket
  5. into the WebSocket as binary frames with backpressure both ways, and runs the client stream through
  6. :class:`tools.bot_desktop.rfb_filter.RfbClientFilter` so keyboard, pointer and clipboard reach Xvnc
  7. only from the viewer that currently holds the lease. noVNC's ``viewOnly`` is UX; this is the gate.
  8. A lease change closes the evicted viewer's socket with 4000 ``control-taken`` so its UI drops back to
  9. Watch mode and reconnects. A viewer a browser tab asked for signs out with that tab: ``hermes webapp
  10. --sign-out`` closes it with 4401, and its ticket opens nothing afterwards.
  11. Design notes. The ticket rides in the query string on purpose: noVNC's Websock owns the socket and
  12. cannot negotiate a subprotocol or add a header, and the ticket is single-use and expires in 30 s, so
  13. a logged URL is spent by the time anyone reads it. The bridge's input gate re-reads ``lease.json`` at
  14. most every ``_LEASE_REFRESH_S`` (250 ms) between in-process ``on_change`` callbacks: that bounds how
  15. long another PROCESS's takeover can go unnoticed, and is the price of not stat-ing the lease file on
  16. the event loop for every pointer move.
  17. """
  18. from __future__ import annotations
  19. import asyncio
  20. import logging
  21. from pathlib import Path
  22. from typing import Optional
  23. from fastapi import APIRouter, WebSocket, WebSocketDisconnect
  24. from hermes_cli.web_server_chat import _ws_request_is_allowed
  25. _log = logging.getLogger(__name__)
  26. router = APIRouter()
  27. _ACTIVITY_STAMP_S = 60.0
  28. _READ_CHUNK = 64 * 1024
  29. _CLOSE_CONTROL_TAKEN = 4000
  30. _CLEAN_CLOSE = frozenset({1000, 1001})
  31. _CLOSE_DESKTOP_GONE = 4001
  32. _CLOSE_BAD_TICKET = 4401
  33. _CLOSE_NOT_ALLOWED = 4403
  34. _CLOSE_PROTOCOL = 1003
  35. _LEASE_REFRESH_S = 0.25
  36. def _should_evict(held: dict, lease, viewer_id: str) -> bool:
  37. """A viewer that held control during this connection and lost it to ANOTHER human is kicked so its
  38. UI repaints; a plain hand-back to the agent, pure watchers and the new holder stay connected. The
  39. hand-back also forgets that this viewer ever held: after it, they are a plain watcher again and a
  40. later takeover by someone else must not evict them. ``held`` is the per-connection memory."""
  41. from tools.bot_desktop import lease as _lease
  42. if lease.holder != _lease.HUMAN:
  43. held["ever"] = False
  44. return False
  45. if lease.viewer_id == viewer_id:
  46. held["ever"] = True
  47. return False
  48. return bool(held["ever"])
  49. def _consume_display_ticket(ws: WebSocket) -> Optional[dict]:
  50. from hermes_cli.dashboard_auth.ws_tickets import TicketInvalid, consume_ticket
  51. ticket = ws.query_params.get("display_ticket", "")
  52. if not ticket:
  53. return None
  54. try:
  55. info = consume_ticket(ticket)
  56. except TicketInvalid:
  57. return None
  58. if info.get("provider") != "bot-desktop" or not info.get("hermes_home"):
  59. return None
  60. return info
  61. @router.websocket("/api/display/ws")
  62. async def display_ws(ws: WebSocket) -> None:
  63. # Host/Origin/client-IP policy is refused BEFORE accept, like every other dashboard route: a
  64. # cross-origin page never gets a completed handshake. Everything after that (ticket, desktop
  65. # state) is refused AFTER accept so the code + reason arrive in a close frame — a close before
  66. # accept surfaces in the browser as an opaque HTTP 403 and the renderer cannot tell "re-observe"
  67. # (4401) from "screen is gone" (4001).
  68. if not _ws_request_is_allowed(ws):
  69. await ws.close(code=_CLOSE_NOT_ALLOWED)
  70. return
  71. await ws.accept()
  72. info = _consume_display_ticket(ws)
  73. if info is None:
  74. await ws.close(code=_CLOSE_BAD_TICKET, reason="display ticket missing, expired or used")
  75. return
  76. if _signed_out(ws, info):
  77. await ws.close(code=_CLOSE_BAD_TICKET, reason="Signed out")
  78. return
  79. await _bridge(ws, info)
  80. def _signed_out(ws: WebSocket, info: dict) -> bool:
  81. """True when the browser tab that asked for this viewer has signed out since (the ticket's
  82. browser credential generation is no longer current); otherwise the viewer is tracked with that
  83. tab's sockets, so signing out closes it too. A ticket a native client minted carries no browser
  84. credential and is left alone."""
  85. from hermes_cli.browser_credential import is_current
  86. from hermes_cli.web_session_sockets import track
  87. admitted = info.get("admitted_with")
  88. if admitted is None:
  89. return False
  90. if not is_current(admitted):
  91. return True
  92. track(ws, admitted)
  93. return False
  94. async def _open_rfb(profile_home: Path):
  95. """``(reader, writer, relay)`` for THIS profile's Xvnc. Gateway-hosted screen: its unix socket. Screen inside
  96. the terminal backend: a ``docker exec`` / ``ssh`` relay whose stdio IS the RFB stream (``relay`` is that
  97. Popen; None for a socket). Raises OSError when nothing is running."""
  98. from hermes_constants import reset_hermes_home_override, set_hermes_home_override
  99. from tools.bot_desktop import runtime as _bd_runtime
  100. token = set_hermes_home_override(profile_home)
  101. try:
  102. if _bd_runtime.sandbox_screen_running():
  103. relay = _bd_runtime.open_rfb_stream()
  104. else:
  105. sock = profile_home / "bot-desktop" / "rfb.sock"
  106. if not sock.exists():
  107. raise OSError("rfb.sock missing")
  108. reader, writer = await asyncio.open_unix_connection(str(sock))
  109. return reader, writer, None
  110. finally:
  111. reset_hermes_home_override(token)
  112. loop = asyncio.get_running_loop()
  113. reader = asyncio.StreamReader()
  114. await loop.connect_read_pipe(lambda: asyncio.StreamReaderProtocol(reader), relay.stdout)
  115. # StreamReaderProtocol (not the bare FlowControlMixin) so ``writer.wait_closed()`` in the bridge's teardown has
  116. # a close waiter to await; on the bare mixin it raises NotImplementedError.
  117. w_transport, w_protocol = await loop.connect_write_pipe(lambda: asyncio.StreamReaderProtocol(asyncio.StreamReader()), relay.stdin)
  118. writer = asyncio.StreamWriter(w_transport, w_protocol, None, loop)
  119. return reader, writer, relay
  120. async def _bridge(ws: WebSocket, info: dict) -> None:
  121. """Pump RFB bytes between the viewer socket (already accepted) and THIS profile's Xvnc, gated by
  122. the lease."""
  123. from hermes_constants import hermes_home_key
  124. from tools.bot_desktop import lease as _lease
  125. from tools.bot_desktop.rfb_filter import RfbClientFilter
  126. profile_home = str(info["hermes_home"])
  127. profile_key = hermes_home_key(profile_home)
  128. viewer_id = str(info.get("viewer_id") or info.get("user_id") or "viewer")
  129. try:
  130. reader, writer, relay = await _open_rfb(Path(profile_home))
  131. except OSError as exc:
  132. _log.warning("display ws: cannot reach RFB endpoint for %s: %s", profile_home, exc)
  133. await ws.close(code=_CLOSE_DESKTOP_GONE, reason="Bot Desktop is not running")
  134. return
  135. loop = asyncio.get_running_loop()
  136. evicted = asyncio.Event()
  137. held = {"ever": _lease.viewer_may_send_input(viewer_id, profile_key=profile_home)}
  138. # Input gate cache: reading lease.json per client message (a stat + read on the event loop for
  139. # every pointer move) is replaced by a decision refreshed on this process's on_change callback
  140. # and by a file re-read at most every _LEASE_REFRESH_S, so another process's takeover still lands.
  141. allowed = {"input": held["ever"], "at": loop.time()}
  142. def _refresh_allowed(lease=None) -> None:
  143. if lease is None:
  144. lease = _lease.get(profile_key=profile_home)
  145. allowed["input"] = lease.holder == _lease.HUMAN and lease.viewer_id == viewer_id
  146. allowed["at"] = loop.time()
  147. def _may_send_input() -> bool:
  148. if loop.time() - allowed["at"] > _LEASE_REFRESH_S:
  149. _refresh_allowed()
  150. return allowed["input"]
  151. def _on_lease(key: str, lease) -> None:
  152. if key != profile_key:
  153. return
  154. loop.call_soon_threadsafe(_refresh_allowed, lease)
  155. if _should_evict(held, lease, viewer_id):
  156. loop.call_soon_threadsafe(evicted.set)
  157. unsubscribe = _lease.on_change(_on_lease)
  158. rfb_filter = RfbClientFilter(_may_send_input)
  159. viewer_closed = asyncio.Event()
  160. activity_file = Path(profile_home) / "bot-desktop" / "activity"
  161. stamped = {"at": 0.0}
  162. def _stamp_activity() -> None:
  163. # An attached viewer is use: the idle auto-stop must not take a screen someone is watching.
  164. if loop.time() - stamped["at"] < _ACTIVITY_STAMP_S:
  165. return
  166. stamped["at"] = loop.time()
  167. try:
  168. activity_file.touch()
  169. except OSError:
  170. pass
  171. async def rfb_to_ws() -> None:
  172. while True:
  173. chunk = await reader.read(_READ_CHUNK)
  174. if not chunk:
  175. return
  176. _stamp_activity()
  177. await ws.send_bytes(chunk) # awaiting the send is the backpressure toward Xvnc
  178. async def ws_to_rfb() -> None:
  179. while True:
  180. message = await ws.receive()
  181. if message.get("type") == "websocket.disconnect":
  182. # 1000/1001 = the viewer closed the window; anything else is a dropped link.
  183. if message.get("code") in _CLEAN_CLOSE:
  184. viewer_closed.set()
  185. return
  186. data = message.get("bytes")
  187. if data is None:
  188. await ws.close(code=_CLOSE_PROTOCOL, reason="RFB is binary")
  189. return
  190. try:
  191. allowed = rfb_filter.feed(data)
  192. except ValueError as exc:
  193. await ws.close(code=_CLOSE_PROTOCOL, reason=str(exc)[:100])
  194. return
  195. if allowed:
  196. writer.write(allowed)
  197. await writer.drain() # backpressure toward the browser
  198. async def watch_eviction() -> None:
  199. await evicted.wait()
  200. await ws.close(code=_CLOSE_CONTROL_TAKEN, reason="control-taken")
  201. tasks = [asyncio.create_task(rfb_to_ws()), asyncio.create_task(ws_to_rfb()),
  202. asyncio.create_task(watch_eviction())]
  203. try:
  204. done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
  205. for t in pending:
  206. t.cancel()
  207. # Cancelled pumps must finish before we tear down the socket they hold, or they outlive the
  208. # bridge on the loop (a viewer reconnecting in a loop piled them up).
  209. await asyncio.gather(*pending, return_exceptions=True)
  210. for t in done:
  211. exc = t.exception()
  212. if exc and not isinstance(exc, (WebSocketDisconnect, ConnectionError)):
  213. _log.debug("display ws ended: %r", exc)
  214. finally:
  215. unsubscribe()
  216. writer.close()
  217. try:
  218. await writer.wait_closed()
  219. except OSError: # Xvnc already gone (ECONNRESET / EPIPE on the FIN)
  220. pass
  221. if relay is not None:
  222. relay.kill() # the exec client; Xvnc inside the sandbox stays up
  223. # Closing the viewer window hands control back. A DROPPED link (laptop lid, Wi-Fi, 1006)
  224. # keeps the human's exclusion: they may be mid-login on that screen and the agent must not
  225. # resume into it. The Desktop reconnects into the same lease, or the human hands back.
  226. if viewer_closed.is_set() and _lease.viewer_may_send_input(viewer_id, profile_key=profile_home):
  227. _lease.release(viewer_id, profile_key=profile_home)
  228. try:
  229. await ws.close()
  230. except Exception: # already closed by the peer or by an eviction
  231. pass