local_desktop_ui.py 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289
  1. """The desktop window bridges for a local viewer's turn on the session authority.
  2. Main's TUI sidecar wired two things per session (``tui_gateway/session_notifications.py::
  3. _wire_desktop_sinks`` and ``tui_gateway/agent_callbacks.py::_agent_cbs``):
  4. * the ``tools.desktop_ui`` emitter, so ``desktop_preview`` open/close, ``focus_pane``,
  5. ``close_terminal`` and the other window events reach the renderer as ``event`` frames;
  6. * the GUI callbacks (``read_preview_callback``, ``drive_preview_callback``,
  7. ``read_terminal_callback``, ``read_window_below_callback``, ``theme_callback``, ``tour_callback``,
  8. ``setup_beat_callback``), answered by the renderer through ``preview.read`` / ``preview.act`` /
  9. ``terminal.read`` / ``window.read`` / ``theme.apply`` / ``theme.pick`` / ``tour`` /
  10. ``setup_beat`` server requests;
  11. * the process-registry sinks (``on_output`` / ``on_close``) that stream a background process into
  12. its read-only terminal tab (``agent.terminal.output``) and let ``close_terminal`` drop the tab
  13. (``terminal.close``).
  14. On the session authority none was installed, so every ``desktop_ui`` tool the session policy
  15. still offers a GUI session failed with "only available in the Hermes desktop app". Here the
  16. emitter and the process sinks are installed once per process; the emitter routes each event to the
  17. turn that raised it (by the turn's bound ``HERMES_SESSION_ID``) through the generation-fenced
  18. execution stream, and the process sinks route by the process's ``session_key`` to the owning
  19. session's event stream (a background process outlives the turn that started it). The callbacks
  20. are installed per turn on the cached agent and restored with the other turn sinks.
  21. Card for ``manage_connections``: ``connection_callback`` publishes ``connection.request`` on the turn, and each later change of an operation is published as ``connection.update`` on the session that owns it (looked up by its session key, the route); the card answers through ``connection.respond`` (``gateway/session_connections.py``).
  22. """
  23. from __future__ import annotations
  24. import json
  25. import logging
  26. import threading
  27. import time
  28. logger = logging.getLogger(__name__)
  29. # Agent attributes ``wire`` installs for one local-viewer turn (restored by ``local_viewer_events``).
  30. CALLBACK_ATTRS = ("read_terminal_callback", "read_preview_callback", "drive_preview_callback",
  31. "read_window_below_callback", "theme_callback", "tour_callback", "setup_beat_callback",
  32. "connection_callback")
  33. # main: tui_gateway/agent_callbacks.py ``_read_block`` budgets and tui_gateway/server.py's preview.act ladder.
  34. _TERMINAL_READ_TIMEOUT_S = 30
  35. _PREVIEW_READ_TIMEOUT_S = 45
  36. _WINDOW_READ_TIMEOUT_S = 30
  37. _PREVIEW_ACTION_TIMEOUT_S = 45
  38. _PREVIEW_ACTION_PROBE_TIMEOUT_S = 10
  39. _PREVIEW_ACTION_REPROBE_COOLDOWN_S = 30
  40. _PREVIEW_ACTION_BRIDGE_UNAVAILABLE = json.dumps({
  41. "success": False,
  42. "error": ("No Hermes Desktop window answered the preview action request. The drive_preview / "
  43. "annotate_preview bridge is served by the desktop app's renderer, which updates "
  44. "separately from this backend, so an app build older than the tool has nothing "
  45. "listening. Update the Hermes Desktop app, open a page with open_preview, and try "
  46. "again in this session after a short cooldown.")})
  47. # main: tui_gateway/theme_bridge.py and tui_gateway/server.py's tour ladder.
  48. _THEME_TIMEOUT_S = 15
  49. _THEME_PICK_TIMEOUT_S = 300
  50. _THEME_UNANSWERED = {"success": False, "error": "No Hermes Desktop window answered the theme request."}
  51. _TOUR_TIMEOUT_S = 45
  52. _TOUR_PROBE_TIMEOUT_S = 10
  53. _TOUR_BRIDGE_UNAVAILABLE = json.dumps({
  54. "success": False,
  55. "error": ("No Hermes Desktop window answered the tour request. The tour is driven by the desktop app's "
  56. "renderer, which updates separately from this backend, so an app build older than the tour tool "
  57. "has nothing listening. Update the Hermes Desktop app and start a new session. Do not retry tour "
  58. "in this session.")})
  59. _lock = threading.Lock()
  60. # turn key (the turn's bound HERMES_SESSION_ID) -> publish(event_type, payload) of the live turn
  61. _turns: dict[str, object] = {}
  62. # canonical session id -> {"state": "answered"|"unanswered", "retry_at": monotonic}
  63. _preview_bridge: dict[str, dict] = {}
  64. # canonical session id -> "answered" | "unanswered" (main: the session record's ``tour_bridge``)
  65. _tour_bridge: dict[str, str] = {}
  66. # process ``session_key`` -> (authority, canonical session id) of the session that last ran a turn with it
  67. _process_routes: dict[str, tuple] = {}
  68. _emitter_installed = False
  69. _process_sinks_installed = False
  70. def _emit(sid, event, payload) -> None:
  71. """``desktop_ui`` sink: publish *event* on the stream of the turn that raised it."""
  72. from gateway.session_context import get_session_env
  73. keys = [str(sid or ""), get_session_env("HERMES_SESSION_ID", "")]
  74. with _lock:
  75. publish = next((_turns[key] for key in keys if key and key in _turns), None)
  76. if publish is None:
  77. # Raised outside a local-viewer turn (a messaging route, a background thread): no window.
  78. raise RuntimeError("no Hermes desktop window is attached to this turn")
  79. publish(event, dict(payload or {}))
  80. def _install_emitter() -> None:
  81. global _emitter_installed
  82. if _emitter_installed:
  83. return
  84. from tools import desktop_ui
  85. desktop_ui.set_emitter(_emit)
  86. _emitter_installed = True
  87. def _publish_process(session, event_type, payload) -> None:
  88. """Publish a process event on the stream of the session that owns the process."""
  89. key = str(getattr(session, "session_key", "") or "") if session is not None else ""
  90. with _lock:
  91. route = _process_routes.get(key) if key else None
  92. if route is None:
  93. raise RuntimeError("no Hermes desktop window owns this process")
  94. authority, session_id = route
  95. live = authority.sessions.get(session_id)
  96. if live is None:
  97. raise RuntimeError("the session that owns this process is no longer live")
  98. live.event_stream.publish(session_id, payload, event_type=event_type)
  99. def _install_process_sinks() -> None:
  100. """main's ``_wire_desktop_sinks``: live output and tab close for background processes (once)."""
  101. global _process_sinks_installed
  102. if _process_sinks_installed:
  103. return
  104. from tools.process_registry import process_registry
  105. if getattr(process_registry, "on_output", None) is None:
  106. process_registry.on_output = lambda session, chunk: _publish_process(
  107. session, "agent.terminal.output", {"process_id": session.id, "chunk": chunk})
  108. if getattr(process_registry, "on_close", None) is None:
  109. process_registry.on_close = lambda session, pid: _publish_process(
  110. session, "terminal.close", {"process_id": pid})
  111. _process_sinks_installed = True
  112. def _install_connection_hook() -> None:
  113. """``connection.update`` for an operation opened by one of our turns, on the session that owns it (same session key and profile, as the sidecar matches its own); any other operation goes to the hook this one wraps (the sidecar's, for its own sessions). Re-wraps whenever another hook replaced it, so install order does not matter."""
  114. from tools.connectors.operation import ConnectionOperation
  115. previous = ConnectionOperation.on_change
  116. if getattr(previous, "_local_desktop_ui", False):
  117. return
  118. def on_change(operation, change, snapshot):
  119. from hermes_constants import hermes_home_key
  120. with _lock:
  121. route = _process_routes.get(operation.session_key)
  122. live = route[0].sessions.get(route[1]) if route is not None else None
  123. if live is None or hermes_home_key(route[0].profile_id) != operation.profile_key:
  124. if previous is not None:
  125. previous(operation, change, snapshot)
  126. return
  127. from tui_gateway.connector_payload import connector_ui_payload
  128. payload = {**snapshot, "settled": snapshot.get("settled_at") is not None, **(change or {}),
  129. "owner": {"type": "session", "session_id": route[1]}}
  130. live.event_stream.publish(route[1], connector_ui_payload(payload), event_type="connection.update")
  131. on_change._local_desktop_ui = True
  132. ConnectionOperation.on_change = staticmethod(on_change)
  133. def _ask_result(turn, agent, method, params, timeout):
  134. """One structured server request; the viewer's ``result`` dict, or None when none answered."""
  135. from gateway.session_server_requests import ask_viewer_result
  136. authority, session_id, _generation = turn._approval_owner
  137. live = authority.sessions.get(session_id)
  138. if live is None:
  139. return None
  140. return ask_viewer_result(live.event_stream, session_id, method, dict(params), timeout=timeout,
  141. cancelled=lambda: bool(getattr(agent, "_interrupt_requested", False)))
  142. def _theme(turn, agent, payload) -> dict:
  143. """main's ``theme_request``: ``theme.apply`` for values, or a ``theme.pick`` card that waits."""
  144. if "picker" not in payload:
  145. answer = _ask(turn, agent, "theme.apply", payload, _THEME_TIMEOUT_S)
  146. if not answer:
  147. return dict(_THEME_UNANSWERED)
  148. try:
  149. return json.loads(answer)
  150. except ValueError:
  151. return {"success": True, "text": answer}
  152. started = time.monotonic()
  153. result = _ask_result(turn, agent, "theme.pick", payload, _THEME_PICK_TIMEOUT_S)
  154. if result is None:
  155. return {"timed_out": True} if time.monotonic() - started >= _THEME_PICK_TIMEOUT_S else {"cancelled": True}
  156. if isinstance(result.get("picked"), dict):
  157. return {"success": True, "picked": result["picked"]}
  158. if isinstance(result.get("said"), str) and result["said"].strip():
  159. return {"said": result["said"].strip()}
  160. return {"cancelled": True}
  161. def _tour(turn, agent, payload) -> str:
  162. """main's ``_tour_request``: probe once per session, then the full deadline or fail fast."""
  163. session_id = turn._approval_owner[1]
  164. with _lock:
  165. state = _tour_bridge.get(session_id)
  166. if state == "unanswered":
  167. return _TOUR_BRIDGE_UNAVAILABLE
  168. answer = _ask(turn, agent, "tour", payload, _TOUR_TIMEOUT_S if state == "answered" else _TOUR_PROBE_TIMEOUT_S)
  169. with _lock:
  170. if answer:
  171. _tour_bridge[session_id] = "answered"
  172. elif state != "answered" and not getattr(agent, "_interrupt_requested", False):
  173. _tour_bridge[session_id] = "unanswered"
  174. return answer or _TOUR_BRIDGE_UNAVAILABLE
  175. def _setup_beat(turn, agent, payload):
  176. from agent.onboarding_flow import CARD_TIMEOUT_S
  177. return _ask_result(turn, agent, "setup_beat", payload, CARD_TIMEOUT_S)
  178. def _ask(turn, agent, method, params, timeout):
  179. """One ``ValueResult`` server request to the session's attached viewers; "" when none answered."""
  180. from gateway.session_server_requests import ask_viewer
  181. authority, session_id, _generation = turn._approval_owner
  182. live = authority.sessions.get(session_id)
  183. if live is None:
  184. return ""
  185. # Automatic window requests (reads, a preview click, the tour) are not handed to a reconnecting
  186. # viewer again: a second delivery would repeat the action.
  187. value = ask_viewer(live.event_stream, session_id, method, dict(params), timeout=timeout, replay=False,
  188. cancelled=lambda: bool(getattr(agent, "_interrupt_requested", False)))
  189. return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
  190. def _read_block(turn, agent, method, timeout):
  191. return lambda start=None, count=None: _ask(
  192. turn, agent, method, {k: v for k, v in (("start", start), ("count", count)) if v is not None}, timeout)
  193. def _preview_action(turn, agent, payload) -> str:
  194. """main's ``_preview_action_request`` ladder: a short probe until the renderer has answered once,
  195. then the full deadline; an unanswered probe fails fast for a cooldown instead of stacking waits."""
  196. session_id = turn._approval_owner[1]
  197. now = time.monotonic()
  198. with _lock:
  199. bridge = _preview_bridge.setdefault(session_id, {})
  200. state = bridge.get("state")
  201. if state == "unanswered" and now < bridge.get("retry_at", 0):
  202. return _PREVIEW_ACTION_BRIDGE_UNAVAILABLE
  203. if state == "unanswered":
  204. bridge["retry_at"] = now + _PREVIEW_ACTION_REPROBE_COOLDOWN_S
  205. answer = _ask(turn, agent, "preview.act", payload,
  206. _PREVIEW_ACTION_TIMEOUT_S if state == "answered" else _PREVIEW_ACTION_PROBE_TIMEOUT_S)
  207. with _lock:
  208. if answer:
  209. bridge["state"] = "answered"
  210. elif bridge.get("state") != "answered" and not getattr(agent, "_interrupt_requested", False):
  211. bridge["state"] = "unanswered"
  212. bridge["retry_at"] = time.monotonic() + _PREVIEW_ACTION_REPROBE_COOLDOWN_S
  213. return answer or _PREVIEW_ACTION_BRIDGE_UNAVAILABLE
  214. def wire(turn, agent):
  215. """Install the window bridges for one local-viewer turn; returns the registry key to release."""
  216. owner = getattr(turn, "_approval_owner", None)
  217. if owner is None:
  218. return None
  219. try:
  220. _install_emitter()
  221. _install_process_sinks()
  222. _install_connection_hook()
  223. except Exception: # health: allow BLE001 -- the window bridges are optional; the turn must run
  224. logger.warning("Desktop UI sink install failed", exc_info=True)
  225. key = str(getattr(turn._ctx, "session_id", "") or owner[1])
  226. publish = turn._publish_execution
  227. process_key = str(getattr(turn._ctx, "session_key", "") or "")
  228. with _lock:
  229. _turns[key] = publish
  230. if process_key:
  231. _process_routes[process_key] = (owner[0], owner[1])
  232. agent.read_terminal_callback = _read_block(turn, agent, "terminal.read", _TERMINAL_READ_TIMEOUT_S)
  233. agent.read_preview_callback = _read_block(turn, agent, "preview.read", _PREVIEW_READ_TIMEOUT_S)
  234. agent.drive_preview_callback = lambda payload: _preview_action(turn, agent, dict(payload or {}))
  235. agent.read_window_below_callback = lambda: _ask(turn, agent, "window.read", {}, _WINDOW_READ_TIMEOUT_S)
  236. agent.theme_callback = lambda payload: _theme(turn, agent, dict(payload or {}))
  237. agent.tour_callback = lambda payload: _tour(turn, agent, dict(payload or {}))
  238. agent.setup_beat_callback = lambda payload: _setup_beat(turn, agent, dict(payload or {}))
  239. # Fire-and-forget: the tool thread waits on its own operation; the card answers by op_id.
  240. agent.connection_callback = lambda payload: publish("connection.request", dict(payload)) and None
  241. return key, publish
  242. def release(token) -> None:
  243. """Drop this turn's routing entry, unless a newer turn of the same session already replaced it."""
  244. if not token:
  245. return
  246. key, publish = token
  247. with _lock:
  248. if _turns.get(key) is publish:
  249. _turns.pop(key, None)