gateway_client.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. """Transport-only RPC client. Closing a viewer never stops its authority."""
  2. from __future__ import annotations
  3. import asyncio
  4. from collections import deque
  5. from contextlib import asynccontextmanager, contextmanager, suppress
  6. import contextvars
  7. import json
  8. import os
  9. from pathlib import Path
  10. import time
  11. # The owner's protocol name; session_contract is dependency-free (no config/plugins load).
  12. from gateway.session_contract import CANONICAL_GATEWAY_PROTOCOL as GATEWAY_WS_PROTOCOL
  13. GATEWAY_WS_TICKET_PREFIX = "hermes-gateway-ticket."
  14. # Per-request answer budget. Owner verbs are admissions/reads that answer promptly; the one that
  15. # legitimately runs long passes its own budget (``rpc(..., _timeout=...)``).
  16. DEFAULT_RPC_TIMEOUT = 30
  17. # A typed ``--api-key`` on a resume that the owner does not bind (``rebind_launch_key``): every CLI says this.
  18. RESUME_KEY_MISMATCH = ("--api-key on resume must be the key this session was launched with "
  19. "(a session launched without --api-key keeps its profile credentials). "
  20. "A session never switches keys: start a new one (without --resume) to use a different key.")
  21. # Canonical ``/compress`` answers only after the summary call(s) and the commit: a long session's
  22. # summary routinely takes more than 30 s (``auxiliary.compression.timeout`` defaults to 120 s per
  23. # request, ``compression.context_total_ceiling_seconds`` to 600 s). Same budget as Desktop's
  24. # ``SESSION_COMPRESS_TIMEOUT_MS`` so every surface gives up at the same point.
  25. COMPRESS_RPC_TIMEOUT = 660
  26. # A full event queue parks the socket reader; a consumer that takes no event for this long has
  27. # stalled and is failed. Same bound as the owner's own send deadline (``tui_gateway/ws.py``).
  28. EVENT_STALL_TIMEOUT = 30
  29. # Events read past a full queue while an RPC answer is awaited (the owner sends a re-attached
  30. # session's frames ahead of its ``session.resume`` answer, and a recovery RPC is awaited from the
  31. # queue's only consumer). Bounded: past either cap the reader waits for the consumer again.
  32. EVENT_OVERFLOW_FRAMES = 16384
  33. EVENT_OVERFLOW_BYTES = 8 * 1024 * 1024
  34. def gateway_ws_target(endpoint, ticket):
  35. """``(url, subprotocols)`` for dialing the owner's canonical WebSocket with a session ticket.
  36. The owner accepts exactly one ticket subprotocol beside ``GATEWAY_WS_PROTOCOL`` and echoes the
  37. latter; callers check ``ws.subprotocol == GATEWAY_WS_PROTOCOL`` after the upgrade."""
  38. url = endpoint.api_origin.replace("https:", "wss:").replace("http:", "ws:") + "/api/ws"
  39. return url, [GATEWAY_WS_PROTOCOL, GATEWAY_WS_TICKET_PREFIX + ticket]
  40. class GatewayClientError(ValueError):
  41. pass
  42. DISCONNECTED = ("Gateway disconnected; turn outcome is unknown. "
  43. "Resume the printed session ID to check whether it completed or was interrupted; do not resend the work.")
  44. # A request refused before it reached any socket: nothing is in doubt, so never DISCONNECTED.
  45. NOT_SENT = "Gateway disconnected before the request was sent; nothing was sent."
  46. class GatewayRPCError(GatewayClientError):
  47. """A received owner refusal, distinct from an ambiguous transport failure.
  48. ``str(exc)`` is the bounded reason code; ``code`` is the frame's numeric JSON-RPC code (``None``
  49. when absent or not an int) and ``data`` its bounded, scalar-only ``error.data`` mapping."""
  50. def __init__(self, reason, code=None, data=None):
  51. super().__init__(reason)
  52. self.code = code
  53. self.data = data or {}
  54. def _bounded_reason(value):
  55. return value if isinstance(value, str) and value.replace("_", "").isalnum() and len(value) <= 80 else None
  56. def _bounded_error_data(data):
  57. """At most 16 short keys whose values are bool/int/None, short strings or short lists of them;
  58. anything else (nested objects, raw remote diagnostics) is dropped, never forwarded."""
  59. def scalar(value):
  60. return value is None or isinstance(value, (bool, int)) or (isinstance(value, str) and len(value) <= 200)
  61. if not isinstance(data, dict):
  62. return {}
  63. out = {}
  64. for key, value in data.items():
  65. if len(out) >= 16 or not isinstance(key, str) or len(key) > 64:
  66. continue
  67. if scalar(value):
  68. out[key] = value
  69. elif isinstance(value, list) and len(value) <= 16 and all(scalar(item) for item in value):
  70. out[key] = list(value)
  71. return out
  72. def rpc_error(error):
  73. """The ``GatewayRPCError`` for one received ``error`` member: only bounded reason codes, never
  74. raw remote diagnostics. A free-text message (``unknown method: x``) falls back to the bounded
  75. ``data.reason`` the owner sends beside it, then to ``request_failed``."""
  76. error = error if isinstance(error, dict) else {}
  77. data = _bounded_error_data(error.get("data"))
  78. code = error.get("code")
  79. reason = _bounded_reason(error.get("message", "request_failed")) or _bounded_reason(data.get("reason"))
  80. return GatewayRPCError(reason or "request_failed",
  81. code if isinstance(code, int) and not isinstance(code, bool) else None, data)
  82. class GatewayUnavailableError(GatewayClientError):
  83. """No ready owner to dial: ``state`` is ``ensure_gateway_runtime``'s verdict (``draining`` while an
  84. update holds the install, ``starting`` past the deadline, ...). Nothing was submitted."""
  85. def __init__(self, message, state, reason_code=None):
  86. super().__init__(message)
  87. self.state, self.reason_code = state, reason_code
  88. class EventQueue(asyncio.Queue):
  89. """The client's bounded event queue plus an ordered overflow behind it: each ``get`` refills the
  90. queue from the overflow, so items arrive in read order and ``qsize``/``full`` see only the queue."""
  91. def __init__(self, maxsize):
  92. super().__init__(maxsize)
  93. self.overflow = deque() # (size, item)
  94. self.overflow_bytes = 0
  95. self.moved = asyncio.Event() # an item left the overflow (or an RPC started waiting)
  96. def _get(self):
  97. item = super()._get()
  98. if self.overflow:
  99. size, held = self.overflow.popleft()
  100. self.overflow_bytes -= size
  101. super()._put(held)
  102. self.moved.set()
  103. return item
  104. def hold(self, item, size=0):
  105. """Queue *item* behind everything already read, never dropping it."""
  106. if not self.overflow:
  107. try:
  108. self.put_nowait(item)
  109. return
  110. except asyncio.QueueFull:
  111. pass
  112. self.overflow.append((size, item))
  113. self.overflow_bytes += size
  114. def put_behind(events, item):
  115. """Queue a local item (ACP's post-ack recovery) behind every frame read so far; never dropped."""
  116. if isinstance(events, EventQueue):
  117. events.hold(item)
  118. else:
  119. events.put_nowait(item)
  120. class GatewayClient:
  121. def __init__(self, websocket):
  122. self.websocket = websocket
  123. self.events = EventQueue(maxsize=4096)
  124. self.pending = {}
  125. self.sequence = 0
  126. self.reader = None
  127. def _read_on(self):
  128. """Keep reading past a full queue: nothing is held, or an awaited answer may sit behind the
  129. held frames (its waiter may be the queue's only consumer) and the overflow has room."""
  130. events = self.events
  131. return not events.overflow or (
  132. len(events.overflow) < EVENT_OVERFLOW_FRAMES and events.overflow_bytes < EVENT_OVERFLOW_BYTES
  133. and any(not future.done() for future in self.pending.values()))
  134. async def _deliver(self, frame, size):
  135. self.events.hold(frame, size)
  136. if self._read_on():
  137. return
  138. # Frames the socket already buffered (a CLI resumed from Ctrl-Z) are read without yielding,
  139. # so the consumer has not had a turn yet: wait for it (or for an RPC that needs the reader).
  140. # Not wait_for: on 3.11 it swallows a close that lands as the wait completes.
  141. async with asyncio.timeout(EVENT_STALL_TIMEOUT):
  142. while not self._read_on():
  143. self.events.moved.clear()
  144. await self.events.moved.wait()
  145. async def __aenter__(self):
  146. self.reader = asyncio.create_task(self._read())
  147. return self
  148. async def __aexit__(self, *exc):
  149. self.reader.cancel()
  150. with suppress(asyncio.CancelledError):
  151. await self.reader
  152. async def _read(self):
  153. from websockets.exceptions import ConnectionClosed
  154. try:
  155. async for raw in self.websocket:
  156. frame = json.loads(raw)
  157. future = self.pending.get(frame.get("id"))
  158. if future is not None and not future.done():
  159. if "error" in frame:
  160. future.set_exception(rpc_error(frame["error"]))
  161. else:
  162. future.set_result(frame.get("result", {}))
  163. elif "method" in frame:
  164. await self._deliver(frame, len(raw))
  165. except (ConnectionClosed, OSError, ValueError, TimeoutError):
  166. pass
  167. finally:
  168. error = GatewayClientError(DISCONNECTED)
  169. for future in self.pending.values():
  170. if not future.done():
  171. future.set_exception(error)
  172. # A stalled consumer is failed rather than silently losing terminal events: behind
  173. # whatever is held, or in place of the oldest queued frame when nothing is.
  174. if self.events.full() and not self.events.overflow:
  175. self.events.get_nowait()
  176. self.events.hold(error)
  177. async def rpc(self, method, *, _timeout=None, **params):
  178. """Send one request and await its answer; ``_timeout`` (seconds, never sent on the wire)
  179. overrides ``DEFAULT_RPC_TIMEOUT`` for a verb that legitimately takes longer (``math.inf``: no
  180. deadline; the answer, a hang-up or the caller's cancel ends the wait)."""
  181. from websockets.protocol import State
  182. if self.websocket.state is not State.OPEN or (self.reader is not None and self.reader.done()):
  183. # A closing socket (a keepalive timeout fails it, then waits up to close_timeout for the
  184. # peer) is never written to, and a reader that ended (closed socket, bad frame, stalled
  185. # consumer) could not read the answer: refuse before writing, so the caller knows nothing left.
  186. raise GatewayClientError(NOT_SENT)
  187. self.sequence += 1
  188. rid = self.sequence
  189. future = asyncio.get_running_loop().create_future()
  190. self.pending[rid] = future
  191. self.events.moved.set() # a reader waiting on the consumer reads on toward this answer
  192. from websockets.exceptions import ConnectionClosed
  193. try:
  194. try:
  195. await self.websocket.send(json.dumps({"jsonrpc": "2.0", "id": rid, "method": method, "params": params}))
  196. except (ConnectionClosed, OSError) as exc:
  197. # The frame may have left before the socket died: the same unknown outcome as a lost answer.
  198. raise GatewayClientError(DISCONNECTED) from exc
  199. budget = DEFAULT_RPC_TIMEOUT if _timeout is None else _timeout
  200. return await asyncio.wait_for(future, None if budget == float("inf") else budget)
  201. finally:
  202. self.pending.pop(rid, None)
  203. if future.done() and not future.cancelled():
  204. future.exception() # retrieved, so a failure nobody awaited is never logged as unretrieved
  205. def private_control_query(home: Path, request: bytes, timeout: float = 5) -> bytes:
  206. """Send one request line to *home*'s gateway control socket as this OS user; the raw answer.
  207. The socket authenticates the peer (same user only), which is what lets a private verb answer."""
  208. if os.name == "nt":
  209. from gateway.runtime_bootstrap_windows import query_runtime_control
  210. return query_runtime_control(home, request, timeout)
  211. from hermes_cli.gateway_runtime_discovery import connect_private
  212. deadline = time.monotonic() + timeout
  213. with connect_private(home, timeout) as peer:
  214. peer.sendall(request)
  215. data = bytearray()
  216. while b"\n" not in data:
  217. budget = deadline - time.monotonic()
  218. if budget <= 0:
  219. raise GatewayClientError("Gateway bootstrap timed out")
  220. peer.settimeout(budget)
  221. chunk = peer.recv(4096)
  222. if not chunk or len(data) + len(chunk) > 65536:
  223. raise GatewayClientError("Invalid gateway bootstrap response")
  224. data.extend(chunk)
  225. return bytes(data)
  226. # An owner that predates ``restart_wait`` refuses the unknown parameter with exactly this answer.
  227. _UNKNOWN_TICKET_PARAMS = "PermissionError: invalid bootstrap parameters"
  228. def _session_ticket(home: Path, endpoint, *, purpose="interactive", scope=None, subject=None,
  229. refuse_restart_wait=False) -> str:
  230. from hermes_cli.gateway_runtime import control_home_for
  231. from hermes_cli.gateway_runtime_discovery import _identify_response
  232. # A served secondary's ticket is minted by the multiplexer's socket, bound to the secondary.
  233. home = control_home_for(home, endpoint)
  234. # ``subject`` names the principal a same-user caller acts for (the standalone dashboard's
  235. # authenticated user) instead of the socket UID; the owner's peer gate stays the boundary.
  236. # ``restart_wait``: the owner refuses the socket at admission if a restart wait has opened by then.
  237. params = {"profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "purpose": purpose,
  238. **({"scope": scope} if scope else {}), **({"subject": subject} if subject else {}),
  239. **({"restart_wait": "refuse"} if refuse_restart_wait else {})}
  240. request = json.dumps({"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n"
  241. answer = private_control_query(home, request)
  242. refused = json.loads(answer.split(b"\n", 1)[0]) if refuse_restart_wait else None
  243. if isinstance(refused, dict) and refused.get("error") == _UNKNOWN_TICKET_PARAMS:
  244. # An owner older than this client (the code on disk was updated under it) cannot hold the
  245. # policy; refusing it outright would strand the caller until it restarts. The pre-dial check stands.
  246. del params["restart_wait"]
  247. answer = private_control_query(home, json.dumps(
  248. {"protocol": 1, "id": 1, "verb": "session-ticket", "params": params}).encode() + b"\n")
  249. grant = _identify_response(answer)
  250. if any(grant.get(k) != v for k, v in {
  251. "profile_id": endpoint.profile_id, "instance_id": endpoint.instance_id, "runtime_protocol": 1,
  252. }.items()) or not isinstance(grant.get("ticket"), str) or not grant["ticket"]:
  253. raise GatewayClientError("Gateway bootstrap identity changed; retry launch")
  254. return grant["ticket"]
  255. # A restart's after-turn wait admits clients so a human can answer a parked prompt (#139041). A machine
  256. # poller held in it (a Kanban worker reading its card's receipt) keeps the turn the wait is waiting
  257. # on alive, so the restart, and ``hermes update`` behind it, is held until its cap.
  258. _REFUSE_RESTART_WAIT: contextvars.ContextVar[bool] = contextvars.ContextVar("refuse_restart_wait", default=False)
  259. @contextmanager
  260. def outside_restart_wait():
  261. """``connect_gateway`` calls inside never attach to an owner in its restart's after-turn wait."""
  262. token = _REFUSE_RESTART_WAIT.set(True)
  263. try:
  264. yield
  265. finally:
  266. _REFUSE_RESTART_WAIT.reset(token)
  267. @asynccontextmanager
  268. async def connect_gateway(endpoint=None, subject=None, *, max_size=8 * 1024 * 1024):
  269. """Attach to this home's owner. ``endpoint``: an already-discovered ready owner to dial as-is
  270. (maintenance that must never start a gateway); default ensures one, starting it if needed.
  271. ``subject``: mint the ticket for this principal (the caller's authenticated dashboard user)
  272. rather than the socket identity; only a peer the owner already trusts may name one.
  273. ``max_size``: the largest frame accepted; a ``session.resume`` answer carries the whole history.
  274. Inside ``outside_restart_wait()`` an owner in a restart's after-turn wait is refused as
  275. ``draining`` (``restart_wait``) instead of attached, and the ticket carries that policy so the
  276. owner also refuses the socket when the wait opens between discovery and the dial."""
  277. from websockets.asyncio.client import connect
  278. from hermes_constants import get_hermes_home
  279. from hermes_cli.gateway_runtime import attachable, ensure_gateway_runtime, gateway_profile_home
  280. from hermes_cli.gateway_runtime_discovery import DiscoveryError
  281. from urllib.parse import urlsplit
  282. remote = "" if endpoint is not None else os.environ.get("HERMES_TUI_GATEWAY_URL", "").strip()
  283. protocols = None
  284. if endpoint is not None:
  285. ticket = await asyncio.to_thread(_session_ticket, gateway_profile_home(get_hermes_home()), endpoint,
  286. subject=subject)
  287. url, protocols = gateway_ws_target(endpoint, ticket)
  288. elif remote:
  289. parsed = urlsplit(remote)
  290. if parsed.scheme not in {"ws", "wss"} or not parsed.hostname:
  291. raise GatewayClientError("Invalid explicit gateway WebSocket URL; no local fallback")
  292. url = remote
  293. else:
  294. # A per-task overlay home is served by the profile its symlinked lock names (#138949).
  295. home = gateway_profile_home(get_hermes_home())
  296. # A client's own start: the daemon it may launch ends itself once idle (no client, adapter,
  297. # cron/kanban work or admission), and the next client starts a fresh one.
  298. result = await asyncio.to_thread(ensure_gateway_runtime, home, idle_exit=True)
  299. if not attachable(result) or (_REFUSE_RESTART_WAIT.get() and result.state != "ready"):
  300. detail = getattr(result, "detail", None)
  301. if result.reason_code == "runtime_exited" and detail:
  302. # A multi-line startup report (redacted traceback tail + where the full log is).
  303. raise GatewayUnavailableError(f"Gateway could not start: {detail}", result.state, result.reason_code)
  304. detail = f" ({detail})" if detail else ""
  305. raise GatewayUnavailableError(f"Gateway {result.state}: {result.reason_code or 'not_ready'}{detail}",
  306. result.state, result.reason_code)
  307. endpoint = result.endpoint
  308. try:
  309. policy: dict = {"refuse_restart_wait": True} if _REFUSE_RESTART_WAIT.get() else {}
  310. ticket = await asyncio.to_thread(_session_ticket, home, endpoint, **policy)
  311. except DiscoveryError as exc:
  312. # The control socket refused the ticket: the verdict discovery gives the same refusal.
  313. detail = exc.bounded_detail(500)
  314. raise GatewayUnavailableError(f"Gateway inaccessible: {exc.reason}" + (f" ({detail})" if detail else ""),
  315. "inaccessible", exc.reason) from exc
  316. url, protocols = gateway_ws_target(endpoint, ticket)
  317. try:
  318. # The gateway is a loopback (or explicitly named) peer, never something to route through the
  319. # user's HTTP(S) proxy; websockets>=14 reads HTTP_PROXY/HTTPS_PROXY by default and a proxy that
  320. # cannot reach 127.0.0.1 turns every launch into a 10 s open timeout.
  321. async with connect(url, subprotocols=protocols, open_timeout=10, max_size=max_size,
  322. proxy=None) as ws:
  323. if protocols and ws.subprotocol != GATEWAY_WS_PROTOCOL:
  324. raise GatewayClientError("Gateway protocol mismatch; update/restart required")
  325. async with GatewayClient(ws) as client:
  326. yield client
  327. except GatewayClientError:
  328. raise
  329. except (OSError, TimeoutError) as exc:
  330. raise GatewayClientError("Gateway connection failed; no local fallback") from exc