gateway_connection.py 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497
  1. """One gateway connection per classic-CLI process (CALLMAP §3.1, §4).
  2. A daemon thread owns an asyncio loop and one ``GatewayClient``. It never calls a HermesCLI
  3. method and never blocks: every event frame goes into a thread-safe ``queue.Queue`` that the
  4. agent thread drains (``GatewayAgent.run_conversation``), and ``session.info``-class frames
  5. also update the plain-attribute ``SessionState`` caches the status bar reads.
  6. Callers on any other thread use ``call()``, which blocks on the loop's future; calling it on
  7. the loop thread itself is a deadlock and raises.
  8. """
  9. from __future__ import annotations
  10. import asyncio
  11. import logging
  12. import queue
  13. import threading
  14. import time
  15. from hermes_cli.gateway_client import (DEFAULT_RPC_TIMEOUT, DISCONNECTED, NOT_SENT, RESUME_KEY_MISMATCH,
  16. GatewayClientError, GatewayRPCError)
  17. logger = logging.getLogger(__name__)
  18. # ``session.resume`` refusals that mean "this viewer may not open that session": the caller keeps the
  19. # session it is on instead of adopting the target (startup resume, bare -c fallback).
  20. RESUME_REFUSALS = frozenset({"not_found", "permission_denied"})
  21. # Resume refusals meaning the session itself is gone for this viewer (gateway/session_controls): the
  22. # above plus profile_mismatch, raised for another profile's session (reconnect, /resume switch).
  23. RESUME_GONE = RESUME_REFUSALS | {"profile_mismatch"}
  24. # Keyed-resume refusals a plain resume answers as main did: the key does not apply to this session
  25. # (``rebind_launch_key``: keyless or non-local, another key held, another fingerprint), or the viewer
  26. # lacks the ``session:control`` a rebind needs but may still read. Any other refusal is the resume's.
  27. _KEY_NOT_TAKEN = frozenset({"admission_conflict", "permission_denied"})
  28. class CallStopped(GatewayClientError):
  29. """The caller stopped waiting (:meth:`GatewayConnection.call_until`); the request was sent."""
  30. def _resume_plain(exc, session_id, typed):
  31. """A keyed ``session.resume`` the owner refused with ``exc``: returns when the caller resumes plain,
  32. else raises. Only a refusal in ``_KEY_NOT_TAKEN`` resumes plain (a keyed startup alias on a Desktop/TUI
  33. session); any other is raised, not hidden by a keyless attach whose first turn fails. ``typed`` (a typed
  34. ``--api-key`` the user resumes with) never resumes plain: ``admission_conflict`` is the thin client's
  35. refusal."""
  36. if typed and str(exc) == "admission_conflict":
  37. raise GatewayClientError(RESUME_KEY_MISMATCH) from None
  38. if typed or str(exc) not in _KEY_NOT_TAKEN:
  39. raise exc
  40. logger.debug("session.resume %s: launch key not taken (%s); resuming without it", session_id, exc)
  41. def lost_before(frame, epoch) -> bool:
  42. """``frame`` is a transport error from a connection older than ``epoch``: a reconnect replaced it,
  43. so it is no outcome of anything submitted on ``epoch``."""
  44. lost = getattr(frame, "connection_epoch", None)
  45. return isinstance(frame, Exception) and lost is not None and epoch is not None and lost < epoch
  46. class SessionState:
  47. """What the gateway last told this viewer about one session (plain attributes, GIL-atomic
  48. reads from the prompt_toolkit render thread; never an RPC per paint)."""
  49. def __init__(self, session_id):
  50. self.session_id = session_id
  51. self.info: dict = {}
  52. self.title: str | None = None
  53. self.todo_state: dict | None = None # the session's {todos, revision}: resume snapshot, then todo.updated
  54. self.revision = None
  55. self.subscription_id = None
  56. self.execution_generation = None
  57. self.messages: list = []
  58. self.usage: dict = {}
  59. # Called (loop thread) with each ``session.usage`` payload; GatewayAgent maps it onto main's
  60. # agent attributes, so a frame landing between turns is never dropped with the turn queue.
  61. self.usage_listeners: list = []
  62. # Called (loop thread) with each ``request.cancel`` payload: the agent thread may be parked in a
  63. # modal answering that very request (another viewer settled it), so it cannot read the queue.
  64. self.request_listeners: list = []
  65. # A gateway-admitted notification turn the idle CLI queued a header for (gateway_tools_notify).
  66. self.admitted_turn: dict | None = None
  67. # The owner's pending admissions as last published (snapshot or ``session.info``), and the
  68. # ``/queue`` inputs this viewer admitted there (``admission_id -> text``), so the turn the owner
  69. # runs for one is painted like a typed turn (``gateway_tools_notify.queue_admitted_turn``).
  70. self.pending: list = []
  71. self.queued_inputs: dict = {}
  72. # Started ``/queue`` admissions waiting for their turn (``admission_id -> receipt-like``), and the
  73. # one whose item the input loop just took (``gateway_tools_notify.arm_adopted``).
  74. self.adopted: dict = {}
  75. self.armed_admission: str | None = None
  76. # Each pending list the owner published (or a snapshot carried) bumps ``pending_seq``; a
  77. # ``/queue`` admission whose ``message.start`` this viewer dropped unpainted (a local turn's
  78. # drain or pump) is in ``missed``. Both make the idle tick read its receipt (``recover_queued``).
  79. self.pending_seq = 0
  80. # ``admission_id -> pending_seq`` when this viewer queued it: the dock shows it only until the
  81. # owner publishes a list after that (``gateway_queue.published_queue``).
  82. self.queued_at: dict = {}
  83. self.recovered_seq = 0
  84. self.missed: set = set()
  85. # ``session.replay_gap``s seen, and how many a re-attach covered: the owner retired this viewer's
  86. # subscription on overflow, so its frames stop until ``GatewayConnection.reattach``.
  87. self.gaps = self.attached_gaps = 0
  88. def adopt_snapshot(self, snapshot):
  89. self.info = dict(snapshot.get("info") or {})
  90. self.revision = snapshot.get("revision", self.revision)
  91. self.subscription_id = snapshot.get("subscription_id", self.subscription_id)
  92. self.execution_generation = snapshot.get("execution_generation", self.execution_generation)
  93. self.messages = list(snapshot.get("messages") or [])
  94. if "todo_state" in snapshot: # null clears an earlier plan; a missing key (older gateway) keeps it
  95. todo_state = snapshot["todo_state"]
  96. self.todo_state = dict(todo_state) if isinstance(todo_state, dict) else None
  97. if isinstance(snapshot.get("pending"), list):
  98. self.pending = list(snapshot["pending"])
  99. self.pending_seq += 1 # a (re)attach replays no frames: the idle tick checks this viewer's /queue rows
  100. if self.info.get("title"):
  101. self.title = self.info["title"]
  102. def runs_route(self, model, provider=""):
  103. """Whether the session runs ``model`` (and ``provider``, when both sides name one)."""
  104. current, current_provider = self.info.get("model"), self.info.get("provider")
  105. return bool(current) and model == current and not (
  106. provider and current_provider and provider != current_provider)
  107. def apply_event(self, kind, payload):
  108. if kind in ("session.info", "session.updated") and isinstance(payload, dict):
  109. self.info.update({k: v for k, v in payload.items() if k != "session_id"})
  110. if isinstance(payload.get("pending"), list):
  111. self.pending = list(payload["pending"])
  112. self.pending_seq += 1
  113. if payload.get("title"):
  114. self.title = payload["title"]
  115. elif kind == "todo.updated" and isinstance(payload, dict):
  116. self.todo_state = {"todos": list(payload.get("todos") or []), "revision": payload.get("revision")}
  117. elif kind == "session.usage" and isinstance(payload, dict):
  118. # The gateway's frame carries the flat counters next to a nested TUI ``usage`` view this
  119. # reader can't map (``session_usage.usage_frame``): the counters win when present.
  120. nested = payload.get("usage")
  121. flat = isinstance(payload.get("api_calls"), int) or not isinstance(nested, dict)
  122. self.apply_usage(payload if flat else nested)
  123. elif kind == "request.cancel" and isinstance(payload, dict):
  124. for listener in list(self.request_listeners):
  125. listener(payload)
  126. elif kind == "session.replay_gap":
  127. self.gaps += 1
  128. def apply_usage(self, usage):
  129. """One session-cumulative usage dict (``session.usage`` payload or ``session.stats`` result)."""
  130. if not isinstance(usage, dict):
  131. return
  132. self.usage = dict(usage)
  133. for listener in list(self.usage_listeners):
  134. listener(self.usage)
  135. class GatewayConnection:
  136. _shared = None
  137. _shared_lock = threading.Lock()
  138. def __init__(self, connect=None):
  139. self._connect = connect
  140. self.events: queue.Queue = queue.Queue()
  141. self.states: dict[str, SessionState] = {}
  142. self.client = None
  143. # (client, epoch) of the open socket, swapped as one: a caller never pairs a socket with
  144. # another life's epoch (``call_tagged``).
  145. self._live = None
  146. self.loop = None
  147. self._thread = None
  148. self._ready = threading.Event()
  149. self._error = None
  150. self._stop = None
  151. self._start_lock = threading.Lock()
  152. self.closed = False
  153. # close() ends the viewer; a gateway hang-up (restart, idle exit, crash) lets the next call
  154. # dial again. Each connect bumps ``epoch``, and its loss is tagged with it (``lost_before``).
  155. self._detached = False
  156. self.epoch = 0
  157. # /bg and /btw answers by task id (hermes_cli/gateway_side.py); created by whichever of the
  158. # waiter or the frame comes first, so an answer faster than the verb's reply is kept.
  159. self._side: dict[str, queue.Queue] = {}
  160. self._side_lock = threading.Lock()
  161. # The wake events of :meth:`call_until` commands in flight, each on its own socket.
  162. self._commands: set[threading.Event] = set()
  163. # This CLI's launch-only key (``--api-key`` or a keyed startup alias), set by the session store:
  164. # an owner restart revokes it, so every resume re-supplies it (the owner's ``rebind_launch_key``).
  165. self.launch_api_key = None
  166. # True when that key is the typed ``--api-key`` (not a keyed startup alias's): a startup resume the
  167. # owner will not bind it to is refused as the thin client refuses it, never attached plain.
  168. self.launch_api_key_typed = False
  169. @classmethod
  170. def shared(cls):
  171. with cls._shared_lock:
  172. if cls._shared is None or cls._shared.final:
  173. cls._shared = cls()
  174. return cls._shared
  175. @property
  176. def final(self) -> bool:
  177. """No call dials again: the viewer closed it, or its first connect failed. ``closed`` is also
  178. true while the gateway is hung up, which the next ``call()``/``start()`` redials."""
  179. return self._detached or (self._error is not None and not self.epoch)
  180. def state(self, session_id) -> SessionState:
  181. state = self.states.get(session_id)
  182. if state is None:
  183. state = self.states[session_id] = SessionState(session_id)
  184. return state
  185. # -- lifecycle ---------------------------------------------------------------------------
  186. def start(self, timeout=90):
  187. deadline = time.monotonic() + timeout
  188. while True:
  189. with self._start_lock:
  190. if self.final:
  191. if self._error is not None:
  192. raise self._error
  193. return
  194. thread = self._thread
  195. if thread is not None and self.closed and thread.is_alive() and thread is not threading.current_thread():
  196. # ``_run`` reports ready before its thread returns: that life is over, so let it
  197. # end instead of reading it as a dial in progress (and re-raising its stale error).
  198. thread.join(max(deadline - time.monotonic(), 0))
  199. if thread is None or not thread.is_alive():
  200. from agent.memory_provider import spawn_context_thread
  201. if thread is not None:
  202. # The gateway hung up, or the last redial failed: dial again (``connect_gateway``
  203. # starts a gateway when none is running) and resubscribe this viewer's sessions.
  204. self._error = None
  205. self._ready.clear()
  206. self.closed = False
  207. thread = self._thread = spawn_context_thread(self._run, name="hermes-gateway-loop")
  208. thread.start()
  209. # Every caller waits out a dial in progress, its own or another thread's.
  210. if not self._ready.wait(max(deadline - time.monotonic(), 0)):
  211. raise GatewayClientError("Gateway connection timed out; no local fallback")
  212. if self._error is not None:
  213. raise self._error
  214. if self.client is not None or self.final:
  215. return
  216. # Hung up right after that dial: let its loop end, then dial again.
  217. thread.join(max(deadline - time.monotonic(), 0))
  218. if time.monotonic() >= deadline:
  219. raise GatewayClientError("Gateway connection timed out; no local fallback")
  220. def _run(self):
  221. try:
  222. asyncio.run(self._main())
  223. except BaseException as exc: # health: allow BLE001 -- the loop thread reports every failure to the waiting caller
  224. if self._error is None:
  225. self._error = exc if isinstance(exc, GatewayClientError) else GatewayClientError(
  226. "Gateway connection failed; no local fallback")
  227. finally:
  228. self.closed = True
  229. self._ready.set()
  230. async def _main(self):
  231. self.loop = asyncio.get_running_loop()
  232. self._stop = asyncio.Event()
  233. connect = self._connect
  234. if connect is None:
  235. from hermes_cli.gateway_client import connect_gateway
  236. connect = connect_gateway
  237. try:
  238. async with connect() as client:
  239. if self.epoch:
  240. await self._resubscribe(client)
  241. self.epoch += 1
  242. self.client = client
  243. self._live = (client, self.epoch)
  244. self._ready.set()
  245. pump = asyncio.create_task(self._pump(client))
  246. stop = asyncio.create_task(self._stop.wait())
  247. await asyncio.wait({pump, stop}, return_when=asyncio.FIRST_COMPLETED)
  248. pump.cancel()
  249. stop.cancel()
  250. except Exception as exc: # health: allow BLE001 -- a transport failure becomes the connect error or the disconnect frame
  251. if not self._ready.is_set():
  252. self._error = exc if isinstance(exc, GatewayClientError) else GatewayClientError(
  253. f"Gateway connection failed; no local fallback ({type(exc).__name__})")
  254. else:
  255. self._put_loss(GatewayClientError(DISCONNECTED))
  256. finally:
  257. self.client = None
  258. self._live = None
  259. def _put_loss(self, error):
  260. error.connection_epoch = self.epoch
  261. self.events.put(error)
  262. async def _resubscribe(self, client):
  263. """A new socket is a new viewer: the gateway streams a session's frames only to connections
  264. that resumed it. Resume each session this viewer had open; one gone meanwhile is dropped. Any
  265. other failure fails the reconnect: ready without the subscription, every prompt is refused on a
  266. live socket that never redials, while a failed redial is retried by the next call."""
  267. for session_id, state in list(self.states.items()):
  268. if state.subscription_id is None:
  269. continue
  270. try:
  271. state.adopt_snapshot(await self._resume(client, session_id))
  272. except GatewayClientError as exc:
  273. if str(exc) not in RESUME_GONE:
  274. raise
  275. logger.debug("session.resume %s after reconnect: %s", session_id, exc)
  276. state.subscription_id = None
  277. def resume(self, call, session_id, *, typed=False):
  278. """``session.resume`` through ``call`` (the store's attach), re-supplying :attr:`launch_api_key`;
  279. a refusal of the keyed resume is :func:`_resume_plain`'s to decide. ``typed``: the user resumes with
  280. a typed ``--api-key``."""
  281. key = self.launch_api_key
  282. if key:
  283. try:
  284. return call("session.resume", session_id=session_id, api_key=key)
  285. except GatewayRPCError as exc:
  286. _resume_plain(exc, session_id, typed)
  287. return call("session.resume", session_id=session_id)
  288. async def _resume(self, client, session_id):
  289. """:meth:`resume` for :meth:`_resubscribe`, awaited on this loop so a cancelled reconnect cancels
  290. its request. A reconnect types nothing: never ``typed``."""
  291. key = self.launch_api_key
  292. if key:
  293. try:
  294. return await client.rpc("session.resume", session_id=session_id, api_key=key)
  295. except GatewayRPCError as exc:
  296. _resume_plain(exc, session_id, typed=False)
  297. return await client.rpc("session.resume", session_id=session_id)
  298. async def _pump(self, client):
  299. while True:
  300. frame = await client.events.get()
  301. if isinstance(frame, Exception):
  302. self._put_loss(frame)
  303. return
  304. params = frame.get("params") if isinstance(frame, dict) else None
  305. if isinstance(params, dict) and self._take_side(params):
  306. continue
  307. if isinstance(params, dict):
  308. sid = params.get("session_id")
  309. if sid in self.states:
  310. try:
  311. self.states[sid].apply_event(params.get("type"), params.get("payload"))
  312. except Exception: # pragma: no cover - a cache update must never stop the pump
  313. logger.debug("session state update failed", exc_info=True)
  314. self.events.put(frame)
  315. def _take_side(self, params):
  316. from hermes_cli.gateway_side import SIDE_EVENTS
  317. payload = params.get("payload")
  318. if params.get("type") not in SIDE_EVENTS or not isinstance(payload, dict) or not payload.get("task_id"):
  319. return False
  320. self.side_waiter(payload["task_id"]).put(payload)
  321. return True
  322. def side_waiter(self, task_id) -> queue.Queue:
  323. with self._side_lock:
  324. return self._side.setdefault(task_id, queue.Queue())
  325. def drop_side_waiter(self, task_id):
  326. with self._side_lock:
  327. self._side.pop(task_id, None)
  328. def close(self):
  329. self._detached = True
  330. with self._side_lock:
  331. for wake in self._commands: # their sockets close too
  332. wake.set()
  333. loop, stop = self.loop, self._stop
  334. if loop is not None and stop is not None and not self.closed:
  335. try:
  336. loop.call_soon_threadsafe(stop.set)
  337. except RuntimeError: # the loop already ended (the gateway hung up first)
  338. pass
  339. if self._thread is not None and self._thread is not threading.current_thread():
  340. self._thread.join(5)
  341. self.closed = True
  342. # -- requests ----------------------------------------------------------------------------
  343. def reply(self, request_id, result):
  344. """Answer one of the gateway's server→client requests (the ``sudo`` prompt): a JSON-RPC
  345. response frame with the request's own id. Never raises; a lost reply times out gateway-side."""
  346. import json
  347. client, loop = self.client, self.loop
  348. if client is None or loop is None:
  349. return False
  350. frame = json.dumps({"jsonrpc": "2.0", "id": request_id, "result": result})
  351. try:
  352. asyncio.run_coroutine_threadsafe(client.websocket.send(frame), loop).result(10)
  353. except Exception as exc: # health: allow BLE001 -- the gateway's own wait bounds a lost answer
  354. logger.debug("server request reply failed: %s", exc)
  355. return False
  356. return True
  357. def call(self, method, *, timeout=None, abandon=None, **params):
  358. return self.call_tagged(method, timeout=timeout, abandon=abandon, **params)[0]
  359. def reattach(self, session_id, abandon=None):
  360. """``session.resume`` after a replay gap; its snapshot names the subscription to detach."""
  361. state = self.state(session_id)
  362. gaps = state.gaps
  363. snapshot = self.call("session.resume", session_id=session_id, abandon=abandon)
  364. state.adopt_snapshot(snapshot)
  365. state.attached_gaps = gaps # a gap during this resume (or a wait given up) still owes one
  366. return snapshot
  367. def _send(self, method, budget, params):
  368. """``(future, epoch)`` of one request sent on the live socket (see :meth:`call_tagged`)."""
  369. if self._thread is not None and threading.current_thread() is self._thread:
  370. raise RuntimeError("GatewayConnection.call() on the gateway loop thread would deadlock")
  371. self.start()
  372. live, loop = self._live, self.loop
  373. if live is None or loop is None:
  374. raise GatewayClientError(NOT_SENT)
  375. client, epoch = live
  376. return asyncio.run_coroutine_threadsafe(client.rpc(method, _timeout=budget, **params), loop), epoch
  377. def call_until(self, method, wake, **params):
  378. """``call`` with no answer deadline, for a verb that lasts as long as the work it runs (a plugin
  379. command's handler), on a socket of its own to this connection's gateway: the gateway answers a
  380. socket's requests one at a time, in arrival order (``tui_gateway/ws.py``), so a handler still
  381. running never holds this connection's reads, prompts and interrupts. One dial and one send,
  382. never retried. Waits for the answer or the transport's error, unless *wake* (a
  383. ``threading.Event``, set by Ctrl+C or :meth:`close`) is set first: then raises
  384. :class:`CallStopped`; the request was sent and its work goes on, but its answer is dropped. The
  385. socket closes on any of these, and its thread is gone before this returns."""
  386. import concurrent.futures
  387. from agent.memory_provider import spawn_context_thread
  388. if self._thread is not None and threading.current_thread() is self._thread:
  389. raise RuntimeError("GatewayConnection.call_until() on the gateway loop thread would deadlock")
  390. self.start() # this viewer's gateway, dialled (or redialled after a hang-up) as for any call
  391. if self.final:
  392. raise GatewayClientError(NOT_SENT)
  393. answer, sent = concurrent.futures.Future(), threading.Event()
  394. answer.add_done_callback(lambda _done: wake.set())
  395. thread = spawn_context_thread(lambda: asyncio.run(self._command(method, params, wake, answer, sent)),
  396. name="hermes-gateway-command")
  397. with self._side_lock:
  398. self._commands.add(wake)
  399. thread.start()
  400. try:
  401. wake.wait()
  402. finally:
  403. wake.set() # also on KeyboardInterrupt: the socket closes
  404. thread.join(10)
  405. with self._side_lock:
  406. self._commands.discard(wake)
  407. if answer.done():
  408. return answer.result()
  409. if sent.is_set():
  410. raise CallStopped(method)
  411. raise GatewayClientError(NOT_SENT)
  412. async def _command(self, method, params, wake, answer, sent):
  413. """The command socket of :meth:`call_until`: dial, send once, settle ``answer`` with the reply or
  414. the transport's error; a set ``wake`` without an answer cancels the dial or the wait."""
  415. connect = self._connect
  416. if connect is None:
  417. from hermes_cli.gateway_client import connect_gateway
  418. connect = connect_gateway
  419. task = asyncio.current_task()
  420. stop = asyncio.ensure_future(asyncio.to_thread(wake.wait))
  421. stop.add_done_callback(lambda _stop: answer.done() or task.cancel())
  422. try:
  423. async with connect() as client:
  424. sent.set() # from here the request may have reached the gateway: never sent again
  425. answer.set_result(await client.rpc(method, _timeout=float("inf"), **params))
  426. except asyncio.CancelledError:
  427. pass
  428. except Exception as exc: # health: allow BLE001 -- every transport failure is the caller's error
  429. if not isinstance(exc, GatewayClientError):
  430. exc = GatewayClientError(DISCONNECTED if sent.is_set() else NOT_SENT)
  431. answer.set_exception(exc)
  432. finally:
  433. wake.set() # ends the watcher above
  434. await asyncio.gather(stop, return_exceptions=True)
  435. def call_tagged(self, method, *, timeout=None, abandon=None, **params):
  436. """``(result, epoch)``: the epoch of the socket that carried the request, so work it admitted
  437. is tied to that gateway life even if another caller redials before this one reads ``epoch``.
  438. ``abandon`` (polled) true stops the wait with :class:`CallStopped`; the answer is dropped."""
  439. from concurrent.futures import wait
  440. budget = DEFAULT_RPC_TIMEOUT if timeout is None else timeout
  441. if abandon is not None and abandon():
  442. raise CallStopped(method)
  443. future, epoch = self._send(method, budget, params)
  444. deadline = time.monotonic() + budget + 5
  445. try:
  446. while abandon is not None and not wait([future], 0.1).done and time.monotonic() < deadline:
  447. if abandon():
  448. raise CallStopped(method)
  449. return future.result(max(deadline - time.monotonic(), 0)), epoch
  450. except TimeoutError as exc:
  451. future.cancel()
  452. raise GatewayClientError(f"Gateway did not answer {method}") from exc