gateway_queue.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. """The classic CLI's ``/queue`` over the gateway's durable FIFO (refs #134889, N5).
  2. On main a prompt queued while a turn ran sat in the CLI's ``_pending_input``: no
  3. ``session_admissions`` row, so an owner restart or a disconnect lost it. Over the session authority
  4. ``/queue <prompt>`` typed while a turn runs is an ordinary queued admission (``prompt.submit
  5. {queued: true}``, the same path Ink and Desktop queue busy input on): the owner commits it at accept,
  6. keeps it across a restart and runs it once, in FIFO order, after the current turn.
  7. The CLI keeps no copy to resend. It remembers only which admissions it queued
  8. (``SessionState.queued_inputs``) so the turn the owner runs for one is painted like a typed turn
  9. (``gateway_tools_notify.queue_admitted_turn`` adopts it), and ``/queue list|edit|rm|clear`` read and
  10. change the owner's queue (``prompt.receipt`` / ``prompt.replace_queued`` / ``prompt.cancel``).
  11. """
  12. from __future__ import annotations
  13. import logging
  14. import uuid
  15. from hermes_cli.gateway_client import GatewayClientError
  16. logger = logging.getLogger(__name__)
  17. def _state(cli, store):
  18. return store.connection.state(cli.session_id)
  19. class QueueOutcomeUnknown(GatewayClientError):
  20. """The submit was sent but no answer came back (timeout, disconnect): the owner may hold it.
  21. No admission id came back, so nothing enters ``queued_inputs``. If the owner did queue it, the row
  22. shows in ``/queue list`` (from the owner's published list) and runs once, but this CLI never adopts
  23. its turn: the reply lands in the session's history, not painted live here. The user is told to
  24. check ``/queue list`` instead of re-sending. This is rare (two lost answers in a row).
  25. """
  26. def submit_queued(cli, store, text: str) -> str:
  27. """Admit ``text`` behind the running turn in the owner's FIFO; returns its admission id.
  28. A refusal (``GatewayRPCError``) or a request never sent raises as is: nothing was queued. A lost
  29. answer is asked once more under the SAME ``submission_id``, which the owner answers with the
  30. admission it already holds (``admit_session_input`` keys it by principal, session and request id),
  31. so the prompt is queued exactly once. Still no answer: :class:`QueueOutcomeUnknown`.
  32. """
  33. from hermes_cli.gateway_client import NOT_SENT, GatewayRPCError
  34. submission = uuid.uuid4().hex
  35. for attempt in (1, 2):
  36. try:
  37. receipt = store.call("prompt.submit", session_id=cli.session_id, submission_id=submission,
  38. text=text, queued=True)
  39. break
  40. except GatewayRPCError:
  41. raise
  42. except GatewayClientError as exc:
  43. if str(exc) == NOT_SENT and attempt == 1:
  44. raise # never reached a socket: nothing is in doubt
  45. if attempt == 2:
  46. raise QueueOutcomeUnknown(str(exc)) from exc
  47. logger.debug("prompt.submit answer lost (%s); asking again under the same identity", exc)
  48. admission = (receipt or {}).get("admission_id")
  49. if not admission:
  50. raise GatewayClientError("the gateway did not admit the queued prompt")
  51. state = _state(cli, store)
  52. state.queued_inputs[admission] = text
  53. state.queued_at[admission] = state.pending_seq
  54. return admission
  55. def queued_rows(cli, store) -> list[tuple[str, str]]:
  56. """``[(admission_id, text)]`` still queued in the owner's FIFO, in run order.
  57. This viewer's own admissions are read back with ``prompt.receipt`` (one it finds started or settled
  58. is left for the idle tick to paint); other viewers' queued rows come from the last pending list the owner published.
  59. """
  60. state = _state(cli, store)
  61. rows: dict[str, tuple[int, str]] = {}
  62. for row in state.pending or ():
  63. if isinstance(row, dict) and row.get("status") == "queued" and row.get("admission_id"):
  64. rows[row["admission_id"]] = (int(row.get("sequence") or 0), str(row.get("text") or ""))
  65. for admission, text in list(state.queued_inputs.items()):
  66. try:
  67. receipt = store.call("prompt.receipt", session_id=cli.session_id, admission_id=admission)
  68. except GatewayClientError as exc:
  69. logger.debug("prompt.receipt %s failed: %s", admission, exc)
  70. rows.setdefault(admission, (1 << 62, text)) # unread: still shown, never dropped
  71. continue
  72. if receipt.get("status") == "queued":
  73. rows[admission] = (int(receipt.get("sequence") or 0), text)
  74. else:
  75. # Started or settled: no longer queued, but its turn may not be painted yet. The idle tick
  76. # paints it (``gateway_tools_notify.recover_queued``); listing never erases that.
  77. rows.pop(admission, None)
  78. state.missed.add(admission)
  79. return [(admission, text) for admission, (_, text) in sorted(rows.items(), key=lambda kv: kv[1][0])]
  80. def published_queue(cli, store) -> list[str]:
  81. """The texts the owner last published as queued, plus this viewer's admissions submitted after that
  82. list (no RPC: the dock repaints on every tick).
  83. The owner's published list is authoritative: an admission it no longer lists (cancelled, replaced or
  84. steered by another client, or started) is not shown as queued, even while ``queued_inputs`` still
  85. owes its turn a paint."""
  86. state = _state(cli, store)
  87. for gone in [admission for admission in state.queued_at if admission not in state.queued_inputs]:
  88. state.queued_at.pop(gone, None)
  89. rows = [row for row in state.pending or () if isinstance(row, dict)]
  90. known = {row.get("admission_id") for row in rows}
  91. fresh = [text for admission, text in state.queued_inputs.items()
  92. if admission not in known and state.queued_at.get(admission, -1) >= state.pending_seq]
  93. return [str(row.get("text") or "") for row in rows if row.get("status") == "queued"] + fresh
  94. # How long the input loop holds a local input behind this viewer's own ``/queue`` admission at most.
  95. OWN_QUEUE_HOLD_S = 2.0
  96. def _own_head(state):
  97. """This viewer's ``/queue`` admission at the head of the owner's queue, not yet adopted; None when
  98. the head is someone else's, nothing is queued, or the FIFO is paused on an ``unknown`` row."""
  99. rows = [row for row in state.pending or () if isinstance(row, dict)]
  100. if any(row.get("status") == "unknown" for row in rows):
  101. return None # paused until the owner resolves that row: never hold typing behind it
  102. queued = sorted((row for row in rows if row.get("status") == "queued" and row.get("admission_id")),
  103. key=lambda row: int(row.get("sequence") or 0))
  104. if queued:
  105. head = queued[0]["admission_id"]
  106. else: # submitted after the last published list (``published_queue``'s fresh rows)
  107. listed = {row.get("admission_id") for row in rows}
  108. head = next((a for a in state.queued_inputs
  109. if a not in listed and state.queued_at.get(a, -1) >= state.pending_seq), None)
  110. return head if head in state.queued_inputs and head not in state.adopted else None
  111. class OwnQueueHold:
  112. """A local input (a slash command, a typed turn) taken while this viewer's own ``/queue`` admission
  113. is the owner's next turn. On main both sat in ``_pending_input`` in entry order; now the admission
  114. is in the owner's FIFO and its turn is adopted only when the owner's ``message.start`` arrives, so
  115. the input loop holds the local item until that admission was taken (or left the queue).
  116. Bounded: an ``unknown`` row (a paused FIFO) never starts a hold; an owner that publishes an idle
  117. list still queuing it (beyond the one a just-settled turn publishes before its claim) releases it;
  118. and ``OWN_QUEUE_HOLD_S`` caps the wait. The admission itself is never touched.
  119. """
  120. def __init__(self, item, state, admission):
  121. import time
  122. self.item, self.state, self.admission = item, state, admission
  123. self.seq = state.pending_seq
  124. # A turn the owner reports running settles with one idle list (still queuing this admission)
  125. # before its drain claims it: that list is expected, a further one means the FIFO is paused.
  126. self.allowed_idle_lists = 1 if state.info.get("running") else 0
  127. self.idle_lists = 0
  128. self.deadline = time.monotonic() + OWN_QUEUE_HOLD_S
  129. self.taken = False
  130. @classmethod
  131. def begin(cls, cli, item):
  132. from cli import _SeededQueryMessage
  133. from hermes_cli.gateway_tools_notify import gateway_store
  134. from tools.process_registry_notifications import TimelineNotification
  135. if getattr(cli, "_agent_running", False) or isinstance(item, (_SeededQueryMessage, TimelineNotification)):
  136. return None # a turn the owner already runs (or a seeded prompt) is never held
  137. store = gateway_store(cli) if getattr(cli, "session_id", None) else None
  138. if store is None:
  139. return None
  140. state = _state(cli, store)
  141. admission = _own_head(state)
  142. return cls(item, state, admission) if admission else None
  143. def take_gateway_turn(self, pending):
  144. """Pop a turn the owner is running now (an adopted admission, a notification header) from the
  145. head of ``_pending_input``: those run before the held item, typed items stay behind it."""
  146. from tools.process_registry_notifications import TimelineNotification
  147. with pending.mutex:
  148. if not pending.queue:
  149. return None
  150. head = pending.queue[0]
  151. if not (getattr(head, "admission_id", None) or isinstance(head, TimelineNotification)):
  152. return None
  153. pending.queue.popleft()
  154. pending.unfinished_tasks = max(0, pending.unfinished_tasks - 1)
  155. if getattr(head, "admission_id", None) == self.admission:
  156. self.taken = True
  157. return head
  158. def done(self) -> bool:
  159. import time
  160. if self.taken or time.monotonic() >= self.deadline:
  161. return True
  162. state = self.state
  163. if _own_head(state) != self.admission:
  164. # Adopted: its item is at the head of ``_pending_input``, taken on the next step. Otherwise
  165. # it started unseen (the idle tick recovers it), was cancelled, or is no longer next.
  166. return self.admission not in state.adopted
  167. if state.pending_seq != self.seq:
  168. self.seq = state.pending_seq
  169. if not state.info.get("running"):
  170. self.idle_lists += 1
  171. return self.idle_lists > self.allowed_idle_lists # idle and still not claiming it: stop waiting
  172. STARTED_REFUSALS = frozenset({"stale_generation", "admission_started"})
  173. def cancel(cli, store, admission: str) -> str:
  174. """Retire one queued admission: ``"cancelled"`` when it never ran, ``"stopped"`` when the drain had
  175. claimed it and the owner interrupted that run instead (``session_controls.cancel``), ``"started"``
  176. when it already ran. A turn that ran is still owed a paint, so only a cancelled one leaves
  177. ``queued_inputs``."""
  178. receipt = store.call("prompt.cancel", session_id=cli.session_id, admission_id=admission) or {}
  179. if receipt.get("status") == "terminal" and receipt.get("outcome") == "cancelled":
  180. state = _state(cli, store)
  181. state.queued_inputs.pop(admission, None)
  182. state.missed.discard(admission)
  183. return "cancelled"
  184. _state(cli, store).missed.add(admission) # the idle tick paints whatever ran
  185. if receipt.get("status") == "started" or receipt.get("outcome") == "interrupted":
  186. return "stopped"
  187. return "started" # it already ran to its end: nothing was stopped
  188. def replace(cli, store, admission: str, text: str) -> str | None:
  189. """``prompt.replace_queued``: the edited prompt is a new admission at the back of the FIFO."""
  190. receipt = store.call("prompt.replace_queued", session_id=cli.session_id, admission_id=admission,
  191. submission_id=uuid.uuid4().hex, text=text)
  192. state = _state(cli, store)
  193. state.queued_inputs.pop(admission, None)
  194. new = (receipt or {}).get("admission_id")
  195. if new:
  196. state.queued_inputs[new] = text
  197. state.queued_at[new] = state.pending_seq
  198. return new