session_ingress_burst.py 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217
  1. """Busy same-sender bursts become one shared-runtime admission.
  2. Main merged a busy photo burst into the pending slot and debounced queue-mode text from one sender
  3. (``BasePlatformAdapter._queue_text_debounce``) so a thought split over two messages ran as one
  4. turn. The authority FIFO admits each input as it arrives, so the burst is held here for main's
  5. bounded window first, then committed once through ``dispatch_shared_busy``. Every absorbed caller
  6. returns only after that commit, with its acceptance copied from it (committed before ACK), and
  7. input that arrives later never commits before the burst. An adapter declaring
  8. ``serial_receive_loop`` (Signal, IRC, ...) delivers a chat's next input only after this returns, and
  9. returning triggers no acknowledgement, so its caller returns once the input is held, as main
  10. returned once merged. Its reader still never runs ahead of what it returned on: a chat holds at
  11. most one uncommitted burst (the next one waits for that commit, main's backpressure), and a
  12. command first waits until every earlier input of that reader has committed (``burst_gate``).
  13. """
  14. import asyncio
  15. from dataclasses import dataclass, field
  16. from functools import partial
  17. import logging
  18. import time
  19. from gateway.platforms.event import MessageEvent, MessageType
  20. from hermes_cli.observability.shared_metrics_busy import note_busy_merged, settle_busy_arrival, stamp_busy_arrival
  21. logger = logging.getLogger(__name__)
  22. @dataclass(eq=False)
  23. class _Burst:
  24. head: MessageEvent
  25. done: asyncio.Future
  26. first_ts: float
  27. last_ts: float
  28. timer: asyncio.Task | None = None
  29. commit: asyncio.Task | None = None
  30. absorbed: list = field(default_factory=list)
  31. def _kind(adapter, event):
  32. if event.message_type == MessageType.PHOTO and not event.internal:
  33. return 'photo'
  34. return 'text' if adapter._is_queue_text_debounce_candidate(event) else None
  35. def _state(adapter, name):
  36. return adapter.__dict__.setdefault(name, {})
  37. def _serial(adapter):
  38. return getattr(adapter, 'serial_receive_loop', False)
  39. def holds_busy_burst(adapter, session_key):
  40. """A burst is held or committing: later input for the route must queue behind it."""
  41. return any(session_key in adapter.__dict__.get(name, ()) for name in ('_shared_bursts', '_shared_burst_commits'))
  42. async def burst_gate(adapter, event, session_key):
  43. """``handle_message``'s entry check. True: a held or committing burst owns the route, so this
  44. shared-runtime input must queue behind it. A command on a serial reader first waits for every
  45. input that reader already returned on, as main's reader reached it only after their commits:
  46. /stop, /new and /reset drop their own chat's held burst (main dropped the debounce buffer),
  47. every other held burst commits now, and every pending commit lands before the command runs."""
  48. if adapter._shared_runtime_owns_input(event):
  49. return holds_busy_burst(adapter, session_key)
  50. if (cmd := event.get_command()) and _serial(adapter):
  51. from hermes_cli.commands import is_interrupt_then_dispatch
  52. if is_interrupt_then_dispatch(cmd):
  53. discard_busy_burst(adapter, session_key)
  54. for key, burst in list(adapter.__dict__.get('_shared_bursts', {}).items()):
  55. _flush(adapter, key, burst)
  56. if pending := list(adapter.__dict__.get('_shared_burst_commits', {}).values()):
  57. await asyncio.wait(pending)
  58. if held := list(adapter.__dict__.get('_uncommitted_inputs', ())):
  59. await asyncio.wait(held) # a non-burst input released behind an earlier one, not durable yet
  60. return False
  61. async def dispatch_busy_burst(adapter, event, session_key):
  62. """``dispatch_shared_busy`` for one busy input, merging a same-sender PHOTO+PHOTO or
  63. queue-mode TEXT+TEXT burst first."""
  64. from gateway.platforms.base import merge_pending_message_event
  65. from gateway.session_ingress import dispatch_shared_busy
  66. stamp_busy_arrival(adapter, event)
  67. kind, bursts = _kind(adapter, event), _state(adapter, '_shared_bursts')
  68. while (burst := bursts.get(session_key)) is not None:
  69. if kind is not None and kind == _kind(adapter, burst.head) and adapter._can_merge_text_debounce_events(
  70. burst.head, event):
  71. merge_pending_message_event({session_key: burst.head}, session_key, event, merge_text=kind == 'text')
  72. burst.absorbed.append(event)
  73. note_busy_merged(burst.head, event)
  74. burst.last_ts = time.monotonic()
  75. _arm(adapter, session_key, burst)
  76. return await _accepted(adapter, burst, event)
  77. await asyncio.shield(_flush(adapter, session_key, burst)) # a different sender or kind: in order
  78. previous = _state(adapter, '_shared_burst_commits').get(session_key)
  79. if kind is not None and previous is not None and not previous.done() and _serial(adapter):
  80. await asyncio.wait({previous}) # one uncommitted burst per chat: the reader waits, as on main
  81. return await dispatch_busy_burst(adapter, event, session_key) # a burst may have started meanwhile
  82. if kind is None:
  83. if previous is not None:
  84. await asyncio.wait({previous})
  85. return await dispatch_shared_busy(adapter, event, session_key)
  86. now = time.monotonic()
  87. burst = bursts[session_key] = _Burst(event, asyncio.get_running_loop().create_future(), now, now)
  88. _arm(adapter, session_key, burst)
  89. return await _accepted(adapter, burst, event)
  90. async def _accepted(adapter, burst, event):
  91. if _serial(adapter):
  92. return # waiting out the hold window here would keep the burst's next input unread
  93. await asyncio.shield(burst.done) # raises the merged admission's own refusal
  94. event._gateway_accepted = burst.head._gateway_accepted
  95. def _track(adapter, task):
  96. adapter._background_tasks.add(task)
  97. task.add_done_callback(adapter._background_tasks.discard)
  98. return task
  99. def _arm(adapter, session_key, burst):
  100. if burst.timer is not None:
  101. burst.timer.cancel()
  102. delay = max(0.0, min(burst.last_ts + adapter._busy_text_debounce_seconds,
  103. burst.first_ts + adapter._busy_text_hard_cap_seconds) - time.monotonic())
  104. async def fire():
  105. await asyncio.sleep(delay)
  106. await asyncio.shield(_flush(adapter, session_key, burst))
  107. burst.timer = _track(adapter, asyncio.create_task(fire()))
  108. def _flush(adapter, session_key, burst):
  109. """Start committing ``burst`` (once), after any earlier commit for the route."""
  110. if burst.commit is None:
  111. bursts, tails = _state(adapter, '_shared_bursts'), _state(adapter, '_shared_burst_commits')
  112. if bursts.get(session_key) is burst:
  113. del bursts[session_key]
  114. if burst.timer is not asyncio.current_task():
  115. burst.timer.cancel()
  116. burst.commit = tails[session_key] = _track(adapter, asyncio.create_task(
  117. _commit(adapter, session_key, burst, tails.get(session_key))))
  118. burst.commit.add_done_callback(lambda task: tails.get(session_key) is task and tails.pop(session_key))
  119. burst.commit.add_done_callback(partial(_settle, burst))
  120. return burst.commit
  121. async def _commit(adapter, session_key, burst, previous):
  122. from gateway.session_ingress import dispatch_shared_busy
  123. if previous is not None:
  124. await asyncio.wait({previous})
  125. try:
  126. # Durable: this task is what the chat's next burst, a command barrier and /restart wait on, so
  127. # it ends once the row is committed, not when its FIFO position is fixed behind a capture.
  128. await dispatch_shared_busy(adapter, burst.head, session_key, durable=True)
  129. except Exception:
  130. if not _serial(adapter):
  131. raise # every absorbed caller sees it (``_settle``)
  132. # No caller waits on a serial reader's burst: log it as its receive loop would have.
  133. logger.exception('[%s] Busy burst for %s was not queued', adapter.name, session_key)
  134. def _settle(burst, task):
  135. """Every absorbed caller sees the merged admission's outcome, refusal or cancellation."""
  136. if burst.done.done():
  137. return
  138. if task.cancelled():
  139. burst.done.cancel()
  140. elif task.exception() is not None:
  141. burst.done.set_exception(task.exception())
  142. else:
  143. burst.done.set_result(None)
  144. def _undurable(adapter):
  145. """Inputs the reader already returned on that are not committed yet: held bursts, burst commits
  146. and busy inputs released before their commit (``session_ingress._hold_until_committed``)."""
  147. pending = [burst.done for burst in adapter.__dict__.get('_shared_bursts', {}).values()]
  148. pending += [*adapter.__dict__.get('_shared_burst_commits', {}).values(),
  149. *adapter.__dict__.get('_uncommitted_inputs', ())]
  150. return [future for future in pending if not future.done()]
  151. async def when_returned_inputs_durable(adapter, persist):
  152. """Run ``persist()`` once every input a serial reader returned on is committed: a reader that
  153. saves its own receive cursor after the call returns (Buzz) must not advance it past a held burst,
  154. or a restart in the hold window loses it. Inline when nothing is pending; otherwise one tracked
  155. task per adapter, so the reader reads on (a lost cursor write only replays inputs admission
  156. already holds). ``persist`` carries the cursor as of this call; while that task waits, a newer
  157. call replaces it, and the task picks the newest in the same step that saw nothing pending, so
  158. an input that arrives during the write's I/O is never in it."""
  159. if getattr(adapter, '_durable_persist', None) is not None:
  160. adapter._durable_persist = persist
  161. return
  162. if not _undurable(adapter):
  163. await persist()
  164. return
  165. adapter._durable_persist = persist
  166. async def later():
  167. try:
  168. while pending := _undurable(adapter):
  169. await asyncio.wait(pending)
  170. finally:
  171. latest = adapter.__dict__.pop('_durable_persist', None)
  172. await latest()
  173. _track(adapter, asyncio.create_task(later()))
  174. def discard_busy_burst(adapter, session_key):
  175. """``/stop`` / ``/new`` drop a held burst as main dropped its debounce buffer; nothing was ACKed."""
  176. burst = adapter.__dict__.get('_shared_bursts', {}).pop(session_key, None)
  177. if burst is not None:
  178. burst.timer.cancel()
  179. settle_busy_arrival(burst.head, discarded=True)
  180. burst.done.done() or burst.done.set_result(None)