session_ingress.py 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568
  1. """Trusted messaging admission and the existing TurnRunner invocation boundary."""
  2. import asyncio
  3. import logging
  4. from contextvars import ContextVar
  5. from contextlib import asynccontextmanager, nullcontext
  6. from dataclasses import replace
  7. from gateway.platforms.event import MessageEvent, ProcessingOutcome
  8. from gateway.session_envelope import restore_native
  9. from gateway.session_producer_lifecycle import close_producer_turn, producer_turn
  10. from hermes_cli.observability.shared_metrics_busy import (
  11. busy_turn_started, note_busy_admitted, note_busy_parked, settle_busy_arrival)
  12. from hermes_state_runtime import RuntimeStoreError
  13. logger = logging.getLogger(__name__)
  14. admission_author = ContextVar('admission_author', default=None)
  15. executing_admission = ContextVar('executing_admission', default=False)
  16. # The live session while the executing turn is one its viewer submitted (``is_viewer_input``), else None.
  17. viewer_turn: ContextVar = ContextVar('viewer_turn', default=None)
  18. # Set by a busy-path caller that must be released once its input is durably queued.
  19. busy_acceptance = ContextVar('busy_acceptance', default=None)
  20. # Set beside it by a caller that must also know when the row is durably committed (a busy burst's
  21. # commit task, which command barriers and /restart wait on): ``busy_acceptance`` may already resolve
  22. # once the input's FIFO position is fixed behind an earlier input of its chat still capturing.
  23. busy_commit = ContextVar('busy_commit', default=None)
  24. # ``_validate_native`` clauses about the message's own content, safe to name to the sender's chat.
  25. _CONTENT_CLAUSES = frozenset({
  26. 'text', 'auto_skill', 'channel_prompt', 'channel_context', 'reply_expected',
  27. 'media_urls', 'media_types', 'media_text_inlined', 'media_types_length', 'media_text_inlined_length',
  28. 'media_filenames', 'media_filenames_length',
  29. })
  30. # Nothing was committed (main's busy-queue cap); /retry answers it before rewriting (#139257).
  31. QUEUE_FULL_REPLY = ('⏳ Too many messages are already waiting in this conversation; '
  32. 'send this again after the current reply.')
  33. async def admit_message(authority, event):
  34. from gateway.session_authority import messaging_admission
  35. accepted, committed = busy_acceptance.get(), busy_commit.get()
  36. busy_acceptance.set(None) # the drain task this schedules must not inherit the busy caller's future
  37. busy_commit.set(None)
  38. ingress = messaging_admission.set(True if accepted is None else accepted)
  39. try:
  40. receipt = await authority.admit_native(event)
  41. if committed is not None and not committed.done():
  42. committed.set_result(None) # the row is durable (any status): a barrier may go on
  43. except RuntimeStoreError as exc:
  44. if exc.reason == 'stale_generation' and exc.field == 'route':
  45. return None # a later /stop or /new of the chat fenced it, as main cancels a queued row
  46. if exc.reason == 'invalid_params' and exc.field:
  47. # Refused before commit (_validate_native): a refusal notice instead of a generic turn error.
  48. # The reply reaches the whole chat, so only content clauses are named; trust and provenance
  49. # clauses (and any new one) stay generic, and gateway.log keeps the clause (``_refuse``).
  50. named = f'{exc.reason}: {exc.field}' if exc.field in _CONTENT_CLAUSES else exc.reason
  51. return (f'⚠️ Hermes refused this message ({named}), so it was not saved '
  52. 'or answered. Ask the operator to check gateway.log.')
  53. if exc.reason != 'queue_full':
  54. raise
  55. # Nothing was committed (main's busy-queue cap): say so instead of a generic turn error.
  56. return QUEUE_FULL_REPLY
  57. finally:
  58. messaging_admission.reset(ingress)
  59. if receipt.status == 'terminal':
  60. return None
  61. # Only the delivery waiter is process-local; execution reads the committed snapshot.
  62. authority.native_waiters.add(receipt.admission_id)
  63. waiter = authority.waiters.setdefault(receipt.admission_id, asyncio.get_running_loop().create_future())
  64. adopters = _marker_adopters(authority)
  65. if getattr(event, '_turn_marker_handoff', False):
  66. # The adapter lifecycle that sends this reply releases the turn's crash marker only once
  67. # the reply is in the delivery ledger; the drain hands its marker to this event.
  68. adopters[receipt.admission_id] = event
  69. # A drain already running can claim, run and settle the row before this caller resumes; one
  70. # that settled it already sent (or owes) the reply itself and resolves no waiter registered now.
  71. from hermes_state_runtime import get_session_admission
  72. row = get_session_admission(authority.db, admission_id=receipt.admission_id)
  73. status = 'terminal' if row is None else row['status'] # None: deleted with its chat, nothing owed
  74. note_busy_admitted(event, receipt.admission_id, status, None if row is None else row.get('outcome'))
  75. if status in ('terminal', 'unknown'):
  76. authority.native_waiters.discard(receipt.admission_id)
  77. if authority.waiters.get(receipt.admission_id) is waiter:
  78. del authority.waiters[receipt.admission_id]
  79. if adopters.get(receipt.admission_id) is event:
  80. del adopters[receipt.admission_id]
  81. return None if status == 'terminal' else pause_notice(authority, receipt.ref, 'unknown_execution')
  82. if accepted is not None and not accepted.done():
  83. accepted.set_result(None)
  84. try:
  85. return await asyncio.shield(waiter)
  86. except RuntimeStoreError as exc:
  87. row = get_session_admission(authority.db, admission_id=receipt.admission_id)
  88. if row is not None and row['status'] == 'terminal':
  89. # The drain refused this head (``_refuse_head``): cancelled, not saved behind a pause.
  90. return (f'⚠️ Hermes did not answer this message ({exc.reason}): the conversation it was '
  91. 'queued for moved, so it was dropped. Send it again to continue.')
  92. if row is not None:
  93. note_busy_parked(event) # still queued (saved): answered after the pause, not dropped
  94. return pause_notice(authority, receipt.ref, exc.reason)
  95. finally:
  96. if adopters.get(receipt.admission_id) is event:
  97. del adopters[receipt.admission_id]
  98. def _marker_adopters(authority):
  99. adopters = getattr(authority, 'marker_adopters', None)
  100. if adopters is None:
  101. adopters = authority.marker_adopters = {}
  102. return adopters
  103. _MARKER_FIELDS = ('_gateway_active_turn_session_key', '_gateway_active_turn_token')
  104. async def _hand_over_turn_marker(authority, admission_id, event):
  105. """The executed turn's durable active-turn marker outlives the handler, as the in-process
  106. turn's does on main, and is released only once its reply is in the delivery ledger (or nothing
  107. is owed). A kill between the terminal commit and that ledger write then leaves a marked turn
  108. whose persisted reply the next unclean boot ledgers and sends once, without new inference,
  109. instead of a settled answer nobody owes. The waiting adapter lifecycle adopts it; a recovered
  110. no-waiter reply keeps it until ``deliver_settled``; any other turn releases it here."""
  111. adopter = _marker_adopters(authority).get(admission_id)
  112. if adopter is not None:
  113. _move_marker(event, adopter)
  114. return
  115. if admission_id in authority.pending_deliveries:
  116. # Released by ``deliver_settled``, also when settlement failed and dropped the delivery.
  117. _held_markers(authority)[admission_id] = event
  118. return
  119. await _release_turn_marker(authority, event)
  120. def _move_marker(source, target):
  121. for name in _MARKER_FIELDS:
  122. if hasattr(source, name):
  123. setattr(target, name, getattr(source, name))
  124. delattr(source, name)
  125. def _held_markers(authority):
  126. held = getattr(authority, 'held_turn_markers', None)
  127. if held is None:
  128. held = authority.held_turn_markers = {}
  129. return held
  130. def _registry(authority, name):
  131. found = getattr(authority, name, None)
  132. if found is None:
  133. found = {}
  134. setattr(authority, name, found)
  135. return found
  136. def waiter_replies(authority):
  137. """admission -> the reply a live native waiter is about to send, kept only until the drain
  138. settles the turn, so a turn fenced ``unknown`` can still owe it to the chat."""
  139. return _registry(authority, 'waiter_replies')
  140. def retain_unsettled_delivery(authority, admission_id):
  141. """Settlement could not commit and the turn is fenced ``unknown``, but this owner keeps its
  142. captured answer for Discard (``resolve_unknown``). The reply stays owed to its destination,
  143. and so does the turn's crash marker: the live waiter is released with a pause notice, so the
  144. marker is taken back from it. Restart while unknown clears the marker (no answer is owed,
  145. ``release_unknown_turn_markers``); a Discard that commits the captured answer makes it owed,
  146. sent once by ``deliver_resolved`` or, after a kill before that send, by the boot ledger."""
  147. delivery = waiter_replies(authority).pop(admission_id, None)
  148. if delivery is not None:
  149. adopter = _marker_adopters(authority).get(admission_id)
  150. if adopter is not None:
  151. _move_marker(adopter, delivery[1])
  152. _held_markers(authority)[admission_id] = delivery[1]
  153. else:
  154. delivery = authority.pending_deliveries.pop(admission_id, None)
  155. if delivery is not None:
  156. _registry(authority, 'unsettled_deliveries')[admission_id] = delivery
  157. async def deliver_resolved(authority, admission_id, *, owed):
  158. """After a Discard committed: send the retained reply when its captured answer committed with
  159. the resolution (``owed``), else release its marker and drop it. Popped first, so a repeated
  160. resolution never sends twice; a turn the drain still owns is delivered by the drain."""
  161. delivery = _registry(authority, 'unsettled_deliveries').pop(admission_id, None)
  162. if delivery is None:
  163. return
  164. if owed:
  165. authority.pending_deliveries[admission_id] = delivery
  166. await deliver_settled(authority, admission_id)
  167. def _take_post_delivery(authority, admission_id, source, route):
  168. """Move the turn's post-delivery slot off its adapter onto this admission. A native waiter (the
  169. messaging adapter task that sends the reply) runs it after its own send instead. A runner or
  170. adapter without the seam (a webhook sink, a bare test double) holds none."""
  171. resolve = getattr(authority.runner, '_adapter_for_source', None)
  172. native = admission_id in authority.native_waiters
  173. slots = getattr(resolve(source), '_post_delivery_callbacks', None) if resolve and source and not native else None
  174. entry = slots.pop(route, None) if slots is not None else None
  175. if entry is not None:
  176. _registry(authority, 'post_delivery')[admission_id] = (slots, route, entry)
  177. def _hand_back_post_delivery(authority, admission_id):
  178. """A native waiter registered after the turn started: its adapter task sends the reply and then
  179. runs the slot, which goes back unless a newer turn's already took it."""
  180. taken = _registry(authority, 'post_delivery').pop(admission_id, None)
  181. if taken is not None:
  182. slots, route, entry = taken
  183. slots.setdefault(route, entry)
  184. async def fire_post_delivery(authority, admission_id):
  185. """Run a settled turn's post-delivery callbacks (background-review release, temp-bubble cleanup,
  186. a deferred /goal or /loop verdict) after its reply was sent, as ``_process_message_background``
  187. does after a messaging reply. Never run, they also pinned the whole turn (its agent, history
  188. and result) in the adapter's slot until restart."""
  189. if admission_id in authority.native_waiters:
  190. _hand_back_post_delivery(authority, admission_id)
  191. return
  192. taken = _registry(authority, 'post_delivery').pop(admission_id, None)
  193. if taken is None:
  194. return
  195. from gateway.platforms.base import _POST_DELIVERY_CALLBACK_TIMEOUT_SECONDS, _split_post_delivery_entry
  196. from gateway.platforms.base_post_delivery import run_post_delivery_callback
  197. callback = _split_post_delivery_entry(taken[2])[1]
  198. with await _execution_scope(authority, None): # off-turn: the owning profile's scope
  199. await run_post_delivery_callback(callback, _POST_DELIVERY_CALLBACK_TIMEOUT_SECONDS)
  200. async def _release_turn_marker(authority, event):
  201. if getattr(event, '_gateway_active_turn_token', None):
  202. await authority.runner._clear_durable_active_turn(event)
  203. def pause_notice(authority, ref, reason):
  204. """The committed input stays queued behind a paused FIFO (a turn lost across an owner
  205. restart, or a head the preflight refused). Tell the platform user once per pause episode
  206. through the ordinary reply path; later messages onto the same pause are admitted silently."""
  207. live = authority.sessions[ref.session_id]
  208. if live.pause_notified:
  209. return None
  210. live.pause_notified = True
  211. if reason == 'runtime_draining':
  212. # Transient: the row runs when the restarted owner drains it, so /reset would be wrong advice.
  213. return '⏳ Hermes is restarting — your message is saved and will be answered when it is back.'
  214. if reason == 'session_busy':
  215. # Transient: queued behind work already running on this conversation (a worker execution).
  216. return '⏳ Your message is saved and will be answered when the work running in this conversation finishes.'
  217. if reason == 'unknown_execution':
  218. # GW-10: the way out is in this chat. /discard acknowledges the lost turn (never replayed)
  219. # through the same fenced resolve_unknown as Desktop/TUI, and the saved messages then run.
  220. # Slack registers no native /discard (its slash cap): the umbrella `/hermes discard` reaches it.
  221. slack = getattr(getattr(live.source, 'platform', None), 'value', None) == 'slack'
  222. discard = '/hermes discard' if slack else '/discard'
  223. return ('⏸️ This conversation is paused: a previous turn did not finish when Hermes restarted, '
  224. f'and only you know whether to send it again. Your message is saved. Send {discard} to drop '
  225. 'the lost turn and continue (it is not re-run), or /reset to start a fresh conversation.')
  226. cause = f'a queued message could not be re-authorized ({reason})'
  227. return (f'⏸️ This conversation is paused: {cause}. Your message is saved but will not be answered here. '
  228. 'Send /reset to start a fresh conversation, or ask the operator to resume this one.')
  229. def row_turn_author(policy, row):
  230. """Who wrote the admitted input, for memory attribution only: the producer's stamp when it
  231. left one, else the forwarded peer bound into the session policy; never grants anything."""
  232. payload = row['payload']
  233. for key in ('local_automation_v1', 'api_turn_v1', 'cli_turn_v1'):
  234. author = (payload.get(key) or {}).get('turn_author')
  235. if author is not None:
  236. return author
  237. from gateway.session_a2a import forward_author
  238. return forward_author(policy)
  239. async def execute_admission(authority, ref, row):
  240. busy_turn_started(row['admission_id'])
  241. from gateway.session_completion_notes import note_if_unviewed
  242. noted = await note_if_unviewed(authority, ref, row) # #134889 LC-7: no viewer, no turn
  243. if noted is not None:
  244. return noted
  245. from gateway.session_policy import policy_for_source
  246. policy = policy_for_source(authority.runner, authority.sessions[ref.session_id].source)
  247. if policy is not None and policy.source == 'cron':
  248. from gateway.session_cron import execute
  249. return await execute(authority, ref, row, policy)
  250. local_policy = policy
  251. from gateway.session_managed_worker import managed_policy, admit_managed
  252. policy = managed_policy(authority, ref)
  253. if policy is not None:
  254. return await admit_managed(authority, ref, row, policy)
  255. live = authority.sessions[ref.session_id]
  256. # The key the turn registers its post-delivery slot under; a rebind mid-turn must not move the take.
  257. route = live.route
  258. native = row['admission_id'] in authority.native_waiters
  259. authority.native_waiters.discard(row['admission_id'])
  260. if 'native_text_v1' in row['payload']:
  261. event = restore_native(row['payload'], authority.runner)
  262. else:
  263. from gateway.session_ingress_media import restore_attachments
  264. event = MessageEvent(text=row['payload']['text'], source=live.source,
  265. message_id=row['admission_id'], **restore_attachments(row['payload']))
  266. if 'local_automation_v1' in row['payload']:
  267. from gateway.session_automation import restore_local_automation
  268. event = restore_local_automation(authority, ref, row)
  269. if event.internal and 'gateway_session_id' in event.metadata:
  270. # Main's ownership proof (802fd5782f3): the session the pre-claim check just validated, so the
  271. # turn's resolver never re-reads (and on a transient failure drops) an admitted completion.
  272. event.metadata['gateway_proven_session_id'] = (
  273. authority.physical_target(ref) if 'native_text_v1' in row['payload'] else event.metadata['gateway_session_id'])
  274. provenance = row['payload'].get('native_text_v1', {}).get('provenance')
  275. home = None
  276. if provenance is not None:
  277. from gateway.session_ingress_context import restore_provenance
  278. home = restore_provenance(authority.runner, event.source, provenance)
  279. from gateway.config import Platform
  280. from gateway.session_api_turn import api_execution, prepare_api_execution
  281. from gateway.session_results import execution_result
  282. is_api = live.source.platform == Platform.API_SERVER
  283. if not (event.internal or is_api or live.source.platform == Platform.LOCAL):
  284. # The claim on a chat route (native or a viewer's submit): a /new before the turn resolves
  285. # its session must not move it onto the fresh conversation (main's /new cancelled its task).
  286. event._claimed_owner = (ref.session_id, row['generation'])
  287. author = event.metadata.get('turn_author')
  288. author_token = admission_author.set(author if author is not None else row_turn_author(local_policy, row))
  289. from gateway.session_operator import is_viewer_input
  290. viewer_token = viewer_turn.set(live if is_viewer_input(row) else None)
  291. api_token = api_execution.set(None)
  292. captured = {}
  293. result_token = execution_result.set(captured)
  294. token = executing_admission.set(True)
  295. # This drain, not the handler's unwind, releases the turn's crash marker (see below).
  296. event._turn_marker_handoff = True
  297. handed = False
  298. try:
  299. async with _admission_scope(authority, home, row, event, captured):
  300. if is_api:
  301. # Committed media resolves under the owning profile's home, like admission did.
  302. prepared = prepare_api_execution(authority, ref, row['payload'])
  303. api_execution.set(prepared)
  304. if isinstance(event.text, list):
  305. # The durable transcript keeps the committed media references (the same
  306. # ``[Image attached ...]`` hints native transports persist), not only the caption.
  307. event.text = '\n'.join(part['text'] for part in prepared['content'] if part.get('type') == 'text')
  308. event.allow_gateway_control = False
  309. event.internal = True # trust comes from the private binding and preclaim, never client JSON
  310. response = await authority.runner._handle_message(event)
  311. result = captured.get('terminal_result', captured.get('result'))
  312. if result is None:
  313. # No TurnRunner result means the handler answered without executing the turn.
  314. # A turn that never ran because it failed (agent initialization raised, history
  315. # unreadable) recorded its own failure via ``record_unexecuted_failure`` on every
  316. # surface; any other reply here is a deliberate notice. An API caller asked for
  317. # work, so its receipt is a failure either way, never a completed apology; so did
  318. # a finite (``chat -q`` / ``-z``) prompt, whose exit code is the only verdict a
  319. # script reads, unless the prompt was a command and the reply is its answer.
  320. result = {'final_response': response or '', 'messages': []}
  321. if is_api or (row['payload'].get('finite') is True and not event.get_command()):
  322. result = {'final_response': '', 'messages': [], 'failed': True, 'completed': False,
  323. 'error': response or 'The admitted turn failed.'}
  324. # The drain commits this under the stream lock so no viewer reads `terminal`
  325. # before the completion event exists in the replay ring.
  326. authority.pending_results[row['admission_id']] = {
  327. 'result': result, 'usage': captured.get('usage', {}),
  328. **({'session_usage': captured['session_usage']} if 'session_usage' in captured else {})}
  329. if not is_api and response:
  330. adapter = authority.runner._adapter_for_source(event.source)
  331. if adapter is not None:
  332. # A recovered turn with no live delivery waiter: the drain sends this only after
  333. # it commits the terminal outcome (deliver_settled), so a failed settlement never
  334. # leaves an externally answered admission started -> unknown. A live waiter
  335. # sends its own; the record only outlives a settlement fenced unknown.
  336. delivery = (adapter, event, live.route, response, home)
  337. if native:
  338. waiter_replies(authority)[row['admission_id']] = delivery
  339. else:
  340. authority.pending_deliveries[row['admission_id']] = delivery
  341. handed = True
  342. await _hand_over_turn_marker(authority, row['admission_id'], event)
  343. return response
  344. finally:
  345. if not native:
  346. # No adapter task awaits this turn to run its post-delivery callbacks: the drain runs them
  347. # once the turn settled and its reply went out (``fire_post_delivery``).
  348. _take_post_delivery(authority, row['admission_id'], event.source, route)
  349. if not handed:
  350. await _release_turn_marker(authority, event)
  351. executing_admission.reset(token)
  352. execution_result.reset(result_token)
  353. api_execution.reset(api_token)
  354. admission_author.reset(author_token)
  355. viewer_turn.reset(viewer_token)
  356. def _launch_body_if_launch_owner(authority):
  357. """A standalone owner's turn binds nothing, but the launch profile's is marked at entry: a second
  358. profile switching this gateway to multiplexing mid-turn must not fail its later credential reads
  359. closed. Any other owner (a hosted named profile) stays unmarked, never reading the launch env."""
  360. from pathlib import Path
  361. from agent.secret_scope import _is_process_home
  362. owner = Path(str(authority.profile_id))
  363. if not owner.is_absolute() or not _is_process_home(owner):
  364. return nullcontext()
  365. from tui_gateway.launch_profile_policy import launch_body
  366. return launch_body()
  367. @asynccontextmanager
  368. async def _admission_scope(authority, home, row, event, captured):
  369. """The execution scope, inside the adapter lifecycle the executor owns for a producer-admitted
  370. turn (``session_producer_lifecycle``; a no-op for every other admission)."""
  371. async with producer_turn(authority, row, event, captured):
  372. with await _execution_scope(authority, home):
  373. yield
  374. async def _execution_scope(authority, home):
  375. """Under multiplex, owner-side execution runs under the OWNING profile's home (agent build,
  376. config, secrets, state.db), never the launch profile's ambient scope; native provenance
  377. (``home``) refines it. A single-profile gateway keeps its ambient scope byte-for-byte.
  378. Secret sources (external CLIs) hydrate off the loop, as main's per-message scope does (#99519)."""
  379. multiplex = getattr(getattr(authority.runner, 'config', None), 'multiplex_profiles', False)
  380. if home is None and not multiplex:
  381. return _launch_body_if_launch_owner(authority)
  382. from gateway.run import _load_profile_secret_scope, _profile_runtime_scope
  383. def load():
  384. from pathlib import Path
  385. scope_home = home
  386. if scope_home is None: # owner_scope's home; symbolic test ids keep the ambient scope
  387. owner = Path(str(authority.profile_id))
  388. scope_home = owner if owner.is_absolute() and owner.is_dir() else None
  389. return scope_home, scope_home and _load_profile_secret_scope(scope_home)
  390. scope_home, secrets = await asyncio.to_thread(load)
  391. return nullcontext() if scope_home is None else _profile_runtime_scope(scope_home, secrets)
  392. async def deliver_settled(authority, admission_id):
  393. """Send a recovered no-waiter turn's reply after its terminal outcome committed. A send
  394. failure cannot rewrite that outcome; it is logged, and inference never runs again."""
  395. held = _held_markers(authority).pop(admission_id, None)
  396. delivery = authority.pending_deliveries.get(admission_id)
  397. if delivery is not None and admission_id in authority.native_waiters:
  398. # ``admit_message`` registered its delivery waiter after this turn started: the drain
  399. # hands that waiter the reply and its adapter lifecycle sends it, so this send is not owed.
  400. authority.native_waiters.discard(admission_id)
  401. _hand_back_post_delivery(authority, admission_id)
  402. authority.pending_deliveries.pop(admission_id)
  403. adopter = _marker_adopters(authority).get(admission_id)
  404. if held is not None and adopter is not None:
  405. _move_marker(held, adopter)
  406. held = None
  407. delivery = None
  408. if delivery is None:
  409. if held is not None:
  410. # Fenced unknown: no answer is owed, so no later boot may deliver one either.
  411. await _release_turn_marker(authority, held)
  412. # Settled with nothing (left) to send: its reply streamed, or it had none.
  413. await close_producer_turn(authority, admission_id, ProcessingOutcome.SUCCESS)
  414. return
  415. adapter, event, session_key, response, home = delivery
  416. delivered = False
  417. try:
  418. with await _execution_scope(authority, home):
  419. delivered = await deliver_response(adapter, event, session_key, response)
  420. except Exception:
  421. logger.exception('Delivery of settled admission %s failed', admission_id)
  422. finally:
  423. # Held until the send returns: an empty map means every settled reply was handed over.
  424. authority.pending_deliveries.pop(admission_id, None)
  425. # The ledgered send already released the marker; an empty or failed send owes no more.
  426. await _release_turn_marker(authority, event)
  427. await close_producer_turn(
  428. authority, admission_id, ProcessingOutcome.SUCCESS if delivered else ProcessingOutcome.FAILURE)
  429. async def deliver_response(adapter, event, session_key, response):
  430. """Send a turn's reply; whether it was delivered (``_process_message_background``'s verdict:
  431. nothing to send, or at least one send that succeeded)."""
  432. from gateway.platforms.base import _thread_metadata_for_event, _mark_notify_metadata
  433. text, ttl = adapter._unwrap_ephemeral(response)
  434. if not text:
  435. return True
  436. extracted = await adapter._extract_response_content(text, event, session_key, is_ephemeral_response=ttl > 0)
  437. metadata = _mark_notify_metadata(_thread_metadata_for_event(event))
  438. results = []
  439. if extracted.text_content:
  440. await adapter._send_final_text(event, session_key, extracted.text_content,
  441. metadata, ttl > 0, ttl, results.append)
  442. await adapter._deliver_attachments(event, extracted, metadata, anything_sent=bool(results),
  443. record_delivery=results.append)
  444. return any(getattr(result, 'success', False) for result in results if result is not None)
  445. async def dispatch_shared_busy(adapter, event, session_key, *, durable=False):
  446. """Serial receive loops (IRC, Signal) await this before reading their next frame, so return
  447. once the follow-up is committed to the FIFO, or queued behind an earlier input of its chat still
  448. capturing (its position is then fixed; acceptance still waits for the commit); a tracked
  449. observer delivers its one reply. *durable*: return only once the row is committed (or the input
  450. was refused, fenced or answered), never on the earlier position release: a busy burst's commit
  451. task, which a command barrier (``/restart`` closing admission) waits on. Otherwise an input
  452. released before its commit is held until it commits or its handler ends, and a serial reader's
  453. next command waits for that too (``burst_gate``): a non-burst busy input (a second video, a voice
  454. note, text outside queue mode) has no burst commit task to wait on."""
  455. delivery_event = replace(event, source=replace(event.source))
  456. accepted = asyncio.get_running_loop().create_future()
  457. committed = asyncio.get_running_loop().create_future()
  458. async def observe():
  459. # Like the first message's lifecycle, this sender adopts the turn's crash marker and
  460. # releases it only once the reply is ledgered (the marker moves onto ``event``).
  461. event._turn_marker_handoff = getattr(adapter, 'gateway_runner', None) is not None
  462. try:
  463. response = await adapter._message_handler(event)
  464. # The ledgered send releases the marker off the event it sends.
  465. _move_marker(event, delivery_event)
  466. delivery_event._turn_marker_handoff = event._turn_marker_handoff
  467. await deliver_response(adapter, delivery_event, session_key, response)
  468. except Exception as exc:
  469. if not accepted.done():
  470. raise # nothing was queued: the receive loop sees the failure as before
  471. logger.error('[%s] Queued busy follow-up failed: %s', adapter.name, exc, exc_info=True)
  472. await adapter._notify_turn_error(delivery_event, exc)
  473. finally:
  474. settle_busy_arrival(event)
  475. # Clears the marker if no ledgered send did, then wakes the drain's bounded wait,
  476. # which listens on the adopter (``event``).
  477. await adapter._release_turn_marker(delivery_event)
  478. await adapter._release_turn_marker(event)
  479. token, commit_token = busy_acceptance.set(accepted), busy_commit.set(committed)
  480. try:
  481. task = asyncio.create_task(observe())
  482. finally:
  483. busy_acceptance.reset(token)
  484. busy_commit.reset(commit_token)
  485. adapter._background_tasks.add(task)
  486. task.add_done_callback(adapter._background_tasks.discard)
  487. try:
  488. await asyncio.wait({accepted, task}, return_when=asyncio.FIRST_COMPLETED)
  489. if durable and not task.done():
  490. await asyncio.wait({committed, task}, return_when=asyncio.FIRST_COMPLETED)
  491. except asyncio.CancelledError:
  492. if not accepted.done():
  493. task.cancel() # cancelled before commit: the input was never ACKed
  494. raise
  495. if task.done():
  496. task.result() # refused, consumed or answered inline before any durable acceptance
  497. elif not committed.done():
  498. _hold_until_committed(adapter, committed, task)
  499. def _hold_until_committed(adapter, committed, task):
  500. """Released before its commit: the reader reads on, but this input is not durable yet. A serial
  501. reader's next command waits for its commit or its handler's end (``burst_gate``), so /restart never
  502. closes admission, and /new never cancels the chat's rows, before it lands."""
  503. settled = asyncio.get_running_loop().create_future()
  504. def settle(_):
  505. if not settled.done():
  506. settled.set_result(None)
  507. committed.add_done_callback(settle)
  508. task.add_done_callback(settle)
  509. held = adapter.__dict__.setdefault('_uncommitted_inputs', set())
  510. held.add(settled)
  511. settled.add_done_callback(held.discard)